From 5aac9f241d60f6092a849f7403e7a6286a54efaf Mon Sep 17 00:00:00 2001 From: "cmzmasek@gmail.com" Date: Wed, 16 Apr 2014 22:27:33 +0000 Subject: [PATCH] inprogress --- .../org/forester/application/msa_compactor.java | 12 +- .../org/forester/msa_compactor/MsaCompactor.java | 284 +++++++++++--------- 2 files changed, 159 insertions(+), 137 deletions(-) diff --git a/forester/java/src/org/forester/application/msa_compactor.java b/forester/java/src/org/forester/application/msa_compactor.java index 765648f..345539a 100644 --- a/forester/java/src/org/forester/application/msa_compactor.java +++ b/forester/java/src/org/forester/application/msa_compactor.java @@ -9,7 +9,6 @@ import java.util.List; import org.forester.io.parsers.FastaParser; import org.forester.io.parsers.GeneralMsaParser; import org.forester.msa.Msa; -import org.forester.msa.Msa.MSA_FORMAT; import org.forester.msa.MsaInferrer; import org.forester.msa_compactor.MsaCompactor; import org.forester.util.CommandLineArguments; @@ -109,19 +108,19 @@ public class msa_compactor { } MsaCompactor mc = null; if ( worst_remove > 0 ) { - mc = MsaCompactor.removeWorstOffenders( msa, worst_remove, realign, norm, path_to_mafft ); + mc = MsaCompactor.removeWorstOffenders( msa, worst_remove, step, realign, norm, path_to_mafft, out ); } else if ( av > 0 ) { - mc = MsaCompactor.reduceGapAverage( msa, av, step, realign, out, 50, path_to_mafft ); + mc = MsaCompactor.reduceGapAverage( msa, av, step, realign, 50, path_to_mafft, out ); } else if ( length > 0 ) { - mc = MsaCompactor.reduceLength( msa, length, step, realign, path_to_mafft ); + mc = MsaCompactor.reduceLength( msa, length, step, realign, path_to_mafft, out ); } //System.out.println( MsaMethods.calcGapRatio( mc.getMsa() ) ); // for( final String id : mc.getRemovedSeqIds() ) { // System.out.println( id ); //} - mc.writeMsa( out, MSA_FORMAT.PHYLIP, ".aln" ); + //mc.writeMsa( out, MSA_FORMAT.PHYLIP, ".aln" ); } catch ( final Exception e ) { e.printStackTrace(); @@ -168,6 +167,9 @@ public class msa_compactor { System.out.println(); System.out.println( " -" + REMOVE_WORST_OFFENDERS_OPTION + "= number of worst offender sequences to remove" ); + System.out.println( " -" + LENGTH_OPTION + "= length" ); + System.out.println( " -" + AV_GAPINESS_OPTION + "= gap %" ); + System.out.println( " -" + STEP_OPTION + "= step" ); System.out.println( " -" + REALIGN_OPTION + " to realign using MAFFT" + mafft_comment ); System.out.println(); System.out.println(); diff --git a/forester/java/src/org/forester/msa_compactor/MsaCompactor.java b/forester/java/src/org/forester/msa_compactor/MsaCompactor.java index 5d3a55c..a7886f1 100644 --- a/forester/java/src/org/forester/msa_compactor/MsaCompactor.java +++ b/forester/java/src/org/forester/msa_compactor/MsaCompactor.java @@ -37,10 +37,11 @@ public class MsaCompactor { final private static NumberFormat NF_3 = new DecimalFormat( "#.###" ); final private static NumberFormat NF_4 = new DecimalFormat( "#.####" ); - private static final boolean VERBOSE = true; + private static final boolean VERBOSE = false; private Msa _msa; - private final SortedSet _removed_seq_ids; + private File _out_file_base; private String _path_to_mafft; + private final SortedSet _removed_seq_ids; static { NF_4.setRoundingMode( RoundingMode.HALF_UP ); NF_3.setRoundingMode( RoundingMode.HALF_UP ); @@ -59,11 +60,16 @@ public class MsaCompactor { return _removed_seq_ids; } - final public void writeMsa( final File outfile, final MSA_FORMAT format, final String suffix ) throws IOException { + final public void setOutFileBase( final File out_file_base ) { + _out_file_base = out_file_base; + } + + final public String writeMsa( final File outfile, final MSA_FORMAT format, final String suffix ) throws IOException { final Double gr = MsaMethods.calcGapRatio( _msa ); - writeMsa( outfile + "_" + _msa.getNumberOfSequences() + "_" + _msa.getLength() + "_" - + ForesterUtil.roundToInt( gr * 100 ) + suffix, - format ); + final String s = outfile + "_" + _msa.getNumberOfSequences() + "_" + _msa.getLength() + "_" + + ForesterUtil.roundToInt( gr * 100 ); + writeMsa( s + suffix, format ); + return s; } final int calcNonGapResidues( final Sequence seq ) { @@ -76,6 +82,24 @@ public class MsaCompactor { return ng; } + Phylogeny pi( final String matrix ) { + final Phylogeny master_phy = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, _msa, true, matrix ); + final int seed = 15; + final int n = 100; + final ResampleableMsa resampleable_msa = new ResampleableMsa( ( BasicMsa ) _msa ); + final int[][] resampled_column_positions = BootstrapResampler.createResampledColumnPositions( _msa.getLength(), + n, + seed ); + final Phylogeny[] eval_phys = new Phylogeny[ n ]; + for( int i = 0; i < n; ++i ) { + resampleable_msa.resample( resampled_column_positions[ i ] ); + eval_phys[ i ] = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, resampleable_msa, false, null ); + } + ConfidenceAssessor.evaluate( "bootstrap", eval_phys, master_phy, true, 1 ); + PhylogenyMethods.extractFastaInformation( master_phy ); + return master_phy; + } + private final GapContribution[] calcGapContribtions( final boolean normalize_for_effective_seq_length ) { final double gappiness[] = calcGappiness(); final GapContribution stats[] = new GapContribution[ _msa.getNumberOfSequences() ]; @@ -116,14 +140,6 @@ public class MsaCompactor { return stats; } - private static DescriptiveStatistics calculateIdentityRatio( final int from, final int to, final Msa msa ) { - final DescriptiveStatistics stats = new BasicDescriptiveStatistics(); - for( int c = from; c <= to; ++c ) { - stats.addValue( MsaMethods.calculateIdentityRatio( msa, c ) ); - } - return stats; - } - private final double[] calcGappiness() { final int l = _msa.getLength(); final double gappiness[] = new double[ l ]; @@ -134,44 +150,36 @@ public class MsaCompactor { return gappiness; } - // Returns null if not path found. - final public static String guessPathToMafft() { - String path; - if ( ForesterUtil.OS_NAME.toLowerCase().indexOf( "win" ) >= 0 ) { - path = "C:\\Program Files\\mafft-win\\mafft.bat"; - if ( MsaInferrer.isInstalled( path ) ) { - return path; - } - } - path = "/usr/local/bin/mafft"; - if ( MsaInferrer.isInstalled( path ) ) { - return path; - } - path = "/usr/bin/mafft"; - if ( MsaInferrer.isInstalled( path ) ) { - return path; - } - path = "/bin/mafft"; - if ( MsaInferrer.isInstalled( path ) ) { - return path; + private Phylogeny inferNJphylogeny( final PWD_DISTANCE_METHOD pwd_distance_method, + final Msa msa, + final boolean write_matrix, + final String matrix_name ) { + BasicSymmetricalDistanceMatrix m = null; + switch ( pwd_distance_method ) { + case KIMURA_DISTANCE: + m = PairwiseDistanceCalculator.calcKimuraDistances( msa ); + break; + case POISSON_DISTANCE: + m = PairwiseDistanceCalculator.calcPoissonDistances( msa ); + break; + case FRACTIONAL_DISSIMILARITY: + m = PairwiseDistanceCalculator.calcFractionalDissimilarities( msa ); + break; + default: + throw new IllegalArgumentException( "invalid pwd method" ); } - path = "mafft"; - if ( MsaInferrer.isInstalled( path ) ) { - return path; + if ( write_matrix ) { + try { + m.write( ForesterUtil.createBufferedWriter( matrix_name ) ); + } + catch ( final IOException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } - return null; - } - - final private void mafft() throws IOException, InterruptedException { - // final MsaInferrer mafft = Mafft - // .createInstance( "/home/czmasek/SOFTWARE/MSA/MAFFT/mafft-7.130-without-extensions/scripts/mafft" ); - final MsaInferrer mafft = Mafft.createInstance( _path_to_mafft ); - final List opts = new ArrayList(); - opts.add( "--maxiterate" ); - opts.add( "1000" ); - opts.add( "--localpair" ); - opts.add( "--quiet" ); - _msa = mafft.infer( _msa.asSequenceList(), opts ); + final NeighborJoiningF nj = NeighborJoiningF.createInstance( false, 5 ); + final Phylogeny phy = nj.execute( m ); + return phy; } private StringBuilder msaStatsAsSB() { @@ -186,6 +194,18 @@ public class MsaCompactor { return sb; } + final private void realignWithMafft() throws IOException, InterruptedException { + // final MsaInferrer mafft = Mafft + // .createInstance( "/home/czmasek/SOFTWARE/MSA/MAFFT/mafft-7.130-without-extensions/scripts/mafft" ); + final MsaInferrer mafft = Mafft.createInstance( _path_to_mafft ); + final List opts = new ArrayList(); + opts.add( "--maxiterate" ); + opts.add( "1000" ); + opts.add( "--localpair" ); + opts.add( "--quiet" ); + _msa = mafft.infer( _msa.asSequenceList(), opts ); + } + final private void removeGapColumns() { _msa = MsaMethods.createInstance().removeGapColumns( 1, 0, _msa ); } @@ -214,9 +234,9 @@ public class MsaCompactor { int counter = step; double gr; do { - removeWorstOffenders( step, 1, false, false ); + removeWorstOffenders( step, 1, false, false, false ); if ( realign ) { - mafft(); + realignWithMafft(); } gr = MsaMethods.calcGapRatio( _msa ); if ( VERBOSE ) { @@ -243,9 +263,9 @@ public class MsaCompactor { } int counter = step; while ( _msa.getLength() > length ) { - removeWorstOffenders( step, 1, false, false ); + removeWorstOffenders( step, 1, false, false, false ); if ( realign ) { - mafft(); + realignWithMafft(); } if ( VERBOSE ) { System.out.println( counter + ": " + msaStatsAsSB() ); @@ -254,86 +274,45 @@ public class MsaCompactor { } } - Phylogeny pi( final String matrix ) { - final Phylogeny master_phy = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, _msa, true, matrix ); - final int seed = 15; - final int n = 100; - final ResampleableMsa resampleable_msa = new ResampleableMsa( ( BasicMsa ) _msa ); - final int[][] resampled_column_positions = BootstrapResampler.createResampledColumnPositions( _msa.getLength(), - n, - seed ); - final Phylogeny[] eval_phys = new Phylogeny[ n ]; - for( int i = 0; i < n; ++i ) { - resampleable_msa.resample( resampled_column_positions[ i ] ); - eval_phys[ i ] = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, resampleable_msa, false, null ); - } - ConfidenceAssessor.evaluate( "bootstrap", eval_phys, master_phy, true, 1 ); - PhylogenyMethods.extractFastaInformation( master_phy ); - return master_phy; - } - - private Phylogeny inferNJphylogeny( final PWD_DISTANCE_METHOD pwd_distance_method, - final Msa msa, - final boolean write_matrix, - final String matrix_name ) { - BasicSymmetricalDistanceMatrix m = null; - switch ( pwd_distance_method ) { - case KIMURA_DISTANCE: - m = PairwiseDistanceCalculator.calcKimuraDistances( msa ); - break; - case POISSON_DISTANCE: - m = PairwiseDistanceCalculator.calcPoissonDistances( msa ); - break; - case FRACTIONAL_DISSIMILARITY: - m = PairwiseDistanceCalculator.calcFractionalDissimilarities( msa ); - break; - default: - throw new IllegalArgumentException( "invalid pwd method" ); - } - if ( write_matrix ) { - try { - m.write( ForesterUtil.createBufferedWriter( matrix_name ) ); - } - catch ( final IOException e ) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - final NeighborJoiningF nj = NeighborJoiningF.createInstance( false, 5 ); - final Phylogeny phy = nj.execute( m ); - return phy; - } - final private void removeWorstOffenders( final int to_remove, final int step, final boolean realign, - final boolean norm ) throws IOException, InterruptedException { - //final Phylogeny a = pi( "a.pwd" ); - //Archaeopteryx.createApplication( a ); + final boolean norm, + final boolean verbose ) throws IOException, InterruptedException { final GapContribution stats[] = calcGapContribtionsStats( norm ); final List to_remove_ids = new ArrayList(); for( int j = 0; j < to_remove; ++j ) { to_remove_ids.add( stats[ j ].getId() ); _removed_seq_ids.add( stats[ j ].getId() ); } - //TODO if verbose/interactive - for( final String id : to_remove_ids ) { + for( int i = 0; i < to_remove_ids.size(); ++i ) { + final String id = to_remove_ids.get( i ); _msa = MsaMethods.removeSequence( _msa, id ); removeGapColumns(); - //System.out.print( id ); - System.out.print( ForesterUtil.pad( id, 20, ' ', false ) ); - System.out.print( "\t" ); - final StringBuilder sb = msaStatsAsSB(); - System.out.println( sb ); - } - //TODO else: - //_msa = MsaMethods.removeSequences( _msa, to_remove_ids ); - //removeGapColumns(); - if ( realign ) { - mafft(); + if ( verbose ) { + System.out.print( ForesterUtil.pad( id, 20, ' ', false ) ); + System.out.print( "\t" ); + final StringBuilder sb = msaStatsAsSB(); + System.out.print( sb ); + System.out.print( "\t" ); + } + if ( ( ( ( i + 1 ) % step ) == 0 ) || ( i == ( to_remove_ids.size() - 1 ) ) ) { + if ( realign ) { + realignWithMafft(); + } + final String s = writeOutfile(); + if ( verbose ) { + System.out.print( "-> " + s ); + } + } + if ( verbose ) { + System.out.println(); + } } - //final Phylogeny b = pi( "b.pwd" ); - //Archaeopteryx.createApplication( b ); + } + + private void setPathToMafft( final String path_to_mafft ) { + _path_to_mafft = path_to_mafft; } final private void writeMsa( final String outfile, final MSA_FORMAT format ) throws IOException { @@ -342,18 +321,52 @@ public class MsaCompactor { w.close(); } + private String writeOutfile() throws IOException { + final String s = writeMsa( _out_file_base, MSA_FORMAT.PHYLIP, ".aln" ); + //writeMsa( _out_file_base, MSA_FORMAT.FASTA, ".fasta" ); + return s; + } + + // Returns null if not path found. + final public static String guessPathToMafft() { + String path; + if ( ForesterUtil.OS_NAME.toLowerCase().indexOf( "win" ) >= 0 ) { + path = "C:\\Program Files\\mafft-win\\mafft.bat"; + if ( MsaInferrer.isInstalled( path ) ) { + return path; + } + } + path = "/usr/local/bin/mafft"; + if ( MsaInferrer.isInstalled( path ) ) { + return path; + } + path = "/usr/bin/mafft"; + if ( MsaInferrer.isInstalled( path ) ) { + return path; + } + path = "/bin/mafft"; + if ( MsaInferrer.isInstalled( path ) ) { + return path; + } + path = "mafft"; + if ( MsaInferrer.isInstalled( path ) ) { + return path; + } + return null; + } + public final static MsaCompactor reduceGapAverage( final Msa msa, final double max_gap_average, final int step, final boolean realign, - final File out, final int minimal_effective_length, - final String path_to_mafft ) throws IOException, - InterruptedException { + final String path_to_mafft, + final File out ) throws IOException, InterruptedException { final MsaCompactor mc = new MsaCompactor( msa ); if ( realign ) { mc.setPathToMafft( path_to_mafft ); } + mc.setOutFileBase( out ); mc.removeViaGapAverage( max_gap_average, step, realign, out, minimal_effective_length ); return mc; } @@ -362,31 +375,38 @@ public class MsaCompactor { final int length, final int step, final boolean realign, - final String path_to_mafft ) throws IOException, - InterruptedException { + final String path_to_mafft, + final File out ) throws IOException, InterruptedException { final MsaCompactor mc = new MsaCompactor( msa ); if ( realign ) { mc.setPathToMafft( path_to_mafft ); } + mc.setOutFileBase( out ); mc.removeViaLength( length, step, realign ); return mc; } public final static MsaCompactor removeWorstOffenders( final Msa msa, final int worst_offenders_to_remove, + final int step, final boolean realign, final boolean norm, - final String path_to_mafft ) throws IOException, - InterruptedException { + final String path_to_mafft, + final File out ) throws IOException, InterruptedException { final MsaCompactor mc = new MsaCompactor( msa ); if ( realign ) { mc.setPathToMafft( path_to_mafft ); } - mc.removeWorstOffenders( worst_offenders_to_remove, 1, realign, norm ); + mc.setOutFileBase( out ); + mc.removeWorstOffenders( worst_offenders_to_remove, step, realign, norm, true ); return mc; } - private void setPathToMafft( final String path_to_mafft ) { - _path_to_mafft = path_to_mafft; + private static DescriptiveStatistics calculateIdentityRatio( final int from, final int to, final Msa msa ) { + final DescriptiveStatistics stats = new BasicDescriptiveStatistics(); + for( int c = from; c <= to; ++c ) { + stats.addValue( MsaMethods.calculateIdentityRatio( msa, c ) ); + } + return stats; } } -- 1.7.10.2