X-Git-Url: http://source.jalview.org/gitweb/?a=blobdiff_plain;f=forester%2Fjava%2Fsrc%2Forg%2Fforester%2Fmsa_compactor%2FMsaCompactor.java;h=2a5c5b185e32385572f634feb66ddb535e4ad7bc;hb=14375b2b2837d1ae5a2d0bb44f8aae9f1916281a;hp=eead263e48fb1d29062388a8e08f85aab70c51c2;hpb=54aa9ca62b9951be30c6a1f95b4d9887691b33f2;p=jalview.git diff --git a/forester/java/src/org/forester/msa_compactor/MsaCompactor.java b/forester/java/src/org/forester/msa_compactor/MsaCompactor.java index eead263..2a5c5b1 100644 --- a/forester/java/src/org/forester/msa_compactor/MsaCompactor.java +++ b/forester/java/src/org/forester/msa_compactor/MsaCompactor.java @@ -13,20 +13,34 @@ import java.util.List; import java.util.SortedSet; import java.util.TreeSet; +import org.forester.evoinference.distance.NeighborJoiningF; +import org.forester.evoinference.distance.PairwiseDistanceCalculator; +import org.forester.evoinference.distance.PairwiseDistanceCalculator.PWD_DISTANCE_METHOD; +import org.forester.evoinference.matrix.distance.BasicSymmetricalDistanceMatrix; +import org.forester.evoinference.tools.BootstrapResampler; +import org.forester.msa.BasicMsa; import org.forester.msa.Mafft; import org.forester.msa.Msa; import org.forester.msa.Msa.MSA_FORMAT; import org.forester.msa.MsaInferrer; import org.forester.msa.MsaMethods; +import org.forester.msa.ResampleableMsa; +import org.forester.phylogeny.Phylogeny; +import org.forester.phylogeny.PhylogenyMethods; import org.forester.sequence.Sequence; +import org.forester.tools.ConfidenceAssessor; +import org.forester.util.BasicDescriptiveStatistics; +import org.forester.util.DescriptiveStatistics; import org.forester.util.ForesterUtil; public class MsaCompactor { final private static NumberFormat NF_3 = new DecimalFormat( "#.###" ); final private static NumberFormat NF_4 = new DecimalFormat( "#.####" ); - private static final boolean VERBOSE = true; + private static final boolean VERBOSE = false; private Msa _msa; + private File _out_file_base; + private String _path_to_mafft; private final SortedSet _removed_seq_ids; static { NF_4.setRoundingMode( RoundingMode.HALF_UP ); @@ -46,11 +60,16 @@ public class MsaCompactor { return _removed_seq_ids; } - final public void writeMsa( final File outfile, final MSA_FORMAT format, final String suffix ) throws IOException { + final public void setOutFileBase( final File out_file_base ) { + _out_file_base = out_file_base; + } + + final public String writeMsa( final File outfile, final MSA_FORMAT format, final String suffix ) throws IOException { final Double gr = MsaMethods.calcGapRatio( _msa ); - writeMsa( outfile + "_" + _msa.getNumberOfSequences() + "_" + _msa.getLength() + "_" - + ForesterUtil.roundToInt( gr * 100 ) + suffix, - format ); + final String s = outfile + "_" + _msa.getNumberOfSequences() + "_" + _msa.getLength() + "_" + + ForesterUtil.roundToInt( gr * 100 ); + writeMsa( s + suffix, format ); + return s; } final int calcNonGapResidues( final Sequence seq ) { @@ -63,6 +82,24 @@ public class MsaCompactor { return ng; } + Phylogeny pi( final String matrix ) { + final Phylogeny master_phy = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, _msa, true, matrix ); + final int seed = 15; + final int n = 100; + final ResampleableMsa resampleable_msa = new ResampleableMsa( ( BasicMsa ) _msa ); + final int[][] resampled_column_positions = BootstrapResampler.createResampledColumnPositions( _msa.getLength(), + n, + seed ); + final Phylogeny[] eval_phys = new Phylogeny[ n ]; + for( int i = 0; i < n; ++i ) { + resampleable_msa.resample( resampled_column_positions[ i ] ); + eval_phys[ i ] = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, resampleable_msa, false, null ); + } + ConfidenceAssessor.evaluate( "bootstrap", eval_phys, master_phy, true, 1 ); + PhylogenyMethods.extractFastaInformation( master_phy ); + return master_phy; + } + private final GapContribution[] calcGapContribtions( final boolean normalize_for_effective_seq_length ) { final double gappiness[] = calcGappiness(); final GapContribution stats[] = new GapContribution[ _msa.getNumberOfSequences() ]; @@ -86,20 +123,20 @@ public class MsaCompactor { final private GapContribution[] calcGapContribtionsStats( final boolean norm ) { final GapContribution stats[] = calcGapContribtions( norm ); Arrays.sort( stats ); - for( final GapContribution stat : stats ) { - final StringBuilder sb = new StringBuilder(); - sb.append( stat.getId() ); - sb.append( "\t" ); - sb.append( NF_4.format( stat.getValue() ) ); - sb.append( "\t" ); - // sb.append( NF_4.format( stat.median() ) ); - // sb.append( "\t" ); - // sb.append( NF_4.format( stat.getMin() ) ); - // sb.append( "\t" ); - // sb.append( NF_4.format( stat.getMax() ) ); - //sb.append( "\t" ); - System.out.println( sb ); - } + // for( final GapContribution stat : stats ) { + // final StringBuilder sb = new StringBuilder(); + // sb.append( stat.getId() ); + // sb.append( "\t" ); + // sb.append( NF_4.format( stat.getValue() ) ); + // sb.append( "\t" ); + // sb.append( NF_4.format( stat.median() ) ); + // sb.append( "\t" ); + // sb.append( NF_4.format( stat.getMin() ) ); + // sb.append( "\t" ); + // sb.append( NF_4.format( stat.getMax() ) ); + //sb.append( "\t" ); + //System.out.println( sb ); + // } return stats; } @@ -113,15 +150,36 @@ public class MsaCompactor { return gappiness; } - final private void mafft() throws IOException, InterruptedException { - final MsaInferrer mafft = Mafft - .createInstance( "/home/czmasek/SOFTWARE/MSA/MAFFT/mafft-7.130-without-extensions/scripts/mafft" ); - final List opts = new ArrayList(); - // opts.add( "--maxiterate" ); - // opts.add( "1000" ); - // opts.add( "--localpair" ); - opts.add( "--quiet" ); - _msa = mafft.infer( _msa.asSequenceList(), opts ); + private Phylogeny inferNJphylogeny( final PWD_DISTANCE_METHOD pwd_distance_method, + final Msa msa, + final boolean write_matrix, + final String matrix_name ) { + BasicSymmetricalDistanceMatrix m = null; + switch ( pwd_distance_method ) { + case KIMURA_DISTANCE: + m = PairwiseDistanceCalculator.calcKimuraDistances( msa ); + break; + case POISSON_DISTANCE: + m = PairwiseDistanceCalculator.calcPoissonDistances( msa ); + break; + case FRACTIONAL_DISSIMILARITY: + m = PairwiseDistanceCalculator.calcFractionalDissimilarities( msa ); + break; + default: + throw new IllegalArgumentException( "invalid pwd method" ); + } + if ( write_matrix ) { + try { + m.write( ForesterUtil.createBufferedWriter( matrix_name ) ); + } + catch ( final IOException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + final NeighborJoiningF nj = NeighborJoiningF.createInstance( false, 5 ); + final Phylogeny phy = nj.execute( m ); + return phy; } private StringBuilder msaStatsAsSB() { @@ -131,9 +189,23 @@ public class MsaCompactor { sb.append( _msa.getLength() ); sb.append( "\t" ); sb.append( NF_3.format( MsaMethods.calcGapRatio( _msa ) ) ); + sb.append( "\t" ); + sb.append( NF_3.format( calculateIdentityRatio( 0, _msa.getLength() - 1, _msa ).arithmeticMean() ) ); return sb; } + final private void realignWithMafft() throws IOException, InterruptedException { + // final MsaInferrer mafft = Mafft + // .createInstance( "/home/czmasek/SOFTWARE/MSA/MAFFT/mafft-7.130-without-extensions/scripts/mafft" ); + final MsaInferrer mafft = Mafft.createInstance( _path_to_mafft ); + final List opts = new ArrayList(); + opts.add( "--maxiterate" ); + opts.add( "1000" ); + opts.add( "--localpair" ); + opts.add( "--quiet" ); + _msa = mafft.infer( _msa.asSequenceList(), opts ); + } + final private void removeGapColumns() { _msa = MsaMethods.createInstance().removeGapColumns( 1, 0, _msa ); } @@ -141,8 +213,46 @@ public class MsaCompactor { final private void removeViaGapAverage( final double mean_gapiness, final int step, final boolean realign, - final File outfile, - final int minimal_effective_length ) throws IOException, + final boolean norm, + final boolean verbose ) throws IOException, InterruptedException { + final GapContribution stats[] = calcGapContribtionsStats( norm ); + final List to_remove_ids = new ArrayList(); + for( final GapContribution gap_gontribution : stats ) { + to_remove_ids.add( gap_gontribution.getId() ); + } + int i = 0; + while ( MsaMethods.calcGapRatio( _msa ) > mean_gapiness ) { + final String id = to_remove_ids.get( i ); + _msa = MsaMethods.removeSequence( _msa, id ); + removeGapColumns(); + if ( verbose ) { + System.out.print( ForesterUtil.pad( id, 20, ' ', false ) ); + System.out.print( "\t" ); + final StringBuilder sb = msaStatsAsSB(); + System.out.print( sb ); + System.out.print( "\t" ); + } + if ( ( ( ( i + 1 ) % step ) == 0 ) || ( MsaMethods.calcGapRatio( _msa ) <= mean_gapiness ) ) { + if ( realign ) { + realignWithMafft(); + } + final String s = writeOutfile(); + if ( verbose ) { + System.out.print( "-> " + s ); + } + } + if ( verbose ) { + System.out.println(); + } + ++i; + } + } + + final private void removeViaGapAverageOLD( final double mean_gapiness, + final int step, + final boolean realign, + final File outfile, + final int minimal_effective_length ) throws IOException, InterruptedException { if ( step < 1 ) { throw new IllegalArgumentException( "step cannot be less than 1" ); @@ -162,9 +272,9 @@ public class MsaCompactor { int counter = step; double gr; do { - removeWorstOffenders( step, 1, false, false ); + removeWorstOffenders( step, 1, false, false, false ); if ( realign ) { - mafft(); + realignWithMafft(); } gr = MsaMethods.calcGapRatio( _msa ); if ( VERBOSE ) { @@ -178,91 +288,178 @@ public class MsaCompactor { } } - final private void removeViaLength( final int length, final int step, final boolean realign ) throws IOException, - InterruptedException { - if ( step < 1 ) { - throw new IllegalArgumentException( "step cannot be less than 1" ); - } - if ( length < 11 ) { - throw new IllegalArgumentException( "target length cannot be less than 1" ); - } - if ( VERBOSE ) { - System.out.println( "orig: " + msaStatsAsSB() ); + final private void removeViaLength( final int length, + final int step, + final boolean realign, + final boolean norm, + final boolean verbose ) throws IOException, InterruptedException { + final GapContribution stats[] = calcGapContribtionsStats( norm ); + final List to_remove_ids = new ArrayList(); + for( final GapContribution gap_gontribution : stats ) { + to_remove_ids.add( gap_gontribution.getId() ); } - int counter = step; + int i = 0; while ( _msa.getLength() > length ) { - removeWorstOffenders( step, 1, false, false ); - if ( realign ) { - mafft(); + final String id = to_remove_ids.get( i ); + _msa = MsaMethods.removeSequence( _msa, id ); + removeGapColumns(); + if ( verbose ) { + System.out.print( ForesterUtil.pad( id, 20, ' ', false ) ); + System.out.print( "\t" ); + final StringBuilder sb = msaStatsAsSB(); + System.out.print( sb ); + System.out.print( "\t" ); } - if ( VERBOSE ) { - System.out.println( counter + ": " + msaStatsAsSB() ); + if ( ( ( ( i + 1 ) % step ) == 0 ) || ( _msa.getLength() <= length ) ) { + if ( realign ) { + realignWithMafft(); + } + final String s = writeOutfile(); + if ( verbose ) { + System.out.print( "-> " + s ); + } } - counter += step; + if ( verbose ) { + System.out.println(); + } + ++i; } } final private void removeWorstOffenders( final int to_remove, final int step, final boolean realign, - final boolean norm ) throws IOException, InterruptedException { + final boolean norm, + final boolean verbose ) throws IOException, InterruptedException { final GapContribution stats[] = calcGapContribtionsStats( norm ); final List to_remove_ids = new ArrayList(); for( int j = 0; j < to_remove; ++j ) { to_remove_ids.add( stats[ j ].getId() ); _removed_seq_ids.add( stats[ j ].getId() ); } - //TODO if verbose/interactve - for( final String id : to_remove_ids ) { + for( int i = 0; i < to_remove_ids.size(); ++i ) { + final String id = to_remove_ids.get( i ); _msa = MsaMethods.removeSequence( _msa, id ); removeGapColumns(); - System.out.print( id ); - System.out.print( "\t" ); - final StringBuilder sb = msaStatsAsSB(); - System.out.println( sb ); - } - //TODO else: - //_msa = MsaMethods.removeSequences( _msa, to_remove_ids ); - //removeGapColumns(); - if ( realign ) { - mafft(); + if ( verbose ) { + System.out.print( ForesterUtil.pad( id, 20, ' ', false ) ); + System.out.print( "\t" ); + final StringBuilder sb = msaStatsAsSB(); + System.out.print( sb ); + System.out.print( "\t" ); + } + if ( ( ( ( i + 1 ) % step ) == 0 ) || ( i == ( to_remove_ids.size() - 1 ) ) ) { + if ( realign ) { + realignWithMafft(); + } + final String s = writeOutfile(); + if ( verbose ) { + System.out.print( "-> " + s ); + } + } + if ( verbose ) { + System.out.println(); + } } } + private void setPathToMafft( final String path_to_mafft ) { + _path_to_mafft = path_to_mafft; + } + final private void writeMsa( final String outfile, final MSA_FORMAT format ) throws IOException { final Writer w = ForesterUtil.createBufferedWriter( outfile ); _msa.write( w, format ); w.close(); } + private String writeOutfile() throws IOException { + final String s = writeMsa( _out_file_base, MSA_FORMAT.PHYLIP, ".aln" ); + //writeMsa( _out_file_base, MSA_FORMAT.FASTA, ".fasta" ); + return s; + } + + // Returns null if not path found. + final public static String guessPathToMafft() { + String path; + if ( ForesterUtil.OS_NAME.toLowerCase().indexOf( "win" ) >= 0 ) { + path = "C:\\Program Files\\mafft-win\\mafft.bat"; + if ( MsaInferrer.isInstalled( path ) ) { + return path; + } + } + path = "/usr/local/bin/mafft"; + if ( MsaInferrer.isInstalled( path ) ) { + return path; + } + path = "/usr/bin/mafft"; + if ( MsaInferrer.isInstalled( path ) ) { + return path; + } + path = "/bin/mafft"; + if ( MsaInferrer.isInstalled( path ) ) { + return path; + } + path = "mafft"; + if ( MsaInferrer.isInstalled( path ) ) { + return path; + } + return null; + } + public final static MsaCompactor reduceGapAverage( final Msa msa, final double max_gap_average, final int step, final boolean realign, - final File out, - final int minimal_effective_length ) throws IOException, - InterruptedException { + final boolean norm, + final String path_to_mafft, + final File out ) throws IOException, InterruptedException { final MsaCompactor mc = new MsaCompactor( msa ); - mc.removeViaGapAverage( max_gap_average, step, realign, out, minimal_effective_length ); + if ( realign ) { + mc.setPathToMafft( path_to_mafft ); + } + mc.setOutFileBase( out ); + mc.removeViaGapAverage( max_gap_average, step, realign, norm, true ); return mc; } public final static MsaCompactor reduceLength( final Msa msa, final int length, final int step, - final boolean realign ) throws IOException, InterruptedException { + final boolean realign, + final boolean norm, + final String path_to_mafft, + final File out ) throws IOException, InterruptedException { final MsaCompactor mc = new MsaCompactor( msa ); - mc.removeViaLength( length, step, realign ); + if ( realign ) { + mc.setPathToMafft( path_to_mafft ); + } + mc.setOutFileBase( out ); + mc.removeViaLength( length, step, realign, norm, true ); return mc; } public final static MsaCompactor removeWorstOffenders( final Msa msa, final int worst_offenders_to_remove, + final int step, final boolean realign, - final boolean norm ) throws IOException, - InterruptedException { + final boolean norm, + final String path_to_mafft, + final File out ) throws IOException, InterruptedException { final MsaCompactor mc = new MsaCompactor( msa ); - mc.removeWorstOffenders( worst_offenders_to_remove, 1, realign, norm ); + if ( realign ) { + mc.setPathToMafft( path_to_mafft ); + } + mc.setOutFileBase( out ); + mc.removeWorstOffenders( worst_offenders_to_remove, step, realign, norm, true ); return mc; } + + private static DescriptiveStatistics calculateIdentityRatio( final int from, final int to, final Msa msa ) { + final DescriptiveStatistics stats = new BasicDescriptiveStatistics(); + for( int c = from; c <= to; ++c ) { + stats.addValue( MsaMethods.calculateIdentityRatio( msa, c ) ); + } + return stats; + } }