2 // FORESTER -- software libraries and applications
3 // for evolutionary biology research and applications.
5 // Copyright (C) 2014 Christian M. Zmasek
6 // Copyright (C) 2014 Sanford-Burnham Medical Research Institute
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
23 // WWW: https://sites.google.com/site/cmzmasek/home/software/forester
25 package org.forester.msa_compactor;
28 import java.io.IOException;
29 import java.io.Writer;
30 import java.math.RoundingMode;
31 import java.text.DecimalFormat;
32 import java.text.NumberFormat;
33 import java.util.ArrayList;
34 import java.util.Arrays;
35 import java.util.List;
36 import java.util.SortedSet;
37 import java.util.TreeSet;
39 import org.forester.evoinference.distance.NeighborJoiningF;
40 import org.forester.evoinference.distance.PairwiseDistanceCalculator;
41 import org.forester.evoinference.distance.PairwiseDistanceCalculator.PWD_DISTANCE_METHOD;
42 import org.forester.evoinference.matrix.distance.BasicSymmetricalDistanceMatrix;
43 import org.forester.evoinference.tools.BootstrapResampler;
44 import org.forester.io.writers.SequenceWriter;
45 import org.forester.io.writers.SequenceWriter.SEQ_FORMAT;
46 import org.forester.msa.DeleteableMsa;
47 import org.forester.msa.Mafft;
48 import org.forester.msa.Msa;
49 import org.forester.msa.Msa.MSA_FORMAT;
50 import org.forester.msa.MsaInferrer;
51 import org.forester.msa.MsaMethods;
52 import org.forester.msa.ResampleableMsa;
53 import org.forester.phylogeny.Phylogeny;
54 import org.forester.phylogeny.PhylogenyMethods;
55 import org.forester.sequence.Sequence;
56 import org.forester.tools.ConfidenceAssessor;
57 import org.forester.util.ForesterUtil;
59 public class MsaCompactor {
61 final private static NumberFormat NF_3 = new DecimalFormat( "#.###" );
62 final private static NumberFormat NF_4 = new DecimalFormat( "#.####" );
63 private double _gap_ratio = -1;
65 private final String _maffts_opts = "--auto";
66 private int _min_length = -1;
68 private DeleteableMsa _msa = null;
69 private boolean _norm = true;
70 private File _out_file_base = null;
71 private MSA_FORMAT _output_format = MSA_FORMAT.FASTA;
72 private String _path_to_mafft = null;
74 private boolean _realign = false;
75 private final SortedSet<String> _removed_seq_ids;
76 private File _removed_seqs_out_base = null;
78 public final void setRemovedSeqsOutBase( final File removed_seqs_out_base ) {
79 _removed_seqs_out_base = removed_seqs_out_base;
81 private boolean _report_aln_mean_identity = false;
82 private int _step = -1;
83 private int _step_for_diagnostics = -1;
84 private final short _longest_id_length;
85 private final ArrayList<Sequence> _removed_seqs;
87 NF_4.setRoundingMode( RoundingMode.HALF_UP );
88 NF_3.setRoundingMode( RoundingMode.HALF_UP );
91 public MsaCompactor( final DeleteableMsa msa ) {
93 _removed_seq_ids = new TreeSet<String>();
94 _longest_id_length = _msa.determineMaxIdLength();
95 _removed_seqs = new ArrayList<Sequence>();
98 final public Msa getMsa() {
102 final public SortedSet<String> getRemovedSeqIds() {
103 return _removed_seq_ids;
106 public final void removeViaGapAverage( final double mean_gapiness ) throws IOException, InterruptedException {
107 final GapContribution stats[] = calcGapContribtionsStats( _norm );
108 final List<String> to_remove_ids = new ArrayList<String>();
109 for( final GapContribution gap_gontribution : stats ) {
110 to_remove_ids.add( gap_gontribution.getId() );
114 while ( MsaMethods.calcGapRatio( _msa ) > mean_gapiness ) {
115 final String id = to_remove_ids.get( i );
116 _removed_seq_ids.add( id );
117 final Sequence deleted = _msa.deleteRow( id );
118 _removed_seqs.add( deleted );
120 if ( ( ( _step > 0 ) && ( ( ( i + 1 ) % _step ) == 0 ) )
121 || ( MsaMethods.calcGapRatio( _msa ) <= mean_gapiness ) ) {
122 printMsaStatsWriteOutfileAndRealign( _realign, id );
125 final MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
126 printMsaProperties( id, msa_prop );
128 System.out.println();
131 if ( _removed_seqs_out_base != null ) {
132 final String msg = writeAndAlignRemovedSeqs();
133 System.out.println( msg );
137 public void removeViaLength( final int length ) throws IOException, InterruptedException {
138 final GapContribution stats[] = calcGapContribtionsStats( _norm );
139 final List<String> to_remove_ids = new ArrayList<String>();
140 for( final GapContribution gap_gontribution : stats ) {
141 to_remove_ids.add( gap_gontribution.getId() );
145 while ( _msa.getLength() > length ) {
146 final String id = to_remove_ids.get( i );
147 _removed_seq_ids.add( id );
148 final Sequence deleted = _msa.deleteRow( id );
149 _removed_seqs.add( deleted );
151 if ( ( ( _step > 0 ) && ( ( ( i + 1 ) % _step ) == 0 ) ) || ( _msa.getLength() <= length ) ) {
152 printMsaStatsWriteOutfileAndRealign( _realign, id );
154 final MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
155 printMsaProperties( id, msa_prop );
156 System.out.println();
159 if ( _removed_seqs_out_base != null ) {
160 final String msg = writeAndAlignRemovedSeqs();
161 System.out.println( msg );
165 public final void removeWorstOffenders( final int to_remove ) throws IOException, InterruptedException {
166 final GapContribution stats[] = calcGapContribtionsStats( _norm );
167 final List<String> to_remove_ids = new ArrayList<String>();
168 for( int j = 0; j < to_remove; ++j ) {
169 to_remove_ids.add( stats[ j ].getId() );
170 _removed_seq_ids.add( stats[ j ].getId() );
173 for( int i = 0; i < to_remove_ids.size(); ++i ) {
174 final String id = to_remove_ids.get( i );
175 _removed_seq_ids.add( id );
176 final Sequence deleted = _msa.deleteRow( id );
177 _removed_seqs.add( deleted );
179 if ( isPrintMsaStatsWriteOutfileAndRealign( i ) || ( i == ( to_remove_ids.size() - 1 ) ) ) {
180 printMsaStatsWriteOutfileAndRealign( _realign, id );
181 System.out.println();
183 else if ( isPrintMsaStats( i ) ) {
184 final MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
185 printMsaProperties( id, msa_prop );
186 System.out.println();
189 if ( _removed_seqs_out_base != null ) {
190 final String msg = writeAndAlignRemovedSeqs();
191 System.out.println( msg );
195 public final List<MsaProperties> chart( final int step, final boolean realign, final boolean norm )
196 throws IOException, InterruptedException {
197 final GapContribution stats[] = calcGapContribtionsStats( norm );
198 final List<String> to_remove_ids = new ArrayList<String>();
199 final List<MsaProperties> msa_props = new ArrayList<MsaProperties>();
200 for( final GapContribution gap_gontribution : stats ) {
201 to_remove_ids.add( gap_gontribution.getId() );
205 final int x = ForesterUtil.roundToInt( _msa.getNumberOfSequences() / 20.0 );
206 MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
207 msa_props.add( msa_prop );
208 printMsaProperties( "", msa_prop );
209 System.out.println();
210 while ( _msa.getNumberOfSequences() > x ) {
211 final String id = to_remove_ids.get( i );
212 _msa.deleteRow( id );
213 if ( realign && isPrintMsaStatsWriteOutfileAndRealign( i ) ) {
216 msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
217 msa_props.add( msa_prop );
218 printMsaProperties( id, msa_prop );
219 System.out.print( "(realigned)" );
220 System.out.println();
222 else if ( isPrintMsaStats( i ) ) {
224 msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
225 msa_props.add( msa_prop );
226 printMsaProperties( id, msa_prop );
227 System.out.println();
234 private final boolean isPrintMsaStats( final int i ) {
235 return ( ( _step_for_diagnostics < 2 ) || ( ( ( i + 1 ) % _step_for_diagnostics ) == 0 ) );
238 private final boolean isPrintMsaStatsWriteOutfileAndRealign( final int i ) {
239 return ( ( _step < 2 ) || ( ( ( i + 1 ) % _step ) == 0 ) );
242 public final void setGapRatio( final double gap_ratio ) {
243 _gap_ratio = gap_ratio;
246 public final void setMinLength( final int min_length ) {
247 _min_length = min_length;
250 public final void setNorm( final boolean norm ) {
254 final public void setOutFileBase( final File out_file_base ) {
255 _out_file_base = out_file_base;
258 public final void setOutputFormat( final MSA_FORMAT output_format ) {
259 _output_format = output_format;
262 public void setPathToMafft( final String path_to_mafft ) {
263 _path_to_mafft = path_to_mafft;
266 public final void setRealign( final boolean realign ) {
270 public final void setStep( final int step ) {
274 public final void setStepForDiagnostics( final int step_for_diagnostics ) {
275 _step_for_diagnostics = step_for_diagnostics;
278 public final void setReportAlnMeanIdentity( final boolean report_aln_mean_identity ) {
279 _report_aln_mean_identity = report_aln_mean_identity;
282 final public String writeMsa( final File outfile, final MSA_FORMAT format, final String suffix ) throws IOException {
283 final Double gr = MsaMethods.calcGapRatio( _msa );
284 final String s = outfile + "_" + _msa.getNumberOfSequences() + "_" + _msa.getLength() + "_"
285 + ForesterUtil.roundToInt( gr * 100 );
286 writeMsa( _msa, s + suffix, format );
290 final public String writeAndAlignRemovedSeqs() throws IOException, InterruptedException {
291 final StringBuilder msg = new StringBuilder();
292 final String n = _removed_seqs_out_base + "_" + _removed_seqs.size() + ".fasta";
293 SequenceWriter.writeSeqs( _removed_seqs, new File( n ), SEQ_FORMAT.FASTA, 100 );
294 msg.append( "wrote " + _removed_seqs.size() + " removed sequences to " + n );
296 final MsaInferrer mafft = Mafft.createInstance( _path_to_mafft );
297 final List<String> opts = new ArrayList<String>();
298 for( final String o : _maffts_opts.split( "\\s" ) ) {
301 final Msa removed_msa = mafft.infer( _removed_seqs, opts );
302 final Double gr = MsaMethods.calcGapRatio( removed_msa );
303 String s = _removed_seqs_out_base + "_" + removed_msa.getNumberOfSequences() + "_"
304 + removed_msa.getLength() + "_" + ForesterUtil.roundToInt( gr * 100 );
306 if ( _output_format == MSA_FORMAT.FASTA ) {
309 else if ( _output_format == MSA_FORMAT.PHYLIP ) {
313 writeMsa( removed_msa, s, _output_format );
314 msg.append( ", and as MSA of length " + removed_msa.getLength() + " to " + s );
316 return msg.toString();
319 final int calcNonGapResidues( final Sequence seq ) {
321 for( int i = 0; i < seq.getLength(); ++i ) {
322 if ( !seq.isGapAt( i ) ) {
329 private final GapContribution[] calcGapContribtions( final boolean normalize_for_effective_seq_length ) {
330 final double gappiness[] = calcGappiness();
331 final GapContribution stats[] = new GapContribution[ _msa.getNumberOfSequences() ];
332 for( int row = 0; row < _msa.getNumberOfSequences(); ++row ) {
333 stats[ row ] = new GapContribution( _msa.getIdentifier( row ) );
334 for( int col = 0; col < _msa.getLength(); ++col ) {
335 if ( !_msa.isGapAt( row, col ) ) {
336 stats[ row ].addToValue( gappiness[ col ] );
339 if ( normalize_for_effective_seq_length ) {
340 stats[ row ].divideValue( calcNonGapResidues( _msa.getSequence( row ) ) );
343 stats[ row ].divideValue( _msa.getLength() );
349 final private GapContribution[] calcGapContribtionsStats( final boolean norm ) {
350 final GapContribution stats[] = calcGapContribtions( norm );
351 Arrays.sort( stats );
355 private final double[] calcGappiness() {
356 final int l = _msa.getLength();
357 final double gappiness[] = new double[ l ];
358 final int seqs = _msa.getNumberOfSequences();
359 for( int i = 0; i < l; ++i ) {
360 gappiness[ i ] = ( double ) MsaMethods.calcGapSumPerColumn( _msa, i ) / seqs;
365 private final Phylogeny inferNJphylogeny( final PWD_DISTANCE_METHOD pwd_distance_method,
367 final boolean write_matrix,
368 final String matrix_name ) {
369 BasicSymmetricalDistanceMatrix m = null;
370 switch ( pwd_distance_method ) {
371 case KIMURA_DISTANCE:
372 m = PairwiseDistanceCalculator.calcKimuraDistances( msa );
374 case POISSON_DISTANCE:
375 m = PairwiseDistanceCalculator.calcPoissonDistances( msa );
377 case FRACTIONAL_DISSIMILARITY:
378 m = PairwiseDistanceCalculator.calcFractionalDissimilarities( msa );
381 throw new IllegalArgumentException( "invalid pwd method" );
383 if ( write_matrix ) {
385 m.write( ForesterUtil.createBufferedWriter( matrix_name ) );
387 catch ( final IOException e ) {
391 final NeighborJoiningF nj = NeighborJoiningF.createInstance( false, 5 );
392 final Phylogeny phy = nj.execute( m );
396 private final Phylogeny pi( final String matrix ) {
397 final Phylogeny master_phy = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, _msa, true, matrix );
400 final ResampleableMsa resampleable_msa = new ResampleableMsa( _msa );
401 final int[][] resampled_column_positions = BootstrapResampler.createResampledColumnPositions( _msa.getLength(),
404 final Phylogeny[] eval_phys = new Phylogeny[ n ];
405 for( int i = 0; i < n; ++i ) {
406 resampleable_msa.resample( resampled_column_positions[ i ] );
407 eval_phys[ i ] = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, resampleable_msa, false, null );
409 ConfidenceAssessor.evaluate( "bootstrap", eval_phys, master_phy, true, 1 );
410 PhylogenyMethods.extractFastaInformation( master_phy );
414 private final void printMsaProperties( final String id, final MsaProperties msa_properties ) {
415 System.out.print( ForesterUtil.pad( id, _longest_id_length, ' ', false ) );
416 System.out.print( "\t" );
417 final StringBuilder sb = msaPropertiesAsSB( msa_properties );
418 System.out.print( sb );
419 System.out.print( "\t" );
422 private final StringBuilder msaPropertiesAsSB( final MsaProperties msa_properties ) {
423 final StringBuilder sb = new StringBuilder();
424 sb.append( msa_properties.getNumberOfSequences() );
426 sb.append( msa_properties.getLength() );
428 sb.append( NF_4.format( msa_properties.getGapRatio() ) );
429 if ( _report_aln_mean_identity /*msa_properties.getAverageIdentityRatio() >= 0*/) {
431 sb.append( NF_4.format( msa_properties.getAverageIdentityRatio() ) );
436 final private void printMsaStatsWriteOutfileAndRealign( final boolean realign, final String id )
437 throws IOException, InterruptedException {
441 final MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
442 printMsaProperties( id, msa_prop );
443 final String s = writeOutfile();
444 System.out.print( "-> " + s + ( realign ? "\t(realigned)" : "" ) );
447 final private void realignWithMafft() throws IOException, InterruptedException {
448 final MsaInferrer mafft = Mafft.createInstance( _path_to_mafft );
449 final List<String> opts = new ArrayList<String>();
450 for( final String o : _maffts_opts.split( "\\s" ) ) {
453 _msa = DeleteableMsa.createInstance( mafft.infer( _msa.asSequenceList(), opts ) );
456 final private void removeGapColumns() {
457 _msa.deleteGapOnlyColumns();
460 final private static void writeMsa( final Msa msa, final String outfile, final MSA_FORMAT format )
462 final Writer w = ForesterUtil.createBufferedWriter( outfile );
463 msa.write( w, format );
467 private final String writeOutfile() throws IOException {
468 final String s = writeMsa( _out_file_base, MSA_FORMAT.PHYLIP, ".aln" );
469 //writeMsa( _out_file_base, MSA_FORMAT.FASTA, ".fasta" );
473 // Returns null if not path found.
474 final public static String guessPathToMafft() {
476 if ( ForesterUtil.OS_NAME.toLowerCase().indexOf( "win" ) >= 0 ) {
477 path = "C:\\Program Files\\mafft-win\\mafft.bat";
478 if ( MsaInferrer.isInstalled( path ) ) {
482 path = "/home/czmasek/SOFTWARE/MSA/MAFFT/mafft-7.130-without-extensions/scripts/mafft";
483 if ( MsaInferrer.isInstalled( path ) ) {
486 path = "/usr/local/bin/mafft";
487 if ( MsaInferrer.isInstalled( path ) ) {
490 path = "/usr/bin/mafft";
491 if ( MsaInferrer.isInstalled( path ) ) {
495 if ( MsaInferrer.isInstalled( path ) ) {
499 if ( MsaInferrer.isInstalled( path ) ) {
505 private final void printTableHeader() {
506 System.out.print( ForesterUtil.pad( "Id", _longest_id_length, ' ', false ) );
507 System.out.print( "\t" );
508 System.out.print( "Seqs" );
509 System.out.print( "\t" );
510 System.out.print( "Length" );
511 System.out.print( "\t" );
512 System.out.print( "Gaps" );
513 System.out.print( "\t" );
514 if ( _report_aln_mean_identity ) {
515 System.out.print( "MSA qual" );
516 System.out.print( "\t" );
518 System.out.println();