2 // FORESTER -- software libraries and applications
3 // for evolutionary biology research and applications.
5 // Copyright (C) 2014 Christian M. Zmasek
6 // Copyright (C) 2014 Sanford-Burnham Medical Research Institute
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
23 // WWW: https://sites.google.com/site/cmzmasek/home/software/forester
25 package org.forester.msa_compactor;
28 import java.io.IOException;
29 import java.io.Writer;
30 import java.math.RoundingMode;
31 import java.text.DecimalFormat;
32 import java.text.NumberFormat;
33 import java.util.ArrayList;
34 import java.util.Arrays;
35 import java.util.List;
36 import java.util.SortedSet;
37 import java.util.TreeSet;
39 import org.forester.archaeopteryx.Archaeopteryx;
40 import org.forester.archaeopteryx.Configuration;
41 import org.forester.evoinference.distance.NeighborJoiningF;
42 import org.forester.evoinference.distance.PairwiseDistanceCalculator;
43 import org.forester.evoinference.distance.PairwiseDistanceCalculator.PWD_DISTANCE_METHOD;
44 import org.forester.evoinference.matrix.distance.BasicSymmetricalDistanceMatrix;
45 import org.forester.evoinference.tools.BootstrapResampler;
46 import org.forester.io.parsers.nhx.NHXParser.TAXONOMY_EXTRACTION;
47 import org.forester.io.parsers.phyloxml.PhyloXmlDataFormatException;
48 import org.forester.io.parsers.util.ParserUtils;
49 import org.forester.io.writers.SequenceWriter;
50 import org.forester.io.writers.SequenceWriter.SEQ_FORMAT;
51 import org.forester.msa.DeleteableMsa;
52 import org.forester.msa.Mafft;
53 import org.forester.msa.Msa;
54 import org.forester.msa.Msa.MSA_FORMAT;
55 import org.forester.msa.MsaInferrer;
56 import org.forester.msa.MsaMethods;
57 import org.forester.msa.ResampleableMsa;
58 import org.forester.phylogeny.Phylogeny;
59 import org.forester.phylogeny.PhylogenyMethods;
60 import org.forester.phylogeny.PhylogenyNode;
61 import org.forester.phylogeny.iterators.PhylogenyNodeIterator;
62 import org.forester.sequence.Sequence;
63 import org.forester.tools.ConfidenceAssessor;
64 import org.forester.util.ForesterUtil;
66 public class MsaCompactor {
68 final private static NumberFormat NF_3 = new DecimalFormat( "#.###" );
69 final private static NumberFormat NF_4 = new DecimalFormat( "#.####" );
70 private double _gap_ratio = -1;
71 private final short _longest_id_length;
73 private String _maffts_opts = "--auto";
74 private int _min_length = -1;
76 private String _infile_name = null;
77 private DeleteableMsa _msa = null;
78 private boolean _norm = true;
79 private File _out_file_base = null;
80 private MSA_FORMAT _output_format = MSA_FORMAT.FASTA;
81 private String _path_to_mafft = null;
83 private boolean _realign = false;
84 private final SortedSet<String> _removed_seq_ids;
85 private final ArrayList<Sequence> _removed_seqs;
86 private File _removed_seqs_out_base = null;
87 private boolean _report_aln_mean_identity = false;
88 private int _step = -1;
89 private int _step_for_diagnostics = -1;
90 private boolean _phylogentic_inference = false;
92 NF_4.setRoundingMode( RoundingMode.HALF_UP );
93 NF_3.setRoundingMode( RoundingMode.HALF_UP );
96 public MsaCompactor( final DeleteableMsa msa ) {
98 _removed_seq_ids = new TreeSet<String>();
99 _longest_id_length = _msa.determineMaxIdLength();
100 _removed_seqs = new ArrayList<Sequence>();
103 public final List<MsaProperties> chart( final int step, final boolean realign, final boolean norm )
104 throws IOException, InterruptedException {
105 final GapContribution stats[] = calcGapContribtionsStats( norm );
106 final List<String> to_remove_ids = new ArrayList<String>();
107 final List<MsaProperties> msa_props = new ArrayList<MsaProperties>();
108 for( final GapContribution gap_gontribution : stats ) {
109 to_remove_ids.add( gap_gontribution.getId() );
111 if ( _phylogentic_inference ) {
112 System.out.println( "calculating phylegentic tree..." );
113 System.out.println();
120 int x = ForesterUtil.roundToInt( _msa.getNumberOfSequences() / 20.0 );
124 MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
125 msa_props.add( msa_prop );
126 printMsaProperties( "", msa_prop );
127 System.out.println();
129 while ( _msa.getNumberOfSequences() > x ) {
130 final String id = to_remove_ids.get( i );
131 _msa.deleteRow( id, false );
132 if ( realign && isPrintMsaStatsWriteOutfileAndRealign( i ) ) {
135 msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
136 msa_props.add( msa_prop );
137 printMsaProperties( id, msa_prop );
138 System.out.print( "(realigned)" );
139 System.out.println();
141 else if ( isPrintMsaStats( i ) ) {
143 msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
144 msa_props.add( msa_prop );
145 printMsaProperties( id, msa_prop );
146 System.out.println();
153 final public void deleteGapColumns( final double max_allowed_gap_ratio ) {
154 _msa.deleteGapColumns( max_allowed_gap_ratio );
157 final public Msa getMsa() {
161 final public SortedSet<String> getRemovedSeqIds() {
162 return _removed_seq_ids;
165 public final void removeSequencesByMinimalLength( final int min_effective_length ) {
166 printMsaProperties( "", new MsaProperties( _msa, _report_aln_mean_identity ) );
167 System.out.println();
168 _msa = DeleteableMsa.createInstance( MsaMethods.removeSequencesByMinimalLength( _msa, min_effective_length ) );
170 printMsaProperties( "", new MsaProperties( _msa, _report_aln_mean_identity ) );
171 System.out.println();
174 public final List<MsaProperties> removeViaGapAverage( final double mean_gapiness ) throws IOException,
175 InterruptedException {
176 final GapContribution stats[] = calcGapContribtionsStats( _norm );
177 final List<String> to_remove_ids = new ArrayList<String>();
178 final List<MsaProperties> msa_props = new ArrayList<MsaProperties>();
179 for( final GapContribution gap_gontribution : stats ) {
180 to_remove_ids.add( gap_gontribution.getId() );
183 MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
184 msa_props.add( msa_prop );
185 printMsaProperties( "", msa_prop );
186 System.out.println();
188 while ( MsaMethods.calcGapRatio( _msa ) > mean_gapiness ) {
189 final String id = to_remove_ids.get( i );
190 _removed_seq_ids.add( id );
191 final Sequence deleted = _msa.deleteRow( id, true );
192 _removed_seqs.add( deleted );
194 if ( isPrintMsaStatsWriteOutfileAndRealign( i ) || ( MsaMethods.calcGapRatio( _msa ) <= mean_gapiness ) ) {
195 msa_prop = printMsaStatsWriteOutfileAndRealign( _realign, id );
196 msa_props.add( msa_prop );
197 System.out.println();
199 else if ( isPrintMsaStats( i ) ) {
200 msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
201 msa_props.add( msa_prop );
202 printMsaProperties( id, msa_prop );
203 System.out.println();
207 if ( _removed_seqs_out_base != null ) {
208 final String msg = writeAndAlignRemovedSeqs();
209 System.out.println();
210 System.out.println( msg );
215 public List<MsaProperties> removeViaLength( final int length ) throws IOException, InterruptedException {
216 final GapContribution stats[] = calcGapContribtionsStats( _norm );
217 final List<String> to_remove_ids = new ArrayList<String>();
218 final List<MsaProperties> msa_props = new ArrayList<MsaProperties>();
219 for( final GapContribution gap_gontribution : stats ) {
220 to_remove_ids.add( gap_gontribution.getId() );
223 MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
224 msa_props.add( msa_prop );
225 printMsaProperties( "", msa_prop );
226 System.out.println();
228 while ( _msa.getLength() > length ) {
229 final String id = to_remove_ids.get( i );
230 _removed_seq_ids.add( id );
231 final Sequence deleted = _msa.deleteRow( id, true );
232 _removed_seqs.add( deleted );
234 if ( isPrintMsaStatsWriteOutfileAndRealign( i ) || ( _msa.getLength() <= length ) ) {
235 msa_prop = printMsaStatsWriteOutfileAndRealign( _realign, id );
236 msa_props.add( msa_prop );
237 System.out.println();
239 else if ( isPrintMsaStats( i ) ) {
240 msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
241 printMsaProperties( id, msa_prop );
242 msa_props.add( msa_prop );
243 System.out.println();
247 if ( _removed_seqs_out_base != null ) {
248 final String msg = writeAndAlignRemovedSeqs();
249 System.out.println();
250 System.out.println( msg );
255 public final List<MsaProperties> removeWorstOffenders( final int to_remove ) throws IOException,
256 InterruptedException {
257 final GapContribution stats[] = calcGapContribtionsStats( _norm );
258 final List<String> to_remove_ids = new ArrayList<String>();
259 final List<MsaProperties> msa_props = new ArrayList<MsaProperties>();
260 for( int j = 0; j < to_remove; ++j ) {
261 to_remove_ids.add( stats[ j ].getId() );
262 _removed_seq_ids.add( stats[ j ].getId() );
265 MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
266 msa_props.add( msa_prop );
267 printMsaProperties( "", msa_prop );
268 System.out.println();
269 for( int i = 0; i < to_remove_ids.size(); ++i ) {
270 final String id = to_remove_ids.get( i );
271 _removed_seq_ids.add( id );
272 final Sequence deleted = _msa.deleteRow( id, true );
273 _removed_seqs.add( deleted );
275 if ( isPrintMsaStatsWriteOutfileAndRealign( i ) || ( i == ( to_remove_ids.size() - 1 ) ) ) {
276 msa_prop = printMsaStatsWriteOutfileAndRealign( _realign, id );
277 msa_props.add( msa_prop );
278 System.out.println();
280 else if ( isPrintMsaStats( i ) ) {
281 msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
282 msa_props.add( msa_prop );
283 printMsaProperties( id, msa_prop );
284 System.out.println();
287 if ( _removed_seqs_out_base != null ) {
288 final String msg = writeAndAlignRemovedSeqs();
289 System.out.println();
290 System.out.println( msg );
295 public final void setGapRatio( final double gap_ratio ) {
296 _gap_ratio = gap_ratio;
299 public final void setMafftOptions( final String maffts_opts ) {
300 _maffts_opts = maffts_opts;
303 public final void setMinLength( final int min_length ) {
304 _min_length = min_length;
307 public final void setNorm( final boolean norm ) {
311 final public void setOutFileBase( final File out_file_base ) {
312 _out_file_base = out_file_base;
315 public final void setOutputFormat( final MSA_FORMAT output_format ) {
316 _output_format = output_format;
319 public void setPathToMafft( final String path_to_mafft ) {
320 _path_to_mafft = path_to_mafft;
323 public final void setRealign( final boolean realign ) {
327 public final void setRemovedSeqsOutBase( final File removed_seqs_out_base ) {
328 _removed_seqs_out_base = removed_seqs_out_base;
331 public final void setReportAlnMeanIdentity( final boolean report_aln_mean_identity ) {
332 _report_aln_mean_identity = report_aln_mean_identity;
335 public final void setStep( final int step ) {
339 public final void setStepForDiagnostics( final int step_for_diagnostics ) {
340 _step_for_diagnostics = step_for_diagnostics;
343 final public String writeAndAlignRemovedSeqs() throws IOException, InterruptedException {
344 final StringBuilder msg = new StringBuilder();
345 final String n = _removed_seqs_out_base + "_" + _removed_seqs.size() + ".fasta";
346 SequenceWriter.writeSeqs( _removed_seqs, new File( n ), SEQ_FORMAT.FASTA, 100 );
347 msg.append( "wrote " + _removed_seqs.size() + " removed sequences to " + "\"" + n + "\"" );
349 final MsaInferrer mafft = Mafft.createInstance( _path_to_mafft );
350 final List<String> opts = new ArrayList<String>();
351 for( final String o : _maffts_opts.split( "\\s" ) ) {
354 final Msa removed_msa = mafft.infer( _removed_seqs, opts );
355 final Double gr = MsaMethods.calcGapRatio( removed_msa );
356 String s = _removed_seqs_out_base + "_" + removed_msa.getNumberOfSequences() + "_"
357 + removed_msa.getLength() + "_" + ForesterUtil.roundToInt( gr * 100 );
358 final String suffix = obtainSuffix();
360 writeMsa( removed_msa, s, _output_format );
361 msg.append( ", and as MSA of length " + removed_msa.getLength() + " to \"" + s + "\"" );
363 return msg.toString();
366 final public String writeMsa( final File outfile ) throws IOException {
367 final Double gr = MsaMethods.calcGapRatio( _msa );
368 final String s = outfile + "_" + _msa.getNumberOfSequences() + "_" + _msa.getLength() + "_"
369 + ForesterUtil.roundToInt( gr * 100 );
370 writeMsa( _msa, s + obtainSuffix(), _output_format );
374 final int calcNonGapResidues( final Sequence seq ) {
376 for( int i = 0; i < seq.getLength(); ++i ) {
377 if ( !seq.isGapAt( i ) ) {
384 private final GapContribution[] calcGapContribtions( final boolean normalize_for_effective_seq_length ) {
385 final double gappiness[] = calcGappiness();
386 final GapContribution stats[] = new GapContribution[ _msa.getNumberOfSequences() ];
387 for( int row = 0; row < _msa.getNumberOfSequences(); ++row ) {
388 stats[ row ] = new GapContribution( _msa.getIdentifier( row ) );
389 for( int col = 0; col < _msa.getLength(); ++col ) {
390 if ( !_msa.isGapAt( row, col ) ) {
391 stats[ row ].addToValue( gappiness[ col ] );
394 if ( normalize_for_effective_seq_length ) {
395 stats[ row ].divideValue( calcNonGapResidues( _msa.getSequence( row ) ) );
398 stats[ row ].divideValue( _msa.getLength() );
404 final private GapContribution[] calcGapContribtionsStats( final boolean norm ) {
405 final GapContribution stats[] = calcGapContribtions( norm );
406 Arrays.sort( stats );
410 private final double[] calcGappiness() {
411 final int l = _msa.getLength();
412 final double gappiness[] = new double[ l ];
413 final int seqs = _msa.getNumberOfSequences();
414 for( int i = 0; i < l; ++i ) {
415 gappiness[ i ] = ( double ) MsaMethods.calcGapSumPerColumn( _msa, i ) / seqs;
420 private final Phylogeny inferNJphylogeny( final PWD_DISTANCE_METHOD pwd_distance_method,
422 final boolean write_matrix,
423 final String matrix_name ) {
424 BasicSymmetricalDistanceMatrix m = null;
425 switch ( pwd_distance_method ) {
426 case KIMURA_DISTANCE:
427 m = PairwiseDistanceCalculator.calcKimuraDistances( msa );
429 case POISSON_DISTANCE:
430 m = PairwiseDistanceCalculator.calcPoissonDistances( msa );
432 case FRACTIONAL_DISSIMILARITY:
433 m = PairwiseDistanceCalculator.calcFractionalDissimilarities( msa );
436 throw new IllegalArgumentException( "invalid pwd method" );
438 if ( write_matrix ) {
440 m.write( ForesterUtil.createBufferedWriter( matrix_name ) );
442 catch ( final IOException e ) {
446 final NeighborJoiningF nj = NeighborJoiningF.createInstance( false, 5 );
447 final Phylogeny phy = nj.execute( m );
451 private final boolean isPrintMsaStats( final int i ) {
452 return ( ( ( _step == 1 ) && ( _step_for_diagnostics == 1 ) ) || ( ( _step_for_diagnostics > 0 ) && ( ( ( i + 1 ) % _step_for_diagnostics ) == 0 ) ) );
455 private final boolean isPrintMsaStatsWriteOutfileAndRealign( final int i ) {
456 return ( ( ( _step == 1 ) && ( _step_for_diagnostics == 1 ) ) || ( ( _step > 0 ) && ( ( ( i + 1 ) % _step ) == 0 ) ) );
459 private final StringBuilder msaPropertiesAsSB( final MsaProperties msa_properties ) {
460 final StringBuilder sb = new StringBuilder();
461 sb.append( msa_properties.getNumberOfSequences() );
463 sb.append( msa_properties.getLength() );
465 sb.append( NF_4.format( msa_properties.getGapRatio() ) );
466 if ( _report_aln_mean_identity ) {
468 sb.append( NF_4.format( msa_properties.getAverageIdentityRatio() ) );
473 private String obtainSuffix() {
474 if ( _output_format == MSA_FORMAT.FASTA ) {
477 else if ( _output_format == MSA_FORMAT.PHYLIP ) {
483 private final Phylogeny pi( final String matrix, final int boostrap ) {
484 final Phylogeny master_phy = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, _msa, true, matrix );
487 final ResampleableMsa resampleable_msa = new ResampleableMsa( _msa );
488 final int[][] resampled_column_positions = BootstrapResampler.createResampledColumnPositions( _msa.getLength(),
491 final Phylogeny[] eval_phys = new Phylogeny[ n ];
492 for( int i = 0; i < n; ++i ) {
493 resampleable_msa.resample( resampled_column_positions[ i ] );
494 eval_phys[ i ] = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, resampleable_msa, false, null );
496 ConfidenceAssessor.evaluate( "bootstrap", eval_phys, master_phy, true, 1 );
497 PhylogenyMethods.extractFastaInformation( master_phy );
501 private final Phylogeny pi() {
502 final Phylogeny phy = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, _msa, false, "" );
503 PhylogenyMethods.midpointRoot( phy );
504 final boolean x = PhylogenyMethods.extractFastaInformation( phy );
506 final PhylogenyNodeIterator it = phy.iteratorExternalForward();
507 while ( it.hasNext() ) {
508 final PhylogenyNode n = it.next();
509 final String name = n.getName().trim();
510 if ( !ForesterUtil.isEmpty( name ) ) {
512 ParserUtils.extractTaxonomyDataFromNodeName( n, TAXONOMY_EXTRACTION.AGGRESSIVE );
514 catch ( final PhyloXmlDataFormatException e ) {
520 final Configuration config = new Configuration();
521 config.setDisplayAsPhylogram( true );
522 Archaeopteryx.createApplication( phy, config, _infile_name );
526 private final void printMsaProperties( final String id, final MsaProperties msa_properties ) {
527 if ( ( _step == 1 ) || ( _step_for_diagnostics == 1 ) ) {
528 System.out.print( ForesterUtil.pad( id, _longest_id_length, ' ', false ) );
529 System.out.print( "\t" );
531 System.out.print( msaPropertiesAsSB( msa_properties ) );
532 System.out.print( "\t" );
535 final private MsaProperties printMsaStatsWriteOutfileAndRealign( final boolean realign, final String id )
536 throws IOException, InterruptedException {
540 final MsaProperties msa_prop = new MsaProperties( _msa, _report_aln_mean_identity );
541 printMsaProperties( id, msa_prop );
542 final String s = writeOutfile();
543 System.out.print( "-> " + s + ( realign ? "\t(realigned)" : "" ) );
547 private final void printTableHeader() {
548 if ( ( _step == 1 ) || ( _step_for_diagnostics == 1 ) ) {
549 System.out.print( ForesterUtil.pad( "Id", _longest_id_length, ' ', false ) );
550 System.out.print( "\t" );
552 System.out.print( "Seqs" );
553 System.out.print( "\t" );
554 System.out.print( "Length" );
555 System.out.print( "\t" );
556 System.out.print( "Gaps" );
557 System.out.print( "\t" );
558 if ( _report_aln_mean_identity ) {
559 System.out.print( "MSA qual" );
560 System.out.print( "\t" );
562 System.out.println();
565 final private void realignWithMafft() throws IOException, InterruptedException {
566 final MsaInferrer mafft = Mafft.createInstance( _path_to_mafft );
567 final List<String> opts = new ArrayList<String>();
568 for( final String o : _maffts_opts.split( "\\s" ) ) {
571 _msa = DeleteableMsa.createInstance( mafft.infer( _msa.asSequenceList(), opts ) );
574 final private void removeGapColumns() {
575 _msa.deleteGapOnlyColumns();
578 private final String writeOutfile() throws IOException {
579 final String s = writeMsa( _out_file_base );
583 // Returns null if not path found.
584 final public static String guessPathToMafft() {
586 if ( ForesterUtil.OS_NAME.toLowerCase().indexOf( "win" ) >= 0 ) {
587 path = "C:\\Program Files\\mafft-win\\mafft.bat";
588 if ( MsaInferrer.isInstalled( path ) ) {
592 path = "/home/czmasek/SOFTWARE/MSA/MAFFT/mafft-7.130-without-extensions/scripts/mafft";
593 if ( MsaInferrer.isInstalled( path ) ) {
596 path = "/usr/local/bin/mafft";
597 if ( MsaInferrer.isInstalled( path ) ) {
600 path = "/usr/bin/mafft";
601 if ( MsaInferrer.isInstalled( path ) ) {
605 if ( MsaInferrer.isInstalled( path ) ) {
609 if ( MsaInferrer.isInstalled( path ) ) {
615 final private static void writeMsa( final Msa msa, final String outfile, final MSA_FORMAT format )
617 final Writer w = ForesterUtil.createBufferedWriter( outfile );
618 msa.write( w, format );
622 public void setPeformPhylogenticInference( final boolean phylogentic_inference ) {
623 _phylogentic_inference = phylogentic_inference;
626 public void setInfileName( final String infile_name ) {
627 _infile_name = infile_name;