2 // FORESTER -- software libraries and applications
3 // for evolutionary biology research and applications.
5 // Copyright (C) 2014 Christian M. Zmasek
6 // Copyright (C) 2014 Sanford-Burnham Medical Research Institute
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
23 // WWW: https://sites.google.com/site/cmzmasek/home/software/forester
25 package org.forester.msa_compactor;
28 import java.io.IOException;
29 import java.io.Writer;
30 import java.math.RoundingMode;
31 import java.text.DecimalFormat;
32 import java.text.NumberFormat;
33 import java.util.ArrayList;
34 import java.util.Arrays;
35 import java.util.List;
36 import java.util.SortedSet;
37 import java.util.TreeSet;
39 import org.forester.evoinference.distance.NeighborJoiningF;
40 import org.forester.evoinference.distance.PairwiseDistanceCalculator;
41 import org.forester.evoinference.distance.PairwiseDistanceCalculator.PWD_DISTANCE_METHOD;
42 import org.forester.evoinference.matrix.distance.BasicSymmetricalDistanceMatrix;
43 import org.forester.evoinference.tools.BootstrapResampler;
44 import org.forester.msa.BasicMsa;
45 import org.forester.msa.Mafft;
46 import org.forester.msa.Msa;
47 import org.forester.msa.Msa.MSA_FORMAT;
48 import org.forester.msa.MsaInferrer;
49 import org.forester.msa.MsaMethods;
50 import org.forester.msa.ResampleableMsa;
51 import org.forester.phylogeny.Phylogeny;
52 import org.forester.phylogeny.PhylogenyMethods;
53 import org.forester.sequence.Sequence;
54 import org.forester.tools.ConfidenceAssessor;
55 import org.forester.util.ForesterUtil;
57 public class MsaCompactor {
59 final private static NumberFormat NF_3 = new DecimalFormat( "#.###" );
60 final private static NumberFormat NF_4 = new DecimalFormat( "#.####" );
61 private final String _maffts_opts = "--retree 1";
63 private File _out_file_base;
64 private String _path_to_mafft;
65 private final SortedSet<String> _removed_seq_ids;
67 NF_4.setRoundingMode( RoundingMode.HALF_UP );
68 NF_3.setRoundingMode( RoundingMode.HALF_UP );
71 private MsaCompactor( final Msa msa ) {
73 _removed_seq_ids = new TreeSet<String>();
76 final public Msa getMsa() {
80 final public SortedSet<String> getRemovedSeqIds() {
81 return _removed_seq_ids;
84 final public void setOutFileBase( final File out_file_base ) {
85 _out_file_base = out_file_base;
88 final public String writeMsa( final File outfile, final MSA_FORMAT format, final String suffix ) throws IOException {
89 final Double gr = MsaMethods.calcGapRatio( _msa );
90 final String s = outfile + "_" + _msa.getNumberOfSequences() + "_" + _msa.getLength() + "_"
91 + ForesterUtil.roundToInt( gr * 100 );
92 writeMsa( s + suffix, format );
96 final int calcNonGapResidues( final Sequence seq ) {
98 for( int i = 0; i < seq.getLength(); ++i ) {
99 if ( !seq.isGapAt( i ) ) {
106 Phylogeny pi( final String matrix ) {
107 final Phylogeny master_phy = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, _msa, true, matrix );
110 final ResampleableMsa resampleable_msa = new ResampleableMsa( ( BasicMsa ) _msa );
111 final int[][] resampled_column_positions = BootstrapResampler.createResampledColumnPositions( _msa.getLength(),
114 final Phylogeny[] eval_phys = new Phylogeny[ n ];
115 for( int i = 0; i < n; ++i ) {
116 resampleable_msa.resample( resampled_column_positions[ i ] );
117 eval_phys[ i ] = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, resampleable_msa, false, null );
119 ConfidenceAssessor.evaluate( "bootstrap", eval_phys, master_phy, true, 1 );
120 PhylogenyMethods.extractFastaInformation( master_phy );
124 private final GapContribution[] calcGapContribtions( final boolean normalize_for_effective_seq_length ) {
125 final double gappiness[] = calcGappiness();
126 final GapContribution stats[] = new GapContribution[ _msa.getNumberOfSequences() ];
127 for( int row = 0; row < _msa.getNumberOfSequences(); ++row ) {
128 stats[ row ] = new GapContribution( _msa.getIdentifier( row ) );
129 for( int col = 0; col < _msa.getLength(); ++col ) {
130 if ( !_msa.isGapAt( row, col ) ) {
131 stats[ row ].addToValue( gappiness[ col ] );
134 if ( normalize_for_effective_seq_length ) {
135 stats[ row ].divideValue( calcNonGapResidues( _msa.getSequence( row ) ) );
138 stats[ row ].divideValue( _msa.getLength() );
144 final private GapContribution[] calcGapContribtionsStats( final boolean norm ) {
145 final GapContribution stats[] = calcGapContribtions( norm );
146 Arrays.sort( stats );
150 private final double[] calcGappiness() {
151 final int l = _msa.getLength();
152 final double gappiness[] = new double[ l ];
153 final int seqs = _msa.getNumberOfSequences();
154 for( int i = 0; i < l; ++i ) {
155 gappiness[ i ] = ( double ) MsaMethods.calcGapSumPerColumn( _msa, i ) / seqs;
160 private Phylogeny inferNJphylogeny( final PWD_DISTANCE_METHOD pwd_distance_method,
162 final boolean write_matrix,
163 final String matrix_name ) {
164 BasicSymmetricalDistanceMatrix m = null;
165 switch ( pwd_distance_method ) {
166 case KIMURA_DISTANCE:
167 m = PairwiseDistanceCalculator.calcKimuraDistances( msa );
169 case POISSON_DISTANCE:
170 m = PairwiseDistanceCalculator.calcPoissonDistances( msa );
172 case FRACTIONAL_DISSIMILARITY:
173 m = PairwiseDistanceCalculator.calcFractionalDissimilarities( msa );
176 throw new IllegalArgumentException( "invalid pwd method" );
178 if ( write_matrix ) {
180 m.write( ForesterUtil.createBufferedWriter( matrix_name ) );
182 catch ( final IOException e ) {
186 final NeighborJoiningF nj = NeighborJoiningF.createInstance( false, 5 );
187 final Phylogeny phy = nj.execute( m );
191 private StringBuilder msaStatsAsSB() {
192 final StringBuilder sb = new StringBuilder();
193 sb.append( _msa.getNumberOfSequences() );
195 sb.append( _msa.getLength() );
197 sb.append( NF_4.format( MsaMethods.calcGapRatio( _msa ) ) );
199 sb.append( NF_4.format( MsaMethods.calculateIdentityRatio( 0, _msa.getLength() - 1, _msa ).arithmeticMean() ) );
203 private final void printMsaStats( final String id ) {
204 System.out.print( ForesterUtil.pad( id, 20, ' ', false ) );
205 System.out.print( "\t" );
206 final StringBuilder sb = msaStatsAsSB();
207 System.out.print( sb );
208 System.out.print( "\t" );
211 final private void realignWithMafft() throws IOException, InterruptedException {
212 // final MsaInferrer mafft = Mafft
213 // .createInstance( "/home/czmasek/SOFTWARE/MSA/MAFFT/mafft-7.130-without-extensions/scripts/mafft" );
214 final MsaInferrer mafft = Mafft.createInstance( _path_to_mafft );
215 final List<String> opts = new ArrayList<String>();
216 for( final String o : _maffts_opts.split( "\\s" ) ) {
219 //opts.add( "--maxiterate" );
220 //opts.add( "1000" );
221 //opts.add( "--localpair" );
222 //opts.add( "--quiet" );
223 _msa = mafft.infer( _msa.asSequenceList(), opts );
226 final private void removeGapColumns() {
227 _msa = MsaMethods.createInstance().removeGapColumns( 1, 0, _msa );
230 final private void removeViaGapAverage( final double mean_gapiness,
232 final boolean realign,
234 final boolean verbose ) throws IOException, InterruptedException {
235 final GapContribution stats[] = calcGapContribtionsStats( norm );
236 final List<String> to_remove_ids = new ArrayList<String>();
237 for( final GapContribution gap_gontribution : stats ) {
238 to_remove_ids.add( gap_gontribution.getId() );
244 while ( MsaMethods.calcGapRatio( _msa ) > mean_gapiness ) {
245 final String id = to_remove_ids.get( i );
246 _msa = MsaMethods.removeSequence( _msa, id );
251 if ( ( ( step > 0 ) && ( ( ( i + 1 ) % step ) == 0 ) )
252 || ( MsaMethods.calcGapRatio( _msa ) <= mean_gapiness ) ) {
256 final String s = writeOutfile();
258 System.out.print( "-> " + s );
262 System.out.println();
268 final private List<MsaProperties> chart( final boolean realign, final boolean norm, final boolean verbose )
269 throws IOException, InterruptedException {
270 final GapContribution stats[] = calcGapContribtionsStats( norm );
271 final List<String> to_remove_ids = new ArrayList<String>();
272 final List<MsaProperties> msa_props = new ArrayList<MsaProperties>();
273 for( final GapContribution gap_gontribution : stats ) {
274 to_remove_ids.add( gap_gontribution.getId() );
280 final int x = ForesterUtil.roundToInt( _msa.getNumberOfSequences() / 20.0 );
281 while ( _msa.getNumberOfSequences() > x ) {
282 final String id = to_remove_ids.get( i );
283 _msa = MsaMethods.removeSequence( _msa, id );
285 msa_props.add( new MsaProperties( _msa ) );
293 System.out.println();
300 final private void removeViaLength( final int length,
302 final boolean realign,
304 final boolean verbose ) throws IOException, InterruptedException {
305 final GapContribution stats[] = calcGapContribtionsStats( norm );
306 final List<String> to_remove_ids = new ArrayList<String>();
307 for( final GapContribution gap_gontribution : stats ) {
308 to_remove_ids.add( gap_gontribution.getId() );
314 while ( _msa.getLength() > length ) {
315 final String id = to_remove_ids.get( i );
316 _msa = MsaMethods.removeSequence( _msa, id );
321 if ( ( ( step > 0 ) && ( ( ( i + 1 ) % step ) == 0 ) ) || ( _msa.getLength() <= length ) ) {
325 final String s = writeOutfile();
327 System.out.print( "-> " + s );
331 System.out.println();
337 final private void removeWorstOffenders( final int to_remove,
339 final boolean realign,
341 final boolean verbose ) throws IOException, InterruptedException {
342 final GapContribution stats[] = calcGapContribtionsStats( norm );
343 final List<String> to_remove_ids = new ArrayList<String>();
344 for( int j = 0; j < to_remove; ++j ) {
345 to_remove_ids.add( stats[ j ].getId() );
346 _removed_seq_ids.add( stats[ j ].getId() );
351 for( int i = 0; i < to_remove_ids.size(); ++i ) {
352 final String id = to_remove_ids.get( i );
353 _msa = MsaMethods.removeSequence( _msa, id );
358 if ( ( ( step > 0 ) && ( ( ( i + 1 ) % step ) == 0 ) ) || ( i == ( to_remove_ids.size() - 1 ) ) ) {
362 final String s = writeOutfile();
364 System.out.print( "-> " + s );
368 System.out.println();
373 private void setPathToMafft( final String path_to_mafft ) {
374 _path_to_mafft = path_to_mafft;
377 final private void writeMsa( final String outfile, final MSA_FORMAT format ) throws IOException {
378 final Writer w = ForesterUtil.createBufferedWriter( outfile );
379 _msa.write( w, format );
383 private String writeOutfile() throws IOException {
384 final String s = writeMsa( _out_file_base, MSA_FORMAT.PHYLIP, ".aln" );
385 //writeMsa( _out_file_base, MSA_FORMAT.FASTA, ".fasta" );
389 // Returns null if not path found.
390 final public static String guessPathToMafft() {
392 if ( ForesterUtil.OS_NAME.toLowerCase().indexOf( "win" ) >= 0 ) {
393 path = "C:\\Program Files\\mafft-win\\mafft.bat";
394 if ( MsaInferrer.isInstalled( path ) ) {
398 path = "/home/czmasek/SOFTWARE/MSA/MAFFT/mafft-7.130-without-extensions/scripts/mafft";
399 if ( MsaInferrer.isInstalled( path ) ) {
402 path = "/usr/local/bin/mafft";
403 if ( MsaInferrer.isInstalled( path ) ) {
406 path = "/usr/bin/mafft";
407 if ( MsaInferrer.isInstalled( path ) ) {
411 if ( MsaInferrer.isInstalled( path ) ) {
415 if ( MsaInferrer.isInstalled( path ) ) {
421 public final static MsaCompactor reduceGapAverage( final Msa msa,
422 final double max_gap_average,
424 final boolean realign,
426 final String path_to_mafft,
427 final File out ) throws IOException, InterruptedException {
428 final MsaCompactor mc = new MsaCompactor( msa );
430 mc.setPathToMafft( path_to_mafft );
432 mc.setOutFileBase( out );
433 mc.removeViaGapAverage( max_gap_average, step, realign, norm, true );
437 public final static MsaCompactor reduceLength( final Msa msa,
440 final boolean realign,
442 final String path_to_mafft,
443 final File out ) throws IOException, InterruptedException {
444 final MsaCompactor mc = new MsaCompactor( msa );
446 mc.setPathToMafft( path_to_mafft );
448 mc.setOutFileBase( out );
449 mc.removeViaLength( length, step, realign, norm, true );
453 public final static MsaCompactor chart( final Msa msa,
454 final boolean realign,
456 final String path_to_mafft ) throws IOException, InterruptedException {
457 final MsaCompactor mc = new MsaCompactor( msa );
459 mc.setPathToMafft( path_to_mafft );
461 final List<MsaProperties> msa_props = mc.chart( realign, norm, true );
462 Chart.display( msa_props );
466 public final static MsaCompactor removeWorstOffenders( final Msa msa,
467 final int worst_offenders_to_remove,
469 final boolean realign,
471 final String path_to_mafft,
472 final File out ) throws IOException, InterruptedException {
473 final MsaCompactor mc = new MsaCompactor( msa );
475 mc.setPathToMafft( path_to_mafft );
477 mc.setOutFileBase( out );
478 mc.removeWorstOffenders( worst_offenders_to_remove, step, realign, norm, true );
482 private final static void printTableHeader() {
483 System.out.print( ForesterUtil.pad( "Id", 20, ' ', false ) );
484 System.out.print( "\t" );
485 System.out.print( "Seqs" );
486 System.out.print( "\t" );
487 System.out.print( "Length" );
488 System.out.print( "\t" );
489 System.out.print( "Gaps" );
490 System.out.print( "\t" );
491 System.out.print( "MSA qual" );
492 System.out.print( "\t" );
493 System.out.println();