2 package org.forester.msa_compactor;
5 import java.io.IOException;
7 import java.math.RoundingMode;
8 import java.text.DecimalFormat;
9 import java.text.NumberFormat;
10 import java.util.ArrayList;
11 import java.util.Arrays;
12 import java.util.List;
13 import java.util.SortedSet;
14 import java.util.TreeSet;
16 import org.forester.archaeopteryx.Archaeopteryx;
17 import org.forester.evoinference.distance.NeighborJoiningF;
18 import org.forester.evoinference.distance.PairwiseDistanceCalculator;
19 import org.forester.evoinference.distance.PairwiseDistanceCalculator.PWD_DISTANCE_METHOD;
20 import org.forester.evoinference.matrix.distance.BasicSymmetricalDistanceMatrix;
21 import org.forester.evoinference.tools.BootstrapResampler;
22 import org.forester.msa.BasicMsa;
23 import org.forester.msa.Mafft;
24 import org.forester.msa.Msa;
25 import org.forester.msa.Msa.MSA_FORMAT;
26 import org.forester.msa.MsaInferrer;
27 import org.forester.msa.MsaMethods;
28 import org.forester.msa.ResampleableMsa;
29 import org.forester.phylogeny.Phylogeny;
30 import org.forester.phylogeny.PhylogenyMethods;
31 import org.forester.sequence.Sequence;
32 import org.forester.tools.ConfidenceAssessor;
33 import org.forester.util.ForesterUtil;
35 public class MsaCompactor {
37 final private static NumberFormat NF_3 = new DecimalFormat( "#.###" );
38 final private static NumberFormat NF_4 = new DecimalFormat( "#.####" );
39 private static final boolean VERBOSE = true;
41 private final SortedSet<String> _removed_seq_ids;
42 private String _path_to_mafft;
44 NF_4.setRoundingMode( RoundingMode.HALF_UP );
45 NF_3.setRoundingMode( RoundingMode.HALF_UP );
48 private MsaCompactor( final Msa msa ) {
50 _removed_seq_ids = new TreeSet<String>();
53 final public Msa getMsa() {
57 final public SortedSet<String> getRemovedSeqIds() {
58 return _removed_seq_ids;
61 final public void writeMsa( final File outfile, final MSA_FORMAT format, final String suffix ) throws IOException {
62 final Double gr = MsaMethods.calcGapRatio( _msa );
63 writeMsa( outfile + "_" + _msa.getNumberOfSequences() + "_" + _msa.getLength() + "_"
64 + ForesterUtil.roundToInt( gr * 100 ) + suffix,
68 final int calcNonGapResidues( final Sequence seq ) {
70 for( int i = 0; i < seq.getLength(); ++i ) {
71 if ( !seq.isGapAt( i ) ) {
78 private final GapContribution[] calcGapContribtions( final boolean normalize_for_effective_seq_length ) {
79 final double gappiness[] = calcGappiness();
80 final GapContribution stats[] = new GapContribution[ _msa.getNumberOfSequences() ];
81 for( int row = 0; row < _msa.getNumberOfSequences(); ++row ) {
82 stats[ row ] = new GapContribution( _msa.getIdentifier( row ) );
83 for( int col = 0; col < _msa.getLength(); ++col ) {
84 if ( !_msa.isGapAt( row, col ) ) {
85 stats[ row ].addToValue( gappiness[ col ] );
88 if ( normalize_for_effective_seq_length ) {
89 stats[ row ].divideValue( calcNonGapResidues( _msa.getSequence( row ) ) );
92 stats[ row ].divideValue( _msa.getLength() );
98 final private GapContribution[] calcGapContribtionsStats( final boolean norm ) {
99 final GapContribution stats[] = calcGapContribtions( norm );
100 Arrays.sort( stats );
101 for( final GapContribution stat : stats ) {
102 final StringBuilder sb = new StringBuilder();
103 sb.append( stat.getId() );
105 sb.append( NF_4.format( stat.getValue() ) );
107 // sb.append( NF_4.format( stat.median() ) );
108 // sb.append( "\t" );
109 // sb.append( NF_4.format( stat.getMin() ) );
110 // sb.append( "\t" );
111 // sb.append( NF_4.format( stat.getMax() ) );
113 System.out.println( sb );
118 private final double[] calcGappiness() {
119 final int l = _msa.getLength();
120 final double gappiness[] = new double[ l ];
121 final int seqs = _msa.getNumberOfSequences();
122 for( int i = 0; i < l; ++i ) {
123 gappiness[ i ] = ( double ) MsaMethods.calcGapSumPerColumn( _msa, i ) / seqs;
128 // Returns null if not path found.
129 final public static String guessPathToMafft() {
131 if ( ForesterUtil.OS_NAME.toLowerCase().indexOf( "win" ) >= 0 ) {
132 path = "C:\\Program Files\\mafft-win\\mafft.bat";
133 if ( MsaInferrer.isInstalled( path ) ) {
138 path = "/usr/local/bin/mafft";
139 if ( MsaInferrer.isInstalled( path ) ) {
142 path = "/usr/bin/mafft";
143 if ( MsaInferrer.isInstalled( path ) ) {
147 if ( MsaInferrer.isInstalled( path ) ) {
151 if ( MsaInferrer.isInstalled( path ) ) {
158 final private void mafft() throws IOException, InterruptedException {
159 // final MsaInferrer mafft = Mafft
160 // .createInstance( "/home/czmasek/SOFTWARE/MSA/MAFFT/mafft-7.130-without-extensions/scripts/mafft" );
162 final MsaInferrer mafft = Mafft
163 .createInstance( _path_to_mafft );
166 final List<String> opts = new ArrayList<String>();
167 opts.add( "--maxiterate" );
169 opts.add( "--localpair" );
170 opts.add( "--quiet" );
171 _msa = mafft.infer( _msa.asSequenceList(), opts );
174 private StringBuilder msaStatsAsSB() {
175 final StringBuilder sb = new StringBuilder();
176 sb.append( _msa.getNumberOfSequences() );
178 sb.append( _msa.getLength() );
180 sb.append( NF_3.format( MsaMethods.calcGapRatio( _msa ) ) );
184 final private void removeGapColumns() {
185 _msa = MsaMethods.createInstance().removeGapColumns( 1, 0, _msa );
188 final private void removeViaGapAverage( final double mean_gapiness,
190 final boolean realign,
192 final int minimal_effective_length ) throws IOException,
193 InterruptedException {
195 throw new IllegalArgumentException( "step cannot be less than 1" );
197 if ( mean_gapiness < 0 ) {
198 throw new IllegalArgumentException( "target average gap ratio cannot be less than 0" );
201 System.out.println( "orig: " + msaStatsAsSB() );
203 if ( minimal_effective_length > 1 ) {
204 _msa = MsaMethods.removeSequencesByMinimalLength( _msa, minimal_effective_length );
206 System.out.println( "short seq removal: " + msaStatsAsSB() );
212 removeWorstOffenders( step, 1, false, false );
216 gr = MsaMethods.calcGapRatio( _msa );
218 System.out.println( counter + ": " + msaStatsAsSB() );
220 // write( outfile, gr );
222 } while ( gr > mean_gapiness );
224 System.out.println( "final: " + msaStatsAsSB() );
228 final private void removeViaLength( final int length, final int step, final boolean realign ) throws IOException,
229 InterruptedException {
231 throw new IllegalArgumentException( "step cannot be less than 1" );
234 throw new IllegalArgumentException( "target length cannot be less than 1" );
237 System.out.println( "orig: " + msaStatsAsSB() );
240 while ( _msa.getLength() > length ) {
241 removeWorstOffenders( step, 1, false, false );
246 System.out.println( counter + ": " + msaStatsAsSB() );
252 Phylogeny pi( final String matrix ) {
253 final Phylogeny master_phy = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, _msa, true, matrix );
256 final ResampleableMsa resampleable_msa = new ResampleableMsa( ( BasicMsa ) _msa );
257 final int[][] resampled_column_positions = BootstrapResampler.createResampledColumnPositions( _msa.getLength(),
260 final Phylogeny[] eval_phys = new Phylogeny[ n ];
261 for( int i = 0; i < n; ++i ) {
262 resampleable_msa.resample( resampled_column_positions[ i ] );
263 eval_phys[ i ] = inferNJphylogeny( PWD_DISTANCE_METHOD.KIMURA_DISTANCE, resampleable_msa, false, null );
265 ConfidenceAssessor.evaluate( "bootstrap", eval_phys, master_phy, true, 1 );
266 PhylogenyMethods.extractFastaInformation( master_phy );
270 private Phylogeny inferNJphylogeny( final PWD_DISTANCE_METHOD pwd_distance_method,
272 final boolean write_matrix,
273 final String matrix_name ) {
274 BasicSymmetricalDistanceMatrix m = null;
275 switch ( pwd_distance_method ) {
276 case KIMURA_DISTANCE:
277 m = PairwiseDistanceCalculator.calcKimuraDistances( msa );
279 case POISSON_DISTANCE:
280 m = PairwiseDistanceCalculator.calcPoissonDistances( msa );
282 case FRACTIONAL_DISSIMILARITY:
283 m = PairwiseDistanceCalculator.calcFractionalDissimilarities( msa );
286 throw new IllegalArgumentException( "invalid pwd method" );
288 if ( write_matrix ) {
290 m.write( ForesterUtil.createBufferedWriter( matrix_name ) );
292 catch ( final IOException e ) {
293 // TODO Auto-generated catch block
297 final NeighborJoiningF nj = NeighborJoiningF.createInstance( false, 5 );
298 final Phylogeny phy = nj.execute( m );
302 final private void removeWorstOffenders( final int to_remove,
304 final boolean realign,
305 final boolean norm ) throws IOException, InterruptedException {
306 final Phylogeny a = pi( "a.pwd" );
307 Archaeopteryx.createApplication( a );
308 final GapContribution stats[] = calcGapContribtionsStats( norm );
309 final List<String> to_remove_ids = new ArrayList<String>();
310 for( int j = 0; j < to_remove; ++j ) {
311 to_remove_ids.add( stats[ j ].getId() );
312 _removed_seq_ids.add( stats[ j ].getId() );
314 //TODO if verbose/interactve
315 for( final String id : to_remove_ids ) {
316 _msa = MsaMethods.removeSequence( _msa, id );
318 System.out.print( id );
319 System.out.print( "\t" );
320 final StringBuilder sb = msaStatsAsSB();
321 System.out.println( sb );
324 //_msa = MsaMethods.removeSequences( _msa, to_remove_ids );
325 //removeGapColumns();
329 final Phylogeny b = pi( "b.pwd" );
330 Archaeopteryx.createApplication( b );
333 final private void writeMsa( final String outfile, final MSA_FORMAT format ) throws IOException {
334 final Writer w = ForesterUtil.createBufferedWriter( outfile );
335 _msa.write( w, format );
339 public final static MsaCompactor reduceGapAverage( final Msa msa,
340 final double max_gap_average,
342 final boolean realign,
344 final int minimal_effective_length,
345 final String path_to_mafft ) throws IOException,
346 InterruptedException {
347 final MsaCompactor mc = new MsaCompactor( msa );
349 mc.setPathToMafft( path_to_mafft );
351 mc.removeViaGapAverage( max_gap_average, step, realign, out, minimal_effective_length );
355 public final static MsaCompactor reduceLength( final Msa msa,
358 final boolean realign,
359 final String path_to_mafft ) throws IOException, InterruptedException {
360 final MsaCompactor mc = new MsaCompactor( msa );
362 mc.setPathToMafft( path_to_mafft );
364 mc.removeViaLength( length, step, realign );
368 public final static MsaCompactor removeWorstOffenders( final Msa msa,
369 final int worst_offenders_to_remove,
370 final boolean realign,
372 final String path_to_mafft ) throws IOException,
373 InterruptedException {
374 final MsaCompactor mc = new MsaCompactor( msa );
376 mc.setPathToMafft( path_to_mafft );
378 mc.removeWorstOffenders( worst_offenders_to_remove, 1, realign, norm );
382 private void setPathToMafft( final String path_to_mafft ) {
383 _path_to_mafft = path_to_mafft;