2 // FORESTER -- software libraries and applications
3 // for evolutionary biology research and applications.
5 // Copyright (C) 2014 Christian M. Zmasek
6 // Copyright (C) 2014 Sanford-Burnham Medical Research Institute
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
23 // WWW: https://sites.google.com/site/cmzmasek/home/software/forester
25 package org.forester.application;
28 import java.io.FileInputStream;
29 import java.io.IOException;
30 import java.math.RoundingMode;
31 import java.text.DecimalFormat;
32 import java.text.NumberFormat;
33 import java.util.ArrayList;
34 import java.util.List;
36 import org.forester.io.parsers.FastaParser;
37 import org.forester.io.parsers.GeneralMsaParser;
38 import org.forester.msa.DeleteableMsa;
39 import org.forester.msa.Msa.MSA_FORMAT;
40 import org.forester.msa.MsaInferrer;
41 import org.forester.msa.MsaMethods;
42 import org.forester.msa_compactor.Chart;
43 import org.forester.msa_compactor.MsaCompactor;
44 import org.forester.msa_compactor.MsaProperties;
45 import org.forester.util.CommandLineArguments;
46 import org.forester.util.DescriptiveStatistics;
47 import org.forester.util.ForesterUtil;
51 java -cp C:\Users\czmasek\SOFTWARE_DEV\ECLIPSE\forester\java\fo
52 rester.jar org.forester.application.msa_compactor Bcl-2_e1_20_mafft -t
55 public class msa_compactor {
57 final private static NumberFormat NF_1 = new DecimalFormat( "0.#" );
58 final private static NumberFormat NF_4 = new DecimalFormat( "0.####" );
60 NF_1.setRoundingMode( RoundingMode.HALF_UP );
61 NF_4.setRoundingMode( RoundingMode.HALF_UP );
63 final static private String HELP_OPTION_1 = "help";
64 final static private String HELP_OPTION_2 = "h";
65 final static private String REMOVE_WORST_OFFENDERS_OPTION = "r";
66 final static private String AV_GAPINESS_OPTION = "g";
67 final static private String STEP_OPTION = "s";
68 final static private String LENGTH_OPTION = "l";
69 final static private String REALIGN_OPTION = "a";
70 final static private String INFO_ONLY_OPTION = "i";
72 final static private String STEP_FOR_DIAGNOSTICS_OPTION = "sd";
73 final static private String MIN_LENGTH_OPTION = "ml";
74 final static private String GAP_RATIO_LENGTH_OPTION = "gr";
75 final static private String REPORT_ENTROPY = "e";
76 final static private String OUTPUT_FORMAT_OPTION = "f";
77 final static private String OUTPUT_REMOVED_SEQS_OPTION = "ro";
78 final static private String MAFFT_OPTIONS = "mo";
79 final static private String PERFORM_PHYLOGENETIC_INFERENCE = "t";
81 final static private String PATH_TO_MAFFT_OPTION = "mafft";
82 final static private String DO_NOT_NORMALIZE_FOR_EFF_LENGTH_OPTION = "nn";
83 final static private String PRG_NAME = "msa_compactor";
84 final static private String PRG_DESC = "multiple sequence aligment compactor";
85 final static private String PRG_VERSION = "0.3";
86 final static private String PRG_DATE = "140508";
87 final static private String E_MAIL = "czmasek@sanfordburham.org";
88 final static private String WWW = "https://sites.google.com/site/cmzmasek/home/software/forester";
90 public static void main( final String args[] ) {
92 final CommandLineArguments cla = new CommandLineArguments( args );
93 if ( cla.isOptionSet( HELP_OPTION_1 ) || cla.isOptionSet( HELP_OPTION_2 )
94 || ( ( cla.getNumberOfNames() < 1 ) || ( cla.getNumberOfNames() > 2 ) ) ) {
98 final File in = cla.getFile( 0 );
100 if ( cla.getNumberOfNames() > 1 ) {
101 out = cla.getFile( 1 );
103 int worst_remove = -1;
107 boolean realign = false;
108 boolean normalize_for_effective_seq_length = true;
109 String path_to_mafft = null;
110 int step_for_diagnostics = 1;
112 double gap_ratio = -1;
113 boolean report_entropy = false;
114 MSA_FORMAT output_format = MSA_FORMAT.FASTA;
115 File removed_seqs_out_base = null;
116 String mafft_options = "--auto";
117 boolean perform_phylogenetic_inference = false;
118 final List<String> allowed_options = new ArrayList<String>();
119 allowed_options.add( REMOVE_WORST_OFFENDERS_OPTION );
120 allowed_options.add( AV_GAPINESS_OPTION );
121 allowed_options.add( LENGTH_OPTION );
122 allowed_options.add( REALIGN_OPTION );
123 allowed_options.add( DO_NOT_NORMALIZE_FOR_EFF_LENGTH_OPTION );
124 allowed_options.add( STEP_OPTION );
125 allowed_options.add( PATH_TO_MAFFT_OPTION );
126 allowed_options.add( STEP_FOR_DIAGNOSTICS_OPTION );
127 allowed_options.add( MIN_LENGTH_OPTION );
128 allowed_options.add( GAP_RATIO_LENGTH_OPTION );
129 allowed_options.add( REPORT_ENTROPY );
130 allowed_options.add( OUTPUT_FORMAT_OPTION );
131 allowed_options.add( OUTPUT_REMOVED_SEQS_OPTION );
132 allowed_options.add( MAFFT_OPTIONS );
133 allowed_options.add( PERFORM_PHYLOGENETIC_INFERENCE );
134 allowed_options.add( INFO_ONLY_OPTION );
135 final String dissallowed_options = cla.validateAllowedOptionsAsString( allowed_options );
136 if ( dissallowed_options.length() > 0 ) {
137 ForesterUtil.fatalError( PRG_NAME, "unknown option(s): " + dissallowed_options );
139 DeleteableMsa msa = null;
140 final FileInputStream is = new FileInputStream( in );
141 if ( FastaParser.isLikelyFasta( in ) ) {
142 msa = DeleteableMsa.createInstance( FastaParser.parseMsa( is ) );
145 msa = DeleteableMsa.createInstance( GeneralMsaParser.parseMsa( is ) );
147 final DescriptiveStatistics initial_msa_stats = MsaMethods.calculateEffectiveLengthStatistics( msa );
148 if (cla.isOptionSet( INFO_ONLY_OPTION ) ) {
149 printInfo( in, msa, initial_msa_stats );
153 final boolean chart_only = ( !cla.isOptionSet( LENGTH_OPTION ) )
154 && ( !cla.isOptionSet( REMOVE_WORST_OFFENDERS_OPTION ) )
155 && ( !cla.isOptionSet( AV_GAPINESS_OPTION ) && ( !cla.isOptionSet( MIN_LENGTH_OPTION ) ) );
156 if ( !chart_only && ( out == null ) ) {
157 ForesterUtil.fatalError( PRG_NAME, "outfile file missing" );
159 if ( cla.isOptionSet( REMOVE_WORST_OFFENDERS_OPTION ) ) {
160 worst_remove = cla.getOptionValueAsInt( REMOVE_WORST_OFFENDERS_OPTION );
161 if ( ( worst_remove < 1 ) || ( worst_remove >= ( msa.getNumberOfSequences() - 1 ) ) ) {
162 ForesterUtil.fatalError( PRG_NAME, "number of worst offender sequences to remove is out of range: "
166 if ( cla.isOptionSet( AV_GAPINESS_OPTION ) ) {
167 if ( cla.isOptionSet( REMOVE_WORST_OFFENDERS_OPTION ) ) {
171 av_gap = cla.getOptionValueAsDouble( AV_GAPINESS_OPTION );
172 if ( ( av_gap < 0 ) || ( av_gap >= 1 ) ) {
173 ForesterUtil.fatalError( PRG_NAME, "target gap-ratio is out of range: " + av_gap );
176 if ( cla.isOptionSet( LENGTH_OPTION ) ) {
177 if ( cla.isOptionSet( REMOVE_WORST_OFFENDERS_OPTION ) || cla.isOptionSet( AV_GAPINESS_OPTION ) ) {
181 length = cla.getOptionValueAsInt( LENGTH_OPTION );
182 if ( length >= msa.getLength() ) {
183 ForesterUtil.fatalError( PRG_NAME,
184 "target length is out of range [longer than MSA (" + msa.getLength()
187 else if ( length < initial_msa_stats.getMin() ) {
188 ForesterUtil.fatalError( PRG_NAME,
189 "target length is out of range [shorter than the shortest sequence ("
190 + initial_msa_stats.getMin() + ") ]: " + length );
193 if ( cla.isOptionSet( MIN_LENGTH_OPTION ) ) {
194 if ( cla.isOptionSet( LENGTH_OPTION ) || cla.isOptionSet( REMOVE_WORST_OFFENDERS_OPTION )
195 || cla.isOptionSet( AV_GAPINESS_OPTION ) || cla.isOptionSet( STEP_OPTION )
196 || cla.isOptionSet( REALIGN_OPTION ) || cla.isOptionSet( PATH_TO_MAFFT_OPTION )
197 || cla.isOptionSet( STEP_FOR_DIAGNOSTICS_OPTION ) || cla.isOptionSet( REPORT_ENTROPY )
198 || cla.isOptionSet( OUTPUT_REMOVED_SEQS_OPTION )
199 || cla.isOptionSet( PERFORM_PHYLOGENETIC_INFERENCE ) ) {
203 min_length = cla.getOptionValueAsInt( MIN_LENGTH_OPTION );
204 if ( ( min_length < 2 ) || ( min_length > initial_msa_stats.getMax() ) ) {
205 ForesterUtil.fatalError( PRG_NAME, "value for minimal sequence length is out of range: "
209 if ( cla.isOptionSet( STEP_OPTION ) ) {
210 step = cla.getOptionValueAsInt( STEP_OPTION );
212 || ( ( step > msa.getNumberOfSequences() ) || ( ( worst_remove > 0 ) && ( step > worst_remove ) ) ) ) {
213 ForesterUtil.fatalError( PRG_NAME, "value for step is out of range: " + step );
216 if ( cla.isOptionSet( REALIGN_OPTION ) ) {
219 if ( cla.isOptionSet( PATH_TO_MAFFT_OPTION ) ) {
221 ForesterUtil.fatalError( PRG_NAME, "no need to indicate path to MAFFT without realigning" );
223 path_to_mafft = cla.getOptionValueAsCleanString( PATH_TO_MAFFT_OPTION );
225 if ( cla.isOptionSet( DO_NOT_NORMALIZE_FOR_EFF_LENGTH_OPTION ) ) {
226 normalize_for_effective_seq_length = false;
228 if ( cla.isOptionSet( STEP_FOR_DIAGNOSTICS_OPTION ) ) {
229 step_for_diagnostics = cla.getOptionValueAsInt( STEP_FOR_DIAGNOSTICS_OPTION );
230 if ( ( step_for_diagnostics < 1 )
231 || ( ( step_for_diagnostics > msa.getNumberOfSequences() ) || ( ( worst_remove > 0 ) && ( step_for_diagnostics > worst_remove ) ) ) ) {
232 ForesterUtil.fatalError( PRG_NAME, "value for diagnostic step is out of range: "
233 + step_for_diagnostics );
236 if ( cla.isOptionSet( GAP_RATIO_LENGTH_OPTION ) ) {
237 gap_ratio = cla.getOptionValueAsDouble( GAP_RATIO_LENGTH_OPTION );
238 if ( ( gap_ratio < 0 ) || ( gap_ratio > 1 ) ) {
239 ForesterUtil.fatalError( PRG_NAME, "gap ratio is out of range: " + gap_ratio );
242 if ( cla.isOptionSet( REPORT_ENTROPY ) ) {
243 report_entropy = true;
245 if ( cla.isOptionSet( OUTPUT_FORMAT_OPTION ) ) {
246 final String fs = cla.getOptionValueAsCleanString( OUTPUT_FORMAT_OPTION );
247 if ( fs.equalsIgnoreCase( "p" ) ) {
248 output_format = MSA_FORMAT.PHYLIP;
250 else if ( fs.equalsIgnoreCase( "f" ) ) {
251 output_format = MSA_FORMAT.FASTA;
253 else if ( fs.equalsIgnoreCase( "n" ) ) {
254 output_format = MSA_FORMAT.NEXUS;
257 ForesterUtil.fatalError( PRG_NAME, "illegal or empty output format option: " + fs );
260 if ( cla.isOptionSet( OUTPUT_REMOVED_SEQS_OPTION ) ) {
261 final String s = cla.getOptionValueAsCleanString( OUTPUT_REMOVED_SEQS_OPTION );
262 removed_seqs_out_base = new File( s );
265 if ( ForesterUtil.isEmpty( path_to_mafft ) ) {
266 path_to_mafft = MsaCompactor.guessPathToMafft();
268 checkPathToMafft( path_to_mafft );
269 if ( cla.isOptionSet( MAFFT_OPTIONS ) ) {
270 mafft_options = cla.getOptionValueAsCleanString( MAFFT_OPTIONS );
271 if ( ForesterUtil.isEmpty( mafft_options ) || ( mafft_options.length() < 3 ) ) {
272 ForesterUtil.fatalError( PRG_NAME, "illegal or empty MAFFT options: " + mafft_options );
276 else if ( cla.isOptionSet( MAFFT_OPTIONS ) ) {
277 ForesterUtil.fatalError( PRG_NAME, "no need to indicate MAFFT options without realigning" );
279 if ( cla.isOptionSet( PERFORM_PHYLOGENETIC_INFERENCE ) ) {
280 perform_phylogenetic_inference = true;
283 if ( ( out != null ) || ( removed_seqs_out_base != null ) ) {
285 .fatalError( PRG_NAME,
286 "chart only, no outfile(s) produced, thus no need to indicate output file(s)" );
288 if ( !realign && cla.isOptionSet( STEP_OPTION ) ) {
289 ForesterUtil.fatalError( PRG_NAME,
290 "chart only, no re-aligning, thus no need to use step for output and re-aligning; use -"
291 + STEP_FOR_DIAGNOSTICS_OPTION + " instead" );
294 if ( perform_phylogenetic_inference ) {
295 if ( step_for_diagnostics != 1 ) {
296 ForesterUtil.fatalError( PRG_NAME,
297 "step for diagnostics reports needs to be set to 1 for tree calculation" );
300 printInfo( in, msa, initial_msa_stats );
302 System.out.println( "Output : " + out );
305 if ( removed_seqs_out_base != null ) {
306 System.out.println( "Write removed sequences to : " + removed_seqs_out_base );
308 if ( worst_remove > 0 ) {
309 System.out.println( "Number of worst offenders to remove : " + worst_remove );
312 System.out.println( "Target gap-ratio : " + av_gap );
315 System.out.println( "Target MSA length : " + length );
317 if ( min_length > 1 ) {
318 System.out.println( "Minimal effective sequence length : " + min_length );
320 if ( gap_ratio > -1 ) {
321 System.out.println( "Maximum allowed gap ratio per column : " + gap_ratio );
323 if ( ( out != null ) || ( removed_seqs_out_base != null ) ) {
324 System.out.print( "Output format : " );
325 if ( output_format == MSA_FORMAT.FASTA ) {
326 System.out.println( "fasta" );
328 else if ( output_format == MSA_FORMAT.PHYLIP ) {
329 System.out.println( "phylip" );
331 else if ( output_format == MSA_FORMAT.NEXUS ) {
332 System.out.println( "nexus" );
335 if ( min_length == -1 ) {
336 if ( chart_only && !realign ) {
337 System.out.println( "Step for output and re-aligning : n/a" );
341 System.out.println( "Step for re-aligning : " + step );
344 System.out.println( "Step for output and re-aligning : " + step );
347 System.out.println( "Step for diagnostics reports : " + step_for_diagnostics );
348 System.out.println( "Calculate normalized Shannon Entropy : " + report_entropy );
349 if ( normalize_for_effective_seq_length ) {
350 System.out.println( "Normalize : with individual, effective sequence lenghts" );
353 System.out.println( "Normalize : with MSA length" );
355 System.out.println( "Realign with MAFFT : " + realign );
357 System.out.println( "MAFFT options : " + mafft_options );
359 System.out.println( "Simple tree (Kimura distances, NJ) : " + perform_phylogenetic_inference );
361 System.out.println();
362 final int initial_number_of_seqs = msa.getNumberOfSequences();
363 List<MsaProperties> msa_props = null;
364 final MsaCompactor mc = new MsaCompactor( msa );
365 mc.setInfileName( in.getName() );
366 if ( ( worst_remove > 0 ) || ( av_gap > 0 ) || ( length > 0 ) || ( min_length != -1 ) ) {
367 mc.setOutputFormat( output_format );
368 mc.setOutFileBase( out );
370 if ( min_length != -1 ) {
371 mc.removeSequencesByMinimalLength( min_length );
374 mc.setPeformPhylogenticInference( perform_phylogenetic_inference );
375 if ( removed_seqs_out_base != null ) {
376 mc.setRemovedSeqsOutBase( removed_seqs_out_base );
378 mc.setNorm( normalize_for_effective_seq_length );
379 mc.setRealign( realign );
381 mc.setPathToMafft( path_to_mafft );
382 mc.setMafftOptions( mafft_options );
385 mc.setStepForDiagnostics( step_for_diagnostics );
386 mc.setCalculateNormalizedShannonEntropy( report_entropy );
387 if ( worst_remove > 0 ) {
388 msa_props = mc.removeWorstOffenders( worst_remove );
390 else if ( av_gap > 0 ) {
391 msa_props = mc.removeViaGapAverage( av_gap );
393 else if ( length > 0 ) {
394 msa_props = mc.removeViaLength( length );
397 msa_props = mc.chart( step, realign, normalize_for_effective_seq_length );
399 Chart.display( msa_props, initial_number_of_seqs, report_entropy, in.getName() );
400 System.out.println();
401 System.out.println( "Final MSA properties" );
402 printMsaInfo( msa, MsaMethods.calculateEffectiveLengthStatistics( msa ));
405 catch ( final IllegalArgumentException iae ) {
406 // iae.printStackTrace(); //TODO remove me
407 ForesterUtil.fatalError( PRG_NAME, iae.getMessage() );
409 catch ( final IOException ioe ) {
410 // ioe.printStackTrace(); //TODO remove me
411 ForesterUtil.fatalError( PRG_NAME, ioe.getMessage() );
413 catch ( final Exception e ) {
414 ForesterUtil.unexpectedFatalError( PRG_NAME, e );
418 private static void printInfo( final File in, DeleteableMsa msa, final DescriptiveStatistics initial_msa_stats ) {
419 ForesterUtil.printProgramInformation( PRG_NAME,
425 ForesterUtil.getForesterLibraryInformation() );
426 System.out.println( "Input MSA : " + in );
427 printMsaInfo( msa, initial_msa_stats );
430 private static void printMsaInfo( DeleteableMsa msa, final DescriptiveStatistics msa_stats ) {
431 System.out.println( "MSA length : " + msa.getLength() );
432 System.out.println( "Number of sequences : " + msa.getNumberOfSequences() );
433 System.out.println( "Median sequence length : " + NF_1.format( msa_stats.median() ) );
434 System.out.println( "Mean sequence length : "
435 + NF_1.format( msa_stats.arithmeticMean() ) );
436 System.out.println( "Max sequence length : " + ( ( int ) msa_stats.getMax() ) );
437 System.out.println( "Min sequence length : " + ( ( int ) msa_stats.getMin() ) );
438 System.out.println( "Gap ratio : "
439 + NF_4.format( MsaMethods.calcGapRatio( msa ) ) );
440 System.out.println( "Mean gap count per sequence : "
441 + NF_1.format( MsaMethods.calcNumberOfGapsStats( msa ).arithmeticMean() ) );
442 System.out.println( "Normalized Shannon Entropy (entn7) : "
443 + NF_4.format( MsaMethods.calcNormalizedShannonsEntropy( 7, msa ) ) );
444 System.out.println( "Normalized Shannon Entropy (entn21) : "
445 + NF_4.format( MsaMethods.calcNormalizedShannonsEntropy( 21, msa ) ) );
448 private static void checkPathToMafft( final String path_to_mafft ) {
449 if ( !ForesterUtil.isEmpty( path_to_mafft ) && MsaInferrer.isInstalled( path_to_mafft ) ) {
452 if ( ForesterUtil.isEmpty( path_to_mafft ) ) {
453 ForesterUtil.fatalError( PRG_NAME, "no MAFFT executable found, use -\"" + PATH_TO_MAFFT_OPTION
454 + "=<path to MAFFT>\" option" );
457 ForesterUtil.fatalError( PRG_NAME, "no MAFFT executable at \"" + path_to_mafft + "\"" );
462 private static void printHelp() {
463 ForesterUtil.printProgramInformation( PRG_NAME,
469 ForesterUtil.getForesterLibraryInformation() );
470 final String path_to_mafft = MsaCompactor.guessPathToMafft();
471 String mafft_comment;
472 if ( !ForesterUtil.isEmpty( path_to_mafft ) ) {
473 mafft_comment = " (using " + path_to_mafft + ")";
476 mafft_comment = " (no path to MAFFT found, use -\"" + PATH_TO_MAFFT_OPTION + "=<path to MAFFT>\" option";
478 System.out.println( "Usage:" );
479 System.out.println();
480 System.out.println( PRG_NAME + " [options] <msa input file> [output file base]" );
481 System.out.println();
482 System.out.println( " options: " );
483 System.out.println();
484 System.out.println( " -" + INFO_ONLY_OPTION
485 + " to only display same basic information about the MSA" );
486 System.out.println( " -" + REMOVE_WORST_OFFENDERS_OPTION
487 + "=<integer> number of worst offender sequences to remove" );
488 System.out.println( " -" + LENGTH_OPTION + "=<integer> target MSA length" );
489 System.out.println( " -" + AV_GAPINESS_OPTION + "=<decimal> target gap-ratio (0.0-1.0)" );
490 System.out.println( " -" + REALIGN_OPTION + " to realign using MAFFT" + mafft_comment );
491 System.out.println( " -" + MAFFT_OPTIONS + "=<string> options for MAFFT (default: --auto)" );
492 System.out.println( " -" + STEP_OPTION + "=<integer> step for output and re-aligning (default: 1)" );
493 System.out.println( " -" + STEP_FOR_DIAGNOSTICS_OPTION
494 + "=<integer> step for diagnostics reports (default: 1)" );
495 System.out.println( " -" + REPORT_ENTROPY
496 + " to calculate normalized Shannon Entropy (not recommended for very large alignments)" );
497 System.out.println( " -" + OUTPUT_FORMAT_OPTION
498 + "=<f|p|n> format for output alignments: f for fasta (default), p for phylip, or n for nexus" );
499 System.out.println( " -" + OUTPUT_REMOVED_SEQS_OPTION + "=<file> to output the removed sequences" );
500 System.out.println( " -" + MIN_LENGTH_OPTION
501 + "=<integer> minimal effecive sequence length (for deleting of shorter sequences)" );
502 System.out.println( " -" + GAP_RATIO_LENGTH_OPTION
503 + "=<decimal> maximal allowed gap ratio per column (for deleting of columms) (0.0-1.0)" );
504 System.out.println( " -" + PERFORM_PHYLOGENETIC_INFERENCE
505 + " to calculate a simple phylogenetic tree (Kimura distances, NJ)" );
506 System.out.println( " -" + DO_NOT_NORMALIZE_FOR_EFF_LENGTH_OPTION
507 + " to normalize gap-contributions with MSA length, instead of individual effective sequence lenghts" );
509 System.out.println();
510 System.out.println();
511 System.out.println();