2 // FORESTER -- software libraries and applications
3 // for evolutionary biology research and applications.
5 // Copyright (C) 2014 Christian M. Zmasek
6 // Copyright (C) 2014 Sanford-Burnham Medical Research Institute
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
23 // WWW: https://sites.google.com/site/cmzmasek/home/software/forester
25 package org.forester.application;
28 import java.io.FileInputStream;
29 import java.io.IOException;
30 import java.util.ArrayList;
31 import java.util.List;
33 import org.forester.io.parsers.FastaParser;
34 import org.forester.io.parsers.GeneralMsaParser;
35 import org.forester.msa.DeleteableMsa;
36 import org.forester.msa.Msa.MSA_FORMAT;
37 import org.forester.msa.MsaInferrer;
38 import org.forester.msa.MsaMethods;
39 import org.forester.msa_compactor.Chart;
40 import org.forester.msa_compactor.MsaCompactor;
41 import org.forester.msa_compactor.MsaProperties;
42 import org.forester.util.CommandLineArguments;
43 import org.forester.util.DescriptiveStatistics;
44 import org.forester.util.ForesterUtil;
46 public class msa_compactor {
48 final static private String HELP_OPTION_1 = "help";
49 final static private String HELP_OPTION_2 = "h";
50 final static private String REMOVE_WORST_OFFENDERS_OPTION = "r";
51 final static private String AV_GAPINESS_OPTION = "g";
52 final static private String STEP_OPTION = "s";
53 final static private String LENGTH_OPTION = "l";
54 final static private String REALIGN_OPTION = "a";
56 final static private String STEP_FOR_DIAGNOSTICS_OPTION = "sd";
57 final static private String MIN_LENGTH_OPTION = "ml";
58 final static private String GAP_RATIO_LENGTH_OPTION = "gr";
59 final static private String REPORT_ALN_MEAN_IDENTITY = "q";
60 final static private String OUTPUT_FORMAT_PHYLIP_OPTION = "p";
61 final static private String OUTPUT_REMOVED_SEQS_OPTION = "ro";
62 final static private String MAFFT_OPTIONS = "mo";
64 final static private String PATH_TO_MAFFT_OPTION = "mafft";
65 final static private String DO_NOT_NORMALIZE_FOR_EFF_LENGTH_OPTION = "nn";
66 final static private String PRG_NAME = "msa_compactor";
67 final static private String PRG_DESC = "multiple sequence aligment compactor";
68 final static private String PRG_VERSION = "0.2";
69 final static private String PRG_DATE = "140428";
70 final static private String E_MAIL = "czmasek@sanfordburham.org";
71 final static private String WWW = "https://sites.google.com/site/cmzmasek/home/software/forester";
73 public static void main( final String args[] ) {
75 final CommandLineArguments cla = new CommandLineArguments( args );
76 if ( cla.isOptionSet( HELP_OPTION_1 ) || cla.isOptionSet( HELP_OPTION_2 )
77 || ( ( cla.getNumberOfNames() < 1 ) || ( cla.getNumberOfNames() > 2 ) ) ) {
81 final File in = cla.getFile( 0 );
83 if ( cla.getNumberOfNames() > 1 ) {
84 out = cla.getFile( 1 );
86 int worst_remove = -1;
90 boolean realign = false;
92 String path_to_mafft = null;
93 int step_for_diagnostics = 1;
95 double gap_ratio = -1;
96 boolean report_aln_mean_identity = false;
97 MSA_FORMAT output_format = MSA_FORMAT.FASTA;
98 File removed_seqs_out_base = null;
99 String mafft_options = "--auto";
100 final List<String> allowed_options = new ArrayList<String>();
101 allowed_options.add( REMOVE_WORST_OFFENDERS_OPTION );
102 allowed_options.add( AV_GAPINESS_OPTION );
103 allowed_options.add( LENGTH_OPTION );
104 allowed_options.add( REALIGN_OPTION );
105 allowed_options.add( DO_NOT_NORMALIZE_FOR_EFF_LENGTH_OPTION );
106 allowed_options.add( STEP_OPTION );
107 allowed_options.add( PATH_TO_MAFFT_OPTION );
108 allowed_options.add( STEP_FOR_DIAGNOSTICS_OPTION );
109 allowed_options.add( MIN_LENGTH_OPTION );
110 allowed_options.add( GAP_RATIO_LENGTH_OPTION );
111 allowed_options.add( REPORT_ALN_MEAN_IDENTITY );
112 allowed_options.add( OUTPUT_FORMAT_PHYLIP_OPTION );
113 allowed_options.add( OUTPUT_REMOVED_SEQS_OPTION );
114 allowed_options.add( MAFFT_OPTIONS );
115 final String dissallowed_options = cla.validateAllowedOptionsAsString( allowed_options );
116 if ( dissallowed_options.length() > 0 ) {
117 ForesterUtil.fatalError( PRG_NAME, "unknown option(s): " + dissallowed_options );
119 DeleteableMsa msa = null;
120 final FileInputStream is = new FileInputStream( in );
121 if ( FastaParser.isLikelyFasta( in ) ) {
122 msa = DeleteableMsa.createInstance( FastaParser.parseMsa( is ) );
125 msa = DeleteableMsa.createInstance( GeneralMsaParser.parse( is ) );
127 final DescriptiveStatistics initial_msa_stats = MsaMethods.calculateEffectiveLengthStatistics( msa );
128 //System.out.println( initial_msa_stats.toString() );
129 if ( ( cla.isOptionSet( LENGTH_OPTION ) || cla.isOptionSet( REMOVE_WORST_OFFENDERS_OPTION ) || cla
130 .isOptionSet( AV_GAPINESS_OPTION ) ) && ( out == null ) ) {
131 ForesterUtil.fatalError( PRG_NAME, "outfile file missing" );
133 if ( cla.isOptionSet( REMOVE_WORST_OFFENDERS_OPTION ) ) {
134 worst_remove = cla.getOptionValueAsInt( REMOVE_WORST_OFFENDERS_OPTION );
135 if ( ( worst_remove < 1 ) || ( worst_remove >= msa.getNumberOfSequences() - 1 ) ) {
136 ForesterUtil.fatalError( PRG_NAME, "number of worst offender sequences to remove is out of range: "
140 if ( cla.isOptionSet( AV_GAPINESS_OPTION ) ) {
141 if ( cla.isOptionSet( REMOVE_WORST_OFFENDERS_OPTION ) ) {
145 av_gap = cla.getOptionValueAsDouble( AV_GAPINESS_OPTION );
146 if ( ( av_gap < 0 ) || ( av_gap >= 1 ) ) {
147 ForesterUtil.fatalError( PRG_NAME, "target gap-ratio is out of range: " + av_gap );
150 if ( cla.isOptionSet( LENGTH_OPTION ) ) {
151 if ( cla.isOptionSet( REMOVE_WORST_OFFENDERS_OPTION ) || cla.isOptionSet( AV_GAPINESS_OPTION ) ) {
155 length = cla.getOptionValueAsInt( LENGTH_OPTION );
156 if ( ( length < 2 ) || ( length >= msa.getLength() ) ) {
157 ForesterUtil.fatalError( PRG_NAME, "target length is out of range: " + length );
160 if ( cla.isOptionSet( STEP_OPTION ) ) {
161 step = cla.getOptionValueAsInt( STEP_OPTION );
163 || ( ( step > msa.getNumberOfSequences() ) || ( ( worst_remove > 0 ) && ( step > worst_remove ) ) ) ) {
164 ForesterUtil.fatalError( PRG_NAME, "value for step is out of range: " + step );
167 if ( cla.isOptionSet( REALIGN_OPTION ) ) {
170 if ( cla.isOptionSet( PATH_TO_MAFFT_OPTION ) ) {
172 ForesterUtil.fatalError( PRG_NAME, "no need to indicate path to MAFFT without realigning" );
174 path_to_mafft = cla.getOptionValueAsCleanString( PATH_TO_MAFFT_OPTION );
176 if ( cla.isOptionSet( DO_NOT_NORMALIZE_FOR_EFF_LENGTH_OPTION ) ) {
180 if ( cla.isOptionSet( STEP_FOR_DIAGNOSTICS_OPTION ) ) {
181 step_for_diagnostics = cla.getOptionValueAsInt( STEP_FOR_DIAGNOSTICS_OPTION );
182 if ( ( step_for_diagnostics < 1 )
183 || ( ( step_for_diagnostics > msa.getNumberOfSequences() ) || ( ( worst_remove > 0 ) && ( step_for_diagnostics > worst_remove ) ) ) ) {
184 ForesterUtil.fatalError( PRG_NAME, "value for diagnostic step is out of range: "
185 + step_for_diagnostics );
188 if ( cla.isOptionSet( MIN_LENGTH_OPTION ) ) {
189 min_length = cla.getOptionValueAsInt( MIN_LENGTH_OPTION );
190 if ( ( min_length < 2 ) || ( min_length > initial_msa_stats.getMax() ) ) {
191 ForesterUtil.fatalError( PRG_NAME, "value for minimal sequence length is out of range: "
195 if ( cla.isOptionSet( MIN_LENGTH_OPTION ) ) {
196 gap_ratio = cla.getOptionValueAsDouble( GAP_RATIO_LENGTH_OPTION );
197 if ( ( gap_ratio < 0 ) || ( gap_ratio > 1 ) ) {
198 ForesterUtil.fatalError( PRG_NAME, "gap ratio is out of range: " + gap_ratio );
201 if ( cla.isOptionSet( REPORT_ALN_MEAN_IDENTITY ) ) {
202 report_aln_mean_identity = true;
204 if ( cla.isOptionSet( OUTPUT_FORMAT_PHYLIP_OPTION ) ) {
205 output_format = MSA_FORMAT.PHYLIP;
207 if ( cla.isOptionSet( OUTPUT_REMOVED_SEQS_OPTION ) ) {
208 final String s = cla.getOptionValueAsCleanString( OUTPUT_REMOVED_SEQS_OPTION );
209 removed_seqs_out_base = new File( s );
212 if ( ForesterUtil.isEmpty( path_to_mafft ) ) {
213 path_to_mafft = MsaCompactor.guessPathToMafft();
215 checkPathToMafft( path_to_mafft );
216 if ( cla.isOptionSet( MAFFT_OPTIONS ) ) {
217 mafft_options = cla.getOptionValueAsCleanString( MAFFT_OPTIONS );
218 if ( ForesterUtil.isEmpty( mafft_options ) || ( mafft_options.length() < 3 ) ) {
219 ForesterUtil.fatalError( PRG_NAME, "gap ratio is out of range: " + gap_ratio );
223 if ( ( !cla.isOptionSet( LENGTH_OPTION ) && !cla.isOptionSet( REMOVE_WORST_OFFENDERS_OPTION ) && !cla
224 .isOptionSet( AV_GAPINESS_OPTION ) ) && ( ( out != null ) || ( removed_seqs_out_base != null ) ) ) {
225 ForesterUtil.fatalError( PRG_NAME,
226 "chart only, no outfile(s) produced (no need to indicate output file(s))" );
228 ForesterUtil.printProgramInformation( PRG_NAME,
234 ForesterUtil.getForesterLibraryInformation() );
236 System.out.println( "Input MSA : " + in );
238 System.out.println( "Output : " + out );
241 System.out.println( "Output : n/a" );
243 if ( removed_seqs_out_base != null ) {
244 System.out.println( "Write removed sequences to : " + removed_seqs_out_base );
246 if ( worst_remove > 0 ) {
247 System.out.println( "Number of worst offenders to remove : " + worst_remove );
249 else if ( av_gap > 0 ) {
250 System.out.println( "Target gap-ratio : " + av_gap );
252 else if ( length > 0 ) {
253 System.out.println( "Target MSA length : " + length );
256 System.out.println( "Chart and diagnostics only : true" );
258 if ( ( out != null ) || ( removed_seqs_out_base != null ) ) {
259 System.out.println( "Output format : "
260 + ( output_format == MSA_FORMAT.FASTA ? "fasta" : "phylip" ) );
262 System.out.println( "Step for output and re-aligning : " + step );
263 System.out.println( "Step for diagnostics reports : " + step_for_diagnostics );
264 System.out.println( "Calculate mean identity : " + report_aln_mean_identity );
266 System.out.println( "Normalize : " + norm );
268 System.out.println( "Realign : " + realign );
270 System.out.println( "MAFFT options : " + mafft_options );
272 if ( min_length > -1 ) {
273 System.out.println( "Minimal effective sequence length : " + min_length );
275 if ( gap_ratio > -1 ) {
276 System.out.println( "Maximum allowed gap ratio per column : " + gap_ratio );
278 System.out.println();
281 final int initial_number_of_seqs = msa.getNumberOfSequences();
282 List<MsaProperties> msa_props = null;
283 if ( worst_remove > 0 ) {
284 final MsaCompactor mc = new MsaCompactor( msa );
285 mc.setRealign( realign );
286 mc.setOutputFormat( output_format );
288 mc.setPathToMafft( path_to_mafft );
289 mc.setMafftOptions( mafft_options );
292 mc.setReportAlnMeanIdentity( report_aln_mean_identity );
293 mc.setOutFileBase( out );
294 if ( removed_seqs_out_base != null ) {
295 mc.setRemovedSeqsOutBase( removed_seqs_out_base );
298 mc.setStepForDiagnostics( step_for_diagnostics );
299 msa_props = mc.removeWorstOffenders( worst_remove );
301 else if ( av_gap > 0 ) {
302 final MsaCompactor mc = new MsaCompactor( msa );
303 mc.setRealign( realign );
304 mc.setOutputFormat( output_format );
306 mc.setPathToMafft( path_to_mafft );
307 mc.setMafftOptions( mafft_options );
310 mc.setReportAlnMeanIdentity( report_aln_mean_identity );
311 mc.setOutFileBase( out );
312 if ( removed_seqs_out_base != null ) {
313 mc.setRemovedSeqsOutBase( removed_seqs_out_base );
316 mc.setStepForDiagnostics( step_for_diagnostics );
317 msa_props = mc.removeViaGapAverage( av_gap );
319 else if ( length > 0 ) {
320 // TODO if < shortest seq -> error
321 final MsaCompactor mc = new MsaCompactor( msa );
322 mc.setRealign( realign );
323 mc.setOutputFormat( output_format );
325 mc.setPathToMafft( path_to_mafft );
326 mc.setMafftOptions( mafft_options );
329 mc.setReportAlnMeanIdentity( report_aln_mean_identity );
330 mc.setOutFileBase( out );
331 if ( removed_seqs_out_base != null ) {
332 mc.setRemovedSeqsOutBase( removed_seqs_out_base );
335 mc.setStepForDiagnostics( step_for_diagnostics );
336 msa_props = mc.removeViaLength( length );
339 //MsaCompactor.chart( msa, step, realign, norm, path_to_mafft );
340 final MsaCompactor mc = new MsaCompactor( msa );
341 mc.setRealign( realign );
343 mc.setPathToMafft( path_to_mafft );
344 mc.setMafftOptions( mafft_options );
347 mc.setReportAlnMeanIdentity( report_aln_mean_identity );
348 mc.setOutFileBase( out );
350 mc.setStepForDiagnostics( step_for_diagnostics );
351 msa_props = mc.chart( step, realign, norm );
353 Chart.display( msa_props, initial_number_of_seqs, report_aln_mean_identity, in.toString() );
355 catch ( final IllegalArgumentException iae ) {
356 iae.printStackTrace(); //TODO remove me
357 ForesterUtil.fatalError( PRG_NAME, iae.getMessage() );
359 catch ( final IOException ioe ) {
360 ioe.printStackTrace(); //TODO remove me
361 ForesterUtil.fatalError( PRG_NAME, ioe.getMessage() );
363 catch ( final Exception e ) {
364 ForesterUtil.unexpectedFatalError( PRG_NAME, e );
368 private static void checkPathToMafft( final String path_to_mafft ) {
369 if ( !ForesterUtil.isEmpty( path_to_mafft ) && MsaInferrer.isInstalled( path_to_mafft ) ) {
370 ForesterUtil.programMessage( PRG_NAME, "using MAFFT at \"" + path_to_mafft + "\"" );
373 if ( ForesterUtil.isEmpty( path_to_mafft ) ) {
374 ForesterUtil.fatalError( PRG_NAME, "no MAFFT executable found, use -\"" + PATH_TO_MAFFT_OPTION
375 + "=<path to MAFFT>\" option" );
378 ForesterUtil.fatalError( PRG_NAME, "no MAFFT executable at \"" + path_to_mafft + "\"" );
383 private static void printHelp() {
384 ForesterUtil.printProgramInformation( PRG_NAME,
390 ForesterUtil.getForesterLibraryInformation() );
391 final String path_to_mafft = MsaCompactor.guessPathToMafft();
392 String mafft_comment;
393 if ( !ForesterUtil.isEmpty( path_to_mafft ) ) {
394 mafft_comment = " (using " + path_to_mafft + ")";
397 mafft_comment = " (no path to MAFFT found, use -\"" + PATH_TO_MAFFT_OPTION + "=<path to MAFFT>\" option";
399 System.out.println( "Usage:" );
400 System.out.println();
401 System.out.println( PRG_NAME + " <options> <msa input file> <output file base>" );
402 System.out.println();
403 System.out.println( " options: " );
404 System.out.println();
405 System.out.println( " -" + REMOVE_WORST_OFFENDERS_OPTION
406 + "=<integer> number of worst offender sequences to remove" );
407 System.out.println( " -" + LENGTH_OPTION + "=<integer> target MSA length" );
408 System.out.println( " -" + AV_GAPINESS_OPTION + "=<decimal> target gap-ratio (0.0-1.0)" );
409 System.out.println( " -" + REALIGN_OPTION + " to realign using MAFFT" + mafft_comment );
410 System.out.println( " -" + MAFFT_OPTIONS + "=<string> options for MAFFT (default: --auto)" );
411 System.out.println( " -" + STEP_OPTION + "=<integer> step for output and re-aligning (default: 1)" );
412 System.out.println( " -" + STEP_FOR_DIAGNOSTICS_OPTION
413 + "=<integer> step for diagnostics reports (default: 1)" );
414 // System.out.println( " -" + MIN_LENGTH_OPTION
415 // + "=<integer> minimal effecive sequence length (for deleting of shorter sequences)" );
416 // System.out.println( " -" + GAP_RATIO_LENGTH_OPTION
417 // + "=<decimal> maximal allowed gap ratio per column (for deleting of columms) (0.0-1.0)" );
420 + REPORT_ALN_MEAN_IDENTITY
421 + " to calculate mean MSA column identity (\"MSA quality\") (not recommended for very large alignments)" );
422 System.out.println( " -" + OUTPUT_FORMAT_PHYLIP_OPTION
423 + " to write output alignments in phylip format instead of fasta" );
424 System.out.println( " -" + OUTPUT_REMOVED_SEQS_OPTION + "=<file> to output the removed sequences" );
425 System.out.println();
426 System.out.println();
427 System.out.println();