2 // FORESTER -- software libraries and applications
3 // for evolutionary biology research and applications.
5 // Copyright (C) 2017 Christian M. Zmasek
6 // Copyright (C) 2017 J. Craig Venter Institute
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
23 // Contact: phyloxml @ gmail . com
24 // WWW: https://sites.google.com/site/cmzmasek/home/software/forester
26 package org.forester.application;
29 import java.io.IOException;
30 import java.text.DecimalFormat;
31 import java.util.ArrayList;
32 import java.util.List;
33 import java.util.SortedMap;
34 import java.util.regex.Pattern;
35 import java.util.regex.PatternSyntaxException;
37 import org.forester.clade_analysis.AnalysisMulti;
38 import org.forester.clade_analysis.Prefix;
39 import org.forester.clade_analysis.ResultMulti;
40 import org.forester.io.parsers.PhylogenyParser;
41 import org.forester.io.parsers.util.ParserUtils;
42 import org.forester.phylogeny.Phylogeny;
43 import org.forester.phylogeny.factories.ParserBasedPhylogenyFactory;
44 import org.forester.phylogeny.factories.PhylogenyFactory;
45 import org.forester.util.BasicTable;
46 import org.forester.util.BasicTableParser;
47 import org.forester.util.CommandLineArguments;
48 import org.forester.util.EasyWriter;
49 import org.forester.util.ForesterUtil;
50 import org.forester.util.UserException;
52 public final class cladinator {
54 final static private String PRG_NAME = "cladinator";
55 final static private String PRG_VERSION = "1.06";
56 final static private String PRG_DATE = "171115";
57 final static private String PRG_DESC = "clades within clades of annotated labels -- analysis of pplacer-type outputs";
58 final static private String E_MAIL = "phyloxml@gmail.com";
59 final static private String WWW = "https://sites.google.com/site/cmzmasek/home/software/forester";
60 final static private String HELP_OPTION_1 = "help";
61 final static private String HELP_OPTION_2 = "h";
62 final static private String SEP_OPTION = "s";
63 final static private String QUERY_PATTERN_OPTION = "q";
64 final static private String SPECIFICS_CUTOFF_OPTION = "c";
65 final static private String MAPPING_FILE_OPTION = "m";
66 final static private String EXTRA_PROCESSING_OPTION1 = "x";
67 final static private String EXTRA_PROCESSING1_SEP_OPTION = "xs";
68 final static private String EXTRA_PROCESSING1_KEEP_EXTRA_OPTION = "xk";
69 final static private String QUIET_OPTION = "Q";
70 final static private String SPECIAL_PROCESSING_OPTION = "S";
71 final static private String VERBOSE_OPTION = "v";
72 final static private String REMOVE_ANNOT_SEP_OPTION = "rs";
73 final static private double SPECIFICS_CUTOFF_DEFAULT = 0.8;
74 final static private String SEP_DEFAULT = ".";
75 final static private Pattern QUERY_PATTERN_DEFAULT = AnalysisMulti.DEFAULT_QUERY_PATTERN_FOR_PPLACER_TYPE;
76 final static private String EXTRA_PROCESSING1_SEP_DEFAULT = "|";
77 final static private boolean EXTRA_PROCESSING1_KEEP_EXTRA_DEFAULT = false;
78 private final static DecimalFormat df = new DecimalFormat( "0.0###" );
80 public static void main( final String args[] ) {
82 ForesterUtil.printProgramInformation( PRG_NAME,
88 ForesterUtil.getForesterLibraryInformation() );
89 CommandLineArguments cla = null;
91 cla = new CommandLineArguments( args );
93 catch ( final Exception e ) {
94 ForesterUtil.fatalError( PRG_NAME, e.getMessage() );
96 if ( cla.isOptionSet( HELP_OPTION_1 ) || cla.isOptionSet( HELP_OPTION_2 ) ) {
101 if ( ( cla.getNumberOfNames() != 1 ) && ( cla.getNumberOfNames() != 2 ) ) {
105 final List<String> allowed_options = new ArrayList<String>();
106 allowed_options.add( SEP_OPTION );
107 allowed_options.add( QUERY_PATTERN_OPTION );
108 allowed_options.add( SPECIFICS_CUTOFF_OPTION );
109 allowed_options.add( MAPPING_FILE_OPTION );
110 allowed_options.add( EXTRA_PROCESSING_OPTION1 );
111 allowed_options.add( EXTRA_PROCESSING1_SEP_OPTION );
112 allowed_options.add( EXTRA_PROCESSING1_KEEP_EXTRA_OPTION );
113 allowed_options.add( SPECIAL_PROCESSING_OPTION );
114 allowed_options.add( VERBOSE_OPTION );
115 allowed_options.add( QUIET_OPTION );
116 allowed_options.add( REMOVE_ANNOT_SEP_OPTION );
117 final String dissallowed_options = cla.validateAllowedOptionsAsString( allowed_options );
118 if ( dissallowed_options.length() > 0 ) {
119 ForesterUtil.fatalError( PRG_NAME, "unknown option(s): " + dissallowed_options );
121 double cutoff_specifics = SPECIFICS_CUTOFF_DEFAULT;
122 if ( cla.isOptionSet( SPECIFICS_CUTOFF_OPTION ) ) {
123 if ( cla.isOptionValueSet( SPECIFICS_CUTOFF_OPTION ) ) {
124 cutoff_specifics = cla.getOptionValueAsDouble( SPECIFICS_CUTOFF_OPTION );
125 if ( cutoff_specifics < 0 ) {
126 ForesterUtil.fatalError( PRG_NAME, "cutoff cannot be negative" );
130 ForesterUtil.fatalError( PRG_NAME, "no value for cutoff for specifics" );
133 String separator = SEP_DEFAULT;
134 if ( cla.isOptionSet( SEP_OPTION ) ) {
135 if ( cla.isOptionValueSet( SEP_OPTION ) ) {
136 separator = cla.getOptionValue( SEP_OPTION );
139 ForesterUtil.fatalError( PRG_NAME, "no value for separator option" );
142 Pattern compiled_query = null;
143 if ( cla.isOptionSet( QUERY_PATTERN_OPTION ) ) {
144 if ( cla.isOptionValueSet( QUERY_PATTERN_OPTION ) ) {
145 final String query_str = cla.getOptionValue( QUERY_PATTERN_OPTION );
147 compiled_query = Pattern.compile( query_str );
149 catch ( final PatternSyntaxException e ) {
150 ForesterUtil.fatalError( PRG_NAME,
151 "error in regular expression: " + query_str + ": " + e.getMessage() );
155 ForesterUtil.fatalError( PRG_NAME, "no value for query pattern option" );
158 File mapping_file = null;
159 if ( cla.isOptionSet( MAPPING_FILE_OPTION ) ) {
160 if ( cla.isOptionValueSet( MAPPING_FILE_OPTION ) ) {
161 final String mapping_file_str = cla.getOptionValue( MAPPING_FILE_OPTION );
162 final String error = ForesterUtil.isReadableFile( mapping_file_str );
163 if ( !ForesterUtil.isEmpty( error ) ) {
164 ForesterUtil.fatalError( PRG_NAME, error );
166 mapping_file = new File( mapping_file_str );
169 ForesterUtil.fatalError( PRG_NAME, "no value for mapping file" );
172 final Pattern pattern = ( compiled_query != null ) ? compiled_query : QUERY_PATTERN_DEFAULT;
173 final File intreefile = cla.getFile( 0 );
174 final String error_intreefile = ForesterUtil.isReadableFile( intreefile );
175 if ( !ForesterUtil.isEmpty( error_intreefile ) ) {
176 ForesterUtil.fatalError( PRG_NAME, error_intreefile );
178 final File outtablefile;
179 if ( cla.getNumberOfNames() > 1 ) {
180 outtablefile = cla.getFile( 1 );
181 final String error_outtablefile = ForesterUtil.isWritableFile( outtablefile );
182 if ( !ForesterUtil.isEmpty( error_outtablefile ) ) {
183 ForesterUtil.fatalError( PRG_NAME, error_outtablefile );
189 final BasicTable<String> t;
190 final SortedMap<String, String> map;
191 if ( mapping_file != null ) {
192 t = BasicTableParser.parse( mapping_file, '\t' );
193 if ( t.getNumberOfColumns() != 2 ) {
194 ForesterUtil.fatalError( PRG_NAME,
195 "mapping file needs to have 2 tab-separated columns, not "
196 + t.getNumberOfColumns() );
198 map = t.getColumnsAsMap( 0, 1 );
204 final boolean extra_processing1;
205 if ( cla.isOptionSet( EXTRA_PROCESSING_OPTION1 ) ) {
206 extra_processing1 = true;
209 extra_processing1 = false;
211 String extra_processing1_sep = EXTRA_PROCESSING1_SEP_DEFAULT;
212 if ( cla.isOptionSet( EXTRA_PROCESSING1_SEP_OPTION ) ) {
213 if ( !extra_processing1 ) {
214 ForesterUtil.fatalError( PRG_NAME,
215 "extra processing is not enabled, cannot set -"
216 + EXTRA_PROCESSING1_SEP_OPTION + " option" );
218 if ( cla.isOptionValueSet( EXTRA_PROCESSING1_SEP_OPTION ) ) {
219 extra_processing1_sep = cla.getOptionValue( EXTRA_PROCESSING1_SEP_OPTION );
222 ForesterUtil.fatalError( PRG_NAME, "no value for extra processing separator" );
225 if ( ( extra_processing1_sep != null ) && extra_processing1_sep.equals( separator ) ) {
226 ForesterUtil.fatalError( PRG_NAME,
227 "extra processing separator must not be the same the annotation-separator" );
229 boolean extra_processing1_keep = EXTRA_PROCESSING1_KEEP_EXTRA_DEFAULT;
230 if ( cla.isOptionSet( EXTRA_PROCESSING1_KEEP_EXTRA_OPTION ) ) {
231 if ( !extra_processing1 ) {
232 ForesterUtil.fatalError( PRG_NAME,
233 "extra processing is not enabled, cannot set -"
234 + EXTRA_PROCESSING1_KEEP_EXTRA_OPTION + " option" );
236 extra_processing1_keep = true;
238 Pattern special_pattern = null;
239 boolean special_processing = false;
240 if ( cla.isOptionSet( SPECIAL_PROCESSING_OPTION ) ) {
241 if ( extra_processing1 == true ) {
243 .fatalError( PRG_NAME,
244 "extra processing cannot be used together with special processing pattern" );
246 if ( cla.isOptionValueSet( SPECIAL_PROCESSING_OPTION ) ) {
247 final String str = cla.getOptionValue( SPECIAL_PROCESSING_OPTION );
249 special_pattern = Pattern.compile( str );
251 catch ( final PatternSyntaxException e ) {
253 .fatalError( PRG_NAME,
254 "error in special processing pattern: " + str + ": " + e.getMessage() );
256 special_processing = true;
259 ForesterUtil.fatalError( PRG_NAME, "no value for special processing pattern" );
262 final boolean remove_annotation_sep;
263 if ( cla.isOptionSet( REMOVE_ANNOT_SEP_OPTION ) ) {
264 remove_annotation_sep = true;
267 remove_annotation_sep = false;
269 final boolean verbose;
270 if ( cla.isOptionSet( VERBOSE_OPTION ) ) {
277 if ( cla.isOptionSet( QUIET_OPTION ) ) {
283 System.out.println( "Input tree : " + intreefile );
284 System.out.println( "Specific-hit support cutoff: " + cutoff_specifics );
285 if ( mapping_file != null ) {
286 System.out.println( "Mapping file : " + mapping_file + " (" + t.getNumberOfRows()
289 System.out.println( "Annotation-separator : " + separator );
290 if ( remove_annotation_sep ) {
291 System.out.println( "Remove anno.-sep. in output: " + remove_annotation_sep );
293 System.out.println( "Query pattern : " + pattern );
294 if ( extra_processing1 ) {
295 System.out.println( "Extra processing : " + extra_processing1 );
296 System.out.println( "Extra processing separator : " + extra_processing1_sep );
297 System.out.println( "Keep extra annotations : " + extra_processing1_keep );
299 if ( special_processing ) {
300 System.out.println( "Special processing : " + special_processing );
301 System.out.println( "Special processing pattern : " + special_pattern );
303 if ( outtablefile != null ) {
304 System.out.println( "Output table : " + outtablefile );
306 Phylogeny phys[] = null;
308 final PhylogenyFactory factory = ParserBasedPhylogenyFactory.getInstance();
309 final PhylogenyParser pp = ParserUtils.createParserDependingOnFileType( intreefile, true );
310 phys = factory.create( intreefile, pp );
312 catch ( final IOException e ) {
313 ForesterUtil.fatalError( PRG_NAME, "Could not read \"" + intreefile + "\" [" + e.getMessage() + "]" );
315 if ( phys.length == 0 ) {
316 ForesterUtil.fatalError( PRG_NAME, "\"" + intreefile + "\" does not contain any trees" );
318 System.out.println( "Number of input trees : " + phys.length );
319 if ( phys.length == 1 ) {
320 System.out.println( "Ext. nodes in input tree : " + phys[ 0 ].getNumberOfExternalNodes() );
323 System.out.println( "Ext. nodes in input tree 1 : " + phys[ 0 ].getNumberOfExternalNodes() );
325 final EasyWriter outtable_writer;
326 if ( outtablefile != null ) {
327 outtable_writer = ForesterUtil.createEasyWriter( outtablefile );
328 outtable_writer.print( "#" + PRG_NAME + " " + PRG_VERSION + " " + PRG_DATE );
329 outtable_writer.print( " Input tree: " + intreefile );
330 outtable_writer.println( " Specific-hit support cutoff: " + cutoff_specifics );
333 outtable_writer = null;
336 for( final Phylogeny phy : phys ) {
338 AnalysisMulti.performMapping( pattern, map, phy, verbose );
340 if ( extra_processing1 ) {
341 AnalysisMulti.performExtraProcessing1( pattern,
343 extra_processing1_sep,
344 extra_processing1_keep,
348 else if ( special_processing ) {
349 AnalysisMulti.performSpecialProcessing1( pattern, phy, separator, special_pattern, verbose );
351 final ResultMulti res = AnalysisMulti.execute( phy, pattern, separator, cutoff_specifics );
353 if ( phys.length == 1 ) {
354 printResult( res, -1, remove_annotation_sep );
357 printResult( res, counter, remove_annotation_sep );
360 if ( outtable_writer != null ) {
361 writeResultToTable( res, outtable_writer, remove_annotation_sep );
362 outtable_writer.flush();
366 if ( outtable_writer != null ) {
367 outtable_writer.flush();
368 outtable_writer.close();
371 catch ( final UserException e ) {
372 ForesterUtil.fatalError( PRG_NAME, e.getMessage() );
374 catch ( final IOException e ) {
375 ForesterUtil.fatalError( PRG_NAME, e.getMessage() );
377 catch ( final Exception e ) {
379 ForesterUtil.fatalError( PRG_NAME, "Unexpected errror!" );
383 private final static void printResult( final ResultMulti res,
385 final boolean remove_annotation_sep ) {
386 System.out.println();
387 if ( counter == -1 ) {
388 System.out.println( "Result for " + res.getQueryNamePrefix() );
391 System.out.println( "Result for " + res.getQueryNamePrefix() + " [tree " + counter + "]" );
393 if ( ( res.getAllMultiHitPrefixes() == null ) | ( res.getAllMultiHitPrefixes().size() < 1 ) ) {
394 System.out.println( " No match to query pattern!" );
397 System.out.println( " Matching Clade(s):" );
398 for( final Prefix prefix : res.getCollapsedMultiHitPrefixes() ) {
399 if ( remove_annotation_sep ) {
400 System.out.println( " " + prefix.toStringRemovSeparator() );
403 System.out.println( " " + prefix );
406 if ( res.isHasSpecificMultiHitsPrefixes() ) {
407 System.out.println();
408 System.out.println( " Specific-hit(s):" );
409 for( final Prefix prefix : res.getSpecificMultiHitPrefixes() ) {
410 if ( remove_annotation_sep ) {
411 System.out.println( " " + prefix.toStringRemovSeparator() );
414 System.out.println( " " + prefix );
417 System.out.println();
418 System.out.println( " Matching Clade(s) with Specific-hit(s):" );
419 for( final Prefix prefix : res.getCollapsedMultiHitPrefixes() ) {
420 if ( remove_annotation_sep ) {
421 System.out.println( " " + prefix.toStringRemovSeparator() );
424 System.out.println( " " + prefix );
426 for( final Prefix spec : res.getSpecificMultiHitPrefixes() ) {
427 if ( spec.getPrefix().startsWith( prefix.getPrefix() ) ) {
428 if ( remove_annotation_sep ) {
429 System.out.println( " " + spec.toStringRemovSeparator() );
432 System.out.println( " " + spec );
438 if ( !ForesterUtil.isEmpty( res.getAllMultiHitPrefixesDown() ) ) {
439 System.out.println();
440 System.out.println( " Matching Down-tree Bracketing Clade(s):" );
441 for( final Prefix prefix : res.getCollapsedMultiHitPrefixesDown() ) {
442 if ( remove_annotation_sep ) {
443 System.out.println( " " + prefix.toStringRemovSeparator() );
446 System.out.println( " " + prefix );
450 if ( !ForesterUtil.isEmpty( res.getAllMultiHitPrefixesUp() ) ) {
451 System.out.println();
452 System.out.println( " Matching Up-tree Bracketing Clade(s):" );
453 for( final Prefix prefix : res.getCollapsedMultiHitPrefixesUp() ) {
454 if ( remove_annotation_sep ) {
455 System.out.println( " " + prefix.toStringRemovSeparator() );
458 System.out.println( " " + prefix );
462 System.out.println();
463 System.out.println( " Total Number of Matches: " + res.getNumberOfMatches() + "/"
464 + res.getReferenceTreeNumberOfExternalNodes() );
466 System.out.println();
469 private final static void writeResultToTable( final ResultMulti res,
471 final boolean remove_annotation_sep )
473 if ( ( res.getAllMultiHitPrefixes() == null ) | ( res.getAllMultiHitPrefixes().size() < 1 ) ) {
474 w.print( res.getQueryNamePrefix() );
476 w.println( "No match to query pattern!" );
479 for( final Prefix prefix : res.getCollapsedMultiHitPrefixes() ) {
480 w.print( res.getQueryNamePrefix() );
482 w.print( "Matching Clades" );
484 if ( remove_annotation_sep ) {
485 w.print( prefix.getPrefixRemovSeparator() );
488 w.print( prefix.getPrefix() );
491 w.print( df.format( prefix.getConfidence() ) );
493 w.print( String.valueOf( res.getNumberOfMatches() ) );
495 w.print( String.valueOf( res.getReferenceTreeNumberOfExternalNodes() ) );
498 if ( res.isHasSpecificMultiHitsPrefixes() ) {
499 for( final Prefix prefix : res.getSpecificMultiHitPrefixes() ) {
500 w.print( res.getQueryNamePrefix() );
502 w.print( "Specific-hits" );
504 if ( remove_annotation_sep ) {
505 w.print( prefix.getPrefixRemovSeparator() );
508 w.print( prefix.getPrefix() );
511 w.print( df.format( prefix.getConfidence() ) );
513 w.print( String.valueOf( res.getNumberOfMatches() ) );
515 w.print( String.valueOf( res.getReferenceTreeNumberOfExternalNodes() ) );
519 if ( !ForesterUtil.isEmpty( res.getAllMultiHitPrefixesDown() ) ) {
520 for( final Prefix prefix : res.getCollapsedMultiHitPrefixesDown() ) {
521 w.print( res.getQueryNamePrefix() );
523 w.print( "Matching Down-tree Bracketing Clades" );
525 if ( remove_annotation_sep ) {
526 w.print( prefix.getPrefixRemovSeparator() );
529 w.print( prefix.getPrefix() );
532 w.print( df.format( prefix.getConfidence() ) );
534 w.print( String.valueOf( res.getNumberOfMatches() ) );
536 w.print( String.valueOf( res.getReferenceTreeNumberOfExternalNodes() ) );
540 if ( !ForesterUtil.isEmpty( res.getAllMultiHitPrefixesUp() ) ) {
541 for( final Prefix prefix : res.getCollapsedMultiHitPrefixesUp() ) {
542 w.print( res.getQueryNamePrefix() );
544 w.print( "Matching Up-tree Bracketing Clades" );
546 if ( remove_annotation_sep ) {
547 w.print( prefix.getPrefixRemovSeparator() );
550 w.print( prefix.getPrefix() );
553 w.print( df.format( prefix.getConfidence() ) );
555 w.print( String.valueOf( res.getNumberOfMatches() ) );
557 w.print( String.valueOf( res.getReferenceTreeNumberOfExternalNodes() ) );
564 private final static void print_help() {
565 System.out.println( "Usage:" );
566 System.out.println();
567 System.out.println( PRG_NAME + " [options] <input tree(s) file> [output table file]" );
568 System.out.println();
569 System.out.println( " options:" );
570 System.out.println( " -" + SPECIFICS_CUTOFF_OPTION
571 + "=<double> : the minimal confidence value for \"specific-hits\" to be reported (default: "
572 + SPECIFICS_CUTOFF_DEFAULT + ")" );
573 System.out.println( " -" + SEP_OPTION + "=<separator> : the annotation-separator to be used (default: "
574 + SEP_DEFAULT + ")" );
575 System.out.println( " -" + MAPPING_FILE_OPTION
576 + "=<mapping table> : to map node names to appropriate annotations (tab-separated, two columns) (default: no mapping)" );
577 System.out.println( " -" + EXTRA_PROCESSING_OPTION1
578 + " : to enable extra processing of annotations (e.g. \"Q16611|A.1.1\" becomes \"A.1.1\")" );
579 System.out.println( " -" + EXTRA_PROCESSING1_SEP_OPTION
580 + "=<separator> : the separator for extra annotations (default: \"" + EXTRA_PROCESSING1_SEP_DEFAULT
582 System.out.println( " -" + EXTRA_PROCESSING1_KEEP_EXTRA_OPTION
583 + " : to keep extra annotations (e.g. \"Q16611|A.1.1\" becomes \"A.1.1.Q16611\")" );
584 System.out.println( " -" + SPECIAL_PROCESSING_OPTION
585 + "=<pattern> : special processing with pattern (e.g. \"(\\d+)([a-z]+)_.+\" for changing \"6q_EF42\" to \"6.q\")" );
586 System.out.println( " -" + REMOVE_ANNOT_SEP_OPTION
587 + " : to remove the annotation-separator in the output" );
588 System.out.println( " -" + VERBOSE_OPTION + " : verbose" );
589 System.out.println( " -" + QUIET_OPTION + " : quiet (for when used in a pipeline)" );
590 System.out.println( " --" + QUERY_PATTERN_OPTION
591 + "=<pattern> : the regular expression pattern for the query (default: \"" + QUERY_PATTERN_DEFAULT
592 + "\" for pplacer output)" );
593 System.out.println();
594 System.out.println( "Examples:" );
595 System.out.println();
596 System.out.println( " " + PRG_NAME + " pp_out_tree.sing.tre result.tsv" );
597 System.out.println( " " + PRG_NAME + " -c=0.5 -s=. pp_out_tree.sing.tre result.tsv" );
598 System.out.println( " " + PRG_NAME + " -c=0.9 -s=_ -m=map.tsv pp_out_trees.sing.tre result.tsv" );
599 System.out.println( " " + PRG_NAME + " -x -xs=& -xk pp_out_trees.sing.tre result.tsv" );
600 System.out.println( " " + PRG_NAME + " -x -xs=\"|\" pp_out_trees.sing.tre result.tsv" );
601 System.out.println( " " + PRG_NAME + " -x -xk -m=map.tsv pp_out_trees.sing.tre result.tsv" );
602 System.out.println( " " + PRG_NAME + " -m=map.tsv -S='(\\d+)([a-z?]*)_.+' pp_out_trees.sing.tre result.tsv" );
603 System.out.println();