2 // FORESTER -- software libraries and applications
3 // for evolutionary biology research and applications.
5 // Copyright (C) 2013 Christian M. Zmasek
7 // This library is free software; you can redistribute it and/or
8 // modify it under the terms of the GNU Lesser General Public
9 // License as published by the Free Software Foundation; either
10 // version 2.1 of the License, or (at your option) any later version.
12 // This library is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 // Lesser General Public License for more details.
17 // You should have received a copy of the GNU Lesser General Public
18 // License along with this library; if not, write to the Free Software
19 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
21 // Contact: phylosoft @ gmail . com
22 // WWW: https://sites.google.com/site/cmzmasek/home/software/forester
24 package org.forester.io.parsers.nhx;
26 import java.io.BufferedReader;
28 import java.io.FileNotFoundException;
29 import java.io.FileReader;
30 import java.io.IOException;
31 import java.io.InputStream;
32 import java.io.InputStreamReader;
33 import java.util.ArrayList;
34 import java.util.List;
35 import java.util.StringTokenizer;
36 import java.util.regex.Matcher;
37 import java.util.regex.Pattern;
39 import org.forester.io.parsers.PhylogenyParser;
40 import org.forester.io.parsers.phyloxml.PhyloXmlDataFormatException;
41 import org.forester.io.parsers.util.ParserUtils;
42 import org.forester.io.parsers.util.PhylogenyParserException;
43 import org.forester.phylogeny.Phylogeny;
44 import org.forester.phylogeny.PhylogenyMethods;
45 import org.forester.phylogeny.PhylogenyNode;
46 import org.forester.phylogeny.data.Accession;
47 import org.forester.phylogeny.data.Confidence;
48 import org.forester.phylogeny.data.DomainArchitecture;
49 import org.forester.phylogeny.data.Event;
50 import org.forester.phylogeny.data.Identifier;
51 import org.forester.phylogeny.data.PhylogenyDataUtil;
52 import org.forester.phylogeny.data.Sequence;
53 import org.forester.phylogeny.data.Taxonomy;
54 import org.forester.phylogeny.iterators.PhylogenyNodeIterator;
55 import org.forester.util.ForesterUtil;
57 public final class NHXParser implements PhylogenyParser {
59 public static final TAXONOMY_EXTRACTION TAXONOMY_EXTRACTION_DEFAULT = TAXONOMY_EXTRACTION.NO;
60 public final static Pattern UC_LETTERS_NUMBERS_PATTERN = Pattern.compile( "^[A-Z0-9]+$" );
61 public final static Pattern NUMBERS_ONLY_PATTERN = Pattern.compile( "^[0-9\\.]+$" );
62 public final static Pattern MB_PROB_PATTERN = Pattern.compile( "prob=([^,]+)" );
63 public final static Pattern MB_PROB_SD_PATTERN = Pattern.compile( "prob_stddev=([^,]+)" );
64 public final static Pattern MB_BL_PATTERN = Pattern.compile( "length_median=([^,]+)" );
65 final static private boolean GUESS_ROOTEDNESS_DEFAULT = true;
66 final static private boolean GUESS_IF_SUPPORT_VALUES = true;
67 final static private boolean IGNORE_QUOTES_DEFAULT = false;
68 final static public boolean REPLACE_UNDERSCORES_DEFAULT = false;
69 final static private byte STRING = 0;
70 final static private byte STRING_BUFFER = 1;
71 final static private byte CHAR_ARRAY = 2;
72 final static private byte BUFFERED_READER = 3;
73 final static private byte STRING_BUILDER = 4;
74 private boolean _guess_rootedness;
75 private boolean _ignore_quotes;
76 private byte _input_type;
77 private int _source_length;
78 private PhylogenyNode _current_node;
79 private StringBuilder _current_anotation;
80 private Object _nhx_source;
81 private int _clade_level;
82 private Phylogeny _current_phylogeny;
83 private TAXONOMY_EXTRACTION _taxonomy_extraction;
84 private boolean _replace_underscores;
85 private boolean _in_comment = false;
86 private boolean _saw_colon = false;
87 private boolean _saw_closing_paren;
88 private boolean _saw_open_bracket = false;
89 private boolean _in_open_bracket = false;
90 private boolean _in_double_quote = false;
91 private boolean _in_single_quote = false;
92 private String _my_source_str = null;
93 private StringBuffer _my_source_sbuff = null;
94 private StringBuilder _my_source_sbuil = null;
95 private char[] _my_source_charary = null;
96 private BufferedReader _my_source_br = null;
98 private Phylogeny _next;
99 private Object _source;
105 public final TAXONOMY_EXTRACTION getTaxonomyExtraction() {
106 return _taxonomy_extraction;
109 public final boolean hasNext() {
110 return _next != null;
113 public final Phylogeny next() throws NHXFormatException, IOException {
114 final Phylogeny phy = _next;
120 public final Phylogeny[] parse() throws IOException {
122 final List<Phylogeny> l = new ArrayList<Phylogeny>();
123 while ( hasNext() ) {
126 final Phylogeny[] p = new Phylogeny[ l.size() ];
127 for( int i = 0; i < l.size(); ++i ) {
133 public final void reset() throws NHXFormatException, IOException {
138 _saw_open_bracket = false;
139 _in_open_bracket = false;
140 _in_double_quote = false;
141 _in_single_quote = false;
143 _current_anotation = new StringBuilder();
144 _current_phylogeny = null;
145 _current_node = null;
146 _my_source_str = null;
147 _my_source_sbuff = null;
148 _my_source_sbuil = null;
149 _my_source_charary = null;
150 _my_source_br = null;
151 determineSourceType( _source );
152 switch ( _input_type ) {
154 _my_source_str = ( String ) _nhx_source;
157 _my_source_sbuff = ( StringBuffer ) _nhx_source;
160 _my_source_sbuil = ( StringBuilder ) _nhx_source;
163 _my_source_charary = ( char[] ) _nhx_source;
165 case BUFFERED_READER:
166 if ( _my_source_br != null ) {
168 _my_source_br.close();
170 catch ( final IOException e ) {
174 _my_source_br = ( BufferedReader ) _nhx_source;
177 throw new RuntimeException( "unknown input type" );
182 public final void setGuessRootedness( final boolean guess_rootedness ) {
183 _guess_rootedness = guess_rootedness;
186 public final void setIgnoreQuotes( final boolean ignore_quotes ) {
187 _ignore_quotes = ignore_quotes;
190 public final void setReplaceUnderscores( final boolean replace_underscores ) {
191 _replace_underscores = replace_underscores;
195 public final void setSource( final Object nhx_source ) throws NHXFormatException, IOException {
196 _source = nhx_source;
200 public final void setTaxonomyExtraction( final TAXONOMY_EXTRACTION taxonomy_extraction ) {
201 _taxonomy_extraction = taxonomy_extraction;
204 private final void determineSourceType( final Object nhx_source ) throws PhylogenyParserException,
205 FileNotFoundException {
206 if ( nhx_source == null ) {
207 throw new PhylogenyParserException( getClass() + ": attempt to parse null object." );
209 else if ( nhx_source instanceof String ) {
210 _input_type = NHXParser.STRING;
211 _source_length = ( ( String ) nhx_source ).length();
212 _nhx_source = nhx_source;
214 else if ( nhx_source instanceof StringBuilder ) {
215 _input_type = NHXParser.STRING_BUILDER;
216 _source_length = ( ( StringBuilder ) nhx_source ).length();
217 _nhx_source = nhx_source;
219 else if ( nhx_source instanceof StringBuffer ) {
220 _input_type = NHXParser.STRING_BUFFER;
221 _source_length = ( ( StringBuffer ) nhx_source ).length();
222 _nhx_source = nhx_source;
224 else if ( nhx_source instanceof StringBuilder ) {
225 _input_type = NHXParser.STRING_BUILDER;
226 _source_length = ( ( StringBuilder ) nhx_source ).length();
227 _nhx_source = nhx_source;
229 else if ( nhx_source instanceof char[] ) {
230 _input_type = NHXParser.CHAR_ARRAY;
231 _source_length = ( ( char[] ) nhx_source ).length;
232 _nhx_source = nhx_source;
234 else if ( nhx_source instanceof File ) {
235 _input_type = NHXParser.BUFFERED_READER;
237 final File f = ( File ) nhx_source;
238 final String error = ForesterUtil.isReadableFile( f );
239 if ( !ForesterUtil.isEmpty( error ) ) {
240 throw new PhylogenyParserException( error );
242 _nhx_source = new BufferedReader( new FileReader( f ) );
244 else if ( nhx_source instanceof InputStream ) {
245 _input_type = NHXParser.BUFFERED_READER;
247 final InputStreamReader isr = new InputStreamReader( ( InputStream ) nhx_source );
248 _nhx_source = new BufferedReader( isr );
251 throw new IllegalArgumentException( getClass() + " can only parse objects of type String,"
252 + " StringBuffer, StringBuilder, char[], File," + " or InputStream "
253 + " [attempt to parse object of " + nhx_source.getClass() + "]." );
257 private final Phylogeny finishPhylogeny() throws PhylogenyParserException, NHXFormatException,
258 PhyloXmlDataFormatException {
259 if ( _current_phylogeny != null ) {
260 parseNHX( _current_anotation != null ? _current_anotation.toString() : "",
261 _current_phylogeny.getRoot(),
262 getTaxonomyExtraction(),
263 isReplaceUnderscores() );
264 if ( GUESS_IF_SUPPORT_VALUES ) {
265 if ( isBranchLengthsLikeBootstrapValues( _current_phylogeny ) ) {
266 moveBranchLengthsToConfidenceValues( _current_phylogeny );
269 if ( isGuessRootedness() ) {
270 final PhylogenyNode root = _current_phylogeny.getRoot();
271 if ( ( root.getDistanceToParent() >= 0.0 ) || !ForesterUtil.isEmpty( root.getName() )
272 || !ForesterUtil.isEmpty( PhylogenyMethods.getSpecies( root ) ) || root.isHasAssignedEvent() ) {
273 _current_phylogeny.setRooted( true );
276 return _current_phylogeny;
281 private final Phylogeny finishSingleNodePhylogeny() throws PhylogenyParserException, NHXFormatException,
282 PhyloXmlDataFormatException {
283 final PhylogenyNode new_node = new PhylogenyNode();
284 parseNHX( _current_anotation.toString(), new_node, getTaxonomyExtraction(), isReplaceUnderscores() );
285 _current_phylogeny = new Phylogeny();
286 _current_phylogeny.setRoot( new_node );
287 return _current_phylogeny;
290 private final void getNext() throws IOException, NHXFormatException {
293 if ( _input_type == BUFFERED_READER ) {
294 final int ci = _my_source_br.read();
303 if ( _i >= _source_length ) {
307 switch ( _input_type ) {
309 c = _my_source_str.charAt( _i );
312 c = _my_source_sbuff.charAt( _i );
315 c = _my_source_sbuil.charAt( _i );
318 c = _my_source_charary[ _i ];
323 if ( !_in_single_quote && !_in_double_quote ) {
327 else if ( !( ( c < 33 ) || ( c > 126 ) ) && _saw_colon
328 && ( ( c != '[' ) && ( c != '.' ) && ( ( c < 48 ) || ( c > 57 ) ) ) ) {
331 if ( _in_open_bracket && ( c == ']' ) ) {
332 _in_open_bracket = false;
335 // \n\t is always ignored,
336 // as is " (34) and ' (39) (space is 32):
337 if ( ( isIgnoreQuotes() && ( ( c < 33 ) || ( c > 126 ) || ( c == 34 ) || ( c == 39 ) || ( ( _clade_level == 0 ) && ( c == ';' ) ) ) )
338 || ( !isIgnoreQuotes() && ( ( c < 32 ) || ( c > 126 ) || ( ( _clade_level == 0 ) && ( c == ';' ) ) ) ) ) {
341 else if ( ( c == 32 ) && ( !_in_single_quote && !_in_double_quote ) ) {
344 else if ( _in_comment ) {
349 else if ( _in_double_quote ) {
351 _in_double_quote = false;
354 _current_anotation.append( c );
357 else if ( c == '"' ) {
358 _in_double_quote = true;
360 else if ( _in_single_quote ) {
362 _in_single_quote = false;
365 _current_anotation.append( c );
368 else if ( c == 39 ) {
369 _in_single_quote = true;
371 else if ( c == '[' ) {
372 _saw_open_bracket = true;
373 _in_open_bracket = true;
375 else if ( _saw_open_bracket ) {
377 // everything not starting with "[&" is considered a comment
378 // unless ":digits and/or . [bootstrap]":
380 _current_anotation.append( "[&" );
382 else if ( _saw_colon ) {
383 _current_anotation.append( "[" + c );
389 // comment consisting just of "[]":
390 _saw_open_bracket = false;
392 else if ( ( c == '(' ) && !_in_open_bracket ) {
393 final Phylogeny phy = processOpenParen();
401 else if ( ( c == ')' ) && !_in_open_bracket ) {
404 else if ( ( c == ',' ) && !_in_open_bracket ) {
408 _current_anotation.append( c );
412 if ( _clade_level != 0 ) {
413 throw new PhylogenyParserException( "error in NH (Newick) formatted data: most likely cause: number of open parens does not equal number of close parens" );
415 if ( _current_phylogeny != null ) {
416 _next = finishPhylogeny();
417 _current_phylogeny = null;
418 _current_anotation = null;
420 else if ( ( _current_anotation != null ) && ( _current_anotation.length() > 0 ) ) {
421 _next = finishSingleNodePhylogeny();
422 _current_anotation = null;
429 private final void init() {
430 setTaxonomyExtraction( TAXONOMY_EXTRACTION_DEFAULT );
431 setReplaceUnderscores( REPLACE_UNDERSCORES_DEFAULT );
432 setGuessRootedness( GUESS_ROOTEDNESS_DEFAULT );
433 setIgnoreQuotes( IGNORE_QUOTES_DEFAULT );
436 private final boolean isGuessRootedness() {
437 return _guess_rootedness;
440 private final boolean isIgnoreQuotes() {
441 return _ignore_quotes;
444 private final boolean isReplaceUnderscores() {
445 return _replace_underscores;
448 private final void processCloseParen() throws PhylogenyParserException, NHXFormatException,
449 PhyloXmlDataFormatException {
450 if ( _clade_level < 0 ) {
451 throw new PhylogenyParserException( "error in NH (Newick)/NHX formatted data: most likely cause: number of close parens is larger than number of open parens" );
454 if ( !_saw_closing_paren ) {
455 final PhylogenyNode new_node = new PhylogenyNode();
456 parseNHX( _current_anotation.toString(), new_node, getTaxonomyExtraction(), isReplaceUnderscores() );
457 _current_anotation = new StringBuilder();
458 _current_node.addAsChild( new_node );
461 parseNHX( _current_anotation.toString(),
462 _current_node.getLastChildNode(),
463 getTaxonomyExtraction(),
464 isReplaceUnderscores() );
465 _current_anotation = new StringBuilder();
467 if ( !_current_node.isRoot() ) {
468 _current_node = _current_node.getParent();
470 _saw_closing_paren = true;
473 private final void processComma() throws PhylogenyParserException, NHXFormatException, PhyloXmlDataFormatException {
474 if ( !_saw_closing_paren ) {
475 final PhylogenyNode new_node = new PhylogenyNode();
476 parseNHX( _current_anotation.toString(), new_node, getTaxonomyExtraction(), isReplaceUnderscores() );
477 if ( _current_node == null ) {
478 throw new NHXFormatException( "format might not be NH or NHX" );
480 _current_node.addAsChild( new_node );
483 parseNHX( _current_anotation.toString(),
484 _current_node.getLastChildNode(),
485 getTaxonomyExtraction(),
486 isReplaceUnderscores() );
488 _current_anotation = new StringBuilder();
489 _saw_closing_paren = false;
492 private final Phylogeny processOpenParen() throws PhylogenyParserException, NHXFormatException,
493 PhyloXmlDataFormatException {
494 Phylogeny phy = null;
495 final PhylogenyNode new_node = new PhylogenyNode();
496 if ( _clade_level == 0 ) {
497 if ( _current_phylogeny != null ) {
498 phy = finishPhylogeny();
501 _current_anotation = new StringBuilder();
502 _current_phylogeny = new Phylogeny();
503 _current_phylogeny.setRoot( new_node );
507 _current_node.addAsChild( new_node );
509 _current_node = new_node;
510 _saw_closing_paren = false;
514 public final static void parseNHX( String s,
515 final PhylogenyNode node_to_annotate,
516 final TAXONOMY_EXTRACTION taxonomy_extraction,
517 final boolean replace_underscores ) throws NHXFormatException,
518 PhyloXmlDataFormatException {
519 if ( ( taxonomy_extraction != TAXONOMY_EXTRACTION.NO ) && replace_underscores ) {
520 throw new IllegalArgumentException( "cannot extract taxonomies and replace under scores at the same time" );
522 if ( ( s != null ) && ( s.length() > 0 ) ) {
523 if ( replace_underscores ) {
524 s = s.replaceAll( "_+", " " );
526 boolean is_nhx = false;
527 final int ob = s.indexOf( "[" );
531 final int cb = s.indexOf( "]" );
533 throw new NHXFormatException( "error in NHX formatted data: no closing \"]\" in \"" + s + "\"" );
535 if ( s.indexOf( "&&NHX" ) == ( ob + 1 ) ) {
536 b = s.substring( ob + 6, cb );
539 // No &&NHX and digits only: is likely to be a support value.
540 final String bracketed = s.substring( ob + 1, cb );
541 final Matcher numbers_only = NUMBERS_ONLY_PATTERN.matcher( bracketed );
542 if ( numbers_only.matches() ) {
543 b = ":" + NHXtags.SUPPORT + bracketed;
545 else if ( s.indexOf( "prob=" ) > -1 ) {
546 processMrBayes3Data( s, node_to_annotate );
549 s = s.substring( 0, ob ) + b;
550 if ( ( s.indexOf( "[" ) > -1 ) || ( s.indexOf( "]" ) > -1 ) ) {
551 throw new NHXFormatException( "error in NHX formatted data: more than one \"]\" or \"[\"" );
554 final StringTokenizer t = new StringTokenizer( s, ":" );
555 if ( t.countTokens() > 0 ) {
556 if ( !s.startsWith( ":" ) ) {
557 node_to_annotate.setName( t.nextToken() );
558 if ( !replace_underscores && ( !is_nhx && ( taxonomy_extraction != TAXONOMY_EXTRACTION.NO ) ) ) {
559 ParserUtils.extractTaxonomyDataFromNodeName( node_to_annotate, taxonomy_extraction );
562 while ( t.hasMoreTokens() ) {
564 if ( s.startsWith( NHXtags.SPECIES_NAME ) ) {
565 if ( !node_to_annotate.getNodeData().isHasTaxonomy() ) {
566 node_to_annotate.getNodeData().setTaxonomy( new Taxonomy() );
568 node_to_annotate.getNodeData().getTaxonomy().setScientificName( s.substring( 2 ) );
570 else if ( s.startsWith( NHXtags.IS_DUPLICATION ) ) {
571 if ( ( s.charAt( 2 ) == 'Y' ) || ( s.charAt( 2 ) == 'T' ) ) {
572 node_to_annotate.getNodeData().setEvent( Event.createSingleDuplicationEvent() );
574 else if ( ( s.charAt( 2 ) == 'N' ) || ( s.charAt( 2 ) == 'F' ) ) {
575 node_to_annotate.getNodeData().setEvent( Event.createSingleSpeciationEvent() );
577 else if ( s.charAt( 2 ) == '?' ) {
578 node_to_annotate.getNodeData().setEvent( Event.createSingleSpeciationOrDuplicationEvent() );
581 throw new NHXFormatException( "error in NHX formatted data: :D=Y or :D=N or :D=?" );
584 else if ( s.startsWith( NHXtags.SUPPORT ) ) {
585 PhylogenyMethods.setConfidence( node_to_annotate, doubleValue( s.substring( 2 ) ) );
587 else if ( s.startsWith( NHXtags.TAXONOMY_ID ) ) {
588 if ( !node_to_annotate.getNodeData().isHasTaxonomy() ) {
589 node_to_annotate.getNodeData().setTaxonomy( new Taxonomy() );
591 node_to_annotate.getNodeData().getTaxonomy().setIdentifier( new Identifier( s.substring( 2 ) ) );
593 else if ( s.startsWith( NHXtags.DOMAIN_STRUCTURE ) ) {
594 if ( !node_to_annotate.getNodeData().isHasSequence() ) {
595 node_to_annotate.getNodeData().setSequence( new Sequence() );
597 node_to_annotate.getNodeData().getSequence()
598 .setDomainArchitecture( new DomainArchitecture( s.substring( 3 ) ) );
600 else if ( s.startsWith( NHXtags.SEQUENCE_ACCESSION ) ) {
601 if ( !node_to_annotate.getNodeData().isHasSequence() ) {
602 node_to_annotate.getNodeData().setSequence( new Sequence() );
604 node_to_annotate.getNodeData().getSequence()
605 .setAccession( new Accession( s.substring( 3 ), "?" ) );
607 else if ( s.startsWith( NHXtags.GENE_NAME ) ) {
608 if ( !node_to_annotate.getNodeData().isHasSequence() ) {
609 node_to_annotate.getNodeData().setSequence( new Sequence() );
611 node_to_annotate.getNodeData().getSequence().setName( s.substring( 3 ) );
613 else if ( s.indexOf( '=' ) < 0 ) {
614 if ( node_to_annotate.getDistanceToParent() != PhylogenyDataUtil.BRANCH_LENGTH_DEFAULT ) {
615 throw new NHXFormatException( "error in NHX formatted data: more than one distance to parent:"
618 node_to_annotate.setDistanceToParent( doubleValue( s ) );
620 } // while ( t.hasMoreTokens() )
625 private final static double doubleValue( final String str ) throws NHXFormatException {
627 return Double.valueOf( str ).doubleValue();
629 catch ( final NumberFormatException ex ) {
630 throw new NHXFormatException( "error in NH/NHX formatted data: failed to parse number from " + "\"" + str
635 private final static boolean isBranchLengthsLikeBootstrapValues( final Phylogeny p ) {
636 final PhylogenyNodeIterator it = p.iteratorExternalForward();
637 final double d0 = it.next().getDistanceToParent();
638 if ( ( d0 < 10 ) || !it.hasNext() ) {
641 while ( it.hasNext() ) {
642 final double d = it.next().getDistanceToParent();
643 if ( ( d != d0 ) || ( d < 10 ) ) {
650 private final static void moveBranchLengthsToConfidenceValues( final Phylogeny p ) {
651 final PhylogenyNodeIterator it = p.iteratorPostorder();
652 while ( it.hasNext() ) {
653 final PhylogenyNode n = it.next();
654 PhylogenyMethods.setBootstrapConfidence( n, n.getDistanceToParent() );
655 n.setDistanceToParent( PhylogenyDataUtil.BRANCH_LENGTH_DEFAULT );
659 private final static void processMrBayes3Data( final String s, final PhylogenyNode node_to_annotate )
660 throws NHXFormatException {
662 final Matcher mb_prob_sd_matcher = MB_PROB_SD_PATTERN.matcher( s );
663 if ( mb_prob_sd_matcher.find() ) {
665 sd = Double.parseDouble( mb_prob_sd_matcher.group( 1 ) );
667 catch ( final NumberFormatException e ) {
668 throw new NHXFormatException( "failed to parse probability standard deviation (Mr Bayes output) from \""
672 final Matcher mb_prob_matcher = MB_PROB_PATTERN.matcher( s );
673 if ( mb_prob_matcher.find() ) {
676 prob = Double.parseDouble( mb_prob_matcher.group( 1 ) );
678 catch ( final NumberFormatException e ) {
679 throw new NHXFormatException( "failed to parse probability (Mr Bayes output) from \"" + s + "\"" );
683 node_to_annotate.getBranchData()
684 .addConfidence( new Confidence( prob, "posterior probability", sd ) );
687 node_to_annotate.getBranchData().addConfidence( new Confidence( prob, "posterior probability" ) );
691 final Matcher mb_bl_matcher = MB_BL_PATTERN.matcher( s );
692 if ( mb_bl_matcher.find() ) {
695 bl = Double.parseDouble( mb_bl_matcher.group( 1 ) );
697 catch ( final NumberFormatException e ) {
698 throw new NHXFormatException( "failed to parse median branch length (Mr Bayes output) from \"" + s
702 node_to_annotate.setDistanceToParent( bl );
707 public static enum TAXONOMY_EXTRACTION {
708 NO, YES, PFAM_STYLE_ONLY;