d4eabfd251f85a6192e0abbcf5f303ae3e119da0
[jalview.git] / forester / java / src / org / forester / io / parsers / nexus / NexusPhylogeniesParser.java
1 // $Id:
2 // FORESTER -- software libraries and applications
3 // for evolutionary biology research and applications.
4 //
5 // Copyright (C) 2008-2009 Christian M. Zmasek
6 // Copyright (C) 2008-2009 Burnham Institute for Medical Research
7 // All rights reserved
8 //
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
13 //
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
18 //
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
22 //
23 // Contact: phylosoft @ gmail . com
24 // WWW: https://sites.google.com/site/cmzmasek/home/software/forester
25
26 package org.forester.io.parsers.nexus;
27
28 import java.io.BufferedReader;
29 import java.io.FileNotFoundException;
30 import java.io.IOException;
31 import java.util.ArrayList;
32 import java.util.HashMap;
33 import java.util.List;
34 import java.util.Map;
35 import java.util.regex.Matcher;
36 import java.util.regex.Pattern;
37
38 import org.forester.archaeopteryx.AptxConstants;
39 import org.forester.io.parsers.IteratingPhylogenyParser;
40 import org.forester.io.parsers.PhylogenyParser;
41 import org.forester.io.parsers.nhx.NHXFormatException;
42 import org.forester.io.parsers.nhx.NHXParser;
43 import org.forester.io.parsers.nhx.NHXParser.TAXONOMY_EXTRACTION;
44 import org.forester.io.parsers.util.ParserUtils;
45 import org.forester.io.parsers.util.PhylogenyParserException;
46 import org.forester.phylogeny.Phylogeny;
47 import org.forester.phylogeny.PhylogenyNode;
48 import org.forester.phylogeny.data.Sequence;
49 import org.forester.phylogeny.iterators.PhylogenyNodeIterator;
50 import org.forester.sequence.BasicSequence;
51 import org.forester.sequence.MolecularSequence;
52 import org.forester.util.ForesterUtil;
53
54 public final class NexusPhylogeniesParser implements IteratingPhylogenyParser, PhylogenyParser {
55
56     private static final String UTF_8 = "UTF-8";
57
58     final private static boolean DEBUG                               = false;
59     
60     final private static String            begin_trees               = NexusConstants.BEGIN_TREES.toLowerCase();
61     final private static String            end                       = NexusConstants.END.toLowerCase();
62     final private static String            endblock                  = "endblock";
63     final private static Pattern           ROOTEDNESS_PATTERN        = Pattern.compile( ".+=\\s*\\[&([R|U])\\].*" );
64     final private static String            taxlabels                 = NexusConstants.TAXLABELS.toLowerCase();
65     final private static Pattern           TITLE_PATTERN             = Pattern.compile( "TITLE.?\\s+([^;]+)",
66                                                                                         Pattern.CASE_INSENSITIVE );
67     final private static String            translate                 = NexusConstants.TRANSLATE.toLowerCase();
68     final private static String            data                      = NexusConstants.BEGIN_CHARACTERS.toLowerCase();
69     final private static String            characters                = NexusConstants.BEGIN_DATA.toLowerCase();
70     final private static String            tree                      = NexusConstants.TREE.toLowerCase();
71     final private static Pattern           TREE_NAME_PATTERN         = Pattern.compile( "\\s*.?Tree\\s+(.+?)\\s*=.+",
72                                                                                         Pattern.CASE_INSENSITIVE );
73     final private static Pattern           TRANSLATE_PATTERN         = Pattern.compile( "([0-9A-Za-z]+)\\s+(.+)" );
74     final private static Pattern           ALN_PATTERN               = Pattern.compile( "(.+)\\s+([A-Za-z-_\\*\\?]+)" );
75     final private static Pattern           DATATYPE_PATTERN          = Pattern.compile( "datatype\\s?.\\s?([a-z]+)" );
76     //final private static Pattern           LINK_TAXA_PATTERN         = Pattern.compile( "link\\s+taxa\\s?.\\s?([^;]+)",
77     //                                                                                    Pattern.CASE_INSENSITIVE );
78     final private static String            utree                     = NexusConstants.UTREE.toLowerCase();
79     private BufferedReader                 _br;
80     private boolean                        _ignore_quotes_in_nh_data = AptxConstants.NH_PARSING_IGNORE_QUOTES_DEFAULT;
81     private boolean                        _in_taxalabels;
82     private boolean                        _in_translate;
83     private boolean                        _in_tree;
84     private boolean                        _in_trees_block;
85     private boolean                        _in_data_block;
86     private boolean                        _is_rooted;
87     private String                         _datatype;
88     private String                         _name;
89     private Phylogeny                      _next;
90     private Object                         _nexus_source;
91     private StringBuilder                  _nh;
92     private boolean                        _replace_underscores      = NHXParser.REPLACE_UNDERSCORES_DEFAULT;
93     private boolean                        _rooted_info_present;
94     private List<String>                   _taxlabels;
95     private TAXONOMY_EXTRACTION            _taxonomy_extraction      = TAXONOMY_EXTRACTION.NO;
96     private String                         _title;
97     private Map<String, String>            _translate_map;
98     private StringBuilder                  _translate_sb;
99     private Map<String, MolecularSequence> _seqs;
100     private final boolean                  _add_sequences            = true;
101
102     @Override
103     public String getName() {
104         return "Nexus Phylogenies Parser";
105     }
106
107     @Override
108     public final boolean hasNext() {
109         return _next != null;
110     }
111
112     @Override
113     public final Phylogeny next() throws NHXFormatException, IOException {
114         final Phylogeny phy = _next;
115         getNext();
116         return phy;
117     }
118
119     @Override
120     public final Phylogeny[] parse() throws IOException {
121         final List<Phylogeny> l = new ArrayList<Phylogeny>();
122         while ( hasNext() ) {
123             l.add( next() );
124         }
125         final Phylogeny[] p = new Phylogeny[ l.size() ];
126         for( int i = 0; i < l.size(); ++i ) {
127             p[ i ] = l.get( i );
128         }
129         reset();
130         return p;
131     }
132
133     @Override
134     public final void reset() throws FileNotFoundException, IOException {
135         _taxlabels = new ArrayList<String>();
136         _translate_map = new HashMap<String, String>();
137         _nh = new StringBuilder();
138         _name = "";
139         _title = "";
140         _translate_sb = null;
141         _next = null;
142         _in_trees_block = false;
143         _in_taxalabels = false;
144         _in_translate = false;
145         _in_tree = false;
146         _rooted_info_present = false;
147         _is_rooted = false;
148         _seqs = new HashMap<String, MolecularSequence>();
149         _br = ParserUtils.createReader( _nexus_source, UTF_8 );
150         getNext();
151     }
152
153     public final void setIgnoreQuotes( final boolean ignore_quotes_in_nh_data ) {
154         _ignore_quotes_in_nh_data = ignore_quotes_in_nh_data;
155     }
156
157     public final void setReplaceUnderscores( final boolean replace_underscores ) {
158         _replace_underscores = replace_underscores;
159     }
160
161     @Override
162     public final void setSource( final Object nexus_source ) throws PhylogenyParserException, IOException {
163         if ( nexus_source == null ) {
164             throw new PhylogenyParserException( "attempt to parse null object" );
165         }
166         _nexus_source = nexus_source;
167         reset();
168     }
169
170     public final void setTaxonomyExtraction( final TAXONOMY_EXTRACTION taxonomy_extraction ) {
171         _taxonomy_extraction = taxonomy_extraction;
172     }
173
174     private final void createPhylogeny( final String title,
175                                         final String name,
176                                         final StringBuilder nhx,
177                                         final boolean rooted_info_present,
178                                         final boolean is_rooted ) throws IOException {
179         _next = null;
180         final NHXParser pars = new NHXParser();
181         pars.setTaxonomyExtraction( _taxonomy_extraction );
182         pars.setReplaceUnderscores( _replace_underscores );
183         pars.setIgnoreQuotes( _ignore_quotes_in_nh_data );
184         if ( rooted_info_present ) {
185             pars.setGuessRootedness( false );
186         }
187         pars.setSource( nhx.toString() );
188         final Phylogeny p = pars.next();
189         if ( p == null ) {
190             throw new PhylogenyParserException( "failed to create phylogeny" );
191         }
192         String myname = null;
193         if ( !ForesterUtil.isEmpty( title ) && !ForesterUtil.isEmpty( name ) ) {
194             myname = title.replace( '_', ' ' ).trim() + " (" + name.trim() + ")";
195         }
196         else if ( !ForesterUtil.isEmpty( title ) ) {
197             myname = title.replace( '_', ' ' ).trim();
198         }
199         else if ( !ForesterUtil.isEmpty( name ) ) {
200             myname = name.trim();
201         }
202         if ( !ForesterUtil.isEmpty( myname ) ) {
203             p.setName( myname );
204         }
205         if ( rooted_info_present ) {
206             p.setRooted( is_rooted );
207         }
208         if ( ( _taxlabels.size() > 0 ) || ( _translate_map.size() > 0 ) ) {
209             final PhylogenyNodeIterator it = p.iteratorExternalForward();
210             while ( it.hasNext() ) {
211                 final PhylogenyNode node = it.next();
212                 if ( ( _translate_map.size() > 0 ) && _translate_map.containsKey( node.getName() ) ) {
213                     node.setName( _translate_map.get( node.getName() ).replaceAll( "['\"]+", "" ) );
214                 }
215                 else if ( _taxlabels.size() > 0 ) {
216                     int i = -1;
217                     try {
218                         i = Integer.parseInt( node.getName() );
219                     }
220                     catch ( final NumberFormatException e ) {
221                         // Ignore.
222                     }
223                     if ( i > 0 ) {
224                         node.setName( _taxlabels.get( i - 1 ).replaceAll( "['\"]+", "" ) );
225                     }
226                 }
227                 if ( !_replace_underscores && ( ( _taxonomy_extraction != TAXONOMY_EXTRACTION.NO ) ) ) {
228                     ParserUtils.extractTaxonomyDataFromNodeName( node, _taxonomy_extraction );
229                 }
230                 else if ( _replace_underscores ) {
231                     if ( !ForesterUtil.isEmpty( node.getName() ) ) {
232                         node.setName( node.getName().replace( '_', ' ' ).trim() );
233                     }
234                 }
235                 if ( _add_sequences ) {
236                     if ( _seqs.containsKey( node.getName() ) ) {
237                         final MolecularSequence s = _seqs.get( node.getName() );
238                         //TODO need to check for uniqueness when adding seqs....
239                         final Sequence ns = new Sequence( s );
240                         ns.setMolecularSequenceAligned( true ); //TODO need to check if all same length
241                         node.getNodeData().addSequence( ns );
242                     }
243                 }
244             }
245         }
246         _next = p;
247     }
248
249     private final void getNext() throws IOException, NHXFormatException {
250         _next = null;
251         String line;
252         while ( ( line = _br.readLine() ) != null ) {
253             if ( DEBUG ) {
254                 System.out.println( line );
255             }
256             line = line.trim();
257             if ( ( line.length() > 0 ) && !line.startsWith( "#" ) && !line.startsWith( ">" ) ) {
258                 line = ForesterUtil.collapseWhiteSpace( line );
259                 line = removeWhiteSpaceBeforeSemicolon( line );
260                 final String line_lc = line.toLowerCase();
261                 if ( line_lc.startsWith( begin_trees ) ) {
262                     _in_trees_block = true;
263                     _in_taxalabels = false;
264                     _in_translate = false;
265                     _in_data_block = false;
266                     _datatype = null;
267                     _title = "";
268                 }
269                 else if ( line_lc.startsWith( taxlabels ) ) {
270                     //TODO need to be taxa block instead
271                     _in_trees_block = false;
272                     _in_taxalabels = true;
273                     _in_translate = false;
274                     _in_data_block = false;
275                     _datatype = null;
276                 }
277                 else if ( line_lc.startsWith( translate ) ) {
278                     _translate_sb = new StringBuilder();
279                     _in_taxalabels = false;
280                     _in_translate = true;
281                     _in_data_block = false;
282                     _datatype = null;
283                 }
284                 else if ( line_lc.startsWith( characters ) || line_lc.startsWith( data ) ) {
285                     _in_taxalabels = false;
286                     _in_trees_block = false;
287                     _in_translate = false;
288                     _in_data_block = true;
289                     _datatype = null;
290                 }
291                 else if ( _in_trees_block ) {
292                     if ( line_lc.startsWith( "title" ) ) {
293                         final Matcher title_m = TITLE_PATTERN.matcher( line );
294                         if ( title_m.lookingAt() ) {
295                             _title = title_m.group( 1 );
296                         }
297                     }
298                     else if ( line_lc.startsWith( "link" ) ) {
299                         //final Matcher link_m = LINK_TAXA_PATTERN.matcher( line );
300                         //if ( link_m.lookingAt() ) {
301                             //final String link = link_m.group( 1 );  //TODO why?
302                        // }
303                     }
304                     else if ( line_lc.startsWith( end ) || line_lc.startsWith( endblock ) ) {
305                         _in_trees_block = false;
306                         _in_tree = false;
307                         _in_translate = false;
308                         if ( _nh.length() > 0 ) {
309                             createPhylogeny( _title, _name, _nh, _rooted_info_present, _is_rooted );
310                             _nh = new StringBuilder();
311                             _name = "";
312                             _rooted_info_present = false;
313                             _is_rooted = false;
314                             if ( _next != null ) {
315                                 return;
316                             }
317                         }
318                     }
319                     else if ( line_lc.startsWith( tree ) || ( line_lc.startsWith( utree ) ) ) {
320                         boolean might = false;
321                         if ( _nh.length() > 0 ) {
322                             might = true;
323                             createPhylogeny( _title, _name, _nh, _rooted_info_present, _is_rooted );
324                             _nh = new StringBuilder();
325                             _name = "";
326                             _rooted_info_present = false;
327                             _is_rooted = false;
328                         }
329                         _in_tree = true;
330                         _nh.append( line.substring( line.indexOf( '=' ) ) );
331                         final Matcher name_matcher = TREE_NAME_PATTERN.matcher( line );
332                         if ( name_matcher.matches() ) {
333                             _name = name_matcher.group( 1 );
334                             _name = _name.replaceAll( "['\"]+", "" );
335                         }
336                         final Matcher rootedness_matcher = ROOTEDNESS_PATTERN.matcher( line );
337                         if ( rootedness_matcher.matches() ) {
338                             final String s = rootedness_matcher.group( 1 );
339                             line = line.replaceAll( "\\[\\&.\\]", "" );
340                             _rooted_info_present = true;
341                             if ( s.toUpperCase().equals( "R" ) ) {
342                                 _is_rooted = true;
343                             }
344                         }
345                         if ( might && ( _next != null ) ) {
346                             return;
347                         }
348                     }
349                     else if ( _in_tree && !_in_translate ) {
350                         _nh.append( line );
351                     }
352                     if ( !line_lc.startsWith( "title" ) && !line_lc.startsWith( "link" ) && !_in_translate
353                             && !line_lc.startsWith( end ) && !line_lc.startsWith( endblock ) && line_lc.endsWith( ";" ) ) {
354                         _in_tree = false;
355                         _in_translate = false;
356                         createPhylogeny( _title, _name, _nh, _rooted_info_present, _is_rooted );
357                         _nh = new StringBuilder();
358                         _name = "";
359                         _rooted_info_present = false;
360                         _is_rooted = false;
361                         if ( _next != null ) {
362                             return;
363                         }
364                     }
365                 }
366                 if ( _in_taxalabels ) {
367                     if ( line_lc.startsWith( end ) || line_lc.startsWith( endblock ) ) {
368                         _in_taxalabels = false;
369                     }
370                     else {
371                         final String[] labels = line.split( "\\s+" );
372                         for( String label : labels ) {
373                             if ( !label.toLowerCase().equals( taxlabels ) ) {
374                                 if ( label.endsWith( ";" ) ) {
375                                     _in_taxalabels = false;
376                                     label = label.substring( 0, label.length() - 1 );
377                                 }
378                                 if ( label.length() > 0 ) {
379                                     _taxlabels.add( label );
380                                 }
381                             }
382                         }
383                     }
384                 }
385                 if ( _in_translate ) {
386                     if ( line_lc.startsWith( end ) || line_lc.startsWith( endblock ) ) {
387                         _in_translate = false;
388                     }
389                     else {
390                         _translate_sb.append( " " );
391                         _translate_sb.append( line.trim() );
392                         if ( line.endsWith( ";" ) ) {
393                             _in_translate = false;
394                             setTranslateKeyValuePairs( _translate_sb );
395                         }
396                     }
397                 }
398                 if ( _in_data_block ) {
399                     if ( line_lc.startsWith( end ) || line_lc.startsWith( endblock ) ) {
400                         _in_data_block = false;
401                         _datatype = null;
402                     }
403                     else if ( line_lc.startsWith( "link" ) ) {
404                      //   final Matcher link_m = LINK_TAXA_PATTERN.matcher( line );
405                      //   if ( link_m.lookingAt() ) {
406                      //       final String link = link_m.group( 1 );
407                      //   }
408                     }
409                     else {
410                         final Matcher datatype_matcher = DATATYPE_PATTERN.matcher( line_lc );
411                         if ( datatype_matcher.find() ) {
412                             _datatype = datatype_matcher.group( 1 );
413                         }
414                         else {
415                             if ( ( _datatype != null )
416                                     && ( _datatype.equals( "protein" ) || _datatype.equals( "dna" ) || _datatype
417                                             .equals( "rna" ) ) ) {
418                                 if ( line.endsWith( ";" ) ) {
419                                     _in_data_block = false;
420                                     line = line.substring( 0, line.length() - 1 );
421                                 }
422                                 final Matcher aln_matcher = ALN_PATTERN.matcher( line );
423                                 if ( aln_matcher.matches() ) {
424                                     final String id = aln_matcher.group( 1 );
425                                     final String seq = aln_matcher.group( 2 );
426                                     MolecularSequence s = null;
427                                     if ( _datatype.equals( "protein" ) ) {
428                                         s = BasicSequence.createAaSequence( id, seq );
429                                     }
430                                     else if ( _datatype.equals( "dna" ) ) {
431                                         s = BasicSequence.createDnaSequence( id, seq );
432                                     }
433                                     else {
434                                         s = BasicSequence.createRnaSequence( id, seq );
435                                     }
436                                     _seqs.put( id, s );
437                                 }
438                             }
439                         }
440                     }
441                 }
442             }
443         }
444         if ( _nh.length() > 0 ) {
445             createPhylogeny( _title, _name, _nh, _rooted_info_present, _is_rooted );
446             if ( _next != null ) {
447                 return;
448             }
449         }
450     }
451
452     private final void setTranslateKeyValuePairs( final StringBuilder translate_sb ) throws IOException {
453         String s = translate_sb.toString().trim();
454         if ( s.endsWith( ";" ) ) {
455             s = s.substring( 0, s.length() - 1 ).trim();
456         }
457         for( String pair : s.split( "," ) ) {
458             String key = "";
459             String value = "";
460             final int ti = pair.toLowerCase().indexOf( "translate" );
461             if ( ti > -1 ) {
462                 pair = pair.substring( ti + 9 );
463             }
464             final Matcher m = TRANSLATE_PATTERN.matcher( pair );
465             if ( m.find() ) {
466                 key = m.group( 1 );
467                 value = m.group( 2 ).replaceAll( "\'", "" ).replaceAll( "\"", "" ).trim();
468             }
469             else {
470                 throw new IOException( "ill-formatted translate values: " + pair );
471             }
472             if ( value.endsWith( ";" ) ) {
473                 value = value.substring( 0, value.length() - 1 );
474             }
475             _translate_map.put( key, value );
476         }
477     }
478
479     private final static String removeWhiteSpaceBeforeSemicolon( final String s ) {
480         return s.replaceAll( "\\s+;", ";" );
481     }
482 }