2 // FORESTER -- software libraries and applications
3 // for evolutionary biology research and applications.
5 // Copyright (C) 2008-2009 Christian M. Zmasek
6 // Copyright (C) 2008-2009 Burnham Institute for Medical Research
7 // Copyright (C) 2000-2001 Washington University School of Medicine
8 // and Howard Hughes Medical Institute
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
25 // Contact: phylosoft @ gmail . com
26 // WWW: www.phylosoft.org/forester
28 package org.forester.sdi;
31 import java.io.FileNotFoundException;
32 import java.io.IOException;
33 import java.util.ArrayList;
34 import java.util.Arrays;
35 import java.util.HashMap;
36 import java.util.HashSet;
37 import java.util.List;
40 import org.forester.datastructures.IntMatrix;
41 import org.forester.io.parsers.PhylogenyParser;
42 import org.forester.io.parsers.nhx.NHXParser;
43 import org.forester.io.parsers.util.ParserUtils;
44 import org.forester.phylogeny.Phylogeny;
45 import org.forester.phylogeny.PhylogenyMethods;
46 import org.forester.phylogeny.PhylogenyNode;
47 import org.forester.phylogeny.factories.ParserBasedPhylogenyFactory;
48 import org.forester.phylogeny.factories.PhylogenyFactory;
49 import org.forester.phylogeny.iterators.PhylogenyNodeIterator;
50 import org.forester.util.ForesterUtil;
52 public final class RIO {
54 private final static boolean ROOT_BY_MINIMIZING_SUM_OF_DUPS = true;
55 private final static boolean ROOT_BY_MINIMIZING_TREE_HEIGHT = true;
56 private Phylogeny[] _analyzed_gene_trees;
57 private HashMap<String, HashMap<String, Integer>> _o_maps;
58 private HashMap<String, HashMap<String, Integer>> _so_maps;
59 private HashMap<String, HashMap<String, Integer>> _up_maps;
60 private List<String> _seq_names;
62 private int _ext_nodes;
65 * Default constructor.
66 * @throws SDIException
68 * @throws RIOException
70 public RIO( final File gene_trees_file, final Phylogeny species_tree, final String query ) throws IOException,
71 SDIException, RIOException {
72 if ( ForesterUtil.isEmpty( query ) ) {
73 throw new IllegalArgumentException( "query is empty" );
76 inferOrthologs( gene_trees_file, species_tree, query );
79 public RIO( final File gene_trees_file, final Phylogeny species_tree ) throws IOException, SDIException,
82 inferOrthologs( gene_trees_file, species_tree, null );
85 public final Phylogeny[] getAnalyzedGeneTrees() {
86 return _analyzed_gene_trees;
90 * Returns the numbers of number of ext nodes in gene trees analyzed (after
93 * @return number of ext nodes in gene trees analyzed (after stripping)
95 public final int getExtNodesOfAnalyzedGeneTrees() {
100 * Returns a HashMap containing the inferred "ultra paralogs" of the
101 * external gene tree node with the sequence name seq_name. Sequence names
102 * are the keys (String), numbers of observations are the values (Int).
103 * "ultra paralogs" are to be inferred by method "inferOrthologs". Throws an
104 * exception if seq_name is not found.
107 * sequence name of a external node of the gene trees
108 * @return HashMap containing the inferred ultra paralogs
109 * (name(String)->value(Int))
111 public final HashMap<String, Integer> getInferredUltraParalogs( final String seq_name ) {
112 if ( _up_maps == null ) {
115 return _up_maps.get( seq_name );
118 public final int getNumberOfSamples() {
123 * Returns a String containg the names of orthologs of the PhylogenyNode
124 * with seq name query_name. The String also contains how many times a
125 * particular ortholog has been observed.
128 * The output order is (per line): Name, Ortholog, Subtree neighbor, Super
132 * The sort priority of this is determined by sort in the following manner:
135 * <li>1 : Ortholog, Super ortholog
136 * <li>2 : Super ortholog, Ortholog
139 * Returns "-" if no putative orthologs have been found (given
140 * threshold_orthologs).
142 * Orthologs are to be inferred by method "inferOrthologs".
144 * (Last modified: 05/08/01)
147 * sequence name of a external node of the gene trees
149 * order and sort priority
150 * @param threshold_orthologs
151 * the minimal number of observations for a a sequence to be
152 * reported as orthologous, in percents (0.0-100.0%)
153 * @param threshold_subtreeneighborings
154 * the minimal number of observations for a a sequence to be
155 * reported as orthologous, in percents (0.0-100.0%)
156 * @return String containing the inferred orthologs, String containing "-"
157 * if no orthologs have been found null in case of error
159 public final StringBuffer inferredOrthologsToString( final String query_name, int sort, double threshold_orthologs ) {
160 HashMap<String, Integer> o_hashmap = null;
161 HashMap<String, Integer> s_hashmap = null;
163 double o = 0.0; // Orthologs.
164 double s = 0.0; // Super orthologs.
167 final ArrayList<ResultLine> nv = new ArrayList<ResultLine>();
168 if ( ( _o_maps == null ) || ( _so_maps == null ) ) {
169 throw new RuntimeException( "orthologs have not been calculated (successfully)" );
171 if ( ( sort < 0 ) || ( sort > 2 ) ) {
174 if ( threshold_orthologs < 0.0 ) {
175 threshold_orthologs = 0.0;
177 else if ( threshold_orthologs > 100.0 ) {
178 threshold_orthologs = 100.0;
180 o_hashmap = getInferredOrthologs( query_name );
181 s_hashmap = getInferredSuperOrthologs( query_name );
182 if ( ( o_hashmap == null ) || ( s_hashmap == null ) ) {
183 throw new RuntimeException( "Orthologs for " + query_name + " were not established" );
185 final StringBuffer orthologs = new StringBuffer();
186 if ( _seq_names.size() > 0 ) {
187 I: for( int i = 0; i < _seq_names.size(); ++i ) {
188 name = _seq_names.get( i );
189 if ( name.equals( query_name ) ) {
192 o = getBootstrapValueFromHash( o_hashmap, name );
193 if ( o < threshold_orthologs ) {
196 s = getBootstrapValueFromHash( s_hashmap, name );
199 nv.add( new ResultLine( name, o, 5 ) );
202 nv.add( new ResultLine( name, o, s, 5 ) );
205 nv.add( new ResultLine( name, s, o, 5 ) );
208 nv.add( new ResultLine( name, o, 5 ) );
210 } // End of I for loop.
211 if ( ( nv != null ) && ( nv.size() > 0 ) ) {
212 orthologs.append( "seq name\t\tortho\ts-ortho" + ForesterUtil.LINE_SEPARATOR );
213 final ResultLine[] nv_array = new ResultLine[ nv.size() ];
214 for( int j = 0; j < nv.size(); ++j ) {
215 nv_array[ j ] = nv.get( j );
217 Arrays.sort( nv_array );
218 for( final ResultLine element : nv_array ) {
219 name = element.getKey();
220 value1 = element.getValue1();
221 value2 = element.getValue2();
222 orthologs.append( addNameAndValues( name, value1, value2, sort ) );
226 // No orthologs found.
227 if ( ( orthologs == null ) || ( orthologs.length() < 1 ) ) {
228 orthologs.append( "-" );
234 * Returns a String containg the names of orthologs of the PhylogenyNode
235 * with seq name query_name. The String also contains how many times a
236 * particular ortholog has been observed. Returns "-" if no putative
237 * orthologs have been found (given threshold_orthologs).
239 * Orthologs are to be inferred by method "inferOrthologs".
242 * sequence name of a external node of the gene trees
243 * @param return_dists
244 * @param threshold_ultra_paralogs
246 * @return String containing the inferred orthologs, String containing "-"
247 * if no orthologs have been found null in case of error
249 public final String inferredUltraParalogsToString( final String query_name, double threshold_ultra_paralogs ) {
250 HashMap<String, Integer> sp_hashmap = null;
251 String name = "", ultra_paralogs = "";
256 final List<ResultLine> nv = new ArrayList<ResultLine>();
257 if ( threshold_ultra_paralogs < 1.0 ) {
258 threshold_ultra_paralogs = 1.0;
260 else if ( threshold_ultra_paralogs > 100.0 ) {
261 threshold_ultra_paralogs = 100.0;
263 if ( _up_maps == null ) {
264 throw new RuntimeException( "Ultra paralogs have not been calculated (successfully)." );
266 sp_hashmap = getInferredUltraParalogs( query_name );
267 if ( sp_hashmap == null ) {
268 throw new RuntimeException( "Ultra paralogs for " + query_name + " were not established" );
270 if ( _seq_names.size() > 0 ) {
271 I: for( int i = 0; i < _seq_names.size(); ++i ) {
272 name = _seq_names.get( i );
273 if ( name.equals( query_name ) ) {
276 sp = getBootstrapValueFromHash( sp_hashmap, name );
277 if ( sp < threshold_ultra_paralogs ) {
280 nv.add( new ResultLine( name, sp, 5 ) );
281 } // End of I for loop.
282 if ( ( nv != null ) && ( nv.size() > 0 ) ) {
283 final ResultLine[] nv_array = new ResultLine[ nv.size() ];
284 for( int j = 0; j < nv.size(); ++j ) {
285 nv_array[ j ] = nv.get( j );
287 Arrays.sort( nv_array );
289 for( final ResultLine element : nv_array ) {
290 name = element.getKey();
291 value1 = element.getValue1();
292 value2 = element.getValue2();
293 ultra_paralogs += addNameAndValues( name, value1, value2, sort );
297 // No ultra paralogs found.
298 if ( ( ultra_paralogs == null ) || ( ultra_paralogs.length() < 1 ) ) {
299 ultra_paralogs = "-";
301 return ultra_paralogs;
304 // Helper method for inferredOrthologsToString.
305 // inferredOrthologsToArrayList,
306 // and inferredUltraParalogsToString.
307 private final double getBootstrapValueFromHash( final HashMap<String, Integer> h, final String name ) {
308 if ( !h.containsKey( name ) ) {
311 final int i = h.get( name );
312 return ( ( i * 100.0 ) / getNumberOfSamples() );
316 * Returns a HashMap containing the inferred orthologs of the external gene
317 * tree node with the sequence name seq_name. Sequence names are the keys
318 * (String), numbers of observations are the values (Int). Orthologs are to
319 * be inferred by method "inferOrthologs". Throws an exception if seq_name
323 * sequence name of a external node of the gene trees
324 * @return HashMap containing the inferred orthologs
325 * (name(String)->value(Int))
327 private final HashMap<String, Integer> getInferredOrthologs( final String seq_name ) {
328 if ( _o_maps == null ) {
331 return _o_maps.get( seq_name );
335 * Returns a HashMap containing the inferred "super orthologs" of the
336 * external gene tree node with the sequence name seq_name. Sequence names
337 * are the keys (String), numbers of observations are the values (Int).
338 * Super orthologs are to be inferred by method "inferOrthologs". Throws an
339 * exception if seq_name is not found.
342 * sequence name of a external node of the gene trees
343 * @return HashMap containing the inferred super orthologs
344 * (name(String)->value(Int))
346 private final HashMap<String, Integer> getInferredSuperOrthologs( final String seq_name ) {
347 if ( _so_maps == null ) {
350 return _so_maps.get( seq_name );
354 * Infers the orthologs (as well the "super orthologs", the "subtree
355 * neighbors", and the "ultra paralogs") for each external node of the gene
356 * Trees in multiple tree File gene_trees_file (=output of PHYLIP NEIGHBOR,
357 * for example). Tallies how many times each sequence is (super-)
358 * orthologous towards the query. Tallies how many times each sequence is
359 * ultra paralogous towards the query. Tallies how many times each sequence
360 * is a subtree neighbor of the query. Gene duplications are inferred using
361 * SDI. Modifies its argument species_tree. Is a little faster than
362 * "inferOrthologs(File,Phylogeny)" since orthologs are only inferred for
365 * To obtain the results use the methods listed below.
367 * @param gene_trees_file
368 * a File containing gene Trees in NH format, which is the result
369 * of performing a bootstrap analysis in PHYLIP
370 * @param species_tree
371 * a species Phylogeny, which has species names in its species
374 * the sequence name of the squence whose orthologs are to be
376 * @throws SDIException
377 * @throws RIOException
378 * @throws IOException
379 * @throws FileNotFoundException
381 private final void inferOrthologs( final File gene_trees_file, final Phylogeny species_tree, final String query )
382 throws SDIException, RIOException, FileNotFoundException, IOException {
383 // Read in first tree to get its sequence names
384 // and strip species_tree.
385 final PhylogenyFactory factory = ParserBasedPhylogenyFactory.getInstance();
386 final PhylogenyParser p = ParserUtils.createParserDependingOnFileType( gene_trees_file, true );
387 if ( p instanceof NHXParser ) {
388 final NHXParser nhx = ( NHXParser ) p;
389 nhx.setReplaceUnderscores( false );
390 nhx.setIgnoreQuotes( true );
391 nhx.setTaxonomyExtraction( NHXParser.TAXONOMY_EXTRACTION.YES );
393 final Phylogeny[] gene_trees = factory.create( gene_trees_file, p );
394 // Removes from species_tree all species not found in gene_tree.
395 PhylogenyMethods.taxonomyBasedDeletionOfExternalNodes( gene_trees[ 0 ], species_tree );
396 if ( species_tree.isEmpty() ) {
397 throw new RIOException( "failed to establish species based mapping between gene and species trees" );
399 if ( !ForesterUtil.isEmpty( query ) ) {
400 PhylogenyMethods.taxonomyBasedDeletionOfExternalNodes( species_tree, gene_trees[ 0 ] );
401 if ( gene_trees[ 0 ].isEmpty() ) {
402 throw new RIOException( "failed to establish species based mapping between gene and species trees" );
404 _seq_names = getAllExternalSequenceNames( gene_trees[ 0 ] );
405 if ( ( _seq_names == null ) || ( _seq_names.size() < 1 ) ) {
406 throw new RIOException( "could not get sequence names" );
408 _o_maps = new HashMap<String, HashMap<String, Integer>>();
409 _so_maps = new HashMap<String, HashMap<String, Integer>>();
410 _up_maps = new HashMap<String, HashMap<String, Integer>>();
411 _o_maps.put( query, new HashMap<String, Integer>( _seq_names.size() ) );
412 _so_maps.put( query, new HashMap<String, Integer>( _seq_names.size() ) );
413 _up_maps.put( query, new HashMap<String, Integer>( _seq_names.size() ) );
415 _analyzed_gene_trees = new Phylogeny[ gene_trees.length ];
417 for( final Phylogeny gt : gene_trees ) {
418 // Removes from gene_tree all species not found in species_tree.
419 PhylogenyMethods.taxonomyBasedDeletionOfExternalNodes( species_tree, gt );
420 if ( gt.isEmpty() ) {
421 throw new RIOException( "failed to establish species based mapping between gene and species trees" );
423 _analyzed_gene_trees[ c++ ] = inferOrthologsHelper( gt, species_tree, query );
425 setNumberOfSamples( gene_trees.length );
428 // Helper method which performs the actual ortholog inference for
429 // the external node with seqname query.
430 private final Phylogeny inferOrthologsHelper( final Phylogeny gene_tree,
431 final Phylogeny species_tree,
432 final String query ) throws SDIException, RIOException {
433 final SDIR sdiunrooted = new SDIR();
434 final Phylogeny assigned_tree = sdiunrooted.infer( gene_tree,
437 RIO.ROOT_BY_MINIMIZING_SUM_OF_DUPS,
438 RIO.ROOT_BY_MINIMIZING_TREE_HEIGHT,
441 setExtNodesOfAnalyzedGeneTrees( assigned_tree.getNumberOfExternalNodes() );
442 if ( !ForesterUtil.isEmpty( query ) ) {
443 final List<PhylogenyNode> nodes = getNodesViaSequenceName( assigned_tree, query );
444 if ( nodes.size() > 1 ) {
445 throw new RIOException( "node named [" + query + "] not unique" );
447 else if ( nodes.isEmpty() ) {
448 throw new RIOException( "no node containing a sequence named [" + query + "] found" );
450 final PhylogenyNode query_node = nodes.get( 0 );
451 updateCounts( _o_maps, query, PhylogenyMethods.getOrthologousNodes( assigned_tree, query_node ) );
452 updateCounts( _so_maps, query, PhylogenyMethods.getSuperOrthologousNodes( query_node ) );
453 updateCounts( _up_maps, query, PhylogenyMethods.getUltraParalogousNodes( query_node ) );
455 return assigned_tree;
458 private final void init() {
467 private final void setExtNodesOfAnalyzedGeneTrees( final int i ) {
471 private final void setNumberOfSamples( int i ) {
478 // Helper for doInferOrthologs( Phylogeny, Phylogeny, String )
479 // and doInferOrthologs( Phylogeny, Phylogeny ).
480 private final void updateCounts( final HashMap<String, HashMap<String, Integer>> counter_map,
481 final String query_seq_name,
482 final List<PhylogenyNode> nodes ) {
483 final HashMap<String, Integer> hash_map = counter_map.get( query_seq_name );
484 if ( hash_map == null ) {
485 throw new RuntimeException( "unexpected error in updateCounts" );
487 for( int j = 0; j < nodes.size(); ++j ) {
489 if ( ( nodes.get( j ) ).getNodeData().isHasSequence()
490 && !ForesterUtil.isEmpty( ( nodes.get( j ) ).getNodeData().getSequence().getName() ) ) {
491 seq_name = ( nodes.get( j ) ).getNodeData().getSequence().getName();
494 seq_name = ( nodes.get( j ) ).getName();
496 if ( hash_map.containsKey( seq_name ) ) {
497 hash_map.put( seq_name, hash_map.get( seq_name ) + 1 );
500 hash_map.put( seq_name, 1 );
505 public final static IntMatrix calculateOrthologTable( final Phylogeny[] analyzed_gene_trees ) throws RIOException {
506 final List<String> labels = new ArrayList<String>();
507 final Set<String> labels_set = new HashSet<String>();
509 for( final PhylogenyNode n : analyzed_gene_trees[ 0 ].getExternalNodes() ) {
510 if ( n.getNodeData().isHasSequence() && !ForesterUtil.isEmpty( n.getNodeData().getSequence().getName() ) ) {
511 label = n.getNodeData().getSequence().getName();
513 else if ( n.getNodeData().isHasSequence()
514 && !ForesterUtil.isEmpty( n.getNodeData().getSequence().getSymbol() ) ) {
515 label = n.getNodeData().getSequence().getSymbol();
517 else if ( !ForesterUtil.isEmpty( n.getName() ) ) {
521 throw new IllegalArgumentException( "node " + n + " has no appropriate label" );
523 if ( labels_set.contains( label ) ) {
524 throw new IllegalArgumentException( "label " + label + " is not unique" );
526 labels_set.add( label );
529 final IntMatrix m = new IntMatrix( labels );
531 for( final Phylogeny gt : analyzed_gene_trees ) {
533 PhylogenyMethods.preOrderReId( gt );
534 final HashMap<String, PhylogenyNode> map = PhylogenyMethods.createNameToExtNodeMap( gt );
535 for( int x = 0; x < m.size(); ++x ) {
536 final String mx = m.getLabel( x );
537 final PhylogenyNode nx = map.get( mx );
539 throw new RIOException( "node \"" + mx + "\" not present in gene tree #" + counter );
543 for( int y = 0; y < m.size(); ++y ) {
544 my = m.getLabel( y );
547 throw new RIOException( "node \"" + my + "\" not present in gene tree #" + counter );
549 if ( !PhylogenyMethods.calculateLCAonTreeWithIdsInPreOrder( nx, ny ).isDuplication() ) {
550 m.inreaseByOne( x, y );
559 * Returns the order in which ortholog (o), "super ortholog" (s) and
560 * distance (d) are returned and sorted (priority of sort always goes from
561 * left to right), given sort. For the meaning of sort
563 * @see #inferredOrthologsToString(String,int,double,double)
566 * determines order and sort priority
567 * @return String indicating the order
569 public final static String getOrder( final int sort ) {
573 order = "orthologies";
576 order = "orthologies > super orthologies";
579 order = "super orthologies > orthologies";
582 order = "orthologies";
588 public final static StringBuffer getOrderHelp() {
589 final StringBuffer sb = new StringBuffer();
590 sb.append( " 0: orthologies" + ForesterUtil.LINE_SEPARATOR );
591 sb.append( " 1: orthologies > super orthologies" + ForesterUtil.LINE_SEPARATOR );
592 sb.append( " 2: super orthologies > orthologies" + ForesterUtil.LINE_SEPARATOR );
596 // Helper method for inferredOrthologsToString
597 // and inferredUltraParalogsToString.
598 private final static String addNameAndValues( final String name,
602 final java.text.DecimalFormat df = new java.text.DecimalFormat( "0.#####" );
603 df.setDecimalSeparatorAlwaysShown( false );
605 if ( name.length() < 8 ) {
606 line += ( name + "\t\t\t" );
608 else if ( name.length() < 16 ) {
609 line += ( name + "\t\t" );
612 line += ( name + "\t" );
616 line += addToLine( value1, df );
620 line += addToLine( value1, df );
621 line += addToLine( value2, df );
624 line += addToLine( value2, df );
625 line += addToLine( value1, df );
628 line += addToLine( value1, df );
632 line += addToLine( value1, df );
633 line += addToLine( value2, df );
636 line += ForesterUtil.LINE_SEPARATOR;
640 // Helper for addNameAndValues.
641 private final static String addToLine( final double value, final java.text.DecimalFormat df ) {
643 if ( value != ResultLine.DEFAULT ) {
644 s = df.format( value ) + "\t";
652 private final static List<String> getAllExternalSequenceNames( final Phylogeny phy ) throws RIOException {
653 final List<String> names = new ArrayList<String>();
654 for( final PhylogenyNodeIterator iter = phy.iteratorExternalForward(); iter.hasNext(); ) {
655 final PhylogenyNode n = iter.next();
656 if ( n.getNodeData().isHasSequence() && !ForesterUtil.isEmpty( n.getNodeData().getSequence().getName() ) ) {
657 names.add( n.getNodeData().getSequence().getName() );
659 else if ( !ForesterUtil.isEmpty( n.getName() ) ) {
660 names.add( n.getName() );
663 throw new RIOException( "node has no (sequence) name: " + n );
669 private final static List<PhylogenyNode> getNodesViaSequenceName( final Phylogeny phy, final String seq_name ) {
670 final List<PhylogenyNode> nodes = new ArrayList<PhylogenyNode>();
671 for( final PhylogenyNodeIterator iter = phy.iteratorPreorder(); iter.hasNext(); ) {
672 final PhylogenyNode n = iter.next();
673 if ( n.getNodeData().isHasSequence() && n.getNodeData().getSequence().getName().equals( seq_name ) ) {
676 if ( !n.getNodeData().isHasSequence() && n.getName().equals( seq_name ) ) {
683 private final class ResultLine implements Comparable<ResultLine> {
685 public static final int DEFAULT = -999;
686 private final String _key;
687 private final double _value1;
688 private final double _value2;
691 ResultLine( final String name, final double value1, final double value2, final int c ) {
696 if ( ( c >= 0 ) && ( c <= 2 ) ) {
701 ResultLine( final String name, final double value1, final int c ) {
705 _value2 = ResultLine.DEFAULT;
712 public int compareTo( final ResultLine n ) {
713 if ( ( getValue1() != ResultLine.DEFAULT ) && ( n.getValue1() != ResultLine.DEFAULT ) ) {
714 if ( getValue1() < n.getValue1() ) {
717 if ( getValue1() > n.getValue1() ) {
721 if ( ( getValue2() != ResultLine.DEFAULT ) && ( n.getValue2() != ResultLine.DEFAULT ) ) {
722 if ( getValue2() < n.getValue2() ) {
725 if ( getValue2() > n.getValue2() ) {
729 return ( getKey().compareTo( n.getKey() ) );
744 private void setSigns() {
746 _p[ 0 ] = _p[ 1 ] = +1;