2 // FORESTER -- software libraries and applications
3 // for evolutionary biology research and applications.
5 // Copyright (C) 2008-2009 Christian M. Zmasek
6 // Copyright (C) 2008-2009 Burnham Institute for Medical Research
7 // Copyright (C) 2000-2001 Washington University School of Medicine
8 // and Howard Hughes Medical Institute
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
25 // Contact: phylosoft @ gmail . com
26 // WWW: www.phylosoft.org/forester
28 package org.forester.sdi;
31 import java.io.IOException;
32 import java.util.ArrayList;
33 import java.util.Arrays;
34 import java.util.HashMap;
35 import java.util.HashSet;
36 import java.util.List;
39 import org.forester.datastructures.IntMatrix;
40 import org.forester.io.parsers.PhylogenyParser;
41 import org.forester.io.parsers.nhx.NHXParser;
42 import org.forester.io.parsers.util.ParserUtils;
43 import org.forester.phylogeny.Phylogeny;
44 import org.forester.phylogeny.PhylogenyMethods;
45 import org.forester.phylogeny.PhylogenyNode;
46 import org.forester.phylogeny.factories.ParserBasedPhylogenyFactory;
47 import org.forester.phylogeny.factories.PhylogenyFactory;
48 import org.forester.phylogeny.iterators.PhylogenyNodeIterator;
49 import org.forester.util.ForesterUtil;
51 public final class RIO {
53 private final static boolean ROOT_BY_MINIMIZING_SUM_OF_DUPS = true;
54 private final static boolean ROOT_BY_MINIMIZING_TREE_HEIGHT = true;
55 private Phylogeny[] _analyzed_gene_trees;
56 private HashMap<String, HashMap<String, Integer>> _o_maps;
57 private HashMap<String, HashMap<String, Integer>> _so_maps;
58 private HashMap<String, HashMap<String, Integer>> _up_maps;
59 private List<String> _seq_names;
61 private int _ext_nodes;
64 * Default constructor.
65 * @throws SDIException
68 public RIO( final File gene_trees_file, final Phylogeny species_tree, final String query ) throws IOException,
70 if ( ForesterUtil.isEmpty( query ) ) {
71 throw new IllegalArgumentException( "query is empty" );
74 inferOrthologs( gene_trees_file, species_tree, query );
77 public RIO( final File gene_trees_file, final Phylogeny species_tree ) throws IOException, SDIException {
79 inferOrthologs( gene_trees_file, species_tree, null );
82 public final Phylogeny[] getAnalyzedGeneTrees() {
83 return _analyzed_gene_trees;
87 * Returns the numbers of number of ext nodes in gene trees analyzed (after
90 * @return number of ext nodes in gene trees analyzed (after stripping)
92 public final int getExtNodesOfAnalyzedGeneTrees() {
97 * Returns a HashMap containing the inferred "ultra paralogs" of the
98 * external gene tree node with the sequence name seq_name. Sequence names
99 * are the keys (String), numbers of observations are the values (Int).
100 * "ultra paralogs" are to be inferred by method "inferOrthologs". Throws an
101 * exception if seq_name is not found.
104 * sequence name of a external node of the gene trees
105 * @return HashMap containing the inferred ultra paralogs
106 * (name(String)->value(Int))
108 public final HashMap<String, Integer> getInferredUltraParalogs( final String seq_name ) {
109 if ( _up_maps == null ) {
112 return _up_maps.get( seq_name );
115 public final int getNumberOfSamples() {
120 * Returns a String containg the names of orthologs of the PhylogenyNode
121 * with seq name query_name. The String also contains how many times a
122 * particular ortholog has been observed.
125 * The output order is (per line): Name, Ortholog, Subtree neighbor, Super
129 * The sort priority of this is determined by sort in the following manner:
132 * <li>1 : Ortholog, Super ortholog
133 * <li>2 : Super ortholog, Ortholog
136 * Returns "-" if no putative orthologs have been found (given
137 * threshold_orthologs).
139 * Orthologs are to be inferred by method "inferOrthologs".
141 * (Last modified: 05/08/01)
144 * sequence name of a external node of the gene trees
146 * order and sort priority
147 * @param threshold_orthologs
148 * the minimal number of observations for a a sequence to be
149 * reported as orthologous, in percents (0.0-100.0%)
150 * @param threshold_subtreeneighborings
151 * the minimal number of observations for a a sequence to be
152 * reported as orthologous, in percents (0.0-100.0%)
153 * @return String containing the inferred orthologs, String containing "-"
154 * if no orthologs have been found null in case of error
156 public final StringBuffer inferredOrthologsToString( final String query_name, int sort, double threshold_orthologs ) {
157 HashMap<String, Integer> o_hashmap = null;
158 HashMap<String, Integer> s_hashmap = null;
160 double o = 0.0; // Orthologs.
161 double s = 0.0; // Super orthologs.
164 final ArrayList<ResultLine> nv = new ArrayList<ResultLine>();
165 if ( ( _o_maps == null ) || ( _so_maps == null ) ) {
166 throw new RuntimeException( "orthologs have not been calculated (successfully)" );
168 if ( ( sort < 0 ) || ( sort > 2 ) ) {
171 if ( threshold_orthologs < 0.0 ) {
172 threshold_orthologs = 0.0;
174 else if ( threshold_orthologs > 100.0 ) {
175 threshold_orthologs = 100.0;
177 o_hashmap = getInferredOrthologs( query_name );
178 s_hashmap = getInferredSuperOrthologs( query_name );
179 if ( ( o_hashmap == null ) || ( s_hashmap == null ) ) {
180 throw new RuntimeException( "Orthologs for " + query_name + " were not established" );
182 final StringBuffer orthologs = new StringBuffer();
183 if ( _seq_names.size() > 0 ) {
184 I: for( int i = 0; i < _seq_names.size(); ++i ) {
185 name = _seq_names.get( i );
186 if ( name.equals( query_name ) ) {
189 o = getBootstrapValueFromHash( o_hashmap, name );
190 if ( o < threshold_orthologs ) {
193 s = getBootstrapValueFromHash( s_hashmap, name );
196 nv.add( new ResultLine( name, o, 5 ) );
199 nv.add( new ResultLine( name, o, s, 5 ) );
202 nv.add( new ResultLine( name, s, o, 5 ) );
205 nv.add( new ResultLine( name, o, 5 ) );
207 } // End of I for loop.
208 if ( ( nv != null ) && ( nv.size() > 0 ) ) {
209 orthologs.append( "seq name\t\tortho\ts-ortho" + ForesterUtil.LINE_SEPARATOR );
210 final ResultLine[] nv_array = new ResultLine[ nv.size() ];
211 for( int j = 0; j < nv.size(); ++j ) {
212 nv_array[ j ] = nv.get( j );
214 Arrays.sort( nv_array );
215 for( final ResultLine element : nv_array ) {
216 name = element.getKey();
217 value1 = element.getValue1();
218 value2 = element.getValue2();
219 orthologs.append( addNameAndValues( name, value1, value2, sort ) );
223 // No orthologs found.
224 if ( ( orthologs == null ) || ( orthologs.length() < 1 ) ) {
225 orthologs.append( "-" );
231 * Returns a String containg the names of orthologs of the PhylogenyNode
232 * with seq name query_name. The String also contains how many times a
233 * particular ortholog has been observed. Returns "-" if no putative
234 * orthologs have been found (given threshold_orthologs).
236 * Orthologs are to be inferred by method "inferOrthologs".
239 * sequence name of a external node of the gene trees
240 * @param return_dists
241 * @param threshold_ultra_paralogs
243 * @return String containing the inferred orthologs, String containing "-"
244 * if no orthologs have been found null in case of error
246 public final String inferredUltraParalogsToString( final String query_name, double threshold_ultra_paralogs ) {
247 HashMap<String, Integer> sp_hashmap = null;
248 String name = "", ultra_paralogs = "";
253 final List<ResultLine> nv = new ArrayList<ResultLine>();
254 if ( threshold_ultra_paralogs < 1.0 ) {
255 threshold_ultra_paralogs = 1.0;
257 else if ( threshold_ultra_paralogs > 100.0 ) {
258 threshold_ultra_paralogs = 100.0;
260 if ( _up_maps == null ) {
261 throw new RuntimeException( "Ultra paralogs have not been calculated (successfully)." );
263 sp_hashmap = getInferredUltraParalogs( query_name );
264 if ( sp_hashmap == null ) {
265 throw new RuntimeException( "Ultra paralogs for " + query_name + " were not established" );
267 if ( _seq_names.size() > 0 ) {
268 I: for( int i = 0; i < _seq_names.size(); ++i ) {
269 name = _seq_names.get( i );
270 if ( name.equals( query_name ) ) {
273 sp = getBootstrapValueFromHash( sp_hashmap, name );
274 if ( sp < threshold_ultra_paralogs ) {
277 nv.add( new ResultLine( name, sp, 5 ) );
278 } // End of I for loop.
279 if ( ( nv != null ) && ( nv.size() > 0 ) ) {
280 final ResultLine[] nv_array = new ResultLine[ nv.size() ];
281 for( int j = 0; j < nv.size(); ++j ) {
282 nv_array[ j ] = nv.get( j );
284 Arrays.sort( nv_array );
286 for( final ResultLine element : nv_array ) {
287 name = element.getKey();
288 value1 = element.getValue1();
289 value2 = element.getValue2();
290 ultra_paralogs += addNameAndValues( name, value1, value2, sort );
294 // No ultra paralogs found.
295 if ( ( ultra_paralogs == null ) || ( ultra_paralogs.length() < 1 ) ) {
296 ultra_paralogs = "-";
298 return ultra_paralogs;
301 // Helper method for inferredOrthologsToString.
302 // inferredOrthologsToArrayList,
303 // and inferredUltraParalogsToString.
304 private final double getBootstrapValueFromHash( final HashMap<String, Integer> h, final String name ) {
305 if ( !h.containsKey( name ) ) {
308 final int i = h.get( name );
309 return ( ( i * 100.0 ) / getNumberOfSamples() );
313 * Returns a HashMap containing the inferred orthologs of the external gene
314 * tree node with the sequence name seq_name. Sequence names are the keys
315 * (String), numbers of observations are the values (Int). Orthologs are to
316 * be inferred by method "inferOrthologs". Throws an exception if seq_name
320 * sequence name of a external node of the gene trees
321 * @return HashMap containing the inferred orthologs
322 * (name(String)->value(Int))
324 private final HashMap<String, Integer> getInferredOrthologs( final String seq_name ) {
325 if ( _o_maps == null ) {
328 return _o_maps.get( seq_name );
332 * Returns a HashMap containing the inferred "super orthologs" of the
333 * external gene tree node with the sequence name seq_name. Sequence names
334 * are the keys (String), numbers of observations are the values (Int).
335 * Super orthologs are to be inferred by method "inferOrthologs". Throws an
336 * exception if seq_name is not found.
339 * sequence name of a external node of the gene trees
340 * @return HashMap containing the inferred super orthologs
341 * (name(String)->value(Int))
343 private final HashMap<String, Integer> getInferredSuperOrthologs( final String seq_name ) {
344 if ( _so_maps == null ) {
347 return _so_maps.get( seq_name );
351 * Infers the orthologs (as well the "super orthologs", the "subtree
352 * neighbors", and the "ultra paralogs") for each external node of the gene
353 * Trees in multiple tree File gene_trees_file (=output of PHYLIP NEIGHBOR,
354 * for example). Tallies how many times each sequence is (super-)
355 * orthologous towards the query. Tallies how many times each sequence is
356 * ultra paralogous towards the query. Tallies how many times each sequence
357 * is a subtree neighbor of the query. Gene duplications are inferred using
358 * SDI. Modifies its argument species_tree. Is a little faster than
359 * "inferOrthologs(File,Phylogeny)" since orthologs are only inferred for
362 * To obtain the results use the methods listed below.
364 * @param gene_trees_file
365 * a File containing gene Trees in NH format, which is the result
366 * of performing a bootstrap analysis in PHYLIP
367 * @param species_tree
368 * a species Phylogeny, which has species names in its species
371 * the sequence name of the squence whose orthologs are to be
373 * @throws SDIException
375 private final void inferOrthologs( final File gene_trees_file, final Phylogeny species_tree, final String query )
376 throws IOException, SDIException {
377 // Read in first tree to get its sequence names
378 // and strip species_tree.
379 final PhylogenyFactory factory = ParserBasedPhylogenyFactory.getInstance();
380 final PhylogenyParser p = ParserUtils.createParserDependingOnFileType( gene_trees_file, true );
381 if ( p instanceof NHXParser ) {
382 final NHXParser nhx = ( NHXParser ) p;
383 nhx.setReplaceUnderscores( false );
384 nhx.setIgnoreQuotes( true );
385 nhx.setTaxonomyExtraction( PhylogenyMethods.TAXONOMY_EXTRACTION.YES );
387 final Phylogeny[] gene_trees = factory.create( gene_trees_file, p );
388 // Removes from species_tree all species not found in gene_tree.
389 PhylogenyMethods.taxonomyBasedDeletionOfExternalNodes( gene_trees[ 0 ], species_tree );
390 if ( !ForesterUtil.isEmpty( query ) ) {
391 PhylogenyMethods.taxonomyBasedDeletionOfExternalNodes( species_tree, gene_trees[ 0 ] );
392 _seq_names = getAllExternalSequenceNames( gene_trees[ 0 ] );
393 if ( ( _seq_names == null ) || ( _seq_names.size() < 1 ) ) {
394 throw new IOException( "could not get sequence names" );
396 _o_maps = new HashMap<String, HashMap<String, Integer>>();
397 _so_maps = new HashMap<String, HashMap<String, Integer>>();
398 _up_maps = new HashMap<String, HashMap<String, Integer>>();
399 _o_maps.put( query, new HashMap<String, Integer>( _seq_names.size() ) );
400 _so_maps.put( query, new HashMap<String, Integer>( _seq_names.size() ) );
401 _up_maps.put( query, new HashMap<String, Integer>( _seq_names.size() ) );
403 _analyzed_gene_trees = new Phylogeny[ gene_trees.length ];
405 for( final Phylogeny gt : gene_trees ) {
406 // Removes from gene_tree all species not found in species_tree.
407 PhylogenyMethods.taxonomyBasedDeletionOfExternalNodes( species_tree, gt );
408 _analyzed_gene_trees[ c++ ] = inferOrthologsHelper( gt, species_tree, query );
410 setNumberOfSamples( gene_trees.length );
413 // Helper method which performs the actual ortholog inference for
414 // the external node with seqname query.
415 private final Phylogeny inferOrthologsHelper( final Phylogeny gene_tree,
416 final Phylogeny species_tree,
417 final String query ) throws SDIException {
418 final SDIR sdiunrooted = new SDIR();
419 final Phylogeny assigned_tree = sdiunrooted.infer( gene_tree,
422 RIO.ROOT_BY_MINIMIZING_SUM_OF_DUPS,
423 RIO.ROOT_BY_MINIMIZING_TREE_HEIGHT,
426 setExtNodesOfAnalyzedGeneTrees( assigned_tree.getNumberOfExternalNodes() );
427 if ( !ForesterUtil.isEmpty( query ) ) {
428 final List<PhylogenyNode> nodes = getNodesViaSequenceName( assigned_tree, query );
429 if ( nodes.size() > 1 ) {
430 throw new IllegalArgumentException( "node named [" + query + "] not unique" );
432 else if ( nodes.isEmpty() ) {
433 throw new IllegalArgumentException( "no node containing a sequence named [" + query + "] found" );
435 final PhylogenyNode query_node = nodes.get( 0 );
436 updateCounts( _o_maps, query, PhylogenyMethods.getOrthologousNodes( assigned_tree, query_node ) );
437 updateCounts( _so_maps, query, PhylogenyMethods.getSuperOrthologousNodes( query_node ) );
438 updateCounts( _up_maps, query, PhylogenyMethods.getUltraParalogousNodes( query_node ) );
440 return assigned_tree;
443 private final void init() {
452 private final void setExtNodesOfAnalyzedGeneTrees( final int i ) {
456 private final void setNumberOfSamples( int i ) {
463 // Helper for doInferOrthologs( Phylogeny, Phylogeny, String )
464 // and doInferOrthologs( Phylogeny, Phylogeny ).
465 private final void updateCounts( final HashMap<String, HashMap<String, Integer>> counter_map,
466 final String query_seq_name,
467 final List<PhylogenyNode> nodes ) {
468 final HashMap<String, Integer> hash_map = counter_map.get( query_seq_name );
469 if ( hash_map == null ) {
470 throw new RuntimeException( "Unexpected failure in method updateHash." );
472 for( int j = 0; j < nodes.size(); ++j ) {
474 if ( ( nodes.get( j ) ).getNodeData().isHasSequence()
475 && !ForesterUtil.isEmpty( ( nodes.get( j ) ).getNodeData().getSequence().getName() ) ) {
476 seq_name = ( nodes.get( j ) ).getNodeData().getSequence().getName();
479 seq_name = ( nodes.get( j ) ).getName();
481 if ( hash_map.containsKey( seq_name ) ) {
482 hash_map.put( seq_name, hash_map.get( seq_name ) + 1 );
485 hash_map.put( seq_name, 1 );
490 public final static IntMatrix calculateOrthologTable( final Phylogeny[] analyzed_gene_trees ) {
491 final List<String> labels = new ArrayList<String>();
492 final Set<String> labels_set = new HashSet<String>();
494 for( final PhylogenyNode n : analyzed_gene_trees[ 0 ].getExternalNodes() ) {
495 if ( n.getNodeData().isHasSequence() && !ForesterUtil.isEmpty( n.getNodeData().getSequence().getName() ) ) {
496 label = n.getNodeData().getSequence().getName();
498 else if ( n.getNodeData().isHasSequence()
499 && !ForesterUtil.isEmpty( n.getNodeData().getSequence().getSymbol() ) ) {
500 label = n.getNodeData().getSequence().getSymbol();
502 else if ( !ForesterUtil.isEmpty( n.getName() ) ) {
506 throw new IllegalArgumentException( "node " + n + " has no appropriate label" );
508 if ( labels_set.contains( label ) ) {
509 throw new IllegalArgumentException( "label " + label + " is not unique" );
511 labels_set.add( label );
514 final IntMatrix m = new IntMatrix( labels );
516 for( final Phylogeny gt : analyzed_gene_trees ) {
518 PhylogenyMethods.preOrderReId( gt );
519 final HashMap<String, PhylogenyNode> map = PhylogenyMethods.createNameToExtNodeMap( gt );
520 for( int x = 0; x < m.size(); ++x ) {
521 final PhylogenyNode nx = map.get( m.getLabel( x ) );
522 for( int y = 0; y < m.size(); ++y ) {
523 if ( !PhylogenyMethods.calculateLCAonTreeWithIdsInPreOrder( nx, map.get( m.getLabel( y ) ) )
525 m.inreaseByOne( x, y );
534 * Returns the order in which ortholog (o), "super ortholog" (s) and
535 * distance (d) are returned and sorted (priority of sort always goes from
536 * left to right), given sort. For the meaning of sort
538 * @see #inferredOrthologsToString(String,int,double,double)
541 * determines order and sort priority
542 * @return String indicating the order
544 public final static String getOrder( final int sort ) {
548 order = "orthologies";
551 order = "orthologies > super orthologies";
554 order = "super orthologies > orthologies";
557 order = "orthologies";
563 public final static StringBuffer getOrderHelp() {
564 final StringBuffer sb = new StringBuffer();
565 sb.append( " 0: orthologies" + ForesterUtil.LINE_SEPARATOR );
566 sb.append( " 1: orthologies > super orthologies" + ForesterUtil.LINE_SEPARATOR );
567 sb.append( " 2: super orthologies > orthologies" + ForesterUtil.LINE_SEPARATOR );
571 // Helper method for inferredOrthologsToString
572 // and inferredUltraParalogsToString.
573 private final static String addNameAndValues( final String name,
577 final java.text.DecimalFormat df = new java.text.DecimalFormat( "0.#####" );
578 df.setDecimalSeparatorAlwaysShown( false );
580 if ( name.length() < 8 ) {
581 line += ( name + "\t\t\t" );
583 else if ( name.length() < 16 ) {
584 line += ( name + "\t\t" );
587 line += ( name + "\t" );
591 line += addToLine( value1, df );
595 line += addToLine( value1, df );
596 line += addToLine( value2, df );
599 line += addToLine( value2, df );
600 line += addToLine( value1, df );
603 line += addToLine( value1, df );
607 line += addToLine( value1, df );
608 line += addToLine( value2, df );
611 line += ForesterUtil.LINE_SEPARATOR;
615 // Helper for addNameAndValues.
616 private final static String addToLine( final double value, final java.text.DecimalFormat df ) {
618 if ( value != ResultLine.DEFAULT ) {
619 s = df.format( value ) + "\t";
627 private final static List<String> getAllExternalSequenceNames( final Phylogeny phy ) {
628 final List<String> names = new ArrayList<String>();
629 for( final PhylogenyNodeIterator iter = phy.iteratorExternalForward(); iter.hasNext(); ) {
630 final PhylogenyNode n = iter.next();
631 if ( n.getNodeData().isHasSequence() && !ForesterUtil.isEmpty( n.getNodeData().getSequence().getName() ) ) {
632 names.add( n.getNodeData().getSequence().getName() );
634 else if ( !ForesterUtil.isEmpty( n.getName() ) ) {
635 names.add( n.getName() );
638 throw new IllegalArgumentException( "node has no (sequence) name: " + n );
644 private final static List<PhylogenyNode> getNodesViaSequenceName( final Phylogeny phy, final String seq_name ) {
645 final List<PhylogenyNode> nodes = new ArrayList<PhylogenyNode>();
646 for( final PhylogenyNodeIterator iter = phy.iteratorPreorder(); iter.hasNext(); ) {
647 final PhylogenyNode n = iter.next();
648 if ( n.getNodeData().isHasSequence() && n.getNodeData().getSequence().getName().equals( seq_name ) ) {
651 if ( !n.getNodeData().isHasSequence() && n.getName().equals( seq_name ) ) {
658 private final class ResultLine implements Comparable<ResultLine> {
660 public static final int DEFAULT = -999;
661 private final String _key;
662 private final double _value1;
663 private final double _value2;
666 ResultLine( final String name, final double value1, final double value2, final int c ) {
671 if ( ( c >= 0 ) && ( c <= 2 ) ) {
676 ResultLine( final String name, final double value1, final int c ) {
680 _value2 = ResultLine.DEFAULT;
687 public int compareTo( final ResultLine n ) {
688 if ( ( getValue1() != ResultLine.DEFAULT ) && ( n.getValue1() != ResultLine.DEFAULT ) ) {
689 if ( getValue1() < n.getValue1() ) {
692 if ( getValue1() > n.getValue1() ) {
696 if ( ( getValue2() != ResultLine.DEFAULT ) && ( n.getValue2() != ResultLine.DEFAULT ) ) {
697 if ( getValue2() < n.getValue2() ) {
700 if ( getValue2() > n.getValue2() ) {
704 return ( getKey().compareTo( n.getKey() ) );
719 private void setSigns() {
721 _p[ 0 ] = _p[ 1 ] = +1;