X-Git-Url: http://source.jalview.org/gitweb/?a=blobdiff_plain;f=forester%2Fjava%2Fsrc%2Forg%2Fforester%2Fsurfacing%2FSurfacingUtil.java;h=a22bdad9d269d3b90e11befef059097b831d0ebd;hb=78b77ac2a4a069d0a2e53dc5e7652838c58acd0c;hp=eef4ff9102c5565695dd0cebf52914bf4683ef91;hpb=756e207d17b773c707ff2315091bcef338ddcf85;p=jalview.git diff --git a/forester/java/src/org/forester/surfacing/SurfacingUtil.java b/forester/java/src/org/forester/surfacing/SurfacingUtil.java index eef4ff9..a22bdad 100644 --- a/forester/java/src/org/forester/surfacing/SurfacingUtil.java +++ b/forester/java/src/org/forester/surfacing/SurfacingUtil.java @@ -41,6 +41,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.PriorityQueue; import java.util.Set; import java.util.SortedMap; @@ -67,9 +68,17 @@ import org.forester.io.writers.PhylogenyWriter; import org.forester.phylogeny.Phylogeny; import org.forester.phylogeny.PhylogenyMethods; import org.forester.phylogeny.PhylogenyNode; +import org.forester.phylogeny.PhylogenyNodeI.NH_CONVERSION_SUPPORT_VALUE_STYLE; import org.forester.phylogeny.data.BinaryCharacters; import org.forester.phylogeny.data.Confidence; import org.forester.phylogeny.iterators.PhylogenyNodeIterator; +import org.forester.protein.BasicDomain; +import org.forester.protein.BasicProtein; +import org.forester.protein.BinaryDomainCombination; +import org.forester.protein.Domain; +import org.forester.protein.DomainId; +import org.forester.protein.Protein; +import org.forester.species.Species; import org.forester.surfacing.DomainSimilarityCalculator.Detailedness; import org.forester.surfacing.DomainSimilarityCalculator.GoAnnotationOutput; import org.forester.surfacing.GenomeWideCombinableDomains.GenomeWideCombinableDomainsSortOrder; @@ -104,6 +113,7 @@ public final class SurfacingUtil { } }; public final static Pattern PATTERN_SP_STYLE_TAXONOMY = Pattern.compile( "^[A-Z0-9]{3,5}$" ); + private static final boolean USE_LAST = true; private SurfacingUtil() { // Hidden constructor. @@ -166,8 +176,38 @@ public final class SurfacingUtil { final String outfilename_for_counts, final String outfilename_for_dc, final String outfilename_for_dc_for_go_mapping, - final String outfilename_for_dc_for_go_mapping_unique ) { + final String outfilename_for_dc_for_go_mapping_unique, + final String outfilename_for_rank_counts, + final String outfilename_for_ancestor_species_counts, + final String outfilename_for_protein_stats, + final Map protein_length_stats_by_dc, + final Map domain_number_stats_by_dc, + final Map domain_length_stats_by_domain ) { try { + // + // if ( protein_length_stats_by_dc != null ) { + // for( final Entry entry : protein_length_stats_by_dc.entrySet() ) { + // System.out.print( entry.getKey().toString() ); + // System.out.print( ": " ); + // double[] a = entry.getValue().getDataAsDoubleArray(); + // for( int i = 0; i < a.length; i++ ) { + // System.out.print( a[ i ] + " " ); + // } + // System.out.println(); + // } + // } + // if ( domain_number_stats_by_dc != null ) { + // for( final Entry entry : domain_number_stats_by_dc.entrySet() ) { + // System.out.print( entry.getKey().toString() ); + // System.out.print( ": " ); + // double[] a = entry.getValue().getDataAsDoubleArray(); + // for( int i = 0; i < a.length; i++ ) { + // System.out.print( a[ i ] + " " ); + // } + // System.out.println(); + // } + // } + // final BufferedWriter out_counts = new BufferedWriter( new FileWriter( outfilename_for_counts ) ); final BufferedWriter out_dc = new BufferedWriter( new FileWriter( outfilename_for_dc ) ); final BufferedWriter out_dc_for_go_mapping = new BufferedWriter( new FileWriter( outfilename_for_dc_for_go_mapping ) ); @@ -187,9 +227,21 @@ public final class SurfacingUtil { } final SortedMap histogram = new TreeMap(); final SortedMap domain_lists = new TreeMap(); + final SortedMap dc_reapp_counts_to_protein_length_stats = new TreeMap(); + final SortedMap dc_reapp_counts_to_domain_number_stats = new TreeMap(); + final SortedMap dc_reapp_counts_to_domain_lengths_stats = new TreeMap(); final SortedMap> domain_lists_go = new TreeMap>(); final SortedMap> domain_lists_go_unique = new TreeMap>(); final Set dcs = dc_gain_counts.keySet(); + final SortedSet more_than_once = new TreeSet(); + final DescriptiveStatistics gained_once_lengths_stats = new BasicDescriptiveStatistics(); + final DescriptiveStatistics gained_once_domain_count_stats = new BasicDescriptiveStatistics(); + final DescriptiveStatistics gained_multiple_times_lengths_stats = new BasicDescriptiveStatistics(); + final DescriptiveStatistics gained_multiple_times_domain_count_stats = new BasicDescriptiveStatistics(); + long gained_multiple_times_domain_length_sum = 0; + long gained_once_domain_length_sum = 0; + long gained_multiple_times_domain_length_count = 0; + long gained_once_domain_length_count = 0; for( final String dc : dcs ) { final int count = dc_gain_counts.get( dc ); if ( histogram.containsKey( count ) ) { @@ -208,6 +260,85 @@ public final class SurfacingUtil { set.addAll( splitDomainCombination( dc ) ); domain_lists_go_unique.put( count, set ); } + if ( protein_length_stats_by_dc != null ) { + if ( !dc_reapp_counts_to_protein_length_stats.containsKey( count ) ) { + dc_reapp_counts_to_protein_length_stats.put( count, new BasicDescriptiveStatistics() ); + } + dc_reapp_counts_to_protein_length_stats.get( count ).addValue( protein_length_stats_by_dc.get( dc ) + .arithmeticMean() ); + } + if ( domain_number_stats_by_dc != null ) { + if ( !dc_reapp_counts_to_domain_number_stats.containsKey( count ) ) { + dc_reapp_counts_to_domain_number_stats.put( count, new BasicDescriptiveStatistics() ); + } + dc_reapp_counts_to_domain_number_stats.get( count ).addValue( domain_number_stats_by_dc.get( dc ) + .arithmeticMean() ); + } + if ( domain_length_stats_by_domain != null ) { + if ( !dc_reapp_counts_to_domain_lengths_stats.containsKey( count ) ) { + dc_reapp_counts_to_domain_lengths_stats.put( count, new BasicDescriptiveStatistics() ); + } + final String[] ds = dc.split( "=" ); + dc_reapp_counts_to_domain_lengths_stats.get( count ).addValue( domain_length_stats_by_domain + .get( ds[ 0 ] ).arithmeticMean() ); + dc_reapp_counts_to_domain_lengths_stats.get( count ).addValue( domain_length_stats_by_domain + .get( ds[ 1 ] ).arithmeticMean() ); + } + if ( count > 1 ) { + more_than_once.add( dc ); + if ( protein_length_stats_by_dc != null ) { + final DescriptiveStatistics s = protein_length_stats_by_dc.get( dc ); + for( final double element : s.getData() ) { + gained_multiple_times_lengths_stats.addValue( element ); + } + } + if ( domain_number_stats_by_dc != null ) { + final DescriptiveStatistics s = domain_number_stats_by_dc.get( dc ); + for( final double element : s.getData() ) { + gained_multiple_times_domain_count_stats.addValue( element ); + } + } + if ( domain_length_stats_by_domain != null ) { + final String[] ds = dc.split( "=" ); + final DescriptiveStatistics s0 = domain_length_stats_by_domain.get( ds[ 0 ] ); + final DescriptiveStatistics s1 = domain_length_stats_by_domain.get( ds[ 1 ] ); + for( final double element : s0.getData() ) { + gained_multiple_times_domain_length_sum += element; + ++gained_multiple_times_domain_length_count; + } + for( final double element : s1.getData() ) { + gained_multiple_times_domain_length_sum += element; + ++gained_multiple_times_domain_length_count; + } + } + } + else { + if ( protein_length_stats_by_dc != null ) { + final DescriptiveStatistics s = protein_length_stats_by_dc.get( dc ); + for( final double element : s.getData() ) { + gained_once_lengths_stats.addValue( element ); + } + } + if ( domain_number_stats_by_dc != null ) { + final DescriptiveStatistics s = domain_number_stats_by_dc.get( dc ); + for( final double element : s.getData() ) { + gained_once_domain_count_stats.addValue( element ); + } + } + if ( domain_length_stats_by_domain != null ) { + final String[] ds = dc.split( "=" ); + final DescriptiveStatistics s0 = domain_length_stats_by_domain.get( ds[ 0 ] ); + final DescriptiveStatistics s1 = domain_length_stats_by_domain.get( ds[ 1 ] ); + for( final double element : s0.getData() ) { + gained_once_domain_length_sum += element; + ++gained_once_domain_length_count; + } + for( final double element : s1.getData() ) { + gained_once_domain_length_sum += element; + ++gained_once_domain_length_count; + } + } + } } final Set histogram_keys = histogram.keySet(); for( final Integer histogram_key : histogram_keys ) { @@ -230,6 +361,134 @@ public final class SurfacingUtil { out_dc.close(); out_dc_for_go_mapping.close(); out_dc_for_go_mapping_unique.close(); + final SortedMap lca_rank_counts = new TreeMap(); + final SortedMap lca_ancestor_species_counts = new TreeMap(); + for( final String dc : more_than_once ) { + final List nodes = new ArrayList(); + for( final PhylogenyNodeIterator it = local_phylogeny_l.iteratorExternalForward(); it.hasNext(); ) { + final PhylogenyNode n = it.next(); + if ( n.getNodeData().getBinaryCharacters().getGainedCharacters().contains( dc ) ) { + nodes.add( n ); + } + } + for( int i = 0; i < nodes.size() - 1; ++i ) { + for( int j = i + 1; j < nodes.size(); ++j ) { + final PhylogenyNode lca = PhylogenyMethods.getInstance().obtainLCA( nodes.get( i ), + nodes.get( j ) ); + String rank = "unknown"; + if ( lca.getNodeData().isHasTaxonomy() + && !ForesterUtil.isEmpty( lca.getNodeData().getTaxonomy().getRank() ) ) { + rank = lca.getNodeData().getTaxonomy().getRank(); + } + addToCountMap( lca_rank_counts, rank ); + String lca_species; + if ( lca.getNodeData().isHasTaxonomy() + && !ForesterUtil.isEmpty( lca.getNodeData().getTaxonomy().getScientificName() ) ) { + lca_species = lca.getNodeData().getTaxonomy().getScientificName(); + } + else if ( lca.getNodeData().isHasTaxonomy() + && !ForesterUtil.isEmpty( lca.getNodeData().getTaxonomy().getCommonName() ) ) { + lca_species = lca.getNodeData().getTaxonomy().getCommonName(); + } + else { + lca_species = lca.getName(); + } + addToCountMap( lca_ancestor_species_counts, lca_species ); + } + } + } + final BufferedWriter out_for_rank_counts = new BufferedWriter( new FileWriter( outfilename_for_rank_counts ) ); + final BufferedWriter out_for_ancestor_species_counts = new BufferedWriter( new FileWriter( outfilename_for_ancestor_species_counts ) ); + ForesterUtil.map2writer( out_for_rank_counts, lca_rank_counts, "\t", ForesterUtil.LINE_SEPARATOR ); + ForesterUtil.map2writer( out_for_ancestor_species_counts, + lca_ancestor_species_counts, + "\t", + ForesterUtil.LINE_SEPARATOR ); + out_for_rank_counts.close(); + out_for_ancestor_species_counts.close(); + if ( !ForesterUtil.isEmpty( outfilename_for_protein_stats ) + && ( ( domain_length_stats_by_domain != null ) || ( protein_length_stats_by_dc != null ) || ( domain_number_stats_by_dc != null ) ) ) { + final BufferedWriter w = new BufferedWriter( new FileWriter( outfilename_for_protein_stats ) ); + w.write( "Domain Lengths: " ); + w.write( "\n" ); + if ( domain_length_stats_by_domain != null ) { + for( final Entry entry : dc_reapp_counts_to_domain_lengths_stats + .entrySet() ) { + w.write( entry.getKey().toString() ); + w.write( "\t" + entry.getValue().arithmeticMean() ); + w.write( "\t" + entry.getValue().median() ); + w.write( "\n" ); + } + } + w.flush(); + w.write( "\n" ); + w.write( "\n" ); + w.write( "Protein Lengths: " ); + w.write( "\n" ); + if ( protein_length_stats_by_dc != null ) { + for( final Entry entry : dc_reapp_counts_to_protein_length_stats + .entrySet() ) { + w.write( entry.getKey().toString() ); + w.write( "\t" + entry.getValue().arithmeticMean() ); + w.write( "\t" + entry.getValue().median() ); + w.write( "\n" ); + } + } + w.flush(); + w.write( "\n" ); + w.write( "\n" ); + w.write( "Number of domains: " ); + w.write( "\n" ); + if ( domain_number_stats_by_dc != null ) { + for( final Entry entry : dc_reapp_counts_to_domain_number_stats + .entrySet() ) { + w.write( entry.getKey().toString() ); + w.write( "\t" + entry.getValue().arithmeticMean() ); + w.write( "\t" + entry.getValue().median() ); + w.write( "\n" ); + } + } + w.flush(); + w.write( "\n" ); + w.write( "\n" ); + w.write( "Gained once, domain lengths:" ); + w.write( "\n" ); + w.write( "N: " + gained_once_domain_length_count ); + w.write( "\n" ); + w.write( "Avg: " + ( ( double ) gained_once_domain_length_sum / gained_once_domain_length_count ) ); + w.write( "\n" ); + w.write( "\n" ); + w.write( "Gained multiple times, domain lengths:" ); + w.write( "\n" ); + w.write( "N: " + gained_multiple_times_domain_length_count ); + w.write( "\n" ); + w.write( "Avg: " + + ( ( double ) gained_multiple_times_domain_length_sum / gained_multiple_times_domain_length_count ) ); + w.write( "\n" ); + w.write( "\n" ); + w.write( "\n" ); + w.write( "\n" ); + w.write( "Gained once, protein lengths:" ); + w.write( "\n" ); + w.write( gained_once_lengths_stats.toString() ); + w.write( "\n" ); + w.write( "\n" ); + w.write( "Gained once, domain counts:" ); + w.write( "\n" ); + w.write( gained_once_domain_count_stats.toString() ); + w.write( "\n" ); + w.write( "\n" ); + w.write( "Gained multiple times, protein lengths:" ); + w.write( "\n" ); + w.write( gained_multiple_times_lengths_stats.toString() ); + w.write( "\n" ); + w.write( "\n" ); + w.write( "Gained multiple times, domain counts:" ); + w.write( "\n" ); + w.write( gained_multiple_times_domain_count_stats.toString() ); + w.flush(); + w.close(); + } } catch ( final IOException e ) { ForesterUtil.printWarningMessage( surfacing.PRG_NAME, "Failure to write: " + e ); @@ -246,6 +505,15 @@ public final class SurfacingUtil { + outfilename_for_dc_for_go_mapping_unique + "]" ); } + private final static void addToCountMap( final Map map, final String s ) { + if ( map.containsKey( s ) ) { + map.put( s, map.get( s ) + 1 ); + } + else { + map.put( s, 1 ); + } + } + public static int calculateOverlap( final Domain domain, final List covered_positions ) { int overlap_count = 0; for( int i = domain.getFrom(); i <= domain.getTo(); ++i ) { @@ -503,7 +771,10 @@ public final class SurfacingUtil { final boolean output_binary_domain_combinations_for_graphs, final List all_binary_domains_combination_gained_fitch, final List all_binary_domains_combination_lost_fitch, - final BinaryDomainCombination.DomainCombinationType dc_type ) { + final BinaryDomainCombination.DomainCombinationType dc_type, + final Map protein_length_stats_by_dc, + final Map domain_number_stats_by_dc, + final Map domain_length_stats_by_domain ) { final String sep = ForesterUtil.LINE_SEPARATOR + "###################" + ForesterUtil.LINE_SEPARATOR; final String date_time = ForesterUtil.getCurrentDateTime(); final SortedSet all_pfams_encountered = new TreeSet(); @@ -611,7 +882,7 @@ public final class SurfacingUtil { randomization = "yes, seed = " + random_number_seed_for_fitch_parsimony; } else { - domain_parsimony.executeFitchParsimonyOnBinaryDomainCombintion( false ); + domain_parsimony.executeFitchParsimonyOnBinaryDomainCombintion( USE_LAST ); } SurfacingUtil.writeMatrixToFile( domain_parsimony.getGainLossMatrix(), outfile_name + surfacing.PARSIMONY_OUTPUT_GL_SUFFIX_FITCH_BINARY_COMBINATIONS, Format.FORESTER ); @@ -720,11 +991,21 @@ public final class SurfacingUtil { parameters_str ); SurfacingUtil.writePhylogenyToFile( local_phylogeny_l, outfile_name + surfacing.BINARY_DOMAIN_COMBINATIONS_PARSIMONY_TREE_OUTPUT_SUFFIX_FITCH ); - calculateIndependentDomainCombinationGains( local_phylogeny_l, outfile_name - + surfacing.INDEPENDENT_DC_GAINS_FITCH_PARS_COUNTS_OUTPUT_SUFFIX, outfile_name - + surfacing.INDEPENDENT_DC_GAINS_FITCH_PARS_DC_OUTPUT_SUFFIX, outfile_name - + surfacing.INDEPENDENT_DC_GAINS_FITCH_PARS_DC_FOR_GO_MAPPING_OUTPUT_SUFFIX, outfile_name - + surfacing.INDEPENDENT_DC_GAINS_FITCH_PARS_DC_FOR_GO_MAPPING_OUTPUT_UNIQUE_SUFFIX ); + calculateIndependentDomainCombinationGains( local_phylogeny_l, + outfile_name + + surfacing.INDEPENDENT_DC_GAINS_FITCH_PARS_COUNTS_OUTPUT_SUFFIX, + outfile_name + + surfacing.INDEPENDENT_DC_GAINS_FITCH_PARS_DC_OUTPUT_SUFFIX, + outfile_name + + surfacing.INDEPENDENT_DC_GAINS_FITCH_PARS_DC_FOR_GO_MAPPING_OUTPUT_SUFFIX, + outfile_name + + surfacing.INDEPENDENT_DC_GAINS_FITCH_PARS_DC_FOR_GO_MAPPING_OUTPUT_UNIQUE_SUFFIX, + outfile_name + "_indep_dc_gains_fitch_lca_ranks.txt", + outfile_name + "_indep_dc_gains_fitch_lca_taxonomies.txt", + outfile_name + "_indep_dc_gains_fitch_protein_statistics.txt", + protein_length_stats_by_dc, + domain_number_stats_by_dc, + domain_length_stats_by_domain ); } } @@ -739,7 +1020,7 @@ public final class SurfacingUtil { writeToNexus( outfile_name + surfacing.NEXUS_SECONDARY_FEATURES, secondary_features_parsimony.createMatrixOfSecondaryFeaturePresenceOrAbsence( null ), phylogeny ); - final Phylogeny local_phylogeny_copy = phylogeny.copy(); + Phylogeny local_phylogeny_copy = phylogeny.copy(); secondary_features_parsimony.executeDolloParsimonyOnSecondaryFeatures( mapping_results_map ); SurfacingUtil.writeMatrixToFile( secondary_features_parsimony.getGainLossMatrix(), outfile_name + surfacing.PARSIMONY_OUTPUT_GL_SUFFIX_DOLLO_SECONDARY_FEATURES, Format.FORESTER ); @@ -777,70 +1058,111 @@ public final class SurfacingUtil { parameters_str ); SurfacingUtil.writePhylogenyToFile( local_phylogeny_copy, outfile_name + surfacing.SECONDARY_FEATURES_PARSIMONY_TREE_OUTPUT_SUFFIX_DOLLO ); + // FITCH DOMAIN COMBINATIONS + // ------------------------- + local_phylogeny_copy = phylogeny.copy(); + final String randomization = "no"; + secondary_features_parsimony.executeFitchParsimonyOnBinaryDomainCombintionOnSecondaryFeatures( USE_LAST ); + preparePhylogeny( local_phylogeny_copy, + secondary_features_parsimony, + date_time, + "Fitch parsimony on secondary binary domain combination presence/absence randomization: " + + randomization, + "fitch_on_binary_domain_combinations_" + outfile_name, + parameters_str ); + SurfacingUtil.writePhylogenyToFile( local_phylogeny_copy, outfile_name + + surfacing.BINARY_DOMAIN_COMBINATIONS_PARSIMONY_TREE_OUTPUT_SUFFIX_FITCH_MAPPED ); + calculateIndependentDomainCombinationGains( local_phylogeny_copy, outfile_name + + surfacing.INDEPENDENT_DC_GAINS_FITCH_PARS_COUNTS_MAPPED_OUTPUT_SUFFIX, outfile_name + + surfacing.INDEPENDENT_DC_GAINS_FITCH_PARS_DC_MAPPED_OUTPUT_SUFFIX, outfile_name + + surfacing.INDEPENDENT_DC_GAINS_FITCH_PARS_DC_FOR_GO_MAPPING_MAPPED_OUTPUT_SUFFIX, outfile_name + + surfacing.INDEPENDENT_DC_GAINS_FITCH_PARS_DC_FOR_GO_MAPPING_MAPPED_OUTPUT_UNIQUE_SUFFIX, outfile_name + + "_MAPPED_indep_dc_gains_fitch_lca_ranks.txt", outfile_name + + "_MAPPED_indep_dc_gains_fitch_lca_taxonomies.txt", null, null, null, null ); } - public static void extractProteinNames( final List proteins, - final List query_domain_ids_nc_order, - final Writer out, - final String separator ) throws IOException { + public static void doit( final List proteins, + final List query_domain_ids_nc_order, + final Writer out, + final String separator, + final String limit_to_species, + final Map> average_protein_lengths_by_dc ) throws IOException { for( final Protein protein : proteins ) { - if ( protein.contains( query_domain_ids_nc_order, true ) ) { - out.write( protein.getSpecies().getSpeciesId() ); - out.write( separator ); - out.write( protein.getProteinId().getId() ); - out.write( separator ); - out.write( "[" ); - final Set visited_domain_ids = new HashSet(); - boolean first = true; - for( final Domain domain : protein.getProteinDomains() ) { - if ( !visited_domain_ids.contains( domain.getDomainId() ) ) { - visited_domain_ids.add( domain.getDomainId() ); - if ( first ) { - first = false; - } - else { - out.write( " " ); + if ( ForesterUtil.isEmpty( limit_to_species ) + || protein.getSpecies().getSpeciesId().equalsIgnoreCase( limit_to_species ) ) { + if ( protein.contains( query_domain_ids_nc_order, true ) ) { + out.write( protein.getSpecies().getSpeciesId() ); + out.write( separator ); + out.write( protein.getProteinId().getId() ); + out.write( separator ); + out.write( "[" ); + final Set visited_domain_ids = new HashSet(); + boolean first = true; + for( final Domain domain : protein.getProteinDomains() ) { + if ( !visited_domain_ids.contains( domain.getDomainId() ) ) { + visited_domain_ids.add( domain.getDomainId() ); + if ( first ) { + first = false; + } + else { + out.write( " " ); + } + out.write( domain.getDomainId().getId() ); + out.write( " {" ); + out.write( "" + domain.getTotalCount() ); + out.write( "}" ); } - out.write( domain.getDomainId().getId() ); - out.write( " {" ); - out.write( "" + domain.getTotalCount() ); - out.write( "}" ); } + out.write( "]" ); + out.write( separator ); + if ( !( ForesterUtil.isEmpty( protein.getDescription() ) || protein.getDescription() + .equals( SurfacingConstants.NONE ) ) ) { + out.write( protein.getDescription() ); + } + out.write( separator ); + if ( !( ForesterUtil.isEmpty( protein.getAccession() ) || protein.getAccession() + .equals( SurfacingConstants.NONE ) ) ) { + out.write( protein.getAccession() ); + } + out.write( SurfacingConstants.NL ); } - out.write( "]" ); - out.write( separator ); - if ( !( ForesterUtil.isEmpty( protein.getDescription() ) || protein.getDescription() - .equals( SurfacingConstants.NONE ) ) ) { - out.write( protein.getDescription() ); - } - out.write( separator ); - if ( !( ForesterUtil.isEmpty( protein.getAccession() ) || protein.getAccession() - .equals( SurfacingConstants.NONE ) ) ) { - out.write( protein.getAccession() ); - } - out.write( SurfacingConstants.NL ); } } out.flush(); } - public static void extractProteinNames( final SortedMap> protein_lists_per_species, - final DomainId domain_id, + public static void extractProteinNames( final List proteins, + final List query_domain_ids_nc_order, final Writer out, - final String separator ) throws IOException { - for( final Species species : protein_lists_per_species.keySet() ) { - for( final Protein protein : protein_lists_per_species.get( species ) ) { - final List domains = protein.getProteinDomains( domain_id ); - if ( domains.size() > 0 ) { - final DescriptiveStatistics stats = new BasicDescriptiveStatistics(); - for( final Domain domain : domains ) { - stats.addValue( domain.getPerSequenceEvalue() ); - } + final String separator, + final String limit_to_species ) throws IOException { + for( final Protein protein : proteins ) { + if ( ForesterUtil.isEmpty( limit_to_species ) + || protein.getSpecies().getSpeciesId().equalsIgnoreCase( limit_to_species ) ) { + if ( protein.contains( query_domain_ids_nc_order, true ) ) { out.write( protein.getSpecies().getSpeciesId() ); out.write( separator ); out.write( protein.getProteinId().getId() ); out.write( separator ); - out.write( "[" + FORMATTER.format( stats.median() ) + "]" ); + out.write( "[" ); + final Set visited_domain_ids = new HashSet(); + boolean first = true; + for( final Domain domain : protein.getProteinDomains() ) { + if ( !visited_domain_ids.contains( domain.getDomainId() ) ) { + visited_domain_ids.add( domain.getDomainId() ); + if ( first ) { + first = false; + } + else { + out.write( " " ); + } + out.write( domain.getDomainId().getId() ); + out.write( " {" ); + out.write( "" + domain.getTotalCount() ); + out.write( "}" ); + } + } + out.write( "]" ); out.write( separator ); if ( !( ForesterUtil.isEmpty( protein.getDescription() ) || protein.getDescription() .equals( SurfacingConstants.NONE ) ) ) { @@ -858,6 +1180,44 @@ public final class SurfacingUtil { out.flush(); } + public static void extractProteinNames( final SortedMap> protein_lists_per_species, + final DomainId domain_id, + final Writer out, + final String separator, + final String limit_to_species ) throws IOException { + for( final Species species : protein_lists_per_species.keySet() ) { + for( final Protein protein : protein_lists_per_species.get( species ) ) { + if ( ForesterUtil.isEmpty( limit_to_species ) + || protein.getSpecies().getSpeciesId().equalsIgnoreCase( limit_to_species ) ) { + final List domains = protein.getProteinDomains( domain_id ); + if ( domains.size() > 0 ) { + final DescriptiveStatistics stats = new BasicDescriptiveStatistics(); + for( final Domain domain : domains ) { + stats.addValue( domain.getPerSequenceEvalue() ); + } + out.write( protein.getSpecies().getSpeciesId() ); + out.write( separator ); + out.write( protein.getProteinId().getId() ); + out.write( separator ); + out.write( "[" + FORMATTER.format( stats.median() ) + "]" ); + out.write( separator ); + if ( !( ForesterUtil.isEmpty( protein.getDescription() ) || protein.getDescription() + .equals( SurfacingConstants.NONE ) ) ) { + out.write( protein.getDescription() ); + } + out.write( separator ); + if ( !( ForesterUtil.isEmpty( protein.getAccession() ) || protein.getAccession() + .equals( SurfacingConstants.NONE ) ) ) { + out.write( protein.getAccession() ); + } + out.write( SurfacingConstants.NL ); + } + } + } + } + out.flush(); + } + public static SortedSet getAllDomainIds( final List gwcd_list ) { final SortedSet all_domains_ids = new TreeSet(); for( final GenomeWideCombinableDomains gwcd : gwcd_list ) { @@ -893,11 +1253,17 @@ public final class SurfacingUtil { final PhylogenyNode n = it.next(); if ( ForesterUtil.isEmpty( n.getName() ) && ( !n.getNodeData().isHasTaxonomy() || ForesterUtil.isEmpty( n.getNodeData().getTaxonomy() - .getScientificName() ) ) ) { + .getScientificName() ) ) + && ( !n.getNodeData().isHasTaxonomy() || ForesterUtil.isEmpty( n.getNodeData().getTaxonomy() + .getCommonName() ) ) ) { if ( n.getParent() != null ) { names.append( " " ); names.append( n.getParent().getName() ); } + final List l = n.getAllExternalDescendants(); + for( final Object object : l ) { + System.out.println( l.toString() ); + } ++c; } } @@ -1055,7 +1421,7 @@ public final class SurfacingUtil { final boolean remove_engulfed_domains, final Protein protein ) { final Protein pruned_protein = new BasicProtein( protein.getProteinId().getId(), protein.getSpecies() - .getSpeciesId() ); + .getSpeciesId(), protein.getLength() ); final List sorted = SurfacingUtil.sortDomainsWithAscendingConfidenceValues( protein ); final List covered_positions = new ArrayList(); for( final Domain domain : sorted ) { @@ -1574,181 +1940,6 @@ public final class SurfacingUtil { ForesterUtil.programMessage( surfacing.PRG_NAME, "Wrote characters detailed HTML list: \"" + filename + "\"" ); } - public static void writeBinaryStatesMatrixToListORIGIG( final Map> domain_id_to_go_ids_map, - final Map go_id_to_term_map, - final GoNameSpace go_namespace_limit, - final boolean domain_combinations, - final CharacterStateMatrix matrix, - final CharacterStateMatrix.GainLossStates state, - final String filename, - final String indentifier_characters_separator, - final String character_separator, - final String title_for_html, - final String prefix_for_html, - final Map>[] domain_id_to_secondary_features_maps, - final SortedSet all_pfams_encountered, - final SortedSet pfams_gained_or_lost, - final String suffix_for_per_node_events_file ) { - if ( ( go_namespace_limit != null ) && ( ( go_id_to_term_map == null ) || ( go_id_to_term_map.size() < 1 ) ) ) { - throw new IllegalArgumentException( "attempt to use GO namespace limit without a GO-id to term map" ); - } - else if ( ( ( domain_id_to_go_ids_map == null ) || ( domain_id_to_go_ids_map.size() < 1 ) ) ) { - throw new IllegalArgumentException( "attempt to output detailed HTML without a Pfam to GO map" ); - } - else if ( ( ( go_id_to_term_map == null ) || ( go_id_to_term_map.size() < 1 ) ) ) { - throw new IllegalArgumentException( "attempt to output detailed HTML without a GO-id to term map" ); - } - final File outfile = new File( filename ); - checkForOutputFileWriteability( outfile ); - final SortedSet sorted_ids = new TreeSet(); - for( int i = 0; i < matrix.getNumberOfIdentifiers(); ++i ) { - sorted_ids.add( matrix.getIdentifier( i ) ); - } - try { - final Writer out = new BufferedWriter( new FileWriter( outfile ) ); - final File per_node_go_mapped_domain_gain_loss_files_base_dir = createBaseDirForPerNodeDomainFiles( surfacing.BASE_DIRECTORY_PER_NODE_DOMAIN_GAIN_LOSS_FILES, - domain_combinations, - state, - filename ); - Writer per_node_go_mapped_domain_gain_loss_outfile_writer = null; - File per_node_go_mapped_domain_gain_loss_outfile = null; - int per_node_counter = 0; - out.write( "" ); - out.write( SurfacingConstants.NL ); - addHtmlHead( out, title_for_html ); - out.write( SurfacingConstants.NL ); - out.write( "" ); - out.write( SurfacingConstants.NL ); - out.write( "

" ); - out.write( SurfacingConstants.NL ); - out.write( title_for_html ); - out.write( SurfacingConstants.NL ); - out.write( "

" ); - out.write( SurfacingConstants.NL ); - out.write( "" ); - out.write( SurfacingConstants.NL ); - for( final String id : sorted_ids ) { - out.write( "" ); - out.write( "" ); - out.write( "" ); - out.write( SurfacingConstants.NL ); - } - out.write( "
" ); - out.write( "" + id + "" ); - writeTaxonomyLinks( out, id ); - out.write( "
" ); - out.write( SurfacingConstants.NL ); - for( final String id : sorted_ids ) { - out.write( SurfacingConstants.NL ); - out.write( "

" ); - out.write( "" + id + "" ); - writeTaxonomyLinks( out, id ); - out.write( "

" ); - out.write( SurfacingConstants.NL ); - out.write( "" ); - out.write( SurfacingConstants.NL ); - out.write( "" ); - out.write( "" ); - out.write( "" ); - out.write( SurfacingConstants.NL ); - out.write( "" ); - out.write( SurfacingConstants.NL ); - per_node_counter = 0; - if ( matrix.getNumberOfCharacters() > 0 ) { - per_node_go_mapped_domain_gain_loss_outfile = new File( per_node_go_mapped_domain_gain_loss_files_base_dir - + ForesterUtil.FILE_SEPARATOR + id + suffix_for_per_node_events_file ); - SurfacingUtil.checkForOutputFileWriteability( per_node_go_mapped_domain_gain_loss_outfile ); - per_node_go_mapped_domain_gain_loss_outfile_writer = ForesterUtil - .createBufferedWriter( per_node_go_mapped_domain_gain_loss_outfile ); - } - else { - per_node_go_mapped_domain_gain_loss_outfile = null; - per_node_go_mapped_domain_gain_loss_outfile_writer = null; - } - for( int c = 0; c < matrix.getNumberOfCharacters(); ++c ) { - // Not nice: - // using null to indicate either UNCHANGED_PRESENT or GAIN. - if ( ( matrix.getState( id, c ) == state ) - || ( ( state == null ) && ( ( matrix.getState( id, c ) == CharacterStateMatrix.GainLossStates.UNCHANGED_PRESENT ) || ( matrix - .getState( id, c ) == CharacterStateMatrix.GainLossStates.GAIN ) ) ) ) { - final String character = matrix.getCharacter( c ); - String domain_0 = ""; - String domain_1 = ""; - if ( character.indexOf( BinaryDomainCombination.SEPARATOR ) > 0 ) { - final String[] s = character.split( BinaryDomainCombination.SEPARATOR ); - if ( s.length != 2 ) { - throw new AssertionError( "this should not have happened: unexpected format for domain combination: [" - + character + "]" ); - } - domain_0 = s[ 0 ]; - domain_1 = s[ 1 ]; - } - else { - domain_0 = character; - } - writeDomainData( domain_id_to_go_ids_map, - go_id_to_term_map, - go_namespace_limit, - out, - domain_0, - domain_1, - prefix_for_html, - character_separator, - domain_id_to_secondary_features_maps, - null ); - all_pfams_encountered.add( domain_0 ); - if ( pfams_gained_or_lost != null ) { - pfams_gained_or_lost.add( domain_0 ); - } - if ( !ForesterUtil.isEmpty( domain_1 ) ) { - all_pfams_encountered.add( domain_1 ); - if ( pfams_gained_or_lost != null ) { - pfams_gained_or_lost.add( domain_1 ); - } - } - if ( per_node_go_mapped_domain_gain_loss_outfile_writer != null ) { - writeDomainsToIndividualFilePerTreeNode( per_node_go_mapped_domain_gain_loss_outfile_writer, - domain_0, - domain_1 ); - per_node_counter++; - } - } - } - if ( per_node_go_mapped_domain_gain_loss_outfile_writer != null ) { - per_node_go_mapped_domain_gain_loss_outfile_writer.close(); - if ( per_node_counter < 1 ) { - per_node_go_mapped_domain_gain_loss_outfile.delete(); - } - per_node_counter = 0; - } - out.write( "
" ); - out.write( "Pfam domain(s)" ); - out.write( "" ); - out.write( "GO term acc" ); - out.write( "" ); - out.write( "GO term" ); - out.write( "" ); - out.write( "Penultimate GO term" ); - out.write( "" ); - out.write( "GO namespace" ); - out.write( "
" ); - out.write( SurfacingConstants.NL ); - out.write( "
" ); - out.write( SurfacingConstants.NL ); - } // for( final String id : sorted_ids ) { - out.write( "" ); - out.write( SurfacingConstants.NL ); - out.write( "" ); - out.write( SurfacingConstants.NL ); - out.flush(); - out.close(); - } - catch ( final IOException e ) { - ForesterUtil.fatalError( surfacing.PRG_NAME, e.getMessage() ); - } - ForesterUtil.programMessage( surfacing.PRG_NAME, "Wrote characters detailed HTML list: \"" + filename + "\"" ); - } - public static void writeDomainCombinationsCountsFile( final String[][] input_file_properties, final File output_dir, final Writer per_genome_domain_promiscuity_statistics_writer, @@ -1927,7 +2118,8 @@ public final class SurfacingUtil { public static DescriptiveStatistics writeDomainSimilaritiesToFile( final StringBuilder html_desc, final StringBuilder html_title, - final Writer w, + final Writer single_writer, + Map split_writers, final SortedSet similarities, final boolean treat_as_binary, final List species_order, @@ -2070,90 +2262,119 @@ public final class SurfacingUtil { System.out.println( "Pearsonian skewness : n/a" ); } } + if ( ( single_writer != null ) && ( ( split_writers == null ) || split_writers.isEmpty() ) ) { + split_writers = new HashMap(); + split_writers.put( '_', single_writer ); + } switch ( print_option ) { case SIMPLE_TAB_DELIMITED: break; case HTML: - w.write( "" ); - w.write( SurfacingConstants.NL ); - addHtmlHead( w, "SURFACING :: " + html_title ); - w.write( SurfacingConstants.NL ); - w.write( "" ); - w.write( SurfacingConstants.NL ); - w.write( html_desc.toString() ); - w.write( SurfacingConstants.NL ); - w.write( "
" ); - w.write( "
" ); - w.write( SurfacingConstants.NL ); - w.write( "
" );
-                w.write( SurfacingConstants.NL );
-                if ( histo != null ) {
-                    w.write( histo.toStringBuffer( 20, '|', 40, 5 ).toString() );
+                for( final Character key : split_writers.keySet() ) {
+                    final Writer w = split_writers.get( key );
+                    w.write( "" );
+                    w.write( SurfacingConstants.NL );
+                    if ( key != '_' ) {
+                        addHtmlHead( w, "DCs (" + html_title + ") " + key.toString().toUpperCase() );
+                    }
+                    else {
+                        addHtmlHead( w, "DCs (" + html_title + ")" );
+                    }
+                    w.write( SurfacingConstants.NL );
+                    w.write( "" );
+                    w.write( SurfacingConstants.NL );
+                    w.write( html_desc.toString() );
+                    w.write( SurfacingConstants.NL );
+                    w.write( "
" );
+                    w.write( "
" );
+                    w.write( SurfacingConstants.NL );
+                    w.write( "
" );
+                    w.write( SurfacingConstants.NL );
+                    if ( histo != null ) {
+                        w.write( histo.toStringBuffer( 20, '|', 40, 5 ).toString() );
+                        w.write( SurfacingConstants.NL );
+                    }
+                    w.write( "
" );
+                    w.write( SurfacingConstants.NL );
+                    w.write( "" );
+                    w.write( SurfacingConstants.NL );
+                    w.write( "" );
+                    w.write( SurfacingConstants.NL );
+                    w.write( "" );
+                    w.write( SurfacingConstants.NL );
+                    w.write( "" );
+                    w.write( SurfacingConstants.NL );
+                    w.write( "" );
+                    w.write( SurfacingConstants.NL );
+                    if ( stats.getN() > 1 ) {
+                        w.write( "" );
+                    }
+                    else {
+                        w.write( "" );
+                    }
+                    w.write( SurfacingConstants.NL );
+                    w.write( "" );
+                    w.write( SurfacingConstants.NL );
+                    if ( stats.getN() > 1 ) {
+                        w.write( "" );
+                    }
+                    else {
+                        w.write( "" );
+                    }
+                    w.write( SurfacingConstants.NL );
+                    w.write( "
N: " + stats.getN() + "
Min: " + stats.getMin() + "
Max: " + stats.getMax() + "
Mean: " + stats.arithmeticMean() + "
SD: " + stats.sampleStandardDeviation() + "
SD: n/a
Median: " + stats.median() + "
Pearsonian skewness: " + stats.pearsonianSkewness() + "
Pearsonian skewness: n/a
" );
+                    w.write( SurfacingConstants.NL );
+                    w.write( "
" );
+                    w.write( SurfacingConstants.NL );
+                    w.write( "
" );
+                    w.write( SurfacingConstants.NL );
+                    w.write( "
" );
+                    w.write( SurfacingConstants.NL );
+                    w.write( "" );
                     w.write( SurfacingConstants.NL );
                 }
-                w.write( "" );
-                w.write( SurfacingConstants.NL );
-                w.write( "
" );
-                w.write( SurfacingConstants.NL );
-                w.write( "" );
-                w.write( SurfacingConstants.NL );
-                w.write( "" );
-                w.write( SurfacingConstants.NL );
-                w.write( "" );
-                w.write( SurfacingConstants.NL );
-                w.write( "" );
-                w.write( SurfacingConstants.NL );
-                if ( stats.getN() > 1 ) {
-                    w.write( "" );
-                }
-                else {
-                    w.write( "" );
-                }
-                w.write( SurfacingConstants.NL );
-                w.write( "" );
-                w.write( SurfacingConstants.NL );
-                if ( stats.getN() > 1 ) {
-                    w.write( "" );
-                }
-                else {
-                    w.write( "" );
-                }
-                w.write( SurfacingConstants.NL );
-                w.write( "
N: " + stats.getN() + "
Min: " + stats.getMin() + "
Max: " + stats.getMax() + "
Mean: " + stats.arithmeticMean() + "
SD: " + stats.sampleStandardDeviation() + "
SD: n/a
Median: " + stats.median() + "
Pearsonian skewness: " + stats.pearsonianSkewness() + "
Pearsonian skewness: n/a
" );
-                w.write( SurfacingConstants.NL );
-                w.write( "
" );
-                w.write( SurfacingConstants.NL );
-                w.write( "
" );
-                w.write( SurfacingConstants.NL );
-                w.write( "
" );
-                w.write( SurfacingConstants.NL );
-                w.write( "" );
-                w.write( SurfacingConstants.NL );
                 break;
         }
-        w.write( SurfacingConstants.NL );
+        for( final Writer w : split_writers.values() ) {
+            w.write( SurfacingConstants.NL );
+        }
         for( final DomainSimilarity similarity : similarities ) {
             if ( ( species_order != null ) && !species_order.isEmpty() ) {
                 ( ( PrintableDomainSimilarity ) similarity ).setSpeciesOrder( species_order );
             }
-            w.write( similarity.toStringBuffer( print_option ).toString() );
-            w.write( SurfacingConstants.NL );
+            if ( single_writer != null ) {
+                single_writer.write( similarity.toStringBuffer( print_option ).toString() );
+            }
+            else {
+                Writer local_writer = split_writers.get( ( similarity.getDomainId().getId().charAt( 0 ) + "" )
+                        .toLowerCase().charAt( 0 ) );
+                if ( local_writer == null ) {
+                    local_writer = split_writers.get( '0' );
+                }
+                local_writer.write( similarity.toStringBuffer( print_option ).toString() );
+            }
+            for( final Writer w : split_writers.values() ) {
+                w.write( SurfacingConstants.NL );
+            }
         }
         switch ( print_option ) {
             case HTML:
-                w.write( SurfacingConstants.NL );
-                w.write( "
" );
-                w.write( SurfacingConstants.NL );
-                w.write( "" );
-                w.write( SurfacingConstants.NL );
-                w.write( "" );
-                w.write( SurfacingConstants.NL );
-                w.write( "" );
-                w.write( SurfacingConstants.NL );
+                for( final Writer w : split_writers.values() ) {
+                    w.write( SurfacingConstants.NL );
+                    w.write( "" );
+                    w.write( SurfacingConstants.NL );
+                    w.write( "" );
+                    w.write( SurfacingConstants.NL );
+                    w.write( "" );
+                    w.write( SurfacingConstants.NL );
+                    w.write( "" );
+                    w.write( SurfacingConstants.NL );
+                }
                 break;
         }
-        w.flush();
-        w.close();
+        for( final Writer w : split_writers.values() ) {
+            w.close();
+        }
         return stats;
     }
 
@@ -2265,7 +2486,7 @@ public final class SurfacingUtil {
             w.write( ForesterUtil.LINE_SEPARATOR );
             my_matrix.writeNexusTaxaBlock( w );
             my_matrix.writeNexusBinaryChractersBlock( w );
-            PhylogenyWriter.writeNexusTreesBlock( w, phylogenies );
+            PhylogenyWriter.writeNexusTreesBlock( w, phylogenies, NH_CONVERSION_SUPPORT_VALUE_STYLE.NONE );
             w.flush();
             w.close();
             ForesterUtil.programMessage( surfacing.PRG_NAME, "Wrote Nexus file: \"" + outfile_name + "\"" );
@@ -2338,17 +2559,32 @@ public final class SurfacingUtil {
         try {
             writer.write( genome );
             writer.write( "\t" );
-            writer.write( stats.arithmeticMean() + "" );
-            writer.write( "\t" );
-            writer.write( stats.sampleStandardDeviation() + "" );
-            writer.write( "\t" );
-            writer.write( stats.median() + "" );
-            writer.write( "\t" );
-            writer.write( stats.getN() + "" );
-            writer.write( "\t" );
-            writer.write( stats.getMin() + "" );
-            writer.write( "\t" );
-            writer.write( stats.getMax() + "" );
+            if ( stats.getN() >= 1 ) {
+                writer.write( stats.arithmeticMean() + "" );
+                writer.write( "\t" );
+                if ( stats.getN() >= 2 ) {
+                    writer.write( stats.sampleStandardDeviation() + "" );
+                }
+                else {
+                    writer.write( "" );
+                }
+                writer.write( "\t" );
+                writer.write( stats.median() + "" );
+                writer.write( "\t" );
+                writer.write( stats.getN() + "" );
+                writer.write( "\t" );
+                writer.write( stats.getMin() + "" );
+                writer.write( "\t" );
+                writer.write( stats.getMax() + "" );
+            }
+            else {
+                writer.write( "\t" );
+                writer.write( "\t" );
+                writer.write( "\t" );
+                writer.write( "0" );
+                writer.write( "\t" );
+                writer.write( "\t" );
+            }
             writer.write( "\n" );
         }
         catch ( final IOException e ) {