From cc2a342b0e38cd850ace29050aa027f084dee26f Mon Sep 17 00:00:00 2001 From: "cmzmasek@gmail.com" Date: Sat, 29 Sep 2012 04:38:29 +0000 Subject: [PATCH] in progress --- .../lib/evo/apps/domain_sequence_extractor.rb | 2 +- .../lib/evo/io/parser/hmmscan_domain_extractor.rb | 432 +++++++++++--------- .../evo/io/parser/hmmsearch_domain_extractor.rb | 38 +- 3 files changed, 244 insertions(+), 228 deletions(-) diff --git a/forester/ruby/evoruby/lib/evo/apps/domain_sequence_extractor.rb b/forester/ruby/evoruby/lib/evo/apps/domain_sequence_extractor.rb index 517042a..457e530 100644 --- a/forester/ruby/evoruby/lib/evo/apps/domain_sequence_extractor.rb +++ b/forester/ruby/evoruby/lib/evo/apps/domain_sequence_extractor.rb @@ -31,7 +31,7 @@ module Evoruby ADD_DOMAIN_NUMBER_OPTION_AS_DIGIT = 'dd' ADD_DOMAIN_NUMBER_OPTION_AS_LETTER = 'dl' ADD_SPECIES = 's' - MIN_LINKER_OPT = 'ml' + MIN_LINKER_OPT = 'ml' TRIM_OPTION = 't' LOG_FILE_SUFFIX = '_domain_seq_extr.log' PASSED_SEQS_SUFFIX = '_domain_seq_extr_passed' diff --git a/forester/ruby/evoruby/lib/evo/io/parser/hmmscan_domain_extractor.rb b/forester/ruby/evoruby/lib/evo/io/parser/hmmscan_domain_extractor.rb index bf7cb71..273f348 100644 --- a/forester/ruby/evoruby/lib/evo/io/parser/hmmscan_domain_extractor.rb +++ b/forester/ruby/evoruby/lib/evo/io/parser/hmmscan_domain_extractor.rb @@ -61,8 +61,12 @@ module Evoruby failed_seqs = Msa.new passed_seqs = Msa.new out_msa_pairs = nil + out_msa_distance_partners = nil + out_msa_singlets = nil if min_linker out_msa_pairs = Msa.new + out_msa_distant_partners = Msa.new + out_msa_singlets = Msa.new end ld = Constants::LINE_DELIMITER @@ -72,15 +76,14 @@ module Evoruby proteins_with_passing_domains = 0 proteins_with_failing_domains = 0 max_domain_copy_number_per_protein = -1 - max_domain_copy_number_sequence = '' - failed_species_counts = Hash.new - passed_species_counts = Hash.new + max_domain_copy_number_sequence = "" prev_sequence = nil prev_number = nil prev_env_from = nil prev_env_to = nil prev_i_e_value = nil + prev_is_pair = false File.open( hmmsearch_output ) do | file | while line = file.gets @@ -95,7 +98,6 @@ module Evoruby next end - sequence = $4 number = $10.to_i out_of = $11.to_i @@ -106,8 +108,9 @@ module Evoruby max_domain_copy_number_sequence = sequence max_domain_copy_number_per_protein = number end - if ( ( ( e_value_threshold.to_f < 0.0 ) || ( i_e_value <= e_value_threshold ) ) && - ( ( length_threshold.to_f <= 0 ) || ( env_to - env_from + 1 ) >= length_threshold.to_f ) ) + if ( ( ( e_value_threshold < 0.0 ) || ( i_e_value <= e_value_threshold ) ) && + ( ( length_threshold <= 0 ) || ( env_to - env_from + 1 ) >= length_threshold.to_f ) ) + extract_domain( sequence, number, out_of, @@ -122,251 +125,284 @@ module Evoruby trim_name , add_species ) domain_pass_counter += 1 - count_species( sequence, passed_species_counts ) + if passed_seqs.find_by_name_start( sequence, true ).length < 1 add_sequence( sequence, in_msa, passed_seqs ) proteins_with_passing_domains += 1 end if min_linker - if sequence == prev_sequence && ( ( ( e_value_threshold.to_f < 0.0 ) || ( prev_i_e_value <= e_value_threshold ) ) && - ( ( length_threshold.to_f <= 0 ) || ( ( prev_env_to - prev_env_from + 1 ) >= length_threshold.to_f ) ) ) - diff = env_from - prev_env_to - if diff <= min_linker + if ( ( e_value_threshold < 0.0 ) || ( prev_i_e_value <= e_value_threshold ) ) && + ( ( length_threshold <= 0 ) || ( ( prev_env_to - prev_env_from + 1 ) >= length_threshold.to_f ) ) + + if sequence != prev_sequence + prev_is_pair = false + end + + if out_of == 1 + + if sequence == prev_sequence + puts "sequence == prev_sequence && out_of == 1" + exit + end extract_domain( sequence, - prev_number.to_s + "/" + number.to_s, + number, out_of, - prev_env_from, + env_from, env_to, in_msa, - out_msa_pairs, + out_msa_singlets, false, true, false, false, trim_name , add_species ) + + elsif sequence == prev_sequence + + if ( env_from - prev_env_to ) <= min_linker ####### + extract_domain( sequence, + prev_number.to_s + "+" + number.to_s, + out_of, + prev_env_from, + env_to, + in_msa, + out_msa_pairs, + false, + true, + false, + false, + trim_name , + add_species ) + prev_is_pair = true + else ####### + if !prev_is_pair + extract_domain( sequence, + prev_number, + out_of, + prev_env_from, + prev_env_to, + in_msa, + out_msa_distant_partners, + false, + true, + false, + false, + trim_name , + add_species ) + end + if number == out_of + extract_domain( sequence, + number, + out_of, + env_from, + env_to, + in_msa, + out_msa_distant_partners, + false, + true, + false, + false, + trim_name , + add_species ) + end + prev_is_pair = false + end ####### + end + prev_sequence = sequence + prev_number = number + prev_env_from = env_from + prev_env_to = env_to + prev_i_e_value = i_e_value end - prev_sequence = sequence - prev_number = number - prev_env_from = env_from - prev_env_to = env_to - prev_i_e_value = i_e_value - end - else - print( domain_fail_counter.to_s + ": " + sequence.to_s + " did not meet threshold(s)" ) - log << domain_fail_counter.to_s + ": " + sequence.to_s + " did not meet threshold(s)" - if ( ( e_value_threshold.to_f >= 0.0 ) && ( i_e_value > e_value_threshold ) ) - print( " iE=" + i_e_value.to_s ) - log << " iE=" + i_e_value.to_s - end - if ( ( length_threshold.to_f > 0 ) && ( env_to - env_from + 1 ) < length_threshold.to_f ) - le = env_to - env_from + 1 - print( " l=" + le.to_s ) - log << " l=" + le.to_s - end - print( Constants::LINE_DELIMITER ) - log << Constants::LINE_DELIMITER - domain_fail_counter += 1 - count_species( sequence, failed_species_counts ) - if failed_seqs.find_by_name_start( sequence, true ).length < 1 - add_sequence( sequence, in_msa, failed_seqs ) - proteins_with_failing_domains += 1 + else + print( domain_fail_counter.to_s + ": " + sequence.to_s + " did not meet threshold(s)" ) + log << domain_fail_counter.to_s + ": " + sequence.to_s + " did not meet threshold(s)" + if ( ( e_value_threshold.to_f >= 0.0 ) && ( i_e_value > e_value_threshold ) ) + print( " iE=" + i_e_value.to_s ) + log << " iE=" + i_e_value.to_s + end + if ( ( length_threshold.to_f > 0 ) && ( env_to - env_from + 1 ) < length_threshold.to_f ) + le = env_to - env_from + 1 + print( " l=" + le.to_s ) + log << " l=" + le.to_s + end + print( Constants::LINE_DELIMITER ) + log << Constants::LINE_DELIMITER + domain_fail_counter += 1 + + if failed_seqs.find_by_name_start( sequence, true ).length < 1 + add_sequence( sequence, in_msa, failed_seqs ) + proteins_with_failing_domains += 1 + end end end end end - end - if domain_pass_counter < 1 - error_msg = "no domain sequences were extracted" - raise StandardError, error_msg - end - - log << Constants::LINE_DELIMITER - puts( "Max domain copy number per protein : " + max_domain_copy_number_per_protein.to_s ) - log << "Max domain copy number per protein : " + max_domain_copy_number_per_protein.to_s - log << Constants::LINE_DELIMITER + if domain_pass_counter < 1 + error_msg = "no domain sequences were extracted" + raise StandardError, error_msg + end - if ( max_domain_copy_number_per_protein > 1 ) - puts( "First protein with this copy number: " + max_domain_copy_number_sequence ) - log << "First protein with this copy number: " + max_domain_copy_number_sequence log << Constants::LINE_DELIMITER - end + puts( "Max domain copy number per protein : " + max_domain_copy_number_per_protein.to_s ) + log << "Max domain copy number per protein : " + max_domain_copy_number_per_protein.to_s + log << Constants::LINE_DELIMITER - io = MsaIO.new() - w = FastaWriter.new() - w.set_line_width( 60 ) - w.clean( true ) + if ( max_domain_copy_number_per_protein > 1 ) + puts( "First protein with this copy number: " + max_domain_copy_number_sequence ) + log << "First protein with this copy number: " + max_domain_copy_number_sequence + log << Constants::LINE_DELIMITER + end - begin - io.write_to_file( out_msa, outfile, w ) - rescue Exception - error_msg = "could not write to \"" + outfile + "\"" - raise IOError, error_msg - end + write_msa( out_msa, outfile ) + write_msa( passed_seqs, passed_seqs_outfile ) + write_msa( failed_seqs, failed_seqs_outfile ) - if out_msa_pairs - begin - io.write_to_file( out_msa_pairs, outfile+"_" + min_linker.to_s, w ) - rescue Exception - error_msg = "could not write to \"" + outfile+"_" + min_linker.to_s + "\"" - raise IOError, error_msg + if out_msa_pairs + write_msa( out_msa_pairs, outfile +"_" + min_linker.to_s ) end - end - begin - io.write_to_file( passed_seqs, passed_seqs_outfile, w ) - rescue Exception - error_msg = "could not write to \"" + passed_seqs_outfile + "\"" - raise IOError, error_msg - end + if out_msa_singlets + write_msa( out_msa_singlets, outfile +"_singles" ) + end - begin - io.write_to_file( failed_seqs, failed_seqs_outfile, w ) - rescue Exception - error_msg = "could not write to \"" + failed_seqs_outfile + "\"" - raise IOError, error_msg - end + if out_msa_distant_partners + write_msa( out_msa_distant_partners, outfile +"_singles" ) + end - log << ld - log << "passing domains : " + domain_pass_counter.to_s + ld - log << "failing domains : " + domain_fail_counter.to_s + ld - log << "proteins with passing domains: " + proteins_with_passing_domains.to_s + ld - log << "proteins with failing domains: " + proteins_with_failing_domains.to_s + ld - log << ld - log << 'passing domains counts per species: ' << ld - passed_species_counts.each_pair { | species, count | log << "#{species}: #{count}" << ld } - log << ld - log << 'failing domains counts per species: ' << ld - failed_species_counts.each_pair { | species, count | log << "#{species}: #{count}" << ld } - log << ld - return domain_pass_counter - - end # parse - - private - - def add_sequence( sequence_name, in_msa, add_to_msa ) - seqs = in_msa.find_by_name_start( sequence_name, true ) - if ( seqs.length < 1 ) - error_msg = "sequence \"" + sequence_name + "\" not found in sequence file" - raise StandardError, error_msg - end - if ( seqs.length > 1 ) - error_msg = "sequence \"" + sequence_name + "\" not unique in sequence file" - raise StandardError, error_msg - end - seq = in_msa.get_sequence( seqs[ 0 ] ) - add_to_msa.add_sequence( seq ) - end - # raises ArgumentError, StandardError - def extract_domain( sequence, - number, - out_of, - seq_from, - seq_to, - in_msa, - out_msa, - add_position, - add_domain_number, - add_domain_number_as_digit, - add_domain_number_as_letter, - trim_name, - add_species ) - if number.is_a?( Fixnum ) && ( number < 1 || out_of < 1 || number > out_of ) - error_msg = "impossible: number=" + number.to_s + ", out of=" + out_of.to_s - raise ArgumentError, error_msg - end - if seq_from < 1 || seq_to < 1 || seq_from >= seq_to - error_msg = "impossible: seq-f=" + seq_from.to_s + ", seq-t=" + seq_to.to_s - raise ArgumentError, error_msg - end - seqs = in_msa.find_by_name_start( sequence, true ) - if seqs.length < 1 - error_msg = "sequence \"" + sequence + "\" not found in sequence file" - raise StandardError, error_msg - end - if seqs.length > 1 - error_msg = "sequence \"" + sequence + "\" not unique in sequence file" - raise StandardError, error_msg - end - # hmmsearch is 1 based, wheres sequences are 0 bases in this package. - seq = in_msa.get_sequence( seqs[ 0 ] ).get_subsequence( seq_from - 1, seq_to - 1 ) + log << ld + log << "passing domains : " + domain_pass_counter.to_s + ld + log << "failing domains : " + domain_fail_counter.to_s + ld + log << "proteins with passing domains: " + proteins_with_passing_domains.to_s + ld + log << "proteins with failing domains: " + proteins_with_failing_domains.to_s + ld + log << ld - orig_name = seq.get_name + return domain_pass_counter - seq.set_name( orig_name.split[ 0 ] ) + end # parse - if add_position - seq.set_name( seq.get_name + "_" + seq_from.to_s + "-" + seq_to.to_s ) - end - if trim_name - seq.set_name( seq.get_name[ 0, seq.get_name.length - TRIM_BY ] ) - end + private - if out_of != 1 - if add_domain_number_as_digit - seq.set_name( seq.get_name + number.to_s ) - elsif add_domain_number_as_letter - if number > 25 - error_msg = 'too many identical domains per sequence, cannot use letters to distinguish them' - raise StandardError, error_msg - end - seq.set_name( seq.get_name + ( number + 96 ).chr ) - elsif add_domain_number - seq.set_name( seq.get_name + "~" + number.to_s + "-" + out_of.to_s ) + def write_msa( msa, filename ) + io = MsaIO.new() + w = FastaWriter.new() + w.set_line_width( 60 ) + w.clean( true ) + begin + io.write_to_file( msa, filename, w ) + rescue Exception + error_msg = "could not write to \"" + filename + "\"" + raise IOError, error_msg end end - # if ( seq.get_name.length > 10 ) - # error_msg = "sequence name [" + seq.get_name + "] is longer than 10 characters" - # raise StandardError, error_msg - # end - if add_species - a = orig_name.rindex "[" - b = orig_name.rindex "]" - unless a && b - error_msg = "species not found in " + orig_name + def add_sequence( sequence_name, in_msa, add_to_msa ) + seqs = in_msa.find_by_name_start( sequence_name, true ) + if ( seqs.length < 1 ) + error_msg = "sequence \"" + sequence_name + "\" not found in sequence file" + raise StandardError, error_msg + end + if ( seqs.length > 1 ) + error_msg = "sequence \"" + sequence_name + "\" not unique in sequence file" raise StandardError, error_msg end - species = orig_name[ a .. b ] - seq.set_name( seq.get_name + " " + species ) + seq = in_msa.get_sequence( seqs[ 0 ] ) + add_to_msa.add_sequence( seq ) end - out_msa.add_sequence( seq ) + # raises ArgumentError, StandardError + def extract_domain( sequence, + number, + out_of, + seq_from, + seq_to, + in_msa, + out_msa, + add_position, + add_domain_number, + add_domain_number_as_digit, + add_domain_number_as_letter, + trim_name, + add_species ) + if number.is_a?( Fixnum ) && ( number < 1 || out_of < 1 || number > out_of ) + error_msg = "impossible: number=" + number.to_s + ", out of=" + out_of.to_s + raise ArgumentError, error_msg + end + if seq_from < 1 || seq_to < 1 || seq_from >= seq_to + error_msg = "impossible: seq-f=" + seq_from.to_s + ", seq-t=" + seq_to.to_s + raise ArgumentError, error_msg + end + seqs = in_msa.find_by_name_start( sequence, true ) + if seqs.length < 1 + error_msg = "sequence \"" + sequence + "\" not found in sequence file" + raise StandardError, error_msg + end + if seqs.length > 1 + error_msg = "sequence \"" + sequence + "\" not unique in sequence file" + raise StandardError, error_msg + end + # hmmsearch is 1 based, wheres sequences are 0 bases in this package. + seq = in_msa.get_sequence( seqs[ 0 ] ).get_subsequence( seq_from - 1, seq_to - 1 ) + + orig_name = seq.get_name - end + seq.set_name( orig_name.split[ 0 ] ) - def count_species( sequence, species_counts_map ) - species = get_species( sequence ) - if species != nil - if !species_counts_map.has_key?( species ) - species_counts_map[ species ] = 1 - else - species_counts_map[ species ] = species_counts_map[ species ] + 1 + if add_position + seq.set_name( seq.get_name + "_" + seq_from.to_s + "-" + seq_to.to_s ) end - end - end - def get_species( sequence_name ) - if sequence_name =~ /^.+_(.+)$/ - return $1 - else - return nil + if trim_name + seq.set_name( seq.get_name[ 0, seq.get_name.length - TRIM_BY ] ) + end + + if out_of != 1 + if add_domain_number_as_digit + seq.set_name( seq.get_name + number.to_s ) + elsif add_domain_number_as_letter + if number > 25 + error_msg = 'too many identical domains per sequence, cannot use letters to distinguish them' + raise StandardError, error_msg + end + seq.set_name( seq.get_name + ( number + 96 ).chr ) + elsif add_domain_number + seq.set_name( seq.get_name + "~" + number.to_s + "-" + out_of.to_s ) + end + end + + # if ( seq.get_name.length > 10 ) + # error_msg = "sequence name [" + seq.get_name + "] is longer than 10 characters" + # raise StandardError, error_msg + # end + + if add_species + a = orig_name.rindex "[" + b = orig_name.rindex "]" + unless a && b + error_msg = "species not found in " + orig_name + raise StandardError, error_msg + end + species = orig_name[ a .. b ] + seq.set_name( seq.get_name + " " + species ) + end + + out_msa.add_sequence( seq ) + end - end - def is_ignorable?( line ) - return ( line !~ /[A-Za-z0-9-]/ || line =~/^#/ ) - end + def is_ignorable?( line ) + return ( line !~ /[A-Za-z0-9-]/ || line =~/^#/ ) + end - end # class HmmscanDomainExtractor + end # class HmmscanDomainExtractor -end # module Evoruby + end # module Evoruby diff --git a/forester/ruby/evoruby/lib/evo/io/parser/hmmsearch_domain_extractor.rb b/forester/ruby/evoruby/lib/evo/io/parser/hmmsearch_domain_extractor.rb index 238d13c..e6d67d9 100644 --- a/forester/ruby/evoruby/lib/evo/io/parser/hmmsearch_domain_extractor.rb +++ b/forester/ruby/evoruby/lib/evo/io/parser/hmmsearch_domain_extractor.rb @@ -64,9 +64,8 @@ module Evoruby proteins_with_passing_domains = 0 proteins_with_failing_domains = 0 max_domain_copy_number_per_protein = -1 - max_domain_copy_number_sequence = '' - failed_species_counts = Hash.new - passed_species_counts = Hash.new + max_domain_copy_number_sequence = "" + File.open( hmmsearch_output ) do | file | while line = file.gets @@ -88,6 +87,7 @@ module Evoruby end if ( ( ( e_value_threshold.to_f < 0.0 ) || ( i_e_value <= e_value_threshold ) ) && ( ( length_threshold.to_f <= 0 ) || ( env_to - env_from + 1 ) >= length_threshold.to_f ) ) + HmmsearchDomainExtractor.extract_domain( sequence, number, out_of, @@ -100,8 +100,9 @@ module Evoruby add_domain_number_as_digit, add_domain_number_as_letter, trim_name ) + domain_pass_counter += 1 - count_species( sequence, passed_species_counts ) + if !passed_seqs.has?( sequence, true, false ) HmmsearchDomainExtractor.add_sequence( sequence, in_msa, passed_seqs ) proteins_with_passing_domains += 1 @@ -121,7 +122,7 @@ module Evoruby print( Constants::LINE_DELIMITER ) log << Constants::LINE_DELIMITER domain_fail_counter += 1 - count_species( sequence, failed_species_counts ) + if !failed_seqs.has?( sequence, true, false ) HmmsearchDomainExtractor.add_sequence( sequence, in_msa, failed_seqs ) proteins_with_failing_domains += 1 @@ -179,19 +180,13 @@ module Evoruby log << "proteins with passing domains: " + proteins_with_passing_domains.to_s + ld log << "proteins with failing domains: " + proteins_with_failing_domains.to_s + ld log << ld - log << 'passing domains counts per species: ' << ld - passed_species_counts.each_pair { | species, count | log << "#{species}: #{count}" << ld } - log << ld - log << 'failing domains counts per species: ' << ld - failed_species_counts.each_pair { | species, count | log << "#{species}: #{count}" << ld } - log << ld + return domain_pass_counter end # parse private - def HmmsearchDomainExtractor.add_sequence( sequence_name, in_msa, add_to_msa ) seqs = in_msa.find_by_name( sequence_name, true, false ) if ( seqs.length < 1 ) @@ -268,24 +263,9 @@ module Evoruby out_msa.add_sequence( seq ) end - def count_species( sequence, species_counts_map ) - species = get_species( sequence ) - if species != nil - if !species_counts_map.has_key?( species ) - species_counts_map[ species ] = 1 - else - species_counts_map[ species ] = species_counts_map[ species ] + 1 - end - end - end + - def get_species( sequence_name ) - if sequence_name =~ /^.+_(.+)$/ - return $1 - else - return nil - end - end + def is_ignorable?( line ) return ( line !~ /[A-Za-z0-9-]/ || line =~/^#/ ) -- 1.7.10.2