X-Git-Url: http://source.jalview.org/gitweb/?a=blobdiff_plain;f=forester%2Fruby%2Fevoruby%2Flib%2Fevo%2Fio%2Fparser%2Fhmmscan_domain_extractor.rb;h=30962da9ade3f56690e70ba076c0891bdb768208;hb=fea8aa1f2d8d2eebfbbb57ede28edc003367c12f;hp=865424bf8f2dda4bd50316b6952fd5aa5b4f648d;hpb=07ac6e752b2d6e466fa350a46fd2a3c89b7ca034;p=jalview.git diff --git a/forester/ruby/evoruby/lib/evo/io/parser/hmmscan_domain_extractor.rb b/forester/ruby/evoruby/lib/evo/io/parser/hmmscan_domain_extractor.rb index 865424b..30962da 100644 --- a/forester/ruby/evoruby/lib/evo/io/parser/hmmscan_domain_extractor.rb +++ b/forester/ruby/evoruby/lib/evo/io/parser/hmmscan_domain_extractor.rb @@ -12,20 +12,20 @@ require 'lib/evo/msa/msa_factory' require 'lib/evo/io/msa_io' require 'lib/evo/io/writer/fasta_writer' require 'lib/evo/io/parser/fasta_parser' +require 'lib/evo/io/parser/hmmscan_parser' + module Evoruby class HmmscanDomainExtractor - TRIM_BY = 2 - def initialize end # raises ArgumentError, IOError, StandardError def parse( domain_id, - hmmsearch_output, + hmmscan_output, fasta_sequence_file, outfile, passed_seqs_outfile, @@ -34,16 +34,13 @@ module Evoruby length_threshold, add_position, add_domain_number, - add_domain_number_as_digit, - add_domain_number_as_letter, - trim_name, add_species, min_linker, log ) - Util.check_file_for_readability( hmmsearch_output ) + Util.check_file_for_readability( hmmscan_output ) Util.check_file_for_readability( fasta_sequence_file ) - Util.check_file_for_writability( outfile ) + Util.check_file_for_writability( outfile + ".fasta" ) Util.check_file_for_writability( passed_seqs_outfile ) Util.check_file_for_writability( failed_seqs_outfile ) @@ -61,227 +58,203 @@ module Evoruby failed_seqs = Msa.new passed_seqs = Msa.new out_msa_pairs = nil - out_msa_distance_partners = nil - out_msa_singlets = nil + out_msa_isolated = nil + out_msa_singles = nil + out_msa_single_domains_protein_seqs = nil + out_msa_close_pairs_protein_seqs = nil + out_msa_close_pairs_only_protein_seqs = nil + out_msa_isolated_protein_seqs = nil + out_msa_isolated_only_protein_seqs = nil + out_msa_isolated_and_close_pair_protein_seqs = nil if min_linker out_msa_pairs = Msa.new - out_msa_distant_partners = Msa.new - out_msa_singlets = Msa.new + out_msa_isolated = Msa.new + out_msa_singles = Msa.new + out_msa_single_domains_protein_seqs = Msa.new + out_msa_close_pairs_protein_seqs = Msa.new + out_msa_close_pairs_only_protein_seqs = Msa.new + out_msa_isolated_protein_seqs = Msa.new + out_msa_isolated_only_protein_seqs = Msa.new + out_msa_isolated_and_close_pair_protein_seqs = Msa.new end ld = Constants::LINE_DELIMITER domain_pass_counter = 0 domain_fail_counter = 0 - proteins_with_passing_domains = 0 proteins_with_failing_domains = 0 max_domain_copy_number_per_protein = -1 max_domain_copy_number_sequence = "" - prev_sequence = nil - prev_number = nil - prev_env_from = nil - prev_env_to = nil - prev_i_e_value = nil - prev_is_pair = false + hmmscan_datas = [] - File.open( hmmsearch_output ) do | file | - while line = file.gets - if !is_ignorable?( line ) && line =~ /^\S+\s+/ + hmmscan_parser = HmmscanParser.new( hmmscan_output ) + results = hmmscan_parser.parse - # tn acc tlen query acc qlen Evalue score bias # of c-E i-E score bias hf ht af at ef et acc desc - # 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 - line =~ /^(\S+)\s+(\S+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.*)/ + results.each do | r | + if domain_id != r.model + next + end - target_name = $1 - if domain_id != target_name - next - end + sequence = r.query + number = r.number + out_of = r.out_of + env_from = r.env_from + env_to = r.env_to + i_e_value = r.i_e_value + + if ( ( ( e_value_threshold < 0.0 ) || ( i_e_value <= e_value_threshold ) ) && + ( ( length_threshold <= 0 ) || ( env_to - env_from + 1 ) >= length_threshold.to_f ) ) + hmmscan_datas << HmmsearchData.new( sequence, number, out_of, env_from, env_to, i_e_value ) + if ( number > max_domain_copy_number_per_protein ) + max_domain_copy_number_sequence = sequence + max_domain_copy_number_per_protein = number + end + else # failed + print( domain_fail_counter.to_s + ": " + sequence.to_s + " did not meet threshold(s)" ) + log << domain_fail_counter.to_s + ": " + sequence.to_s + " did not meet threshold(s)" + if ( ( e_value_threshold.to_f >= 0.0 ) && ( i_e_value > e_value_threshold ) ) + print( " iE=" + i_e_value.to_s ) + log << " iE=" + i_e_value.to_s + end + if ( ( length_threshold.to_f > 0 ) && ( env_to - env_from + 1 ) < length_threshold.to_f ) + le = env_to - env_from + 1 + print( " l=" + le.to_s ) + log << " l=" + le.to_s + end + print( ld ) + log << ld + domain_fail_counter += 1 + end + + if number > out_of + error_msg = "number > out_of ! (this should not have happened)" + raise StandardError, error_msg + end - sequence = $4 - number = $10.to_i - out_of = $11.to_i - env_from = $20.to_i - env_to = $21.to_i - i_e_value = $13.to_f - if ( number > max_domain_copy_number_per_protein ) - max_domain_copy_number_sequence = sequence - max_domain_copy_number_per_protein = number + if number == out_of + if !hmmscan_datas.empty? + process_hmmscan_datas( hmmscan_datas, + in_msa, + add_position, + add_domain_number, + add_species, + out_msa, + out_msa_singles, + out_msa_pairs, + out_msa_isolated, + min_linker, + out_msa_single_domains_protein_seqs, + out_msa_close_pairs_protein_seqs, + out_msa_close_pairs_only_protein_seqs, + out_msa_isolated_protein_seqs, + out_msa_isolated_only_protein_seqs, + out_msa_isolated_and_close_pair_protein_seqs ) + domain_pass_counter += hmmscan_datas.length + if passed_seqs.find_by_name_start( sequence, true ).length < 1 + add_sequence( sequence, in_msa, passed_seqs ) + else + error_msg = "this should not have happened" + raise StandardError, error_msg end - if ( ( ( e_value_threshold < 0.0 ) || ( i_e_value <= e_value_threshold ) ) && - ( ( length_threshold <= 0 ) || ( env_to - env_from + 1 ) >= length_threshold.to_f ) ) - - extract_domain( sequence, - number, - out_of, - env_from, - env_to, - in_msa, - out_msa, - add_position, - add_domain_number, - add_domain_number_as_digit, - add_domain_number_as_letter, - trim_name , - add_species ) - domain_pass_counter += 1 - - if passed_seqs.find_by_name_start( sequence, true ).length < 1 - add_sequence( sequence, in_msa, passed_seqs ) - proteins_with_passing_domains += 1 - end - - if min_linker - if ( ( e_value_threshold < 0.0 ) || ( prev_i_e_value <= e_value_threshold ) ) && - ( ( length_threshold <= 0 ) || ( ( prev_env_to - prev_env_from + 1 ) >= length_threshold.to_f ) ) - - if sequence != prev_sequence - prev_is_pair = false - end - - if out_of == 1 - - if sequence == prev_sequence - puts "sequence == prev_sequence && out_of == 1" - exit - end - extract_domain( sequence, - number, - out_of, - env_from, - env_to, - in_msa, - out_msa_singlets, - false, - true, - false, - false, - trim_name , - add_species ) - - elsif sequence == prev_sequence - - if ( env_from - prev_env_to ) <= min_linker ####### - extract_domain( sequence, - prev_number.to_s + "+" + number.to_s, - out_of, - prev_env_from, - env_to, - in_msa, - out_msa_pairs, - false, - true, - false, - false, - trim_name , - add_species ) - prev_is_pair = true - else ####### - if !prev_is_pair - extract_domain( sequence, - prev_number, - out_of, - prev_env_from, - prev_env_to, - in_msa, - out_msa_distant_partners, - false, - true, - false, - false, - trim_name , - add_species ) - end - if number == out_of - extract_domain( sequence, - number, - out_of, - env_from, - env_to, - in_msa, - out_msa_distant_partners, - false, - true, - false, - false, - trim_name , - add_species ) - end - prev_is_pair = false - end ####### - - end - prev_sequence = sequence - prev_number = number - prev_env_from = env_from - prev_env_to = env_to - prev_i_e_value = i_e_value - end - - else - print( domain_fail_counter.to_s + ": " + sequence.to_s + " did not meet threshold(s)" ) - log << domain_fail_counter.to_s + ": " + sequence.to_s + " did not meet threshold(s)" - if ( ( e_value_threshold.to_f >= 0.0 ) && ( i_e_value > e_value_threshold ) ) - print( " iE=" + i_e_value.to_s ) - log << " iE=" + i_e_value.to_s - end - if ( ( length_threshold.to_f > 0 ) && ( env_to - env_from + 1 ) < length_threshold.to_f ) - le = env_to - env_from + 1 - print( " l=" + le.to_s ) - log << " l=" + le.to_s - end - print( Constants::LINE_DELIMITER ) - log << Constants::LINE_DELIMITER - domain_fail_counter += 1 - - if failed_seqs.find_by_name_start( sequence, true ).length < 1 - add_sequence( sequence, in_msa, failed_seqs ) - proteins_with_failing_domains += 1 - end - end + else + if failed_seqs.find_by_name_start( sequence, true ).length < 1 + add_sequence( sequence, in_msa, failed_seqs ) + proteins_with_failing_domains += 1 + else + error_msg = "this should not have happened" + raise StandardError, error_msg end - end # if !is_ignorable?( line ) && line =~ /^\S+\s+/ - end # while line = file.gets - end # File.open( hmmsearch_output ) do | file | + end + hmmscan_datas.clear + end + end if domain_pass_counter < 1 error_msg = "no domain sequences were extracted" - raise StandardError, error_msg + raise IOError, error_msg end - log << Constants::LINE_DELIMITER + log << ld puts( "Max domain copy number per protein : " + max_domain_copy_number_per_protein.to_s ) log << "Max domain copy number per protein : " + max_domain_copy_number_per_protein.to_s - log << Constants::LINE_DELIMITER + log << ld if ( max_domain_copy_number_per_protein > 1 ) puts( "First protein with this copy number: " + max_domain_copy_number_sequence ) log << "First protein with this copy number: " + max_domain_copy_number_sequence - log << Constants::LINE_DELIMITER + log << ld end - write_msa( out_msa, outfile ) + write_msa( out_msa, outfile + ".fasta" ) write_msa( passed_seqs, passed_seqs_outfile ) write_msa( failed_seqs, failed_seqs_outfile ) if out_msa_pairs - write_msa( out_msa_pairs, outfile +"_" + min_linker.to_s ) + write_msa( out_msa_pairs, outfile +"_" + min_linker.to_s + ".fasta") end - if out_msa_singlets - write_msa( out_msa_singlets, outfile +"_singles" ) + if out_msa_singles + write_msa( out_msa_singles, outfile +"_singles.fasta") end - if out_msa_distant_partners - write_msa( out_msa_distant_partners, outfile +"_singles" ) + if out_msa_isolated + write_msa( out_msa_isolated, outfile + "_" + min_linker.to_s + "_isolated.fasta"); end + if out_msa_single_domains_protein_seqs + write_msa( out_msa_single_domains_protein_seqs, outfile +"_proteins_with_singles.fasta" ) + end + + if out_msa_close_pairs_protein_seqs + write_msa( out_msa_close_pairs_protein_seqs, outfile + "_" + min_linker.to_s + "_proteins_close_pairs.fasta" ) + end + + if out_msa_close_pairs_only_protein_seqs + write_msa( out_msa_close_pairs_only_protein_seqs, outfile + "_" + min_linker.to_s + "_proteins_with_close_pairs_only.fasta" ) + end + + if out_msa_isolated_protein_seqs + write_msa( out_msa_isolated_protein_seqs, outfile + "_" + min_linker.to_s + "_proteins_with_isolated_domains.fasta" ) + end + + if out_msa_isolated_only_protein_seqs + write_msa( out_msa_isolated_only_protein_seqs, outfile + "_" + min_linker.to_s + "_proteins_with_isolated_domains_only.fasta" ) + end + + if out_msa_isolated_and_close_pair_protein_seqs + write_msa( out_msa_isolated_and_close_pair_protein_seqs, outfile + "_" + min_linker.to_s + "_proteins_with_isolated_and_close_pairs.fasta" ) + end + + if min_linker + if ( out_msa_single_domains_protein_seqs.get_number_of_seqs + + out_msa_close_pairs_only_protein_seqs.get_number_of_seqs + + out_msa_isolated_only_protein_seqs.get_number_of_seqs + + out_msa_isolated_and_close_pair_protein_seqs.get_number_of_seqs ) != passed_seqs.get_number_of_seqs + error_msg = "this should not have happened" + raise StandardError, error_msg + end + end log << ld - log << "passing domains : " + domain_pass_counter.to_s + ld - log << "failing domains : " + domain_fail_counter.to_s + ld - log << "proteins with passing domains: " + proteins_with_passing_domains.to_s + ld - log << "proteins with failing domains: " + proteins_with_failing_domains.to_s + ld + log << "passing domains : " + domain_pass_counter.to_s + ld + log << "failing domains : " + domain_fail_counter.to_s + ld + log << "input proteins : " + in_msa.get_number_of_seqs.to_s + ld + log << "proteins with passing domains : " + passed_seqs.get_number_of_seqs.to_s + ld + log << "proteins with no passing domains : " + proteins_with_failing_domains.to_s + ld + if min_linker + log << "min linker length : " + min_linker.to_s + ld + log << "single domains : " + out_msa_singles.get_number_of_seqs.to_s + ld + log << "domains in close pairs : " + (2 * out_msa_pairs.get_number_of_seqs).to_s + ld + log << "isolated domains : " + out_msa_isolated.get_number_of_seqs.to_s + ld + log << "proteins wih single domains : " + out_msa_single_domains_protein_seqs.get_number_of_seqs.to_s + ld + log << "proteins wih close pair domains : " + out_msa_close_pairs_protein_seqs.get_number_of_seqs.to_s + ld + log << "proteins wih close pair domains only : " + out_msa_close_pairs_only_protein_seqs.get_number_of_seqs.to_s + ld + log << "proteins wih isolated domains : " + out_msa_isolated_protein_seqs.get_number_of_seqs.to_s + ld + log << "proteins wih isolated domains only : " + out_msa_isolated_only_protein_seqs.get_number_of_seqs.to_s + ld + log << "proteins wih close pair and isolated domains: " + out_msa_isolated_and_close_pair_protein_seqs.get_number_of_seqs.to_s + ld + end + log << ld return domain_pass_counter @@ -319,7 +292,150 @@ module Evoruby add_to_msa.add_sequence( seq ) end - # raises ArgumentError, StandardError + def process_hmmscan_datas( hmmscan_datas, + in_msa, + add_position, + add_domain_number, + add_species, + out_msa, + out_msa_singles, + out_msa_pairs, + out_msa_isolated, + min_linker, + out_msa_single_domains_protein_seqs, + out_msa_close_pairs_protein_seqs, + out_msa_close_pairs_only_protein_seqs, + out_msa_isolated_protein_seqs, + out_msa_isolated_only_protein_seqs, + out_msa_isolated_and_close_pair_protein_seqs ) + + actual_out_of = hmmscan_datas.size + saw_close_pair = false + saw_isolated = false + + seq_name = "" + prev_seq_name = nil + + hmmscan_datas.each_with_index do |hmmscan_data, index| + if hmmscan_data.number < ( index + 1 ) + error_msg = "hmmscan_data.number < ( index + 1 ) (this should not have happened)" + raise StandardError, error_msg + end + + seq_name = hmmscan_data.seq_name + + extract_domain( seq_name, + index + 1, + actual_out_of, + hmmscan_data.env_from, + hmmscan_data.env_to, + in_msa, + out_msa, + add_position, + add_domain_number, + add_species ) + + if min_linker + if actual_out_of == 1 + extract_domain( seq_name, + 1, + 1, + hmmscan_data.env_from, + hmmscan_data.env_to, + in_msa, + out_msa_singles, + add_position, + add_domain_number, + add_species ) + if out_msa_single_domains_protein_seqs.find_by_name_start( seq_name, true ).length < 1 + add_sequence( seq_name, in_msa, out_msa_single_domains_protein_seqs ) + else + error_msg = "this should not have happened" + raise StandardError, error_msg + end + + else + first = index == 0 + last = index == hmmscan_datas.length - 1 + + if ( ( first && ( ( hmmscan_datas[ index + 1 ].env_from - hmmscan_data.env_to ) > min_linker) ) || + ( last && ( ( hmmscan_data.env_from - hmmscan_datas[ index - 1 ].env_to ) > min_linker ) ) || + ( !first && !last && ( ( hmmscan_datas[ index + 1 ].env_from - hmmscan_data.env_to ) > min_linker ) && + ( ( hmmscan_data.env_from - hmmscan_datas[ index - 1 ].env_to ) > min_linker ) ) ) + + extract_domain( seq_name, + index + 1, + actual_out_of, + hmmscan_data.env_from, + hmmscan_data.env_to, + in_msa, + out_msa_isolated, + add_position, + add_domain_number, + add_species ) + saw_isolated = true + + elsif !first + extract_domain( seq_name, + index.to_s + "+" + ( index + 1 ).to_s, + actual_out_of, + hmmscan_datas[ index - 1 ].env_from, + hmmscan_data.env_to, + in_msa, + out_msa_pairs, + add_position, + add_domain_number, + add_species ) + saw_close_pair = true + end + end + end + if prev_seq_name && prev_seq_name != seq_name + error_msg = "this should not have happened" + raise StandardError, error_msg + end + prev_seq_name = seq_name + end # each + if saw_isolated + if out_msa_isolated_protein_seqs.find_by_name_start( seq_name, true ).length < 1 + add_sequence( seq_name, in_msa, out_msa_isolated_protein_seqs ) + else + error_msg = "this should not have happened" + raise StandardError, error_msg + end + end + if saw_close_pair + if out_msa_close_pairs_protein_seqs.find_by_name_start( seq_name, true ).length < 1 + add_sequence( seq_name, in_msa, out_msa_close_pairs_protein_seqs ) + else + error_msg = "this should not have happened" + raise StandardError, error_msg + end + end + if saw_close_pair && saw_isolated + if out_msa_isolated_and_close_pair_protein_seqs.find_by_name_start( seq_name, true ).length < 1 + add_sequence( seq_name, in_msa, out_msa_isolated_and_close_pair_protein_seqs ) + else + error_msg = "this should not have happened" + raise StandardError, error_msg + end + elsif saw_close_pair + if out_msa_close_pairs_only_protein_seqs.find_by_name_start( seq_name, true ).length < 1 + add_sequence( seq_name, in_msa, out_msa_close_pairs_only_protein_seqs ) + else + error_msg = "this should not have happened" + raise StandardError, error_msg + end + elsif saw_isolated + if out_msa_isolated_only_protein_seqs.find_by_name_start( seq_name, true ).length < 1 + add_sequence( seq_name, in_msa, out_msa_isolated_only_protein_seqs ) + else + error_msg = "this should not have happened" + raise StandardError, error_msg + end + end + end # def process_hmmscan_data + def extract_domain( sequence, number, out_of, @@ -329,26 +445,23 @@ module Evoruby out_msa, add_position, add_domain_number, - add_domain_number_as_digit, - add_domain_number_as_letter, - trim_name, add_species ) if number.is_a?( Fixnum ) && ( number < 1 || out_of < 1 || number > out_of ) error_msg = "impossible: number=" + number.to_s + ", out of=" + out_of.to_s - raise ArgumentError, error_msg + raise StandardError, error_msg end if seq_from < 1 || seq_to < 1 || seq_from >= seq_to error_msg = "impossible: seq-f=" + seq_from.to_s + ", seq-t=" + seq_to.to_s - raise ArgumentError, error_msg + raise StandardError, error_msg end seqs = in_msa.find_by_name_start( sequence, true ) if seqs.length < 1 error_msg = "sequence \"" + sequence + "\" not found in sequence file" - raise StandardError, error_msg + raise IOError, error_msg end if seqs.length > 1 error_msg = "sequence \"" + sequence + "\" not unique in sequence file" - raise StandardError, error_msg + raise IOError, error_msg end # hmmsearch is 1 based, wheres sequences are 0 bases in this package. seq = in_msa.get_sequence( seqs[ 0 ] ).get_subsequence( seq_from - 1, seq_to - 1 ) @@ -361,29 +474,10 @@ module Evoruby seq.set_name( seq.get_name + "_" + seq_from.to_s + "-" + seq_to.to_s ) end - if trim_name - seq.set_name( seq.get_name[ 0, seq.get_name.length - TRIM_BY ] ) + if out_of != 1 && add_domain_number + seq.set_name( seq.get_name + "~" + number.to_s + "-" + out_of.to_s ) end - if out_of != 1 - if add_domain_number_as_digit - seq.set_name( seq.get_name + number.to_s ) - elsif add_domain_number_as_letter - if number > 25 - error_msg = 'too many identical domains per sequence, cannot use letters to distinguish them' - raise StandardError, error_msg - end - seq.set_name( seq.get_name + ( number + 96 ).chr ) - elsif add_domain_number - seq.set_name( seq.get_name + "~" + number.to_s + "-" + out_of.to_s ) - end - end - - # if ( seq.get_name.length > 10 ) - # error_msg = "sequence name [" + seq.get_name + "] is longer than 10 characters" - # raise StandardError, error_msg - # end - if add_species a = orig_name.rindex "[" b = orig_name.rindex "]" @@ -403,5 +497,17 @@ module Evoruby end # class HmmscanDomainExtractor + class HmmsearchData + def initialize( seq_name, number, out_of, env_from, env_to, i_e_value ) + @seq_name = seq_name + @number = number + @out_of = out_of + @env_from = env_from + @env_to = env_to + @i_e_value = i_e_value + end + attr_reader :seq_name, :number, :out_of, :env_from, :env_to, :i_e_value + end + end # module Evoruby