2 # = lib/evo/io/parser/hmmscan_domain_extractor.rb - HmmscanDomainExtractor class
4 # Copyright:: Copyright (C) 2017 Christian M. Zmasek
5 # License:: GNU Lesser General Public License (LGPL)
7 # Last modified: 2017/02/20
9 require 'lib/evo/util/constants'
10 require 'lib/evo/msa/msa_factory'
11 require 'lib/evo/io/msa_io'
12 require 'lib/evo/io/writer/fasta_writer'
13 require 'lib/evo/io/parser/fasta_parser'
14 require 'lib/evo/io/parser/hmmscan_parser'
17 class HmmscanDomainExtractor
19 ADD_TO_CLOSE_PAIRS = 0
23 # raises ArgumentError, IOError, StandardError
38 Util.check_file_for_readability( hmmscan_output )
39 Util.check_file_for_readability( fasta_sequence_file )
40 Util.check_file_for_writability( outfile + ".fasta" )
41 Util.check_file_for_writability( passed_seqs_outfile )
42 Util.check_file_for_writability( failed_seqs_outfile )
45 factory = MsaFactory.new()
46 in_msa = factory.create_msa_from_file( fasta_sequence_file, FastaParser.new() )
48 if ( in_msa == nil || in_msa.get_number_of_seqs() < 1 )
49 error_msg = "could not find fasta sequences in " + fasta_sequence_file
50 raise IOError, error_msg
58 out_msa_isolated = nil
60 out_msa_single_domains_protein_seqs = nil
61 out_msa_close_pairs_protein_seqs = nil
62 out_msa_close_pairs_only_protein_seqs = nil
63 out_msa_isolated_protein_seqs = nil
64 out_msa_isolated_only_protein_seqs = nil
65 out_msa_isolated_and_close_pair_protein_seqs = nil
67 out_msa_pairs = Msa.new
68 out_msa_isolated = Msa.new
69 out_msa_singles = Msa.new
70 out_msa_single_domains_protein_seqs = Msa.new
71 out_msa_close_pairs_protein_seqs = Msa.new
72 out_msa_close_pairs_only_protein_seqs = Msa.new
73 out_msa_isolated_protein_seqs = Msa.new
74 out_msa_isolated_only_protein_seqs = Msa.new
75 out_msa_isolated_and_close_pair_protein_seqs = Msa.new
78 ld = Constants::LINE_DELIMITER
80 domain_pass_counter = 0
81 domain_fail_counter = 0
82 passing_domains_per_protein = 0
83 proteins_with_failing_domains = 0
84 domain_not_present_counter = 0
86 max_domain_copy_number_per_protein = -1
87 max_domain_copy_number_sequence = ""
88 passing_target_length_sum = 0
89 overall_target_length_sum = 0
90 overall_target_length_min = 10000000
91 overall_target_length_max = -1
92 passing_target_length_min = 10000000
93 passing_target_length_max = -1
95 overall_target_ie_min = 10000000
96 overall_target_ie_max = -1
97 passing_target_ie_min = 10000000
98 passing_target_ie_max = -1
102 hmmscan_parser = HmmscanParser.new( hmmscan_output )
103 results = hmmscan_parser.parse
108 results.each do | r |
110 if ( prev_query != nil ) && ( r.query != prev_query )
112 passing_domains_per_protein = 0
114 log << domain_not_present_counter.to_s + ": " + prev_query.to_s + " lacks target domain" + ld
115 domain_not_present_counter += 1
122 if domain_id != r.model
131 env_from = r.env_from
133 i_e_value = r.i_e_value
136 length = 1 + env_to - env_from
138 overall_target_length_sum += length
139 if length > overall_target_length_max
140 overall_target_length_max = length
142 if length < overall_target_length_min
143 overall_target_length_min = length
146 if i_e_value > overall_target_ie_max
147 overall_target_ie_max = i_e_value
149 if i_e_value < overall_target_ie_min
150 overall_target_ie_min = i_e_value
153 if ( ( ( e_value_threshold < 0.0 ) || ( i_e_value <= e_value_threshold ) ) &&
154 ( ( length_threshold <= 0 ) || ( length >= length_threshold.to_f ) ) )
155 hmmscan_datas << HmmsearchData.new( sequence, number, out_of, env_from, env_to, i_e_value )
156 passing_target_length_sum += length
157 passing_domains_per_protein += 1
158 if length > passing_target_length_max
159 passing_target_length_max = length
161 if length < passing_target_length_min
162 passing_target_length_min = length
164 if i_e_value > passing_target_ie_max
165 passing_target_ie_max = i_e_value
167 if i_e_value < passing_target_ie_min
168 passing_target_ie_min = i_e_value
170 if ( passing_domains_per_protein > max_domain_copy_number_per_protein )
171 max_domain_copy_number_sequence = sequence
172 max_domain_copy_number_per_protein = passing_domains_per_protein
175 log << domain_fail_counter.to_s + ": " + sequence.to_s + " fails threshold(s)"
176 if ( ( e_value_threshold.to_f >= 0.0 ) && ( i_e_value > e_value_threshold ) )
177 log << " iE=" + i_e_value.to_s
179 if ( ( length_threshold.to_f > 0 ) && ( env_to - env_from + 1 ) < length_threshold.to_f )
180 le = env_to - env_from + 1
181 log << " l=" + le.to_s
184 domain_fail_counter += 1
188 error_msg = "number > out_of (this should not have happened)"
189 raise StandardError, error_msg
193 if !hmmscan_datas.empty?
194 process_hmmscan_datas( hmmscan_datas,
204 out_msa_single_domains_protein_seqs,
205 out_msa_close_pairs_protein_seqs,
206 out_msa_close_pairs_only_protein_seqs,
207 out_msa_isolated_protein_seqs,
208 out_msa_isolated_only_protein_seqs,
209 out_msa_isolated_and_close_pair_protein_seqs )
210 domain_pass_counter += hmmscan_datas.length
211 if passed_seqs.find_by_name_start( sequence, true ).length < 1
212 add_sequence( sequence, in_msa, passed_seqs )
214 error_msg = "this should not have happened"
215 raise StandardError, error_msg
218 if failed_seqs.find_by_name_start( sequence, true ).length < 1
219 add_sequence( sequence, in_msa, failed_seqs )
220 proteins_with_failing_domains += 1
222 error_msg = "this should not have happened"
223 raise StandardError, error_msg
229 end # results.each do | r |
231 if (prev_query != nil) && (!saw_target)
232 log << domain_not_present_counter.to_s + ": " + prev_query.to_s + " lacks target domain" + ld
233 domain_not_present_counter += 1
236 if domain_pass_counter < 1
237 error_msg = "no domain sequences were extracted"
238 raise IOError, error_msg
241 if ( domain_not_present_counter + passed_seqs.get_number_of_seqs + proteins_with_failing_domains ) != protein_counter
242 error_msg = "not present + passing + not passing != proteins in sequence (fasta) file (this should not have happened)"
243 raise StandardError, error_msg
250 avg_pass = ( passing_target_length_sum / domain_pass_counter )
251 puts( "Passing target domain lengths: average: " + avg_pass.to_s )
252 log << "Passing target domain lengths: average: " + avg_pass.to_s
254 puts( "Passing target domain lengths: min-max: " + passing_target_length_min.to_s + " - " + passing_target_length_max.to_s)
255 log << "Passing target domain lengths: min-max: " + passing_target_length_min.to_s + " - " + passing_target_length_max.to_s
257 puts( "Passing target domain iE: min-max: " + passing_target_ie_min.to_s + " - " + passing_target_ie_max.to_s)
258 log << "Passing target domain iE: min-max: " + passing_target_ie_min.to_s + " - " + passing_target_ie_max.to_s
260 puts( "Passing target domains: sum: " + domain_pass_counter.to_s )
261 log << "Passing target domains: sum: " + domain_pass_counter.to_s
265 sum = domain_pass_counter + domain_fail_counter
266 avg_all = overall_target_length_sum / sum
267 puts( "All target domain lengths: average: " + avg_all.to_s )
268 log << "All target domain lengths: average: " + avg_all.to_s
270 puts( "All target domain lengths: min-max: " + overall_target_length_min.to_s + " - " + overall_target_length_max.to_s)
271 log << "All target domain lengths: min-max: " + overall_target_length_min.to_s + " - " + overall_target_length_max.to_s
273 puts( "All target target domain iE: min-max: " + overall_target_ie_min.to_s + " - " + overall_target_ie_max.to_s)
274 log << "All target target domain iE: min-max: " + overall_target_ie_min.to_s + " - " + overall_target_ie_max.to_s
276 puts( "All target domains: sum: " + sum.to_s )
277 log << "All target domains: sum: " + sum.to_s
280 puts( "Proteins with passing target domain(s): " + passed_seqs.get_number_of_seqs.to_s )
281 puts( "Proteins with no passing target domain: " + proteins_with_failing_domains.to_s )
282 puts( "Proteins with no target domain : " + domain_not_present_counter.to_s )
287 puts( "Max target domain copy number per protein: " + max_domain_copy_number_per_protein.to_s )
288 log << "Max target domain copy number per protein: " + max_domain_copy_number_per_protein.to_s
291 if ( max_domain_copy_number_per_protein > 1 )
292 puts( "First target protein with this copy number: " + max_domain_copy_number_sequence )
293 log << "First target protein with this copy number: " + max_domain_copy_number_sequence
297 write_msa( out_msa, outfile + ".fasta" )
298 write_msa( passed_seqs, passed_seqs_outfile )
299 write_msa( failed_seqs, failed_seqs_outfile )
302 write_msa( out_msa_pairs, outfile + "_" + min_linker.to_s + ".fasta")
306 write_msa( out_msa_singles, outfile + "_singles.fasta")
310 write_msa( out_msa_isolated, outfile + "_" + min_linker.to_s + "_isolated.fasta");
313 if out_msa_single_domains_protein_seqs
314 write_msa( out_msa_single_domains_protein_seqs, outfile + "_proteins_with_singles.fasta" )
317 if out_msa_close_pairs_protein_seqs
318 write_msa( out_msa_close_pairs_protein_seqs, outfile + "_" + min_linker.to_s + "_proteins_close_pairs.fasta" )
321 if out_msa_close_pairs_only_protein_seqs
322 write_msa( out_msa_close_pairs_only_protein_seqs, outfile + "_" + min_linker.to_s + "_proteins_with_close_pairs_only.fasta" )
325 if out_msa_isolated_protein_seqs
326 write_msa( out_msa_isolated_protein_seqs, outfile + "_" + min_linker.to_s + "_proteins_with_isolated_domains.fasta" )
329 if out_msa_isolated_only_protein_seqs
330 write_msa( out_msa_isolated_only_protein_seqs, outfile + "_" + min_linker.to_s + "_proteins_with_isolated_domains_only.fasta" )
333 if out_msa_isolated_and_close_pair_protein_seqs
334 write_msa( out_msa_isolated_and_close_pair_protein_seqs, outfile + "_" + min_linker.to_s + "_proteins_with_isolated_and_close_pairs.fasta" )
338 if ( out_msa_single_domains_protein_seqs.get_number_of_seqs +
339 out_msa_close_pairs_only_protein_seqs.get_number_of_seqs +
340 out_msa_isolated_only_protein_seqs.get_number_of_seqs +
341 out_msa_isolated_and_close_pair_protein_seqs.get_number_of_seqs ) != passed_seqs.get_number_of_seqs
342 error_msg = "this should not have happened"
343 raise StandardError, error_msg
348 log << "passing target domains : " + domain_pass_counter.to_s + ld
349 log << "failing target domains : " + domain_fail_counter.to_s + ld
350 log << "proteins in sequence (fasta) file : " + in_msa.get_number_of_seqs.to_s + ld
351 log << "proteins in hmmscan outputfile : " + protein_counter.to_s + ld
352 log << "proteins with passing target domain(s) : " + passed_seqs.get_number_of_seqs.to_s + ld
353 log << "proteins with no passing target domain : " + proteins_with_failing_domains.to_s + ld
354 log << "proteins with no target domain : " + domain_not_present_counter.to_s + ld
356 log << "min linker length : " + min_linker.to_s + ld
357 log << "single domains : " + out_msa_singles.get_number_of_seqs.to_s + ld
358 log << "domains in close pairs : " + (2 * out_msa_pairs.get_number_of_seqs).to_s + ld
359 log << "isolated domains : " + out_msa_isolated.get_number_of_seqs.to_s + ld
360 log << "proteins with single domains : " + out_msa_single_domains_protein_seqs.get_number_of_seqs.to_s + ld
361 log << "proteins with close pair domains : " + out_msa_close_pairs_protein_seqs.get_number_of_seqs.to_s + ld
362 log << "proteins with close pair domains only : " + out_msa_close_pairs_only_protein_seqs.get_number_of_seqs.to_s + ld
363 log << "proteins with isolated domains : " + out_msa_isolated_protein_seqs.get_number_of_seqs.to_s + ld
364 log << "proteins with isolated domains only : " + out_msa_isolated_only_protein_seqs.get_number_of_seqs.to_s + ld
365 log << "proteins with close pair and isolated domains: " + out_msa_isolated_and_close_pair_protein_seqs.get_number_of_seqs.to_s + ld
370 return domain_pass_counter
376 def write_msa( msa, filename )
378 w = FastaWriter.new()
379 w.set_line_width( 60 )
382 io.write_to_file( msa, filename, w )
384 error_msg = "could not write to \"" + filename + "\""
385 raise IOError, error_msg
389 def add_sequence( sequence_name, in_msa, add_to_msa )
390 seqs = in_msa.find_by_name_start( sequence_name, true )
391 if ( seqs.length < 1 )
392 error_msg = "sequence \"" + sequence_name + "\" not found in sequence file"
393 raise StandardError, error_msg
395 if ( seqs.length > 1 )
396 error_msg = "sequence \"" + sequence_name + "\" not unique in sequence file"
397 raise StandardError, error_msg
399 seq = in_msa.get_sequence( seqs[ 0 ] )
400 add_to_msa.add_sequence( seq )
403 def process_hmmscan_datas( hmmscan_datas,
413 out_msa_single_domains_protein_seqs,
414 out_msa_close_pairs_protein_seqs,
415 out_msa_close_pairs_only_protein_seqs,
416 out_msa_isolated_protein_seqs,
417 out_msa_isolated_only_protein_seqs,
418 out_msa_isolated_and_close_pair_protein_seqs )
420 actual_out_of = hmmscan_datas.size
421 saw_close_pair = false
427 hmmscan_datas.each_with_index do |hmmscan_data, index|
428 if hmmscan_data.number < ( index + 1 )
429 error_msg = "hmmscan_data.number < ( index + 1 ) (this should not have happened)"
430 raise StandardError, error_msg
433 seq_name = hmmscan_data.seq_name
435 extract_domain( seq_name,
438 hmmscan_data.env_from,
447 if actual_out_of == 1
448 extract_domain( seq_name,
451 hmmscan_data.env_from,
458 if out_msa_single_domains_protein_seqs.find_by_name_start( seq_name, true ).length < 1
459 add_sequence( seq_name, in_msa, out_msa_single_domains_protein_seqs )
461 error_msg = "this should not have happened"
462 raise StandardError, error_msg
467 last = index == hmmscan_datas.length - 1
469 if ( ( first && ( ( hmmscan_datas[ index + 1 ].env_from - hmmscan_data.env_to ) > min_linker) ) ||
470 ( last && ( ( hmmscan_data.env_from - hmmscan_datas[ index - 1 ].env_to ) > min_linker ) ) ||
471 ( !first && !last && ( ( hmmscan_datas[ index + 1 ].env_from - hmmscan_data.env_to ) > min_linker ) &&
472 ( ( hmmscan_data.env_from - hmmscan_datas[ index - 1 ].env_to ) > min_linker ) ) )
474 extract_domain( seq_name,
477 hmmscan_data.env_from,
488 from = hmmscan_datas[ index - 1 ].env_from
489 to = hmmscan_data.env_to
491 if ADD_TO_CLOSE_PAIRS > 0
492 from = from - ADD_TO_CLOSE_PAIRS
496 to = to + ADD_TO_CLOSE_PAIRS
497 temp_seqs = in_msa.find_by_name_start( seq_name, true )
498 temp_seq = in_msa.get_sequence( temp_seqs[ 0 ] )
499 if to > temp_seq.get_length
500 to = temp_seq.get_length
504 extract_domain( seq_name,
505 index.to_s + "+" + ( index + 1 ).to_s,
514 saw_close_pair = true
518 if prev_seq_name && prev_seq_name != seq_name
519 error_msg = "this should not have happened"
520 raise StandardError, error_msg
522 prev_seq_name = seq_name
525 if out_msa_isolated_protein_seqs.find_by_name_start( seq_name, true ).length < 1
526 add_sequence( seq_name, in_msa, out_msa_isolated_protein_seqs )
528 error_msg = "this should not have happened"
529 raise StandardError, error_msg
533 if out_msa_close_pairs_protein_seqs.find_by_name_start( seq_name, true ).length < 1
534 add_sequence( seq_name, in_msa, out_msa_close_pairs_protein_seqs )
536 error_msg = "this should not have happened"
537 raise StandardError, error_msg
540 if saw_close_pair && saw_isolated
541 if out_msa_isolated_and_close_pair_protein_seqs.find_by_name_start( seq_name, true ).length < 1
542 add_sequence( seq_name, in_msa, out_msa_isolated_and_close_pair_protein_seqs )
544 error_msg = "this should not have happened"
545 raise StandardError, error_msg
548 if out_msa_close_pairs_only_protein_seqs.find_by_name_start( seq_name, true ).length < 1
549 add_sequence( seq_name, in_msa, out_msa_close_pairs_only_protein_seqs )
551 error_msg = "this should not have happened"
552 raise StandardError, error_msg
555 if out_msa_isolated_only_protein_seqs.find_by_name_start( seq_name, true ).length < 1
556 add_sequence( seq_name, in_msa, out_msa_isolated_only_protein_seqs )
558 error_msg = "this should not have happened"
559 raise StandardError, error_msg
562 end # def process_hmmscan_data
564 def extract_domain( sequence,
574 if number.is_a?( Fixnum ) && ( number < 1 || out_of < 1 || number > out_of )
575 error_msg = "number=" + number.to_s + ", out of=" + out_of.to_s
576 raise StandardError, error_msg
578 if seq_from < 1 || seq_to < 1 || seq_from >= seq_to
579 error_msg = "impossible: seq-from=" + seq_from.to_s + ", seq-to=" + seq_to.to_s
580 raise StandardError, error_msg
582 seqs = in_msa.find_by_name_start( sequence, true )
584 error_msg = "sequence \"" + sequence + "\" not found in sequence file"
585 raise IOError, error_msg
588 error_msg = "sequence \"" + sequence + "\" not unique in sequence file"
589 raise IOError, error_msg
591 # hmmscan is 1 based, whereas sequences are 0 bases in this package.
592 seq = in_msa.get_sequence( seqs[ 0 ] ).get_subsequence( seq_from - 1, seq_to - 1 )
594 orig_name = seq.get_name
596 seq.set_name( orig_name.split[ 0 ] )
599 seq.set_name( seq.get_name + "_" + seq_from.to_s + "-" + seq_to.to_s )
602 if out_of != 1 && add_domain_number
603 seq.set_name( seq.get_name + "~" + number.to_s + "-" + out_of.to_s )
607 a = orig_name.rindex "["
608 b = orig_name.rindex "]"
610 error_msg = "species not found in " + orig_name
611 raise StandardError, error_msg
613 species = orig_name[ a .. b ]
614 seq.set_name( seq.get_name + " " + species )
616 out_msa.add_sequence( seq )
619 def is_ignorable?( line )
620 return ( line !~ /[A-Za-z0-9-]/ || line =~/^#/ )
623 end # class HmmscanDomainExtractor
626 def initialize( seq_name, number, out_of, env_from, env_to, i_e_value )
632 @i_e_value = i_e_value
634 attr_reader :seq_name, :number, :out_of, :env_from, :env_to, :i_e_value