3 import java.io.IOException;
4 import java.text.ParseException;
5 import java.util.ArrayList;
6 import java.util.Arrays;
7 import java.util.HashMap;
8 import java.util.Hashtable;
11 import java.util.Map.Entry;
12 import java.util.TreeMap;
14 import jalview.bin.Cache;
15 import jalview.datamodel.DBRefEntry;
16 import jalview.datamodel.DBRefSource;
17 import jalview.datamodel.FeatureProperties;
18 import jalview.datamodel.Mapping;
19 import jalview.datamodel.Sequence;
20 import jalview.datamodel.SequenceFeature;
21 import jalview.datamodel.SequenceI;
22 import jalview.util.DBRefUtils;
23 import jalview.util.DnaUtils;
24 import jalview.util.MapList;
25 import jalview.util.MappingUtils;
28 * A base class to support parsing of GenBank, EMBL or DDBJ flat file format
29 * data. Example files (rather than formal specifications) are provided at
32 * https://ena-docs.readthedocs.io/en/latest/submit/fileprep/flat-file-example.html
33 * https://www.ncbi.nlm.nih.gov/Sitemap/samplerecord.html
36 * or to compare the same entry, see
39 * https://www.ebi.ac.uk/ena/browser/api/embl/X81322.1
40 * https://www.ncbi.nlm.nih.gov/nuccore/X81322.1
43 * The feature table part of the file has a common definition, only the start of
44 * each line is formatted differently in GenBank and EMBL. See
45 * http://www.insdc.org/files/feature_table.html#7.1.
47 public abstract class FlatFile extends AlignFile
49 protected static final String LOCATION = "location";
51 protected static final String QUOTE = "\"";
53 protected static final String DOUBLED_QUOTE = QUOTE + QUOTE;
55 protected static final String WHITESPACE = "\\s+";
58 * Removes leading or trailing double quotes (") unless doubled, and changes
59 * any 'escaped' (doubled) double quotes to single characters. As per the
60 * Feature Table specification for Qualifiers, Free Text.
65 protected static String removeQuotes(String value)
71 if (value.startsWith(QUOTE) && !value.startsWith(DOUBLED_QUOTE))
73 value = value.substring(1);
75 if (value.endsWith(QUOTE) && !value.endsWith(DOUBLED_QUOTE))
77 value = value.substring(0, value.length() - 1);
79 value = value.replace(DOUBLED_QUOTE, QUOTE);
84 * Truncates (if necessary) the exon intervals to match 3 times the length of
85 * the protein; also accepts 3 bases longer (for stop codon not included in
88 * @param proteinLength
90 * an array of [start, end, start, end...] intervals
91 * @return the same array (if unchanged) or a truncated copy
93 protected static int[] adjustForProteinLength(int proteinLength,
96 if (proteinLength <= 0 || exon == null)
100 int expectedCdsLength = proteinLength * 3;
101 int exonLength = MappingUtils.getLength(Arrays.asList(exon));
104 * if exon length matches protein, or is shorter, or longer by the
105 * length of a stop codon (3 bases), then leave it unchanged
107 if (expectedCdsLength >= exonLength
108 || expectedCdsLength == exonLength - 3)
116 origxon = new int[exon.length];
117 System.arraycopy(exon, 0, origxon, 0, exon.length);
119 for (int x = 0; x < exon.length; x += 2)
121 cdspos += Math.abs(exon[x + 1] - exon[x]) + 1;
122 if (expectedCdsLength <= cdspos)
124 // advanced beyond last codon.
126 if (expectedCdsLength != cdspos)
129 // .println("Truncating final exon interval on region by "
130 // + (cdspos - cdslength));
134 * shrink the final exon - reduce end position if forward
135 * strand, increase it if reverse
137 if (exon[x + 1] >= exon[x])
139 endxon = exon[x + 1] - cdspos + expectedCdsLength;
143 endxon = exon[x + 1] + cdspos - expectedCdsLength;
151 // and trim the exon interval set if necessary
152 int[] nxon = new int[sxpos + 2];
153 System.arraycopy(exon, 0, nxon, 0, sxpos + 2);
154 nxon[sxpos + 1] = endxon; // update the end boundary for the new exon
162 * values parsed from the data file
164 protected String sourceDb;
166 protected String accession;
168 protected String version;
170 protected String description;
172 protected int length = 128;
174 protected List<DBRefEntry> dbrefs;
176 protected String sequenceString;
178 protected Map<String, CdsData> cds;
185 * @throws IOException
187 public FlatFile(FileParse fp, String sourceId) throws IOException
189 super(false, fp); // don't parse immediately
190 this.sourceDb = sourceId;
191 dbrefs = new ArrayList<>();
194 * using TreeMap gives CDS sequences in alphabetical, so readable, order
196 cds = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
200 * Parses one (GenBank or EMBL format) CDS feature, saves the parsed data, and
201 * returns the next line
205 * @throws IOException
207 protected String parseCDSFeature(String location) throws IOException
212 * parse location, which can be over >1 line e.g. EAW51554
214 CdsData data = new CdsData();
215 StringBuilder sb = new StringBuilder().append(location);
216 line = parseFeatureQualifier(sb, LOCATION);
217 data.cdsLocation = sb.toString();
221 if (!isFeatureContinuationLine(line))
223 // e.g. start of next feature "FT source..."
228 * extract qualifier, e.g. FT /protein_id="CAA37824.1"
229 * - the value may extend over more than one line
230 * - if the value has enclosing quotes, these are removed
231 * - escaped double quotes ("") are reduced to a single character
233 int slashPos = line.indexOf('/');
236 Cache.log.error("Unexpected EMBL line ignored: " + line);
240 int eqPos = line.indexOf('=', slashPos + 1);
243 // can happen, e.g. /ribosomal_slippage
247 String qualifier = line.substring(slashPos + 1, eqPos);
248 String value = line.substring(eqPos + 1);
249 value = removeQuotes(value);
250 sb = new StringBuilder().append(value);
251 line = parseFeatureQualifier(sb, qualifier);
252 String featureValue = sb.toString();
254 if ("protein_id".equals(qualifier))
256 data.proteinId = featureValue;
258 else if ("codon_start".equals(qualifier))
262 data.codonStart = Integer.parseInt(featureValue.trim());
263 } catch (NumberFormatException e)
265 Cache.log.error("Invalid codon_start in XML for " + this.accession
266 + ": " + e.getMessage());
269 else if ("db_xref".equals(qualifier))
271 String[] parts = featureValue.split(":");
272 if (parts.length == 2)
274 String db = parts[0].trim();
275 db = DBRefUtils.getCanonicalName(db);
276 DBRefEntry dbref = new DBRefEntry(db, "0", parts[1].trim());
277 data.xrefs.add(dbref);
280 else if ("product".equals(qualifier))
282 data.proteinName = featureValue;
284 else if ("translation".equals(qualifier))
286 data.translation = featureValue;
288 else if (!"".equals(featureValue))
290 // throw anything else into the additional properties hash
291 data.cdsProps.put(qualifier, featureValue);
295 if (data.proteinId != null)
297 this.cds.put(data.proteinId, data);
301 Cache.log.error("Ignoring CDS feature with no protein_id for "
302 + sourceDb + ":" + accession);
308 protected abstract boolean isFeatureContinuationLine(String line);
311 * Output (print) is not (yet) implemented for flat file format
314 public String print(SequenceI[] seqs, boolean jvsuffix)
320 * Constructs and saves the sequence from parsed components
322 protected void buildSequence()
324 if (this.accession == null || this.sequenceString == null)
326 Cache.log.error("Failed to parse data from EMBL");
330 String name = this.accession;
331 if (this.sourceDb != null)
333 name = this.sourceDb + "|" + name;
335 SequenceI seq = new Sequence(name, this.sequenceString);
336 seq.setDescription(this.description);
339 * add a DBRef to itself
341 DBRefEntry selfRef = new DBRefEntry(sourceDb, version, accession);
342 int[] startEnd = new int[] { 1, seq.getLength() };
343 selfRef.setMap(new Mapping(null, startEnd, startEnd, 1, 1));
344 seq.addDBRef(selfRef);
346 for (DBRefEntry dbref : this.dbrefs)
351 processCDSFeatures(seq);
353 seq.deriveSequence();
359 * Process the CDS features, including generation of cross-references and
360 * mappings to the protein products (translation)
364 protected void processCDSFeatures(SequenceI seq)
367 * record protein products found to avoid duplication i.e. >1 CDS with
368 * the same /protein_id [though not sure I can find an example of this]
370 Map<String, SequenceI> proteins = new HashMap<>();
371 for (CdsData data : cds.values())
373 processCDSFeature(seq, data, proteins);
378 * Processes data for one parsed CDS feature to
380 * <li>create a protein product sequence for the translation</li>
381 * <li>create a cross-reference to protein with mapping from dna</li>
382 * <li>add a CDS feature to the sequence for each CDS start-end range</li>
383 * <li>add any CDS dbrefs to the sequence and to the protein product</li>
389 * map of protein products so far derived from CDS data
391 void processCDSFeature(SequenceI dna, CdsData data,
392 Map<String, SequenceI> proteins)
395 * parse location into a list of [start, end, start, end] positions
397 int[] exons = getCdsRanges(this.accession, data.cdsLocation);
399 MapList maplist = buildMappingToProtein(dna, exons, data);
403 for (int xint = 0; exons != null && xint < exons.length - 1; xint += 2)
405 int exonStart = exons[xint];
406 int exonEnd = exons[xint + 1];
407 int begin = Math.min(exonStart, exonEnd);
408 int end = Math.max(exonStart, exonEnd);
410 String desc = String.format("Exon %d for protein EMBLCDS:%s",
411 exonNumber, data.proteinId);
413 SequenceFeature sf = new SequenceFeature("CDS", desc, begin, end,
415 for (Entry<String, String> val : data.cdsProps.entrySet())
417 sf.setValue(val.getKey(), val.getValue());
420 sf.setEnaLocation(data.cdsLocation);
421 boolean forwardStrand = exonStart <= exonEnd;
422 sf.setStrand(forwardStrand ? "+" : "-");
423 sf.setPhase(String.valueOf(data.codonStart - 1));
424 sf.setValue(FeatureProperties.EXONPOS, exonNumber);
425 sf.setValue(FeatureProperties.EXONPRODUCT, data.proteinName);
427 dna.addSequenceFeature(sf);
430 boolean hasUniprotDbref = false;
431 for (DBRefEntry xref : data.xrefs)
434 if (xref.getSource().equals(DBRefSource.UNIPROT))
437 * construct (or find) the sequence for (data.protein_id, data.translation)
439 SequenceI protein = buildProteinProduct(dna, xref, data, proteins);
440 Mapping map = new Mapping(protein, maplist);
441 map.setMappedFromId(data.proteinId);
445 * add DBRefs with mappings from dna to protein and the inverse
447 DBRefEntry db1 = new DBRefEntry(sourceDb, version, accession);
448 db1.setMap(new Mapping(dna, maplist.getInverse()));
449 protein.addDBRef(db1);
451 hasUniprotDbref = true;
456 * if we have a product (translation) but no explicit Uniprot dbref
457 * (example: EMBL M19487 protein_id AAB02592.1)
458 * then construct mappings to an assumed EMBLCDSPROTEIN accession
460 if (!hasUniprotDbref)
462 SequenceI protein = proteins.get(data.proteinId);
465 protein = new Sequence(data.proteinId, data.translation);
466 protein.setDescription(data.proteinName);
467 proteins.put(data.proteinId, protein);
469 // assuming CDSPROTEIN sequence version = dna version (?!)
470 DBRefEntry db1 = new DBRefEntry(DBRefSource.EMBLCDSProduct,
471 this.version, data.proteinId);
472 protein.addDBRef(db1);
474 DBRefEntry dnaToEmblProteinRef = new DBRefEntry(
475 DBRefSource.EMBLCDSProduct, this.version, data.proteinId);
476 Mapping map = new Mapping(protein, maplist);
477 map.setMappedFromId(data.proteinId);
478 dnaToEmblProteinRef.setMap(map);
479 dna.addDBRef(dnaToEmblProteinRef);
483 * comment brought forward from EmblXmlSource, lines 447-451:
484 * TODO: if retrieved from EMBLCDS, add a DBRef back to the parent EMBL
485 * sequence with the exon map; if given a dataset reference, search
486 * dataset for parent EMBL sequence if it exists and set its map;
487 * make a new feature annotating the coding contig
492 * Computes a mapping from CDS positions in DNA sequence to protein product
493 * positions, with allowance for stop codon or incomplete start codon
500 MapList buildMappingToProtein(final SequenceI dna, final int[] exons,
503 MapList dnaToProteinMapping = null;
504 int peptideLength = data.translation.length();
506 int[] proteinRange = new int[] { 1, peptideLength };
507 if (exons != null && exons.length > 0)
510 * We were able to parse 'location'; do a final
511 * product length truncation check
513 int[] cdsRanges = adjustForProteinLength(peptideLength, exons);
514 dnaToProteinMapping = new MapList(cdsRanges, proteinRange, 3, 1);
519 * workaround until we handle all 'location' formats fully
520 * e.g. X53828.1:60..1058 or <123..>289
522 Cache.log.error(String.format(
523 "Implementation Notice: EMBLCDS location '%s'not properly supported yet"
524 + " - Making up the CDNA region of (%s:%s)... may be incorrect",
525 data.cdsLocation, sourceDb, this.accession));
527 int completeCodonsLength = 1 - data.codonStart + dna.getLength();
528 int mappedDnaEnd = dna.getEnd();
529 if (peptideLength * 3 == completeCodonsLength)
531 // this might occur for CDS sequences where no features are marked
532 Cache.log.warn("Assuming no stop codon at end of cDNA fragment");
533 mappedDnaEnd = dna.getEnd();
535 else if ((peptideLength + 1) * 3 == completeCodonsLength)
537 Cache.log.warn("Assuming stop codon at end of cDNA fragment");
538 mappedDnaEnd = dna.getEnd() - 3;
541 if (mappedDnaEnd != -1)
543 int[] cdsRanges = new int[] {
544 dna.getStart() + (data.codonStart - 1), mappedDnaEnd };
545 dnaToProteinMapping = new MapList(cdsRanges, proteinRange, 3, 1);
549 return dnaToProteinMapping;
553 * Constructs a sequence for the protein product for the CDS data (if there is
554 * one), and dbrefs with mappings from CDS to protein and the reverse
562 SequenceI buildProteinProduct(SequenceI dna, DBRefEntry xref,
563 CdsData data, Map<String, SequenceI> proteins)
566 * check we have some data to work with
568 if (data.proteinId == null || data.translation == null)
574 * Construct the protein sequence (if not already seen)
576 String proteinSeqName = xref.getSource() + "|" + xref.getAccessionId();
577 SequenceI protein = proteins.get(proteinSeqName);
580 protein = new Sequence(proteinSeqName, data.translation, 1,
581 data.translation.length());
582 protein.setDescription(data.proteinName != null ? data.proteinName
583 : "Protein Product from " + sourceDb);
584 proteins.put(proteinSeqName, protein);
591 * Returns the CDS location as a single array of [start, end, start, end...]
592 * positions. If on the reverse strand, these will be in descending order.
598 protected int[] getCdsRanges(String accession, String location)
600 if (location == null)
607 List<int[]> ranges = DnaUtils.parseLocation(location);
608 return MappingUtils.rangeListToArray(ranges);
609 } catch (ParseException e)
612 String.format("Not parsing inexact CDS location %s in ENA %s",
613 location, accession));
619 * Reads the value of a feature (FT) qualifier from one or more lines of the
620 * file, and returns the next line after that. Values are appended to the
621 * string buffer, which should be already primed with the value read from the
622 * first line for the qualifier (with any leading double quote removed).
623 * Enclosing double quotes are removed, and escaped (repeated) double quotes
624 * reduced to one only. For example for
627 * FT /note="gene_id=hCG28070.3
628 * FT ""foobar"" isoform=CRA_b"
629 * the returned value is
630 * gene_id=hCG28070.3 "foobar" isoform=CRA_b
633 * Note the side-effect of this method, to advance data reading to the next
634 * line after the feature qualifier (which could be another qualifier, a
635 * different feature, a non-feature line, or null at end of file).
638 * a string buffer primed with the first line of the value
639 * @param qualifierName
641 * @throws IOException
643 String parseFeatureQualifier(StringBuilder sb, String qualifierName)
647 while ((line = nextLine()) != null)
649 if (!isFeatureContinuationLine(line))
651 break; // reached next feature or other input line
653 String[] tokens = line.split(WHITESPACE);
654 if (tokens.length < 2)
656 Cache.log.error("Ignoring bad EMBL line for " + this.accession
660 if (tokens[1].startsWith("/"))
662 break; // next feature qualifier
666 * heuristic rule: most multi-line value (e.g. /product) are text,
667 * so add a space for word boundary at a new line; not for translation
669 if (!"translation".equals(qualifierName)
670 && !LOCATION.equals(qualifierName))
676 * remove trailing " and unescape doubled ""
678 String data = removeQuotes(tokens[1]);
686 * Reads and saves the sequence, read from the lines following the ORIGIN
687 * (GenBank) or SQ (EMBL) line. Whitespace and position counters are
688 * discarded. Returns the next line following the sequence data (the next line
689 * that doesn't start with whitespace).
691 * @throws IOException
693 protected String parseSequence() throws IOException
695 StringBuilder sb = new StringBuilder(this.length);
696 String line = nextLine();
697 while (line != null && line.startsWith(" "))
700 String[] blocks = line.split(WHITESPACE);
703 * the first or last block on each line might be a position count - omit
705 for (int i = 0; i < blocks.length; i++)
709 Long.parseLong(blocks[i]);
710 // position counter - ignore it
711 } catch (NumberFormatException e)
713 // sequence data - append it
714 sb.append(blocks[i]);
719 this.sequenceString = sb.toString();
725 * Processes a feature line. If it declares a feature type of interest
726 * (currently, only CDS is processed), processes all of the associated lines
727 * (feature qualifiers), and returns the next line after that, otherwise
728 * simply returns the next line.
731 * the first line for the feature (with initial FT omitted for EMBL
734 * @throws IOException
736 protected String parseFeature(String line) throws IOException
738 String[] tokens = line.trim().split(WHITESPACE);
739 if (tokens.length < 2 || !"CDS".equals(tokens[0]))
744 return parseCDSFeature(tokens[1]);
749 * A data bean class to hold values parsed from one CDS Feature
753 String translation; // from /translation qualifier
755 String cdsLocation; // the raw value e.g. join(1..1234,2012..2837)
757 int codonStart = 1; // from /codon_start qualifier
759 String proteinName; // from /product qualifier; used for protein description
761 String proteinId; // from /protein_id qualifier
763 List<DBRefEntry> xrefs = new ArrayList<>(); // from /db_xref qualifiers
765 Map<String, String> cdsProps = new Hashtable<>(); // other qualifiers