3 import java.io.IOException;
4 import java.text.ParseException;
5 import java.util.ArrayList;
6 import java.util.HashMap;
7 import java.util.Hashtable;
10 import java.util.Map.Entry;
12 import jalview.bin.Cache;
13 import jalview.datamodel.DBRefEntry;
14 import jalview.datamodel.FeatureProperties;
15 import jalview.datamodel.Mapping;
16 import jalview.datamodel.Sequence;
17 import jalview.datamodel.SequenceFeature;
18 import jalview.datamodel.SequenceI;
19 import jalview.util.DBRefUtils;
20 import jalview.util.DnaUtils;
21 import jalview.util.MappingUtils;
24 * A class that provides selective parsing of the EMBL flatfile format.
26 * The initial implementation is limited to extracting fields used by Jalview
27 * after fetching an EMBL or EMBLCDS entry:
30 * accession, version, sequence, xref
31 * and (for CDS feature) location, protein_id, product, codon_start, translation
34 * For a complete parser, it may be best to adopt that provided in
35 * https://github.com/enasequence/sequencetools/tree/master/src/main/java/uk/ac/ebi/embl/flatfile
36 * (but note this has a dependency on the Apache Commons library)
39 * @see ftp://ftp.ebi.ac.uk/pub/databases/ena/sequence/release/doc/usrman.txt
40 * @see ftp://ftp.ebi.ac.uk/pub/databases/embl/doc/FT_current.html
42 public class EmblFlatFile extends AlignFile // FileParse
45 * A data bean class to hold values parsed from one CDS Feature (FT)
49 String translation; // from CDS feature /translation
51 String cdsLocation; // CDS /location raw value
53 int codonStart = 1; // from CDS /codon_start
55 String proteinName; // from CDS /product; used for protein description
57 String proteinId; // from CDS /protein_id
59 Map<String, String> cdsProps = new Hashtable<>(); // CDS other qualifiers
62 private static final String WHITESPACE = "\\s+";
64 private String sourceDb;
67 * values parsed from the EMBL flatfile record
69 private String accession; // from ID (first token)
71 private String version; // from ID (second token)
73 private String description; // from (first) DE line
75 private int length = 128; // from ID (7th token), with usable default
77 private List<DBRefEntry> dbrefs; // from DR and also CDS /db_xref qualifiers
79 private String sequenceString; // from SQ lines
81 private List<CdsData> cds;
90 public EmblFlatFile(FileParse fp, String sourceId) throws IOException
92 super(false, fp); // don't parse immediately
93 this.sourceDb = sourceId;
94 dbrefs = new ArrayList<>();
95 cds = new ArrayList<>();
99 * Parses the flatfile, and if successful, saves as an annotated sequence
100 * which may be retrieved by calling {@code getSequence()}
102 * @throws IOException
104 public void parse() throws IOException
106 String line = nextLine();
109 if (line.startsWith("ID"))
111 line = parseID(line);
113 else if (line.startsWith("DE"))
115 line = parseDE(line);
117 else if (line.startsWith("DR"))
119 line = parseDR(line);
121 else if (line.startsWith("SQ"))
125 else if (line.startsWith("FT"))
127 line = parseFT(line);
138 * Extracts and saves the primary accession and version (SV value) from an ID
139 * line, or null if not found. Returns the next line after the one processed.
142 * @throws IOException
144 String parseID(String line) throws IOException
146 String[] tokens = line.substring(2).split(";");
149 * first is primary accession
151 String token = tokens[0].trim();
152 if (!token.isEmpty())
154 this.accession = token;
158 * second token is 'SV versionNo'
160 if (tokens.length > 1)
162 token = tokens[1].trim();
163 if (token.startsWith("SV"))
165 String[] bits = token.trim().split(WHITESPACE);
166 this.version = bits[bits.length - 1];
171 * seventh token is 'length BP'
173 if (tokens.length > 6)
175 token = tokens[6].trim();
176 String[] bits = token.trim().split(WHITESPACE);
179 this.length = Integer.valueOf(bits[0]);
180 } catch (NumberFormatException e)
182 Cache.log.error("bad length read in flatfile, line: " + line);
190 * Reads sequence description from the first DE line found. Any trailing
191 * period is discarded. If there are multiple DE lines, only the first (short
192 * description) is read, the rest are ignored.
196 * @throws IOException
198 String parseDE(String line) throws IOException
200 String desc = line.substring(2).trim();
201 if (desc.endsWith("."))
203 desc = desc.substring(0, desc.length() - 1);
205 this.description = desc;
208 * pass over any additional DE lines
210 while ((line = nextLine()) != null)
212 if (!line.startsWith("DE"))
222 * Processes one DR line and saves as a DBRefEntry cross-reference. Returns
223 * the line following the line processed.
226 * @throws IOException
228 String parseDR(String line) throws IOException
230 String[] tokens = line.substring(2).split(";");
231 if (tokens.length > 1)
234 * ensure UniProtKB/Swiss-Prot converted to UNIPROT
236 String db = tokens[0].trim();
237 db = DBRefUtils.getCanonicalName(db);
238 String acc = tokens[1].trim();
239 if (acc.endsWith("."))
241 acc = acc.substring(0, acc.length() - 1);
243 String version = "0";
244 if (tokens.length > 2)
246 String secondaryId = tokens[2].trim();
247 if (!secondaryId.isEmpty())
249 // todo: is this right? secondary id is not a version number
250 // version = secondaryId;
253 this.dbrefs.add(new DBRefEntry(db, version, acc));
260 * Reads and saves the sequence, read from the lines following the SQ line.
261 * Whitespace and position counters are discarded. Returns the next line
262 * following the sequence data (the next line that doesn't start with
265 * @throws IOException
267 String parseSQ() throws IOException
269 StringBuilder sb = new StringBuilder(this.length);
270 String line = nextLine();
271 while (line != null && line.startsWith(" "))
274 String[] blocks = line.split(WHITESPACE);
277 * omit the last block (position counter) on each line
279 for (int i = 0; i < blocks.length - 1; i++)
281 sb.append(blocks[i]);
285 this.sequenceString = sb.toString();
291 * Processes an FT line. If it declares a feature type of interest (currently,
292 * only CDS is processed), processes all of the associated lines (feature
293 * qualifiers), and returns the next line after that, otherwise simply returns
298 * @throws IOException
300 String parseFT(String line) throws IOException
302 String[] tokens = line.split(WHITESPACE);
303 if (tokens.length < 3 || !"CDS".equals(tokens[1]))
308 CdsData data = new CdsData();
309 data.cdsLocation = tokens[2];
314 if (!line.startsWith("FT ")) // 4 spaces
316 // e.g. start of next feature "FT source..."
321 * extract qualifier, e.g. FT /protein_id="CAA37824.1"
323 int slashPos = line.indexOf('/');
326 Cache.log.error("Unexpected EMBL line ignored: " + line);
329 int eqPos = line.indexOf('=', slashPos + 1);
332 Cache.log.error("Unexpected EMBL line ignored: " + line);
335 String qualifier = line.substring(slashPos + 1, eqPos);
336 String value = line.substring(eqPos + 1);
337 if (value.startsWith("\"") && value.endsWith("\""))
339 value = value.substring(1, value.length() - 1);
342 if ("protein_id".equals(qualifier))
344 data.proteinId = value;
347 else if ("codon_start".equals(qualifier))
351 data.codonStart = Integer.parseInt(value.trim());
352 } catch (NumberFormatException e)
354 Cache.log.error("Invalid codon_start in XML for " + this.accession
355 + ": " + e.getMessage());
359 else if ("db_xref".equals(qualifier))
361 String[] parts = value.split(":");
362 if (parts.length == 2)
364 String db = parts[0].trim();
365 db = DBRefUtils.getCanonicalName(db);
366 DBRefEntry dbref = new DBRefEntry(db, "0", parts[1].trim());
367 this.dbrefs.add(dbref);
371 else if ("product".equals(qualifier))
373 // sometimes name is returned e.g. for V00488
374 data.proteinName = value;
377 else if ("translation".equals(qualifier))
379 line = readTranslation(value, data);
381 else if (!"".equals(value))
383 // throw anything else into the additional properties hash
384 data.cdsProps.put(qualifier, value);
395 * Reads and returns the CDS translation from one or more lines of the file,
396 * and returns the next line after that
399 * the first line of the translation (likely quoted)
402 * @throws IOException
404 String readTranslation(String value, CdsData data) throws IOException
406 StringBuilder sb = new StringBuilder(this.length / 3 + 1);
407 sb.append(value.replace("\"", ""));
410 while ((line = nextLine()) != null)
412 if (!line.startsWith("FT "))
414 break; // reached next feature or other input line
416 String[] tokens = line.split(WHITESPACE);
417 if (tokens.length < 2)
419 Cache.log.error("Ignoring bad EMBL line: " + line);
422 if (tokens[1].startsWith("/"))
424 break; // next feature qualifier
426 sb.append(tokens[1].replace("\"", ""));
429 data.translation = sb.toString();
435 * Constructs and saves the sequence from parsed components
437 void assembleSequence()
439 String name = this.accession;
440 if (this.sourceDb != null)
442 name = this.sourceDb + "|" + name;
444 SequenceI seq = new Sequence(name, this.sequenceString);
445 seq.setDescription(this.description);
448 * add a DBRef to itself
450 DBRefEntry selfRef = new DBRefEntry(sourceDb, version, accession);
451 int[] startEnd = new int[] { 1, seq.getLength() };
452 selfRef.setMap(new Mapping(null, startEnd, startEnd, 1, 1));
453 seq.addDBRef(selfRef);
455 for (DBRefEntry dbref : this.dbrefs)
462 seq.deriveSequence();
468 * Process the CDS features, including generation of cross-references and
469 * mappings to the protein products (translation)
473 protected void processAllCDS(SequenceI seq)
476 * record protein products found to avoid duplication i.e. >1 CDS with
477 * the same /protein_id [though not sure I can find an example of this]
479 Map<String, SequenceI> proteins = new HashMap<>();
480 for (CdsData data : cds)
482 processOneCDS(seq, data, proteins);
487 * Processes the parsed CDS feature data to
489 * <li>add a CDS feature to the sequence for each CDS start-end range</li>
490 * <li>create a protein product sequence for the translation</li>
491 * <li>create a cross-reference to protein with mapping from dna</li>
492 * <li>add any CDS dbrefs to the sequence and to the protein product</li>
498 * map of protein products so far derived from CDS data
500 void processOneCDS(SequenceI dna, CdsData data,
501 Map<String, SequenceI> proteins)
504 * parse location into a list of [start, end, start, end] positions
506 int[] exons = getCdsRanges(this.accession, data.cdsLocation);
509 for (int xint = 0; exons != null && xint < exons.length - 1; xint += 2)
511 int exonStart = exons[xint];
512 int exonEnd = exons[xint + 1];
513 int begin = Math.min(exonStart, exonEnd);
514 int end = Math.max(exonStart, exonEnd);
516 String desc = String.format("Exon %d for protein EMBLCDS:%s",
517 exonNumber, data.proteinId);
519 SequenceFeature sf = new SequenceFeature("CDS", desc, begin, end,
521 for (Entry<String, String> val : data.cdsProps.entrySet())
523 sf.setValue(val.getKey(), val.getValue());
526 sf.setEnaLocation(data.cdsLocation);
527 boolean forwardStrand = exonStart <= exonEnd;
528 sf.setStrand(forwardStrand ? "+" : "-");
529 sf.setPhase(String.valueOf(data.codonStart - 1));
530 sf.setValue(FeatureProperties.EXONPOS, exonNumber);
531 sf.setValue(FeatureProperties.EXONPRODUCT, data.proteinName);
533 dna.addSequenceFeature(sf);
535 linkProteinProduct(dna, data, proteins);
540 * Constructs a sequence for the protein product for the CDS data (if there is
541 * one), and dbrefs with mappings from CDS to protein and the reverse
547 void linkProteinProduct(SequenceI dna, CdsData data, Map<String, SequenceI> proteins)
550 * check we have some data to work with
552 if (data.proteinId == null || data.translation == null)
558 * Construct the protein sequence (if not already seen)
560 SequenceI protein = proteins.get(data.proteinId);
563 protein = new Sequence(data.proteinId, data.translation, 1,
564 data.translation.length());
565 protein.setDescription(data.proteinName != null ? data.proteinName
566 : "Protein Product from " + sourceDb);
567 proteins.put(data.proteinId, protein);
572 * Returns the CDS location as a single array of [start, end, start, end...]
573 * positions. If on the reverse strand, these will be in descending order.
579 protected int[] getCdsRanges(String accession, String location)
581 if (location == null)
588 List<int[]> ranges = DnaUtils.parseLocation(location);
589 return MappingUtils.listToArray(ranges);
590 } catch (ParseException e)
593 String.format("Not parsing inexact CDS location %s in ENA %s",
594 location, accession));
600 * Output (print) is not implemented for EMBL flat file format
603 public String print(SequenceI[] seqs, boolean jvsuffix)