import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.TreeMap;
import jalview.bin.Cache;
import jalview.datamodel.DBRefEntry;
+import jalview.datamodel.DBRefSource;
import jalview.datamodel.FeatureProperties;
import jalview.datamodel.Mapping;
import jalview.datamodel.Sequence;
import jalview.datamodel.SequenceI;
import jalview.util.DBRefUtils;
import jalview.util.DnaUtils;
+import jalview.util.MapList;
import jalview.util.MappingUtils;
/**
*/
public class EmblFlatFile extends AlignFile // FileParse
{
+ private static final String QUOTE = "\"";
+
+ private static final String DOUBLED_QUOTE = QUOTE + QUOTE;
+
/**
* A data bean class to hold values parsed from one CDS Feature (FT)
*/
String proteinId; // from CDS /protein_id
+ List<DBRefEntry> xrefs = new ArrayList<>(); // from CDS /db_xref qualifiers
+
Map<String, String> cdsProps = new Hashtable<>(); // CDS other qualifiers
}
private int length = 128; // from ID (7th token), with usable default
- private List<DBRefEntry> dbrefs; // from DR and also CDS /db_xref qualifiers
+ private List<DBRefEntry> dbrefs; // from DR
private String sequenceString; // from SQ lines
- private List<CdsData> cds;
+ /*
+ * parsed CDS data fields, keyed by protein_id
+ */
+ private Map<String, CdsData> cds;
/**
* Constructor
super(false, fp); // don't parse immediately
this.sourceDb = sourceId;
dbrefs = new ArrayList<>();
- cds = new ArrayList<>();
+
+ /*
+ * using TreeMap gives CDS sequences in alphabetical, so readable, order
+ */
+ cds = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
}
/**
*
* @throws IOException
*/
+ @Override
public void parse() throws IOException
{
String line = nextLine();
line = nextLine();
}
}
- assembleSequence();
+ buildSequence();
}
/**
return nextLine();
}
+ /*
+ * parse location - which may be over more than one line e.g. EAW51554
+ */
CdsData data = new CdsData();
- data.cdsLocation = tokens[2];
+ StringBuilder sb = new StringBuilder().append(tokens[2]);
+ line = parseFeatureQualifier(sb, "CDS");
+ data.cdsLocation = sb.toString();
- line = nextLine();
while (line != null)
{
if (!line.startsWith("FT ")) // 4 spaces
/*
* extract qualifier, e.g. FT /protein_id="CAA37824.1"
+ * - the value may extend over more than one line
+ * - if the value has enclosing quotes, these are removed
+ * - escaped double quotes ("") are reduced to a single character
*/
int slashPos = line.indexOf('/');
if (slashPos == -1)
{
Cache.log.error("Unexpected EMBL line ignored: " + line);
+ line = nextLine();
continue;
}
int eqPos = line.indexOf('=', slashPos + 1);
if (eqPos == -1)
{
- Cache.log.error("Unexpected EMBL line ignored: " + line);
+ // can happen, e.g. /ribosomal_slippage
+ // Cache.log.error("Unexpected EMBL line ignored: " + line);
+ line = nextLine();
continue;
}
String qualifier = line.substring(slashPos + 1, eqPos);
String value = line.substring(eqPos + 1);
- if (value.startsWith("\"") && value.endsWith("\""))
- {
- value = value.substring(1, value.length() - 1);
- }
+ value = removeQuotes(value);
+ sb = new StringBuilder().append(value);
+ line = parseFeatureQualifier(sb, qualifier);
+ String featureValue = sb.toString();
if ("protein_id".equals(qualifier))
{
- data.proteinId = value;
- line = nextLine();
+ data.proteinId = featureValue;
}
else if ("codon_start".equals(qualifier))
{
try
{
- data.codonStart = Integer.parseInt(value.trim());
+ data.codonStart = Integer.parseInt(featureValue.trim());
} catch (NumberFormatException e)
{
Cache.log.error("Invalid codon_start in XML for " + this.accession
+ ": " + e.getMessage());
}
- line = nextLine();
}
else if ("db_xref".equals(qualifier))
{
- String[] parts = value.split(":");
+ String[] parts = featureValue.split(":");
if (parts.length == 2)
{
String db = parts[0].trim();
db = DBRefUtils.getCanonicalName(db);
DBRefEntry dbref = new DBRefEntry(db, "0", parts[1].trim());
- this.dbrefs.add(dbref);
+ data.xrefs.add(dbref);
}
- line = nextLine();
}
else if ("product".equals(qualifier))
{
- // sometimes name is returned e.g. for V00488
- data.proteinName = value;
- line = nextLine();
+ data.proteinName = featureValue;
}
else if ("translation".equals(qualifier))
{
- line = readTranslation(value, data);
+ data.translation = featureValue;
}
- else if (!"".equals(value))
+ else if (!"".equals(featureValue))
{
// throw anything else into the additional properties hash
- data.cdsProps.put(qualifier, value);
- line = nextLine();
+ data.cdsProps.put(qualifier, featureValue);
}
}
- this.cds.add(data);
+ if (data.proteinId != null)
+ {
+ this.cds.put(data.proteinId, data);
+ }
+ else
+ {
+ Cache.log.error("Ignoring CDS feature with no protein_id for "
+ + sourceDb + ":" + accession);
+ }
return line;
}
/**
- * Reads and returns the CDS translation from one or more lines of the file,
- * and returns the next line after that
+ * Removes leading or trailing double quotes (") unless doubled, and changes
+ * any 'escaped' (doubled) double quotes to single characters. As per the
+ * Feature Table specification for Qualifiers, Free Text.
*
* @param value
- * the first line of the translation (likely quoted)
- * @param data
* @return
- * @throws IOException
*/
- String readTranslation(String value, CdsData data) throws IOException
+ static String removeQuotes(String value)
{
- StringBuilder sb = new StringBuilder(this.length / 3 + 1);
- sb.append(value.replace("\"", ""));
+ if (value == null)
+ {
+ return null;
+ }
+ if (value.startsWith(QUOTE) && !value.startsWith(DOUBLED_QUOTE))
+ {
+ value = value.substring(1);
+ }
+ if (value.endsWith(QUOTE) && !value.endsWith(DOUBLED_QUOTE))
+ {
+ value = value.substring(0, value.length() - 1);
+ }
+ value = value.replace(DOUBLED_QUOTE, QUOTE);
+ return value;
+ }
+ /**
+ * Reads the value of a feature (FT) qualifier from one or more lines of the
+ * file, and returns the next line after that. Values are appended to the
+ * string buffer, which should be already primed with the value read from the
+ * first line for the qualifier (with any leading double quote removed).
+ * Enclosing double quotes are removed, and escaped (repeated) double quotes
+ * reduced to one only. For example for
+ *
+ * <pre>
+ * FT /note="gene_id=hCG28070.3
+ * FT ""foobar"" isoform=CRA_b"
+ * the returned value is
+ * gene_id=hCG28070.3 "foobar" isoform=CRA_b
+ * </pre>
+ *
+ * Note the side-effect of this method, to advance data reading to the next
+ * line after the feature qualifier.
+ *
+ * @param sb
+ * a string buffer primed with the first line of the value
+ * @param qualifierName
+ * @return
+ * @throws IOException
+ */
+ String parseFeatureQualifier(StringBuilder sb, String qualifierName)
+ throws IOException
+ {
String line;
while ((line = nextLine()) != null)
{
String[] tokens = line.split(WHITESPACE);
if (tokens.length < 2)
{
- Cache.log.error("Ignoring bad EMBL line: " + line);
+ Cache.log.error("Ignoring bad EMBL line for " + this.accession
+ + ": " + line);
break;
}
if (tokens[1].startsWith("/"))
{
break; // next feature qualifier
}
- sb.append(tokens[1].replace("\"", ""));
- }
- data.translation = sb.toString();
+ /*
+ * heuristic rule: most multi-line value (e.g. /product) are text,
+ * so add a space for word boundary at a new line; not for translation
+ */
+ if (!"translation".equals(qualifierName)
+ && !"CDS".equals(qualifierName))
+ {
+ sb.append(" ");
+ }
+
+ /*
+ * remove trailing " and unescape doubled ""
+ */
+ String data = removeQuotes(tokens[1]);
+ sb.append(data);
+ }
return line;
}
/**
* Constructs and saves the sequence from parsed components
*/
- void assembleSequence()
+ void buildSequence()
{
+ if (this.accession == null || this.sequenceString == null)
+ {
+ Cache.log.error("Failed to parse data from EMBL");
+ return;
+ }
+
String name = this.accession;
if (this.sourceDb != null)
{
seq.addDBRef(dbref);
}
- processAllCDS(seq);
+ processCDSFeatures(seq);
seq.deriveSequence();
*
* @param seq
*/
- protected void processAllCDS(SequenceI seq)
+ protected void processCDSFeatures(SequenceI seq)
{
/*
* record protein products found to avoid duplication i.e. >1 CDS with
* the same /protein_id [though not sure I can find an example of this]
*/
Map<String, SequenceI> proteins = new HashMap<>();
- for (CdsData data : cds)
+ for (CdsData data : cds.values())
{
- processOneCDS(seq, data, proteins);
+ processCDSFeature(seq, data, proteins);
}
}
/**
- * Processes the parsed CDS feature data to
+ * Processes data for one parsed CDS feature to
* <ul>
- * <li>add a CDS feature to the sequence for each CDS start-end range</li>
* <li>create a protein product sequence for the translation</li>
* <li>create a cross-reference to protein with mapping from dna</li>
+ * <li>add a CDS feature to the sequence for each CDS start-end range</li>
* <li>add any CDS dbrefs to the sequence and to the protein product</li>
* </ul>
*
* @param proteins
* map of protein products so far derived from CDS data
*/
- void processOneCDS(SequenceI dna, CdsData data,
+ void processCDSFeature(SequenceI dna, CdsData data,
Map<String, SequenceI> proteins)
{
/*
* parse location into a list of [start, end, start, end] positions
*/
int[] exons = getCdsRanges(this.accession, data.cdsLocation);
+
+ MapList maplist = buildMappingToProtein(dna, exons, data);
+
int exonNumber = 0;
for (int xint = 0; exons != null && xint < exons.length - 1; xint += 2)
sf.setValue(FeatureProperties.EXONPRODUCT, data.proteinName);
dna.addSequenceFeature(sf);
+ }
+
+ boolean hasUniprotDbref = false;
+ for (DBRefEntry xref : data.xrefs)
+ {
+ dna.addDBRef(xref);
+ if (xref.getSource().equals(DBRefSource.UNIPROT))
+ {
+ /*
+ * construct (or find) the sequence for (data.protein_id, data.translation)
+ */
+ SequenceI protein = buildProteinProduct(dna, xref, data, proteins);
+ Mapping map = new Mapping(protein, maplist);
+ map.setMappedFromId(data.proteinId);
+ xref.setMap(map);
+
+ /*
+ * add DBRefs with mappings from dna to protein and the inverse
+ */
+ DBRefEntry db1 = new DBRefEntry(sourceDb, version, accession);
+ db1.setMap(new Mapping(dna, maplist.getInverse()));
+ protein.addDBRef(db1);
+
+ hasUniprotDbref = true;
+ }
+ }
+
+ /*
+ * if we have a product (translation) but no explicit Uniprot dbref
+ * (example: EMBL M19487 protein_id AAB02592.1)
+ * then construct mappings to an assumed EMBLCDSPROTEIN accession
+ */
+ if (!hasUniprotDbref)
+ {
+ SequenceI protein = proteins.get(data.proteinId);
+ if (protein == null)
+ {
+ protein = new Sequence(data.proteinId, data.translation);
+ protein.setDescription(data.proteinName);
+ proteins.put(data.proteinId, protein);
+ }
+ // assuming CDSPROTEIN sequence version = dna version (?!)
+ DBRefEntry db1 = new DBRefEntry(DBRefSource.EMBLCDSProduct,
+ this.version, data.proteinId);
+ protein.addDBRef(db1);
+
+ DBRefEntry dnaToEmblProteinRef = new DBRefEntry(
+ DBRefSource.EMBLCDSProduct, this.version, data.proteinId);
+ Mapping map = new Mapping(protein, maplist);
+ map.setMappedFromId(data.proteinId);
+ dnaToEmblProteinRef.setMap(map);
+ dna.addDBRef(dnaToEmblProteinRef);
+ }
- linkProteinProduct(dna, data, proteins);
+ /*
+ * comment brought forward from EmblXmlSource, lines 447-451:
+ * TODO: if retrieved from EMBLCDS, add a DBRef back to the parent EMBL
+ * sequence with the exon map; if given a dataset reference, search
+ * dataset for parent EMBL sequence if it exists and set its map;
+ * make a new feature annotating the coding contig
+ */
+ }
+
+ /**
+ * Computes a mapping from CDS positions in DNA sequence to protein product
+ * positions, with allowance for stop codon or incomplete start codon
+ *
+ * @param dna
+ * @param exons
+ * @param data
+ * @return
+ */
+ MapList buildMappingToProtein(final SequenceI dna, final int[] exons,
+ final CdsData data)
+ {
+ MapList dnaToProteinMapping = null;
+ int peptideLength = data.translation.length();
+
+ int[] proteinRange = new int[] { 1, peptideLength };
+ if (exons != null && exons.length > 0)
+ {
+ /*
+ * We were able to parse 'location'; do a final
+ * product length truncation check
+ */
+ int[] cdsRanges = adjustForProteinLength(peptideLength, exons);
+ dnaToProteinMapping = new MapList(cdsRanges, proteinRange, 3, 1);
+ }
+ else
+ {
+ /*
+ * workaround until we handle all 'location' formats fully
+ * e.g. X53828.1:60..1058 or <123..>289
+ */
+ Cache.log.error(String.format(
+ "Implementation Notice: EMBLCDS location '%s'not properly supported yet"
+ + " - Making up the CDNA region of (%s:%s)... may be incorrect",
+ data.cdsLocation, sourceDb, this.accession));
+
+ int completeCodonsLength = 1 - data.codonStart + dna.getLength();
+ int mappedDnaEnd = dna.getEnd();
+ if (peptideLength * 3 == completeCodonsLength)
+ {
+ // this might occur for CDS sequences where no features are marked
+ Cache.log.warn("Assuming no stop codon at end of cDNA fragment");
+ mappedDnaEnd = dna.getEnd();
+ }
+ else if ((peptideLength + 1) * 3 == completeCodonsLength)
+ {
+ Cache.log.warn("Assuming stop codon at end of cDNA fragment");
+ mappedDnaEnd = dna.getEnd() - 3;
+ }
+
+ if (mappedDnaEnd != -1)
+ {
+ int[] cdsRanges = new int[] {
+ dna.getStart() + (data.codonStart - 1), mappedDnaEnd };
+ dnaToProteinMapping = new MapList(cdsRanges, proteinRange, 3, 1);
+ }
}
+
+ return dnaToProteinMapping;
}
/**
* one), and dbrefs with mappings from CDS to protein and the reverse
*
* @param dna
+ * @param xref
* @param data
* @param proteins
+ * @return
*/
- void linkProteinProduct(SequenceI dna, CdsData data, Map<String, SequenceI> proteins)
+ SequenceI buildProteinProduct(SequenceI dna, DBRefEntry xref,
+ CdsData data, Map<String, SequenceI> proteins)
{
/*
* check we have some data to work with
*/
if (data.proteinId == null || data.translation == null)
{
- return;
+ return null;
}
-
+
/*
* Construct the protein sequence (if not already seen)
*/
- SequenceI protein = proteins.get(data.proteinId);
+ String proteinSeqName = xref.getSource() + "|" + xref.getAccessionId();
+ SequenceI protein = proteins.get(proteinSeqName);
if (protein == null)
{
- protein = new Sequence(data.proteinId, data.translation, 1,
+ protein = new Sequence(proteinSeqName, data.translation, 1,
data.translation.length());
protein.setDescription(data.proteinName != null ? data.proteinName
: "Protein Product from " + sourceDb);
- proteins.put(data.proteinId, protein);
+ proteins.put(proteinSeqName, protein);
}
+
+ return protein;
}
/**
{
return null;
}
+
+ /**
+ * Truncates (if necessary) the exon intervals to match 3 times the length of
+ * the protein; also accepts 3 bases longer (for stop codon not included in
+ * protein)
+ *
+ * @param proteinLength
+ * @param exon
+ * an array of [start, end, start, end...] intervals
+ * @return the same array (if unchanged) or a truncated copy
+ */
+ static int[] adjustForProteinLength(int proteinLength, int[] exon)
+ {
+ if (proteinLength <= 0 || exon == null)
+ {
+ return exon;
+ }
+ int expectedCdsLength = proteinLength * 3;
+ int exonLength = MappingUtils.getLength(Arrays.asList(exon));
+
+ /*
+ * if exon length matches protein, or is shorter, or longer by the
+ * length of a stop codon (3 bases), then leave it unchanged
+ */
+ if (expectedCdsLength >= exonLength
+ || expectedCdsLength == exonLength - 3)
+ {
+ return exon;
+ }
+
+ int origxon[];
+ int sxpos = -1;
+ int endxon = 0;
+ origxon = new int[exon.length];
+ System.arraycopy(exon, 0, origxon, 0, exon.length);
+ int cdspos = 0;
+ for (int x = 0; x < exon.length; x += 2)
+ {
+ cdspos += Math.abs(exon[x + 1] - exon[x]) + 1;
+ if (expectedCdsLength <= cdspos)
+ {
+ // advanced beyond last codon.
+ sxpos = x;
+ if (expectedCdsLength != cdspos)
+ {
+ // System.err
+ // .println("Truncating final exon interval on region by "
+ // + (cdspos - cdslength));
+ }
+
+ /*
+ * shrink the final exon - reduce end position if forward
+ * strand, increase it if reverse
+ */
+ if (exon[x + 1] >= exon[x])
+ {
+ endxon = exon[x + 1] - cdspos + expectedCdsLength;
+ }
+ else
+ {
+ endxon = exon[x + 1] + cdspos - expectedCdsLength;
+ }
+ break;
+ }
+ }
+
+ if (sxpos != -1)
+ {
+ // and trim the exon interval set if necessary
+ int[] nxon = new int[sxpos + 2];
+ System.arraycopy(exon, 0, nxon, 0, sxpos + 2);
+ nxon[sxpos + 1] = endxon; // update the end boundary for the new exon
+ // set
+ exon = nxon;
+ }
+ return exon;
+ }
}
git://source.jalview.org / jalview.git / blobdiff