*/
package jalview.ws.dbsources;
+import jalview.analysis.SequenceIdMatcher;
+import jalview.bin.Cache;
import jalview.datamodel.Alignment;
import jalview.datamodel.AlignmentI;
+import jalview.datamodel.DBRefEntry;
+import jalview.datamodel.DBRefSource;
+import jalview.datamodel.FeatureProperties;
+import jalview.datamodel.Mapping;
+import jalview.datamodel.Sequence;
+import jalview.datamodel.SequenceFeature;
import jalview.datamodel.SequenceI;
-import jalview.datamodel.xdb.embl.EmblEntry;
-import jalview.datamodel.xdb.embl.EmblFile;
+import jalview.util.DBRefUtils;
+import jalview.util.DnaUtils;
+import jalview.util.MapList;
+import jalview.util.MappingUtils;
import jalview.util.MessageManager;
import jalview.ws.ebi.EBIFetchClient;
+import jalview.xml.binding.embl.EntryType;
+import jalview.xml.binding.embl.EntryType.Feature;
+import jalview.xml.binding.embl.EntryType.Feature.Qualifier;
+import jalview.xml.binding.embl.XrefType;
import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.text.ParseException;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Hashtable;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.regex.Pattern;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.stream.FactoryConfigurationError;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
public abstract class EmblXmlSource extends EbiFileRetrievedProxy
{
*/
private static final String EMBL_NOT_FOUND_REPLY = "ERROR 12 No entries found.";
+ private static final Pattern SPACE_PATTERN = Pattern.compile(" ");
+
public EmblXmlSource()
{
super();
}
/**
- * retrieve and parse an emblxml file
+ * Retrieves and parses an emblxml file, and returns an alignment containing
+ * the parsed sequences, or null if none were found
*
* @param emprefx
- * either EMBL or EMBLCDS strings are allowed - anything else will
- * not retrieve emblxml
+ * "EMBL" or "EMBLCDS" - anything else will not retrieve emblxml
* @param query
* @return
* @throws Exception
public AlignmentI getEmblSequenceRecords(String emprefx, String query,
File reply) throws Exception
{
- EmblFile efile = null;
- List<SequenceI> seqs = new ArrayList<>();
-
+ List<EntryType> entries = null;
if (reply != null && reply.exists())
{
file = reply.getAbsolutePath();
if (reply.length() > EMBL_NOT_FOUND_REPLY.length())
{
- efile = EmblFile.getEmblFile(reply);
+ InputStream is = new FileInputStream(reply);
+ entries = getEmblEntries(is);
}
}
* EmbFile reads something like (e.g.) this ungrammatical phrase
* Entry: <acc> display type is either not supported or entry is not found.
*/
+ AlignmentI al = null;
+ List<SequenceI> seqs = new ArrayList<>();
List<SequenceI> peptides = new ArrayList<>();
- if (efile != null && efile.getEntries() != null)
+ if (entries != null)
{
- for (EmblEntry entry : efile.getEntries())
+ for (EntryType entry : entries)
{
- SequenceI seq = entry.getSequence(emprefx, peptides);
+ SequenceI seq = getSequence(emprefx, entry, peptides);
if (seq != null)
{
seqs.add(seq.deriveSequence());
// place DBReferences on dataset and refer
}
}
+ if (!seqs.isEmpty())
+ {
+ al = new Alignment(seqs.toArray(new SequenceI[seqs.size()]));
+ }
+ else
+ {
+ System.out.println(
+ "No record found for '" + emprefx + ":" + query + "'");
+ }
}
- AlignmentI al = null;
- if (!seqs.isEmpty())
- {
- al = new Alignment(seqs.toArray(new SequenceI[seqs.size()]));
- }
stopQuery();
return al;
}
+ /**
+ * Reads the XML reply from file and unmarshals it to Java objects. Answers a
+ * (possibly empty) list of <code>EntryType</code> objects.
+ *
+ * is
+ *
+ * @return
+ */
+ List<EntryType> getEmblEntries(InputStream is)
+ {
+ List<EntryType> entries = new ArrayList<>();
+ try
+ {
+ JAXBContext jc = JAXBContext.newInstance("jalview.xml.binding.embl");
+ XMLStreamReader streamReader = XMLInputFactory.newInstance()
+ .createXMLStreamReader(is);
+ javax.xml.bind.Unmarshaller um = jc.createUnmarshaller();
+ jalview.xml.binding.embl.ROOT root = (jalview.xml.binding.embl.ROOT) um
+ .unmarshal(streamReader);
+
+ /*
+ * document root contains either "entry" or "entrySet"
+ */
+ if (root == null)
+ {
+ return entries;
+ }
+ if (root.getEntrySet() != null)
+ {
+ entries = root.getEntrySet().getEntry();
+ }
+ else if (root.getEntry() != null)
+ {
+ entries.add(root.getEntry());
+ }
+ } catch (JAXBException | XMLStreamException
+ | FactoryConfigurationError e)
+ {
+ e.printStackTrace();
+ }
+ return entries;
+ }
+
+ /**
+ * A helper method to parse XML data and construct a sequence, with any
+ * available database references and features
+ *
+ * @param emprefx
+ * @param entry
+ * @param peptides
+ * @return
+ */
+ SequenceI getSequence(String sourceDb, EntryType entry,
+ List<SequenceI> peptides)
+ {
+ String seqString = entry.getSequence();
+ if (seqString == null)
+ {
+ return null;
+ }
+ seqString = seqString.replace(" ", "").replace("\n", "").replace("\t",
+ "");
+ String accession = entry.getAccession();
+ SequenceI dna = new Sequence(sourceDb + "|" + accession, seqString);
+
+ dna.setDescription(entry.getDescription());
+ String sequenceVersion = String.valueOf(entry.getVersion().intValue());
+ DBRefEntry selfRref = new DBRefEntry(sourceDb, sequenceVersion,
+ accession);
+ dna.addDBRef(selfRref);
+ selfRref.setMap(
+ new Mapping(null, new int[]
+ { 1, dna.getLength() }, new int[] { 1, dna.getLength() }, 1,
+ 1));
+
+ /*
+ * add db references
+ */
+ List<XrefType> dbRefs = entry.getXref();
+ if (dbRefs != null)
+ {
+ for (XrefType dbref : dbRefs)
+ {
+ String acc = dbref.getId();
+ String source = DBRefUtils.getCanonicalName(dbref.getDb());
+ String version = dbref.getSecondaryId();
+ dna.addDBRef(new DBRefEntry(source, version, acc));
+ }
+ }
+
+ SequenceIdMatcher matcher = new SequenceIdMatcher(peptides);
+ try
+ {
+ List<Feature> features = entry.getFeature();
+ if (features != null)
+ {
+ for (Feature feature : features)
+ {
+ if (FeatureProperties.isCodingFeature(sourceDb,
+ feature.getName()))
+ {
+ parseCodingFeature(entry, feature, sourceDb, dna, peptides,
+ matcher);
+ }
+ }
+ }
+ } catch (Exception e)
+ {
+ System.err.println("EMBL Record Features parsing error!");
+ System.err
+ .println("Please report the following to help@jalview.org :");
+ System.err.println("EMBL Record " + accession);
+ System.err.println("Resulted in exception: " + e.getMessage());
+ e.printStackTrace(System.err);
+ }
+
+ return dna;
+ }
+
+ /**
+ * Extracts coding region and product from a CDS feature and decorates it with
+ * annotations
+ *
+ * @param entry
+ * @param feature
+ * @param sourceDb
+ * @param dna
+ * @param peptides
+ * @param matcher
+ */
+ void parseCodingFeature(EntryType entry, Feature feature, String sourceDb,
+ SequenceI dna, List<SequenceI> peptides,
+ SequenceIdMatcher matcher)
+ {
+ final boolean isEmblCdna = sourceDb.equals(DBRefSource.EMBLCDS);
+ final String accession = entry.getAccession();
+ final String sequenceVersion = entry.getVersion().toString();
+
+ int[] exons = getCdsRanges(entry.getAccession(), feature);
+
+ String translation = null;
+ String proteinName = "";
+ String proteinId = null;
+ Map<String, String> vals = new Hashtable<>();
+
+ /*
+ * codon_start 1/2/3 in EMBL corresponds to phase 0/1/2 in CDS
+ * (phase is required for CDS features in GFF3 format)
+ */
+ int codonStart = 1;
+
+ /*
+ * parse qualifiers, saving protein translation, protein id,
+ * codon start position, product (name), and 'other values'
+ */
+ if (feature.getQualifier() != null)
+ {
+ for (Qualifier q : feature.getQualifier())
+ {
+ String qname = q.getName();
+ String value = q.getValue();
+ value = value == null ? ""
+ : value.trim().replace(" ", "").replace("\n", "")
+ .replace("\t", "");
+ if (qname.equals("translation"))
+ {
+ translation = value;
+ }
+ else if (qname.equals("protein_id"))
+ {
+ proteinId = value;
+ }
+ else if (qname.equals("codon_start"))
+ {
+ try
+ {
+ codonStart = Integer.parseInt(value.trim());
+ } catch (NumberFormatException e)
+ {
+ System.err.println("Invalid codon_start in XML for "
+ + entry.getAccession() + ": " + e.getMessage());
+ }
+ }
+ else if (qname.equals("product"))
+ {
+ // sometimes name is returned e.g. for V00488
+ proteinName = value;
+ }
+ else
+ {
+ // throw anything else into the additional properties hash
+ if (!"".equals(value))
+ {
+ vals.put(qname, value);
+ }
+ }
+ }
+ }
+
+ DBRefEntry proteinToEmblProteinRef = null;
+ exons = MappingUtils.removeStartPositions(codonStart - 1, exons);
+
+ SequenceI product = null;
+ Mapping dnaToProteinMapping = null;
+ if (translation != null && proteinName != null && proteinId != null)
+ {
+ int translationLength = translation.length();
+
+ /*
+ * look for product in peptides list, if not found, add it
+ */
+ product = matcher.findIdMatch(proteinId);
+ if (product == null)
+ {
+ product = new Sequence(proteinId, translation, 1,
+ translationLength);
+ product.setDescription(((proteinName.length() == 0)
+ ? "Protein Product from " + sourceDb
+ : proteinName));
+ peptides.add(product);
+ matcher.add(product);
+ }
+
+ // we have everything - create the mapping and perhaps the protein
+ // sequence
+ if (exons == null || exons.length == 0)
+ {
+ /*
+ * workaround until we handle dna location for CDS sequence
+ * e.g. location="X53828.1:60..1058" correctly
+ */
+ System.err.println(
+ "Implementation Notice: EMBLCDS records not properly supported yet - Making up the CDNA region of this sequence... may be incorrect ("
+ + sourceDb + ":" + entry.getAccession() + ")");
+ int dnaLength = dna.getLength();
+ if (translationLength * 3 == (1 - codonStart + dnaLength))
+ {
+ System.err.println(
+ "Not allowing for additional stop codon at end of cDNA fragment... !");
+ // this might occur for CDS sequences where no features are marked
+ exons = new int[] { dna.getStart() + (codonStart - 1),
+ dna.getEnd() };
+ dnaToProteinMapping = new Mapping(product, exons,
+ new int[]
+ { 1, translationLength }, 3, 1);
+ }
+ if ((translationLength + 1) * 3 == (1 - codonStart + dnaLength))
+ {
+ System.err.println(
+ "Allowing for additional stop codon at end of cDNA fragment... will probably cause an error in VAMSAs!");
+ exons = new int[] { dna.getStart() + (codonStart - 1),
+ dna.getEnd() - 3 };
+ dnaToProteinMapping = new Mapping(product, exons,
+ new int[]
+ { 1, translationLength }, 3, 1);
+ }
+ }
+ else
+ {
+ // Trim the exon mapping if necessary - the given product may only be a
+ // fragment of a larger protein. (EMBL:AY043181 is an example)
+
+ if (isEmblCdna)
+ {
+ // TODO: Add a DbRef back to the parent EMBL sequence with the exon
+ // map
+ // if given a dataset reference, search dataset for parent EMBL
+ // sequence if it exists and set its map
+ // make a new feature annotating the coding contig
+ }
+ else
+ {
+ // final product length truncation check
+ int[] cdsRanges = adjustForProteinLength(translationLength,
+ exons);
+ dnaToProteinMapping = new Mapping(product, cdsRanges,
+ new int[]
+ { 1, translationLength }, 3, 1);
+ if (product != null)
+ {
+ /*
+ * make xref with mapping from protein to EMBL dna
+ */
+ DBRefEntry proteinToEmblRef = new DBRefEntry(DBRefSource.EMBL,
+ sequenceVersion, proteinId,
+ new Mapping(dnaToProteinMapping.getMap().getInverse()));
+ product.addDBRef(proteinToEmblRef);
+
+ /*
+ * make xref from protein to EMBLCDS; we assume here that the
+ * CDS sequence version is same as dna sequence (?!)
+ */
+ MapList proteinToCdsMapList = new MapList(
+ new int[]
+ { 1, translationLength },
+ new int[]
+ { 1 + (codonStart - 1),
+ (codonStart - 1) + 3 * translationLength },
+ 1, 3);
+ DBRefEntry proteinToEmblCdsRef = new DBRefEntry(
+ DBRefSource.EMBLCDS, sequenceVersion, proteinId,
+ new Mapping(proteinToCdsMapList));
+ product.addDBRef(proteinToEmblCdsRef);
+
+ /*
+ * make 'direct' xref from protein to EMBLCDSPROTEIN
+ */
+ proteinToEmblProteinRef = new DBRefEntry(proteinToEmblCdsRef);
+ proteinToEmblProteinRef.setSource(DBRefSource.EMBLCDSProduct);
+ proteinToEmblProteinRef.setMap(null);
+ product.addDBRef(proteinToEmblProteinRef);
+ }
+ }
+ }
+
+ /*
+ * add cds features to dna sequence
+ */
+ String cds = feature.getName(); // "CDS"
+ for (int xint = 0; exons != null
+ && xint < exons.length - 1; xint += 2)
+ {
+ int exonStart = exons[xint];
+ int exonEnd = exons[xint + 1];
+ int begin = Math.min(exonStart, exonEnd);
+ int end = Math.max(exonStart, exonEnd);
+ int exonNumber = xint / 2 + 1;
+ String desc = String.format("Exon %d for protein '%s' EMBLCDS:%s",
+ exonNumber, proteinName, proteinId);
+
+ SequenceFeature sf = makeCdsFeature(cds, desc, begin, end, sourceDb,
+ vals);
+
+ sf.setEnaLocation(feature.getLocation());
+ boolean forwardStrand = exonStart <= exonEnd;
+ sf.setStrand(forwardStrand ? "+" : "-");
+ sf.setPhase(String.valueOf(codonStart - 1));
+ sf.setValue(FeatureProperties.EXONPOS, exonNumber);
+ sf.setValue(FeatureProperties.EXONPRODUCT, proteinName);
+
+ dna.addSequenceFeature(sf);
+ }
+ }
+
+ /*
+ * add feature dbRefs to sequence, and mappings for Uniprot xrefs
+ */
+ boolean hasUniprotDbref = false;
+ List<XrefType> xrefs = feature.getXref();
+ if (xrefs != null)
+ {
+ boolean mappingUsed = false;
+ for (XrefType xref : xrefs)
+ {
+ /*
+ * ensure UniProtKB/Swiss-Prot converted to UNIPROT
+ */
+ String source = DBRefUtils.getCanonicalName(xref.getDb());
+ DBRefEntry dbref = new DBRefEntry(source, xref.getSecondaryId(),
+ xref.getId());
+ DBRefEntry proteinDbRef = new DBRefEntry(dbref.getSource(),
+ dbref.getVersion(), dbref.getAccessionId());
+ if (source.equals(DBRefSource.UNIPROT))
+ {
+ String proteinSeqName = DBRefSource.UNIPROT + "|"
+ + dbref.getAccessionId();
+ if (dnaToProteinMapping != null
+ && dnaToProteinMapping.getTo() != null)
+ {
+ if (mappingUsed)
+ {
+ /*
+ * two or more Uniprot xrefs for the same CDS -
+ * each needs a distinct Mapping (as to a different sequence)
+ */
+ dnaToProteinMapping = new Mapping(dnaToProteinMapping);
+ }
+ mappingUsed = true;
+
+ /*
+ * try to locate the protein mapped to (possibly by a
+ * previous CDS feature); if not found, construct it from
+ * the EMBL translation
+ */
+ SequenceI proteinSeq = matcher.findIdMatch(proteinSeqName);
+ if (proteinSeq == null)
+ {
+ proteinSeq = new Sequence(proteinSeqName,
+ product.getSequenceAsString());
+ matcher.add(proteinSeq);
+ peptides.add(proteinSeq);
+ }
+ dnaToProteinMapping.setTo(proteinSeq);
+ dnaToProteinMapping.setMappedFromId(proteinId);
+ proteinSeq.addDBRef(proteinDbRef);
+ dbref.setMap(dnaToProteinMapping);
+ }
+ hasUniprotDbref = true;
+ }
+ if (product != null)
+ {
+ /*
+ * copy feature dbref to our protein product
+ */
+ DBRefEntry pref = proteinDbRef;
+ pref.setMap(null); // reference is direct
+ product.addDBRef(pref);
+ // Add converse mapping reference
+ if (dnaToProteinMapping != null)
+ {
+ Mapping pmap = new Mapping(dna,
+ dnaToProteinMapping.getMap().getInverse());
+ pref = new DBRefEntry(sourceDb, sequenceVersion, accession);
+ pref.setMap(pmap);
+ if (dnaToProteinMapping.getTo() != null)
+ {
+ dnaToProteinMapping.getTo().addDBRef(pref);
+ }
+ }
+ }
+ dna.addDBRef(dbref);
+ }
+ }
+
+ /*
+ * if we have a product (translation) but no explicit Uniprot dbref
+ * (example: EMBL AAFI02000057 protein_id EAL65544.1)
+ * then construct mappings to an assumed EMBLCDSPROTEIN accession
+ */
+ if (!hasUniprotDbref && product != null)
+ {
+ if (proteinToEmblProteinRef == null)
+ {
+ // assuming CDSPROTEIN sequence version = dna version (?!)
+ proteinToEmblProteinRef = new DBRefEntry(DBRefSource.EMBLCDSProduct,
+ sequenceVersion, proteinId);
+ }
+ product.addDBRef(proteinToEmblProteinRef);
+
+ if (dnaToProteinMapping != null
+ && dnaToProteinMapping.getTo() != null)
+ {
+ DBRefEntry dnaToEmblProteinRef = new DBRefEntry(
+ DBRefSource.EMBLCDSProduct, sequenceVersion,
+ proteinId);
+ dnaToEmblProteinRef.setMap(dnaToProteinMapping);
+ dnaToProteinMapping.setMappedFromId(proteinId);
+ dna.addDBRef(dnaToEmblProteinRef);
+ }
+ }
+ }
+
@Override
public boolean isDnaCoding()
{
return true;
}
+ /**
+ * Returns the CDS positions as a single array of [start, end, start, end...]
+ * positions. If on the reverse strand, these will be in descending order.
+ *
+ * @param accession
+ * @param feature
+ * @return
+ */
+ protected int[] getCdsRanges(String accession, Feature feature)
+ {
+ String location = feature.getLocation();
+ if (location == null)
+ {
+ return new int[] {};
+ }
+
+ try
+ {
+ List<int[]> ranges = DnaUtils.parseLocation(location);
+ return listToArray(ranges);
+ } catch (ParseException e)
+ {
+ Cache.log.warn(
+ String.format("Not parsing inexact CDS location %s in ENA %s",
+ location, accession));
+ return new int[] {};
+ }
+ }
+
+ /**
+ * Converts a list of [start, end] ranges to a single array of [start, end,
+ * start, end ...]
+ *
+ * @param ranges
+ * @return
+ */
+ int[] listToArray(List<int[]> ranges)
+ {
+ int[] result = new int[ranges.size() * 2];
+ int i = 0;
+ for (int[] range : ranges)
+ {
+ result[i++] = range[0];
+ result[i++] = range[1];
+ }
+ return result;
+ }
+
+ /**
+ * Helper method to construct a SequenceFeature for one cds range
+ *
+ * @param type
+ * feature type ("CDS")
+ * @param desc
+ * description
+ * @param begin
+ * start position
+ * @param end
+ * end position
+ * @param group
+ * feature group
+ * @param vals
+ * map of 'miscellaneous values' for feature
+ * @return
+ */
+ protected SequenceFeature makeCdsFeature(String type, String desc,
+ int begin, int end, String group, Map<String, String> vals)
+ {
+ SequenceFeature sf = new SequenceFeature(type, desc, begin, end, group);
+ if (!vals.isEmpty())
+ {
+ StringBuilder sb = new StringBuilder();
+ boolean first = true;
+ for (Entry<String, String> val : vals.entrySet())
+ {
+ if (!first)
+ {
+ sb.append(";");
+ }
+ sb.append(val.getKey()).append("=").append(val.getValue());
+ first = false;
+ sf.setValue(val.getKey(), val.getValue());
+ }
+ sf.setAttributes(sb.toString());
+ }
+ return sf;
+ }
+
+ /**
+ * Truncates (if necessary) the exon intervals to match 3 times the length of
+ * the protein; also accepts 3 bases longer (for stop codon not included in
+ * protein)
+ *
+ * @param proteinLength
+ * @param exon
+ * an array of [start, end, start, end...] intervals
+ * @return the same array (if unchanged) or a truncated copy
+ */
+ static int[] adjustForProteinLength(int proteinLength, int[] exon)
+ {
+ if (proteinLength <= 0 || exon == null)
+ {
+ return exon;
+ }
+ int expectedCdsLength = proteinLength * 3;
+ int exonLength = MappingUtils.getLength(Arrays.asList(exon));
+
+ /*
+ * if exon length matches protein, or is shorter, or longer by the
+ * length of a stop codon (3 bases), then leave it unchanged
+ */
+ if (expectedCdsLength >= exonLength
+ || expectedCdsLength == exonLength - 3)
+ {
+ return exon;
+ }
+
+ int origxon[];
+ int sxpos = -1;
+ int endxon = 0;
+ origxon = new int[exon.length];
+ System.arraycopy(exon, 0, origxon, 0, exon.length);
+ int cdspos = 0;
+ for (int x = 0; x < exon.length; x += 2)
+ {
+ cdspos += Math.abs(exon[x + 1] - exon[x]) + 1;
+ if (expectedCdsLength <= cdspos)
+ {
+ // advanced beyond last codon.
+ sxpos = x;
+ if (expectedCdsLength != cdspos)
+ {
+ // System.err
+ // .println("Truncating final exon interval on region by "
+ // + (cdspos - cdslength));
+ }
+
+ /*
+ * shrink the final exon - reduce end position if forward
+ * strand, increase it if reverse
+ */
+ if (exon[x + 1] >= exon[x])
+ {
+ endxon = exon[x + 1] - cdspos + expectedCdsLength;
+ }
+ else
+ {
+ endxon = exon[x + 1] + cdspos - expectedCdsLength;
+ }
+ break;
+ }
+ }
+
+ if (sxpos != -1)
+ {
+ // and trim the exon interval set if necessary
+ int[] nxon = new int[sxpos + 2];
+ System.arraycopy(exon, 0, nxon, 0, sxpos + 2);
+ nxon[sxpos + 1] = endxon; // update the end boundary for the new exon
+ // set
+ exon = nxon;
+ }
+ return exon;
+ }
+
}
git://source.jalview.org / jalview.git / blobdiff