X-Git-Url: http://source.jalview.org/gitweb/?a=blobdiff_plain;f=src%2Fjalview%2Fanalysis%2FCrossRef.java;h=4ba7e416e7d115d183d68ce50878e21f1377b892;hb=37de9310bec3501cbc6381e0c3dcb282fcaad812;hp=81fad277f8546b314de35d6a10a937f0ee7d5b15;hpb=20ff4429a9545eeea6857818b55ec29eaf0388ad;p=jalview.git diff --git a/src/jalview/analysis/CrossRef.java b/src/jalview/analysis/CrossRef.java index 81fad27..4ba7e41 100644 --- a/src/jalview/analysis/CrossRef.java +++ b/src/jalview/analysis/CrossRef.java @@ -24,6 +24,7 @@ import jalview.datamodel.AlignedCodonFrame; import jalview.datamodel.Alignment; import jalview.datamodel.AlignmentI; import jalview.datamodel.DBRefEntry; +import jalview.datamodel.DBRefSource; import jalview.datamodel.Mapping; import jalview.datamodel.Sequence; import jalview.datamodel.SequenceFeature; @@ -106,6 +107,16 @@ public class CrossRef findXrefSourcesForSequence(seq, dna, sources); } } + sources.remove(DBRefSource.EMBL); // hack to prevent EMBL xrefs resulting in + // redundant datasets + if (dna) + { + sources.remove(DBRefSource.ENSEMBL); // hack to prevent Ensembl and + // EnsemblGenomes xref option shown + // from cdna panel + sources.remove(DBRefSource.ENSEMBLGENOMES); + } + // redundant datasets return sources; } @@ -209,8 +220,7 @@ public class CrossRef rseqs = new ArrayList(); AlignedCodonFrame cf = new AlignedCodonFrame(); - matcher = new SequenceIdMatcher( - dataset.getSequences()); + matcher = new SequenceIdMatcher(dataset.getSequences()); for (SequenceI seq : fromSeqs) { @@ -222,6 +232,9 @@ public class CrossRef boolean found = false; DBRefEntry[] xrfs = DBRefUtils .selectDbRefs(!fromDna, dss.getDBRefs()); + // ENST & ENSP comes in to both Protein and nucleotide, so we need to + // filter them + // out later. if ((xrfs == null || xrfs.length == 0) && dataset != null) { /* @@ -249,11 +262,15 @@ public class CrossRef List sourceRefs = DBRefUtils.searchRefsForSource(xrfs, source); Iterator refIterator = sourceRefs.iterator(); + // At this point, if we are retrieving Ensembl, we still don't filter out + // ENST when looking for protein crossrefs. while (refIterator.hasNext()) { DBRefEntry xref = refIterator.next(); found = false; - if (xref.hasMap()) + // we're only interested in coding cross-references, not + // locus->transcript + if (xref.hasMap() && xref.getMap().getMap().isTripletMap()) { SequenceI mappedTo = xref.getMap().getTo(); if (mappedTo != null) @@ -289,20 +306,28 @@ public class CrossRef if (!rseqs.contains(matchInDataset)) { rseqs.add(matchInDataset); - // need to try harder to only add unique mappings - if (xref.getMap().getMap().isTripletMap() - && dataset.getMapping(seq, matchInDataset) == null - && cf.getMappingBetween(seq, matchInDataset) == null) + } + // even if rseqs contained matchInDataset - check mappings between + // these seqs are added + // need to try harder to only add unique mappings + if (xref.getMap().getMap().isTripletMap() + && dataset.getMapping(seq, matchInDataset) == null + && cf.getMappingBetween(seq, matchInDataset) == null) + { + // materialise a mapping for highlighting between these + // sequences + if (fromDna) { - // materialise a mapping for highlighting between these sequences - if (fromDna) - { - cf.addMap(dss, matchInDataset, xref.getMap().getMap(), xref.getMap().getMappedFromId()); - } else { - cf.addMap(matchInDataset, dss, xref.getMap().getMap().getInverse(), xref.getMap().getMappedFromId()); - } + cf.addMap(dss, matchInDataset, xref.getMap().getMap(), + xref.getMap().getMappedFromId()); + } + else + { + cf.addMap(matchInDataset, dss, xref.getMap().getMap() + .getInverse(), xref.getMap().getMappedFromId()); } } + refIterator.remove(); continue; } @@ -332,7 +357,9 @@ public class CrossRef { SequenceI matchedSeq = matcher.findIdMatch(xref.getSource() + "|" + xref.getAccessionId()); - if (matchedSeq != null) + // if there was a match, check it's at least the right type of + // molecule! + if (matchedSeq != null && matchedSeq.isProtein() == fromDna) { if (constructMapping(seq, matchedSeq, xref, cf, fromDna)) { @@ -381,6 +408,16 @@ public class CrossRef SequenceI[] retrieved = null; SequenceI dss = seq.getDatasetSequence() == null ? seq : seq .getDatasetSequence(); + // first filter in case we are retrieving crossrefs that have already been + // retrieved. this happens for cases where a database record doesn't yield + // protein products for CDS + removeAlreadyRetrievedSeqs(sourceRefs, fromDna); + if (sourceRefs.size() == 0) + { + // no more work to do! We already had all requested sequence records in + // the dataset. + return; + } try { retrieved = sftch.getSequences(sourceRefs, !fromDna); @@ -394,126 +431,241 @@ public class CrossRef if (retrieved != null) { - updateDbrefMappings(seq, xrfs, retrieved, cf, fromDna); + boolean addedXref = false; + List newDsSeqs = new ArrayList(), doNotAdd = new ArrayList(); + for (SequenceI retrievedSequence : retrieved) { // dataset gets contaminated ccwith non-ds sequences. why ??! // try: Ensembl -> Nuc->Ensembl, Nuc->Uniprot-->Protein->EMBL-> SequenceI retrievedDss = retrievedSequence.getDatasetSequence() == null ? retrievedSequence : retrievedSequence.getDatasetSequence(); - DBRefEntry[] dbr = retrievedSequence.getDBRefs(); - if (dbr != null) + addedXref |= importCrossRefSeq(cf, newDsSeqs, doNotAdd, dss, + retrievedDss); + } + if (!addedXref) + { + // try again, after looking for matching IDs + // shouldn't need to do this unless the dbref mechanism has broken. + updateDbrefMappings(seq, xrfs, retrieved, cf, fromDna); + for (SequenceI retrievedSequence : retrieved) { - for (DBRefEntry dbref : dbr) + // dataset gets contaminated ccwith non-ds sequences. why ??! + // try: Ensembl -> Nuc->Ensembl, Nuc->Uniprot-->Protein->EMBL-> + SequenceI retrievedDss = retrievedSequence.getDatasetSequence() == null ? retrievedSequence + : retrievedSequence.getDatasetSequence(); + addedXref |= importCrossRefSeq(cf, newDsSeqs, doNotAdd, dss, + retrievedDss); + } + } + for (SequenceI newToSeq : newDsSeqs) + { + if (!doNotAdd.contains(newToSeq) + && dataset.findIndex(newToSeq) == -1) + { + dataset.addSequence(newToSeq); + matcher.add(newToSeq); + } + } + } + } + + /** + * Search dataset for sequences with a primary reference contained in + * sourceRefs. + * + * @param sourceRefs + * - list of references to filter. + * @param fromDna + * - type of sequence to search for matching primary reference. + */ + private void removeAlreadyRetrievedSeqs(List sourceRefs, + boolean fromDna) + { + DBRefEntry[] dbrSourceSet = sourceRefs.toArray(new DBRefEntry[0]); + for (SequenceI sq : dataset.getSequences()) + { + boolean dupeFound = false; + // !fromDna means we are looking only for nucleotide sequences, not + // protein + if (sq.isProtein() == fromDna) + { + for (DBRefEntry dbr : sq.getPrimaryDBRefs()) + { + for (DBRefEntry found : DBRefUtils.searchRefs(dbrSourceSet, dbr)) { - // find any entry where we should put in the sequence being - // cross-referenced into the map - Mapping map = dbref.getMap(); - if (map != null) + sourceRefs.remove(found); + dupeFound = true; + } + } + } + if (dupeFound) + { + // rebuild the search array from the filtered sourceRefs list + dbrSourceSet = sourceRefs.toArray(new DBRefEntry[0]); + } + } + } + + /** + * process sequence retrieved via a dbref on source sequence to resolve and + * transfer data + * + * @param cf + * @param sourceSequence + * @param retrievedSequence + * @return true if retrieveSequence was imported + */ + private boolean importCrossRefSeq(AlignedCodonFrame cf, + List newDsSeqs, List doNotAdd, + SequenceI sourceSequence, SequenceI retrievedSequence) + { + /** + * set when retrievedSequence has been verified as a crossreference for + * sourceSequence + */ + boolean imported = false; + DBRefEntry[] dbr = retrievedSequence.getDBRefs(); + if (dbr != null) + { + for (DBRefEntry dbref : dbr) + { + SequenceI matched = findInDataset(dbref); + if (matched == sourceSequence) + { + // verified retrieved and source sequence cross-reference each other + imported = true; + } + // find any entry where we should put in the sequence being + // cross-referenced into the map + Mapping map = dbref.getMap(); + if (map != null) + { + if (map.getTo() != null && map.getMap() != null) + { + if (map.getTo() == sourceSequence) + { + // already called to import once, and most likely this sequence + // already imported ! + continue; + } + if (matched == null) { - if (map.getTo() != null && map.getMap() != null) + /* + * sequence is new to dataset, so save a reference so it can be added. + */ + newDsSeqs.add(map.getTo()); + continue; + } + + /* + * there was a matching sequence in dataset, so now, check to see if we can update the map.getTo() sequence to the existing one. + */ + + try + { + // compare ms with dss and replace with dss in mapping + // if map is congruent + SequenceI ms = map.getTo(); + // TODO findInDataset requires exact sequence match but + // 'congruent' test is only for the mapped part + // maybe not a problem in practice since only ENA provide a + // mapping and it is to the full protein translation of CDS + // matcher.findIdMatch(map.getTo()); + // TODO addendum: if matched is shorter than getTo, this will fail + // - when it should really succeed. + int sf = map.getMap().getToLowest(); + int st = map.getMap().getToHighest(); + SequenceI mappedrg = ms.getSubSequence(sf, st); + if (mappedrg.getLength() > 0 + && ms.getSequenceAsString().equals( + matched.getSequenceAsString())) { - // TODO findInDataset requires exact sequence match but - // 'congruent' test is only for the mapped part - // maybe not a problem in practice since only ENA provide a - // mapping and it is to the full protein translation of CDS - SequenceI matched = findInDataset(dbref); - // matcher.findIdMatch(map.getTo()); - if (matched != null) + /* + * sequences were a match, + */ + String msg = "Mapping updated from " + ms.getName() + + " to retrieved crossreference " + + matched.getName(); + System.out.println(msg); + + DBRefEntry[] toRefs = map.getTo().getDBRefs(); + if (toRefs != null) { /* - * already got an xref to this sequence; update this - * map to point to the same sequence, and add - * any new dbrefs to it + * transfer database refs */ - DBRefEntry[] toRefs = map.getTo().getDBRefs(); - if (toRefs != null) + for (DBRefEntry ref : toRefs) { - for (DBRefEntry ref : toRefs) + if (dbref.getSrcAccString().equals( + ref.getSrcAccString())) { - matched.addDBRef(ref); // add or update mapping + continue; // avoid overwriting the ref on source sequence } + matched.addDBRef(ref); // add or update mapping } - map.setTo(matched); } - else - { - dataset.addSequence(map.getTo()); - matcher.add(map.getTo()); - } - try - { - // compare ms with dss and replace with dss in mapping - // if map is congruent - SequenceI ms = map.getTo(); - int sf = map.getMap().getToLowest(); - int st = map.getMap().getToHighest(); - SequenceI mappedrg = ms.getSubSequence(sf, st); - // SequenceI loc = dss.getSubSequence(sf, st); - if (mappedrg.getLength() > 0 - && ms.getSequenceAsString().equals( - dss.getSequenceAsString())) - // && mappedrg.getSequenceAsString().equals( - // loc.getSequenceAsString())) - { - String msg = "Mapping updated from " + ms.getName() - + " to retrieved crossreference " - + dss.getName(); - System.out.println(msg); - map.setTo(dss); + doNotAdd.add(map.getTo()); + map.setTo(matched); - /* - * give the reverse reference the inverse mapping - * (if it doesn't have one already) - */ - setReverseMapping(dss, dbref, cf); + /* + * give the reverse reference the inverse mapping + * (if it doesn't have one already) + */ + setReverseMapping(matched, dbref, cf); + /* + * copy sequence features as well, avoiding + * duplication (e.g. same variation from two + * transcripts) + */ + SequenceFeature[] sfs = ms.getSequenceFeatures(); + if (sfs != null) + { + for (SequenceFeature feat : sfs) + { /* - * copy sequence features as well, avoiding - * duplication (e.g. same variation from two - * transcripts) + * make a flyweight feature object which ignores Parent + * attribute in equality test; this avoids creating many + * otherwise duplicate exon features on genomic sequence */ - SequenceFeature[] sfs = ms.getSequenceFeatures(); - if (sfs != null) + SequenceFeature newFeature = new SequenceFeature(feat) { - for (SequenceFeature feat : sfs) + @Override + public boolean equals(Object o) { - /* - * make a flyweight feature object which ignores Parent - * attribute in equality test; this avoids creating many - * otherwise duplicate exon features on genomic sequence - */ - SequenceFeature newFeature = new SequenceFeature( - feat) - { - @Override - public boolean equals(Object o) - { - return super.equals(o, true); - } - }; - dss.addSequenceFeature(newFeature); + return super.equals(o, true); } - } + }; + matched.addSequenceFeature(newFeature); } - cf.addMap(retrievedDss, map.getTo(), map.getMap()); - } catch (Exception e) - { - System.err - .println("Exception when consolidating Mapped sequence set..."); - e.printStackTrace(System.err); } + } + cf.addMap(retrievedSequence, map.getTo(), map.getMap()); + } catch (Exception e) + { + System.err + .println("Exception when consolidating Mapped sequence set..."); + e.printStackTrace(System.err); } } } - retrievedSequence.updatePDBIds(); - rseqs.add(retrievedDss); - dataset.addSequence(retrievedDss); - matcher.add(retrievedDss); } } + if (imported) + { + retrievedSequence.updatePDBIds(); + rseqs.add(retrievedSequence); + if (dataset.findIndex(retrievedSequence) == -1) + { + dataset.addSequence(retrievedSequence); + matcher.add(retrievedSequence); + } + } + return imported; } + /** * Sets the inverse sequence mapping in the corresponding dbref of the mapped * to sequence (if any). This is used after fetching a cross-referenced @@ -556,9 +708,12 @@ public class CrossRef } /** - * Returns the first identical sequence in the dataset if any, else null + * Returns null or the first sequence in the dataset which is identical to + * xref.mapTo, and has a) a primary dbref matching xref, or if none found, the + * first one with an ID source|xrefacc * * @param xref + * with map and mapped-to sequence * @return */ SequenceI findInDataset(DBRefEntry xref) @@ -572,22 +727,42 @@ public class CrossRef String name2 = xref.getSource() + "|" + name; SequenceI dss = mapsTo.getDatasetSequence() == null ? mapsTo : mapsTo .getDatasetSequence(); + // first check ds if ds is directly referenced + if (dataset.findIndex(dss) > -1) + { + return dss; + } + DBRefEntry template = new DBRefEntry(xref.getSource(), null, + xref.getAccessionId()); + /** + * remember the first ID match - in case we don't find a match to template + */ + SequenceI firstIdMatch = null; for (SequenceI seq : dataset.getSequences()) { + // first check primary refs. + List match = DBRefUtils.searchRefs(seq.getPrimaryDBRefs() + .toArray(new DBRefEntry[0]), template); + if (match != null && match.size() == 1 && sameSequence(seq, dss)) + { + return seq; + } /* * clumsy alternative to using SequenceIdMatcher which currently * returns sequences with a dbref to the matched accession id * which we don't want */ - if (name.equals(seq.getName()) || seq.getName().startsWith(name2)) + if (firstIdMatch == null + && (name.equals(seq.getName()) || seq.getName().startsWith( + name2))) { if (sameSequence(seq, dss)) { - return seq; + firstIdMatch = seq; } } } - return null; + return firstIdMatch; } /** @@ -690,8 +865,8 @@ public class CrossRef MapList mapping = null; SequenceI dsmapFrom = mapFrom.getDatasetSequence() == null ? mapFrom : mapFrom.getDatasetSequence(); - SequenceI dsmapTo = mapFrom.getDatasetSequence() == null ? mapTo - : mapTo.getDatasetSequence(); + SequenceI dsmapTo = mapTo.getDatasetSequence() == null ? mapTo : mapTo + .getDatasetSequence(); /* * look for a reverse mapping, if found make its inverse. * Note - we do this on dataset sequences only. @@ -736,14 +911,16 @@ public class CrossRef /* * and add a reverse DbRef with the inverse mapping */ - if (mapFrom.getDatasetSequence() != null - && mapFrom.getDatasetSequence().getSourceDBRef() != null) + if (mapFrom.getDatasetSequence() != null && false) + // && mapFrom.getDatasetSequence().getSourceDBRef() != null) { - DBRefEntry dbref = new DBRefEntry(mapFrom.getDatasetSequence() - .getSourceDBRef()); - dbref.setMap(new Mapping(mapFrom.getDatasetSequence(), mapping - .getInverse())); - mapTo.addDBRef(dbref); + // possible need to search primary references... except, why doesn't xref + // == getSourceDBRef ?? + // DBRefEntry dbref = new DBRefEntry(mapFrom.getDatasetSequence() + // .getSourceDBRef()); + // dbref.setMap(new Mapping(mapFrom.getDatasetSequence(), mapping + // .getInverse())); + // mapTo.addDBRef(dbref); } if (fromDna) @@ -819,8 +996,8 @@ public class CrossRef * * @return true if relationship found and sequence added. */ - boolean searchDataset(boolean fromDna, SequenceI fromSeq, - DBRefEntry xrf, List foundSeqs, AlignedCodonFrame mappings, + boolean searchDataset(boolean fromDna, SequenceI fromSeq, DBRefEntry xrf, + List foundSeqs, AlignedCodonFrame mappings, boolean direct) { boolean found = false; @@ -881,37 +1058,38 @@ public class CrossRef // } if (!cands.isEmpty()) { - if (!foundSeqs.contains(nxt)) + if (foundSeqs.contains(nxt)) { - found = true; - foundSeqs.add(nxt); - if (mappings != null && !direct) + continue; + } + found = true; + foundSeqs.add(nxt); + if (mappings != null && !direct) + { + /* + * if the matched sequence has mapped dbrefs to + * protein product / cdna, add equivalent mappings to + * our source sequence + */ + for (DBRefEntry candidate : cands) { - /* - * if the matched sequence has mapped dbrefs to - * protein product / cdna, add equivalent mappings to - * our source sequence - */ - for (DBRefEntry candidate : cands) + Mapping mapping = candidate.getMap(); + if (mapping != null) { - Mapping mapping = candidate.getMap(); - if (mapping != null) + MapList map = mapping.getMap(); + if (mapping.getTo() != null + && map.getFromRatio() != map.getToRatio()) { - MapList map = mapping.getMap(); - if (mapping.getTo() != null - && map.getFromRatio() != map.getToRatio()) + /* + * add a mapping, as from dna to peptide sequence + */ + if (map.getFromRatio() == 3) { - /* - * add a mapping, as from dna to peptide sequence - */ - if (map.getFromRatio() == 3) - { - mappings.addMap(nxt, fromSeq, map); - } - else - { - mappings.addMap(nxt, fromSeq, map.getInverse()); - } + mappings.addMap(nxt, fromSeq, map); + } + else + { + mappings.addMap(nxt, fromSeq, map.getInverse()); } } }