2 * Jalview - A Sequence Alignment Editor and Viewer ($$Version-Rel$$)
3 * Copyright (C) $$Year-Rel$$ The Jalview Authors
5 * This file is part of Jalview.
7 * Jalview is free software: you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation, either version 3
10 * of the License, or (at your option) any later version.
12 * Jalview is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty
14 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR
15 * PURPOSE. See the GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with Jalview. If not, see <http://www.gnu.org/licenses/>.
19 * The Jalview Authors are detailed in the 'AUTHORS' file.
21 package jalview.analysis;
23 import jalview.datamodel.AlignedCodonFrame;
24 import jalview.datamodel.Alignment;
25 import jalview.datamodel.AlignmentI;
26 import jalview.datamodel.DBRefEntry;
27 import jalview.datamodel.Mapping;
28 import jalview.datamodel.Sequence;
29 import jalview.datamodel.SequenceFeature;
30 import jalview.datamodel.SequenceI;
31 import jalview.util.DBRefUtils;
32 import jalview.util.MapList;
33 import jalview.ws.SequenceFetcherFactory;
34 import jalview.ws.seqfetcher.ASequenceFetcher;
36 import java.util.ArrayList;
37 import java.util.Iterator;
38 import java.util.List;
41 * Functions for cross-referencing sequence databases.
49 * the dataset of the alignment for which we are searching for
50 * cross-references; in some cases we may resolve xrefs by
51 * searching in the dataset
53 private AlignmentI dataset;
56 * the sequences for which we are seeking cross-references
58 private SequenceI[] fromSeqs;
61 * matcher built from dataset
63 SequenceIdMatcher matcher;
66 * sequences found by cross-ref searches to fromSeqs
68 List<SequenceI> rseqs;
74 * the sequences for which we are seeking cross-references
76 * the containing alignment dataset (may be searched to resolve
79 public CrossRef(SequenceI[] seqs, AlignmentI ds)
82 dataset = ds.getDataset() == null ? ds : ds.getDataset();
86 * Returns a list of distinct database sources for which sequences have either
88 * <li>a (dna-to-protein or protein-to-dna) cross-reference</li>
89 * <li>an indirect cross-reference - a (dna-to-protein or protein-to-dna)
90 * reference from another sequence in the dataset which has a cross-reference
91 * to a direct DBRefEntry on the given sequence</li>
95 * - when true, cross-references *from* dna returned. When false,
96 * cross-references *from* protein are returned
99 public List<String> findXrefSourcesForSequences(boolean dna)
101 List<String> sources = new ArrayList<String>();
102 for (SequenceI seq : fromSeqs)
106 findXrefSourcesForSequence(seq, dna, sources);
113 * Returns a list of distinct database sources for which a sequence has either
115 * <li>a (dna-to-protein or protein-to-dna) cross-reference</li>
116 * <li>an indirect cross-reference - a (dna-to-protein or protein-to-dna)
117 * reference from another sequence in the dataset which has a cross-reference
118 * to a direct DBRefEntry on the given sequence</li>
122 * the sequence whose dbrefs we are searching against
124 * when true, context is DNA - so sources identifying protein
125 * products will be returned.
127 * a list of sources to add matches to
129 void findXrefSourcesForSequence(SequenceI seq, boolean fromDna,
130 List<String> sources)
133 * first find seq's xrefs (dna-to-peptide or peptide-to-dna)
135 DBRefEntry[] rfs = DBRefUtils.selectDbRefs(!fromDna, seq.getDBRefs());
136 addXrefsToSources(rfs, sources);
140 * find sequence's direct (dna-to-dna, peptide-to-peptide) xrefs
142 DBRefEntry[] lrfs = DBRefUtils.selectDbRefs(fromDna, seq.getDBRefs());
143 List<SequenceI> foundSeqs = new ArrayList<SequenceI>();
146 * find sequences in the alignment which xref one of these DBRefs
147 * i.e. is xref-ed to a common sequence identifier
149 searchDatasetXrefs(fromDna, seq, lrfs, foundSeqs, null);
152 * add those sequences' (dna-to-peptide or peptide-to-dna) dbref sources
154 for (SequenceI rs : foundSeqs)
156 DBRefEntry[] xrs = DBRefUtils
157 .selectDbRefs(!fromDna, rs.getDBRefs());
158 addXrefsToSources(xrs, sources);
164 * Helper method that adds the source identifiers of some cross-references to
165 * a (non-redundant) list of database sources
170 void addXrefsToSources(DBRefEntry[] xrefs, List<String> sources)
174 for (DBRefEntry ref : xrefs)
177 * avoid duplication e.g. ENSEMBL and Ensembl
179 String source = DBRefUtils.getCanonicalName(ref.getSource());
180 if (!sources.contains(source))
189 * Attempts to find cross-references from the sequences provided in the
190 * constructor to the given source database. Cross-references may be found
192 * <li>in dbrefs on the sequence which hold a mapping to a sequence
194 * <li>provided with a fetched sequence (e.g. ENA translation), or</li>
195 * <li>populated previously after getting cross-references</li>
197 * <li>as other sequences in the alignment which share a dbref identifier with
199 * <li>by fetching from the remote database</li>
201 * The cross-referenced sequences, and mappings to them, are added to the
205 * @return cross-referenced sequences (as dataset sequences)
207 public Alignment findXrefSequences(String source, boolean fromDna)
210 rseqs = new ArrayList<SequenceI>();
211 AlignedCodonFrame cf = new AlignedCodonFrame();
212 matcher = new SequenceIdMatcher(
213 dataset.getSequences());
215 for (SequenceI seq : fromSeqs)
218 while (dss.getDatasetSequence() != null)
220 dss = dss.getDatasetSequence();
222 boolean found = false;
223 DBRefEntry[] xrfs = DBRefUtils
224 .selectDbRefs(!fromDna, dss.getDBRefs());
225 if ((xrfs == null || xrfs.length == 0) && dataset != null)
228 * found no suitable dbrefs on sequence - look for sequences in the
229 * alignment which share a dbref with this one
231 DBRefEntry[] lrfs = DBRefUtils.selectDbRefs(fromDna,
235 * find sequences (except this one!), of complementary type,
236 * which have a dbref to an accession id for this sequence,
237 * and add them to the results
239 found = searchDatasetXrefs(fromDna, dss, lrfs, rseqs, cf);
241 if (xrfs == null && !found)
244 * no dbref to source on this sequence or matched
245 * complementary sequence in the dataset
249 List<DBRefEntry> sourceRefs = DBRefUtils.searchRefsForSource(xrfs,
251 Iterator<DBRefEntry> refIterator = sourceRefs.iterator();
252 while (refIterator.hasNext())
254 DBRefEntry xref = refIterator.next();
258 SequenceI mappedTo = xref.getMap().getTo();
259 if (mappedTo != null)
262 * dbref contains the sequence it maps to; add it to the
263 * results unless we have done so already (could happen if
264 * fetching xrefs for sequences which have xrefs in common)
265 * for example: UNIPROT {P0CE19, P0CE20} -> EMBL {J03321, X06707}
269 * problem: matcher.findIdMatch() is lenient - returns a sequence
270 * with a dbref to the search arg e.g. ENST for ENSP - wrong
271 * but findInDataset() matches ENSP when looking for Uniprot...
273 SequenceI matchInDataset = findInDataset(xref);
274 if (matchInDataset != null && xref.getMap().getTo() != null
275 && matchInDataset != xref.getMap().getTo())
278 .println("Implementation problem (reopen JAL-2154): CrossRef.findInDataset seems to have recovered a different sequence than the one explicitly mapped for xref."
282 + xref.getMap().getTo()
286 /*matcher.findIdMatch(mappedTo);*/
287 if (matchInDataset != null)
289 if (!rseqs.contains(matchInDataset))
291 rseqs.add(matchInDataset);
292 // need to try harder to only add unique mappings
293 if (xref.getMap().getMap().isTripletMap()
294 && dataset.getMapping(seq, matchInDataset) == null
295 && cf.getMappingBetween(seq, matchInDataset) == null)
297 // materialise a mapping for highlighting between these sequences
300 cf.addMap(dss, matchInDataset, xref.getMap().getMap(), xref.getMap().getMappedFromId());
302 cf.addMap(matchInDataset, dss, xref.getMap().getMap().getInverse(), xref.getMap().getMappedFromId());
306 refIterator.remove();
309 // TODO: need to determine if this should be a deriveSequence
310 SequenceI rsq = new Sequence(mappedTo);
312 if (xref.getMap().getMap().isTripletMap())
314 // get sense of map correct for adding to product alignment.
317 // map is from dna seq to a protein product
318 cf.addMap(dss, rsq, xref.getMap().getMap(), xref.getMap()
323 // map should be from protein seq to its coding dna
324 cf.addMap(rsq, dss, xref.getMap().getMap().getInverse(),
325 xref.getMap().getMappedFromId());
333 SequenceI matchedSeq = matcher.findIdMatch(xref.getSource() + "|"
334 + xref.getAccessionId());
335 if (matchedSeq != null)
337 if (constructMapping(seq, matchedSeq, xref, cf, fromDna))
346 // do a bit more work - search for sequences with references matching
347 // xrefs on this sequence.
348 found = searchDataset(fromDna, dss, xref, rseqs, cf, false);
352 refIterator.remove();
357 * fetch from source database any dbrefs we haven't resolved up to here
359 if (!sourceRefs.isEmpty())
361 retrieveCrossRef(sourceRefs, seq, xrfs, fromDna, cf);
365 Alignment ral = null;
366 if (rseqs.size() > 0)
368 ral = new Alignment(rseqs.toArray(new SequenceI[rseqs.size()]));
371 dataset.addCodonFrame(cf);
377 private void retrieveCrossRef(List<DBRefEntry> sourceRefs, SequenceI seq,
378 DBRefEntry[] xrfs, boolean fromDna, AlignedCodonFrame cf)
380 ASequenceFetcher sftch = SequenceFetcherFactory.getSequenceFetcher();
381 SequenceI[] retrieved = null;
382 SequenceI dss = seq.getDatasetSequence() == null ? seq : seq
383 .getDatasetSequence();
386 retrieved = sftch.getSequences(sourceRefs, !fromDna);
387 } catch (Exception e)
390 .println("Problem whilst retrieving cross references for Sequence : "
395 if (retrieved != null)
397 updateDbrefMappings(seq, xrfs, retrieved, cf, fromDna);
398 for (SequenceI retrievedSequence : retrieved)
400 // dataset gets contaminated ccwith non-ds sequences. why ??!
401 // try: Ensembl -> Nuc->Ensembl, Nuc->Uniprot-->Protein->EMBL->
402 SequenceI retrievedDss = retrievedSequence.getDatasetSequence() == null ? retrievedSequence
403 : retrievedSequence.getDatasetSequence();
404 DBRefEntry[] dbr = retrievedSequence.getDBRefs();
407 for (DBRefEntry dbref : dbr)
409 // find any entry where we should put in the sequence being
410 // cross-referenced into the map
411 Mapping map = dbref.getMap();
414 if (map.getTo() != null && map.getMap() != null)
416 // TODO findInDataset requires exact sequence match but
417 // 'congruent' test is only for the mapped part
418 // maybe not a problem in practice since only ENA provide a
419 // mapping and it is to the full protein translation of CDS
420 SequenceI matched = findInDataset(dbref);
421 // matcher.findIdMatch(map.getTo());
425 * already got an xref to this sequence; update this
426 * map to point to the same sequence, and add
427 * any new dbrefs to it
429 DBRefEntry[] toRefs = map.getTo().getDBRefs();
432 for (DBRefEntry ref : toRefs)
434 matched.addDBRef(ref); // add or update mapping
441 if (dataset.findIndex(map.getTo()) == -1)
443 dataset.addSequence(map.getTo());
444 matcher.add(map.getTo());
449 // compare ms with dss and replace with dss in mapping
450 // if map is congruent
451 SequenceI ms = map.getTo();
452 int sf = map.getMap().getToLowest();
453 int st = map.getMap().getToHighest();
454 SequenceI mappedrg = ms.getSubSequence(sf, st);
455 // SequenceI loc = dss.getSubSequence(sf, st);
456 if (mappedrg.getLength() > 0
457 && ms.getSequenceAsString().equals(
458 dss.getSequenceAsString()))
459 // && mappedrg.getSequenceAsString().equals(
460 // loc.getSequenceAsString()))
462 String msg = "Mapping updated from " + ms.getName()
463 + " to retrieved crossreference "
465 System.out.println(msg);
469 * give the reverse reference the inverse mapping
470 * (if it doesn't have one already)
472 setReverseMapping(dss, dbref, cf);
475 * copy sequence features as well, avoiding
476 * duplication (e.g. same variation from two
479 SequenceFeature[] sfs = ms.getSequenceFeatures();
482 for (SequenceFeature feat : sfs)
485 * make a flyweight feature object which ignores Parent
486 * attribute in equality test; this avoids creating many
487 * otherwise duplicate exon features on genomic sequence
489 SequenceFeature newFeature = new SequenceFeature(
493 public boolean equals(Object o)
495 return super.equals(o, true);
498 dss.addSequenceFeature(newFeature);
502 cf.addMap(retrievedDss, map.getTo(), map.getMap());
503 } catch (Exception e)
506 .println("Exception when consolidating Mapped sequence set...");
507 e.printStackTrace(System.err);
513 retrievedSequence.updatePDBIds();
514 rseqs.add(retrievedDss);
515 if (dataset.findIndex(retrievedDss) == -1)
517 dataset.addSequence(retrievedDss);
518 matcher.add(retrievedDss);
524 * Sets the inverse sequence mapping in the corresponding dbref of the mapped
525 * to sequence (if any). This is used after fetching a cross-referenced
526 * sequence, if the fetched sequence has a mapping to the original sequence,
527 * to set the mapping in the original sequence's dbref.
530 * the sequence mapped from
534 void setReverseMapping(SequenceI mapFrom, DBRefEntry dbref,
535 AlignedCodonFrame mappings)
537 SequenceI mapTo = dbref.getMap().getTo();
542 DBRefEntry[] dbrefs = mapTo.getDBRefs();
547 for (DBRefEntry toRef : dbrefs)
549 if (toRef.hasMap() && mapFrom == toRef.getMap().getTo())
552 * found the reverse dbref; update its mapping if null
554 if (toRef.getMap().getMap() == null)
556 MapList inverse = dbref.getMap().getMap().getInverse();
557 toRef.getMap().setMap(inverse);
558 mappings.addMap(mapTo, mapFrom, inverse);
565 * Returns the first identical sequence in the dataset if any, else null
570 SequenceI findInDataset(DBRefEntry xref)
572 if (xref == null || !xref.hasMap() || xref.getMap().getTo() == null)
576 SequenceI mapsTo = xref.getMap().getTo();
577 String name = xref.getAccessionId();
578 String name2 = xref.getSource() + "|" + name;
579 SequenceI dss = mapsTo.getDatasetSequence() == null ? mapsTo : mapsTo
580 .getDatasetSequence();
581 for (SequenceI seq : dataset.getSequences())
584 * clumsy alternative to using SequenceIdMatcher which currently
585 * returns sequences with a dbref to the matched accession id
586 * which we don't want
588 if (name.equals(seq.getName()) || seq.getName().startsWith(name2))
590 if (sameSequence(seq, dss))
600 * Answers true if seq1 and seq2 contain exactly the same characters (ignoring
601 * case), else false. This method compares the lengths, then each character in
602 * turn, in order to 'fail fast'. For case-sensitive comparison, it would be
603 * possible to use Arrays.equals(seq1.getSequence(), seq2.getSequence()).
609 // TODO move to Sequence / SequenceI
610 static boolean sameSequence(SequenceI seq1, SequenceI seq2)
616 if (seq1 == null || seq2 == null)
620 char[] c1 = seq1.getSequence();
621 char[] c2 = seq2.getSequence();
622 if (c1.length != c2.length)
626 for (int i = 0; i < c1.length; i++)
628 int diff = c1[i] - c2[i];
630 * same char or differ in case only ('a'-'A' == 32)
632 if (diff != 0 && diff != 32 && diff != -32)
641 * Updates any empty mappings in the cross-references with one to a compatible
642 * retrieved sequence if found, and adds any new mappings to the
650 void updateDbrefMappings(SequenceI mapFrom, DBRefEntry[] xrefs,
651 SequenceI[] retrieved, AlignedCodonFrame acf, boolean fromDna)
653 SequenceIdMatcher idMatcher = new SequenceIdMatcher(retrieved);
654 for (DBRefEntry xref : xrefs)
658 String targetSeqName = xref.getSource() + "|"
659 + xref.getAccessionId();
660 SequenceI[] matches = idMatcher.findAllIdMatches(targetSeqName);
665 for (SequenceI seq : matches)
667 constructMapping(mapFrom, seq, xref, acf, fromDna);
674 * Tries to make a mapping between sequences. If successful, adds the mapping
675 * to the dbref and the mappings collection and answers true, otherwise
676 * answers false. The following methods of making are mapping are tried in
679 * <li>if 'mapTo' holds a mapping to 'mapFrom', take the inverse; this is, for
680 * example, the case after fetching EMBL cross-references for a Uniprot
682 * <li>else check if the dna translates exactly to the protein (give or take
683 * start and stop codons></li>
684 * <li>else try to map based on CDS features on the dna sequence</li>
693 boolean constructMapping(SequenceI mapFrom, SequenceI mapTo,
694 DBRefEntry xref, AlignedCodonFrame mappings, boolean fromDna)
696 MapList mapping = null;
697 SequenceI dsmapFrom = mapFrom.getDatasetSequence() == null ? mapFrom
698 : mapFrom.getDatasetSequence();
699 SequenceI dsmapTo = mapFrom.getDatasetSequence() == null ? mapTo
700 : mapTo.getDatasetSequence();
702 * look for a reverse mapping, if found make its inverse.
703 * Note - we do this on dataset sequences only.
705 if (dsmapTo.getDBRefs() != null)
707 for (DBRefEntry dbref : dsmapTo.getDBRefs())
709 String name = dbref.getSource() + "|" + dbref.getAccessionId();
710 if (dbref.hasMap() && dsmapFrom.getName().startsWith(name))
713 * looks like we've found a map from 'mapTo' to 'mapFrom'
714 * - invert it to make the mapping the other way
716 MapList reverse = dbref.getMap().getMap().getInverse();
717 xref.setMap(new Mapping(dsmapTo, reverse));
718 mappings.addMap(mapFrom, dsmapTo, reverse);
726 mapping = AlignmentUtils.mapCdnaToProtein(mapTo, mapFrom);
730 mapping = AlignmentUtils.mapCdnaToProtein(mapFrom, mapTo);
733 mapping = mapping.getInverse();
740 xref.setMap(new Mapping(mapTo, mapping));
743 * and add a reverse DbRef with the inverse mapping
745 if (mapFrom.getDatasetSequence() != null
746 && mapFrom.getDatasetSequence().getSourceDBRef() != null)
748 DBRefEntry dbref = new DBRefEntry(mapFrom.getDatasetSequence()
750 dbref.setMap(new Mapping(mapFrom.getDatasetSequence(), mapping
752 mapTo.addDBRef(dbref);
757 AlignmentUtils.computeProteinFeatures(mapFrom, mapTo, mapping);
758 mappings.addMap(mapFrom, mapTo, mapping);
762 mappings.addMap(mapTo, mapFrom, mapping.getInverse());
769 * find references to lrfs in the cross-reference set of each sequence in
770 * dataset (that is not equal to sequenceI) Identifies matching DBRefEntry
771 * based on source and accession string only - Map and Version are nulled.
774 * - true if context was searching from Dna sequences, false if
775 * context was searching from Protein sequences
779 * @return true if matches were found.
781 private boolean searchDatasetXrefs(boolean fromDna, SequenceI sequenceI,
782 DBRefEntry[] lrfs, List<SequenceI> foundSeqs, AlignedCodonFrame cf)
784 boolean found = false;
789 for (int i = 0; i < lrfs.length; i++)
791 DBRefEntry xref = new DBRefEntry(lrfs[i]);
793 xref.setVersion(null);
795 found |= searchDataset(fromDna, sequenceI, xref, foundSeqs, cf, false);
801 * Searches dataset for DBRefEntrys matching the given one (xrf) and adds the
802 * associated sequence to rseqs
805 * true if context was searching for refs *from* dna sequence, false
806 * if context was searching for refs *from* protein sequence
808 * a sequence to ignore (start point of search)
810 * a cross-reference to try to match
812 * result list to add to
814 * a set of sequence mappings to add to
816 * - indicates the type of relationship between returned sequences,
817 * xrf, and sequenceI that is required.
819 * <li>direct implies xrf is a primary reference for sequenceI AND
820 * the sequences to be located (eg a uniprot ID for a protein
821 * sequence, and a uniprot ref on a transcript sequence).</li>
822 * <li>indirect means xrf is a cross reference with respect to
823 * sequenceI or all the returned sequences (eg a genomic reference
824 * associated with a locus and one or more transcripts)</li>
826 * @return true if relationship found and sequence added.
828 boolean searchDataset(boolean fromDna, SequenceI fromSeq,
829 DBRefEntry xrf, List<SequenceI> foundSeqs, AlignedCodonFrame mappings,
832 boolean found = false;
837 if (dataset.getSequences() == null)
839 System.err.println("Empty dataset sequence set - NO VECTOR");
843 synchronized (ds = dataset.getSequences())
845 for (SequenceI nxt : ds)
849 if (nxt.getDatasetSequence() != null)
852 .println("Implementation warning: CrossRef initialised with a dataset alignment with non-dataset sequences in it! ("
853 + nxt.getDisplayId(true)
854 + " has ds reference "
855 + nxt.getDatasetSequence().getDisplayId(true)
858 if (nxt == fromSeq || nxt == fromSeq.getDatasetSequence())
863 * only look at same molecule type if 'direct', or
864 * complementary type if !direct
867 boolean isDna = !nxt.isProtein();
868 if (direct ? (isDna != fromDna) : (isDna == fromDna))
870 // skip this sequence because it is wrong molecule type
875 // look for direct or indirect references in common
876 DBRefEntry[] poss = nxt.getDBRefs();
877 List<DBRefEntry> cands = null;
879 // todo: indirect specifies we select either direct references to nxt
880 // that match xrf which is indirect to sequenceI, or indirect
881 // references to nxt that match xrf which is direct to sequenceI
882 cands = DBRefUtils.searchRefs(poss, xrf);
885 // poss = DBRefUtils.selectDbRefs(nxt.isProtein()!fromDna, poss);
886 // cands = DBRefUtils.searchRefs(poss, xrf);
888 if (!cands.isEmpty())
890 if (!foundSeqs.contains(nxt))
894 if (mappings != null && !direct)
897 * if the matched sequence has mapped dbrefs to
898 * protein product / cdna, add equivalent mappings to
899 * our source sequence
901 for (DBRefEntry candidate : cands)
903 Mapping mapping = candidate.getMap();
906 MapList map = mapping.getMap();
907 if (mapping.getTo() != null
908 && map.getFromRatio() != map.getToRatio())
911 * add a mapping, as from dna to peptide sequence
913 if (map.getFromRatio() == 3)
915 mappings.addMap(nxt, fromSeq, map);
919 mappings.addMap(nxt, fromSeq, map.getInverse());