X-Git-Url: http://source.jalview.org/gitweb/?a=blobdiff_plain;ds=sidebyside;f=src%2Fjalview%2Fext%2Fensembl%2FEnsemblSeqProxy.java;h=c903de3a3dafa4df60cc2374dbf3ad08f68910e8;hb=da31d3e5aebfcdbb1ae7084bbc75bbc3539ee2e6;hp=137c9b065357bcdec1b416e989f5206c25d0f713;hpb=8504d9c48d03acee51397e5fefa3df660ad0a6cc;p=jalview.git

diff --git a/src/jalview/ext/ensembl/EnsemblSeqProxy.java b/src/jalview/ext/ensembl/EnsemblSeqProxy.java
index 137c9b0..c903de3 100644
--- a/src/jalview/ext/ensembl/EnsemblSeqProxy.java
+++ b/src/jalview/ext/ensembl/EnsemblSeqProxy.java
@@ -1,178 +1,956 @@
+/*
+ * Jalview - A Sequence Alignment Editor and Viewer ($$Version-Rel$$)
+ * Copyright (C) $$Year-Rel$$ The Jalview Authors
+ * 
+ * This file is part of Jalview.
+ * 
+ * Jalview is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License 
+ * as published by the Free Software Foundation, either version 3
+ * of the License, or (at your option) any later version.
+ *  
+ * Jalview is distributed in the hope that it will be useful, but 
+ * WITHOUT ANY WARRANTY; without even the implied warranty 
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR 
+ * PURPOSE.  See the GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with Jalview.  If not, see <http://www.gnu.org/licenses/>.
+ * The Jalview Authors are detailed in the 'AUTHORS' file.
+ */
 package jalview.ext.ensembl;
 
+import jalview.analysis.AlignmentUtils;
+import jalview.analysis.Dna;
+import jalview.bin.Cache;
+import jalview.datamodel.Alignment;
 import jalview.datamodel.AlignmentI;
+import jalview.datamodel.DBRefEntry;
 import jalview.datamodel.DBRefSource;
+import jalview.datamodel.Mapping;
+import jalview.datamodel.Sequence;
+import jalview.datamodel.SequenceFeature;
 import jalview.datamodel.SequenceI;
+import jalview.datamodel.features.SequenceFeatures;
 import jalview.exceptions.JalviewException;
-import jalview.ext.ensembl.SeqFetcher.EnsemblSeqType;
-import jalview.io.FastaFile;
-import jalview.io.FileParse;
+import jalview.io.gff.SequenceOntologyFactory;
+import jalview.io.gff.SequenceOntologyI;
+import jalview.util.Comparison;
 import jalview.util.DBRefUtils;
-import jalview.ws.seqfetcher.DbSourceProxy;
-import jalview.ws.seqfetcher.DbSourceProxyImpl;
+import jalview.util.IntRangeComparator;
+import jalview.util.MapList;
 
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
-import com.stevesoft.pat.Regex;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
 
-public abstract class EnsemblSeqProxy extends DbSourceProxyImpl implements
-        DbSourceProxy
+/**
+ * Base class for Ensembl sequence fetchers
+ * 
+ * @see http://rest.ensembl.org/documentation/info/sequence_id
+ * @author gmcarstairs
+ */
+public abstract class EnsemblSeqProxy extends EnsemblRestClient
 {
-  SeqFetcher sf;
+  private static final String ALLELES = "alleles";
 
-  public EnsemblSeqProxy() throws Exception
+  protected static final String NAME = "Name";
+
+  protected static final String DESCRIPTION = "description";
+
+  /*
+   * enum for 'type' parameter to the /sequence REST service
+   */
+  public enum EnsemblSeqType
   {
-    sf = new SeqFetcher();
-    addDbSourceProperty(DBRefSource.MULTIACC);
-    addDbSourceProperty(DBRefSource.SEQDB);
-    // decide whether these need to be filtered according to return type
-    addDbSourceProperty(DBRefSource.PROTSEQDB);
-    addDbSourceProperty(DBRefSource.DNACODINGSEQDB);
-    addDbSourceProperty(DBRefSource.DNASEQDB);
+    /**
+     * type=genomic to fetch full dna including introns
+     */
+    GENOMIC("genomic"),
+
+    /**
+     * type=cdna to fetch coding dna including UTRs
+     */
+    CDNA("cdna"),
+
+    /**
+     * type=cds to fetch coding dna excluding UTRs
+     */
+    CDS("cds"),
+
+    /**
+     * type=protein to fetch peptide product sequence
+     */
+    PROTEIN("protein");
+
+    /*
+     * the value of the 'type' parameter to fetch this version of 
+     * an Ensembl sequence
+     */
+    private String type;
+
+    EnsemblSeqType(String t)
+    {
+      type = t;
+    }
+
+    public String getType()
+    {
+      return type;
+    }
+
   }
 
-  @Override
-  public String getDbSource()
+  /**
+   * Default constructor (to use rest.ensembl.org)
+   */
+  public EnsemblSeqProxy()
   {
-    return "ENSEMBL";
+    super();
   }
 
+  /**
+   * Constructor given the target domain to fetch data from
+   */
+  public EnsemblSeqProxy(String d)
+  {
+    super(d);
+  }
 
+  /**
+   * Makes the sequence queries to Ensembl's REST service and returns an
+   * alignment consisting of the returned sequences.
+   */
   @Override
-  public String getDbVersion()
+  public AlignmentI getSequenceRecords(String query) throws Exception
   {
-    return "0"; // sf.getVersion();
+    // TODO use a String... query vararg instead?
+
+    // danger: accession separator used as a regex here, a string elsewhere
+    // in this case it is ok (it is just a space), but (e.g.) '\' would not be
+    List<String> allIds = Arrays
+            .asList(query.split(getAccessionSeparator()));
+    AlignmentI alignment = null;
+    inProgress = true;
+
+    /*
+     * execute queries, if necessary in batches of the
+     * maximum allowed number of ids
+     */
+    int maxQueryCount = getMaximumQueryCount();
+    for (int v = 0, vSize = allIds.size(); v < vSize; v += maxQueryCount)
+    {
+      int p = Math.min(vSize, v + maxQueryCount);
+      List<String> ids = allIds.subList(v, p);
+      try
+      {
+        alignment = fetchSequences(ids, alignment);
+      } catch (Throwable r)
+      {
+        inProgress = false;
+        String msg = "Aborting ID retrieval after " + v
+                + " chunks. Unexpected problem (" + r.getLocalizedMessage()
+                + ")";
+        System.err.println(msg);
+        r.printStackTrace();
+        break;
+      }
+    }
+
+    if (alignment == null)
+    {
+      return null;
+    }
+
+    /*
+     * fetch and transfer genomic sequence features,
+     * fetch protein product and add as cross-reference
+     */
+    for (String accId : allIds)
+    {
+      addFeaturesAndProduct(accId, alignment);
+    }
+
+    for (SequenceI seq : alignment.getSequences())
+    {
+      getCrossReferences(seq);
+    }
+
+    return alignment;
   }
 
-  @Override
-  public String getAccessionSeparator()
+  /**
+   * Fetches Ensembl features using the /overlap REST endpoint, and adds them to
+   * the sequence in the alignment. Also fetches the protein product, maps it
+   * from the CDS features of the sequence, and saves it as a cross-reference of
+   * the dna sequence.
+   * 
+   * @param accId
+   * @param alignment
+   */
+  protected void addFeaturesAndProduct(String accId, AlignmentI alignment)
+  {
+    if (alignment == null)
+    {
+      return;
+    }
+
+    try
+    {
+      /*
+       * get 'dummy' genomic sequence with gene, transcript, 
+       * exon, cds and variation features
+       */
+      SequenceI genomicSequence = null;
+      EnsemblFeatures gffFetcher = new EnsemblFeatures(getDomain());
+      EnsemblFeatureType[] features = getFeaturesToFetch();
+      AlignmentI geneFeatures = gffFetcher.getSequenceRecords(accId,
+              features);
+      if (geneFeatures != null && geneFeatures.getHeight() > 0)
+      {
+        genomicSequence = geneFeatures.getSequenceAt(0);
+      }
+      if (genomicSequence != null)
+      {
+        /*
+         * transfer features to the query sequence
+         */
+        SequenceI querySeq = alignment.findName(accId, true);
+        if (transferFeatures(accId, genomicSequence, querySeq))
+        {
+
+          /*
+           * fetch and map protein product, and add it as a cross-reference
+           * of the retrieved sequence
+           */
+          addProteinProduct(querySeq);
+        }
+      }
+    } catch (IOException e)
+    {
+      System.err.println(
+              "Error transferring Ensembl features: " + e.getMessage());
+    }
+  }
+
+  /**
+   * Returns those sequence feature types to fetch from Ensembl. We may want
+   * features either because they are of interest to the user, or as means to
+   * identify the locations of the sequence on the genomic sequence (CDS
+   * features identify CDS, exon features identify cDNA etc).
+   * 
+   * @return
+   */
+  protected abstract EnsemblFeatureType[] getFeaturesToFetch();
+
+  /**
+   * Fetches and maps the protein product, and adds it as a cross-reference of
+   * the retrieved sequence
+   */
+  protected void addProteinProduct(SequenceI querySeq)
+  {
+    String accId = querySeq.getName();
+    try
+    {
+      AlignmentI protein = new EnsemblProtein(getDomain())
+              .getSequenceRecords(accId);
+      if (protein == null || protein.getHeight() == 0)
+      {
+        System.out.println("No protein product found for " + accId);
+        return;
+      }
+      SequenceI proteinSeq = protein.getSequenceAt(0);
+
+      /*
+       * need dataset sequences (to be the subject of mappings)
+       */
+      proteinSeq.createDatasetSequence();
+      querySeq.createDatasetSequence();
+
+      MapList mapList = AlignmentUtils.mapCdsToProtein(querySeq,
+              proteinSeq);
+      if (mapList != null)
+      {
+        // clunky: ensure Uniprot xref if we have one is on mapped sequence
+        SequenceI ds = proteinSeq.getDatasetSequence();
+        // TODO: Verify ensp primary ref is on proteinSeq.getDatasetSequence()
+        Mapping map = new Mapping(ds, mapList);
+        DBRefEntry dbr = new DBRefEntry(getDbSource(),
+                getEnsemblDataVersion(), proteinSeq.getName(), map);
+        querySeq.getDatasetSequence().addDBRef(dbr);
+        DBRefEntry[] uprots = DBRefUtils.selectRefs(ds.getDBRefs(),
+                new String[]
+                { DBRefSource.UNIPROT });
+        DBRefEntry[] upxrefs = DBRefUtils.selectRefs(querySeq.getDBRefs(),
+                new String[]
+                { DBRefSource.UNIPROT });
+        if (uprots != null)
+        {
+          for (DBRefEntry up : uprots)
+          {
+            // locate local uniprot ref and map
+            List<DBRefEntry> upx = DBRefUtils.searchRefs(upxrefs,
+                    up.getAccessionId());
+            DBRefEntry upxref;
+            if (upx.size() != 0)
+            {
+              upxref = upx.get(0);
+
+              if (upx.size() > 1)
+              {
+                Cache.log.warn(
+                        "Implementation issue - multiple uniprot acc on product sequence.");
+              }
+            }
+            else
+            {
+              upxref = new DBRefEntry(DBRefSource.UNIPROT,
+                      getEnsemblDataVersion(), up.getAccessionId());
+            }
+
+            Mapping newMap = new Mapping(ds, mapList);
+            upxref.setVersion(getEnsemblDataVersion());
+            upxref.setMap(newMap);
+            if (upx.size() == 0)
+            {
+              // add the new uniprot ref
+              querySeq.getDatasetSequence().addDBRef(upxref);
+            }
+
+          }
+        }
+
+        /*
+         * copy exon features to protein, compute peptide variants from dna 
+         * variants and add as features on the protein sequence ta-da
+         */
+        AlignmentUtils.computeProteinFeatures(querySeq, proteinSeq,
+                mapList);
+      }
+    } catch (Exception e)
+    {
+      System.err
+              .println(String.format("Error retrieving protein for %s: %s",
+                      accId, e.getMessage()));
+    }
+  }
+
+  /**
+   * Get database xrefs from Ensembl, and attach them to the sequence
+   * 
+   * @param seq
+   */
+  protected void getCrossReferences(SequenceI seq)
   {
-    return " ";
+    while (seq.getDatasetSequence() != null)
+    {
+      seq = seq.getDatasetSequence();
+    }
+
+    EnsemblXref xrefFetcher = new EnsemblXref(getDomain(), getDbSource(),
+            getEnsemblDataVersion());
+    List<DBRefEntry> xrefs = xrefFetcher.getCrossReferences(seq.getName());
+    for (DBRefEntry xref : xrefs)
+    {
+      seq.addDBRef(xref);
+    }
+
+    /*
+     * and add a reference to itself
+     */
+    DBRefEntry self = new DBRefEntry(getDbSource(), getEnsemblDataVersion(),
+            seq.getName());
+    seq.addDBRef(self);
   }
 
+  /**
+   * Fetches sequences for the list of accession ids and adds them to the
+   * alignment. Returns the extended (or created) alignment.
+   * 
+   * @param ids
+   * @param alignment
+   * @return
+   * @throws JalviewException
+   * @throws IOException
+   */
+  protected AlignmentI fetchSequences(List<String> ids,
+          AlignmentI alignment) throws JalviewException, IOException
+  {
+    if (!isEnsemblAvailable())
+    {
+      inProgress = false;
+      throw new JalviewException("ENSEMBL Rest API not available.");
+    }
+    BufferedReader br = getSequenceReader(ids);
+    if (br == null)
+    {
+      return alignment;
+    }
+
+    List<SequenceI> seqs = parseSequenceJson(br);
+
+    if (seqs.isEmpty())
+    {
+      throw new IOException("No data returned for " + ids);
+    }
+
+    if (seqs.size() != ids.size())
+    {
+      System.out.println(String.format(
+              "Only retrieved %d sequences for %d query strings",
+              seqs.size(), ids.size()));
+    }
+
+    if (!seqs.isEmpty())
+    {
+      AlignmentI seqal = new Alignment(
+              seqs.toArray(new SequenceI[seqs.size()]));
+      for (SequenceI seq : seqs)
+      {
+        if (seq.getDescription() == null)
+        {
+          seq.setDescription(getDbName());
+        }
+        String name = seq.getName();
+        if (ids.contains(name)
+                || ids.contains(name.replace("ENSP", "ENST")))
+        {
+          // TODO JAL-3077 use true accession version in dbref
+          DBRefEntry dbref = DBRefUtils.parseToDbRef(seq, getDbSource(),
+                  getEnsemblDataVersion(), name);
+          seq.addDBRef(dbref);
+        }
+      }
+      if (alignment == null)
+      {
+        alignment = seqal;
+      }
+      else
+      {
+        alignment.append(seqal);
+      }
+    }
+    return alignment;
+  }
+
+  /**
+   * Parses a JSON response into a list of sequences
+   * 
+   * @param br
+   * @return
+   * @see http://rest.ensembl.org/documentation/info/sequence_id
+   */
+  protected List<SequenceI> parseSequenceJson(BufferedReader br)
+  {
+    JSONParser jp = new JSONParser();
+    List<SequenceI> result = new ArrayList<>();
+    try
+    {
+      /*
+       * for now, assumes only one sequence returned; refactor if needed
+       * in future to handle a JSONArray with more than one
+       */
+      final JSONObject val = (JSONObject) jp.parse(br);
+      Object s = val.get("desc");
+      String desc = s == null ? null : s.toString();
+      s = val.get("id");
+      String id = s == null ? null : s.toString();
+      s = val.get("seq");
+      String seq = s == null ? null : s.toString();
+      Sequence sequence = new Sequence(id, seq);
+      if (desc != null)
+      {
+        sequence.setDescription(desc);
+      }
+      // todo JAL-3077 make a DBRefEntry with true accession version
+      // s = val.get("version");
+      // String version = s == null ? "0" : s.toString();
+      // DBRefEntry dbref = new DBRefEntry(getDbSource(), version, id);
+      // sequence.addDBRef(dbref);
+      result.add(sequence);
+    } catch (ParseException | IOException e)
+    {
+      System.err.println("Error processing JSON response: " + e.toString());
+      // ignore
+    }
+    return result;
+  }
+
+  /**
+   * Returns the URL for the REST call
+   * 
+   * @return
+   * @throws MalformedURLException
+   */
   @Override
-  public Regex getAccessionValidator()
+  protected URL getUrl(List<String> ids) throws MalformedURLException
   {
-    return new Regex("((ENSP|ENST|ENSG|CCDS)[0-9.]{3,})");
+    /*
+     * a single id is included in the URL path
+     * multiple ids go in the POST body instead
+     */
+    StringBuffer urlstring = new StringBuffer(128);
+    urlstring.append(getDomain() + "/sequence/id");
+    if (ids.size() == 1)
+    {
+      urlstring.append("/").append(ids.get(0));
+    }
+    // @see https://github.com/Ensembl/ensembl-rest/wiki/Output-formats
+    urlstring.append("?type=").append(getSourceEnsemblType().getType());
+    urlstring.append(("&Accept=application/json"));
+    urlstring.append(("&Content-Type=application/json"));
+
+    String objectType = getObjectType();
+    if (objectType != null)
+    {
+      urlstring.append("&").append(OBJECT_TYPE).append("=")
+              .append(objectType);
+    }
+
+    URL url = new URL(urlstring.toString());
+    return url;
+  }
+
+  /**
+   * Override this method to specify object_type request parameter
+   * 
+   * @return
+   */
+  protected String getObjectType()
+  {
+    return null;
   }
 
+  /**
+   * A sequence/id POST request currently allows up to 50 queries
+   * 
+   * @see http://rest.ensembl.org/documentation/info/sequence_id_post
+   */
   @Override
-  public String getTestQuery()
+  public int getMaximumQueryCount()
   {
-    return "ENSP00000288602";
+    return 50;
   }
 
   @Override
-  public boolean isValidReference(String accession)
+  protected boolean useGetRequest()
   {
-    return getAccessionValidator().search(accession);
+    return false;
   }
 
-  private volatile boolean inProgress = false;
+  /**
+   * 
+   * @return the configured sequence return type for this source
+   */
+  protected abstract EnsemblSeqType getSourceEnsemblType();
 
-  @Override
-  public AlignmentI getSequenceRecords(String queries) throws Exception
+  /**
+   * Returns a list of [start, end] genomic ranges corresponding to the sequence
+   * being retrieved.
+   * 
+   * The correspondence between the frames of reference is made by locating
+   * those features on the genomic sequence which identify the retrieved
+   * sequence. Specifically
+   * <ul>
+   * <li>genomic sequence is identified by "transcript" features with
+   * ID=transcript:transcriptId</li>
+   * <li>cdna sequence is identified by "exon" features with
+   * Parent=transcript:transcriptId</li>
+   * <li>cds sequence is identified by "CDS" features with
+   * Parent=transcript:transcriptId</li>
+   * </ul>
+   * 
+   * The returned ranges are sorted to run forwards (for positive strand) or
+   * backwards (for negative strand). Aborts and returns null if both positive
+   * and negative strand are found (this should not normally happen).
+   * 
+   * @param sourceSequence
+   * @param accId
+   * @param start
+   *          the start position of the sequence we are mapping to
+   * @return
+   */
+  protected MapList getGenomicRangesFromFeatures(SequenceI sourceSequence,
+          String accId, int start)
   {
-    inProgress = true;
-    List<String> tids, ids = new ArrayList<String>();
-    tids = Arrays.asList(queries.split(" +"));
-    AlignmentI rtn = null;
-    for (int v = 0, vSize = tids.size(); v < vSize; v += 50)
+    List<SequenceFeature> sfs = sourceSequence.getFeatures()
+            .getPositionalFeatures();
+    if (sfs.isEmpty())
     {
-      int p = v + 50;
-      if (p > vSize)
-      {
-        p = vSize;
-      }
-      ;
-      ids = tids.subList(v, p);
-      try
+      return null;
+    }
+
+    /*
+     * generously initial size for number of cds regions
+     * (worst case titin Q8WZ42 has c. 313 exons)
+     */
+    List<int[]> regions = new ArrayList<>(100);
+    int mappedLength = 0;
+    int direction = 1; // forward
+    boolean directionSet = false;
+
+    for (SequenceFeature sf : sfs)
+    {
+      /*
+       * accept the target feature type or a specialisation of it
+       * (e.g. coding_exon for exon)
+       */
+      if (identifiesSequence(sf, accId))
       {
-        if (!sf.isEnsemblAvailable())
+        int strand = sf.getStrand();
+        strand = strand == 0 ? 1 : strand; // treat unknown as forward
+
+        if (directionSet && strand != direction)
         {
-          inProgress = false;
-          throw new JalviewException("ENSEMBL Rest API not available.");
+          // abort - mix of forward and backward
+          System.err.println(
+                  "Error: forward and backward strand for " + accId);
+          return null;
         }
-        FileParse fp = new FileParse(sf.getSequenceReader(
-                getSourceEnsemblType(), ids));
-        FastaFile fr = new FastaFile(fp);
-        if (fr.hasWarningMessage())
+        direction = strand;
+        directionSet = true;
+
+        /*
+         * add to CDS ranges, semi-sorted forwards/backwards
+         */
+        if (strand < 0)
         {
-          System.out
-                  .println("Warning when retrieving " + ids.size() + " ids"
-                          + ids.toString() + "\n" + fr.getWarningMessage());
+          regions.add(0, new int[] { sf.getEnd(), sf.getBegin() });
         }
-        else if (fr.getSeqs().size() != ids.size())
+        else
         {
-          System.out.println("Only retrieved " + fr.getSeqs().size()
-                  + " sequences for " + ids.size() + " query strings.");
+          regions.add(new int[] { sf.getBegin(), sf.getEnd() });
         }
-        if (fr.getSeqs().size() > 0)
+        mappedLength += Math.abs(sf.getEnd() - sf.getBegin() + 1);
+
+        if (!isSpliceable())
         {
-          AlignmentI seqal = new jalview.datamodel.Alignment(
-                  fr.getSeqsAsArray());
-          for (SequenceI sq:seqal.getSequences())
-          {
-            if (ids.contains((sq.getName())))
-            {
-              DBRefUtils.parseToDbRef(sq, "ENSEMBL", "0", sq.getName());
-            }
-          }
-          if (rtn == null)
-          {
-            rtn = seqal;
-          }
-          else
-          {
-            rtn.append(seqal);
-          }
+          /*
+           * 'gene' sequence is contiguous so we can stop as soon as its
+           * identifying feature has been found
+           */
+          break;
         }
-      } catch (Throwable r)
+      }
+    }
+
+    if (regions.isEmpty())
+    {
+      System.out.println("Failed to identify target sequence for " + accId
+              + " from genomic features");
+      return null;
+    }
+
+    /*
+     * a final sort is needed since Ensembl returns CDS sorted within source
+     * (havana / ensembl_havana)
+     */
+    Collections.sort(regions, direction == 1 ? IntRangeComparator.ASCENDING
+            : IntRangeComparator.DESCENDING);
+
+    List<int[]> to = Arrays
+            .asList(new int[]
+            { start, start + mappedLength - 1 });
+
+    return new MapList(regions, to, 1, 1);
+  }
+
+  /**
+   * Answers true if the sequence being retrieved may occupy discontiguous
+   * regions on the genomic sequence.
+   */
+  protected boolean isSpliceable()
+  {
+    return true;
+  }
+
+  /**
+   * Returns true if the sequence feature marks positions of the genomic
+   * sequence feature which are within the sequence being retrieved. For
+   * example, an 'exon' feature whose parent is the target transcript marks the
+   * cdna positions of the transcript.
+   * 
+   * @param sf
+   * @param accId
+   * @return
+   */
+  protected abstract boolean identifiesSequence(SequenceFeature sf,
+          String accId);
+
+  /**
+   * Transfers the sequence feature to the target sequence, locating its start
+   * and end range based on the mapping. Features which do not overlap the
+   * target sequence are ignored.
+   * 
+   * @param sf
+   * @param targetSequence
+   * @param mapping
+   *          mapping from the sequence feature's coordinates to the target
+   *          sequence
+   * @param forwardStrand
+   */
+  protected void transferFeature(SequenceFeature sf,
+          SequenceI targetSequence, MapList mapping, boolean forwardStrand)
+  {
+    int start = sf.getBegin();
+    int end = sf.getEnd();
+    int[] mappedRange = mapping.locateInTo(start, end);
+
+    if (mappedRange != null)
+    {
+      String group = sf.getFeatureGroup();
+      if (".".equals(group))
       {
-        inProgress = false;
-        if (rtn != null)
-        {
-          System.err.println("Aborting ID retrieval after " + v
-                  + " chunks.");
-          r.printStackTrace();
-        }
-        else
-        {
+        group = getDbSource();
+      }
+      int newBegin = Math.min(mappedRange[0], mappedRange[1]);
+      int newEnd = Math.max(mappedRange[0], mappedRange[1]);
+      SequenceFeature copy = new SequenceFeature(sf, newBegin, newEnd,
+              group, sf.getScore());
+      targetSequence.addSequenceFeature(copy);
 
-          throw new JalviewException("Aborting ID retrieval after " + v
-                  + " chunks. Unexpected problem ("
-                  + r.getLocalizedMessage() + ")", r);
-        }
+      /*
+       * for sequence_variant on reverse strand, have to convert the allele
+       * values to their complements
+       */
+      if (!forwardStrand && SequenceOntologyFactory.getInstance()
+              .isA(sf.getType(), SequenceOntologyI.SEQUENCE_VARIANT))
+      {
+        reverseComplementAlleles(copy);
+      }
+    }
+  }
+
+  /**
+   * Change the 'alleles' value of a feature by converting to complementary
+   * bases, and also update the feature description to match
+   * 
+   * @param sf
+   */
+  static void reverseComplementAlleles(SequenceFeature sf)
+  {
+    final String alleles = (String) sf.getValue(ALLELES);
+    if (alleles == null)
+    {
+      return;
+    }
+    StringBuilder complement = new StringBuilder(alleles.length());
+    for (String allele : alleles.split(","))
+    {
+      reverseComplementAllele(complement, allele);
+    }
+    String comp = complement.toString();
+    sf.setValue(ALLELES, comp);
+    sf.setDescription(comp);
+
+    /*
+     * replace value of "alleles=" in sf.ATTRIBUTES as well
+     * so 'output as GFF' shows reverse complement alleles
+     */
+    String atts = sf.getAttributes();
+    if (atts != null)
+    {
+      atts = atts.replace(ALLELES + "=" + alleles, ALLELES + "=" + comp);
+      sf.setAttributes(atts);
+    }
+  }
+
+  /**
+   * Makes the 'reverse complement' of the given allele and appends it to the
+   * buffer, after a comma separator if not the first
+   * 
+   * @param complement
+   * @param allele
+   */
+  static void reverseComplementAllele(StringBuilder complement,
+          String allele)
+  {
+    if (complement.length() > 0)
+    {
+      complement.append(",");
+    }
 
+    /*
+     * some 'alleles' are actually descriptive terms 
+     * e.g. HGMD_MUTATION, PhenCode_variation
+     * - we don't want to 'reverse complement' these
+     */
+    if (!Comparison.isNucleotideSequence(allele, true))
+    {
+      complement.append(allele);
+    }
+    else
+    {
+      for (int i = allele.length() - 1; i >= 0; i--)
+      {
+        complement.append(Dna.getComplement(allele.charAt(i)));
       }
     }
-    inProgress = false;
-    return rtn;
   }
 
   /**
+   * Transfers features from sourceSequence to targetSequence
    * 
-   * @return the configured sequence return type for this source
+   * @param accessionId
+   * @param sourceSequence
+   * @param targetSequence
+   * @return true if any features were transferred, else false
    */
-  protected abstract EnsemblSeqType getSourceEnsemblType();
+  protected boolean transferFeatures(String accessionId,
+          SequenceI sourceSequence, SequenceI targetSequence)
+  {
+    if (sourceSequence == null || targetSequence == null)
+    {
+      return false;
+    }
 
-  @Override
-  public boolean queryInProgress()
+//    long start = System.currentTimeMillis();
+    List<SequenceFeature> sfs = sourceSequence.getFeatures()
+            .getPositionalFeatures();
+    MapList mapping = getGenomicRangesFromFeatures(sourceSequence,
+            accessionId, targetSequence.getStart());
+    if (mapping == null)
+    {
+      return false;
+    }
+
+    boolean result = transferFeatures(sfs, targetSequence, mapping,
+            accessionId);
+//    System.out.println("transferFeatures (" + (sfs.size()) + " --> "
+//            + targetSequence.getFeatures().getFeatureCount(true) + ") to "
+//            + targetSequence.getName() + " took "
+//            + (System.currentTimeMillis() - start) + "ms");
+    return result;
+  }
+
+  /**
+   * Transfer features to the target sequence. The start/end positions are
+   * converted using the mapping. Features which do not overlap are ignored.
+   * Features whose parent is not the specified identifier are also ignored.
+   * 
+   * @param sfs
+   * @param targetSequence
+   * @param mapping
+   * @param parentId
+   * @return
+   */
+  protected boolean transferFeatures(List<SequenceFeature> sfs,
+          SequenceI targetSequence, MapList mapping, String parentId)
   {
-    return inProgress;
+    final boolean forwardStrand = mapping.isFromForwardStrand();
+
+    /*
+     * sort features by start position (which corresponds to end
+     * position descending if reverse strand) so as to add them in
+     * 'forwards' order to the target sequence
+     */
+    SequenceFeatures.sortFeatures(sfs, forwardStrand);
+
+    boolean transferred = false;
+    for (SequenceFeature sf : sfs)
+    {
+      if (retainFeature(sf, parentId))
+      {
+        transferFeature(sf, targetSequence, mapping, forwardStrand);
+        transferred = true;
+      }
+    }
+    return transferred;
   }
 
-  @Override
-  public StringBuffer getRawRecords()
+  /**
+   * Answers true if the feature type is one we want to keep for the sequence.
+   * Some features are only retrieved in order to identify the sequence range,
+   * and may then be discarded as redundant information (e.g. "CDS" feature for
+   * a CDS sequence).
+   */
+  @SuppressWarnings("unused")
+  protected boolean retainFeature(SequenceFeature sf, String accessionId)
   {
-    return null;
+    return true; // override as required
+  }
+
+  /**
+   * Answers true if the feature has a Parent which refers to the given
+   * accession id, or if the feature has no parent. Answers false if the
+   * feature's Parent is for a different accession id.
+   * 
+   * @param sf
+   * @param identifier
+   * @return
+   */
+  protected boolean featureMayBelong(SequenceFeature sf, String identifier)
+  {
+    String parent = (String) sf.getValue(PARENT);
+    // using contains to allow for prefix "gene:", "transcript:" etc
+    if (parent != null
+            && !parent.toUpperCase().contains(identifier.toUpperCase()))
+    {
+      // this genomic feature belongs to a different transcript
+      return false;
+    }
+    return true;
   }
 
   @Override
-  public int getTier()
+  public String getDescription()
+  {
+    return "Ensembl " + getSourceEnsemblType().getType()
+            + " sequence with variant features";
+  }
+
+  /**
+   * Returns a (possibly empty) list of features on the sequence which have the
+   * specified sequence ontology term (or a sub-type of it), and the given
+   * identifier as parent
+   * 
+   * @param sequence
+   * @param term
+   * @param parentId
+   * @return
+   */
+  protected List<SequenceFeature> findFeatures(SequenceI sequence,
+          String term, String parentId)
+  {
+    List<SequenceFeature> result = new ArrayList<>();
+
+    List<SequenceFeature> sfs = sequence.getFeatures()
+            .getFeaturesByOntology(term);
+    for (SequenceFeature sf : sfs)
+    {
+      String parent = (String) sf.getValue(PARENT);
+      if (parent != null && parent.equalsIgnoreCase(parentId))
+      {
+        result.add(sf);
+      }
+    }
+
+    return result;
+  }
+
+  /**
+   * Answers true if the feature type is either 'NMD_transcript_variant' or
+   * 'transcript' or one of its sub-types in the Sequence Ontology. This is
+   * needed because NMD_transcript_variant behaves like 'transcript' in Ensembl
+   * although strictly speaking it is not (it is a sub-type of
+   * sequence_variant).
+   * 
+   * @param featureType
+   * @return
+   */
+  public static boolean isTranscript(String featureType)
   {
-    return 0;
+    return SequenceOntologyI.NMD_TRANSCRIPT_VARIANT.equals(featureType)
+            || SequenceOntologyFactory.getInstance().isA(featureType,
+                    SequenceOntologyI.TRANSCRIPT);
   }
 }