JAL-2850 improved use of htsjdk api to read contig info

[jalview.git] / src / jalview / io / vcf / VCFLoader.java

package jalview.io.vcf;

import jalview.analysis.AlignmentUtils;
import jalview.analysis.Dna;
import jalview.api.AlignViewControllerGuiI;
import jalview.bin.Cache;
import jalview.datamodel.AlignmentI;
import jalview.datamodel.DBRefEntry;
import jalview.datamodel.GeneLociI;
import jalview.datamodel.Mapping;
import jalview.datamodel.SequenceFeature;
import jalview.datamodel.SequenceI;
import jalview.datamodel.features.FeatureAttributeType;
import jalview.datamodel.features.FeatureSource;
import jalview.datamodel.features.FeatureSources;
import jalview.ext.ensembl.EnsemblMap;
import jalview.ext.htsjdk.VCFReader;
import jalview.io.gff.Gff3Helper;
import jalview.io.gff.SequenceOntologyI;
import jalview.util.MapList;
import jalview.util.MappingUtils;
import jalview.util.MessageManager;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import htsjdk.samtools.SAMException;
import htsjdk.samtools.SAMSequenceDictionary;
import htsjdk.samtools.SAMSequenceRecord;
import htsjdk.samtools.util.CloseableIterator;
import htsjdk.variant.variantcontext.Allele;
import htsjdk.variant.variantcontext.VariantContext;
import htsjdk.variant.vcf.VCFHeader;
import htsjdk.variant.vcf.VCFHeaderLine;
import htsjdk.variant.vcf.VCFHeaderLineCount;
import htsjdk.variant.vcf.VCFHeaderLineType;
import htsjdk.variant.vcf.VCFInfoHeaderLine;

/**
 * A class to read VCF data (using the htsjdk) and add variants as sequence
 * features on dna and any related protein product sequences
 * 
 * @author gmcarstairs
 */
public class VCFLoader
{
  /**
   * A class to model the mapping from sequence to VCF coordinates. Cases include
   * <ul>
   * <li>a direct 1:1 mapping where the sequence is one of the VCF contigs</li>
   * <li>a mapping of sequence to chromosomal coordinates, where sequence and VCF
   * use the same reference assembly</li>
   * <li>a modified mapping of sequence to chromosomal coordinates, where sequence
   * and VCF use different reference assembles</li>
   * </ul>
   */
  class VCFMap
  {
    final String chromosome;

    final MapList map;

    VCFMap(String chr, MapList m)
    {
      chromosome = chr;
      map = m;
    }
  }

  /*
   * Lookup keys, and default values, for Preference entries that describe
   * patterns for VCF and VEP fields to capture 
   */
  private static final String VEP_FIELDS_PREF = "VEP_FIELDS";

  private static final String VCF_FIELDS_PREF = "VCF_FIELDS";

  private static final String DEFAULT_VCF_FIELDS = ".*";

  private static final String DEFAULT_VEP_FIELDS = ".*";// "Allele,Consequence,IMPACT,SWISSPROT,SIFT,PolyPhen,CLIN_SIG";

  /*
   * keys to fields of VEP CSQ consequence data
   * see https://www.ensembl.org/info/docs/tools/vep/vep_formats.html
   */
  private static final String ALLELE_KEY = "Allele";

  private static final String ALLELE_NUM_KEY = "ALLELE_NUM"; // 0 (ref), 1...
  private static final String FEATURE_KEY = "Feature"; // Ensembl stable id

  /*
   * default VCF INFO key for VEP consequence data
   * NB this can be overridden running VEP with --vcf_info_field
   * - we don't handle this case (require identifier to be CSQ)
   */
  private static final String CSQ_FIELD = "CSQ";

  /*
   * separator for fields in consequence data is '|'
   */
  private static final String PIPE_REGEX = "\\|";

  /*
   * key for Allele Frequency output by VEP
   * see http://www.ensembl.org/info/docs/tools/vep/vep_formats.html
   */
  private static final String ALLELE_FREQUENCY_KEY = "AF";

  /*
   * delimiter that separates multiple consequence data blocks
   */
  private static final String COMMA = ",";

  /*
   * the feature group assigned to a VCF variant in Jalview
   */
  private static final String FEATURE_GROUP_VCF = "VCF";

  /*
   * internal delimiter used to build keys for assemblyMappings
   * 
   */
  private static final String EXCL = "!";

  /*
   * the alignment we are associating VCF data with
   */
  private AlignmentI al;

  /*
   * mappings between VCF and sequence reference assembly regions, as 
   * key = "species!chromosome!fromAssembly!toAssembly
   * value = Map{fromRange, toRange}
   */
  private Map<String, Map<int[], int[]>> assemblyMappings;

  /*
   * holds details of the VCF header lines (metadata)
   */
  private VCFHeader header;

  /*
   * a Dictionary of contigs (if present) referenced in the VCF file
   */
  private SAMSequenceDictionary dictionary;

  /*
   * the position (0...) of field in each block of
   * CSQ (consequence) data (if declared in the VCF INFO header for CSQ)
   * see http://www.ensembl.org/info/docs/tools/vep/vep_formats.html
   */
  private int csqAlleleFieldIndex = -1;
  private int csqAlleleNumberFieldIndex = -1;
  private int csqFeatureFieldIndex = -1;

  /*
   * a unique identifier under which to save metadata about feature
   * attributes (selected INFO field data)
   */
  private String sourceId;

  /*
   * The INFO IDs of data that is both present in the VCF file, and
   * also matched by any filters for data of interest
   */
  List<String> vcfFieldsOfInterest;

  /*
   * The field offsets and identifiers for VEP (CSQ) data that is both present
   * in the VCF file, and also matched by any filters for data of interest
   * for example 0 -> Allele, 1 -> Consequence, ..., 36 -> SIFT, ...
   */
  Map<Integer, String> vepFieldsOfInterest;

  /**
   * Constructor given an alignment context
   * 
   * @param alignment
   */
  public VCFLoader(AlignmentI alignment)
  {
    al = alignment;

    // map of species!chromosome!fromAssembly!toAssembly to {fromRange, toRange}
    assemblyMappings = new HashMap<>();
  }

  /**
   * Starts a new thread to query and load VCF variant data on to the alignment
   * <p>
   * This method is not thread safe - concurrent threads should use separate
   * instances of this class.
   * 
   * @param filePath
   * @param gui
   */
  public void loadVCF(final String filePath,
          final AlignViewControllerGuiI gui)
  {
    if (gui != null)
    {
      gui.setStatus(MessageManager.getString("label.searching_vcf"));
    }

    new Thread()
    {

      @Override
      public void run()
      {
        VCFLoader.this.doLoad(filePath, gui);
      }

    }.start();
  }

  /**
   * Loads VCF on to an alignment - provided it can be related to one or more
   * sequence's chromosomal coordinates
   * 
   * @param filePath
   * @param gui
   *          optional callback handler for messages
   */
  protected void doLoad(String filePath, AlignViewControllerGuiI gui)
  {
    VCFReader reader = null;
    try
    {
      // long start = System.currentTimeMillis();
      reader = new VCFReader(filePath);

      header = reader.getFileHeader();

      try
      {
        dictionary = header.getSequenceDictionary();
      } catch (SAMException e)
      {
        // ignore - thrown if any contig line lacks length info
      }

      sourceId = filePath;

      saveMetadata(sourceId);

      /*
       * get offset of CSQ ALLELE_NUM and Feature if declared
       */
      parseCsqHeader();

      VCFHeaderLine ref = header
              .getOtherHeaderLine(VCFHeader.REFERENCE_KEY);
      String vcfAssembly = ref.getValue();

      int varCount = 0;
      int seqCount = 0;

      /*
       * query for VCF overlapping each sequence in turn
       */
      for (SequenceI seq : al.getSequences())
      {
        int added = loadSequenceVCF(seq, reader, vcfAssembly);
        if (added > 0)
        {
          seqCount++;
          varCount += added;
          transferAddedFeatures(seq);
        }
      }
      if (gui != null)
      {
        // long elapsed = System.currentTimeMillis() - start;
        String msg = MessageManager.formatMessage("label.added_vcf",
                varCount, seqCount);
        gui.setStatus(msg);
        if (gui.getFeatureSettingsUI() != null)
        {
          gui.getFeatureSettingsUI().discoverAllFeatureData();
        }
      }
    } catch (Throwable e)
    {
      System.err.println("Error processing VCF: " + e.getMessage());
      e.printStackTrace();
      if (gui != null)
      {
        gui.setStatus("Error occurred - see console for details");
      }
    } finally
    {
      if (reader != null)
      {
        try
        {
          reader.close();
        } catch (IOException e)
        {
          // ignore
        }
      }
      header = null;
      dictionary = null;
    }
  }

  /**
   * Reads metadata (such as INFO field descriptions and datatypes) and saves
   * them for future reference
   * 
   * @param theSourceId
   */
  void saveMetadata(String theSourceId)
  {
    List<Pattern> vcfFieldPatterns = getFieldMatchers(VCF_FIELDS_PREF,
            DEFAULT_VCF_FIELDS);
    vcfFieldsOfInterest = new ArrayList<>();

    FeatureSource metadata = new FeatureSource(theSourceId);

    for (VCFInfoHeaderLine info : header.getInfoHeaderLines())
    {
      String attributeId = info.getID();
      String desc = info.getDescription();
      VCFHeaderLineType type = info.getType();
      FeatureAttributeType attType = null;
      switch (type)
      {
      case Character:
        attType = FeatureAttributeType.Character;
        break;
      case Flag:
        attType = FeatureAttributeType.Flag;
        break;
      case Float:
        attType = FeatureAttributeType.Float;
        break;
      case Integer:
        attType = FeatureAttributeType.Integer;
        break;
      case String:
        attType = FeatureAttributeType.String;
        break;
      }
      metadata.setAttributeName(attributeId, desc);
      metadata.setAttributeType(attributeId, attType);

      if (isFieldWanted(attributeId, vcfFieldPatterns))
      {
        vcfFieldsOfInterest.add(attributeId);
      }
    }

    FeatureSources.getInstance().addSource(theSourceId, metadata);
  }

  /**
   * Answers true if the field id is matched by any of the filter patterns, else
   * false. Matching is against regular expression patterns, and is not
   * case-sensitive.
   * 
   * @param id
   * @param filters
   * @return
   */
  private boolean isFieldWanted(String id, List<Pattern> filters)
  {
    for (Pattern p : filters)
    {
      if (p.matcher(id.toUpperCase()).matches())
      {
        return true;
      }
    }
    return false;
  }

  /**
   * Records 'wanted' fields defined in the CSQ INFO header (if there is one).
   * Also records the position of selected fields (Allele, ALLELE_NUM, Feature)
   * required for processing.
   * <p>
   * CSQ fields are declared in the CSQ INFO Description e.g.
   * <p>
   * Description="Consequence ...from ... VEP. Format: Allele|Consequence|...
   */
  protected void parseCsqHeader()
  {
    List<Pattern> vepFieldFilters = getFieldMatchers(VEP_FIELDS_PREF,
            DEFAULT_VEP_FIELDS);
    vepFieldsOfInterest = new HashMap<>();

    VCFInfoHeaderLine csqInfo = header.getInfoHeaderLine(CSQ_FIELD);
    if (csqInfo == null)
    {
      return;
    }

    /*
     * parse out the pipe-separated list of CSQ fields; we assume here that
     * these form the last part of the description, and contain no spaces
     */
    String desc = csqInfo.getDescription();
    int spacePos = desc.lastIndexOf(" ");
    desc = desc.substring(spacePos + 1);

    if (desc != null)
    {
      String[] format = desc.split(PIPE_REGEX);
      int index = 0;
      for (String field : format)
      {
        if (ALLELE_NUM_KEY.equals(field))
        {
          csqAlleleNumberFieldIndex = index;
        }
        if (ALLELE_KEY.equals(field))
        {
          csqAlleleFieldIndex = index;
        }
        if (FEATURE_KEY.equals(field))
        {
          csqFeatureFieldIndex = index;
        }

        if (isFieldWanted(field, vepFieldFilters))
        {
          vepFieldsOfInterest.put(index, field);
        }

        index++;
      }
    }
  }

  /**
   * Reads the Preference value for the given key, with default specified if no
   * preference set. The value is interpreted as a comma-separated list of
   * regular expressions, and converted into a list of compiled patterns ready
   * for matching. Patterns are forced to upper-case for non-case-sensitive
   * matching.
   * <p>
   * This supports user-defined filters for fields of interest to capture while
   * processing data. For example, VCF_FIELDS = AF,AC* would mean that VCF INFO
   * fields with an ID of AF, or starting with AC, would be matched.
   * 
   * @param key
   * @param def
   * @return
   */
  private List<Pattern> getFieldMatchers(String key, String def)
  {
    String pref = Cache.getDefault(key, def);
    List<Pattern> patterns = new ArrayList<>();
    String[] tokens = pref.split(",");
    for (String token : tokens)
    {
      try
      {
      patterns.add(Pattern.compile(token.toUpperCase()));
      } catch (PatternSyntaxException e)
      {
        System.err.println("Invalid pattern ignored: " + token);
      }
    }
    return patterns;
  }

  /**
   * Transfers VCF features to sequences to which this sequence has a mapping.
   * If the mapping is 3:1, computes peptide variants from nucleotide variants.
   * 
   * @param seq
   */
  protected void transferAddedFeatures(SequenceI seq)
  {
    DBRefEntry[] dbrefs = seq.getDBRefs();
    if (dbrefs == null)
    {
      return;
    }
    for (DBRefEntry dbref : dbrefs)
    {
      Mapping mapping = dbref.getMap();
      if (mapping == null || mapping.getTo() == null)
      {
        continue;
      }

      SequenceI mapTo = mapping.getTo();
      MapList map = mapping.getMap();
      if (map.getFromRatio() == 3)
      {
        /*
         * dna-to-peptide product mapping
         */
        AlignmentUtils.computeProteinFeatures(seq, mapTo, map);
      }
      else
      {
        /*
         * nucleotide-to-nucleotide mapping e.g. transcript to CDS
         */
        List<SequenceFeature> features = seq.getFeatures()
                .getPositionalFeatures(SequenceOntologyI.SEQUENCE_VARIANT);
        for (SequenceFeature sf : features)
        {
          if (FEATURE_GROUP_VCF.equals(sf.getFeatureGroup()))
          {
            transferFeature(sf, mapTo, map);
          }
        }
      }
    }
  }

  /**
   * Tries to add overlapping variants read from a VCF file to the given
   * sequence, and returns the number of variant features added. Note that this
   * requires the sequence to hold information as to its species, chromosomal
   * positions and reference assembly, in order to be able to map the VCF
   * variants to the sequence (or not)
   * 
   * @param seq
   * @param reader
   * @param vcfAssembly
   * @return
   */
  protected int loadSequenceVCF(SequenceI seq, VCFReader reader,
          String vcfAssembly)
  {
    VCFMap vcfMap = getVcfMap(seq, vcfAssembly);
    if (vcfMap == null)
    {
      return 0;
    }

    return addVcfVariants(seq, reader, vcfMap, vcfAssembly);
  }

  /**
   * Answers a map from sequence coordinates to VCF chromosome ranges
   * 
   * @param seq
   * @param vcfAssembly
   * @return
   */
  private VCFMap getVcfMap(SequenceI seq, String vcfAssembly)
  {
    /*
     * simplest case: sequence has id and length matching a VCF contig
     */
    VCFMap vcfMap = null;
    if (dictionary != null)
    {
      vcfMap = getContigMap(seq);
    }
    if (vcfMap != null)
    {
      return vcfMap;
    }

    /*
     * otherwise, map to VCF from chromosomal coordinates 
     * of the sequence (if known)
     */
    GeneLociI seqCoords = seq.getGeneLoci();
    if (seqCoords == null)
    {
      Cache.log.warn(String.format(
              "Can't query VCF for %s as chromosome coordinates not known",
              seq.getName()));
      return null;
    }

    String species = seqCoords.getSpeciesId();
    String chromosome = seqCoords.getChromosomeId();
    String seqRef = seqCoords.getAssemblyId();
    MapList map = seqCoords.getMap();

    if (!vcfSpeciesMatchesSequence(vcfAssembly, species))
    {
      return null;
    }

    if (vcfAssemblyMatchesSequence(vcfAssembly, seqRef))
    {
      return new VCFMap(chromosome, map);
    }

    if (!"GRCh38".equalsIgnoreCase(seqRef) // Ensembl
            || !vcfAssembly.contains("Homo_sapiens_assembly19")) // gnomAD
    {
      return null;
    }

    /*
     * map chromosomal coordinates from sequence to VCF if the VCF
     * data has a different reference assembly to the sequence
     */
    // TODO generalise for cases other than GRCh38 -> GRCh37 !
    // - or get the user to choose in a dialog

    List<int[]> toVcfRanges = new ArrayList<>();
    List<int[]> fromSequenceRanges = new ArrayList<>();
    String toRef = "GRCh37";

    for (int[] range : map.getToRanges())
    {
      int[] fromRange = map.locateInFrom(range[0], range[1]);
      if (fromRange == null)
      {
        // corrupted map?!?
        continue;
      }

      int[] newRange = mapReferenceRange(range, chromosome, "human", seqRef,
              toRef);
      if (newRange == null)
      {
        Cache.log.error(
                String.format("Failed to map %s:%s:%s:%d:%d to %s", species,
                        chromosome, seqRef, range[0], range[1], toRef));
        continue;
      }
      else
      {
        toVcfRanges.add(newRange);
        fromSequenceRanges.add(fromRange);
      }
    }

    return new VCFMap(chromosome,
            new MapList(fromSequenceRanges, toVcfRanges, 1, 1));
  }

  /**
   * If the sequence id matches a contig declared in the VCF file, and the
   * sequence length matches the contig length, then returns a 1:1 map of the
   * sequence to the contig, else returns null
   * 
   * @param seq
   * @return
   */
  private VCFMap getContigMap(SequenceI seq)
  {
    String id = seq.getName();
    SAMSequenceRecord contig = dictionary.getSequence(id);
    if (contig != null)
    {
      int len = seq.getLength();
      if (len == contig.getSequenceLength())
      {
        MapList map = new MapList(new int[] { 1, len },
                new int[]
                { 1, len }, 1, 1);
        return new VCFMap(id, map);
      }
    }
    return null;
  }

  /**
   * Answers true if we determine that the VCF data uses the same reference
   * assembly as the sequence, else false
   * 
   * @param vcfAssembly
   * @param seqRef
   * @return
   */
  private boolean vcfAssemblyMatchesSequence(String vcfAssembly,
          String seqRef)
  {
    // TODO improve on this stub, which handles gnomAD and
    // hopes for the best for other cases

    if ("GRCh38".equalsIgnoreCase(seqRef) // Ensembl
            && vcfAssembly.contains("Homo_sapiens_assembly19")) // gnomAD
    {
      return false;
    }
    return true;
  }

  /**
   * Answers true if the species inferred from the VCF reference identifier
   * matches that for the sequence
   * 
   * @param vcfAssembly
   * @param speciesId
   * @return
   */
  boolean vcfSpeciesMatchesSequence(String vcfAssembly, String speciesId)
  {
    // PROBLEM 1
    // there are many aliases for species - how to equate one with another?
    // PROBLEM 2
    // VCF ##reference header is an unstructured URI - how to extract species?
    // perhaps check if ref includes any (Ensembl) alias of speciesId??
    // TODO ask the user to confirm this??

    if (vcfAssembly.contains("Homo_sapiens") // gnomAD exome data example
            && "HOMO_SAPIENS".equals(speciesId)) // Ensembl species id
    {
      return true;
    }

    if (vcfAssembly.contains("c_elegans") // VEP VCF response example
            && "CAENORHABDITIS_ELEGANS".equals(speciesId)) // Ensembl
    {
      return true;
    }

    // this is not a sustainable solution...

    return false;
  }

  /**
   * Queries the VCF reader for any variants that overlap the mapped chromosome
   * ranges of the sequence, and adds as variant features. Returns the number of
   * overlapping variants found.
   * 
   * @param seq
   * @param reader
   * @param map
   *          mapping from sequence to VCF coordinates
   * @param vcfAssembly
   *          the '##reference' identifier for the VCF reference assembly
   * @return
   */
  protected int addVcfVariants(SequenceI seq, VCFReader reader,
          VCFMap map, String vcfAssembly)
  {
    boolean forwardStrand = map.map.isToForwardStrand();

    /*
     * query the VCF for overlaps of each contiguous chromosomal region
     */
    int count = 0;

    for (int[] range : map.map.getToRanges())
    {
      int vcfStart = Math.min(range[0], range[1]);
      int vcfEnd = Math.max(range[0], range[1]);
      CloseableIterator<VariantContext> variants = reader
              .query(map.chromosome, vcfStart, vcfEnd);
      while (variants.hasNext())
      {
        VariantContext variant = variants.next();

        int[] featureRange = map.map.locateInFrom(variant.getStart(),
                variant.getEnd());

        if (featureRange != null)
        {
          int featureStart = Math.min(featureRange[0], featureRange[1]);
          int featureEnd = Math.max(featureRange[0], featureRange[1]);
          count += addAlleleFeatures(seq, variant, featureStart, featureEnd,
                  forwardStrand);
        }
      }
      variants.close();
    }

    return count;
  }

  /**
   * A convenience method to get the AF value for the given alternate allele
   * index
   * 
   * @param variant
   * @param alleleIndex
   * @return
   */
  protected float getAlleleFrequency(VariantContext variant, int alleleIndex)
  {
    float score = 0f;
    String attributeValue = getAttributeValue(variant,
            ALLELE_FREQUENCY_KEY, alleleIndex);
    if (attributeValue != null)
    {
      try
      {
        score = Float.parseFloat(attributeValue);
      } catch (NumberFormatException e)
      {
        // leave as 0
      }
    }

    return score;
  }

  /**
   * A convenience method to get an attribute value for an alternate allele
   * 
   * @param variant
   * @param attributeName
   * @param alleleIndex
   * @return
   */
  protected String getAttributeValue(VariantContext variant,
          String attributeName, int alleleIndex)
  {
    Object att = variant.getAttribute(attributeName);

    if (att instanceof String)
    {
      return (String) att;
    }
    else if (att instanceof ArrayList)
    {
      return ((List<String>) att).get(alleleIndex);
    }

    return null;
  }

  /**
   * Adds one variant feature for each allele in the VCF variant record, and
   * returns the number of features added.
   * 
   * @param seq
   * @param variant
   * @param featureStart
   * @param featureEnd
   * @param forwardStrand
   * @return
   */
  protected int addAlleleFeatures(SequenceI seq, VariantContext variant,
          int featureStart, int featureEnd, boolean forwardStrand)
  {
    int added = 0;

    /*
     * Javadoc says getAlternateAlleles() imposes no order on the list returned
     * so we proceed defensively to get them in strict order
     */
    int altAlleleCount = variant.getAlternateAlleles().size();
    for (int i = 0; i < altAlleleCount; i++)
    {
      added += addAlleleFeature(seq, variant, i, featureStart, featureEnd,
              forwardStrand);
    }
    return added;
  }

  /**
   * Inspects one allele and attempts to add a variant feature for it to the
   * sequence. We extract as much as possible of the additional data associated
   * with this allele to store in the feature's key-value map. Answers the
   * number of features added (0 or 1).
   * 
   * @param seq
   * @param variant
   * @param altAlleleIndex
   *          (0, 1..)
   * @param featureStart
   * @param featureEnd
   * @param forwardStrand
   * @return
   */
  protected int addAlleleFeature(SequenceI seq, VariantContext variant,
          int altAlleleIndex, int featureStart, int featureEnd,
          boolean forwardStrand)
  {
    String reference = variant.getReference().getBaseString();
    Allele alt = variant.getAlternateAllele(altAlleleIndex);
    String allele = alt.getBaseString();

    /*
     * build the ref,alt allele description e.g. "G,A", using the base
     * complement if the sequence is on the reverse strand
     */
    // TODO check how structural variants are shown on reverse strand
    StringBuilder sb = new StringBuilder();
    sb.append(forwardStrand ? reference : Dna.reverseComplement(reference));
    sb.append(COMMA);
    sb.append(forwardStrand ? allele : Dna.reverseComplement(allele));
    String alleles = sb.toString(); // e.g. G,A

    String type = SequenceOntologyI.SEQUENCE_VARIANT;
    float score = getAlleleFrequency(variant, altAlleleIndex);

    SequenceFeature sf = new SequenceFeature(type, alleles, featureStart,
            featureEnd, score, FEATURE_GROUP_VCF);
    sf.setSource(sourceId);

    sf.setValue(Gff3Helper.ALLELES, alleles);

    addAlleleProperties(variant, seq, sf, altAlleleIndex);

    seq.addSequenceFeature(sf);

    return 1;
  }

  /**
   * Add any allele-specific VCF key-value data to the sequence feature
   * 
   * @param variant
   * @param seq
   * @param sf
   * @param altAlelleIndex
   *          (0, 1..)
   */
  protected void addAlleleProperties(VariantContext variant, SequenceI seq,
          SequenceFeature sf, final int altAlelleIndex)
  {
    Map<String, Object> atts = variant.getAttributes();

    for (Entry<String, Object> att : atts.entrySet())
    {
      String key = att.getKey();

      /*
       * extract Consequence data (if present) that we are able to
       * associated with the allele for this variant feature
       */
      if (CSQ_FIELD.equals(key))
      {
        addConsequences(variant, seq, sf, altAlelleIndex);
        continue;
      }

      /*
       * filter out fields we don't want to capture
       */
      if (!vcfFieldsOfInterest.contains(key))
      {
        continue;
      }

      /*
       * we extract values for other data which are allele-specific; 
       * these may be per alternate allele (INFO[key].Number = 'A') 
       * or per allele including reference (INFO[key].Number = 'R') 
       */
      VCFInfoHeaderLine infoHeader = header.getInfoHeaderLine(key);
      if (infoHeader == null)
      {
        /*
         * can't be sure what data belongs to this allele, so
         * play safe and don't take any
         */
        continue;
      }

      VCFHeaderLineCount number = infoHeader.getCountType();
      int index = altAlelleIndex;
      if (number == VCFHeaderLineCount.R)
      {
        /*
         * one value per allele including reference, so bump index
         * e.g. the 3rd value is for the  2nd alternate allele
         */
        index++;
      }
      else if (number != VCFHeaderLineCount.A)
      {
        /*
         * don't save other values as not allele-related
         */
        continue;
      }

      /*
       * take the index'th value
       */
      String value = getAttributeValue(variant, key, index);
      if (value != null)
      {
        sf.setValue(key, value);
      }
    }
  }

  /**
   * Inspects CSQ data blocks (consequences) and adds attributes on the sequence
   * feature for the current allele (and transcript if applicable)
   * <p>
   * Allele matching: if field ALLELE_NUM is present, it must match
   * altAlleleIndex. If not present, then field Allele value must match the VCF
   * Allele.
   * <p>
   * Transcript matching: if sequence name can be identified to at least one of
   * the consequences' Feature values, then select only consequences that match
   * the value (i.e. consequences for the current transcript sequence). If not,
   * take all consequences (this is the case when adding features to the gene
   * sequence).
   * 
   * @param variant
   * @param seq
   * @param sf
   * @param altAlelleIndex
   *          (0, 1..)
   */
  protected void addConsequences(VariantContext variant, SequenceI seq,
          SequenceFeature sf, int altAlelleIndex)
  {
    Object value = variant.getAttribute(CSQ_FIELD);

    if (value == null || !(value instanceof ArrayList<?>))
    {
      return;
    }

    List<String> consequences = (List<String>) value;

    /*
     * if CSQ data includes 'Feature', and any value matches the sequence name,
     * then restrict consequence data to only the matching value (transcript)
     * i.e. just pick out consequences for the transcript the variant feature is on
     */
    String seqName = seq.getName()== null ? "" : seq.getName().toLowerCase();
    String matchFeature = null;
    if (csqFeatureFieldIndex > -1)
    {
      for (String consequence : consequences)
      {
        String[] csqFields = consequence.split(PIPE_REGEX);
        if (csqFields.length > csqFeatureFieldIndex)
        {
          String featureIdentifier = csqFields[csqFeatureFieldIndex];
          if (featureIdentifier.length() > 4
                  && seqName.indexOf(featureIdentifier.toLowerCase()) > -1)
          {
            matchFeature = featureIdentifier;
          }
        }
      }
    }

    /*
     * inspect CSQ consequences; where possible restrict to the consequence
     * associated with the current transcript (Feature)
     */
    SortedMap<String, String> csqValues = new TreeMap<>(
            String.CASE_INSENSITIVE_ORDER);

    for (String consequence : consequences)
    {
      String[] csqFields = consequence.split(PIPE_REGEX);

      if (includeConsequence(csqFields, matchFeature, variant,
              altAlelleIndex))
      {
        /*
         * inspect individual fields of this consequence, copying non-null
         * values which are 'fields of interest'
         */
        int i = 0;
        for (String field : csqFields)
        {
          if (field != null && field.length() > 0)
          {
            String id = vepFieldsOfInterest.get(i);
            if (id != null)
            {
              csqValues.put(id, field);
            }
          }
          i++;
        }
      }
    }

    if (!csqValues.isEmpty())
    {
      sf.setValue(CSQ_FIELD, csqValues);
    }
  }

  /**
   * Answers true if we want to associate this block of consequence data with
   * the specified alternate allele of the VCF variant.
   * <p>
   * If consequence data includes the ALLELE_NUM field, then this has to match
   * altAlleleIndex. Otherwise the Allele field of the consequence data has to
   * match the allele value.
   * <p>
   * Optionally (if matchFeature is not null), restrict to only include
   * consequences whose Feature value matches. This allows us to attach
   * consequences to their respective transcripts.
   * 
   * @param csqFields
   * @param matchFeature
   * @param variant
   * @param altAlelleIndex
   *          (0, 1..)
   * @return
   */
  protected boolean includeConsequence(String[] csqFields,
          String matchFeature, VariantContext variant, int altAlelleIndex)
  {
    /*
     * check consequence is for the current transcript
     */
    if (matchFeature != null)
    {
      if (csqFields.length <= csqFeatureFieldIndex)
      {
        return false;
      }
      String featureIdentifier = csqFields[csqFeatureFieldIndex];
      if (!featureIdentifier.equals(matchFeature))
      {
        return false; // consequence is for a different transcript
      }
    }

    /*
     * if ALLELE_NUM is present, it must match altAlleleIndex
     * NB first alternate allele is 1 for ALLELE_NUM, 0 for altAlleleIndex
     */
    if (csqAlleleNumberFieldIndex > -1)
    {
      if (csqFields.length <= csqAlleleNumberFieldIndex)
      {
        return false;
      }
      String alleleNum = csqFields[csqAlleleNumberFieldIndex];
      return String.valueOf(altAlelleIndex + 1).equals(alleleNum);
    }

    /*
     * else consequence allele must match variant allele
     */
    if (csqAlleleFieldIndex > -1 && csqFields.length > csqAlleleFieldIndex)
    {
      String csqAllele = csqFields[csqAlleleFieldIndex];
      String vcfAllele = variant.getAlternateAllele(altAlelleIndex)
              .getBaseString();
      return csqAllele.equals(vcfAllele);
    }

    return false;
  }

  /**
   * A convenience method to complement a dna base and return the string value
   * of its complement
   * 
   * @param reference
   * @return
   */
  protected String complement(byte[] reference)
  {
    return String.valueOf(Dna.getComplement((char) reference[0]));
  }

  /**
   * Determines the location of the query range (chromosome positions) in a
   * different reference assembly.
   * <p>
   * If the range is just a subregion of one for which we already have a mapping
   * (for example, an exon sub-region of a gene), then the mapping is just
   * computed arithmetically.
   * <p>
   * Otherwise, calls the Ensembl REST service that maps from one assembly
   * reference's coordinates to another's
   * 
   * @param queryRange
   *          start-end chromosomal range in 'fromRef' coordinates
   * @param chromosome
   * @param species
   * @param fromRef
   *          assembly reference for the query coordinates
   * @param toRef
   *          assembly reference we wish to translate to
   * @return the start-end range in 'toRef' coordinates
   */
  protected int[] mapReferenceRange(int[] queryRange, String chromosome,
          String species, String fromRef, String toRef)
  {
    /*
     * first try shorcut of computing the mapping as a subregion of one
     * we already have (e.g. for an exon, if we have the gene mapping)
     */
    int[] mappedRange = findSubsumedRangeMapping(queryRange, chromosome,
            species, fromRef, toRef);
    if (mappedRange != null)
    {
      return mappedRange;
    }

    /*
     * call (e.g.) http://rest.ensembl.org/map/human/GRCh38/17:45051610..45109016:1/GRCh37
     */
    EnsemblMap mapper = new EnsemblMap();
    int[] mapping = mapper.getAssemblyMapping(species, chromosome, fromRef,
            toRef, queryRange);

    if (mapping == null)
    {
      // mapping service failure
      return null;
    }

    /*
     * save mapping for possible future re-use
     */
    String key = makeRangesKey(chromosome, species, fromRef, toRef);
    if (!assemblyMappings.containsKey(key))
    {
      assemblyMappings.put(key, new HashMap<int[], int[]>());
    }

    assemblyMappings.get(key).put(queryRange, mapping);

    return mapping;
  }

  /**
   * If we already have a 1:1 contiguous mapping which subsumes the given query
   * range, this method just calculates and returns the subset of that mapping,
   * else it returns null. In practical terms, if a gene has a contiguous
   * mapping between (for example) GRCh37 and GRCh38, then we assume that its
   * subsidiary exons occupy unchanged relative positions, and just compute
   * these as offsets, rather than do another lookup of the mapping.
   * <p>
   * If in future these assumptions prove invalid (e.g. for bacterial dna?!),
   * simply remove this method or let it always return null.
   * <p>
   * Warning: many rapid calls to the /map service map result in a 429 overload
   * error response
   * 
   * @param queryRange
   * @param chromosome
   * @param species
   * @param fromRef
   * @param toRef
   * @return
   */
  protected int[] findSubsumedRangeMapping(int[] queryRange, String chromosome,
          String species, String fromRef, String toRef)
  {
    String key = makeRangesKey(chromosome, species, fromRef, toRef);
    if (assemblyMappings.containsKey(key))
    {
      Map<int[], int[]> mappedRanges = assemblyMappings.get(key);
      for (Entry<int[], int[]> mappedRange : mappedRanges.entrySet())
      {
        int[] fromRange = mappedRange.getKey();
        int[] toRange = mappedRange.getValue();
        if (fromRange[1] - fromRange[0] == toRange[1] - toRange[0])
        {
          /*
           * mapping is 1:1 in length, so we trust it to have no discontinuities
           */
          if (MappingUtils.rangeContains(fromRange, queryRange))
          {
            /*
             * fromRange subsumes our query range
             */
            int offset = queryRange[0] - fromRange[0];
            int mappedRangeFrom = toRange[0] + offset;
            int mappedRangeTo = mappedRangeFrom + (queryRange[1] - queryRange[0]);
            return new int[] { mappedRangeFrom, mappedRangeTo };
          }
        }
      }
    }
    return null;
  }

  /**
   * Transfers the sequence feature to the target sequence, locating its start
   * and end range based on the mapping. Features which do not overlap the
   * target sequence are ignored.
   * 
   * @param sf
   * @param targetSequence
   * @param mapping
   *          mapping from the feature's coordinates to the target sequence
   */
  protected void transferFeature(SequenceFeature sf,
          SequenceI targetSequence, MapList mapping)
  {
    int[] mappedRange = mapping.locateInTo(sf.getBegin(), sf.getEnd());
  
    if (mappedRange != null)
    {
      String group = sf.getFeatureGroup();
      int newBegin = Math.min(mappedRange[0], mappedRange[1]);
      int newEnd = Math.max(mappedRange[0], mappedRange[1]);
      SequenceFeature copy = new SequenceFeature(sf, newBegin, newEnd,
              group, sf.getScore());
      targetSequence.addSequenceFeature(copy);
    }
  }

  /**
   * Formats a ranges map lookup key
   * 
   * @param chromosome
   * @param species
   * @param fromRef
   * @param toRef
   * @return
   */
  protected static String makeRangesKey(String chromosome, String species,
          String fromRef, String toRef)
  {
    return species + EXCL + chromosome + EXCL + fromRef + EXCL
            + toRef;
  }
}