X-Git-Url: http://source.jalview.org/gitweb/?a=blobdiff_plain;f=src%2Fjalview%2Fio%2Fvcf%2FVCFLoader.java;h=9d98b7e6c675c2967424790c4f22e77c86161df1;hb=b87ae5ac68939a1b964682046e8b07958fae219a;hp=ddcecfe2a802b89b06347d0839ed2c68ef7030b7;hpb=44b1e9ba73a6d6a321126e153aece11699eb1d2a;p=jalview.git diff --git a/src/jalview/io/vcf/VCFLoader.java b/src/jalview/io/vcf/VCFLoader.java index ddcecfe..9d98b7e 100644 --- a/src/jalview/io/vcf/VCFLoader.java +++ b/src/jalview/io/vcf/VCFLoader.java @@ -1,20 +1,18 @@ package jalview.io.vcf; -import htsjdk.samtools.util.CloseableIterator; -import htsjdk.variant.variantcontext.Allele; -import htsjdk.variant.variantcontext.VariantContext; -import htsjdk.variant.vcf.VCFHeader; -import htsjdk.variant.vcf.VCFHeaderLine; - import jalview.analysis.AlignmentUtils; import jalview.analysis.Dna; import jalview.api.AlignViewControllerGuiI; +import jalview.bin.Cache; import jalview.datamodel.AlignmentI; import jalview.datamodel.DBRefEntry; import jalview.datamodel.GeneLociI; import jalview.datamodel.Mapping; import jalview.datamodel.SequenceFeature; import jalview.datamodel.SequenceI; +import jalview.datamodel.features.FeatureAttributeType; +import jalview.datamodel.features.FeatureSource; +import jalview.datamodel.features.FeatureSources; import jalview.ext.ensembl.EnsemblMap; import jalview.ext.htsjdk.VCFReader; import jalview.io.gff.Gff3Helper; @@ -24,10 +22,25 @@ import jalview.util.MappingUtils; import jalview.util.MessageManager; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +import htsjdk.samtools.SAMException; +import htsjdk.samtools.SAMSequenceDictionary; +import htsjdk.samtools.SAMSequenceRecord; +import htsjdk.samtools.util.CloseableIterator; +import htsjdk.variant.variantcontext.Allele; +import htsjdk.variant.variantcontext.VariantContext; +import htsjdk.variant.vcf.VCFHeader; +import htsjdk.variant.vcf.VCFHeaderLine; +import htsjdk.variant.vcf.VCFHeaderLineCount; +import htsjdk.variant.vcf.VCFHeaderLineType; +import htsjdk.variant.vcf.VCFInfoHeaderLine; /** * A class to read VCF data (using the htsjdk) and add variants as sequence @@ -37,12 +50,92 @@ import java.util.Map.Entry; */ public class VCFLoader { + /** + * A class to model the mapping from sequence to VCF coordinates. Cases include + * + */ + class VCFMap + { + final String chromosome; + + final MapList map; + + VCFMap(String chr, MapList m) + { + chromosome = chr; + map = m; + } + + @Override + public String toString() + { + return chromosome + ":" + map.toString(); + } + } + + /* + * Lookup keys, and default values, for Preference entries that describe + * patterns for VCF and VEP fields to capture + */ + private static final String VEP_FIELDS_PREF = "VEP_FIELDS"; + + private static final String VCF_FIELDS_PREF = "VCF_FIELDS"; + + private static final String DEFAULT_VCF_FIELDS = ".*"; + + private static final String DEFAULT_VEP_FIELDS = ".*";// "Allele,Consequence,IMPACT,SWISSPROT,SIFT,PolyPhen,CLIN_SIG"; + + /* + * keys to fields of VEP CSQ consequence data + * see https://www.ensembl.org/info/docs/tools/vep/vep_formats.html + */ + private static final String CSQ_CONSEQUENCE_KEY = "Consequence"; + private static final String CSQ_ALLELE_KEY = "Allele"; + private static final String CSQ_ALLELE_NUM_KEY = "ALLELE_NUM"; // 0 (ref), 1... + private static final String CSQ_FEATURE_KEY = "Feature"; // Ensembl stable id + + /* + * default VCF INFO key for VEP consequence data + * NB this can be overridden running VEP with --vcf_info_field + * - we don't handle this case (require identifier to be CSQ) + */ + private static final String CSQ_FIELD = "CSQ"; + + /* + * separator for fields in consequence data is '|' + */ + private static final String PIPE_REGEX = "\\|"; + + /* + * key for Allele Frequency output by VEP + * see http://www.ensembl.org/info/docs/tools/vep/vep_formats.html + */ + private static final String ALLELE_FREQUENCY_KEY = "AF"; + + /* + * delimiter that separates multiple consequence data blocks + */ + private static final String COMMA = ","; + + /* + * the feature group assigned to a VCF variant in Jalview + */ private static final String FEATURE_GROUP_VCF = "VCF"; + /* + * internal delimiter used to build keys for assemblyMappings + * + */ private static final String EXCL = "!"; /* - * the alignment we are associated VCF data with + * the alignment we are associating VCF data with */ private AlignmentI al; @@ -53,6 +146,48 @@ public class VCFLoader */ private Map> assemblyMappings; + /* + * holds details of the VCF header lines (metadata) + */ + private VCFHeader header; + + /* + * a Dictionary of contigs (if present) referenced in the VCF file + */ + private SAMSequenceDictionary dictionary; + + /* + * the position (0...) of field in each block of + * CSQ (consequence) data (if declared in the VCF INFO header for CSQ) + * see http://www.ensembl.org/info/docs/tools/vep/vep_formats.html + */ + private int csqConsequenceFieldIndex = -1; + private int csqAlleleFieldIndex = -1; + private int csqAlleleNumberFieldIndex = -1; + private int csqFeatureFieldIndex = -1; + + // todo the same fields for SnpEff ANN data if wanted + // see http://snpeff.sourceforge.net/SnpEff_manual.html#input + + /* + * a unique identifier under which to save metadata about feature + * attributes (selected INFO field data) + */ + private String sourceId; + + /* + * The INFO IDs of data that is both present in the VCF file, and + * also matched by any filters for data of interest + */ + List vcfFieldsOfInterest; + + /* + * The field offsets and identifiers for VEP (CSQ) data that is both present + * in the VCF file, and also matched by any filters for data of interest + * for example 0 -> Allele, 1 -> Consequence, ..., 36 -> SIFT, ... + */ + Map vepFieldsOfInterest; + /** * Constructor given an alignment context * @@ -63,7 +198,7 @@ public class VCFLoader al = alignment; // map of species!chromosome!fromAssembly!toAssembly to {fromRange, toRange} - assemblyMappings = new HashMap>(); + assemblyMappings = new HashMap<>(); } /** @@ -97,10 +232,11 @@ public class VCFLoader /** * Loads VCF on to an alignment - provided it can be related to one or more - * sequence's chromosomal coordinates. + * sequence's chromosomal coordinates * * @param filePath * @param gui + * optional callback handler for messages */ protected void doLoad(String filePath, AlignViewControllerGuiI gui) { @@ -110,13 +246,28 @@ public class VCFLoader // long start = System.currentTimeMillis(); reader = new VCFReader(filePath); - VCFHeader header = reader.getFileHeader(); + header = reader.getFileHeader(); + + try + { + dictionary = header.getSequenceDictionary(); + } catch (SAMException e) + { + // ignore - thrown if any contig line lacks length info + } + + sourceId = filePath; + + saveMetadata(sourceId); + + /* + * get offset of CSQ ALLELE_NUM and Feature if declared + */ + parseCsqHeader(); + VCFHeaderLine ref = header .getOtherHeaderLine(VCFHeader.REFERENCE_KEY); - // check if reference is wrt assembly19 (GRCh37) - // todo may need to allow user to specify reference assembly? - boolean isRefGrch37 = (ref != null && ref.getValue().contains( - "assembly19")); + String vcfAssembly = ref.getValue(); int varCount = 0; int seqCount = 0; @@ -126,7 +277,7 @@ public class VCFLoader */ for (SequenceI seq : al.getSequences()) { - int added = loadVCF(seq, reader, isRefGrch37); + int added = loadSequenceVCF(seq, reader, vcfAssembly); if (added > 0) { seqCount++; @@ -140,7 +291,10 @@ public class VCFLoader String msg = MessageManager.formatMessage("label.added_vcf", varCount, seqCount); gui.setStatus(msg); - gui.getFeatureSettingsUI().discoverAllFeatureData(); + if (gui.getFeatureSettingsUI() != null) + { + gui.getFeatureSettingsUI().discoverAllFeatureData(); + } } } catch (Throwable e) { @@ -162,12 +316,180 @@ public class VCFLoader // ignore } } + header = null; + dictionary = null; + } + } + + /** + * Reads metadata (such as INFO field descriptions and datatypes) and saves + * them for future reference + * + * @param theSourceId + */ + void saveMetadata(String theSourceId) + { + List vcfFieldPatterns = getFieldMatchers(VCF_FIELDS_PREF, + DEFAULT_VCF_FIELDS); + vcfFieldsOfInterest = new ArrayList<>(); + + FeatureSource metadata = new FeatureSource(theSourceId); + + for (VCFInfoHeaderLine info : header.getInfoHeaderLines()) + { + String attributeId = info.getID(); + String desc = info.getDescription(); + VCFHeaderLineType type = info.getType(); + FeatureAttributeType attType = null; + switch (type) + { + case Character: + attType = FeatureAttributeType.Character; + break; + case Flag: + attType = FeatureAttributeType.Flag; + break; + case Float: + attType = FeatureAttributeType.Float; + break; + case Integer: + attType = FeatureAttributeType.Integer; + break; + case String: + attType = FeatureAttributeType.String; + break; + } + metadata.setAttributeName(attributeId, desc); + metadata.setAttributeType(attributeId, attType); + + if (isFieldWanted(attributeId, vcfFieldPatterns)) + { + vcfFieldsOfInterest.add(attributeId); + } + } + + FeatureSources.getInstance().addSource(theSourceId, metadata); + } + + /** + * Answers true if the field id is matched by any of the filter patterns, else + * false. Matching is against regular expression patterns, and is not + * case-sensitive. + * + * @param id + * @param filters + * @return + */ + private boolean isFieldWanted(String id, List filters) + { + for (Pattern p : filters) + { + if (p.matcher(id.toUpperCase()).matches()) + { + return true; + } + } + return false; + } + + /** + * Records 'wanted' fields defined in the CSQ INFO header (if there is one). + * Also records the position of selected fields (Allele, ALLELE_NUM, Feature) + * required for processing. + *

+ * CSQ fields are declared in the CSQ INFO Description e.g. + *

+ * Description="Consequence ...from ... VEP. Format: Allele|Consequence|... + */ + protected void parseCsqHeader() + { + List vepFieldFilters = getFieldMatchers(VEP_FIELDS_PREF, + DEFAULT_VEP_FIELDS); + vepFieldsOfInterest = new HashMap<>(); + + VCFInfoHeaderLine csqInfo = header.getInfoHeaderLine(CSQ_FIELD); + if (csqInfo == null) + { + return; + } + + /* + * parse out the pipe-separated list of CSQ fields; we assume here that + * these form the last part of the description, and contain no spaces + */ + String desc = csqInfo.getDescription(); + int spacePos = desc.lastIndexOf(" "); + desc = desc.substring(spacePos + 1); + + if (desc != null) + { + String[] format = desc.split(PIPE_REGEX); + int index = 0; + for (String field : format) + { + if (CSQ_CONSEQUENCE_KEY.equals(field)) + { + csqConsequenceFieldIndex = index; + } + if (CSQ_ALLELE_NUM_KEY.equals(field)) + { + csqAlleleNumberFieldIndex = index; + } + if (CSQ_ALLELE_KEY.equals(field)) + { + csqAlleleFieldIndex = index; + } + if (CSQ_FEATURE_KEY.equals(field)) + { + csqFeatureFieldIndex = index; + } + + if (isFieldWanted(field, vepFieldFilters)) + { + vepFieldsOfInterest.put(index, field); + } + + index++; + } + } + } + + /** + * Reads the Preference value for the given key, with default specified if no + * preference set. The value is interpreted as a comma-separated list of + * regular expressions, and converted into a list of compiled patterns ready + * for matching. Patterns are forced to upper-case for non-case-sensitive + * matching. + *

+ * This supports user-defined filters for fields of interest to capture while + * processing data. For example, VCF_FIELDS = AF,AC* would mean that VCF INFO + * fields with an ID of AF, or starting with AC, would be matched. + * + * @param key + * @param def + * @return + */ + private List getFieldMatchers(String key, String def) + { + String pref = Cache.getDefault(key, def); + List patterns = new ArrayList<>(); + String[] tokens = pref.split(","); + for (String token : tokens) + { + try + { + patterns.add(Pattern.compile(token.toUpperCase())); + } catch (PatternSyntaxException e) + { + System.err.println("Invalid pattern ignored: " + token); + } } + return patterns; } /** * Transfers VCF features to sequences to which this sequence has a mapping. - * If the mapping is 1:3, computes peptide variants from nucleotide variants. + * If the mapping is 3:1, computes peptide variants from nucleotide variants. * * @param seq */ @@ -200,7 +522,6 @@ public class VCFLoader /* * nucleotide-to-nucleotide mapping e.g. transcript to CDS */ - // TODO no DBRef to CDS is added to transcripts List features = seq.getFeatures() .getPositionalFeatures(SequenceOntologyI.SEQUENCE_VARIANT); for (SequenceFeature sf : features) @@ -216,217 +537,731 @@ public class VCFLoader /** * Tries to add overlapping variants read from a VCF file to the given - * sequence, and returns the number of overlapping variants found. Note that - * this requires the sequence to hold information as to its chromosomal - * positions and reference, in order to be able to map the VCF variants to the - * sequence. + * sequence, and returns the number of variant features added. Note that this + * requires the sequence to hold information as to its species, chromosomal + * positions and reference assembly, in order to be able to map the VCF + * variants to the sequence (or not) * * @param seq * @param reader - * @param isVcfRefGrch37 + * @param vcfAssembly * @return */ - protected int loadVCF(SequenceI seq, VCFReader reader, - boolean isVcfRefGrch37) + protected int loadSequenceVCF(SequenceI seq, VCFReader reader, + String vcfAssembly) { - int count = 0; - GeneLociI seqCoords = seq.getGeneLoci(); - if (seqCoords == null) + VCFMap vcfMap = getVcfMap(seq, vcfAssembly); + if (vcfMap == null) { return 0; } - List seqChromosomalContigs = seqCoords.getMap().getToRanges(); - for (int[] range : seqChromosomalContigs) + /* + * work with the dataset sequence here + */ + SequenceI dss = seq.getDatasetSequence(); + if (dss == null) { - count += addVcfVariants(seq, reader, range, isVcfRefGrch37); + dss = seq; } - - return count; + return addVcfVariants(dss, reader, vcfMap, vcfAssembly); } /** - * Queries the VCF reader for any variants that overlap the given chromosome - * region of the sequence, and adds as variant features. Returns the number of - * overlapping variants found. + * Answers a map from sequence coordinates to VCF chromosome ranges * * @param seq - * @param reader - * @param range - * start-end range of a sequence region in its chromosomal - * coordinates - * @param isVcfRefGrch37 - * true if the VCF is with reference to GRCh37 + * @param vcfAssembly * @return */ - protected int addVcfVariants(SequenceI seq, VCFReader reader, - int[] range, boolean isVcfRefGrch37) + private VCFMap getVcfMap(SequenceI seq, String vcfAssembly) { + /* + * simplest case: sequence has id and length matching a VCF contig + */ + VCFMap vcfMap = null; + if (dictionary != null) + { + vcfMap = getContigMap(seq); + } + if (vcfMap != null) + { + return vcfMap; + } + + /* + * otherwise, map to VCF from chromosomal coordinates + * of the sequence (if known) + */ GeneLociI seqCoords = seq.getGeneLoci(); + if (seqCoords == null) + { + Cache.log.warn(String.format( + "Can't query VCF for %s as chromosome coordinates not known", + seq.getName())); + return null; + } + String species = seqCoords.getSpeciesId(); String chromosome = seqCoords.getChromosomeId(); String seqRef = seqCoords.getAssemblyId(); - String species = seqCoords.getSpeciesId(); + MapList map = seqCoords.getMap(); - // TODO handle species properly - if ("".equals(species)) + if (!vcfSpeciesMatchesSequence(vcfAssembly, species)) { - species = "human"; + return null; + } + + if (vcfAssemblyMatchesSequence(vcfAssembly, seqRef)) + { + return new VCFMap(chromosome, map); + } + + if (!"GRCh38".equalsIgnoreCase(seqRef) // Ensembl + || !vcfAssembly.contains("Homo_sapiens_assembly19")) // gnomAD + { + return null; } /* - * map chromosomal coordinates from GRCh38 (sequence) to - * GRCh37 (VCF) if necessary + * map chromosomal coordinates from sequence to VCF if the VCF + * data has a different reference assembly to the sequence */ - // TODO generalise for other assemblies and species - int offset = 0; - String fromRef = "GRCh38"; - if (fromRef.equalsIgnoreCase(seqRef) && isVcfRefGrch37) - { - String toRef = "GRCh37"; - int[] newRange = mapReferenceRange(range, chromosome, species, - fromRef, toRef); + // TODO generalise for cases other than GRCh38 -> GRCh37 ! + // - or get the user to choose in a dialog + + List toVcfRanges = new ArrayList<>(); + List fromSequenceRanges = new ArrayList<>(); + String toRef = "GRCh37"; + + for (int[] range : map.getToRanges()) + { + int[] fromRange = map.locateInFrom(range[0], range[1]); + if (fromRange == null) + { + // corrupted map?!? + continue; + } + + int[] newRange = mapReferenceRange(range, chromosome, "human", seqRef, + toRef); if (newRange == null) { - System.err.println(String.format( - "Failed to map %s:%s:%s:%d:%d to %s", species, chromosome, - fromRef, range[0], range[1], toRef)); - return 0; + Cache.log.error( + String.format("Failed to map %s:%s:%s:%d:%d to %s", species, + chromosome, seqRef, range[0], range[1], toRef)); + continue; + } + else + { + toVcfRanges.add(newRange); + fromSequenceRanges.add(fromRange); } - offset = newRange[0] - range[0]; - range = newRange; } - boolean forwardStrand = range[0] <= range[1]; + return new VCFMap(chromosome, + new MapList(fromSequenceRanges, toVcfRanges, 1, 1)); + } + + /** + * If the sequence id matches a contig declared in the VCF file, and the + * sequence length matches the contig length, then returns a 1:1 map of the + * sequence to the contig, else returns null + * + * @param seq + * @return + */ + private VCFMap getContigMap(SequenceI seq) + { + String id = seq.getName(); + SAMSequenceRecord contig = dictionary.getSequence(id); + if (contig != null) + { + int len = seq.getLength(); + if (len == contig.getSequenceLength()) + { + MapList map = new MapList(new int[] { 1, len }, + new int[] + { 1, len }, 1, 1); + return new VCFMap(id, map); + } + } + return null; + } + + /** + * Answers true if we determine that the VCF data uses the same reference + * assembly as the sequence, else false + * + * @param vcfAssembly + * @param seqRef + * @return + */ + private boolean vcfAssemblyMatchesSequence(String vcfAssembly, + String seqRef) + { + // TODO improve on this stub, which handles gnomAD and + // hopes for the best for other cases + + if ("GRCh38".equalsIgnoreCase(seqRef) // Ensembl + && vcfAssembly.contains("Homo_sapiens_assembly19")) // gnomAD + { + return false; + } + return true; + } + + /** + * Answers true if the species inferred from the VCF reference identifier + * matches that for the sequence + * + * @param vcfAssembly + * @param speciesId + * @return + */ + boolean vcfSpeciesMatchesSequence(String vcfAssembly, String speciesId) + { + // PROBLEM 1 + // there are many aliases for species - how to equate one with another? + // PROBLEM 2 + // VCF ##reference header is an unstructured URI - how to extract species? + // perhaps check if ref includes any (Ensembl) alias of speciesId?? + // TODO ask the user to confirm this?? + + if (vcfAssembly.contains("Homo_sapiens") // gnomAD exome data example + && "HOMO_SAPIENS".equals(speciesId)) // Ensembl species id + { + return true; + } + + if (vcfAssembly.contains("c_elegans") // VEP VCF response example + && "CAENORHABDITIS_ELEGANS".equals(speciesId)) // Ensembl + { + return true; + } + + // this is not a sustainable solution... + + return false; + } + + /** + * Queries the VCF reader for any variants that overlap the mapped chromosome + * ranges of the sequence, and adds as variant features. Returns the number of + * overlapping variants found. + * + * @param seq + * @param reader + * @param map + * mapping from sequence to VCF coordinates + * @param vcfAssembly + * the '##reference' identifier for the VCF reference assembly + * @return + */ + protected int addVcfVariants(SequenceI seq, VCFReader reader, + VCFMap map, String vcfAssembly) + { + boolean forwardStrand = map.map.isToForwardStrand(); /* - * query the VCF for overlaps - * (convert a reverse strand range to forwards) + * query the VCF for overlaps of each contiguous chromosomal region */ int count = 0; - MapList mapping = seqCoords.getMap(); - int fromLocus = Math.min(range[0], range[1]); - int toLocus = Math.max(range[0], range[1]); - CloseableIterator variants = reader.query(chromosome, - fromLocus, toLocus); - while (variants.hasNext()) + for (int[] range : map.map.getToRanges()) { - /* - * get variant location in sequence chromosomal coordinates - */ - VariantContext variant = variants.next(); - - /* - * we can only process SNP variants (which can be reported - * as part of a MIXED variant record - */ - if (!variant.isSNP() && !variant.isMixed()) + int vcfStart = Math.min(range[0], range[1]); + int vcfEnd = Math.max(range[0], range[1]); + CloseableIterator variants = reader + .query(map.chromosome, vcfStart, vcfEnd); + while (variants.hasNext()) { - continue; + VariantContext variant = variants.next(); + + int[] featureRange = map.map.locateInFrom(variant.getStart(), + variant.getEnd()); + + if (featureRange != null) + { + int featureStart = Math.min(featureRange[0], featureRange[1]); + int featureEnd = Math.max(featureRange[0], featureRange[1]); + count += addAlleleFeatures(seq, variant, featureStart, featureEnd, + forwardStrand); + } } + variants.close(); + } - count++; - int start = variant.getStart() - offset; - int end = variant.getEnd() - offset; + return count; + } - /* - * convert chromosomal location to sequence coordinates - * - null if a partially overlapping feature - */ - int[] seqLocation = mapping.locateInFrom(start, end); - if (seqLocation != null) + /** + * A convenience method to get the AF value for the given alternate allele + * index + * + * @param variant + * @param alleleIndex + * @return + */ + protected float getAlleleFrequency(VariantContext variant, int alleleIndex) + { + float score = 0f; + String attributeValue = getAttributeValue(variant, + ALLELE_FREQUENCY_KEY, alleleIndex); + if (attributeValue != null) + { + try + { + score = Float.parseFloat(attributeValue); + } catch (NumberFormatException e) { - addVariantFeatures(seq, variant, seqLocation[0], seqLocation[1], - forwardStrand); + // leave as 0 } } - variants.close(); + return score; + } - return count; + /** + * A convenience method to get an attribute value for an alternate allele + * + * @param variant + * @param attributeName + * @param alleleIndex + * @return + */ + protected String getAttributeValue(VariantContext variant, + String attributeName, int alleleIndex) + { + Object att = variant.getAttribute(attributeName); + + if (att instanceof String) + { + return (String) att; + } + else if (att instanceof ArrayList) + { + return ((List) att).get(alleleIndex); + } + + return null; } /** - * Inspects the VCF variant record, and adds variant features to the sequence. - * Only SNP variants are added, not INDELs. - *

- * If the sequence maps to the reverse strand of the chromosome, reference and - * variant bases are recorded as their complements (C/G, A/T). + * Adds one variant feature for each allele in the VCF variant record, and + * returns the number of features added. * * @param seq * @param variant * @param featureStart * @param featureEnd * @param forwardStrand + * @return */ - protected void addVariantFeatures(SequenceI seq, VariantContext variant, + protected int addAlleleFeatures(SequenceI seq, VariantContext variant, int featureStart, int featureEnd, boolean forwardStrand) { - byte[] reference = variant.getReference().getBases(); - if (reference.length != 1) + int added = 0; + + /* + * Javadoc says getAlternateAlleles() imposes no order on the list returned + * so we proceed defensively to get them in strict order + */ + int altAlleleCount = variant.getAlternateAlleles().size(); + for (int i = 0; i < altAlleleCount; i++) + { + added += addAlleleFeature(seq, variant, i, featureStart, featureEnd, + forwardStrand); + } + return added; + } + + /** + * Inspects one allele and attempts to add a variant feature for it to the + * sequence. The additional data associated with this allele is extracted to + * store in the feature's key-value map. Answers the number of features added (0 + * or 1). + * + * @param seq + * @param variant + * @param altAlleleIndex + * (0, 1..) + * @param featureStart + * @param featureEnd + * @param forwardStrand + * @return + */ + protected int addAlleleFeature(SequenceI seq, VariantContext variant, + int altAlleleIndex, int featureStart, int featureEnd, + boolean forwardStrand) + { + String reference = variant.getReference().getBaseString(); + Allele alt = variant.getAlternateAllele(altAlleleIndex); + String allele = alt.getBaseString(); + + /* + * insertion after a genomic base, if on reverse strand, has to be + * converted to insertion of complement after the preceding position + */ + int referenceLength = reference.length(); + if (!forwardStrand && allele.length() > referenceLength + && allele.startsWith(reference)) + { + featureStart -= referenceLength; + featureEnd = featureStart; + char insertAfter = seq.getCharAt(featureStart - seq.getStart()); + reference = Dna.reverseComplement(String.valueOf(insertAfter)); + allele = allele.substring(referenceLength) + reference; + } + + /* + * build the ref,alt allele description e.g. "G,A", using the base + * complement if the sequence is on the reverse strand + */ + StringBuilder sb = new StringBuilder(); + sb.append(forwardStrand ? reference : Dna.reverseComplement(reference)); + sb.append(COMMA); + sb.append(forwardStrand ? allele : Dna.reverseComplement(allele)); + String alleles = sb.toString(); // e.g. G,A + + /* + * pick out the consequence data (if any) that is for the current allele + * and feature (transcript) that matches the current sequence + */ + String consequence = getConsequenceForAlleleAndFeature(variant, CSQ_FIELD, + altAlleleIndex, csqAlleleFieldIndex, + csqAlleleNumberFieldIndex, seq.getName().toLowerCase(), + csqFeatureFieldIndex); + + /* + * pick out the ontology term for the consequence type + */ + String type = SequenceOntologyI.SEQUENCE_VARIANT; + if (consequence != null) + { + type = getOntologyTerm(seq, variant, altAlleleIndex, + consequence); + } + + float score = getAlleleFrequency(variant, altAlleleIndex); + + SequenceFeature sf = new SequenceFeature(type, alleles, featureStart, + featureEnd, score, FEATURE_GROUP_VCF); + sf.setSource(sourceId); + + sf.setValue(Gff3Helper.ALLELES, alleles); + + addAlleleProperties(variant, seq, sf, altAlleleIndex, consequence); + + seq.addSequenceFeature(sf); + + return 1; + } + + /** + * Determines the Sequence Ontology term to use for the variant feature type in + * Jalview. The default is 'sequence_variant', but a more specific term is used + * if: + *

    + *
  • VEP (or SnpEff) Consequence annotation is included in the VCF
  • + *
  • sequence id can be matched to VEP Feature (or SnpEff Feature_ID)
  • + *
+ * + * @param seq + * @param variant + * @param altAlleleIndex + * @param consequence + * @return + * @see http://www.sequenceontology.org/browser/current_svn/term/SO:0001060 + */ + String getOntologyTerm(SequenceI seq, VariantContext variant, + int altAlleleIndex, String consequence) + { + String type = SequenceOntologyI.SEQUENCE_VARIANT; + + if (csqAlleleFieldIndex == -1) // && snpEffAlleleFieldIndex == -1 { /* - * sorry, we don't handle INDEL variants + * no Consequence data so we can't refine the ontology term */ - return; + return type; } /* - * for now we extract allele frequency as feature score; note - * this attribute is String for a simple SNP, but List if - * multiple alleles at the locus; we extract for the simple case only + * can we associate Consequence data with this allele and feature (transcript)? + * if so, prefer the consequence term from that data */ - Object af = variant.getAttribute("AF"); - float score = 0f; - if (af instanceof String) + if (consequence != null) { - try + String[] csqFields = consequence.split(PIPE_REGEX); + if (csqFields.length > csqConsequenceFieldIndex) { - score = Float.parseFloat((String) af); - } catch (NumberFormatException e) + type = csqFields[csqConsequenceFieldIndex]; + } + } + else + { + // todo the same for SnpEff consequence data matching if wanted + } + + /* + * if of the form (e.g.) missense_variant&splice_region_variant, + * just take the first ('most severe') consequence + */ + if (type != null) + { + int pos = type.indexOf('&'); + if (pos > 0) { - // leave as 0 + type = type.substring(0, pos); } } + return type; + } - StringBuilder sb = new StringBuilder(); - sb.append(forwardStrand ? (char) reference[0] : complement(reference)); + /** + * Returns matched consequence data if it can be found, else null. + *
    + *
  • inspects the VCF data for key 'vcfInfoId'
  • + *
  • splits this on comma (to distinct consequences)
  • + *
  • returns the first consequence (if any) where
  • + *
      + *
    • the allele matches the altAlleleIndex'th allele of variant
    • + *
    • the feature matches the sequence name (e.g. transcript id)
    • + *
    + *
+ * If matched, the consequence is returned (as pipe-delimited fields). + * + * @param variant + * @param vcfInfoId + * @param altAlleleIndex + * @param alleleFieldIndex + * @param alleleNumberFieldIndex + * @param seqName + * @param featureFieldIndex + * @return + */ + private String getConsequenceForAlleleAndFeature(VariantContext variant, + String vcfInfoId, int altAlleleIndex, int alleleFieldIndex, + int alleleNumberFieldIndex, + String seqName, int featureFieldIndex) + { + if (alleleFieldIndex == -1 || featureFieldIndex == -1) + { + return null; + } + Object value = variant.getAttribute(vcfInfoId); + + if (value == null || !(value instanceof List)) + { + return null; + } /* - * inspect alleles and record SNP variants (as the variant - * record could be MIXED and include INDEL and SNP alleles) - * warning: getAlleles gives no guarantee as to the order - * in which they are returned + * inspect each consequence in turn (comma-separated blocks + * extracted by htsjdk) */ - for (Allele allele : variant.getAlleles()) + List consequences = (List) value; + + for (String consequence : consequences) { - if (!allele.isReference()) + String[] csqFields = consequence.split(PIPE_REGEX); + if (csqFields.length > featureFieldIndex) { - byte[] alleleBase = allele.getBases(); - if (alleleBase.length == 1) + String featureIdentifier = csqFields[featureFieldIndex]; + if (featureIdentifier.length() > 4 + && seqName.indexOf(featureIdentifier.toLowerCase()) > -1) { - sb.append(",").append( - forwardStrand ? (char) alleleBase[0] - : complement(alleleBase)); + /* + * feature (transcript) matched - now check for allele match + */ + if (matchAllele(variant, altAlleleIndex, csqFields, + alleleFieldIndex, alleleNumberFieldIndex)) + { + return consequence; + } } } } - String alleles = sb.toString(); // e.g. G,A,C - - String type = SequenceOntologyI.SEQUENCE_VARIANT; + return null; + } - SequenceFeature sf = new SequenceFeature(type, alleles, featureStart, - featureEnd, score, FEATURE_GROUP_VCF); + private boolean matchAllele(VariantContext variant, int altAlleleIndex, + String[] csqFields, int alleleFieldIndex, + int alleleNumberFieldIndex) + { + /* + * if ALLELE_NUM is present, it must match altAlleleIndex + * NB first alternate allele is 1 for ALLELE_NUM, 0 for altAlleleIndex + */ + if (alleleNumberFieldIndex > -1) + { + if (csqFields.length <= alleleNumberFieldIndex) + { + return false; + } + String alleleNum = csqFields[alleleNumberFieldIndex]; + return String.valueOf(altAlleleIndex + 1).equals(alleleNum); + } - sf.setValue(Gff3Helper.ALLELES, alleles); + /* + * else consequence allele must match variant allele + */ + if (alleleFieldIndex > -1 && csqFields.length > alleleFieldIndex) + { + String csqAllele = csqFields[alleleFieldIndex]; + String vcfAllele = variant.getAlternateAllele(altAlleleIndex) + .getBaseString(); + return csqAllele.equals(vcfAllele); + } + return false; + } + /** + * Add any allele-specific VCF key-value data to the sequence feature + * + * @param variant + * @param seq + * @param sf + * @param altAlelleIndex + * (0, 1..) + * @param consequence + * if not null, the consequence specific to this sequence (transcript + * feature) and allele + */ + protected void addAlleleProperties(VariantContext variant, SequenceI seq, + SequenceFeature sf, final int altAlelleIndex, String consequence) + { Map atts = variant.getAttributes(); + for (Entry att : atts.entrySet()) { - sf.setValue(att.getKey(), att.getValue()); + String key = att.getKey(); + + /* + * extract Consequence data (if present) that we are able to + * associated with the allele for this variant feature + */ + if (CSQ_FIELD.equals(key)) + { + addConsequences(variant, seq, sf, consequence); + continue; + } + + /* + * filter out fields we don't want to capture + */ + if (!vcfFieldsOfInterest.contains(key)) + { + continue; + } + + /* + * we extract values for other data which are allele-specific; + * these may be per alternate allele (INFO[key].Number = 'A') + * or per allele including reference (INFO[key].Number = 'R') + */ + VCFInfoHeaderLine infoHeader = header.getInfoHeaderLine(key); + if (infoHeader == null) + { + /* + * can't be sure what data belongs to this allele, so + * play safe and don't take any + */ + continue; + } + + VCFHeaderLineCount number = infoHeader.getCountType(); + int index = altAlelleIndex; + if (number == VCFHeaderLineCount.R) + { + /* + * one value per allele including reference, so bump index + * e.g. the 3rd value is for the 2nd alternate allele + */ + index++; + } + else if (number != VCFHeaderLineCount.A) + { + /* + * don't save other values as not allele-related + */ + continue; + } + + /* + * take the index'th value + */ + String value = getAttributeValue(variant, key, index); + if (value != null) + { + sf.setValue(key, value); + } + } + } + + /** + * Inspects CSQ data blocks (consequences) and adds attributes on the sequence + * feature. + *

+ * If myConsequence is not null, then this is the specific + * consequence data (pipe-delimited fields) that is for the current allele and + * transcript (sequence) being processed) + * + * @param variant + * @param seq + * @param sf + * @param myConsequence + */ + protected void addConsequences(VariantContext variant, SequenceI seq, + SequenceFeature sf, String myConsequence) + { + Object value = variant.getAttribute(CSQ_FIELD); + // TODO if CSQ not present, try ANN (for SnpEff consequence data)? + + if (value == null || !(value instanceof List)) + { + return; + } + + List consequences = (List) value; + + /* + * inspect CSQ consequences; restrict to the consequence + * associated with the current transcript (Feature) + */ + Map csqValues = new HashMap<>(); + + for (String consequence : consequences) + { + if (myConsequence == null || myConsequence.equals(consequence)) + { + String[] csqFields = consequence.split(PIPE_REGEX); + + /* + * inspect individual fields of this consequence, copying non-null + * values which are 'fields of interest' + */ + int i = 0; + for (String field : csqFields) + { + if (field != null && field.length() > 0) + { + String id = vepFieldsOfInterest.get(i); + if (id != null) + { + csqValues.put(id, field); + } + } + i++; + } + } + } + + if (!csqValues.isEmpty()) + { + sf.setValue(CSQ_FIELD, csqValues); } - seq.addSequenceFeature(sf); } /** @@ -480,8 +1315,8 @@ public class VCFLoader * call (e.g.) http://rest.ensembl.org/map/human/GRCh38/17:45051610..45109016:1/GRCh37 */ EnsemblMap mapper = new EnsemblMap(); - int[] mapping = mapper.getMapping(species, chromosome, fromRef, toRef, - queryRange); + int[] mapping = mapper.getAssemblyMapping(species, chromosome, fromRef, + toRef, queryRange); if (mapping == null) {