badData;
+
/**
* Constructor given a VCF file
*
@@ -262,8 +321,14 @@ public class VCFLoader
*/
public SequenceI loadVCFContig(String contig)
{
- String ref = header.getOtherHeaderLine(VCFHeader.REFERENCE_KEY)
- .getValue();
+ VCFHeaderLine headerLine = header
+ .getOtherHeaderLine(VCFHeader.REFERENCE_KEY);
+ if (headerLine == null)
+ {
+ Console.error("VCF reference header not found");
+ return null;
+ }
+ String ref = headerLine.getValue();
if (ref.startsWith("file://"))
{
ref = ref.substring(7);
@@ -282,7 +347,7 @@ public class VCFLoader
}
else
{
- System.err.println("VCF reference not found: " + ref);
+ Console.error("VCF reference not found: " + ref);
}
return seq;
@@ -301,7 +366,7 @@ public class VCFLoader
{
VCFHeaderLine ref = header
.getOtherHeaderLine(VCFHeader.REFERENCE_KEY);
- String reference = ref.getValue();
+ String reference = ref == null ? null : ref.getValue();
setSpeciesAndAssembly(reference);
@@ -358,9 +423,9 @@ public class VCFLoader
/**
* Attempts to determine and save the species and genome assembly version to
- * which the VCF data applies. This may be done by parsing the {@code reference}
- * header line, configured in a property file, or (potentially) confirmed
- * interactively by the user.
+ * which the VCF data applies. This may be done by parsing the
+ * {@code reference} header line, configured in a property file, or
+ * (potentially) confirmed interactively by the user.
*
* The saved values should be identifiers valid for Ensembl's REST service
* {@code map} endpoint, so they can be used (if necessary) to retrieve the
@@ -373,7 +438,13 @@ public class VCFLoader
*/
protected void setSpeciesAndAssembly(String reference)
{
- vcfSpecies = DEFAULT_SPECIES;
+ if (reference == null)
+ {
+ Console.error("No VCF ##reference found, defaulting to "
+ + DEFAULT_REFERENCE + ":" + DEFAULT_SPECIES);
+ reference = DEFAULT_REFERENCE; // default to GRCh37 if not specified
+ }
+ reference = reference.toLowerCase(Locale.ROOT);
/*
* for a non-human species, or other assembly identifier,
@@ -388,7 +459,7 @@ public class VCFLoader
String[] tokens = token.split("=");
if (tokens.length == 2)
{
- if (reference.contains(tokens[0].trim().toLowerCase()))
+ if (reference.contains(tokens[0].trim().toLowerCase(Locale.ROOT)))
{
vcfAssembly = tokens[1].trim();
break;
@@ -396,6 +467,7 @@ public class VCFLoader
}
}
+ vcfSpecies = DEFAULT_SPECIES;
prop = Cache.getProperty(VCF_SPECIES);
if (prop != null)
{
@@ -404,7 +476,7 @@ public class VCFLoader
String[] tokens = token.split("=");
if (tokens.length == 2)
{
- if (reference.contains(tokens[0].trim().toLowerCase()))
+ if (reference.contains(tokens[0].trim().toLowerCase(Locale.ROOT)))
{
vcfSpecies = tokens[1].trim();
break;
@@ -509,7 +581,7 @@ public class VCFLoader
{
for (Pattern p : filters)
{
- if (p.matcher(id.toUpperCase()).matches())
+ if (p.matcher(id.toUpperCase(Locale.ROOT)).matches())
{
return true;
}
@@ -603,7 +675,7 @@ public class VCFLoader
{
try
{
- patterns.add(Pattern.compile(token.toUpperCase()));
+ patterns.add(Pattern.compile(token.toUpperCase(Locale.ROOT)));
} catch (PatternSyntaxException e)
{
System.err.println("Invalid pattern ignored: " + token);
@@ -614,13 +686,12 @@ public class VCFLoader
/**
* Transfers VCF features to sequences to which this sequence has a mapping.
- * If the mapping is 3:1, computes peptide variants from nucleotide variants.
*
* @param seq
*/
protected void transferAddedFeatures(SequenceI seq)
{
- DBRefEntry[] dbrefs = seq.getDBRefs();
+ List dbrefs = seq.getDBRefs();
if (dbrefs == null)
{
return;
@@ -640,7 +711,8 @@ public class VCFLoader
/*
* dna-to-peptide product mapping
*/
- AlignmentUtils.computeProteinFeatures(seq, mapTo, map);
+ // JAL-3187 render on the fly instead
+ // AlignmentUtils.computeProteinFeatures(seq, mapTo, map);
}
else
{
@@ -661,8 +733,8 @@ public class VCFLoader
}
/**
- * Tries to add overlapping variants read from a VCF file to the given sequence,
- * and returns the number of variant features added
+ * Tries to add overlapping variants read from a VCF file to the given
+ * sequence, and returns the number of variant features added
*
* @param seq
* @return
@@ -714,7 +786,7 @@ public class VCFLoader
GeneLociI seqCoords = seq.getGeneLoci();
if (seqCoords == null)
{
- Cache.log.warn(String.format(
+ Console.warn(String.format(
"Can't query VCF for %s as chromosome coordinates not known",
seq.getName()));
return null;
@@ -723,13 +795,13 @@ public class VCFLoader
String species = seqCoords.getSpeciesId();
String chromosome = seqCoords.getChromosomeId();
String seqRef = seqCoords.getAssemblyId();
- MapList map = seqCoords.getMap();
+ MapList map = seqCoords.getMapping();
// note this requires the configured species to match that
// returned with the Ensembl sequence; todo: support aliases?
if (!vcfSpecies.equalsIgnoreCase(species))
{
- Cache.log.warn("No VCF loaded to " + seq.getName()
+ Console.warn("No VCF loaded to " + seq.getName()
+ " as species not matched");
return null;
}
@@ -759,10 +831,9 @@ public class VCFLoader
vcfAssembly);
if (newRange == null)
{
- Cache.log.error(
- String.format("Failed to map %s:%s:%s:%d:%d to %s", species,
- chromosome, seqRef, range[0], range[1],
- vcfAssembly));
+ Console.error(String.format("Failed to map %s:%s:%s:%d:%d to %s",
+ species, chromosome, seqRef, range[0], range[1],
+ vcfAssembly));
continue;
}
else
@@ -803,40 +874,6 @@ public class VCFLoader
}
/**
- * Answers true if the species inferred from the VCF reference identifier
- * matches that for the sequence
- *
- * @param vcfAssembly
- * @param speciesId
- * @return
- */
- boolean vcfSpeciesMatchesSequence(String vcfAssembly, String speciesId)
- {
- // PROBLEM 1
- // there are many aliases for species - how to equate one with another?
- // PROBLEM 2
- // VCF ##reference header is an unstructured URI - how to extract species?
- // perhaps check if ref includes any (Ensembl) alias of speciesId??
- // TODO ask the user to confirm this??
-
- if (vcfAssembly.contains("Homo_sapiens") // gnomAD exome data example
- && "HOMO_SAPIENS".equals(speciesId)) // Ensembl species id
- {
- return true;
- }
-
- if (vcfAssembly.contains("c_elegans") // VEP VCF response example
- && "CAENORHABDITIS_ELEGANS".equals(speciesId)) // Ensembl
- {
- return true;
- }
-
- // this is not a sustainable solution...
-
- return false;
- }
-
- /**
* Queries the VCF reader for any variants that overlap the mapped chromosome
* ranges of the sequence, and adds as variant features. Returns the number of
* overlapping variants found.
@@ -859,24 +896,42 @@ public class VCFLoader
{
int vcfStart = Math.min(range[0], range[1]);
int vcfEnd = Math.max(range[0], range[1]);
- CloseableIterator variants = reader
- .query(map.chromosome, vcfStart, vcfEnd);
- while (variants.hasNext())
+ try
{
- VariantContext variant = variants.next();
+ CloseableIterator variants = reader
+ .query(map.chromosome, vcfStart, vcfEnd);
+ while (variants.hasNext())
+ {
+ VariantContext variant = variants.next();
- int[] featureRange = map.map.locateInFrom(variant.getStart(),
- variant.getEnd());
+ int[] featureRange = map.map.locateInFrom(variant.getStart(),
+ variant.getEnd());
- if (featureRange != null)
- {
- int featureStart = Math.min(featureRange[0], featureRange[1]);
- int featureEnd = Math.max(featureRange[0], featureRange[1]);
- count += addAlleleFeatures(seq, variant, featureStart, featureEnd,
- forwardStrand);
+ /*
+ * only take features whose range is fully mappable to sequence positions
+ */
+ if (featureRange != null)
+ {
+ int featureStart = Math.min(featureRange[0], featureRange[1]);
+ int featureEnd = Math.max(featureRange[0], featureRange[1]);
+ if (featureEnd - featureStart == variant.getEnd()
+ - variant.getStart())
+ {
+ count += addAlleleFeatures(seq, variant, featureStart,
+ featureEnd, forwardStrand);
+ }
+ }
}
+ variants.close();
+ } catch (TribbleException e)
+ {
+ /*
+ * RuntimeException throwable by htsjdk
+ */
+ String msg = String.format("Error reading VCF for %s:%d-%d: %s ",
+ map.chromosome, vcfStart, vcfEnd, e.getLocalizedMessage());
+ Console.error(msg);
}
- variants.close();
}
return count;
@@ -939,8 +994,8 @@ public class VCFLoader
/**
* Inspects one allele and attempts to add a variant feature for it to the
* sequence. The additional data associated with this allele is extracted to
- * store in the feature's key-value map. Answers the number of features added (0
- * or 1).
+ * store in the feature's key-value map. Answers the number of features added
+ * (0 or 1).
*
* @param seq
* @param variant
@@ -988,10 +1043,10 @@ public class VCFLoader
* pick out the consequence data (if any) that is for the current allele
* and feature (transcript) that matches the current sequence
*/
- String consequence = getConsequenceForAlleleAndFeature(variant, CSQ_FIELD,
- altAlleleIndex, csqAlleleFieldIndex,
- csqAlleleNumberFieldIndex, seq.getName().toLowerCase(),
- csqFeatureFieldIndex);
+ String consequence = getConsequenceForAlleleAndFeature(variant,
+ CSQ_FIELD, altAlleleIndex, csqAlleleFieldIndex,
+ csqAlleleNumberFieldIndex,
+ seq.getName().toLowerCase(Locale.ROOT), csqFeatureFieldIndex);
/*
* pick out the ontology term for the consequence type
@@ -1006,7 +1061,20 @@ public class VCFLoader
featureEnd, FEATURE_GROUP_VCF);
sf.setSource(sourceId);
- sf.setValue(Gff3Helper.ALLELES, alleles);
+ /*
+ * save the derived alleles as a named attribute; this will be
+ * needed when Jalview computes derived peptide variants
+ */
+ addFeatureAttribute(sf, Gff3Helper.ALLELES, alleles);
+
+ /*
+ * add selected VCF fixed column data as feature attributes
+ */
+ addFeatureAttribute(sf, VCF_POS, String.valueOf(variant.getStart()));
+ addFeatureAttribute(sf, VCF_ID, variant.getID());
+ addFeatureAttribute(sf, VCF_QUAL,
+ String.valueOf(variant.getPhredScaledQual()));
+ addFeatureAttribute(sf, VCF_FILTER, getFilter(variant));
addAlleleProperties(variant, sf, altAlleleIndex, consequence);
@@ -1016,9 +1084,56 @@ public class VCFLoader
}
/**
- * Determines the Sequence Ontology term to use for the variant feature type in
- * Jalview. The default is 'sequence_variant', but a more specific term is used
- * if:
+ * Answers the VCF FILTER value for the variant - or an approximation to it.
+ * This field is either PASS, or a semi-colon separated list of filters not
+ * passed. htsjdk saves filters as a HashSet, so the order when reassembled
+ * into a list may be different.
+ *
+ * @param variant
+ * @return
+ */
+ String getFilter(VariantContext variant)
+ {
+ Set filters = variant.getFilters();
+ if (filters.isEmpty())
+ {
+ return NO_VALUE;
+ }
+ Iterator iterator = filters.iterator();
+ String first = iterator.next();
+ if (filters.size() == 1)
+ {
+ return first;
+ }
+
+ StringBuilder sb = new StringBuilder(first);
+ while (iterator.hasNext())
+ {
+ sb.append(";").append(iterator.next());
+ }
+
+ return sb.toString();
+ }
+
+ /**
+ * Adds one feature attribute unless the value is null, empty or '.'
+ *
+ * @param sf
+ * @param key
+ * @param value
+ */
+ void addFeatureAttribute(SequenceFeature sf, String key, String value)
+ {
+ if (value != null && !value.isEmpty() && !NO_VALUE.equals(value))
+ {
+ sf.setValue(key, value);
+ }
+ }
+
+ /**
+ * Determines the Sequence Ontology term to use for the variant feature type
+ * in Jalview. The default is 'sequence_variant', but a more specific term is
+ * used if:
*
* - VEP (or SnpEff) Consequence annotation is included in the VCF
* - sequence id can be matched to VEP Feature (or SnpEff Feature_ID)
@@ -1096,8 +1211,7 @@ public class VCFLoader
*/
private String getConsequenceForAlleleAndFeature(VariantContext variant,
String vcfInfoId, int altAlleleIndex, int alleleFieldIndex,
- int alleleNumberFieldIndex,
- String seqName, int featureFieldIndex)
+ int alleleNumberFieldIndex, String seqName, int featureFieldIndex)
{
if (alleleFieldIndex == -1 || featureFieldIndex == -1)
{
@@ -1122,8 +1236,8 @@ public class VCFLoader
if (csqFields.length > featureFieldIndex)
{
String featureIdentifier = csqFields[featureFieldIndex];
- if (featureIdentifier.length() > 4
- && seqName.indexOf(featureIdentifier.toLowerCase()) > -1)
+ if (featureIdentifier.length() > 4 && seqName
+ .indexOf(featureIdentifier.toLowerCase(Locale.ROOT)) > -1)
{
/*
* feature (transcript) matched - now check for allele match
@@ -1209,14 +1323,6 @@ public class VCFLoader
}
/*
- * filter out fields we don't want to capture
- */
- if (!vcfFieldsOfInterest.contains(key))
- {
- continue;
- }
-
- /*
* we extract values for other data which are allele-specific;
* these may be per alternate allele (INFO[key].Number = 'A')
* or per allele including reference (INFO[key].Number = 'R')
@@ -1253,14 +1359,85 @@ public class VCFLoader
* take the index'th value
*/
String value = getAttributeValue(variant, key, index);
- if (value != null)
+ if (value != null && isValid(variant, key, value))
{
- sf.setValue(key, value);
+ /*
+ * decode colon, semicolon, equals sign, percent sign, comma (only)
+ * as required by the VCF specification (para 1.2)
+ */
+ value = StringUtils.urlDecode(value, VCF_ENCODABLE);
+ addFeatureAttribute(sf, key, value);
}
}
}
/**
+ * Answers true for '.', null, or an empty value, or if the INFO type is
+ * String. If the INFO type is Integer or Float, answers false if the value is
+ * not in valid format.
+ *
+ * @param variant
+ * @param infoId
+ * @param value
+ * @return
+ */
+ protected boolean isValid(VariantContext variant, String infoId,
+ String value)
+ {
+ if (value == null || value.isEmpty() || NO_VALUE.equals(value))
+ {
+ return true;
+ }
+ VCFInfoHeaderLine infoHeader = header.getInfoHeaderLine(infoId);
+ if (infoHeader == null)
+ {
+ Console.error("Field " + infoId + " has no INFO header");
+ return false;
+ }
+ VCFHeaderLineType infoType = infoHeader.getType();
+ try
+ {
+ if (infoType == VCFHeaderLineType.Integer)
+ {
+ Integer.parseInt(value);
+ }
+ else if (infoType == VCFHeaderLineType.Float)
+ {
+ Float.parseFloat(value);
+ }
+ } catch (NumberFormatException e)
+ {
+ logInvalidValue(variant, infoId, value);
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Logs an error message for malformed data; duplicate messages (same id and
+ * value) are not logged
+ *
+ * @param variant
+ * @param infoId
+ * @param value
+ */
+ private void logInvalidValue(VariantContext variant, String infoId,
+ String value)
+ {
+ if (badData == null)
+ {
+ badData = new HashSet<>();
+ }
+ String token = infoId + ":" + value;
+ if (!badData.contains(token))
+ {
+ badData.add(token);
+ Console.error(String.format("Invalid VCF data at %s:%d %s=%s",
+ variant.getContig(), variant.getStart(), infoId, value));
+ }
+ }
+
+ /**
* Inspects CSQ data blocks (consequences) and adds attributes on the sequence
* feature.
*
@@ -1308,6 +1485,11 @@ public class VCFLoader
String id = vepFieldsOfInterest.get(i);
if (id != null)
{
+ /*
+ * VCF spec requires encoding of special characters e.g. '='
+ * so decode them here before storing
+ */
+ field = StringUtils.urlDecode(field, VCF_ENCODABLE);
csqValues.put(id, field);
}
}
@@ -1417,8 +1599,8 @@ public class VCFLoader
* @param toRef
* @return
*/
- protected int[] findSubsumedRangeMapping(int[] queryRange, String chromosome,
- String species, String fromRef, String toRef)
+ protected int[] findSubsumedRangeMapping(int[] queryRange,
+ String chromosome, String species, String fromRef, String toRef)
{
String key = makeRangesKey(chromosome, species, fromRef, toRef);
if (assemblyMappings.containsKey(key))
@@ -1440,7 +1622,8 @@ public class VCFLoader
*/
int offset = queryRange[0] - fromRange[0];
int mappedRangeFrom = toRange[0] + offset;
- int mappedRangeTo = mappedRangeFrom + (queryRange[1] - queryRange[0]);
+ int mappedRangeTo = mappedRangeFrom
+ + (queryRange[1] - queryRange[0]);
return new int[] { mappedRangeFrom, mappedRangeTo };
}
}
@@ -1463,7 +1646,7 @@ public class VCFLoader
SequenceI targetSequence, MapList mapping)
{
int[] mappedRange = mapping.locateInTo(sf.getBegin(), sf.getEnd());
-
+
if (mappedRange != null)
{
String group = sf.getFeatureGroup();
@@ -1487,7 +1670,6 @@ public class VCFLoader
protected static String makeRangesKey(String chromosome, String species,
String fromRef, String toRef)
{
- return species + EXCL + chromosome + EXCL + fromRef + EXCL
- + toRef;
+ return species + EXCL + chromosome + EXCL + fromRef + EXCL + toRef;
}
}