X-Git-Url: http://source.jalview.org/gitweb/?a=blobdiff_plain;f=src%2Fjalview%2Fws%2Fsifts%2FSiftsClient.java;h=58e0b29e4819e895c1c34a102a3996baa599c335;hb=HEAD;hp=e1b539e21baee0b4b0b0b00cb6368b25b8b67e89;hpb=f1b12e688d0b261d04553a858e5b440cfc158a45;p=jalview.git diff --git a/src/jalview/ws/sifts/SiftsClient.java b/src/jalview/ws/sifts/SiftsClient.java index e1b539e..58e0b29 100644 --- a/src/jalview/ws/sifts/SiftsClient.java +++ b/src/jalview/ws/sifts/SiftsClient.java @@ -20,27 +20,7 @@ */ package jalview.ws.sifts; -import jalview.analysis.AlignSeq; -import jalview.analysis.scoremodels.ScoreMatrix; -import jalview.analysis.scoremodels.ScoreModels; -import jalview.api.DBRefEntryI; -import jalview.api.SiftsClientI; -import jalview.datamodel.DBRefEntry; -import jalview.datamodel.DBRefSource; -import jalview.datamodel.SequenceI; -import jalview.io.StructureFile; -import jalview.schemes.ResidueProperties; -import jalview.structure.StructureMapping; -import jalview.util.Comparison; -import jalview.util.DBRefUtils; -import jalview.util.Format; -import jalview.xml.binding.sifts.Entry; -import jalview.xml.binding.sifts.Entry.Entity; -import jalview.xml.binding.sifts.Entry.Entity.Segment; -import jalview.xml.binding.sifts.Entry.Entity.Segment.ListMapRegion.MapRegion; -import jalview.xml.binding.sifts.Entry.Entity.Segment.ListResidue.Residue; -import jalview.xml.binding.sifts.Entry.Entity.Segment.ListResidue.Residue.CrossRefDb; -import jalview.xml.binding.sifts.Entry.Entity.Segment.ListResidue.Residue.ResidueDetail; +import java.util.Locale; import java.io.File; import java.io.FileInputStream; @@ -67,12 +47,37 @@ import java.util.TreeMap; import java.util.zip.GZIPInputStream; import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBElement; import javax.xml.bind.Unmarshaller; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamReader; -import MCview.Atom; -import MCview.PDBChain; +import jalview.analysis.AlignSeq; +import jalview.analysis.scoremodels.ScoreMatrix; +import jalview.analysis.scoremodels.ScoreModels; +import jalview.api.DBRefEntryI; +import jalview.api.SiftsClientI; +import jalview.bin.Console; +import jalview.datamodel.DBRefEntry; +import jalview.datamodel.DBRefSource; +import jalview.datamodel.SequenceI; +import jalview.io.BackupFiles; +import jalview.io.StructureFile; +import jalview.schemes.ResidueProperties; +import jalview.structure.StructureMapping; +import jalview.util.Comparison; +import jalview.util.DBRefUtils; +import jalview.util.Format; +import jalview.util.Platform; +import jalview.xml.binding.sifts.Entry; +import jalview.xml.binding.sifts.Entry.Entity; +import jalview.xml.binding.sifts.Entry.Entity.Segment; +import jalview.xml.binding.sifts.Entry.Entity.Segment.ListMapRegion.MapRegion; +import jalview.xml.binding.sifts.Entry.Entity.Segment.ListResidue.Residue; +import jalview.xml.binding.sifts.Entry.Entity.Segment.ListResidue.Residue.CrossRefDb; +import jalview.xml.binding.sifts.Entry.Entity.Segment.ListResidue.Residue.ResidueDetail; +import mc_view.Atom; +import mc_view.PDBChain; public class SiftsClient implements SiftsClientI { @@ -108,9 +113,11 @@ public class SiftsClient implements SiftsClientI private static final int PDB_ATOM_POS = 1; + private static final int PDBE_POS = 2; + private static final String NOT_OBSERVED = "Not_Observed"; - private static final String SIFTS_FTP_BASE_URL = "http://ftp.ebi.ac.uk/pub/databases/msd/sifts/xml/"; + private static final String SIFTS_SPLIT_FTP_BASE_URL = "https://ftp.ebi.ac.uk/pub/databases/msd/sifts/split_xml/"; private final static String NEWLINE = System.lineSeparator(); @@ -121,6 +128,7 @@ public class SiftsClient implements SiftsClientI private enum CoordinateSys { UNIPROT("UniProt"), PDB("PDBresnum"), PDBe("PDBe"); + private String name; private CoordinateSys(String name) @@ -138,6 +146,7 @@ public class SiftsClient implements SiftsClientI { NAME_SEC_STRUCTURE("nameSecondaryStructure"), CODE_SEC_STRUCTURE("codeSecondaryStructure"), ANNOTATION("Annotation"); + private String code; private ResidueDetailType(String code) @@ -180,12 +189,13 @@ public class SiftsClient implements SiftsClientI try (InputStream in = new FileInputStream(siftFile); GZIPInputStream gzis = new GZIPInputStream(in);) { - // System.out.println("File : " + siftFile.getAbsolutePath()); + // jalview.bin.Console.outPrintln("File : " + siftFile.getAbsolutePath()); JAXBContext jc = JAXBContext.newInstance("jalview.xml.binding.sifts"); XMLStreamReader streamReader = XMLInputFactory.newInstance() .createXMLStreamReader(gzis); Unmarshaller um = jc.createUnmarshaller(); - return (Entry) um.unmarshal(streamReader); + JAXBElement jbe = um.unmarshal(streamReader, Entry.class); + return jbe.getValue(); } catch (Exception e) { e.printStackTrace(); @@ -212,27 +222,28 @@ public class SiftsClient implements SiftsClientI } String siftsFileName = SiftsSettings.getSiftDownloadDirectory() - + pdbId.toLowerCase() + ".xml.gz"; + + pdbId.toLowerCase(Locale.ROOT) + ".xml.gz"; File siftsFile = new File(siftsFileName); if (siftsFile.exists()) { // The line below is required for unit testing... don't comment it out!!! - System.out.println(">>> SIFTS File already downloaded for " + pdbId); + jalview.bin.Console + .outPrintln(">>> SIFTS File already downloaded for " + pdbId); if (isFileOlderThanThreshold(siftsFile, SiftsSettings.getCacheThresholdInDays())) { File oldSiftsFile = new File(siftsFileName + "_old"); - siftsFile.renameTo(oldSiftsFile); + BackupFiles.moveFileToFile(siftsFile, oldSiftsFile); try { - siftsFile = downloadSiftsFile(pdbId.toLowerCase()); + siftsFile = downloadSiftsFile(pdbId.toLowerCase(Locale.ROOT)); oldSiftsFile.delete(); return siftsFile; } catch (IOException e) { e.printStackTrace(); - oldSiftsFile.renameTo(siftsFile); + BackupFiles.moveFileToFile(oldSiftsFile, siftsFile); return new File(siftsFileName); } } @@ -243,7 +254,7 @@ public class SiftsClient implements SiftsClientI } try { - siftsFile = downloadSiftsFile(pdbId.toLowerCase()); + siftsFile = downloadSiftsFile(pdbId.toLowerCase(Locale.ROOT)); } catch (IOException e) { throw new SiftsException(e.getMessage()); @@ -272,7 +283,7 @@ public class SiftsClient implements SiftsClientI diffInDays = (int) ((new Date().getTime() - attr.lastModifiedTime().toMillis()) / (1000 * 60 * 60 * 24)); - // System.out.println("Diff in days : " + diffInDays); + // jalview.bin.Console.outPrintln("Diff in days : " + diffInDays); } catch (IOException e) { e.printStackTrace(); @@ -296,22 +307,37 @@ public class SiftsClient implements SiftsClientI pdbId = pdbId.replace(".cif", ""); } String siftFile = pdbId + ".xml.gz"; - String siftsFileFTPURL = SIFTS_FTP_BASE_URL + siftFile; - String downloadedSiftsFile = SiftsSettings.getSiftDownloadDirectory() - + siftFile; - File siftsDownloadDir = new File( - SiftsSettings.getSiftDownloadDirectory()); - if (!siftsDownloadDir.exists()) + String siftsFileFTPURL = getDownloadUrlFor(siftFile); + + /* + * Download the file from URL to either + * Java: directory of cached downloaded SIFTS files + * Javascript: temporary 'file' (in-memory cache) + */ + File downloadTo = null; + if (Platform.isJS()) { - siftsDownloadDir.mkdirs(); + downloadTo = File.createTempFile(siftFile, ".xml.gz"); + } + else + { + downloadTo = new File( + SiftsSettings.getSiftDownloadDirectory() + siftFile); + File siftsDownloadDir = new File( + SiftsSettings.getSiftDownloadDirectory()); + if (!siftsDownloadDir.exists()) + { + siftsDownloadDir.mkdirs(); + } } - // System.out.println(">> Download ftp url : " + siftsFileFTPURL); + + // jalview.bin.Console.outPrintln(">> Download ftp url : " + + // siftsFileFTPURL); // long now = System.currentTimeMillis(); URL url = new URL(siftsFileFTPURL); URLConnection conn = url.openConnection(); InputStream inputStream = conn.getInputStream(); - FileOutputStream outputStream = new FileOutputStream( - downloadedSiftsFile); + FileOutputStream outputStream = new FileOutputStream(downloadTo); byte[] buffer = new byte[BUFFER_SIZE]; int bytesRead = -1; while ((bytesRead = inputStream.read(buffer)) != -1) @@ -320,9 +346,19 @@ public class SiftsClient implements SiftsClientI } outputStream.close(); inputStream.close(); - // System.out.println(">>> File downloaded : " + downloadedSiftsFile + // jalview.bin.Console.outPrintln(">>> File downloaded : " + + // downloadedSiftsFile // + " took " + (System.currentTimeMillis() - now) + "ms"); - return new File(downloadedSiftsFile); + return downloadTo; + } + + public static String getDownloadUrlFor(String siftFile) + { + String durl = SIFTS_SPLIT_FTP_BASE_URL + siftFile.substring(1, 3) + "/" + + siftFile; + Console.trace("SIFTS URL for " + siftFile + " is " + durl); + return durl; + } /** @@ -335,7 +371,7 @@ public class SiftsClient implements SiftsClientI public static boolean deleteSiftsFileByPDBId(String pdbId) { File siftsFile = new File(SiftsSettings.getSiftDownloadDirectory() - + pdbId.toLowerCase() + ".xml.gz"); + + pdbId.toLowerCase(Locale.ROOT) + ".xml.gz"); if (siftsFile.exists()) { return siftsFile.delete(); @@ -409,8 +445,8 @@ public class SiftsClient implements SiftsClientI .getMapRegion(); for (MapRegion mapRegion : mapRegions) { - accessions - .add(mapRegion.getDb().getDbAccessionId().toLowerCase()); + accessions.add(mapRegion.getDb().getDbAccessionId() + .toLowerCase(Locale.ROOT)); } } } @@ -427,8 +463,8 @@ public class SiftsClient implements SiftsClientI seq = seq.getDatasetSequence(); } structId = (chain == null) ? pdbId : pdbId + "|" + chain; - System.out.println("Getting SIFTS mapping for " + structId + ": seq " - + seq.getName()); + jalview.bin.Console.outPrintln("Getting SIFTS mapping for " + structId + + ": seq " + seq.getName()); final StringBuilder mappingDetails = new StringBuilder(128); PrintStream ps = new PrintStream(System.out) @@ -458,9 +494,9 @@ public class SiftsClient implements SiftsClientI public HashMap getGreedyMapping(String entityId, SequenceI seq, java.io.PrintStream os) throws SiftsException { - List omitNonObserved = new ArrayList(); - int nonObservedShiftIndex = 0; - // System.out.println("Generating mappings for : " + entityId); + List omitNonObserved = new ArrayList<>(); + int nonObservedShiftIndex = 0, pdbeNonObserved = 0; + // jalview.bin.Console.outPrintln("Generating mappings for : " + entityId); Entity entity = null; entity = getEntityById(entityId); String originalSeq = AlignSeq.extractGaps( @@ -480,9 +516,11 @@ public class SiftsClient implements SiftsClientI HashSet dbRefAccessionIdsString = new HashSet(); for (DBRefEntry dbref : seq.getDBRefs()) { - dbRefAccessionIdsString.add(dbref.getAccessionId().toLowerCase()); + dbRefAccessionIdsString + .add(dbref.getAccessionId().toLowerCase(Locale.ROOT)); } - dbRefAccessionIdsString.add(sourceDBRef.getAccessionId().toLowerCase()); + dbRefAccessionIdsString + .add(sourceDBRef.getAccessionId().toLowerCase(Locale.ROOT)); curDBRefAccessionIdsString = dbRefAccessionIdsString; curSourceDBRef = sourceDBRef.getAccessionId(); @@ -490,7 +528,7 @@ public class SiftsClient implements SiftsClientI TreeMap resNumMap = new TreeMap(); List segments = entity.getSegment(); SegmentHelperPojo shp = new SegmentHelperPojo(seq, mapping, resNumMap, - omitNonObserved, nonObservedShiftIndex); + omitNonObserved, nonObservedShiftIndex, pdbeNonObserved); processSegments(segments, shp); try { @@ -510,20 +548,23 @@ public class SiftsClient implements SiftsClientI if (mapping.isEmpty()) { - throw new SiftsException("SIFTS mapping failed"); + throw new SiftsException("SIFTS mapping failed for " + entityId + + " and " + seq.getName()); } - // also construct a mapping object between the seq-coord sys and the PDB seq's coord sys + // also construct a mapping object between the seq-coord sys and the PDB + // seq's coord sys Integer[] keys = mapping.keySet().toArray(new Integer[0]); Arrays.sort(keys); seqStart = keys[0]; seqEnd = keys[keys.length - 1]; - List from=new ArrayList<>(),to=new ArrayList<>(); - int[]_cfrom=null,_cto=null; + List from = new ArrayList<>(), to = new ArrayList<>(); + int[] _cfrom = null, _cto = null; String matchedSeq = originalSeq; - if (seqStart != UNASSIGNED) // fixme! seqStart can map to -1 for a pdb sequence that starts <-1 + if (seqStart != UNASSIGNED) // fixme! seqStart can map to -1 for a pdb + // sequence that starts <-1 { - for (int seqps:keys) + for (int seqps : keys) { int pdbpos = mapping.get(seqps)[PDBE_POS]; if (pdbpos == UNASSIGNED) @@ -531,19 +572,23 @@ public class SiftsClient implements SiftsClientI // not correct - pdbpos might be -1, but leave it for now continue; } - if (_cfrom==null || seqps!=_cfrom[1]+1) + if (_cfrom == null || seqps != _cfrom[1] + 1) { - _cfrom = new int[] { seqps,seqps}; + _cfrom = new int[] { seqps, seqps }; from.add(_cfrom); _cto = null; // discontinuity - } else { - _cfrom[1]= seqps; } - if (_cto==null || pdbpos!=1+_cto[1]) + else + { + _cfrom[1] = seqps; + } + if (_cto == null || pdbpos != 1 + _cto[1]) { - _cto = new int[] { pdbpos,pdbpos}; + _cto = new int[] { pdbpos, pdbpos }; to.add(_cto); - } else { + } + else + { _cto[1] = pdbpos; } } @@ -565,8 +610,7 @@ public class SiftsClient implements SiftsClientI ; seqFromPdbMapping = new jalview.datamodel.Mapping(null, _cto, _cfrom, - 1, - 1); + 1, 1); pdbStart = mapping.get(seqStart)[PDB_RES_POS]; pdbEnd = mapping.get(seqEnd)[PDB_RES_POS]; int orignalSeqStart = seq.getStart(); @@ -623,11 +667,13 @@ public class SiftsClient implements SiftsClientI int firstPDBResNum = UNASSIGNED; for (Segment segment : segments) { - // System.out.println("Mapping segments : " + segment.getSegId() + "\\"s + // jalview.bin.Console.outPrintln("Mapping segments : " + + // segment.getSegId() + "\\"s // + segStartEnd); List residues = segment.getListResidue().getResidue(); for (Residue residue : residues) { + boolean isObserved = isResidueObserved(residue); int pdbeIndex = getLeadingIntegerValue(residue.getDbResNum(), UNASSIGNED); int currSeqIndex = UNASSIGNED; @@ -638,6 +684,19 @@ public class SiftsClient implements SiftsClientI if (cRefDb.getDbSource().equalsIgnoreCase(DBRefSource.PDB)) { pdbRefDb = cRefDb; + if (firstPDBResNum == UNASSIGNED) + { + firstPDBResNum = getLeadingIntegerValue(cRefDb.getDbResNum(), + UNASSIGNED); + } + else + { + if (isObserved) + { + // after we find the first observed residue we just increment + firstPDBResNum++; + } + } } if (cRefDb.getDbCoordSys().equalsIgnoreCase(seqCoordSys.getName()) && isAccessionMatched(cRefDb.getDbAccessionId())) @@ -650,35 +709,45 @@ public class SiftsClient implements SiftsClientI } } } + if (!isObserved) + { + ++pdbeNonObservedCount; + } if (seqCoordSys == seqCoordSys.PDB) // FIXME: is seqCoordSys ever PDBe // ??? { // if the sequence has a primary reference to the PDB, then we are // dealing with a sequence extracted directly from the PDB. In that // case, numbering is PDBe - non-observed residues - currSeqIndex = pdbeIndex; + currSeqIndex = seq.getStart() - 1 + pdbeIndex; } - if (currSeqIndex == UNASSIGNED) + if (!isObserved) { - continue; + if (seqCoordSys != CoordinateSys.UNIPROT) // FIXME: PDB or PDBe only + // here + { + // mapping to PDB or PDBe so we need to bookkeep for the + // non-observed + // SEQRES positions + omitNonObserved.add(currSeqIndex); + ++nonObservedShiftIndex; + } } - if (!isResidueObserved(residue) - && seqCoordSys != seqCoordSys.UNIPROT) + if (currSeqIndex == UNASSIGNED) { - // mapping to PDB or PDBe so we need to bookkeep for the non-observed - // SEQRES positions - omitNonObserved.add(currSeqIndex); - ++nonObservedShiftIndex; + // change in logic - unobserved residues with no currSeqIndex + // corresponding are still counted in both nonObservedShiftIndex and + // pdbeIndex... + continue; } - // if (currSeqIndex >= seq.getStart() && currSeqIndex <= seqlength) // // true - // numbering - // is - // not - // up - // to - // seq.getEnd() + // numbering + // is + // not + // up + // to + // seq.getEnd() { int resNum = (pdbRefDb == null) @@ -687,18 +756,18 @@ public class SiftsClient implements SiftsClientI : getLeadingIntegerValue(pdbRefDb.getDbResNum(), UNASSIGNED); - if (isResidueObserved(residue) - || seqCoordSys == CoordinateSys.UNIPROT) + if (isObserved) { char resCharCode = ResidueProperties .getSingleCharacterCode(ResidueProperties .getCanonicalAminoAcid(residue.getDbResName())); resNumMap.put(currSeqIndex, String.valueOf(resCharCode)); - } - mapping.put(currSeqIndex - nonObservedShiftIndex, - new int[] - { Integer.valueOf(resNum), UNASSIGNED }); + int[] mappingcols = new int[] { Integer.valueOf(resNum), + UNASSIGNED, isObserved ? firstPDBResNum : UNASSIGNED }; + + mapping.put(currSeqIndex - nonObservedShiftIndex, mappingcols); + } } } } @@ -841,14 +910,15 @@ public class SiftsClient implements SiftsClientI { boolean isStrictMatch = true; return isStrictMatch ? curSourceDBRef.equalsIgnoreCase(accession) - : curDBRefAccessionIdsString.contains(accession.toLowerCase()); + : curDBRefAccessionIdsString + .contains(accession.toLowerCase(Locale.ROOT)); } private boolean isFoundInSiftsEntry(String accessionId) { Set siftsDBRefs = getAllMappingAccession(); return accessionId != null - && siftsDBRefs.contains(accessionId.toLowerCase()); + && siftsDBRefs.contains(accessionId.toLowerCase(Locale.ROOT)); } /** @@ -867,8 +937,8 @@ public class SiftsClient implements SiftsClientI // Arrays.sort(keys); int firstIndex = keys[0]; int lastIndex = keys[keys.length - 1]; - // System.out.println("Min value " + firstIndex); - // System.out.println("Max value " + lastIndex); + // jalview.bin.Console.outPrintln("Min value " + firstIndex); + // jalview.bin.Console.outPrintln("Max value " + lastIndex); for (int x = firstIndex; x <= lastIndex; x++) { if (!resNumMap.containsKey(x) && !omitNonObserved.contains(x)) @@ -901,7 +971,8 @@ public class SiftsClient implements SiftsClientI */ public Entity getEntityByMostOptimalMatchedId(String chainId) { - // System.out.println("---> advanced greedy entityId matching block + // jalview.bin.Console.outPrintln("---> advanced greedy entityId matching + // block // entered.."); List entities = siftsEntry.getEntity(); SiftsEntitySortPojo[] sPojo = new SiftsEntitySortPojo[entities.size()]; @@ -937,8 +1008,9 @@ public class SiftsClient implements SiftsClientI ++count; } Arrays.sort(sPojo, Collections.reverseOrder()); - // System.out.println("highest matched entity : " + sPojo[0].entityId); - // System.out.println("highest matched pid : " + sPojo[0].pid); + // jalview.bin.Console.outPrintln("highest matched entity : " + + // sPojo[0].entityId); + // jalview.bin.Console.outPrintln("highest matched pid : " + sPojo[0].pid); if (sPojo[0].entityId != null) { @@ -988,15 +1060,35 @@ public class SiftsClient implements SiftsClientI private int nonObservedShiftIndex; + /** + * count of number of 'not observed' positions in the PDB record's SEQRES + * (total number of residues with coordinates == length(SEQRES) - + * pdbeNonObserved + */ + private int pdbeNonObserved; + public SegmentHelperPojo(SequenceI seq, HashMap mapping, TreeMap resNumMap, - List omitNonObserved, int nonObservedShiftIndex) + List omitNonObserved, int nonObservedShiftIndex, + int pdbeNonObserved) { setSeq(seq); setMapping(mapping); setResNumMap(resNumMap); setOmitNonObserved(omitNonObserved); setNonObservedShiftIndex(nonObservedShiftIndex); + setPdbeNonObserved(pdbeNonObserved); + + } + + public void setPdbeNonObserved(int pdbeNonObserved2) + { + this.pdbeNonObserved = pdbeNonObserved2; + } + + public int getPdbeNonObserved() + { + return pdbeNonObserved; } public SequenceI getSeq() @@ -1048,6 +1140,7 @@ public class SiftsClient implements SiftsClientI { this.nonObservedShiftIndex = nonObservedShiftIndex; } + } @Override