JAL-3624 use temp file for SIFT download in JS, correct unmarshalling
[jalview.git] / src / jalview / ws / sifts / SiftsClient.java
index 50cf5cb..4fb9ca9 100644 (file)
  */
 package jalview.ws.sifts;
 
-import jalview.analysis.AlignSeq;
-import jalview.api.DBRefEntryI;
-import jalview.api.SiftsClientI;
-import jalview.datamodel.DBRefEntry;
-import jalview.datamodel.DBRefSource;
-import jalview.datamodel.SequenceI;
-import jalview.schemes.ResidueProperties;
-import jalview.structure.StructureMapping;
-import jalview.util.Format;
-import jalview.xml.binding.sifts.Entry;
-import jalview.xml.binding.sifts.Entry.Entity;
-import jalview.xml.binding.sifts.Entry.Entity.Segment;
-import jalview.xml.binding.sifts.Entry.Entity.Segment.ListMapRegion.MapRegion;
-import jalview.xml.binding.sifts.Entry.Entity.Segment.ListResidue.Residue;
-import jalview.xml.binding.sifts.Entry.Entity.Segment.ListResidue.Residue.CrossRefDb;
-import jalview.xml.binding.sifts.Entry.Entity.Segment.ListResidue.Residue.ResidueDetail;
-import jalview.xml.binding.sifts.Entry.ListDB.Db;
-
 import java.io.File;
 import java.io.FileInputStream;
-import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -58,48 +39,81 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.TreeMap;
 import java.util.zip.GZIPInputStream;
 
 import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBException;
+import javax.xml.bind.JAXBElement;
 import javax.xml.bind.Unmarshaller;
-import javax.xml.stream.FactoryConfigurationError;
 import javax.xml.stream.XMLInputFactory;
-import javax.xml.stream.XMLStreamException;
 import javax.xml.stream.XMLStreamReader;
 
-import MCview.Atom;
-import MCview.PDBChain;
-import MCview.PDBfile;
+import jalview.analysis.AlignSeq;
+import jalview.analysis.scoremodels.ScoreMatrix;
+import jalview.analysis.scoremodels.ScoreModels;
+import jalview.api.DBRefEntryI;
+import jalview.api.SiftsClientI;
+import jalview.datamodel.DBRefEntry;
+import jalview.datamodel.DBRefSource;
+import jalview.datamodel.SequenceI;
+import jalview.io.StructureFile;
+import jalview.schemes.ResidueProperties;
+import jalview.structure.StructureMapping;
+import jalview.util.Comparison;
+import jalview.util.DBRefUtils;
+import jalview.util.Format;
+import jalview.util.Platform;
+import jalview.xml.binding.sifts.Entry;
+import jalview.xml.binding.sifts.Entry.Entity;
+import jalview.xml.binding.sifts.Entry.Entity.Segment;
+import jalview.xml.binding.sifts.Entry.Entity.Segment.ListMapRegion.MapRegion;
+import jalview.xml.binding.sifts.Entry.Entity.Segment.ListResidue.Residue;
+import jalview.xml.binding.sifts.Entry.Entity.Segment.ListResidue.Residue.CrossRefDb;
+import jalview.xml.binding.sifts.Entry.Entity.Segment.ListResidue.Residue.ResidueDetail;
+import mc_view.Atom;
+import mc_view.PDBChain;
 
 public class SiftsClient implements SiftsClientI
 {
+  /*
+   * for use in mocking out file fetch for tests only
+   * - reset to null after testing!
+   */
+  private static File mockSiftsFile;
+
   private Entry siftsEntry;
 
-  private PDBfile pdb;
+  private StructureFile pdb;
 
   private String pdbId;
 
   private String structId;
 
-  private String segStartEnd;
-
   private CoordinateSys seqCoordSys = CoordinateSys.UNIPROT;
 
+  /**
+   * PDB sequence position to sequence coordinate mapping as derived from SIFTS
+   * record for the identified SeqCoordSys Used for lift-over from sequence
+   * derived from PDB (with first extracted PDBRESNUM as 'start' to the sequence
+   * being annotated with PDB data
+   */
+  private jalview.datamodel.Mapping seqFromPdbMapping;
+
   private static final int BUFFER_SIZE = 4096;
 
-  public static final int UNASSIGNED = -1;
+  public static final int UNASSIGNED = Integer.MIN_VALUE;
 
   private static final int PDB_RES_POS = 0;
 
   private static final int PDB_ATOM_POS = 1;
 
-  private static final String NOT_FOUND = "Not_Found";
+  private static final int PDBE_POS = 2;
 
   private static final String NOT_OBSERVED = "Not_Observed";
 
-  private static final String SIFTS_FTP_BASE_URL = "ftp://ftp.ebi.ac.uk/pub/databases/msd/sifts/xml/";
+  private static final String SIFTS_FTP_BASE_URL = "http://ftp.ebi.ac.uk/pub/databases/msd/sifts/xml/";
 
   private final static String NEWLINE = System.lineSeparator();
 
@@ -107,7 +121,7 @@ public class SiftsClient implements SiftsClientI
 
   private HashSet<String> curDBRefAccessionIdsString;
 
-  public enum CoordinateSys
+  private enum CoordinateSys
   {
     UNIPROT("UniProt"), PDB("PDBresnum"), PDBe("PDBe");
     private String name;
@@ -123,10 +137,10 @@ public class SiftsClient implements SiftsClientI
     }
   };
 
-  public enum ResidueDetailType
+  private enum ResidueDetailType
   {
-    NAME_SEC_STRUCTURE("nameSecondaryStructure"), CODE_SEC_STRUCTURE(
-            "codeSecondaryStructure"), ANNOTATION("Annotation");
+    NAME_SEC_STRUCTURE("nameSecondaryStructure"),
+    CODE_SEC_STRUCTURE("codeSecondaryStructure"), ANNOTATION("Annotation");
     private String code;
 
     private ResidueDetailType(String code)
@@ -147,31 +161,15 @@ public class SiftsClient implements SiftsClientI
    * @param pdbId
    * @throws SiftsException
    */
-  public SiftsClient(PDBfile pdb) throws SiftsException
+  public SiftsClient(StructureFile pdb) throws SiftsException
   {
     this.pdb = pdb;
-    this.pdbId = pdb.id;
+    this.pdbId = pdb.getId();
     File siftsFile = getSiftsFile(pdbId);
     siftsEntry = parseSIFTs(siftsFile);
   }
 
   /**
-   * Construct an instance of SiftsClient using the supplied SIFTs file. Note:
-   * The SIFTs file should correspond to the PDB Id in PDBfile instance
-   * 
-   * @param pdbId
-   * @param siftsFile
-   * @throws SiftsException
-   * @throws Exception
-   */
-  public SiftsClient(PDBfile pdb, File siftsFile) throws SiftsException
-  {
-    this.pdb = pdb;
-    this.pdbId = pdb.id;
-    siftsEntry = parseSIFTs(siftsFile);
-  }
-
-  /**
    * Parse the given SIFTs File and return a JAXB POJO of parsed data
    * 
    * @param siftFile
@@ -190,24 +188,9 @@ public class SiftsClient implements SiftsClientI
       XMLStreamReader streamReader = XMLInputFactory.newInstance()
               .createXMLStreamReader(gzis);
       Unmarshaller um = jc.createUnmarshaller();
-      return (Entry) um.unmarshal(streamReader);
-    } catch (JAXBException e)
-    {
-      e.printStackTrace();
-      throw new SiftsException(e.getMessage());
-    } catch (FileNotFoundException e)
-    {
-      e.printStackTrace();
-      throw new SiftsException(e.getMessage());
-    } catch (XMLStreamException e)
-    {
-      e.printStackTrace();
-      throw new SiftsException(e.getMessage());
-    } catch (FactoryConfigurationError e)
-    {
-      e.printStackTrace();
-      throw new SiftsException(e.getMessage());
-    } catch (IOException e)
+      JAXBElement<Entry> jbe = um.unmarshal(streamReader, Entry.class);
+      return jbe.getValue();
+    } catch (Exception e)
     {
       e.printStackTrace();
       throw new SiftsException(e.getMessage());
@@ -224,8 +207,17 @@ public class SiftsClient implements SiftsClientI
    */
   public static File getSiftsFile(String pdbId) throws SiftsException
   {
-    File siftsFile = new File(SiftsSettings.getSiftDownloadDirectory()
-            + pdbId.toLowerCase() + ".xml.gz");
+    /*
+     * return mocked file if it has been set
+     */
+    if (mockSiftsFile != null)
+    {
+      return mockSiftsFile;
+    }
+
+    String siftsFileName = SiftsSettings.getSiftDownloadDirectory()
+            + pdbId.toLowerCase() + ".xml.gz";
+    File siftsFile = new File(siftsFileName);
     if (siftsFile.exists())
     {
       // The line below is required for unit testing... don't comment it out!!!
@@ -234,12 +226,32 @@ public class SiftsClient implements SiftsClientI
       if (isFileOlderThanThreshold(siftsFile,
               SiftsSettings.getCacheThresholdInDays()))
       {
-        // System.out.println("Downloaded file is out of date, hence re-downloading...");
-        siftsFile = downloadSiftsFile(pdbId.toLowerCase());
+        File oldSiftsFile = new File(siftsFileName + "_old");
+        siftsFile.renameTo(oldSiftsFile);
+        try
+        {
+          siftsFile = downloadSiftsFile(pdbId.toLowerCase());
+          oldSiftsFile.delete();
+          return siftsFile;
+        } catch (IOException e)
+        {
+          e.printStackTrace();
+          oldSiftsFile.renameTo(siftsFile);
+          return new File(siftsFileName);
+        }
+      }
+      else
+      {
+        return siftsFile;
       }
-      return siftsFile;
     }
-    siftsFile = downloadSiftsFile(pdbId.toLowerCase());
+    try
+    {
+      siftsFile = downloadSiftsFile(pdbId.toLowerCase());
+    } catch (IOException e)
+    {
+      throw new SiftsException(e.getMessage());
+    }
     return siftsFile;
   }
 
@@ -261,8 +273,9 @@ public class SiftsClient implements SiftsClientI
     try
     {
       attr = Files.readAttributes(filePath, BasicFileAttributes.class);
-      diffInDays = (int) ((new Date().getTime() - attr.lastModifiedTime()
-              .toMillis()) / (1000 * 60 * 60 * 24));
+      diffInDays = (int) ((new Date().getTime()
+              - attr.lastModifiedTime().toMillis())
+              / (1000 * 60 * 60 * 24));
       // System.out.println("Diff in days : " + diffInDays);
     } catch (IOException e)
     {
@@ -277,8 +290,10 @@ public class SiftsClient implements SiftsClientI
    * @param pdbId
    * @return downloaded SIFTs XML file
    * @throws SiftsException
+   * @throws IOException
    */
-  public static File downloadSiftsFile(String pdbId) throws SiftsException
+  public static File downloadSiftsFile(String pdbId)
+          throws SiftsException, IOException
   {
     if (pdbId.contains(".cif"))
     {
@@ -286,36 +301,47 @@ public class SiftsClient implements SiftsClientI
     }
     String siftFile = pdbId + ".xml.gz";
     String siftsFileFTPURL = SIFTS_FTP_BASE_URL + siftFile;
-    String downloadedSiftsFile = SiftsSettings.getSiftDownloadDirectory()
-            + siftFile;
-    File siftsDownloadDir = new File(
-            SiftsSettings.getSiftDownloadDirectory());
-    if (!siftsDownloadDir.exists())
+    
+    /*
+     * Download the file from URL to either
+     * Java: directory of cached downloaded SIFTS files
+     * Javascript: temporary 'file' (in-memory cache)
+     */
+    File downloadTo = null;
+    if (Platform.isJS())
     {
-      siftsDownloadDir.mkdirs();
+      downloadTo = File.createTempFile(siftFile, ".xml.gz");
     }
-    try
+    else
     {
-      // System.out.println(">> Download ftp url : " + siftsFileFTPURL);
-      URL url = new URL(siftsFileFTPURL);
-      URLConnection conn = url.openConnection();
-      InputStream inputStream = conn.getInputStream();
-      FileOutputStream outputStream = new FileOutputStream(
-              downloadedSiftsFile);
-      byte[] buffer = new byte[BUFFER_SIZE];
-      int bytesRead = -1;
-      while ((bytesRead = inputStream.read(buffer)) != -1)
+      downloadTo = new File(
+              SiftsSettings.getSiftDownloadDirectory() + siftFile);
+      File siftsDownloadDir = new File(
+              SiftsSettings.getSiftDownloadDirectory());
+      if (!siftsDownloadDir.exists())
       {
-        outputStream.write(buffer, 0, bytesRead);
+        siftsDownloadDir.mkdirs();
       }
-      outputStream.close();
-      inputStream.close();
-      // System.out.println(">>> File downloaded : " + downloadedSiftsFile);
-    } catch (IOException ex)
+    }
+
+    // System.out.println(">> Download ftp url : " + siftsFileFTPURL);
+    // long now = System.currentTimeMillis();
+    URL url = new URL(siftsFileFTPURL);
+    URLConnection conn = url.openConnection();
+    InputStream inputStream = conn.getInputStream();
+    FileOutputStream outputStream = new FileOutputStream(
+            downloadTo);
+    byte[] buffer = new byte[BUFFER_SIZE];
+    int bytesRead = -1;
+    while ((bytesRead = inputStream.read(buffer)) != -1)
     {
-      throw new SiftsException(ex.getMessage());
+      outputStream.write(buffer, 0, bytesRead);
     }
-    return new File(downloadedSiftsFile);
+    outputStream.close();
+    inputStream.close();
+    // System.out.println(">>> File downloaded : " + downloadedSiftsFile
+    // + " took " + (System.currentTimeMillis() - now) + "ms");
+    return downloadTo;
   }
 
   /**
@@ -348,39 +374,29 @@ public class SiftsClient implements SiftsClientI
   public DBRefEntryI getValidSourceDBRef(SequenceI seq)
           throws SiftsException
   {
-    DBRefEntryI sourceDBRef = null;
-    sourceDBRef = seq.getSourceDBRef();
-    if (sourceDBRef != null && isValidDBRefEntry(sourceDBRef))
+    List<DBRefEntry> dbRefs = seq.getPrimaryDBRefs();
+    if (dbRefs == null || dbRefs.size() < 1)
     {
-      return sourceDBRef;
+      throw new SiftsException(
+              "Source DBRef could not be determined. DBRefs might not have been retrieved.");
     }
-    else
+
+    for (DBRefEntry dbRef : dbRefs)
     {
-      DBRefEntry[] dbRefs = seq.getDBRefs();
-      if (dbRefs == null || dbRefs.length < 1)
+      if (dbRef == null || dbRef.getAccessionId() == null
+              || dbRef.getSource() == null)
       {
-        throw new SiftsException("Could not get source DB Ref");
+        continue;
       }
-
-      for (DBRefEntryI dbRef : dbRefs)
+      String canonicalSource = DBRefUtils
+              .getCanonicalName(dbRef.getSource());
+      if (isValidDBRefEntry(dbRef)
+              && (canonicalSource.equalsIgnoreCase(DBRefSource.UNIPROT)
+                      || canonicalSource.equalsIgnoreCase(DBRefSource.PDB)))
       {
-        if (dbRef == null || dbRef.getAccessionId() == null
-                || dbRef.getSource() == null)
-        {
-          continue;
-        }
-        if (isFoundInSiftsEntry(dbRef.getAccessionId())
-                && (dbRef.getSource().equalsIgnoreCase(DBRefSource.UNIPROT) || dbRef
-                        .getSource().equalsIgnoreCase(DBRefSource.PDB)))
-        {
-          return dbRef;
-        }
+        return dbRef;
       }
     }
-    if (sourceDBRef != null && isValidDBRefEntry(sourceDBRef))
-    {
-      return sourceDBRef;
-    }
     throw new SiftsException("Could not get source DB Ref");
   }
 
@@ -392,7 +408,7 @@ public class SiftsClient implements SiftsClientI
    *          - DBRefEntry to validate
    * @return true validation is successful otherwise false is returned.
    */
-  private boolean isValidDBRefEntry(DBRefEntryI entry)
+  boolean isValidDBRefEntry(DBRefEntryI entry)
   {
     return entry != null && entry.getAccessionId() != null
             && isFoundInSiftsEntry(entry.getAccessionId());
@@ -412,7 +428,8 @@ public class SiftsClient implements SiftsClientI
                 .getMapRegion();
         for (MapRegion mapRegion : mapRegions)
         {
-          accessions.add(mapRegion.getDb().getDbAccessionId());
+          accessions
+                  .add(mapRegion.getDb().getDbAccessionId().toLowerCase());
         }
       }
     }
@@ -423,9 +440,14 @@ public class SiftsClient implements SiftsClientI
   public StructureMapping getSiftsStructureMapping(SequenceI seq,
           String pdbFile, String chain) throws SiftsException
   {
+    SequenceI aseq = seq;
+    while (seq.getDatasetSequence() != null)
+    {
+      seq = seq.getDatasetSequence();
+    }
     structId = (chain == null) ? pdbId : pdbId + "|" + chain;
-    System.out.println("Getting mapping for: " + pdbId + "|" + chain
-            + " : seq- " + seq.getName());
+    System.out.println("Getting SIFTS mapping for " + structId + ": seq "
+            + seq.getName());
 
     final StringBuilder mappingDetails = new StringBuilder(128);
     PrintStream ps = new PrintStream(System.out)
@@ -445,8 +467,9 @@ public class SiftsClient implements SiftsClientI
     HashMap<Integer, int[]> mapping = getGreedyMapping(chain, seq, ps);
 
     String mappingOutput = mappingDetails.toString();
-    StructureMapping siftsMapping = new StructureMapping(seq, pdbFile,
-            pdbId, chain, mapping, mappingOutput);
+    StructureMapping siftsMapping = new StructureMapping(aseq, pdbFile,
+            pdbId, chain, mapping, mappingOutput, seqFromPdbMapping);
+
     return siftsMapping;
   }
 
@@ -454,21 +477,18 @@ public class SiftsClient implements SiftsClientI
   public HashMap<Integer, int[]> getGreedyMapping(String entityId,
           SequenceI seq, java.io.PrintStream os) throws SiftsException
   {
-    ArrayList<Integer> omitNonObserved = new ArrayList<Integer>();
-    int nonObservedShiftIndex = 0;
-    System.out.println("Generating mappings for : " + entityId);
+    List<Integer> omitNonObserved = new ArrayList<>();
+    int nonObservedShiftIndex = 0,pdbeNonObserved=0;
+    // System.out.println("Generating mappings for : " + entityId);
     Entity entity = null;
     entity = getEntityById(entityId);
     String originalSeq = AlignSeq.extractGaps(
             jalview.util.Comparison.GapChars, seq.getSequenceAsString());
     HashMap<Integer, int[]> mapping = new HashMap<Integer, int[]>();
-    DBRefEntryI sourceDBRef = seq.getSourceDBRef();
-    if (sourceDBRef == null)
-    {
-      sourceDBRef = getValidSourceDBRef(seq);
-      // TODO ensure sequence start/end is in the same coordinate system and
-      // consistent with the choosen sourceDBRef
-    }
+    DBRefEntryI sourceDBRef;
+    sourceDBRef = getValidSourceDBRef(seq);
+    // TODO ensure sequence start/end is in the same coordinate system and
+    // consistent with the choosen sourceDBRef
 
     // set sequence coordinate system - default value is UniProt
     if (sourceDBRef.getSource().equalsIgnoreCase(DBRefSource.PDB))
@@ -488,82 +508,9 @@ public class SiftsClient implements SiftsClientI
 
     TreeMap<Integer, String> resNumMap = new TreeMap<Integer, String>();
     List<Segment> segments = entity.getSegment();
-    for (Segment segment : segments)
-    {
-      segStartEnd = segment.getStart() + " - " + segment.getEnd();
-      System.out.println("Mappging segments : " + segment.getSegId() + "\\"
-              + segStartEnd);
-      List<Residue> residues = segment.getListResidue().getResidue();
-      for (Residue residue : residues)
-      {
-        int currSeqIndex = UNASSIGNED;
-        List<CrossRefDb> cRefDbs = residue.getCrossRefDb();
-        CrossRefDb pdbRefDb = null;
-        for (CrossRefDb cRefDb : cRefDbs)
-        {
-          if (cRefDb.getDbSource().equalsIgnoreCase(DBRefSource.PDB))
-          {
-            pdbRefDb = cRefDb;
-          }
-          if (cRefDb.getDbCoordSys()
-                  .equalsIgnoreCase(seqCoordSys.getName())
-                  && isAccessionMatched(cRefDb.getDbAccessionId()))
-          {
-            String resNumIndexString = cRefDb.getDbResNum()
-                    .equalsIgnoreCase("None") ? String.valueOf(UNASSIGNED)
-                    : cRefDb.getDbResNum();
-            try
-            {
-              currSeqIndex = Integer.valueOf(resNumIndexString);
-            } catch (NumberFormatException nfe)
-            {
-              currSeqIndex = Integer.valueOf(resNumIndexString
-                      .split("[a-zA-Z]")[0]);
-            }
-            if (pdbRefDb != null)
-            {
-              break;// exit loop if pdb and uniprot are already found
-            }
-          }
-        }
-        if (currSeqIndex == UNASSIGNED)
-        {
-          continue;
-        }
-        if (currSeqIndex >= seq.getStart() && currSeqIndex <= seq.getEnd())
-        {
-          int resNum;
-          try
-          {
-            resNum = (pdbRefDb == null) ? Integer.valueOf(residue
-                    .getDbResNum()) : Integer.valueOf(pdbRefDb
-                    .getDbResNum());
-          } catch (NumberFormatException nfe)
-          {
-            resNum = (pdbRefDb == null) ? Integer.valueOf(residue
-                    .getDbResNum()) : Integer.valueOf(pdbRefDb
-                    .getDbResNum().split("[a-zA-Z]")[0]);
-            continue;
-          }
-
-          if (isResidueObserved(residue)
-                  || seqCoordSys == CoordinateSys.UNIPROT)
-          {
-            char resCharCode = ResidueProperties
-                    .getSingleCharacterCode(ResidueProperties
-                            .getCanonicalAminoAcid(residue.getDbResName()));
-            resNumMap.put(currSeqIndex, String.valueOf(resCharCode));
-          }
-          else
-          {
-            omitNonObserved.add(currSeqIndex);
-            ++nonObservedShiftIndex;
-          }
-          mapping.put(currSeqIndex - nonObservedShiftIndex, new int[] {
-              Integer.valueOf(resNum), UNASSIGNED });
-        }
-      }
-    }
+    SegmentHelperPojo shp = new SegmentHelperPojo(seq, mapping, resNumMap,
+            omitNonObserved, nonObservedShiftIndex,pdbeNonObserved);
+    processSegments(segments, shp);
     try
     {
       populateAtomPositions(entityId, mapping);
@@ -580,25 +527,73 @@ public class SiftsClient implements SiftsClientI
     int pdbStart = UNASSIGNED;
     int pdbEnd = UNASSIGNED;
 
-    Integer[] keys = mapping.keySet().toArray(new Integer[0]);
-    Arrays.sort(keys);
-    if (keys.length < 1)
+    if (mapping.isEmpty())
     {
-      throw new SiftsException(">>> Empty SIFTS mapping generated!!");
+      throw new SiftsException("SIFTS mapping failed");
     }
+    // also construct a mapping object between the seq-coord sys and the PDB seq's coord sys
+
+    Integer[] keys = mapping.keySet().toArray(new Integer[0]);
+    Arrays.sort(keys);
     seqStart = keys[0];
     seqEnd = keys[keys.length - 1];
-
+    List<int[]> from=new ArrayList<>(),to=new ArrayList<>();
+    int[]_cfrom=null,_cto=null;
     String matchedSeq = originalSeq;
-    if (seqStart != UNASSIGNED)
+    if (seqStart != UNASSIGNED) // fixme! seqStart can map to -1 for a pdb sequence that starts <-1
     {
+      for (int seqps:keys)
+      {
+        int pdbpos = mapping.get(seqps)[PDBE_POS];
+        if (pdbpos == UNASSIGNED)
+        {
+          // not correct - pdbpos might be -1, but leave it for now
+          continue;
+        }
+        if (_cfrom==null || seqps!=_cfrom[1]+1)
+        {
+          _cfrom = new int[] { seqps,seqps};
+          from.add(_cfrom);
+          _cto = null; // discontinuity
+        } else {
+          _cfrom[1]= seqps;
+        }
+        if (_cto==null || pdbpos!=1+_cto[1])
+        {
+          _cto = new int[] { pdbpos,pdbpos};
+          to.add(_cto);
+        } else {
+          _cto[1] = pdbpos;
+        }
+      }
+      _cfrom = new int[from.size() * 2];
+      _cto = new int[to.size() * 2];
+      int p = 0;
+      for (int[] range : from)
+      {
+        _cfrom[p++] = range[0];
+        _cfrom[p++] = range[1];
+      }
+      ;
+      p = 0;
+      for (int[] range : to)
+      {
+        _cto[p++] = range[0];
+        _cto[p++] = range[1];
+      }
+      ;
+
+      seqFromPdbMapping = new jalview.datamodel.Mapping(null, _cto, _cfrom,
+              1,
+              1);
       pdbStart = mapping.get(seqStart)[PDB_RES_POS];
       pdbEnd = mapping.get(seqEnd)[PDB_RES_POS];
       int orignalSeqStart = seq.getStart();
       if (orignalSeqStart >= 1)
       {
-        int subSeqStart = (seqStart >= orignalSeqStart) ? seqStart
-                - orignalSeqStart : 0;
+        int subSeqStart = (seqStart >= orignalSeqStart)
+                ? seqStart - orignalSeqStart
+                : 0;
         int subSeqEnd = seqEnd - (orignalSeqStart - 1);
         subSeqEnd = originalSeq.length() < subSeqEnd ? originalSeq.length()
                 : subSeqEnd;
@@ -619,22 +614,162 @@ public class SiftsClient implements SiftsClientI
     if (os != null)
     {
       MappingOutputPojo mop = new MappingOutputPojo();
-      mop.setSeqStart(pdbStart);
-      mop.setSeqEnd(pdbEnd);
+      mop.setSeqStart(seqStart);
+      mop.setSeqEnd(seqEnd);
       mop.setSeqName(seq.getName());
       mop.setSeqResidue(matchedSeq);
 
-      mop.setStrStart(seqStart);
-      mop.setStrEnd(seqEnd);
+      mop.setStrStart(pdbStart);
+      mop.setStrEnd(pdbEnd);
       mop.setStrName(structId);
       mop.setStrResidue(targetStrucSeqs.toString());
 
       mop.setType("pep");
       os.print(getMappingOutput(mop).toString());
+      os.println();
     }
     return mapping;
   }
 
+  void processSegments(List<Segment> segments, SegmentHelperPojo shp)
+  {
+    SequenceI seq = shp.getSeq();
+    HashMap<Integer, int[]> mapping = shp.getMapping();
+    TreeMap<Integer, String> resNumMap = shp.getResNumMap();
+    List<Integer> omitNonObserved = shp.getOmitNonObserved();
+    int nonObservedShiftIndex = shp.getNonObservedShiftIndex();
+    int pdbeNonObservedCount = shp.getPdbeNonObserved();
+    int firstPDBResNum = UNASSIGNED;
+    for (Segment segment : segments)
+    {
+      // System.out.println("Mapping segments : " + segment.getSegId() + "\\"s
+      // + segStartEnd);
+      List<Residue> residues = segment.getListResidue().getResidue();
+      for (Residue residue : residues)
+      {
+        boolean isObserved = isResidueObserved(residue);
+        int pdbeIndex = getLeadingIntegerValue(residue.getDbResNum(),
+                UNASSIGNED);
+        int currSeqIndex = UNASSIGNED;
+        List<CrossRefDb> cRefDbs = residue.getCrossRefDb();
+        CrossRefDb pdbRefDb = null;
+        for (CrossRefDb cRefDb : cRefDbs)
+        {
+          if (cRefDb.getDbSource().equalsIgnoreCase(DBRefSource.PDB))
+          {
+            pdbRefDb = cRefDb;
+            if (firstPDBResNum == UNASSIGNED)
+            {
+              firstPDBResNum = getLeadingIntegerValue(cRefDb.getDbResNum(),
+                      UNASSIGNED);
+            }
+            else
+            {
+              if (isObserved)
+              {
+                // after we find the first observed residue we just increment
+                firstPDBResNum++;
+              }
+            }
+          }
+          if (cRefDb.getDbCoordSys().equalsIgnoreCase(seqCoordSys.getName())
+                  && isAccessionMatched(cRefDb.getDbAccessionId()))
+          {
+            currSeqIndex = getLeadingIntegerValue(cRefDb.getDbResNum(),
+                    UNASSIGNED);
+            if (pdbRefDb != null)
+            {
+              break;// exit loop if pdb and uniprot are already found
+            }
+          }
+        }
+        if (!isObserved)
+        {
+          ++pdbeNonObservedCount;
+        }
+        if (seqCoordSys == seqCoordSys.PDB) // FIXME: is seqCoordSys ever PDBe
+                                            // ???
+        {
+          // if the sequence has a primary reference to the PDB, then we are
+          // dealing with a sequence extracted directly from the PDB. In that
+          // case, numbering is PDBe - non-observed residues
+          currSeqIndex = seq.getStart() - 1 + pdbeIndex;
+        }
+        if (!isObserved)
+        {
+          if (seqCoordSys != CoordinateSys.UNIPROT) // FIXME: PDB or PDBe only
+                                                    // here
+          {
+            // mapping to PDB or PDBe so we need to bookkeep for the
+            // non-observed
+            // SEQRES positions
+            omitNonObserved.add(currSeqIndex);
+            ++nonObservedShiftIndex;
+          }
+        }
+        if (currSeqIndex == UNASSIGNED)
+        {
+          // change in logic - unobserved residues with no currSeqIndex
+          // corresponding are still counted in both nonObservedShiftIndex and
+          // pdbeIndex...
+          continue;
+        }
+        // if (currSeqIndex >= seq.getStart() && currSeqIndex <= seqlength) //
+        // true
+                                                                         // numbering
+                                                                         // is
+                                                                         // not
+                                                                         // up
+                                                                         // to
+                                                                         // seq.getEnd()
+        {
+
+          int resNum = (pdbRefDb == null)
+                  ? getLeadingIntegerValue(residue.getDbResNum(),
+                          UNASSIGNED)
+                  : getLeadingIntegerValue(pdbRefDb.getDbResNum(),
+                          UNASSIGNED);
+
+          if (isObserved)
+          {
+            char resCharCode = ResidueProperties
+                    .getSingleCharacterCode(ResidueProperties
+                            .getCanonicalAminoAcid(residue.getDbResName()));
+            resNumMap.put(currSeqIndex, String.valueOf(resCharCode));
+
+            int[] mappingcols = new int[] { Integer.valueOf(resNum),
+                UNASSIGNED, isObserved ? firstPDBResNum : UNASSIGNED };
+
+            mapping.put(currSeqIndex - nonObservedShiftIndex, mappingcols);
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * Get the leading integer part of a string that begins with an integer.
+   * 
+   * @param input
+   *          - the string input to process
+   * @param failValue
+   *          - value returned if unsuccessful
+   * @return
+   */
+  static int getLeadingIntegerValue(String input, int failValue)
+  {
+    if (input == null)
+    {
+      return failValue;
+    }
+    String[] parts = input.split("(?=\\D)(?<=\\d)");
+    if (parts != null && parts.length > 0 && parts[0].matches("[0-9]+"))
+    {
+      return Integer.valueOf(parts[0]);
+    }
+    return failValue;
+  }
+
   /**
    * 
    * @param chainId
@@ -643,22 +778,33 @@ public class SiftsClient implements SiftsClientI
    *          Two dimension array of residue index versus atom position
    * @throws IllegalArgumentException
    *           Thrown if chainId or mapping is null
+   * @throws SiftsException
    */
-  void populateAtomPositions(String chainId,
-          HashMap<Integer, int[]> mapping) throws IllegalArgumentException
+  void populateAtomPositions(String chainId, Map<Integer, int[]> mapping)
+          throws IllegalArgumentException, SiftsException
   {
-    PDBChain chain = pdb.findChain(chainId);
-    if (chain == null || mapping == null)
-    {
-      throw new IllegalArgumentException(
-              "Chain id or mapping must not be null.");
-    }
-    for (int[] map : mapping.values())
+    try
     {
-      if (map[PDB_RES_POS] != UNASSIGNED)
+      PDBChain chain = pdb.findChain(chainId);
+
+      if (chain == null || mapping == null)
+      {
+        throw new IllegalArgumentException(
+                "Chain id or mapping must not be null.");
+      }
+      for (int[] map : mapping.values())
       {
-        map[PDB_ATOM_POS] = getAtomIndex(map[PDB_RES_POS], chain.atoms);
+        if (map[PDB_RES_POS] != UNASSIGNED)
+        {
+          map[PDB_ATOM_POS] = getAtomIndex(map[PDB_RES_POS], chain.atoms);
+        }
       }
+    } catch (NullPointerException e)
+    {
+      throw new SiftsException(e.getMessage());
+    } catch (Exception e)
+    {
+      throw new SiftsException(e.getMessage());
     }
   }
 
@@ -695,7 +841,7 @@ public class SiftsClient implements SiftsClientI
    */
   private boolean isResidueObserved(Residue residue)
   {
-    HashSet<String> annotations = getResidueAnnotaitons(residue,
+    Set<String> annotations = getResidueAnnotaitons(residue,
             ResidueDetailType.ANNOTATION);
     if (annotations == null || annotations.isEmpty())
     {
@@ -718,7 +864,7 @@ public class SiftsClient implements SiftsClientI
    * @param type
    * @return
    */
-  private HashSet<String> getResidueAnnotaitons(Residue residue,
+  private Set<String> getResidueAnnotaitons(Residue residue,
           ResidueDetailType type)
   {
     HashSet<String> foundAnnotations = new HashSet<String>();
@@ -743,8 +889,9 @@ public class SiftsClient implements SiftsClientI
 
   private boolean isFoundInSiftsEntry(String accessionId)
   {
+    Set<String> siftsDBRefs = getAllMappingAccession();
     return accessionId != null
-            && getAllMappingAccession().contains(accessionId);
+            && siftsDBRefs.contains(accessionId.toLowerCase());
   }
 
   /**
@@ -752,19 +899,19 @@ public class SiftsClient implements SiftsClientI
    * 
    * @param resNumMap
    */
-  void padWithGaps(TreeMap<Integer, String> resNumMap,
-          ArrayList<Integer> omitNonObserved)
+  void padWithGaps(Map<Integer, String> resNumMap,
+          List<Integer> omitNonObserved)
   {
     if (resNumMap == null || resNumMap.isEmpty())
     {
       return;
     }
     Integer[] keys = resNumMap.keySet().toArray(new Integer[0]);
-    Arrays.sort(keys);
+    // Arrays.sort(keys);
     int firstIndex = keys[0];
     int lastIndex = keys[keys.length - 1];
-    System.out.println("Min value " + firstIndex);
-    System.out.println("Max value " + lastIndex);
+    // System.out.println("Min value " + firstIndex);
+    // System.out.println("Max value " + lastIndex);
     for (int x = firstIndex; x <= lastIndex; x++)
     {
       if (!resNumMap.containsKey(x) && !omitNonObserved.contains(x))
@@ -774,33 +921,11 @@ public class SiftsClient implements SiftsClientI
     }
   }
 
-
-
   @Override
   public Entity getEntityById(String id) throws SiftsException
   {
-    // Sometimes SIFTS mappings are wrongly swapped between different chains of
-    // a PDB entry. This results to wrong mappings being generated. The boolean
-    // flag 'isGetEntityIdDirectly, determines whether an entity to process is
-    // determined by a greedy heuristic search or by just matching the Chain Id
-    // directly against the entity Id tag. Setting the default value to 'false'
-    // utilise the heuristic search which always produces correct mappings but
-    // less optimised processing, where as changing the value to 'true'
-    // optimises performance but might result to incorrect mapping in some cases
-    // where SIFTS mappings are wrongly swapped between different chains.
-    boolean isGetEntityIdDirectly = false;
-    if (isGetEntityIdDirectly)
-    {
-      List<Entity> entities = siftsEntry.getEntity();
-      for (Entity entity : entities)
-      {
-        if (!entity.getEntityId().equalsIgnoreCase(id))
-        {
-          continue;
-        }
-        return entity;
-      }
-    }
+    // Determines an entity to process by performing a heuristic matching of all
+    // Entities with the given chainId and choosing the best matching Entity
     Entity entity = getEntityByMostOptimalMatchedId(id);
     if (entity != null)
     {
@@ -819,7 +944,8 @@ public class SiftsClient implements SiftsClientI
    */
   public Entity getEntityByMostOptimalMatchedId(String chainId)
   {
-    // System.out.println("---> advanced greedy entityId matching block entered..");
+    // System.out.println("---> advanced greedy entityId matching block
+    // entered..");
     List<Entity> entities = siftsEntry.getEntity();
     SiftsEntitySortPojo[] sPojo = new SiftsEntitySortPojo[entities.size()];
     int count = 0;
@@ -849,15 +975,20 @@ public class SiftsClient implements SiftsClientI
           }
         }
       }
-      sPojo[count].pid = 100 * (sPojo[count].chainIdFreq / sPojo[count].resCount);
+      sPojo[count].pid = (100 * sPojo[count].chainIdFreq)
+              / sPojo[count].resCount;
       ++count;
     }
     Arrays.sort(sPojo, Collections.reverseOrder());
-    System.out.println("highest matched entity : " + sPojo[0].entityId);
-    System.out.println("highest matched pid : " + sPojo[0].pid);
+    // System.out.println("highest matched entity : " + sPojo[0].entityId);
+    // System.out.println("highest matched pid : " + sPojo[0].pid);
 
     if (sPojo[0].entityId != null)
     {
+      if (sPojo[0].pid < 1)
+      {
+        return null;
+      }
       for (Entity entity : entities)
       {
         if (!entity.getEntityId().equalsIgnoreCase(sPojo[0].entityId))
@@ -870,8 +1001,8 @@ public class SiftsClient implements SiftsClientI
     return null;
   }
 
-  public class SiftsEntitySortPojo implements
-          Comparable<SiftsEntitySortPojo>
+  private class SiftsEntitySortPojo
+          implements Comparable<SiftsEntitySortPojo>
   {
     public String entityId;
 
@@ -888,22 +1019,102 @@ public class SiftsClient implements SiftsClientI
     }
   }
 
-  @Override
-  public String[] getEntryDBs()
+  private class SegmentHelperPojo
   {
-    System.out.println("\nListing DB entries...");
-    List<String> availDbs = new ArrayList<String>();
-    List<Db> dbs = siftsEntry.getListDB().getDb();
-    for (Db db : dbs)
+    private SequenceI seq;
+
+    private HashMap<Integer, int[]> mapping;
+
+    private TreeMap<Integer, String> resNumMap;
+
+    private List<Integer> omitNonObserved;
+
+    private int nonObservedShiftIndex;
+
+    /**
+     * count of number of 'not observed' positions in the PDB record's SEQRES
+     * (total number of residues with coordinates == length(SEQRES) -
+     * pdbeNonObserved
+     */
+    private int pdbeNonObserved;
+
+    public SegmentHelperPojo(SequenceI seq, HashMap<Integer, int[]> mapping,
+            TreeMap<Integer, String> resNumMap,
+            List<Integer> omitNonObserved, int nonObservedShiftIndex,
+            int pdbeNonObserved)
+    {
+      setSeq(seq);
+      setMapping(mapping);
+      setResNumMap(resNumMap);
+      setOmitNonObserved(omitNonObserved);
+      setNonObservedShiftIndex(nonObservedShiftIndex);
+      setPdbeNonObserved(pdbeNonObserved);
+
+    }
+
+    public void setPdbeNonObserved(int pdbeNonObserved2)
+    {
+      this.pdbeNonObserved = pdbeNonObserved2;
+    }
+
+    public int getPdbeNonObserved()
+    {
+      return pdbeNonObserved;
+    }
+    public SequenceI getSeq()
+    {
+      return seq;
+    }
+
+    public void setSeq(SequenceI seq)
+    {
+      this.seq = seq;
+    }
+
+    public HashMap<Integer, int[]> getMapping()
+    {
+      return mapping;
+    }
+
+    public void setMapping(HashMap<Integer, int[]> mapping)
+    {
+      this.mapping = mapping;
+    }
+
+    public TreeMap<Integer, String> getResNumMap()
+    {
+      return resNumMap;
+    }
+
+    public void setResNumMap(TreeMap<Integer, String> resNumMap)
+    {
+      this.resNumMap = resNumMap;
+    }
+
+    public List<Integer> getOmitNonObserved()
+    {
+      return omitNonObserved;
+    }
+
+    public void setOmitNonObserved(List<Integer> omitNonObserved)
+    {
+      this.omitNonObserved = omitNonObserved;
+    }
+
+    public int getNonObservedShiftIndex()
+    {
+      return nonObservedShiftIndex;
+    }
+
+    public void setNonObservedShiftIndex(int nonObservedShiftIndex)
     {
-      availDbs.add(db.getDbSource());
-      System.out.println(db.getDbSource() + " | " + db.getDbCoordSys());
+      this.nonObservedShiftIndex = nonObservedShiftIndex;
     }
-    return availDbs.toArray(new String[0]);
+
   }
 
   @Override
-  public StringBuffer getMappingOutput(MappingOutputPojo mp)
+  public StringBuilder getMappingOutput(MappingOutputPojo mp)
           throws SiftsException
   {
     String seqRes = mp.getSeqResidue();
@@ -925,10 +1136,10 @@ public class SiftsClient implements SiftsClientI
     int nochunks = ((seqRes.length()) / len)
             + ((seqRes.length()) % len > 0 ? 1 : 0);
     // output mappings
-    StringBuffer output = new StringBuffer();
+    StringBuilder output = new StringBuilder(512);
     output.append(NEWLINE);
-    output.append("Sequence \u27f7 Structure mapping details").append(
-            NEWLINE);
+    output.append("Sequence \u27f7 Structure mapping details")
+            .append(NEWLINE);
     output.append("Method: SIFTS");
     output.append(NEWLINE).append(NEWLINE);
 
@@ -946,12 +1157,13 @@ public class SiftsClient implements SiftsClientI
     output.append(String.valueOf(pdbEnd));
     output.append(NEWLINE).append(NEWLINE);
 
+    ScoreMatrix pam250 = ScoreModels.getInstance().getPam250();
     int matchedSeqCount = 0;
     for (int j = 0; j < nochunks; j++)
     {
       // Print the first aligned sequence
-      output.append(new Format("%" + (maxid) + "s").form(seqName)).append(
-              " ");
+      output.append(new Format("%" + (maxid) + "s").form(seqName))
+              .append(" ");
 
       for (int i = 0; i < len; i++)
       {
@@ -964,25 +1176,29 @@ public class SiftsClient implements SiftsClientI
       output.append(NEWLINE);
       output.append(new Format("%" + (maxid) + "s").form(" ")).append(" ");
 
-      // Print out the matching chars
+      /*
+       * Print out the match symbols:
+       * | for exact match (ignoring case)
+       * . if PAM250 score is positive
+       * else a space
+       */
       for (int i = 0; i < len; i++)
       {
         try
         {
           if ((i + (j * len)) < seqRes.length())
           {
-            if (seqRes.charAt(i + (j * len)) == strRes
-                    .charAt(i + (j * len))
-                    && !jalview.util.Comparison.isGap(seqRes.charAt(i
-                            + (j * len))))
+            char c1 = seqRes.charAt(i + (j * len));
+            char c2 = strRes.charAt(i + (j * len));
+            boolean sameChar = Comparison.isSameResidue(c1, c2, false);
+            if (sameChar && !Comparison.isGap(c1))
             {
               matchedSeqCount++;
               output.append("|");
             }
             else if (type.equals("pep"))
             {
-              if (ResidueProperties.getPAM250(seqRes.charAt(i + (j * len)),
-                      strRes.charAt(i + (j * len))) > 0)
+              if (pam250.getPairwiseScore(c1, c2) > 0)
               {
                 output.append(".");
               }
@@ -1019,10 +1235,9 @@ public class SiftsClient implements SiftsClientI
     {
       throw new SiftsException(">>> Low PID detected for SIFTs mapping...");
     }
-    output.append("Length of alignment = " + seqRes.length()).append(
-            NEWLINE);
+    output.append("Length of alignment = " + seqRes.length())
+            .append(NEWLINE);
     output.append(new Format("Percentage ID = %2.2f").form(pid));
-    output.append(NEWLINE);
     return output;
   }
 
@@ -1045,12 +1260,6 @@ public class SiftsClient implements SiftsClientI
   }
 
   @Override
-  public String getDbEvidence()
-  {
-    return siftsEntry.getDbEvidence();
-  }
-
-  @Override
   public String getDbSource()
   {
     return siftsEntry.getDbSource();
@@ -1062,4 +1271,9 @@ public class SiftsClient implements SiftsClientI
     return siftsEntry.getDbVersion();
   }
 
+  public static void setMockSiftsFile(File file)
+  {
+    mockSiftsFile = file;
+  }
+
 }