JAL-3633 ensure check both http:// or https:// for urls
[jalview.git] / src / jalview / project / Jalview2XML.java
index 3a9060e..aa4cd6c 100644 (file)
  */
 package jalview.project;
 
+import static jalview.math.RotatableMatrix.Axis.X;
+import static jalview.math.RotatableMatrix.Axis.Y;
+import static jalview.math.RotatableMatrix.Axis.Z;
+
+import java.awt.Color;
+import java.awt.Font;
+import java.awt.Rectangle;
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.lang.reflect.InvocationTargetException;
+import java.math.BigInteger;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.GregorianCalendar;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Hashtable;
+import java.util.IdentityHashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.Vector;
+import java.util.jar.JarEntry;
+import java.util.jar.JarInputStream;
+import java.util.jar.JarOutputStream;
+
+import javax.swing.JInternalFrame;
+import javax.swing.SwingUtilities;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBElement;
+import javax.xml.bind.Marshaller;
+import javax.xml.datatype.DatatypeConfigurationException;
+import javax.xml.datatype.DatatypeFactory;
+import javax.xml.datatype.XMLGregorianCalendar;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamReader;
+
 import jalview.analysis.Conservation;
+import jalview.analysis.PCA;
+import jalview.analysis.scoremodels.ScoreModels;
+import jalview.analysis.scoremodels.SimilarityParams;
 import jalview.api.FeatureColourI;
 import jalview.api.ViewStyleI;
+import jalview.api.analysis.ScoreModelI;
+import jalview.api.analysis.SimilarityParamsI;
 import jalview.api.structures.JalviewStructureDisplayI;
 import jalview.bin.Cache;
 import jalview.datamodel.AlignedCodonFrame;
 import jalview.datamodel.Alignment;
 import jalview.datamodel.AlignmentAnnotation;
 import jalview.datamodel.AlignmentI;
+import jalview.datamodel.DBRefEntry;
+import jalview.datamodel.GeneLocus;
 import jalview.datamodel.GraphLine;
 import jalview.datamodel.PDBEntry;
+import jalview.datamodel.Point;
 import jalview.datamodel.RnaViewerModel;
 import jalview.datamodel.SequenceFeature;
 import jalview.datamodel.SequenceGroup;
@@ -48,19 +111,21 @@ import jalview.gui.AlignmentPanel;
 import jalview.gui.AppVarna;
 import jalview.gui.ChimeraViewFrame;
 import jalview.gui.Desktop;
-import jalview.gui.FeatureRenderer;
-import jalview.gui.Jalview2XML_V1;
 import jalview.gui.JvOptionPane;
 import jalview.gui.OOMWarning;
+import jalview.gui.PCAPanel;
 import jalview.gui.PaintRefresher;
 import jalview.gui.SplitFrame;
 import jalview.gui.StructureViewer;
 import jalview.gui.StructureViewer.ViewerType;
 import jalview.gui.StructureViewerBase;
 import jalview.gui.TreePanel;
+import jalview.io.BackupFiles;
 import jalview.io.DataSourceType;
 import jalview.io.FileFormat;
 import jalview.io.NewickFile;
+import jalview.math.Matrix;
+import jalview.math.MatrixI;
 import jalview.renderer.ResidueShaderI;
 import jalview.schemes.AnnotationColourGradient;
 import jalview.schemes.ColourSchemeI;
@@ -71,13 +136,16 @@ import jalview.schemes.UserColourScheme;
 import jalview.structure.StructureSelectionManager;
 import jalview.structures.models.AAStructureBindingModel;
 import jalview.util.Format;
+import jalview.util.HttpUtils;
 import jalview.util.MessageManager;
 import jalview.util.Platform;
 import jalview.util.StringUtils;
 import jalview.util.jarInputStreamProvider;
 import jalview.util.matcher.Condition;
 import jalview.viewmodel.AlignmentViewport;
+import jalview.viewmodel.PCAModel;
 import jalview.viewmodel.ViewportRanges;
+import jalview.viewmodel.seqfeatures.FeatureRendererModel;
 import jalview.viewmodel.seqfeatures.FeatureRendererSettings;
 import jalview.viewmodel.seqfeatures.FeaturesDisplayed;
 import jalview.ws.jws2.Jws2Discoverer;
@@ -92,6 +160,8 @@ import jalview.xml.binding.jalview.Annotation;
 import jalview.xml.binding.jalview.Annotation.ThresholdLine;
 import jalview.xml.binding.jalview.AnnotationColourScheme;
 import jalview.xml.binding.jalview.AnnotationElement;
+import jalview.xml.binding.jalview.DoubleMatrix;
+import jalview.xml.binding.jalview.DoubleVector;
 import jalview.xml.binding.jalview.Feature;
 import jalview.xml.binding.jalview.Feature.OtherData;
 import jalview.xml.binding.jalview.FeatureMatcherSet.CompoundMatcher;
@@ -106,6 +176,11 @@ import jalview.xml.binding.jalview.JalviewModel.JSeq.Pdbids;
 import jalview.xml.binding.jalview.JalviewModel.JSeq.Pdbids.StructureState;
 import jalview.xml.binding.jalview.JalviewModel.JSeq.RnaViewer;
 import jalview.xml.binding.jalview.JalviewModel.JSeq.RnaViewer.SecondaryStructure;
+import jalview.xml.binding.jalview.JalviewModel.PcaViewer;
+import jalview.xml.binding.jalview.JalviewModel.PcaViewer.Axis;
+import jalview.xml.binding.jalview.JalviewModel.PcaViewer.SeqPointMax;
+import jalview.xml.binding.jalview.JalviewModel.PcaViewer.SeqPointMin;
+import jalview.xml.binding.jalview.JalviewModel.PcaViewer.SequencePoint;
 import jalview.xml.binding.jalview.JalviewModel.Tree;
 import jalview.xml.binding.jalview.JalviewModel.UserColours;
 import jalview.xml.binding.jalview.JalviewModel.Viewport;
@@ -118,6 +193,7 @@ import jalview.xml.binding.jalview.MapListType.MapListTo;
 import jalview.xml.binding.jalview.Mapping;
 import jalview.xml.binding.jalview.NoValueColour;
 import jalview.xml.binding.jalview.ObjectFactory;
+import jalview.xml.binding.jalview.PcaDataType;
 import jalview.xml.binding.jalview.Pdbentry.Property;
 import jalview.xml.binding.jalview.Sequence;
 import jalview.xml.binding.jalview.Sequence.DBRef;
@@ -126,54 +202,6 @@ import jalview.xml.binding.jalview.SequenceSet.SequenceSetProperties;
 import jalview.xml.binding.jalview.ThresholdType;
 import jalview.xml.binding.jalview.VAMSAS;
 
-import java.awt.Color;
-import java.awt.Font;
-import java.awt.Rectangle;
-import java.io.BufferedReader;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.lang.reflect.InvocationTargetException;
-import java.math.BigInteger;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Enumeration;
-import java.util.GregorianCalendar;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Hashtable;
-import java.util.IdentityHashMap;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.Vector;
-import java.util.jar.JarEntry;
-import java.util.jar.JarInputStream;
-import java.util.jar.JarOutputStream;
-
-import javax.swing.JInternalFrame;
-import javax.swing.SwingUtilities;
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBElement;
-import javax.xml.bind.Marshaller;
-import javax.xml.datatype.DatatypeConfigurationException;
-import javax.xml.datatype.DatatypeFactory;
-import javax.xml.datatype.XMLGregorianCalendar;
-import javax.xml.stream.XMLInputFactory;
-import javax.xml.stream.XMLStreamReader;
-
 /**
  * Write out the current jalview desktop state as a Jalview XML stream.
  * 
@@ -186,12 +214,27 @@ import javax.xml.stream.XMLStreamReader;
  */
 public class Jalview2XML
 {
+
+  // BH 2018 we add the .jvp binary extension to J2S so that
+  // it will declare that binary when we do the file save from the browser
+
+  static
+  {
+    Platform.addJ2SBinaryType(".jvp?");
+  }
+
   private static final String VIEWER_PREFIX = "viewer_";
 
   private static final String RNA_PREFIX = "rna_";
 
   private static final String UTF_8 = "UTF-8";
 
+  /**
+   * prefix for recovering datasets for alignments with multiple views where
+   * non-existent dataset IDs were written for some views
+   */
+  private static final String UNIQSEQSETID = "uniqueSeqSetId.";
+
   // use this with nextCounter() to make unique names for entities
   private int counter = 0;
 
@@ -423,7 +466,7 @@ public class Jalview2XML
       public boolean isResolvable()
       {
         return super.isResolvable() && mp.getTo() != null;
-      };
+      }
 
       @Override
       boolean resolve()
@@ -523,24 +566,30 @@ public class Jalview2XML
   public void saveState(File statefile)
   {
     FileOutputStream fos = null;
+
     try
     {
+
       fos = new FileOutputStream(statefile);
+
       JarOutputStream jout = new JarOutputStream(fos);
       saveState(jout);
+      fos.close();
 
     } catch (Exception e)
     {
+      Cache.log.error("Couln't write Jalview state to " + statefile, e);
       // TODO: inform user of the problem - they need to know if their data was
       // not saved !
       if (errorMessage == null)
       {
-        errorMessage = "Couldn't write Jalview Archive to output file '"
+        errorMessage = "Did't write Jalview Archive to output file '"
                 + statefile + "' - See console error log for details";
       }
       else
       {
-        errorMessage += "(output file was '" + statefile + "')";
+        errorMessage += "(Didn't write Jalview Archive to output file '"
+                + statefile + ")";
       }
       e.printStackTrace();
     } finally
@@ -649,7 +698,6 @@ public class Jalview2XML
       } catch (Exception foo)
       {
       }
-      ;
       jout.close();
     } catch (Exception ex)
     {
@@ -710,7 +758,12 @@ public class Jalview2XML
   {
     try
     {
-      FileOutputStream fos = new FileOutputStream(jarFile);
+      // create backupfiles object and get new temp filename destination
+      boolean doBackup = BackupFiles.getEnabled();
+      BackupFiles backupfiles = doBackup ? new BackupFiles(jarFile) : null;
+      FileOutputStream fos = new FileOutputStream(doBackup ? 
+              backupfiles.getTempFilePath() : jarFile);
+
       JarOutputStream jout = new JarOutputStream(fos);
       List<AlignFrame> frames = new ArrayList<>();
 
@@ -730,9 +783,16 @@ public class Jalview2XML
       } catch (Exception foo)
       {
       }
-      ;
       jout.close();
-      return true;
+      boolean success = true;
+
+      if (doBackup)
+      {
+        backupfiles.setWriteSuccess(success);
+        success = backupfiles.rollBackupsAndRenameTempFile();
+      }
+
+      return success;
     } catch (Exception ex)
     {
       errorMessage = "Couldn't Write alignment view to Jalview Archive - see error output for details";
@@ -901,7 +961,7 @@ public class Jalview2XML
         else
         {
           vamsasSeq = createVamsasSequence(id, jds);
-//          vamsasSet.addSequence(vamsasSeq);
+          // vamsasSet.addSequence(vamsasSeq);
           vamsasSet.getSequence().add(vamsasSeq);
           vamsasSetIds.put(id, vamsasSeq);
           seqRefIds.put(id, jds);
@@ -1032,25 +1092,28 @@ public class Jalview2XML
             if (frames[f] instanceof StructureViewerBase)
             {
               StructureViewerBase viewFrame = (StructureViewerBase) frames[f];
-              matchedFile = saveStructureState(ap, jds, pdb, entry, viewIds,
+              matchedFile = saveStructureViewer(ap, jds, pdb, entry, viewIds,
                       matchedFile, viewFrame);
               /*
                * Only store each structure viewer's state once in the project
                * jar. First time through only (storeDS==false)
                */
               String viewId = viewFrame.getViewId();
+              String viewerType = viewFrame.getViewerType().toString();
               if (!storeDS && !viewIds.contains(viewId))
               {
                 viewIds.add(viewId);
-                try
+                File viewerState = viewFrame.saveSession();
+                if (viewerState != null)
                 {
-                  String viewerState = viewFrame.getStateInfo();
-                  writeJarEntry(jout, getViewerJarEntryName(viewId),
-                          viewerState.getBytes());
-                } catch (IOException e)
+                  copyFileToJar(jout, viewerState.getPath(),
+                          getViewerJarEntryName(viewId), viewerType);
+                }
+                else
                 {
-                  System.err.println(
-                          "Error saving viewer state: " + e.getMessage());
+                  Cache.log.error("Failed to save viewer state for "
+                          +
+                          viewerType);
                 }
               }
             }
@@ -1072,7 +1135,7 @@ public class Jalview2XML
             if (!pdbfiles.contains(pdbId))
             {
               pdbfiles.add(pdbId);
-              copyFileToJar(jout, matchedFile, pdbId);
+              copyFileToJar(jout, matchedFile, pdbId, pdbId);
             }
           }
 
@@ -1209,6 +1272,9 @@ public class Jalview2XML
               tree.setXpos(tp.getX());
               tree.setYpos(tp.getY());
               tree.setId(makeHashCode(tp, null));
+              tree.setLinkToAllViews(
+                      tp.getTreeCanvas().isApplyToAllViews());
+
               // jms.addTree(tree);
               object.getTree().add(tree);
             }
@@ -1217,6 +1283,24 @@ public class Jalview2XML
       }
     }
 
+    /*
+     * save PCA viewers
+     */
+    if (!storeDS && Desktop.desktop != null)
+    {
+      for (JInternalFrame frame : Desktop.desktop.getAllFrames())
+      {
+        if (frame instanceof PCAPanel)
+        {
+          PCAPanel panel = (PCAPanel) frame;
+          if (panel.getAlignViewport().getAlignment() == jal)
+          {
+            savePCA(panel, object);
+          }
+        }
+      }
+    }
+
     // SAVE ANNOTATIONS
     /**
      * store forward refs from an annotationRow to any groups
@@ -1321,7 +1405,7 @@ public class Jalview2XML
         }
       }
 
-      //jms.setJGroup(groups);
+      // jms.setJGroup(groups);
       Object group;
       for (JGroup grp : groups)
       {
@@ -1435,11 +1519,14 @@ public class Jalview2XML
       view.setFollowHighlight(av.isFollowHighlight());
       view.setFollowSelection(av.followSelection);
       view.setIgnoreGapsinConsensus(av.isIgnoreGapsConsensus());
+      view.setShowComplementFeatures(av.isShowComplementFeatures());
+      view.setShowComplementFeaturesOnTop(
+              av.isShowComplementFeaturesOnTop());
       if (av.getFeaturesDisplayed() != null)
       {
         FeatureSettings fs = new FeatureSettings();
 
-        FeatureRenderer fr = ap.getSeqPanel().seqCanvas
+        FeatureRendererModel fr = ap.getSeqPanel().seqCanvas
                 .getFeatureRenderer();
         String[] renderOrder = fr.getRenderOrder().toArray(new String[0]);
 
@@ -1595,6 +1682,7 @@ public class Jalview2XML
       // using save and then load
       try
       {
+        fileName = fileName.replace('\\', '/');
         System.out.println("Writing jar entry " + fileName);
         JarEntry entry = new JarEntry(fileName);
         jout.putNextEntry(entry);
@@ -1624,6 +1712,196 @@ public class Jalview2XML
   }
 
   /**
+   * Writes PCA viewer attributes and computed values to an XML model object and
+   * adds it to the JalviewModel. Any exceptions are reported by logging.
+   */
+  protected void savePCA(PCAPanel panel, JalviewModel object)
+  {
+    try
+    {
+      PcaViewer viewer = new PcaViewer();
+      viewer.setHeight(panel.getHeight());
+      viewer.setWidth(panel.getWidth());
+      viewer.setXpos(panel.getX());
+      viewer.setYpos(panel.getY());
+      viewer.setTitle(panel.getTitle());
+      PCAModel pcaModel = panel.getPcaModel();
+      viewer.setScoreModelName(pcaModel.getScoreModelName());
+      viewer.setXDim(panel.getSelectedDimensionIndex(X));
+      viewer.setYDim(panel.getSelectedDimensionIndex(Y));
+      viewer.setZDim(panel.getSelectedDimensionIndex(Z));
+      viewer.setBgColour(
+              panel.getRotatableCanvas().getBackgroundColour().getRGB());
+      viewer.setScaleFactor(panel.getRotatableCanvas().getScaleFactor());
+      float[] spMin = panel.getRotatableCanvas().getSeqMin();
+      SeqPointMin spmin = new SeqPointMin();
+      spmin.setXPos(spMin[0]);
+      spmin.setYPos(spMin[1]);
+      spmin.setZPos(spMin[2]);
+      viewer.setSeqPointMin(spmin);
+      float[] spMax = panel.getRotatableCanvas().getSeqMax();
+      SeqPointMax spmax = new SeqPointMax();
+      spmax.setXPos(spMax[0]);
+      spmax.setYPos(spMax[1]);
+      spmax.setZPos(spMax[2]);
+      viewer.setSeqPointMax(spmax);
+      viewer.setShowLabels(panel.getRotatableCanvas().isShowLabels());
+      viewer.setLinkToAllViews(
+              panel.getRotatableCanvas().isApplyToAllViews());
+      SimilarityParamsI sp = pcaModel.getSimilarityParameters();
+      viewer.setIncludeGaps(sp.includeGaps());
+      viewer.setMatchGaps(sp.matchGaps());
+      viewer.setIncludeGappedColumns(sp.includeGappedColumns());
+      viewer.setDenominateByShortestLength(sp.denominateByShortestLength());
+
+      /*
+       * sequence points on display
+       */
+      for (jalview.datamodel.SequencePoint spt : pcaModel
+              .getSequencePoints())
+      {
+        SequencePoint point = new SequencePoint();
+        point.setSequenceRef(seqHash(spt.getSequence()));
+        point.setXPos(spt.coord.x);
+        point.setYPos(spt.coord.y);
+        point.setZPos(spt.coord.z);
+        viewer.getSequencePoint().add(point);
+      }
+
+      /*
+       * (end points of) axes on display
+       */
+      for (Point p : panel.getRotatableCanvas().getAxisEndPoints())
+      {
+
+        Axis axis = new Axis();
+        axis.setXPos(p.x);
+        axis.setYPos(p.y);
+        axis.setZPos(p.z);
+        viewer.getAxis().add(axis);
+      }
+
+      /*
+       * raw PCA data (note we are not restoring PCA inputs here -
+       * alignment view, score model, similarity parameters)
+       */
+      PcaDataType data = new PcaDataType();
+      viewer.setPcaData(data);
+      PCA pca = pcaModel.getPcaData();
+
+      DoubleMatrix pm = new DoubleMatrix();
+      saveDoubleMatrix(pca.getPairwiseScores(), pm);
+      data.setPairwiseMatrix(pm);
+
+      DoubleMatrix tm = new DoubleMatrix();
+      saveDoubleMatrix(pca.getTridiagonal(), tm);
+      data.setTridiagonalMatrix(tm);
+
+      DoubleMatrix eigenMatrix = new DoubleMatrix();
+      data.setEigenMatrix(eigenMatrix);
+      saveDoubleMatrix(pca.getEigenmatrix(), eigenMatrix);
+
+      object.getPcaViewer().add(viewer);
+    } catch (Throwable t)
+    {
+      Cache.log.error("Error saving PCA: " + t.getMessage());
+    }
+  }
+
+  /**
+   * Stores values from a matrix into an XML element, including (if present) the
+   * D or E vectors
+   * 
+   * @param m
+   * @param xmlMatrix
+   * @see #loadDoubleMatrix(DoubleMatrix)
+   */
+  protected void saveDoubleMatrix(MatrixI m, DoubleMatrix xmlMatrix)
+  {
+    xmlMatrix.setRows(m.height());
+    xmlMatrix.setColumns(m.width());
+    for (int i = 0; i < m.height(); i++)
+    {
+      DoubleVector row = new DoubleVector();
+      for (int j = 0; j < m.width(); j++)
+      {
+        row.getV().add(m.getValue(i, j));
+      }
+      xmlMatrix.getRow().add(row);
+    }
+    if (m.getD() != null)
+    {
+      DoubleVector dVector = new DoubleVector();
+      for (double d : m.getD())
+      {
+        dVector.getV().add(d);
+      }
+      xmlMatrix.setD(dVector);
+    }
+    if (m.getE() != null)
+    {
+      DoubleVector eVector = new DoubleVector();
+      for (double e : m.getE())
+      {
+        eVector.getV().add(e);
+      }
+      xmlMatrix.setE(eVector);
+    }
+  }
+
+  /**
+   * Loads XML matrix data into a new Matrix object, including the D and/or E
+   * vectors (if present)
+   * 
+   * @param mData
+   * @return
+   * @see Jalview2XML#saveDoubleMatrix(MatrixI, DoubleMatrix)
+   */
+  protected MatrixI loadDoubleMatrix(DoubleMatrix mData)
+  {
+    int rows = mData.getRows();
+    double[][] vals = new double[rows][];
+
+    for (int i = 0; i < rows; i++)
+    {
+      List<Double> dVector = mData.getRow().get(i).getV();
+      vals[i] = new double[dVector.size()];
+      int dvi = 0;
+      for (Double d : dVector)
+      {
+        vals[i][dvi++] = d;
+      }
+    }
+
+    MatrixI m = new Matrix(vals);
+
+    if (mData.getD() != null)
+    {
+      List<Double> dVector = mData.getD().getV();
+      double[] vec = new double[dVector.size()];
+      int dvi = 0;
+      for (Double d : dVector)
+      {
+        vec[dvi++] = d;
+      }
+      m.setD(vec);
+    }
+    if (mData.getE() != null)
+    {
+      List<Double> dVector = mData.getE().getV();
+      double[] vec = new double[dVector.size()];
+      int dvi = 0;
+      for (Double d : dVector)
+      {
+        vec[dvi++] = d;
+      }
+      m.setE(vec);
+    }
+
+    return m;
+  }
+
+  /**
    * Save any Varna viewers linked to this sequence. Writes an rnaViewer element
    * for each viewer, with
    * <ul>
@@ -1705,7 +1983,7 @@ public class Jalview2XML
 
                 String varnaStateFile = varna.getStateInfo(model.rna);
                 jarEntryName = RNA_PREFIX + viewId + "_" + nextCounter();
-                copyFileToJar(jout, varnaStateFile, jarEntryName);
+                copyFileToJar(jout, varnaStateFile, jarEntryName, "Varna");
                 rnaSessions.put(model, jarEntryName);
               }
               SecondaryStructure ss = new SecondaryStructure();
@@ -1729,58 +2007,48 @@ public class Jalview2XML
    * @param jout
    * @param infilePath
    * @param jarEntryName
+   * @param msg
+   *          additional identifying info to log to the console
    */
   protected void copyFileToJar(JarOutputStream jout, String infilePath,
-          String jarEntryName)
+          String jarEntryName, String msg)
   {
-    DataInputStream dis = null;
-    try
+    try (InputStream is = new FileInputStream(infilePath))
     {
       File file = new File(infilePath);
       if (file.exists() && jout != null)
       {
-        dis = new DataInputStream(new FileInputStream(file));
-        byte[] data = new byte[(int) file.length()];
-        dis.readFully(data);
-        writeJarEntry(jout, jarEntryName, data);
+        System.out.println(
+                "Writing jar entry " + jarEntryName + " (" + msg + ")");
+        jout.putNextEntry(new JarEntry(jarEntryName));
+        copyAll(is, jout);
+        jout.closeEntry();
+        // dis = new DataInputStream(new FileInputStream(file));
+        // byte[] data = new byte[(int) file.length()];
+        // dis.readFully(data);
+        // writeJarEntry(jout, jarEntryName, data);
       }
     } catch (Exception ex)
     {
       ex.printStackTrace();
-    } finally
-    {
-      if (dis != null)
-      {
-        try
-        {
-          dis.close();
-        } catch (IOException e)
-        {
-          // ignore
-        }
-      }
     }
   }
 
   /**
-   * Write the data to a new entry of given name in the output jar file
+   * Copies input to output, in 4K buffers; handles any data (text or binary)
    * 
-   * @param jout
-   * @param jarEntryName
-   * @param data
+   * @param in
+   * @param out
    * @throws IOException
    */
-  protected void writeJarEntry(JarOutputStream jout, String jarEntryName,
-          byte[] data) throws IOException
+  protected void copyAll(InputStream in, OutputStream out)
+          throws IOException
   {
-    if (jout != null)
+    byte[] buffer = new byte[4096];
+    int bytesRead = 0;
+    while ((bytesRead = in.read(buffer)) != -1)
     {
-      System.out.println("Writing jar entry " + jarEntryName);
-      jout.putNextEntry(new JarEntry(jarEntryName));
-      DataOutputStream dout = new DataOutputStream(jout);
-      dout.write(data, 0, data.length);
-      dout.flush();
-      jout.closeEntry();
+      out.write(buffer, 0, bytesRead);
     }
   }
 
@@ -1797,7 +2065,7 @@ public class Jalview2XML
    * @param viewFrame
    * @return
    */
-  protected String saveStructureState(AlignmentPanel ap, SequenceI jds,
+  protected String saveStructureViewer(AlignmentPanel ap, SequenceI jds,
           Pdbids pdb, PDBEntry entry, List<String> viewIds,
           String matchedFile, StructureViewerBase viewFrame)
   {
@@ -1851,7 +2119,7 @@ public class Jalview2XML
           final String viewId = viewFrame.getViewId();
           state.setViewId(viewId);
           state.setAlignwithAlignPanel(viewFrame.isUsedforaligment(ap));
-          state.setColourwithAlignPanel(viewFrame.isUsedforcolourby(ap));
+          state.setColourwithAlignPanel(viewFrame.isUsedForColourBy(ap));
           state.setColourByJmol(viewFrame.isColouredByViewer());
           state.setType(viewFrame.getViewerType().toString());
           // pdb.addStructureState(state);
@@ -2235,7 +2503,7 @@ public class Jalview2XML
     vamsasSeq.setName(jds.getName());
     vamsasSeq.setSequence(jds.getSequenceAsString());
     vamsasSeq.setDescription(jds.getDescription());
-    jalview.datamodel.DBRefEntry[] dbrefs = null;
+    List<DBRefEntry> dbrefs = null;
     if (jds.getDatasetSequence() != null)
     {
       vamsasSeq.setDsseqid(seqHash(jds.getDatasetSequence()));
@@ -2251,21 +2519,29 @@ public class Jalview2XML
         parentseq = jds;
       }
     }
+
+    /*
+     * save any dbrefs; special subclass GeneLocus is flagged as 'locus'
+     */
     if (dbrefs != null)
     {
-      for (int d = 0; d < dbrefs.length; d++)
+      for (int d = 0, nd = dbrefs.size(); d < nd; d++)
       {
         DBRef dbref = new DBRef();
-        dbref.setSource(dbrefs[d].getSource());
-        dbref.setVersion(dbrefs[d].getVersion());
-        dbref.setAccessionId(dbrefs[d].getAccessionId());
-        if (dbrefs[d].hasMap())
+        DBRefEntry ref = dbrefs.get(d);
+        dbref.setSource(ref.getSource());
+        dbref.setVersion(ref.getVersion());
+        dbref.setAccessionId(ref.getAccessionId());
+        if (ref instanceof GeneLocus)
         {
-          Mapping mp = createVamsasMapping(dbrefs[d].getMap(), parentseq,
+          dbref.setLocus(true);
+        }
+        if (ref.hasMap())
+        {
+          Mapping mp = createVamsasMapping(ref.getMap(), parentseq,
                   jds, recurse);
           dbref.setMapping(mp);
         }
-        // vamsasSeq.addDBRef(dbref);
         vamsasSeq.getDBRef().add(dbref);
       }
     }
@@ -2457,7 +2733,7 @@ public class Jalview2XML
    * @param file
    *          - HTTP URL or filename
    */
-  public AlignFrame loadJalviewAlign(final String file)
+  public AlignFrame loadJalviewAlign(final Object file)
   {
 
     jalview.gui.AlignFrame af = null;
@@ -2491,7 +2767,7 @@ public class Jalview2XML
           public void run()
           {
             setLoadingFinishedForNewStructureViewers();
-          };
+          }
         });
       } catch (Exception x)
       {
@@ -2501,17 +2777,22 @@ public class Jalview2XML
     return af;
   }
 
-  private jarInputStreamProvider createjarInputStreamProvider(
-          final String file) throws MalformedURLException
-  {
-    URL url = null;
-    errorMessage = null;
-    uniqueSetSuffix = null;
-    seqRefIds = null;
-    viewportsAdded.clear();
-    frefedSequence = null;
+       @SuppressWarnings("unused")
+       private jarInputStreamProvider createjarInputStreamProvider(final Object ofile) throws MalformedURLException {
 
-    if (file.startsWith("http://"))
+               // BH 2018 allow for bytes already attached to File object
+               try {
+                       String file = (ofile instanceof File ? ((File) ofile).getCanonicalPath() : ofile.toString());
+      byte[] bytes = Platform.isJS() ? Platform.getFileBytes((File) ofile)
+              : null;
+                       URL url = null;
+                       errorMessage = null;
+                       uniqueSetSuffix = null;
+                       seqRefIds = null;
+                       viewportsAdded.clear();
+                       frefedSequence = null;
+
+    if (HttpUtils.startsWithHttpOrHttps(file))
     {
       url = new URL(file);
     }
@@ -2519,26 +2800,31 @@ public class Jalview2XML
     return new jarInputStreamProvider()
     {
 
-      @Override
-      public JarInputStream getJarInputStream() throws IOException
-      {
-        if (_url != null)
-        {
-          return new JarInputStream(_url.openStream());
-        }
-        else
-        {
-          return new JarInputStream(new FileInputStream(file));
-        }
-      }
-
-      @Override
-      public String getFilename()
-      {
-        return file;
-      }
-    };
-  }
+                               @Override
+                               public JarInputStream getJarInputStream() throws IOException {
+                                       if (bytes != null) {
+//                                             System.out.println("Jalview2XML: opening byte jarInputStream for bytes.length=" + bytes.length);
+                                               return new JarInputStream(new ByteArrayInputStream(bytes));
+                                       }
+                                       if (_url != null) {
+//                                             System.out.println("Jalview2XML: opening url jarInputStream for " + _url);
+                                               return new JarInputStream(_url.openStream());
+                                       } else {
+//                                             System.out.println("Jalview2XML: opening file jarInputStream for " + file);
+                                               return new JarInputStream(new FileInputStream(file));
+                                       }
+                               }
+
+                               @Override
+                               public String getFilename() {
+                                       return file;
+                               }
+                       };
+               } catch (IOException e) {
+                       e.printStackTrace();
+                       return null;
+               }
+       }
 
   /**
    * Recover jalview session from a jalview project archive. Caller may
@@ -2580,9 +2866,6 @@ public class Jalview2XML
 
         if (jarentry != null && jarentry.getName().endsWith(".xml"))
         {
-          InputStreamReader in = new InputStreamReader(jin, UTF_8);
-          // JalviewModel object = new JalviewModel();
-
           JAXBContext jc = JAXBContext
                   .newInstance("jalview.xml.binding.jalview");
           XMLStreamReader streamReader = XMLInputFactory.newInstance()
@@ -2592,11 +2875,6 @@ public class Jalview2XML
                   .unmarshal(streamReader, JalviewModel.class);
           JalviewModel object = jbe.getValue();
 
-          /*
-          Unmarshaller unmar = new Unmarshaller(object);
-          unmar.setValidation(false);
-          object = (JalviewModel) unmar.unmarshal(in);
-          */
           if (true) // !skipViewport(object))
           {
             _af = loadFromObject(object, file, true, jprovider);
@@ -2643,16 +2921,7 @@ public class Jalview2XML
       ex.printStackTrace(System.err);
       if (attemptversion1parse)
       {
-        // Is Version 1 Jar file?
-        try
-        {
-          af = new Jalview2XML_V1(raiseGUI).LoadJalviewAlign(jprovider);
-        } catch (Exception ex2)
-        {
-          System.err.println("Exception whilst loading as jalviewXMLV1:");
-          ex2.printStackTrace();
-          af = null;
-        }
+        // used to attempt to parse as V1 castor-generated xml
       }
       if (Desktop.instance != null)
       {
@@ -2905,53 +3174,42 @@ public class Jalview2XML
    * @param prefix
    *          a prefix for the temporary file name, must be at least three
    *          characters long
-   * @param origFile
+   * @param suffixModel
    *          null or original file - so new file can be given the same suffix
    *          as the old one
    * @return
    */
   protected String copyJarEntry(jarInputStreamProvider jprovider,
-          String jarEntryName, String prefix, String origFile)
+          String jarEntryName, String prefix, String suffixModel)
   {
-    BufferedReader in = null;
-    PrintWriter out = null;
     String suffix = ".tmp";
-    if (origFile == null)
+    if (suffixModel == null)
     {
-      origFile = jarEntryName;
+      suffixModel = jarEntryName;
     }
-    int sfpos = origFile.lastIndexOf(".");
-    if (sfpos > -1 && sfpos < (origFile.length() - 3))
+    int sfpos = suffixModel.lastIndexOf(".");
+    if (sfpos > -1 && sfpos < (suffixModel.length() - 1))
     {
-      suffix = "." + origFile.substring(sfpos + 1);
+      suffix = "." + suffixModel.substring(sfpos + 1);
     }
-    try
-    {
-      JarInputStream jin = jprovider.getJarInputStream();
-      /*
-       * if (jprovider.startsWith("http://")) { jin = new JarInputStream(new
-       * URL(jprovider).openStream()); } else { jin = new JarInputStream(new
-       * FileInputStream(jprovider)); }
-       */
 
+    try (JarInputStream jin = jprovider.getJarInputStream())
+    {
       JarEntry entry = null;
       do
       {
         entry = jin.getNextJarEntry();
       } while (entry != null && !entry.getName().equals(jarEntryName));
+
       if (entry != null)
       {
-        in = new BufferedReader(new InputStreamReader(jin, UTF_8));
+        // in = new BufferedReader(new InputStreamReader(jin, UTF_8));
         File outFile = File.createTempFile(prefix, suffix);
         outFile.deleteOnExit();
-        out = new PrintWriter(new FileOutputStream(outFile));
-        String data;
-
-        while ((data = in.readLine()) != null)
+        try (OutputStream os = new FileOutputStream(outFile))
         {
-          out.println(data);
+          copyAll(jin, os);
         }
-        out.flush();
         String t = outFile.getAbsolutePath();
         return t;
       }
@@ -2962,22 +3220,6 @@ public class Jalview2XML
     } catch (Exception ex)
     {
       ex.printStackTrace();
-    } finally
-    {
-      if (in != null)
-      {
-        try
-        {
-          in.close();
-        } catch (IOException e)
-        {
-          // ignore
-        }
-      }
-      if (out != null)
-      {
-        out.close();
-      }
     }
 
     return null;
@@ -3030,6 +3272,28 @@ public class Jalview2XML
             : null;
 
     // ////////////////////////////////
+    // INITIALISE ALIGNMENT SEQUENCESETID AND VIEWID
+    //
+    //
+    // If we just load in the same jar file again, the sequenceSetId
+    // will be the same, and we end up with multiple references
+    // to the same sequenceSet. We must modify this id on load
+    // so that each load of the file gives a unique id
+
+    /**
+     * used to resolve correct alignment dataset for alignments with multiple
+     * views
+     */
+    String uniqueSeqSetId = null;
+    String viewId = null;
+    if (view != null)
+    {
+      uniqueSeqSetId = view.getSequenceSetId() + uniqueSetSuffix;
+      viewId = (view.getId() == null ? null
+              : view.getId() + uniqueSetSuffix);
+    }
+
+    // ////////////////////////////////
     // LOAD SEQUENCES
 
     List<SequenceI> hiddenSeqs = null;
@@ -3056,8 +3320,10 @@ public class Jalview2XML
                   || tmpSeq.getEnd() != jseq.getEnd())
           {
             System.err.println(
-                    "Warning JAL-2154 regression: updating start/end for sequence "
-                            + tmpSeq.toString() + " to " + jseq);
+                    String.format("Warning JAL-2154 regression: updating start/end for sequence %s from %d/%d to %d/%d",
+                            tmpSeq.getName(), tmpSeq.getStart(),
+                            tmpSeq.getEnd(), jseq.getStart(),
+                            jseq.getEnd()));
           }
         }
         else
@@ -3098,8 +3364,6 @@ public class Jalview2XML
       }
 
       if (safeBoolean(jseq.isViewreference()))
-      // if (jseq.isViewreference() != null
-      // && jseq.isViewreference().booleanValue())
       {
         referenceseqForView = tmpseqs.get(tmpseqs.size() - 1);
       }
@@ -3152,7 +3416,7 @@ public class Jalview2XML
 
       // finally, verify all data in vamsasSet is actually present in al
       // passing on flag indicating if it is actually a stored dataset
-      recoverDatasetFor(vamsasSet, al, isdsal);
+      recoverDatasetFor(vamsasSet, al, isdsal, uniqueSeqSetId);
     }
 
     if (referenceseqForView != null)
@@ -3385,8 +3649,7 @@ public class Jalview2XML
           // annotation.setAutoCalculated(true);
           // }
         }
-        if (autoForView || // (annotation.hasAutoCalculated() &&
-                annotation.isAutoCalculated())
+        if (autoForView || annotation.isAutoCalculated())
         {
           // remove ID - we don't recover annotation from other views for
           // view-specific annotation
@@ -3595,7 +3858,7 @@ public class Jalview2XML
           }
           else
           {
-            cs = ColourSchemeProperty.getColourScheme(al,
+            cs = ColourSchemeProperty.getColourScheme(null, al,
                     jGroup.getColour());
           }
         }
@@ -3693,13 +3956,6 @@ public class Jalview2XML
     // ///////////////////////////////
     // LOAD VIEWPORT
 
-    // If we just load in the same jar file again, the sequenceSetId
-    // will be the same, and we end up with multiple references
-    // to the same sequenceSet. We must modify this id on load
-    // so that each load of the file gives a unique id
-    String uniqueSeqSetId = view.getSequenceSetId() + uniqueSetSuffix;
-    String viewId = (view.getId() == null ? null
-            : view.getId() + uniqueSetSuffix);
     AlignFrame af = null;
     AlignViewport av = null;
     // now check to see if we really need to create a new viewport.
@@ -3781,6 +4037,7 @@ public class Jalview2XML
     if (loadTreesAndStructures)
     {
       loadTrees(jalviewModel, view, af, av, ap);
+      loadPCAViewers(jalviewModel, ap);
       loadPDBStructures(jprovider, jseqs, af, ap);
       loadRnaViewers(jprovider, jseqs, ap);
     }
@@ -3929,8 +4186,8 @@ public class Jalview2XML
           // TODO: verify 'associate with all views' works still
           tp.getTreeCanvas().setViewport(av); // af.viewport;
           tp.getTreeCanvas().setAssociatedPanel(ap); // af.alignPanel;
-
         }
+        tp.getTreeCanvas().setApplyToAllViews(tree.isLinkToAllViews());
         if (tp == null)
         {
           warn("There was a problem recovering stored Newick tree: \n"
@@ -4033,10 +4290,15 @@ public class Jalview2XML
             }
             if (!structureViewers.containsKey(sviewid))
             {
+              String viewerType = structureState.getType();
+              if (viewerType == null) // pre Jalview 2.9
+              {
+                viewerType = ViewerType.JMOL.toString();
+              }
               structureViewers.put(sviewid,
                       new StructureViewerModel(x, y, width, height, false,
                               false, true, structureState.getViewId(),
-                              structureState.getType()));
+                              viewerType));
               // Legacy pre-2.7 conversion JAL-823 :
               // do not assume any view has to be linked for colour by
               // sequence
@@ -4137,246 +4399,17 @@ public class Jalview2XML
       return;
     }
 
-    /*
-     * From 2.9: stateData.type contains JMOL or CHIMERA, data is in jar entry
-     * "viewer_"+stateData.viewId
-     */
-    if (ViewerType.CHIMERA.toString().equals(stateData.getType()))
-    {
-      createChimeraViewer(viewerData, af, jprovider);
-    }
-    else
-    {
-      /*
-       * else Jmol (if pre-2.9, stateData contains JMOL state string)
-       */
-      createJmolViewer(viewerData, af, jprovider);
-    }
-  }
-
-  /**
-   * Create a new Chimera viewer.
-   * 
-   * @param data
-   * @param af
-   * @param jprovider
-   */
-  protected void createChimeraViewer(
-          Entry<String, StructureViewerModel> viewerData, AlignFrame af,
-          jarInputStreamProvider jprovider)
-  {
-    StructureViewerModel data = viewerData.getValue();
-    String chimeraSessionFile = data.getStateData();
-
-    /*
-     * Copy Chimera session from jar entry "viewer_"+viewId to a temporary file
-     * 
-     * NB this is the 'saved' viewId as in the project file XML, _not_ the
-     * 'uniquified' sviewid used to reconstruct the viewer here
-     */
-    String viewerJarEntryName = getViewerJarEntryName(data.getViewId());
-    chimeraSessionFile = copyJarEntry(jprovider, viewerJarEntryName,
-            "chimera", null);
-
-    Set<Entry<File, StructureData>> fileData = data.getFileData()
-            .entrySet();
-    List<PDBEntry> pdbs = new ArrayList<>();
-    List<SequenceI[]> allseqs = new ArrayList<>();
-    for (Entry<File, StructureData> pdb : fileData)
-    {
-      String filePath = pdb.getValue().getFilePath();
-      String pdbId = pdb.getValue().getPdbId();
-      // pdbs.add(new PDBEntry(filePath, pdbId));
-      pdbs.add(new PDBEntry(pdbId, null, PDBEntry.Type.PDB, filePath));
-      final List<SequenceI> seqList = pdb.getValue().getSeqList();
-      SequenceI[] seqs = seqList.toArray(new SequenceI[seqList.size()]);
-      allseqs.add(seqs);
-    }
-
-    boolean colourByChimera = data.isColourByViewer();
-    boolean colourBySequence = data.isColourWithAlignPanel();
-
-    // TODO use StructureViewer as a factory here, see JAL-1761
-    final PDBEntry[] pdbArray = pdbs.toArray(new PDBEntry[pdbs.size()]);
-    final SequenceI[][] seqsArray = allseqs
-            .toArray(new SequenceI[allseqs.size()][]);
-    String newViewId = viewerData.getKey();
-
-    ChimeraViewFrame cvf = new ChimeraViewFrame(chimeraSessionFile,
-            af.alignPanel, pdbArray, seqsArray, colourByChimera,
-            colourBySequence, newViewId);
-    cvf.setSize(data.getWidth(), data.getHeight());
-    cvf.setLocation(data.getX(), data.getY());
-  }
-
-  /**
-   * Create a new Jmol window. First parse the Jmol state to translate filenames
-   * loaded into the view, and record the order in which files are shown in the
-   * Jmol view, so we can add the sequence mappings in same order.
-   * 
-   * @param viewerData
-   * @param af
-   * @param jprovider
-   */
-  protected void createJmolViewer(
-          final Entry<String, StructureViewerModel> viewerData,
-          AlignFrame af, jarInputStreamProvider jprovider)
-  {
-    final StructureViewerModel svattrib = viewerData.getValue();
-    String state = svattrib.getStateData();
-
-    /*
-     * Pre-2.9: state element value is the Jmol state string
-     * 
-     * 2.9+: @type is "JMOL", state data is in a Jar file member named "viewer_"
-     * + viewId
-     */
-    if (ViewerType.JMOL.toString().equals(svattrib.getType()))
-    {
-      state = readJarEntry(jprovider,
-              getViewerJarEntryName(svattrib.getViewId()));
-    }
-
-    List<String> pdbfilenames = new ArrayList<>();
-    List<SequenceI[]> seqmaps = new ArrayList<>();
-    List<String> pdbids = new ArrayList<>();
-    StringBuilder newFileLoc = new StringBuilder(64);
-    int cp = 0, ncp, ecp;
-    Map<File, StructureData> oldFiles = svattrib.getFileData();
-    while ((ncp = state.indexOf("load ", cp)) > -1)
-    {
-      do
-      {
-        // look for next filename in load statement
-        newFileLoc.append(state.substring(cp,
-                ncp = (state.indexOf("\"", ncp + 1) + 1)));
-        String oldfilenam = state.substring(ncp,
-                ecp = state.indexOf("\"", ncp));
-        // recover the new mapping data for this old filename
-        // have to normalize filename - since Jmol and jalview do
-        // filename
-        // translation differently.
-        StructureData filedat = oldFiles.get(new File(oldfilenam));
-        if (filedat == null)
-        {
-          String reformatedOldFilename = oldfilenam.replaceAll("/", "\\\\");
-          filedat = oldFiles.get(new File(reformatedOldFilename));
-        }
-        newFileLoc.append(Platform.escapeString(filedat.getFilePath()));
-        pdbfilenames.add(filedat.getFilePath());
-        pdbids.add(filedat.getPdbId());
-        seqmaps.add(filedat.getSeqList().toArray(new SequenceI[0]));
-        newFileLoc.append("\"");
-        cp = ecp + 1; // advance beyond last \" and set cursor so we can
-                      // look for next file statement.
-      } while ((ncp = state.indexOf("/*file*/", cp)) > -1);
-    }
-    if (cp > 0)
-    {
-      // just append rest of state
-      newFileLoc.append(state.substring(cp));
-    }
-    else
-    {
-      System.err.print("Ignoring incomplete Jmol state for PDB ids: ");
-      newFileLoc = new StringBuilder(state);
-      newFileLoc.append("; load append ");
-      for (File id : oldFiles.keySet())
-      {
-        // add this and any other pdb files that should be present in
-        // the viewer
-        StructureData filedat = oldFiles.get(id);
-        newFileLoc.append(filedat.getFilePath());
-        pdbfilenames.add(filedat.getFilePath());
-        pdbids.add(filedat.getPdbId());
-        seqmaps.add(filedat.getSeqList().toArray(new SequenceI[0]));
-        newFileLoc.append(" \"");
-        newFileLoc.append(filedat.getFilePath());
-        newFileLoc.append("\"");
-
-      }
-      newFileLoc.append(";");
-    }
-
-    if (newFileLoc.length() == 0)
-    {
-      return;
-    }
-    int histbug = newFileLoc.indexOf("history = ");
-    if (histbug > -1)
-    {
-      /*
-       * change "history = [true|false];" to "history = [1|0];"
-       */
-      histbug += 10;
-      int diff = histbug == -1 ? -1 : newFileLoc.indexOf(";", histbug);
-      String val = (diff == -1) ? null
-              : newFileLoc.substring(histbug, diff);
-      if (val != null && val.length() >= 4)
-      {
-        if (val.contains("e")) // eh? what can it be?
-        {
-          if (val.trim().equals("true"))
-          {
-            val = "1";
-          }
-          else
-          {
-            val = "0";
-          }
-          newFileLoc.replace(histbug, diff, val);
-        }
-      }
-    }
-
-    final String[] pdbf = pdbfilenames
-            .toArray(new String[pdbfilenames.size()]);
-    final String[] id = pdbids.toArray(new String[pdbids.size()]);
-    final SequenceI[][] sq = seqmaps
-            .toArray(new SequenceI[seqmaps.size()][]);
-    final String fileloc = newFileLoc.toString();
-    final String sviewid = viewerData.getKey();
-    final AlignFrame alf = af;
-    final Rectangle rect = new Rectangle(svattrib.getX(), svattrib.getY(),
-            svattrib.getWidth(), svattrib.getHeight());
+    String type = stateData.getType();
     try
     {
-      javax.swing.SwingUtilities.invokeAndWait(new Runnable()
-      {
-        @Override
-        public void run()
-        {
-          JalviewStructureDisplayI sview = null;
-          try
-          {
-            sview = new StructureViewer(
-                    alf.alignPanel.getStructureSelectionManager())
-                            .createView(StructureViewer.ViewerType.JMOL,
-                                    pdbf, id, sq, alf.alignPanel, svattrib,
-                                    fileloc, rect, sviewid);
-            addNewStructureViewer(sview);
-          } catch (OutOfMemoryError ex)
-          {
-            new OOMWarning("restoring structure view for PDB id " + id,
-                    (OutOfMemoryError) ex.getCause());
-            if (sview != null && sview.isVisible())
-            {
-              sview.closeViewer(false);
-              sview.setVisible(false);
-              sview.dispose();
-            }
-          }
-        }
-      });
-    } catch (InvocationTargetException ex)
-    {
-      warn("Unexpected error when opening Jmol view.", ex);
-
-    } catch (InterruptedException e)
+      ViewerType viewerType = ViewerType.valueOf(type);
+      createStructureViewer(viewerType, viewerData, af, jprovider);
+    } catch (IllegalArgumentException | NullPointerException e)
     {
-      // e.printStackTrace();
+      // TODO JAL-3619 show error dialog / offer an alternative viewer
+      Cache.log.error(
+              "Invalid structure viewer type: " + type);
     }
-
   }
 
   /**
@@ -4581,7 +4614,18 @@ public class Jalview2XML
   {
     AlignFrame af = null;
     af = new AlignFrame(al, safeInt(view.getWidth()),
-            safeInt(view.getHeight()), uniqueSeqSetId, viewId);
+            safeInt(view.getHeight()), uniqueSeqSetId, viewId) 
+//    {
+//     
+//     @Override
+//     protected void processKeyEvent(java.awt.event.KeyEvent e) {
+//             System.out.println("Jalview2XML   AF " + e);
+//             super.processKeyEvent(e);
+//             
+//     }
+//     
+//    }
+    ;
 
     af.setFileName(file, FileFormat.Jalview);
 
@@ -4713,25 +4757,27 @@ public class Jalview2XML
       }
       else
       {
-        cs = ColourSchemeProperty.getColourScheme(al, view.getBgColour());
+        cs = ColourSchemeProperty.getColourScheme(af.getViewport(), al,
+                view.getBgColour());
       }
     }
 
+    /*
+     * turn off 'alignment colour applies to all groups'
+     * while restoring global colour scheme
+     */
+    viewport.setColourAppliesToAllGroups(false);
     viewport.setGlobalColourScheme(cs);
     viewport.getResidueShading().setThreshold(pidThreshold,
             view.isIgnoreGapsinConsensus());
     viewport.getResidueShading()
             .setConsensus(viewport.getSequenceConsensusHash());
-    viewport.setColourAppliesToAllGroups(false);
-
     if (safeBoolean(view.isConservationSelected()) && cs != null)
     {
       viewport.getResidueShading()
               .setConservationInc(safeInt(view.getConsThreshold()));
     }
-
     af.changeColour(cs);
-
     viewport.setColourAppliesToAllGroups(true);
 
     viewport
@@ -4749,11 +4795,14 @@ public class Jalview2XML
     viewport.setShowNPFeats(safeBoolean(view.isShowNPfeatureTooltip()));
     viewport.setShowGroupConsensus(view.isShowGroupConsensus());
     viewport.setShowGroupConservation(view.isShowGroupConservation());
+    viewport.setShowComplementFeatures(view.isShowComplementFeatures());
+    viewport.setShowComplementFeaturesOnTop(
+            view.isShowComplementFeaturesOnTop());
 
     // recover feature settings
     if (jm.getFeatureSettings() != null)
     {
-      FeatureRenderer fr = af.alignPanel.getSeqPanel().seqCanvas
+      FeatureRendererModel fr = af.alignPanel.getSeqPanel().seqCanvas
               .getFeatureRenderer();
       FeaturesDisplayed fdi;
       viewport.setFeaturesDisplayed(fdi = new FeaturesDisplayed());
@@ -4807,7 +4856,8 @@ public class Jalview2XML
           float min = safeFloat(safeFloat(setting.getMin()));
           float max = setting.getMax() == null ? 1f
                   : setting.getMax().floatValue();
-          FeatureColourI gc = new FeatureColour(minColour, maxColour,
+          FeatureColourI gc = new FeatureColour(maxColour, minColour,
+                  maxColour,
                   noValueColour, min, max);
           if (setting.getAttributeName().size() > 0)
           {
@@ -4852,7 +4902,7 @@ public class Jalview2XML
         }
         else
         {
-          featureOrder.put(featureType, new Float(
+          featureOrder.put(featureType, Float.valueOf(
                   fs / jm.getFeatureSettings().getSetting().size()));
         }
         if (safeBoolean(setting.isDisplay()))
@@ -4864,7 +4914,7 @@ public class Jalview2XML
       for (int gs = 0; gs < jm.getFeatureSettings().getGroup().size(); gs++)
       {
         Group grp = jm.getFeatureSettings().getGroup().get(gs);
-        fgtable.put(grp.getName(), new Boolean(grp.isDisplay()));
+        fgtable.put(grp.getName(), Boolean.valueOf(grp.isDisplay()));
       }
       // FeatureRendererSettings frs = new FeatureRendererSettings(renderOrder,
       // fgtable, featureColours, jms.getFeatureSettings().hasTransparency() ?
@@ -5009,7 +5059,7 @@ public class Jalview2XML
     else
     {
       cs = new AnnotationColourGradient(matchedAnnotation,
-              ColourSchemeProperty.getColourScheme(al,
+              ColourSchemeProperty.getColourScheme(af.getViewport(), al,
                       viewAnnColour.getColourScheme()),
               safeInt(viewAnnColour.getAboveThreshold()));
     }
@@ -5198,15 +5248,53 @@ public class Jalview2XML
   }
 
   private void recoverDatasetFor(SequenceSet vamsasSet, AlignmentI al,
-          boolean ignoreUnrefed)
+          boolean ignoreUnrefed, String uniqueSeqSetId)
   {
     jalview.datamodel.AlignmentI ds = getDatasetFor(
             vamsasSet.getDatasetId());
-    Vector dseqs = null;
+    AlignmentI xtant_ds = ds;
+    if (xtant_ds == null)
+    {
+      // good chance we are about to create a new dataset, but check if we've
+      // seen some of the dataset sequence IDs before.
+      // TODO: skip this check if we are working with project generated by
+      // version 2.11 or later
+      xtant_ds = checkIfHasDataset(vamsasSet.getSequence());
+      if (xtant_ds != null)
+      {
+        ds = xtant_ds;
+        addDatasetRef(vamsasSet.getDatasetId(), ds);
+      }
+    }
+    Vector<SequenceI> dseqs = null;
+    if (!ignoreUnrefed)
+    {
+      // recovering an alignment View
+      AlignmentI seqSetDS = getDatasetFor(UNIQSEQSETID + uniqueSeqSetId);
+      if (seqSetDS != null)
+      {
+        if (ds != null && ds != seqSetDS)
+        {
+          warn("JAL-3171 regression: Overwriting a dataset reference for an alignment"
+                  + " - CDS/Protein crossreference data may be lost");
+          if (xtant_ds != null)
+          {
+            // This can only happen if the unique sequence set ID was bound to a
+            // dataset that did not contain any of the sequences in the view
+            // currently being restored.
+            warn("JAL-3171 SERIOUS!  TOTAL CONFUSION - please consider contacting the Jalview Development team so they can investigate why your project caused this message to be displayed.");
+          }
+        }
+        ds = seqSetDS;
+        addDatasetRef(vamsasSet.getDatasetId(), ds);
+      }
+    }
     if (ds == null)
     {
+      // try even harder to restore dataset
+      AlignmentI xtantDS = checkIfHasDataset(vamsasSet.getSequence());
       // create a list of new dataset sequences
-      dseqs = new Vector();
+      dseqs = new Vector<>();
     }
     for (int i = 0, iSize = vamsasSet.getSequence().size(); i < iSize; i++)
     {
@@ -5227,10 +5315,58 @@ public class Jalview2XML
     if (al.getDataset() == null && !ignoreUnrefed)
     {
       al.setDataset(ds);
+      // register dataset for the alignment's uniqueSeqSetId for legacy projects
+      addDatasetRef(UNIQSEQSETID + uniqueSeqSetId, ds);
     }
+    updateSeqDatasetBinding(vamsasSet.getSequence(), ds);
   }
 
   /**
+   * XML dataset sequence ID to materialised dataset reference
+   */
+  HashMap<String, AlignmentI> seqToDataset = new HashMap<>();
+
+  /**
+   * @return the first materialised dataset reference containing a dataset
+   *         sequence referenced in the given view
+   * @param list
+   *          - sequences from the view
+   */
+  AlignmentI checkIfHasDataset(List<Sequence> list)
+  {
+    for (Sequence restoredSeq : list)
+    {
+      AlignmentI datasetFor = seqToDataset.get(restoredSeq.getDsseqid());
+      if (datasetFor != null)
+      {
+        return datasetFor;
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Register ds as the containing dataset for the dataset sequences referenced
+   * by sequences in list
+   * 
+   * @param list
+   *          - sequences in a view
+   * @param ds
+   */
+  void updateSeqDatasetBinding(List<Sequence> list, AlignmentI ds)
+  {
+    for (Sequence restoredSeq : list)
+    {
+      AlignmentI prevDS = seqToDataset.put(restoredSeq.getDsseqid(), ds);
+      if (prevDS != null && prevDS != ds)
+      {
+        warn("Dataset sequence appears in many datasets: "
+                + restoredSeq.getDsseqid());
+        // TODO: try to merge!
+      }
+    }
+  }
+  /**
    * 
    * @param vamsasSeq
    *          sequence definition to create/merge dataset sequence for
@@ -5246,7 +5382,8 @@ public class Jalview2XML
    *          vamsasSeq array ordering, to preserve ordering of dataset
    */
   private void ensureJalviewDatasetSequence(Sequence vamsasSeq,
-          AlignmentI ds, Vector dseqs, boolean ignoreUnrefed, int vseqpos)
+          AlignmentI ds, Vector<SequenceI> dseqs, boolean ignoreUnrefed,
+          int vseqpos)
   {
     // JBP TODO: Check this is called for AlCodonFrames to support recovery of
     // xRef Codon Maps
@@ -5451,13 +5588,29 @@ public class Jalview2XML
     return datasetId;
   }
 
+  /**
+   * Add any saved DBRefEntry's to the sequence. An entry flagged as 'locus' is
+   * constructed as a special subclass GeneLocus.
+   * 
+   * @param datasetSequence
+   * @param sequence
+   */
   private void addDBRefs(SequenceI datasetSequence, Sequence sequence)
   {
     for (int d = 0; d < sequence.getDBRef().size(); d++)
     {
       DBRef dr = sequence.getDBRef().get(d);
-      jalview.datamodel.DBRefEntry entry = new jalview.datamodel.DBRefEntry(
-              dr.getSource(), dr.getVersion(), dr.getAccessionId());
+      DBRefEntry entry;
+      if (dr.isLocus())
+      {
+        entry = new GeneLocus(dr.getSource(), dr.getVersion(),
+                dr.getAccessionId());
+      }
+      else
+      {
+        entry = new DBRefEntry(dr.getSource(), dr.getVersion(),
+                dr.getAccessionId());
+      }
       if (dr.getMapping() != null)
       {
         entry.setMap(addMapping(dr.getMapping()));
@@ -5489,15 +5642,16 @@ public class Jalview2XML
     jalview.datamodel.Mapping jmap = new jalview.datamodel.Mapping(dsto, fr,
             fto, m.getMapFromUnit().intValue(),
             m.getMapToUnit().intValue());
-    // if (m.getMappingChoice() != null)
-    // {
-    // MappingChoice mc = m.getMappingChoice();
+
+    /*
+     * (optional) choice of dseqFor or Sequence
+     */
     if (m.getDseqFor() != null)
     {
       String dsfor = m.getDseqFor();
       if (seqRefIds.containsKey(dsfor))
       {
-        /**
+        /*
          * recover from hash
          */
         jmap.setTo(seqRefIds.get(dsfor));
@@ -5507,9 +5661,9 @@ public class Jalview2XML
         frefedSequence.add(newMappingRef(dsfor, jmap));
       }
     }
-    else
+    else if (m.getSequence() != null)
     {
-      /**
+      /*
        * local sequence definition
        */
       Sequence ms = m.getSequence();
@@ -5566,6 +5720,10 @@ public class Jalview2XML
     initSeqRefs();
     JalviewModel jm = saveState(ap, null, null, null);
 
+    addDatasetRef(
+            jm.getVamsasModel().getSequenceSet().get(0).getDatasetId(),
+            ap.getAlignment().getDataset());
+
     uniqueSetSuffix = "";
     // jm.getJalviewModelSequence().getViewport(0).setId(null);
     jm.getViewport().get(0).setId(null);
@@ -5827,6 +5985,299 @@ public class Jalview2XML
   }
 
   /**
+   * Loads any saved PCA viewers
+   * 
+   * @param jms
+   * @param ap
+   */
+  protected void loadPCAViewers(JalviewModel model, AlignmentPanel ap)
+  {
+    try
+    {
+      List<PcaViewer> pcaviewers = model.getPcaViewer();
+      for (PcaViewer viewer : pcaviewers)
+      {
+        String modelName = viewer.getScoreModelName();
+        SimilarityParamsI params = new SimilarityParams(
+                viewer.isIncludeGappedColumns(), viewer.isMatchGaps(),
+                viewer.isIncludeGaps(),
+                viewer.isDenominateByShortestLength());
+
+        /*
+         * create the panel (without computing the PCA)
+         */
+        PCAPanel panel = new PCAPanel(ap, modelName, params);
+
+        panel.setTitle(viewer.getTitle());
+        panel.setBounds(new Rectangle(viewer.getXpos(), viewer.getYpos(),
+                viewer.getWidth(), viewer.getHeight()));
+
+        boolean showLabels = viewer.isShowLabels();
+        panel.setShowLabels(showLabels);
+        panel.getRotatableCanvas().setShowLabels(showLabels);
+        panel.getRotatableCanvas()
+                .setBgColour(new Color(viewer.getBgColour()));
+        panel.getRotatableCanvas()
+                .setApplyToAllViews(viewer.isLinkToAllViews());
+
+        /*
+         * load PCA output data
+         */
+        ScoreModelI scoreModel = ScoreModels.getInstance()
+                .getScoreModel(modelName, ap);
+        PCA pca = new PCA(null, scoreModel, params);
+        PcaDataType pcaData = viewer.getPcaData();
+
+        MatrixI pairwise = loadDoubleMatrix(pcaData.getPairwiseMatrix());
+        pca.setPairwiseScores(pairwise);
+
+        MatrixI triDiag = loadDoubleMatrix(pcaData.getTridiagonalMatrix());
+        pca.setTridiagonal(triDiag);
+
+        MatrixI result = loadDoubleMatrix(pcaData.getEigenMatrix());
+        pca.setEigenmatrix(result);
+
+        panel.getPcaModel().setPCA(pca);
+
+        /*
+         * we haven't saved the input data! (JAL-2647 to do)
+         */
+        panel.setInputData(null);
+
+        /*
+         * add the sequence points for the PCA display
+         */
+        List<jalview.datamodel.SequencePoint> seqPoints = new ArrayList<>();
+        for (SequencePoint sp : viewer.getSequencePoint())
+        {
+          String seqId = sp.getSequenceRef();
+          SequenceI seq = seqRefIds.get(seqId);
+          if (seq == null)
+          {
+            throw new IllegalStateException(
+                    "Unmatched seqref for PCA: " + seqId);
+          }
+          Point pt = new Point(sp.getXPos(), sp.getYPos(), sp.getZPos());
+          jalview.datamodel.SequencePoint seqPoint = new jalview.datamodel.SequencePoint(
+                  seq, pt);
+          seqPoints.add(seqPoint);
+        }
+        panel.getRotatableCanvas().setPoints(seqPoints, seqPoints.size());
+
+        /*
+         * set min-max ranges and scale after setPoints (which recomputes them)
+         */
+        panel.getRotatableCanvas().setScaleFactor(viewer.getScaleFactor());
+        SeqPointMin spMin = viewer.getSeqPointMin();
+        float[] min = new float[] { spMin.getXPos(), spMin.getYPos(),
+            spMin.getZPos() };
+        SeqPointMax spMax = viewer.getSeqPointMax();
+        float[] max = new float[] { spMax.getXPos(), spMax.getYPos(),
+            spMax.getZPos() };
+        panel.getRotatableCanvas().setSeqMinMax(min, max);
+
+        // todo: hold points list in PCAModel only
+        panel.getPcaModel().setSequencePoints(seqPoints);
+
+        panel.setSelectedDimensionIndex(viewer.getXDim(), X);
+        panel.setSelectedDimensionIndex(viewer.getYDim(), Y);
+        panel.setSelectedDimensionIndex(viewer.getZDim(), Z);
+
+        // is this duplication needed?
+        panel.setTop(seqPoints.size() - 1);
+        panel.getPcaModel().setTop(seqPoints.size() - 1);
+
+        /*
+         * add the axes' end points for the display
+         */
+        for (int i = 0; i < 3; i++)
+        {
+          Axis axis = viewer.getAxis().get(i);
+          panel.getRotatableCanvas().getAxisEndPoints()[i] = new Point(
+                  axis.getXPos(), axis.getYPos(), axis.getZPos());
+        }
+
+        Desktop.addInternalFrame(panel, MessageManager.formatMessage(
+                "label.calc_title", "PCA", modelName), 475, 450);
+      }
+    } catch (Exception ex)
+    {
+      Cache.log.error("Error loading PCA: " + ex.toString());
+    }
+  }
+
+  /**
+   * Creates a new structure viewer window
+   * 
+   * @param viewerType
+   * @param viewerData
+   * @param af
+   * @param jprovider
+   */
+  protected void createStructureViewer(
+          ViewerType viewerType, final Entry<String, StructureViewerModel> viewerData,
+          AlignFrame af, jarInputStreamProvider jprovider)
+  {
+    final StructureViewerModel viewerModel = viewerData.getValue();
+    String sessionFilePath = null;
+
+    if (viewerType == ViewerType.JMOL)
+    {
+      sessionFilePath = rewriteJmolSession(viewerModel, jprovider);
+    }
+    else
+    {
+      String viewerJarEntryName = getViewerJarEntryName(
+              viewerModel.getViewId());
+      sessionFilePath = copyJarEntry(jprovider,
+              viewerJarEntryName,
+              "viewerSession", ".tmp");
+    }
+    final String sessionPath = sessionFilePath;
+    final String sviewid = viewerData.getKey();
+    try
+    {
+      SwingUtilities.invokeAndWait(new Runnable()
+      {
+        @Override
+        public void run()
+        {
+          JalviewStructureDisplayI sview = null;
+          try
+          {
+            sview = StructureViewer.createView(viewerType, af.alignPanel,
+                    viewerModel, sessionPath, sviewid);
+            addNewStructureViewer(sview);
+          } catch (OutOfMemoryError ex)
+          {
+            new OOMWarning("Restoring structure view for "
+                    + viewerType,
+                    (OutOfMemoryError) ex.getCause());
+            if (sview != null && sview.isVisible())
+            {
+              sview.closeViewer(false);
+              sview.setVisible(false);
+              sview.dispose();
+            }
+          }
+        }
+      });
+    } catch (InvocationTargetException | InterruptedException ex)
+    {
+      warn("Unexpected error when opening " + viewerType
+              + " structure viewer", ex);
+    }
+  }
+
+  /**
+   * Rewrites a Jmol session script, saves it to a temporary file, and returns
+   * the path of the file. "load file" commands are rewritten to change the
+   * original PDB file names to those created as the Jalview project is loaded.
+   * 
+   * @param svattrib
+   * @param jprovider
+   * @return
+   */
+  private String rewriteJmolSession(StructureViewerModel svattrib,
+          jarInputStreamProvider jprovider)
+  {
+    String state = svattrib.getStateData(); // Jalview < 2.9
+    if (state == null || state.isEmpty()) // Jalview >= 2.9
+    {
+      String jarEntryName = getViewerJarEntryName(svattrib.getViewId());
+      state = readJarEntry(jprovider, jarEntryName);
+    }
+    // TODO or simpler? for each key in oldFiles,
+    // replace key.getPath() in state with oldFiles.get(key).getFilePath()
+    // (allowing for different path escapings)
+    StringBuilder rewritten = new StringBuilder(state.length());
+    int cp = 0, ncp, ecp;
+    Map<File, StructureData> oldFiles = svattrib.getFileData();
+    while ((ncp = state.indexOf("load ", cp)) > -1)
+    {
+      do
+      {
+        // look for next filename in load statement
+        rewritten.append(state.substring(cp,
+                ncp = (state.indexOf("\"", ncp + 1) + 1)));
+        String oldfilenam = state.substring(ncp,
+                ecp = state.indexOf("\"", ncp));
+        // recover the new mapping data for this old filename
+        // have to normalize filename - since Jmol and jalview do
+        // filename translation differently.
+        StructureData filedat = oldFiles.get(new File(oldfilenam));
+        if (filedat == null)
+        {
+          String reformatedOldFilename = oldfilenam.replaceAll("/", "\\\\");
+          filedat = oldFiles.get(new File(reformatedOldFilename));
+        }
+        rewritten
+                .append(Platform.escapeBackslashes(filedat.getFilePath()));
+        rewritten.append("\"");
+        cp = ecp + 1; // advance beyond last \" and set cursor so we can
+                      // look for next file statement.
+      } while ((ncp = state.indexOf("/*file*/", cp)) > -1);
+    }
+    if (cp > 0)
+    {
+      // just append rest of state
+      rewritten.append(state.substring(cp));
+    }
+    else
+    {
+      System.err.print("Ignoring incomplete Jmol state for PDB ids: ");
+      rewritten = new StringBuilder(state);
+      rewritten.append("; load append ");
+      for (File id : oldFiles.keySet())
+      {
+        // add pdb files that should be present in the viewer
+        StructureData filedat = oldFiles.get(id);
+        rewritten.append(" \"").append(filedat.getFilePath()).append("\"");
+      }
+      rewritten.append(";");
+    }
+
+    if (rewritten.length() == 0)
+    {
+      return null;
+    }
+    final String history = "history = ";
+    int historyIndex = rewritten.indexOf(history);
+    if (historyIndex > -1)
+    {
+      /*
+       * change "history = [true|false];" to "history = [1|0];"
+       */
+      historyIndex += history.length();
+      String val = rewritten.substring(historyIndex, historyIndex + 5);
+      if (val.startsWith("true"))
+      {
+        rewritten.replace(historyIndex, historyIndex + 4, "1");
+      }
+      else if (val.startsWith("false"))
+      {
+        rewritten.replace(historyIndex, historyIndex + 5, "0");
+      }
+    }
+
+    try
+    {
+      File tmp = File.createTempFile("viewerSession", ".tmp");
+      try (OutputStream os = new FileOutputStream(tmp))
+      {
+        InputStream is = new ByteArrayInputStream(
+                rewritten.toString().getBytes());
+        copyAll(is, os);
+        return tmp.getAbsolutePath();
+      }
+    } catch (IOException e)
+    {
+      Cache.log.error("Error restoring Jmol session: " + e.toString());
+    }
+    return null;
+  }
+
+  /**
    * Populates an XML model of the feature colour scheme for one feature type
    * 
    * @param featureType
@@ -6089,7 +6540,7 @@ public class Jalview2XML
         noValueColour = maxcol;
       }
   
-      colour = new FeatureColour(mincol, maxcol, noValueColour,
+      colour = new FeatureColour(maxcol, mincol, maxcol, noValueColour,
               safeFloat(colourModel.getMin()),
               safeFloat(colourModel.getMax()));
       final List<String> attributeName = colourModel.getAttributeName();