Merge branch 'features/JAL-1767pcaInProject' into bug/JAL-3171_maintain_datasets_acro...
[jalview.git] / src / jalview / gui / Jalview2XML.java
index b357234..42760f3 100644 (file)
  */
 package jalview.gui;
 
+import static jalview.math.RotatableMatrix.Axis.X;
+import static jalview.math.RotatableMatrix.Axis.Y;
+import static jalview.math.RotatableMatrix.Axis.Z;
+
 import jalview.analysis.Conservation;
+import jalview.analysis.PCA;
+import jalview.analysis.scoremodels.ScoreModels;
+import jalview.analysis.scoremodels.SimilarityParams;
 import jalview.api.FeatureColourI;
 import jalview.api.ViewStyleI;
+import jalview.api.analysis.ScoreModelI;
+import jalview.api.analysis.SimilarityParamsI;
 import jalview.api.structures.JalviewStructureDisplayI;
 import jalview.bin.Cache;
 import jalview.datamodel.AlignedCodonFrame;
@@ -31,24 +40,37 @@ import jalview.datamodel.AlignmentAnnotation;
 import jalview.datamodel.AlignmentI;
 import jalview.datamodel.GraphLine;
 import jalview.datamodel.PDBEntry;
+import jalview.datamodel.Point;
 import jalview.datamodel.RnaViewerModel;
 import jalview.datamodel.SequenceFeature;
 import jalview.datamodel.SequenceGroup;
 import jalview.datamodel.SequenceI;
 import jalview.datamodel.StructureViewerModel;
 import jalview.datamodel.StructureViewerModel.StructureData;
+import jalview.datamodel.features.FeatureMatcher;
+import jalview.datamodel.features.FeatureMatcherI;
+import jalview.datamodel.features.FeatureMatcherSet;
+import jalview.datamodel.features.FeatureMatcherSetI;
 import jalview.ext.varna.RnaModel;
 import jalview.gui.StructureViewer.ViewerType;
 import jalview.io.DataSourceType;
 import jalview.io.FileFormat;
+import jalview.math.Matrix;
+import jalview.math.MatrixI;
 import jalview.renderer.ResidueShaderI;
 import jalview.schemabinding.version2.AlcodMap;
 import jalview.schemabinding.version2.AlcodonFrame;
 import jalview.schemabinding.version2.Annotation;
 import jalview.schemabinding.version2.AnnotationColours;
 import jalview.schemabinding.version2.AnnotationElement;
+import jalview.schemabinding.version2.Axis;
 import jalview.schemabinding.version2.CalcIdParam;
+import jalview.schemabinding.version2.CompoundMatcher;
+import jalview.schemabinding.version2.D;
 import jalview.schemabinding.version2.DBRef;
+import jalview.schemabinding.version2.DoubleMatrix;
+import jalview.schemabinding.version2.E;
+import jalview.schemabinding.version2.EigenMatrix;
 import jalview.schemabinding.version2.Features;
 import jalview.schemabinding.version2.Group;
 import jalview.schemabinding.version2.HiddenColumns;
@@ -60,21 +82,34 @@ import jalview.schemabinding.version2.MapListFrom;
 import jalview.schemabinding.version2.MapListTo;
 import jalview.schemabinding.version2.Mapping;
 import jalview.schemabinding.version2.MappingChoice;
+import jalview.schemabinding.version2.MatchCondition;
+import jalview.schemabinding.version2.MatcherSet;
 import jalview.schemabinding.version2.OtherData;
+import jalview.schemabinding.version2.PairwiseMatrix;
+import jalview.schemabinding.version2.PcaData;
+import jalview.schemabinding.version2.PcaViewer;
 import jalview.schemabinding.version2.PdbentryItem;
 import jalview.schemabinding.version2.Pdbids;
 import jalview.schemabinding.version2.Property;
 import jalview.schemabinding.version2.RnaViewer;
+import jalview.schemabinding.version2.Row;
 import jalview.schemabinding.version2.SecondaryStructure;
+import jalview.schemabinding.version2.SeqPointMax;
+import jalview.schemabinding.version2.SeqPointMin;
 import jalview.schemabinding.version2.Sequence;
+import jalview.schemabinding.version2.SequencePoint;
 import jalview.schemabinding.version2.SequenceSet;
 import jalview.schemabinding.version2.SequenceSetProperties;
 import jalview.schemabinding.version2.Setting;
 import jalview.schemabinding.version2.StructureState;
 import jalview.schemabinding.version2.ThresholdLine;
 import jalview.schemabinding.version2.Tree;
+import jalview.schemabinding.version2.TridiagonalMatrix;
 import jalview.schemabinding.version2.UserColours;
 import jalview.schemabinding.version2.Viewport;
+import jalview.schemabinding.version2.types.ColourThreshTypeType;
+import jalview.schemabinding.version2.types.FeatureMatcherByType;
+import jalview.schemabinding.version2.types.NoValueColour;
 import jalview.schemes.AnnotationColourGradient;
 import jalview.schemes.ColourSchemeI;
 import jalview.schemes.ColourSchemeProperty;
@@ -83,11 +118,14 @@ import jalview.schemes.ResidueProperties;
 import jalview.schemes.UserColourScheme;
 import jalview.structure.StructureSelectionManager;
 import jalview.structures.models.AAStructureBindingModel;
+import jalview.util.Format;
 import jalview.util.MessageManager;
 import jalview.util.Platform;
 import jalview.util.StringUtils;
 import jalview.util.jarInputStreamProvider;
+import jalview.util.matcher.Condition;
 import jalview.viewmodel.AlignmentViewport;
+import jalview.viewmodel.PCAModel;
 import jalview.viewmodel.ViewportRanges;
 import jalview.viewmodel.seqfeatures.FeatureRendererSettings;
 import jalview.viewmodel.seqfeatures.FeaturesDisplayed;
@@ -115,6 +153,7 @@ import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -155,6 +194,12 @@ public class Jalview2XML
 
   private static final String UTF_8 = "UTF-8";
 
+  /**
+   * prefix for recovering datasets for alignments with multiple views where
+   * non-existent dataset IDs were written for some views
+   */
+  private static final String UNIQSEQSETID = "uniqueSeqSetId.";
+
   // use this with nextCounter() to make unique names for entities
   private int counter = 0;
 
@@ -216,34 +261,6 @@ public class Jalview2XML
     }
   }
 
-  void clearSeqRefs()
-  {
-    if (_cleartables)
-    {
-      if (seqRefIds != null)
-      {
-        seqRefIds.clear();
-      }
-      if (seqsToIds != null)
-      {
-        seqsToIds.clear();
-      }
-      if (incompleteSeqs != null)
-      {
-        incompleteSeqs.clear();
-      }
-      // seqRefIds = null;
-      // seqsToIds = null;
-    }
-    else
-    {
-      // do nothing
-      warn("clearSeqRefs called when _cleartables was not set. Doing nothing.");
-      // seqRefIds = new Hashtable();
-      // seqsToIds = new IdentityHashMap();
-    }
-  }
-
   void initSeqRefs()
   {
     if (seqsToIds == null)
@@ -907,15 +924,33 @@ public class Jalview2XML
         }
         if (sf.otherDetails != null)
         {
-          String key;
-          Iterator<String> keys = sf.otherDetails.keySet().iterator();
-          while (keys.hasNext())
+          /*
+           * save feature attributes, which may be simple strings or
+           * map valued (have sub-attributes)
+           */
+          for (Entry<String, Object> entry : sf.otherDetails.entrySet())
           {
-            key = keys.next();
-            OtherData keyValue = new OtherData();
-            keyValue.setKey(key);
-            keyValue.setValue(sf.otherDetails.get(key).toString());
-            features.addOtherData(keyValue);
+            String key = entry.getKey();
+            Object value = entry.getValue();
+            if (value instanceof Map<?, ?>)
+            {
+              for (Entry<String, Object> subAttribute : ((Map<String, Object>) value)
+                      .entrySet())
+              {
+                OtherData otherData = new OtherData();
+                otherData.setKey(key);
+                otherData.setKey2(subAttribute.getKey());
+                otherData.setValue(subAttribute.getValue().toString());
+                features.addOtherData(otherData);
+              }
+            }
+            else
+            {
+              OtherData otherData = new OtherData();
+              otherData.setKey(key);
+              otherData.setValue(value.toString());
+              features.addOtherData(otherData);
+            }
           }
         }
 
@@ -1084,7 +1119,7 @@ public class Jalview2XML
 
     // SAVE TREES
     // /////////////////////////////////
-    if (!storeDS && av.currentTree != null)
+    if (!storeDS && av.getCurrentTree() != null)
     {
       // FIND ANY ASSOCIATED TREES
       // NOT IMPLEMENTED FOR HEADLESS STATE AT PRESENT
@@ -1098,13 +1133,13 @@ public class Jalview2XML
           {
             TreePanel tp = (TreePanel) frames[t];
 
-            if (tp.treeCanvas.av.getAlignment() == jal)
+            if (tp.getTreeCanvas().getViewport().getAlignment() == jal)
             {
               Tree tree = new Tree();
               tree.setTitle(tp.getTitle());
-              tree.setCurrentTree((av.currentTree == tp.getTree()));
+              tree.setCurrentTree((av.getCurrentTree() == tp.getTree()));
               tree.setNewick(tp.getTree().print());
-              tree.setThreshold(tp.treeCanvas.threshold);
+              tree.setThreshold(tp.getTreeCanvas().getThreshold());
 
               tree.setFitToWindow(tp.fitToWindow.getState());
               tree.setFontName(tp.getTreeFont().getName());
@@ -1120,6 +1155,7 @@ public class Jalview2XML
               tree.setXpos(tp.getX());
               tree.setYpos(tp.getY());
               tree.setId(makeHashCode(tp, null));
+              tree.setLinkToAllViews(tp.treeCanvas.applyToAllViews);
               jms.addTree(tree);
             }
           }
@@ -1127,6 +1163,24 @@ public class Jalview2XML
       }
     }
 
+    /*
+     * save PCA viewers
+     */
+    if (!storeDS && Desktop.desktop != null)
+    {
+      for (JInternalFrame frame : Desktop.desktop.getAllFrames())
+      {
+        if (frame instanceof PCAPanel)
+        {
+          PCAPanel panel = (PCAPanel) frame;
+          if (panel.av.getAlignment() == jal)
+          {
+            savePCA(panel, jms);
+          }
+        }
+      }
+    }
+
     // SAVE ANNOTATIONS
     /**
      * store forward refs from an annotationRow to any groups
@@ -1243,7 +1297,7 @@ public class Jalview2XML
       {
         view.setComplementId(av.getCodingComplement().getViewId());
       }
-      view.setViewName(av.viewName);
+      view.setViewName(av.getViewName());
       view.setGatheredViews(av.isGatherViewsHere());
 
       Rectangle size = ap.av.getExplodedGeometry();
@@ -1341,19 +1395,33 @@ public class Jalview2XML
       {
         jalview.schemabinding.version2.FeatureSettings fs = new jalview.schemabinding.version2.FeatureSettings();
 
-        String[] renderOrder = ap.getSeqPanel().seqCanvas
-                .getFeatureRenderer().getRenderOrder()
-                .toArray(new String[0]);
+        FeatureRenderer fr = ap.getSeqPanel().seqCanvas
+                .getFeatureRenderer();
+        String[] renderOrder = fr.getRenderOrder().toArray(new String[0]);
 
         Vector<String> settingsAdded = new Vector<>();
         if (renderOrder != null)
         {
           for (String featureType : renderOrder)
           {
-            FeatureColourI fcol = ap.getSeqPanel().seqCanvas
-                    .getFeatureRenderer().getFeatureStyle(featureType);
             Setting setting = new Setting();
             setting.setType(featureType);
+
+            /*
+             * save any filter for the feature type
+             */
+            FeatureMatcherSetI filter = fr.getFeatureFilter(featureType);
+            if (filter != null)  {
+              Iterator<FeatureMatcherI> filters = filter.getMatchers().iterator();
+              FeatureMatcherI firstFilter = filters.next();
+              setting.setMatcherSet(Jalview2XML.marshalFilter(
+                      firstFilter, filters, filter.isAnded()));
+            }
+
+            /*
+             * save colour scheme for the feature type
+             */
+            FeatureColourI fcol = fr.getFeatureStyle(featureType);
             if (!fcol.isSimpleColour())
             {
               setting.setColour(fcol.getMaxColour().getRGB());
@@ -1361,8 +1429,25 @@ public class Jalview2XML
               setting.setMin(fcol.getMin());
               setting.setMax(fcol.getMax());
               setting.setColourByLabel(fcol.isColourByLabel());
+              if (fcol.isColourByAttribute())
+              {
+                setting.setAttributeName(fcol.getAttributeName());
+              }
               setting.setAutoScale(fcol.isAutoScaled());
               setting.setThreshold(fcol.getThreshold());
+              Color noColour = fcol.getNoColour();
+              if (noColour == null)
+              {
+                setting.setNoValueColour(NoValueColour.NONE);
+              }
+              else if (noColour.equals(fcol.getMaxColour()))
+              {
+                setting.setNoValueColour(NoValueColour.MAX);
+              }
+              else
+              {
+                setting.setNoValueColour(NoValueColour.MIN);
+              }
               // -1 = No threshold, 0 = Below, 1 = Above
               setting.setThreshstate(fcol.isAboveThreshold() ? 1
                       : (fcol.isBelowThreshold() ? 0 : -1));
@@ -1374,7 +1459,7 @@ public class Jalview2XML
 
             setting.setDisplay(
                     av.getFeaturesDisplayed().isVisible(featureType));
-            float rorder = ap.getSeqPanel().seqCanvas.getFeatureRenderer()
+            float rorder = fr
                     .getOrder(featureType);
             if (rorder > -1)
             {
@@ -1386,8 +1471,7 @@ public class Jalview2XML
         }
 
         // is groups actually supposed to be a map here ?
-        Iterator<String> en = ap.getSeqPanel().seqCanvas
-                .getFeatureRenderer().getFeatureGroups().iterator();
+        Iterator<String> en = fr.getFeatureGroups().iterator();
         Vector<String> groupsAdded = new Vector<>();
         while (en.hasNext())
         {
@@ -1398,8 +1482,7 @@ public class Jalview2XML
           }
           Group g = new Group();
           g.setName(grp);
-          g.setDisplay(((Boolean) ap.getSeqPanel().seqCanvas
-                  .getFeatureRenderer().checkGroupVisibility(grp, false))
+          g.setDisplay(((Boolean) fr.checkGroupVisibility(grp, false))
                           .booleanValue());
           fs.addGroup(g);
           groupsAdded.addElement(grp);
@@ -1417,9 +1500,10 @@ public class Jalview2XML
         }
         else
         {
-          ArrayList<int[]> hiddenRegions = hidden.getHiddenColumnsCopy();
-          for (int[] region : hiddenRegions)
+          Iterator<int[]> hiddenRegions = hidden.iterator();
+          while (hiddenRegions.hasNext())
           {
+            int[] region = hiddenRegions.next();
             HiddenColumns hc = new HiddenColumns();
             hc.setStart(region[0]);
             hc.setEnd(region[1]);
@@ -1474,6 +1558,165 @@ public class Jalview2XML
   }
 
   /**
+   * Writes PCA viewer attributes and computed values to an XML model object and adds it to the JalviewModel. Any exceptions are reported by logging.
+   */
+  protected void savePCA(PCAPanel panel, JalviewModelSequence jms)
+  {
+    try
+    {
+      PcaViewer viewer = new PcaViewer();
+      viewer.setHeight(panel.getHeight());
+      viewer.setWidth(panel.getWidth());
+      viewer.setXpos(panel.getX());
+      viewer.setYpos(panel.getY());
+      viewer.setTitle(panel.getTitle());
+      PCAModel pcaModel = panel.pcaModel;
+      viewer.setScoreModelName(pcaModel.getScoreModelName());
+      viewer.setXDim(panel.getSelectedDimensionIndex(X));
+      viewer.setYDim(panel.getSelectedDimensionIndex(Y));
+      viewer.setZDim(panel.getSelectedDimensionIndex(Z));
+      viewer.setBgColour(panel.rc.getBackgroundColour().getRGB());
+      viewer.setScaleFactor(panel.rc.scaleFactor);
+      float[] spMin = panel.rc.getSeqMin();
+      SeqPointMin spmin = new SeqPointMin();
+      spmin.setXPos(spMin[0]);
+      spmin.setYPos(spMin[1]);
+      spmin.setZPos(spMin[2]);
+      viewer.setSeqPointMin(spmin);
+      float[] spMax = panel.rc.getSeqMax();
+      SeqPointMax spmax = new SeqPointMax();
+      spmax.setXPos(spMax[0]);
+      spmax.setYPos(spMax[1]);
+      spmax.setZPos(spMax[2]);
+      viewer.setSeqPointMax(spmax);
+      viewer.setShowLabels(panel.rc.showLabels);
+      viewer.setLinkToAllViews(panel.rc.applyToAllViews);
+      SimilarityParamsI sp = pcaModel.getSimilarityParameters();
+      viewer.setIncludeGaps(sp.includeGaps());
+      viewer.setMatchGaps(sp.matchGaps());
+      viewer.setIncludeGappedColumns(sp.includeGappedColumns());
+      viewer.setDenominateByShortestLength(sp.denominateByShortestLength());
+
+      /*
+       * sequence points on display
+       */
+      for (jalview.datamodel.SequencePoint spt : pcaModel
+              .getSequencePoints())
+      {
+        SequencePoint point = new SequencePoint();
+        point.setSequenceRef(seqHash(spt.getSequence()));
+        point.setXPos(spt.coord.x);
+        point.setYPos(spt.coord.y);
+        point.setZPos(spt.coord.z);
+        viewer.addSequencePoint(point);
+      }
+
+      /*
+       * (end points of) axes on display
+       */
+      for (Point p : panel.rc.axisEndPoints)
+      {
+        Axis axis = new Axis();
+        axis.setXPos(p.x);
+        axis.setYPos(p.y);
+        axis.setZPos(p.z);
+        viewer.addAxis(axis);
+      }
+
+      /*
+       * raw PCA data (note we are not restoring PCA inputs here -
+       * alignment view, score model, similarity parameters)
+       */
+      PcaData data = new PcaData();
+      viewer.setPcaData(data);
+      PCA pca = pcaModel.getPcaData();
+
+      PairwiseMatrix pm = new PairwiseMatrix();
+      saveDoubleMatrix(pca.getPairwiseScores(), pm);
+      data.setPairwiseMatrix(pm);
+
+      TridiagonalMatrix tm = new TridiagonalMatrix();
+      saveDoubleMatrix(pca.getTridiagonal(), tm);
+      data.setTridiagonalMatrix(tm);
+
+      EigenMatrix eigenMatrix = new EigenMatrix();
+      data.setEigenMatrix(eigenMatrix);
+      saveDoubleMatrix(pca.getEigenmatrix(), eigenMatrix);
+
+      jms.addPcaViewer(viewer);
+    } catch (Throwable t)
+    {
+      Cache.log.error("Error saving PCA: " + t.getMessage());
+    }
+  }
+
+  /**
+   * Stores values from a matrix into an XML element, including (if present) the
+   * D or E vectors
+   * 
+   * @param m
+   * @param xmlMatrix
+   * @see #loadDoubleMatrix(DoubleMatrix)
+   */
+  protected void saveDoubleMatrix(MatrixI m, DoubleMatrix xmlMatrix)
+  {
+    xmlMatrix.setRows(m.height());
+    xmlMatrix.setColumns(m.width());
+    for (int i = 0; i < m.height(); i++)
+    {
+      Row row = new Row();
+      for (int j = 0; j < m.width(); j++)
+      {
+        row.addV(m.getValue(i, j));
+      }
+      xmlMatrix.addRow(row);
+    }
+    if (m.getD() != null)
+    {
+      D dVector = new D();
+      dVector.setV(m.getD());
+      xmlMatrix.setD(dVector);
+    }
+    if (m.getE() != null)
+    {
+      E eVector = new E();
+      eVector.setV(m.getE());
+      xmlMatrix.setE(eVector);
+    }
+  }
+
+  /**
+   * Loads XML matrix data into a new Matrix object, including the D and/or E
+   * vectors (if present)
+   * 
+   * @param mData
+   * @return
+   * @see Jalview2XML#saveDoubleMatrix(MatrixI, DoubleMatrix)
+   */
+  protected MatrixI loadDoubleMatrix(DoubleMatrix mData)
+  {
+    int rows = mData.getRows();
+    double[][] vals = new double[rows][];
+
+    for (int i = 0; i < rows; i++)
+    {
+      vals[i] = mData.getRow(i).getV();
+    }
+
+    MatrixI m = new Matrix(vals);
+    
+    if (mData.getD() != null) {
+      m.setD(mData.getD().getV());
+    }
+    if (mData.getE() != null)
+    {
+      m.setE(mData.getE().getV());
+    }
+
+    return m;
+  }
+
+  /**
    * Save any Varna viewers linked to this sequence. Writes an rnaViewer element
    * for each viewer, with
    * <ul>
@@ -2834,6 +3077,28 @@ public class Jalview2XML
             : null;
 
     // ////////////////////////////////
+    // INITIALISE ALIGNMENT SEQUENCESETID AND VIEWID
+    //
+    //
+    // If we just load in the same jar file again, the sequenceSetId
+    // will be the same, and we end up with multiple references
+    // to the same sequenceSet. We must modify this id on load
+    // so that each load of the file gives a unique id
+
+    /**
+     * used to resolve correct alignment dataset for alignments with multiple
+     * views
+     */
+    String uniqueSeqSetId = null;
+    String viewId = null;
+    if (view != null)
+    {
+      uniqueSeqSetId = view.getSequenceSetId() + uniqueSetSuffix;
+      viewId = (view.getId() == null ? null
+              : view.getId() + uniqueSetSuffix);
+    }
+
+    // ////////////////////////////////
     // LOAD SEQUENCES
 
     List<SequenceI> hiddenSeqs = null;
@@ -2951,7 +3216,7 @@ public class Jalview2XML
 
       // finally, verify all data in vamsasSet is actually present in al
       // passing on flag indicating if it is actually a stored dataset
-      recoverDatasetFor(vamsasSet, al, isdsal);
+      recoverDatasetFor(vamsasSet, al, isdsal, uniqueSeqSetId);
     }
 
     if (referenceseqForView != null)
@@ -2990,19 +3255,46 @@ public class Jalview2XML
                     features[f].getEnd(), features[f].getScore(),
                     features[f].getFeatureGroup());
             sf.setStatus(features[f].getStatus());
+
+            /*
+             * load any feature attributes - include map-valued attributes
+             */
+            Map<String, Map<String, String>> mapAttributes = new HashMap<>();
             for (int od = 0; od < features[f].getOtherDataCount(); od++)
             {
               OtherData keyValue = features[f].getOtherData(od);
-              if (keyValue.getKey().startsWith("LINK"))
+              String attributeName = keyValue.getKey();
+              String attributeValue = keyValue.getValue();
+              if (attributeName.startsWith("LINK"))
               {
-                sf.addLink(keyValue.getValue());
+                sf.addLink(attributeValue);
               }
               else
               {
-                sf.setValue(keyValue.getKey(), keyValue.getValue());
+                String subAttribute = keyValue.getKey2();
+                if (subAttribute == null)
+                {
+                  // simple string-valued attribute
+                  sf.setValue(attributeName, attributeValue);
+                }
+                else
+                {
+                  // attribute 'key' has sub-attribute 'key2'
+                  if (!mapAttributes.containsKey(attributeName))
+                  {
+                    mapAttributes.put(attributeName, new HashMap<>());
+                  }
+                  mapAttributes.get(attributeName).put(subAttribute,
+                          attributeValue);
+                }
               }
-
             }
+            for (Entry<String, Map<String, String>> mapAttribute : mapAttributes
+                    .entrySet())
+            {
+              sf.setValue(mapAttribute.getKey(), mapAttribute.getValue());
+            }
+
             // adds feature to datasequence's feature set (since Jalview 2.10)
             al.getSequenceAt(i).addSequenceFeature(sf);
           }
@@ -3472,13 +3764,6 @@ public class Jalview2XML
     // ///////////////////////////////
     // LOAD VIEWPORT
 
-    // If we just load in the same jar file again, the sequenceSetId
-    // will be the same, and we end up with multiple references
-    // to the same sequenceSet. We must modify this id on load
-    // so that each load of the file gives a unique id
-    String uniqueSeqSetId = view.getSequenceSetId() + uniqueSetSuffix;
-    String viewId = (view.getId() == null ? null
-            : view.getId() + uniqueSetSuffix);
     AlignFrame af = null;
     AlignViewport av = null;
     // now check to see if we really need to create a new viewport.
@@ -3560,6 +3845,7 @@ public class Jalview2XML
     if (loadTreesAndStructures)
     {
       loadTrees(jms, view, af, av, ap);
+      loadPCAViewers(jms, ap);
       loadPDBStructures(jprovider, jseqs, af, ap);
       loadRnaViewers(jprovider, jseqs, ap);
     }
@@ -3702,12 +3988,11 @@ public class Jalview2XML
           tp.setTitle(tree.getTitle());
           tp.setBounds(new Rectangle(tree.getXpos(), tree.getYpos(),
                   tree.getWidth(), tree.getHeight()));
-          tp.av = av; // af.viewport; // TODO: verify 'associate with all
+          tp.setViewport(av); // af.viewport; // TODO: verify 'associate with all
           // views'
           // works still
-          tp.treeCanvas.av = av; // af.viewport;
-          tp.treeCanvas.ap = ap; // af.alignPanel;
-
+          tp.getTreeCanvas().setViewport(av); // af.viewport;
+          tp.getTreeCanvas().setAssociatedPanel(ap); // af.alignPanel;
         }
         if (tp == null)
         {
@@ -3734,7 +4019,8 @@ public class Jalview2XML
         tp.showBootstrap(tree.getShowBootstrap());
         tp.showDistances(tree.getShowDistances());
 
-        tp.treeCanvas.threshold = tree.getThreshold();
+        tp.getTreeCanvas().setThreshold(tree.getThreshold());
+        tp.treeCanvas.applyToAllViews = tree.isLinkToAllViews();
 
         if (tree.getCurrentTree())
         {
@@ -4250,7 +4536,8 @@ public class Jalview2XML
       StructureData filedat = oldFiles.get(id);
       String pdbFile = filedat.getFilePath();
       SequenceI[] seq = filedat.getSeqList().toArray(new SequenceI[0]);
-      binding.getSsm().setMapping(seq, null, pdbFile, DataSourceType.FILE);
+      binding.getSsm().setMapping(seq, null, pdbFile, DataSourceType.FILE,
+              null);
       binding.addSequenceForStructFile(pdbFile, seq);
     }
     // and add the AlignmentPanel's reference to the view panel
@@ -4466,7 +4753,7 @@ public class Jalview2XML
 
     if (view.getViewName() != null)
     {
-      af.viewport.viewName = view.getViewName();
+      af.viewport.setViewName(view.getViewName());
       af.setInitialTabVisible();
     }
     af.setBounds(view.getXpos(), view.getYpos(), view.getWidth(),
@@ -4577,9 +4864,11 @@ public class Jalview2XML
       af.viewport.setShowGroupConservation(false);
     }
 
-    // recover featre settings
+    // recover feature settings
     if (jms.getFeatureSettings() != null)
     {
+      FeatureRenderer fr = af.alignPanel.getSeqPanel().seqCanvas
+              .getFeatureRenderer();
       FeaturesDisplayed fdi;
       af.viewport.setFeaturesDisplayed(fdi = new FeaturesDisplayed());
       String[] renderOrder = new String[jms.getFeatureSettings()
@@ -4591,14 +4880,51 @@ public class Jalview2XML
               .getSettingCount(); fs++)
       {
         Setting setting = jms.getFeatureSettings().getSetting(fs);
+        String featureType = setting.getType();
+
+        /*
+         * restore feature filters (if any)
+         */
+        MatcherSet filters = setting.getMatcherSet();
+        if (filters != null)
+        {
+          FeatureMatcherSetI filter = Jalview2XML
+                  .unmarshalFilter(featureType, filters);
+          if (!filter.isEmpty())
+          {
+            fr.setFeatureFilter(featureType, filter);
+          }
+        }
+
+        /*
+         * restore feature colour scheme
+         */
+        Color maxColour = new Color(setting.getColour());
         if (setting.hasMincolour())
         {
-          FeatureColourI gc = setting.hasMin()
-                  ? new FeatureColour(new Color(setting.getMincolour()),
-                          new Color(setting.getColour()), setting.getMin(),
-                          setting.getMax())
-                  : new FeatureColour(new Color(setting.getMincolour()),
-                          new Color(setting.getColour()), 0, 1);
+          /*
+           * minColour is always set unless a simple colour
+           * (including for colour by label though it doesn't use it)
+           */
+          Color minColour = new Color(setting.getMincolour());
+          Color noValueColour = minColour;
+          NoValueColour noColour = setting.getNoValueColour();
+          if (noColour == NoValueColour.NONE)
+          {
+            noValueColour = null;
+          }
+          else if (noColour == NoValueColour.MAX)
+          {
+            noValueColour = maxColour;
+          }
+          float min = setting.hasMin() ? setting.getMin() : 0f;
+          float max = setting.hasMin() ? setting.getMax() : 1f;
+          FeatureColourI gc = new FeatureColour(minColour, maxColour,
+                  noValueColour, min, max);
+          if (setting.getAttributeNameCount() > 0)
+          {
+            gc.setAttributeName(setting.getAttributeName());
+          }
           if (setting.hasThreshold())
           {
             gc.setThreshold(setting.getThreshold());
@@ -4623,26 +4949,26 @@ public class Jalview2XML
             gc.setColourByLabel(setting.getColourByLabel());
           }
           // and put in the feature colour table.
-          featureColours.put(setting.getType(), gc);
+          featureColours.put(featureType, gc);
         }
         else
         {
-          featureColours.put(setting.getType(),
-                  new FeatureColour(new Color(setting.getColour())));
+          featureColours.put(featureType,
+                  new FeatureColour(maxColour));
         }
-        renderOrder[fs] = setting.getType();
+        renderOrder[fs] = featureType;
         if (setting.hasOrder())
         {
-          featureOrder.put(setting.getType(), setting.getOrder());
+          featureOrder.put(featureType, setting.getOrder());
         }
         else
         {
-          featureOrder.put(setting.getType(), new Float(
+          featureOrder.put(featureType, new Float(
                   fs / jms.getFeatureSettings().getSettingCount()));
         }
         if (setting.getDisplay())
         {
-          fdi.setVisible(setting.getType());
+          fdi.setVisible(featureType);
         }
       }
       Map<String, Boolean> fgtable = new Hashtable<>();
@@ -4656,9 +4982,7 @@ public class Jalview2XML
       // jms.getFeatureSettings().getTransparency() : 0.0, featureOrder);
       FeatureRendererSettings frs = new FeatureRendererSettings(renderOrder,
               fgtable, featureColours, 1.0f, featureOrder);
-      af.alignPanel.getSeqPanel().seqCanvas.getFeatureRenderer()
-              .transferSettings(frs);
-
+      fr.transferSettings(frs);
     }
 
     if (view.getHiddenColumnsCount() > 0)
@@ -4984,13 +5308,25 @@ public class Jalview2XML
   }
 
   private void recoverDatasetFor(SequenceSet vamsasSet, AlignmentI al,
-          boolean ignoreUnrefed)
+          boolean ignoreUnrefed, String uniqueSeqSetId)
   {
     jalview.datamodel.AlignmentI ds = getDatasetFor(
             vamsasSet.getDatasetId());
     Vector dseqs = null;
     if (ds == null)
     {
+      if (!ignoreUnrefed)
+      {
+        // try to resolve the dataset via uniqueSeqSetId
+        ds = getDatasetFor(UNIQSEQSETID + uniqueSeqSetId);
+        if (ds != null)
+        {
+          addDatasetRef(vamsasSet.getDatasetId(), ds);
+        }
+      }
+    }
+    if (ds == null)
+    {
       // create a list of new dataset sequences
       dseqs = new Vector();
     }
@@ -5013,6 +5349,8 @@ public class Jalview2XML
     if (al.getDataset() == null && !ignoreUnrefed)
     {
       al.setDataset(ds);
+      // register dataset for the alignment's uniqueSeqSetId for legacy projects
+      addDatasetRef(UNIQSEQSETID + uniqueSeqSetId, ds);
     }
   }
 
@@ -5342,28 +5680,28 @@ public class Jalview2XML
 
   }
 
-  public jalview.gui.AlignmentPanel copyAlignPanel(AlignmentPanel ap,
-          boolean keepSeqRefs)
+  /**
+   * Provides a 'copy' of an alignment view (on action New View) by 'saving' the
+   * view as XML (but not to file), and then reloading it
+   * 
+   * @param ap
+   * @return
+   */
+  public AlignmentPanel copyAlignPanel(AlignmentPanel ap)
   {
     initSeqRefs();
     JalviewModel jm = saveState(ap, null, null, null);
 
-    if (!keepSeqRefs)
-    {
-      clearSeqRefs();
-      jm.getJalviewModelSequence().getViewport(0).setSequenceSetId(null);
-    }
-    else
-    {
-      uniqueSetSuffix = "";
-      jm.getJalviewModelSequence().getViewport(0).setId(null); // we don't
-      // overwrite the
-      // view we just
-      // copied
-    }
+    addDatasetRef(jm.getVamsasModel().getSequenceSet()[0].getDatasetId(),
+            ap.getAlignment().getDataset());
+
+    uniqueSetSuffix = "";
+    jm.getJalviewModelSequence().getViewport(0).setId(null);
+    // we don't overwrite the view we just copied
+
     if (this.frefedSequence == null)
     {
-      frefedSequence = new Vector();
+      frefedSequence = new Vector<>();
     }
 
     viewportsAdded.clear();
@@ -5383,32 +5721,8 @@ public class Jalview2XML
     return af.alignPanel;
   }
 
-  /**
-   * flag indicating if hashtables should be cleared on finalization TODO this
-   * flag may not be necessary
-   */
-  private final boolean _cleartables = true;
-
   private Hashtable jvids2vobj;
 
-  /*
-   * (non-Javadoc)
-   * 
-   * @see java.lang.Object#finalize()
-   */
-  @Override
-  protected void finalize() throws Throwable
-  {
-    // really make sure we have no buried refs left.
-    if (_cleartables)
-    {
-      clearSeqRefs();
-    }
-    this.seqRefIds = null;
-    this.seqsToIds = null;
-    super.finalize();
-  }
-
   private void warn(String msg)
   {
     warn(msg, null);
@@ -5639,4 +5953,409 @@ public class Jalview2XML
   {
     return counter++;
   }
+
+  /**
+   * Loads any saved PCA viewers
+   * 
+   * @param jms
+   * @param ap
+   */
+  protected void loadPCAViewers(JalviewModelSequence jms, AlignmentPanel ap)
+  {
+    try
+    {
+      for (int t = 0; t < jms.getPcaViewerCount(); t++)
+      {
+        PcaViewer viewer = jms.getPcaViewer(t);
+        String modelName = viewer.getScoreModelName();
+        SimilarityParamsI params = new SimilarityParams(
+                viewer.isIncludeGappedColumns(),
+                viewer.isMatchGaps(), viewer.isIncludeGaps(),
+                viewer.isDenominateByShortestLength());
+
+        /*
+         * create the panel (without computing the PCA)
+         */
+        PCAPanel panel = new PCAPanel(ap, modelName, params);
+
+        panel.setTitle(viewer.getTitle());
+        panel.setBounds(new Rectangle(viewer.getXpos(), viewer.getYpos(),
+                viewer.getWidth(), viewer.getHeight()));
+  
+        boolean showLabels = viewer.isShowLabels();
+        panel.setShowLabels(showLabels);
+        panel.rc.showLabels = showLabels;
+        panel.rc.bgColour = new Color(viewer.getBgColour());
+        panel.rc.applyToAllViews = viewer.isLinkToAllViews();
+
+        /*
+         * load PCA output data
+         */
+        ScoreModelI scoreModel = ScoreModels.getInstance()
+                .getScoreModel(modelName, ap);
+        PCA pca = new PCA(null, scoreModel, params);
+        PcaData pcaData = viewer.getPcaData();
+
+        MatrixI pairwise = loadDoubleMatrix(pcaData.getPairwiseMatrix());
+        pca.setPairwiseScores(pairwise);
+
+        MatrixI triDiag = loadDoubleMatrix(pcaData.getTridiagonalMatrix());
+        pca.setTridiagonal(triDiag);
+
+        MatrixI result = loadDoubleMatrix(pcaData.getEigenMatrix());
+        pca.setEigenmatrix(result);
+
+        panel.pcaModel.setPCA(pca);
+
+        /*
+         * we haven't saved the input data! (JAL-2647 to do)
+         */
+        panel.setInputData(null);
+
+        /*
+         * add the sequence points for the PCA display
+         */
+        List<jalview.datamodel.SequencePoint> seqPoints = new ArrayList<>();
+        for (SequencePoint sp : viewer.getSequencePoint())
+        {
+          String seqId = sp.getSequenceRef();
+          SequenceI seq = seqRefIds.get(seqId);
+          if (seq == null)
+          {
+            throw new IllegalStateException(
+                    "Unmatched seqref for PCA: " + seqId);
+          }
+          Point pt = new Point(sp.getXPos(), sp.getYPos(), sp.getZPos());
+          jalview.datamodel.SequencePoint seqPoint = new jalview.datamodel.SequencePoint(
+                  seq, pt);
+          seqPoints.add(seqPoint);
+        }
+        panel.rc.setPoints(seqPoints, seqPoints.size());
+
+        /*
+         * set min-max ranges and scale after setPoints (which recomputes them)
+         */
+        panel.rc.scaleFactor = viewer.getScaleFactor();
+        SeqPointMin spMin = viewer.getSeqPointMin();
+        float[] min = new float[] { spMin.getXPos(), spMin.getYPos(),
+            spMin.getZPos() };
+        SeqPointMax spMax = viewer.getSeqPointMax();
+        float[] max = new float[] { spMax.getXPos(), spMax.getYPos(),
+            spMax.getZPos() };
+        panel.rc.setSeqMinMax(min, max);
+
+        // todo: hold points list in PCAModel only
+        panel.pcaModel.setSequencePoints(seqPoints);
+
+        panel.setSelectedDimensionIndex(viewer.getXDim(), X);
+        panel.setSelectedDimensionIndex(viewer.getYDim(), Y);
+        panel.setSelectedDimensionIndex(viewer.getZDim(), Z);
+
+        // is this duplication needed?
+        panel.top = seqPoints.size() - 1;
+        panel.pcaModel.setTop(seqPoints.size() - 1);
+
+        /*
+         * add the axes' end points for the display
+         */
+        for (int i = 0; i < 3; i++)
+        {
+          Axis axis = viewer.getAxis(i);
+          panel.rc.axisEndPoints[i] = new Point(axis.getXPos(),
+                  axis.getYPos(), axis.getZPos());
+        }
+
+        Desktop.addInternalFrame(panel, MessageManager.formatMessage(
+                "label.calc_title", "PCA", modelName), 475, 450);
+      }
+    } catch (Exception ex)
+    {
+      Cache.log.error("Error loading PCA: " + ex.toString());
+    }
+  }
+
+  /**
+   * Populates an XML model of the feature colour scheme for one feature type
+   * 
+   * @param featureType
+   * @param fcol
+   * @return
+   */
+  protected static jalview.schemabinding.version2.Colour marshalColour(
+          String featureType, FeatureColourI fcol)
+  {
+    jalview.schemabinding.version2.Colour col = new jalview.schemabinding.version2.Colour();
+    if (fcol.isSimpleColour())
+    {
+      col.setRGB(Format.getHexString(fcol.getColour()));
+    }
+    else
+    {
+      col.setRGB(Format.getHexString(fcol.getMaxColour()));
+      col.setMin(fcol.getMin());
+      col.setMax(fcol.getMax());
+      col.setMinRGB(jalview.util.Format.getHexString(fcol.getMinColour()));
+      col.setAutoScale(fcol.isAutoScaled());
+      col.setThreshold(fcol.getThreshold());
+      col.setColourByLabel(fcol.isColourByLabel());
+      col.setThreshType(fcol.isAboveThreshold() ? ColourThreshTypeType.ABOVE
+              : (fcol.isBelowThreshold() ? ColourThreshTypeType.BELOW
+                      : ColourThreshTypeType.NONE));
+      if (fcol.isColourByAttribute())
+      {
+        col.setAttributeName(fcol.getAttributeName());
+      }
+      Color noColour = fcol.getNoColour();
+      if (noColour == null)
+      {
+        col.setNoValueColour(NoValueColour.NONE);
+      }
+      else if (noColour == fcol.getMaxColour())
+      {
+        col.setNoValueColour(NoValueColour.MAX);
+      }
+      else
+      {
+        col.setNoValueColour(NoValueColour.MIN);
+      }
+    }
+    col.setName(featureType);
+    return col;
+  }
+
+  /**
+   * Populates an XML model of the feature filter(s) for one feature type
+   * 
+   * @param firstMatcher
+   *          the first (or only) match condition)
+   * @param filter
+   *          remaining match conditions (if any)
+   * @param and
+   *          if true, conditions are and-ed, else or-ed
+   */
+  protected static MatcherSet marshalFilter(FeatureMatcherI firstMatcher,
+          Iterator<FeatureMatcherI> filters, boolean and)
+  {
+    MatcherSet result = new MatcherSet();
+  
+    if (filters.hasNext())
+    {
+      /*
+       * compound matcher
+       */
+      CompoundMatcher compound = new CompoundMatcher();
+      compound.setAnd(and);
+      MatcherSet matcher1 = marshalFilter(firstMatcher,
+              Collections.emptyIterator(), and);
+      compound.addMatcherSet(matcher1);
+      FeatureMatcherI nextMatcher = filters.next();
+      MatcherSet matcher2 = marshalFilter(nextMatcher, filters, and);
+      compound.addMatcherSet(matcher2);
+      result.setCompoundMatcher(compound);
+    }
+    else
+    {
+      /*
+       * single condition matcher
+       */
+      MatchCondition matcherModel = new MatchCondition();
+      matcherModel.setCondition(
+              firstMatcher.getMatcher().getCondition().getStableName());
+      matcherModel.setValue(firstMatcher.getMatcher().getPattern());
+      if (firstMatcher.isByAttribute())
+      {
+        matcherModel.setBy(FeatureMatcherByType.BYATTRIBUTE);
+        matcherModel.setAttributeName(firstMatcher.getAttribute());
+      }
+      else if (firstMatcher.isByLabel())
+      {
+        matcherModel.setBy(FeatureMatcherByType.BYLABEL);
+      }
+      else if (firstMatcher.isByScore())
+      {
+        matcherModel.setBy(FeatureMatcherByType.BYSCORE);
+      }
+      result.setMatchCondition(matcherModel);
+    }
+  
+    return result;
+  }
+
+  /**
+   * Loads one XML model of a feature filter to a Jalview object
+   * 
+   * @param featureType
+   * @param matcherSetModel
+   * @return
+   */
+  protected static FeatureMatcherSetI unmarshalFilter(
+          String featureType, MatcherSet matcherSetModel)
+  {
+    FeatureMatcherSetI result = new FeatureMatcherSet();
+    try
+    {
+      unmarshalFilterConditions(result, matcherSetModel, true);
+    } catch (IllegalStateException e)
+    {
+      // mixing AND and OR conditions perhaps
+      System.err.println(
+              String.format("Error reading filter conditions for '%s': %s",
+                      featureType, e.getMessage()));
+      // return as much as was parsed up to the error
+    }
+  
+    return result;
+  }
+
+  /**
+   * Adds feature match conditions to matcherSet as unmarshalled from XML
+   * (possibly recursively for compound conditions)
+   * 
+   * @param matcherSet
+   * @param matcherSetModel
+   * @param and
+   *          if true, multiple conditions are AND-ed, else they are OR-ed
+   * @throws IllegalStateException
+   *           if AND and OR conditions are mixed
+   */
+  protected static void unmarshalFilterConditions(
+          FeatureMatcherSetI matcherSet, MatcherSet matcherSetModel,
+          boolean and)
+  {
+    MatchCondition mc = matcherSetModel.getMatchCondition();
+    if (mc != null)
+    {
+      /*
+       * single condition
+       */
+      FeatureMatcherByType filterBy = mc.getBy();
+      Condition cond = Condition.fromString(mc.getCondition());
+      String pattern = mc.getValue();
+      FeatureMatcherI matchCondition = null;
+      if (filterBy == FeatureMatcherByType.BYLABEL)
+      {
+        matchCondition = FeatureMatcher.byLabel(cond, pattern);
+      }
+      else if (filterBy == FeatureMatcherByType.BYSCORE)
+      {
+        matchCondition = FeatureMatcher.byScore(cond, pattern);
+  
+      }
+      else if (filterBy == FeatureMatcherByType.BYATTRIBUTE)
+      {
+        String[] attNames = mc.getAttributeName();
+        matchCondition = FeatureMatcher.byAttribute(cond, pattern,
+                attNames);
+      }
+  
+      /*
+       * note this throws IllegalStateException if AND-ing to a 
+       * previously OR-ed compound condition, or vice versa
+       */
+      if (and)
+      {
+        matcherSet.and(matchCondition);
+      }
+      else
+      {
+        matcherSet.or(matchCondition);
+      }
+    }
+    else
+    {
+      /*
+       * compound condition
+       */
+      MatcherSet[] matchers = matcherSetModel.getCompoundMatcher()
+              .getMatcherSet();
+      boolean anded = matcherSetModel.getCompoundMatcher().getAnd();
+      if (matchers.length == 2)
+      {
+        unmarshalFilterConditions(matcherSet, matchers[0], anded);
+        unmarshalFilterConditions(matcherSet, matchers[1], anded);
+      }
+      else
+      {
+        System.err.println("Malformed compound filter condition");
+      }
+    }
+  }
+
+  /**
+   * Loads one XML model of a feature colour to a Jalview object
+   * 
+   * @param colourModel
+   * @return
+   */
+  protected static FeatureColourI unmarshalColour(
+          jalview.schemabinding.version2.Colour colourModel)
+  {
+    FeatureColourI colour = null;
+  
+    if (colourModel.hasMax())
+    {
+      Color mincol = null;
+      Color maxcol = null;
+      Color noValueColour = null;
+  
+      try
+      {
+        mincol = new Color(Integer.parseInt(colourModel.getMinRGB(), 16));
+        maxcol = new Color(Integer.parseInt(colourModel.getRGB(), 16));
+      } catch (Exception e)
+      {
+        Cache.log.warn("Couldn't parse out graduated feature color.", e);
+      }
+  
+      NoValueColour noCol = colourModel.getNoValueColour();
+      if (noCol == NoValueColour.MIN)
+      {
+        noValueColour = mincol;
+      }
+      else if (noCol == NoValueColour.MAX)
+      {
+        noValueColour = maxcol;
+      }
+  
+      colour = new FeatureColour(mincol, maxcol, noValueColour,
+              colourModel.getMin(),
+              colourModel.getMax());
+      String[] attributes = colourModel.getAttributeName();
+      if (attributes != null && attributes.length > 0)
+      {
+        colour.setAttributeName(attributes);
+      }
+      if (colourModel.hasAutoScale())
+      {
+        colour.setAutoScaled(colourModel.getAutoScale());
+      }
+      if (colourModel.hasColourByLabel())
+      {
+        colour.setColourByLabel(colourModel.getColourByLabel());
+      }
+      if (colourModel.hasThreshold())
+      {
+        colour.setThreshold(colourModel.getThreshold());
+      }
+      ColourThreshTypeType ttyp = colourModel.getThreshType();
+      if (ttyp != null)
+      {
+        if (ttyp == ColourThreshTypeType.ABOVE)
+        {
+          colour.setAboveThreshold(true);
+        }
+        else if (ttyp == ColourThreshTypeType.BELOW)
+        {
+          colour.setBelowThreshold(true);
+        }
+      }
+    }
+    else
+    {
+      Color color = new Color(Integer.parseInt(colourModel.getRGB(), 16));
+      colour = new FeatureColour(color);
+    }
+  
+    return colour;
+  }
 }