X-Git-Url: http://source.jalview.org/gitweb/?a=blobdiff_plain;f=src%2Fjalview%2Fgui%2FJalview2XML.java;h=d18c9b3ed4e52286be46df211128f8772119c90d;hb=ab67e99fb7336d0f4411488d9f843c720ab6b418;hp=1a82a6826b21719717038efad46e8cf3cba3ccf7;hpb=7c25229cd1f73806388e1d08f3f67f05cd332133;p=jalview.git

diff --git a/src/jalview/gui/Jalview2XML.java b/src/jalview/gui/Jalview2XML.java
index 1a82a68..d18c9b3 100644
--- a/src/jalview/gui/Jalview2XML.java
+++ b/src/jalview/gui/Jalview2XML.java
@@ -20,6 +20,7 @@
  */
 package jalview.gui;
 
+import jalview.analysis.Conservation;
 import jalview.api.FeatureColourI;
 import jalview.api.ViewStyleI;
 import jalview.api.structures.JalviewStructureDisplayI;
@@ -28,6 +29,7 @@ import jalview.datamodel.AlignedCodonFrame;
 import jalview.datamodel.Alignment;
 import jalview.datamodel.AlignmentAnnotation;
 import jalview.datamodel.AlignmentI;
+import jalview.datamodel.GraphLine;
 import jalview.datamodel.PDBEntry;
 import jalview.datamodel.RnaViewerModel;
 import jalview.datamodel.SequenceGroup;
@@ -36,6 +38,9 @@ import jalview.datamodel.StructureViewerModel;
 import jalview.datamodel.StructureViewerModel.StructureData;
 import jalview.ext.varna.RnaModel;
 import jalview.gui.StructureViewer.ViewerType;
+import jalview.io.DataSourceType;
+import jalview.io.FileFormat;
+import jalview.renderer.ResidueShaderI;
 import jalview.schemabinding.version2.AlcodMap;
 import jalview.schemabinding.version2.AlcodonFrame;
 import jalview.schemabinding.version2.Annotation;
@@ -73,7 +78,6 @@ import jalview.schemes.AnnotationColourGradient;
 import jalview.schemes.ColourSchemeI;
 import jalview.schemes.ColourSchemeProperty;
 import jalview.schemes.FeatureColour;
-import jalview.schemes.ResidueColourScheme;
 import jalview.schemes.ResidueProperties;
 import jalview.schemes.UserColourScheme;
 import jalview.structure.StructureSelectionManager;
@@ -83,6 +87,7 @@ import jalview.util.Platform;
 import jalview.util.StringUtils;
 import jalview.util.jarInputStreamProvider;
 import jalview.viewmodel.AlignmentViewport;
+import jalview.viewmodel.ViewportRanges;
 import jalview.viewmodel.seqfeatures.FeatureRendererSettings;
 import jalview.viewmodel.seqfeatures.FeaturesDisplayed;
 import jalview.ws.jws2.Jws2Discoverer;
@@ -108,6 +113,7 @@ import java.lang.reflect.InvocationTargetException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -125,7 +131,6 @@ import java.util.jar.JarInputStream;
 import java.util.jar.JarOutputStream;
 
 import javax.swing.JInternalFrame;
-import javax.swing.JOptionPane;
 import javax.swing.SwingUtilities;
 
 import org.exolab.castor.xml.Marshaller;
@@ -165,7 +170,9 @@ public class Jalview2XML
    */
   Map<String, SequenceI> seqRefIds = null;
 
-  Vector<Object[]> frefedSequence = null;
+  Map<String, SequenceI> incompleteSeqs = null;
+
+  List<SeqFref> frefedSequence = null;
 
   boolean raiseGUI = true; // whether errors are raised in dialog boxes or not
 
@@ -173,13 +180,13 @@ public class Jalview2XML
    * Map of reconstructed AlignFrame objects that appear to have come from
    * SplitFrame objects (have a dna/protein complement view).
    */
-  private Map<Viewport, AlignFrame> splitFrameCandidates = new HashMap<Viewport, AlignFrame>();
+  private Map<Viewport, AlignFrame> splitFrameCandidates = new HashMap<>();
 
   /*
    * Map from displayed rna structure models to their saved session state jar
    * entry names
    */
-  private Map<RnaModel, String> rnaSessions = new HashMap<RnaModel, String>();
+  private Map<RnaModel, String> rnaSessions = new HashMap<>();
 
   /**
    * create/return unique hash string for sq
@@ -220,6 +227,10 @@ public class Jalview2XML
       {
         seqsToIds.clear();
       }
+      if (incompleteSeqs != null)
+      {
+        incompleteSeqs.clear();
+      }
       // seqRefIds = null;
       // seqsToIds = null;
     }
@@ -236,11 +247,19 @@ public class Jalview2XML
   {
     if (seqsToIds == null)
     {
-      seqsToIds = new IdentityHashMap<SequenceI, String>();
+      seqsToIds = new IdentityHashMap<>();
     }
     if (seqRefIds == null)
     {
-      seqRefIds = new HashMap<String, SequenceI>();
+      seqRefIds = new HashMap<>();
+    }
+    if (incompleteSeqs == null)
+    {
+      incompleteSeqs = new HashMap<>();
+    }
+    if (frefedSequence == null)
+    {
+      frefedSequence = new ArrayList<>();
     }
   }
 
@@ -253,78 +272,185 @@ public class Jalview2XML
     this.raiseGUI = raiseGUI;
   }
 
+  /**
+   * base class for resolving forward references to sequences by their ID
+   * 
+   * @author jprocter
+   *
+   */
+  abstract class SeqFref
+  {
+    String sref;
+
+    String type;
+
+    public SeqFref(String _sref, String type)
+    {
+      sref = _sref;
+      this.type = type;
+    }
+
+    public String getSref()
+    {
+      return sref;
+    }
+
+    public SequenceI getSrefSeq()
+    {
+      return seqRefIds.get(sref);
+    }
+
+    public boolean isResolvable()
+    {
+      return seqRefIds.get(sref) != null;
+    }
+
+    public SequenceI getSrefDatasetSeq()
+    {
+      SequenceI sq = seqRefIds.get(sref);
+      if (sq != null)
+      {
+        while (sq.getDatasetSequence() != null)
+        {
+          sq = sq.getDatasetSequence();
+        }
+      }
+      return sq;
+    }
+
+    /**
+     * @return true if the forward reference was fully resolved
+     */
+    abstract boolean resolve();
+
+    @Override
+    public String toString()
+    {
+      return type + " reference to " + sref;
+    }
+  }
+
+  /**
+   * create forward reference for a mapping
+   * 
+   * @param sref
+   * @param _jmap
+   * @return
+   */
+  public SeqFref newMappingRef(final String sref,
+          final jalview.datamodel.Mapping _jmap)
+  {
+    SeqFref fref = new SeqFref(sref, "Mapping")
+    {
+      public jalview.datamodel.Mapping jmap = _jmap;
+
+      @Override
+      boolean resolve()
+      {
+        SequenceI seq = getSrefDatasetSeq();
+        if (seq == null)
+        {
+          return false;
+        }
+        jmap.setTo(seq);
+        return true;
+      }
+    };
+    return fref;
+  }
+
+  public SeqFref newAlcodMapRef(final String sref,
+          final AlignedCodonFrame _cf, final jalview.datamodel.Mapping _jmap)
+  {
+
+    SeqFref fref = new SeqFref(sref, "Codon Frame")
+    {
+      AlignedCodonFrame cf = _cf;
+
+      public jalview.datamodel.Mapping mp = _jmap;
+
+      @Override
+      public boolean isResolvable()
+      {
+        return super.isResolvable() && mp.getTo() != null;
+      };
+
+      @Override
+      boolean resolve()
+      {
+        SequenceI seq = getSrefDatasetSeq();
+        if (seq == null)
+        {
+          return false;
+        }
+        cf.addMap(seq, mp.getTo(), mp.getMap());
+        return true;
+      }
+    };
+    return fref;
+  }
+
   public void resolveFrefedSequences()
   {
-    if (frefedSequence.size() > 0)
+    Iterator<SeqFref> nextFref = frefedSequence.iterator();
+    int toresolve = frefedSequence.size();
+    int unresolved = 0, failedtoresolve = 0;
+    while (nextFref.hasNext())
     {
-      int r = 0, rSize = frefedSequence.size();
-      while (r < rSize)
+      SeqFref ref = nextFref.next();
+      if (ref.isResolvable())
       {
-        Object[] ref = frefedSequence.elementAt(r);
-        if (ref != null)
+        try
         {
-          String sref = (String) ref[0];
-          if (seqRefIds.containsKey(sref))
+          if (ref.resolve())
           {
-            if (ref[1] instanceof jalview.datamodel.Mapping)
-            {
-              SequenceI seq = seqRefIds.get(sref);
-              while (seq.getDatasetSequence() != null)
-              {
-                seq = seq.getDatasetSequence();
-              }
-              ((jalview.datamodel.Mapping) ref[1]).setTo(seq);
-            }
-            else
-            {
-              if (ref[1] instanceof jalview.datamodel.AlignedCodonFrame)
-              {
-                SequenceI seq = seqRefIds.get(sref);
-                while (seq.getDatasetSequence() != null)
-                {
-                  seq = seq.getDatasetSequence();
-                }
-                if (ref[2] != null
-                        && ref[2] instanceof jalview.datamodel.Mapping)
-                {
-                  jalview.datamodel.Mapping mp = (jalview.datamodel.Mapping) ref[2];
-                  ((jalview.datamodel.AlignedCodonFrame) ref[1]).addMap(
-                          seq, mp.getTo(), mp.getMap());
-                }
-                else
-                {
-                  System.err
-                          .println("IMPLEMENTATION ERROR: Unimplemented forward sequence references for AlcodonFrames involving "
-                                  + ref[2].getClass() + " type objects.");
-                }
-              }
-              else
-              {
-                System.err
-                        .println("IMPLEMENTATION ERROR: Unimplemented forward sequence references for "
-                                + ref[1].getClass() + " type objects.");
-              }
-            }
-            frefedSequence.remove(r);
-            rSize--;
+            nextFref.remove();
           }
           else
           {
-            System.err
-                    .println("IMPLEMENTATION WARNING: Unresolved forward reference for hash string "
-                            + ref[0]
-                            + " with objecttype "
-                            + ref[1].getClass());
-            r++;
+            failedtoresolve++;
           }
+        } catch (Exception x)
+        {
+          System.err
+                  .println("IMPLEMENTATION ERROR: Failed to resolve forward reference for sequence "
+                          + ref.getSref());
+          x.printStackTrace();
+          failedtoresolve++;
         }
-        else
+      }
+      else
+      {
+        unresolved++;
+      }
+    }
+    if (unresolved > 0)
+    {
+      System.err.println("Jalview Project Import: There were " + unresolved
+              + " forward references left unresolved on the stack.");
+    }
+    if (failedtoresolve > 0)
+    {
+      System.err.println("SERIOUS! " + failedtoresolve
+              + " resolvable forward references failed to resolve.");
+    }
+    if (incompleteSeqs != null && incompleteSeqs.size() > 0)
+    {
+      System.err.println("Jalview Project Import: There are "
+              + incompleteSeqs.size()
+              + " sequences which may have incomplete metadata.");
+      if (incompleteSeqs.size() < 10)
+      {
+        for (SequenceI s : incompleteSeqs.values())
         {
-          // empty reference
-          frefedSequence.remove(r);
-          rSize--;
+          System.err.println(s.toString());
         }
       }
+      else
+      {
+        System.err
+                .println("Too many to report. Skipping output of incomplete sequences.");
+      }
     }
   }
 
@@ -332,9 +458,9 @@ public class Jalview2XML
    * This maintains a map of viewports, the key being the seqSetId. Important to
    * set historyItem and redoList for multiple views
    */
-  Map<String, AlignViewport> viewportsAdded = new HashMap<String, AlignViewport>();
+  Map<String, AlignViewport> viewportsAdded = new HashMap<>();
 
-  Map<String, AlignmentAnnotation> annotationIds = new HashMap<String, AlignmentAnnotation>();
+  Map<String, AlignmentAnnotation> annotationIds = new HashMap<>();
 
   String uniqueSetSuffix = "";
 
@@ -396,8 +522,21 @@ public class Jalview2XML
     {
       return;
     }
+    saveAllFrames(Arrays.asList(frames), jout);
+  }
 
-    Hashtable<String, AlignFrame> dsses = new Hashtable<String, AlignFrame>();
+  /**
+   * core method for storing state for a set of AlignFrames.
+   * 
+   * @param frames
+   *          - frames involving all data to be exported (including containing
+   *          splitframes)
+   * @param jout
+   *          - project output stream
+   */
+  private void saveAllFrames(List<AlignFrame> frames, JarOutputStream jout)
+  {
+    Hashtable<String, AlignFrame> dsses = new Hashtable<>();
 
     /*
      * ensure cached data is clear before starting
@@ -412,13 +551,13 @@ public class Jalview2XML
       // NOTE UTF-8 MUST BE USED FOR WRITING UNICODE CHARS
       // //////////////////////////////////////////////////
 
-      List<String> shortNames = new ArrayList<String>();
-      List<String> viewIds = new ArrayList<String>();
+      List<String> shortNames = new ArrayList<>();
+      List<String> viewIds = new ArrayList<>();
 
       // REVERSE ORDER
-      for (int i = frames.length - 1; i > -1; i--)
+      for (int i = frames.size() - 1; i > -1; i--)
       {
-        AlignFrame af = frames[i];
+        AlignFrame af = frames.get(i);
         // skip ?
         if (skipList != null
                 && skipList
@@ -521,30 +660,20 @@ public class Jalview2XML
   {
     try
     {
-      int ap = 0;
-      int apSize = af.alignPanels.size();
       FileOutputStream fos = new FileOutputStream(jarFile);
       JarOutputStream jout = new JarOutputStream(fos);
-      Hashtable<String, AlignFrame> dsses = new Hashtable<String, AlignFrame>();
-      List<String> viewIds = new ArrayList<String>();
+      List<AlignFrame> frames = new ArrayList<>();
 
-      for (AlignmentPanel apanel : af.alignPanels)
+      // resolve splitframes
+      if (af.getViewport().getCodingComplement() != null)
       {
-        String jfileName = apSize == 1 ? fileName : fileName + ap;
-        ap++;
-        if (!jfileName.endsWith(".xml"))
-        {
-          jfileName = jfileName + ".xml";
-        }
-        saveState(apanel, jfileName, jout, viewIds);
-        String dssid = getDatasetIdRef(af.getViewport().getAlignment()
-                .getDataset());
-        if (!dsses.containsKey(dssid))
-        {
-          dsses.put(dssid, af);
-        }
+        frames = ((SplitFrame) af.getSplitViewContainer()).getAlignFrames();
+      }
+      else
+      {
+        frames.add(af);
       }
-      writeDatasetFor(dsses, fileName, jout);
+      saveAllFrames(frames, jout);
       try
       {
         jout.flush();
@@ -619,14 +748,15 @@ public class Jalview2XML
   {
     if (viewIds == null)
     {
-      viewIds = new ArrayList<String>();
+      viewIds = new ArrayList<>();
     }
 
     initSeqRefs();
 
-    List<UserColourScheme> userColours = new ArrayList<UserColourScheme>();
+    List<UserColourScheme> userColours = new ArrayList<>();
 
     AlignViewport av = ap.av;
+    ViewportRanges vpRanges = av.getRanges();
 
     JalviewModel object = new JalviewModel();
     object.setVamsasModel(new jalview.schemabinding.version2.VamsasModel());
@@ -677,38 +807,43 @@ public class Jalview2XML
     }
 
     JSeq jseq;
-    Set<String> calcIdSet = new HashSet<String>();
-
+    Set<String> calcIdSet = new HashSet<>();
+    // record the set of vamsas sequence XML POJO we create.
+    HashMap<String, Sequence> vamsasSetIds = new HashMap<>();
     // SAVE SEQUENCES
     for (final SequenceI jds : rjal.getSequences())
     {
       final SequenceI jdatasq = jds.getDatasetSequence() == null ? jds
               : jds.getDatasetSequence();
       String id = seqHash(jds);
-
-      if (seqRefIds.get(id) != null)
-      {
-        // This happens for two reasons: 1. multiple views are being serialised.
-        // 2. the hashCode has collided with another sequence's code. This DOES
-        // HAPPEN! (PF00072.15.stk does this)
-        // JBPNote: Uncomment to debug writing out of files that do not read
-        // back in due to ArrayOutOfBoundExceptions.
-        // System.err.println("vamsasSeq backref: "+id+"");
-        // System.err.println(jds.getName()+"
-        // "+jds.getStart()+"-"+jds.getEnd()+" "+jds.getSequenceAsString());
-        // System.err.println("Hashcode: "+seqHash(jds));
-        // SequenceI rsq = (SequenceI) seqRefIds.get(id + "");
-        // System.err.println(rsq.getName()+"
-        // "+rsq.getStart()+"-"+rsq.getEnd()+" "+rsq.getSequenceAsString());
-        // System.err.println("Hashcode: "+seqHash(rsq));
-      }
-      else
-      {
-        vamsasSeq = createVamsasSequence(id, jds);
-        vamsasSet.addSequence(vamsasSeq);
-        seqRefIds.put(id, jds);
+      if (vamsasSetIds.get(id) == null)
+      {
+        if (seqRefIds.get(id) != null && !storeDS)
+        {
+          // This happens for two reasons: 1. multiple views are being
+          // serialised.
+          // 2. the hashCode has collided with another sequence's code. This
+          // DOES
+          // HAPPEN! (PF00072.15.stk does this)
+          // JBPNote: Uncomment to debug writing out of files that do not read
+          // back in due to ArrayOutOfBoundExceptions.
+          // System.err.println("vamsasSeq backref: "+id+"");
+          // System.err.println(jds.getName()+"
+          // "+jds.getStart()+"-"+jds.getEnd()+" "+jds.getSequenceAsString());
+          // System.err.println("Hashcode: "+seqHash(jds));
+          // SequenceI rsq = (SequenceI) seqRefIds.get(id + "");
+          // System.err.println(rsq.getName()+"
+          // "+rsq.getStart()+"-"+rsq.getEnd()+" "+rsq.getSequenceAsString());
+          // System.err.println("Hashcode: "+seqHash(rsq));
+        }
+        else
+        {
+          vamsasSeq = createVamsasSequence(id, jds);
+          vamsasSet.addSequence(vamsasSeq);
+          vamsasSetIds.put(id, vamsasSeq);
+          seqRefIds.put(id, jds);
+        }
       }
-
       jseq = new JSeq();
       jseq.setStart(jds.getStart());
       jseq.setEnd(jds.getEnd());
@@ -727,8 +862,7 @@ public class Jalview2XML
           if (av.isHiddenRepSequence(jds))
           {
             jalview.datamodel.SequenceI[] reps = av
-                    .getRepresentedSequences(jds)
-                    .getSequencesInOrder(rjal);
+                    .getRepresentedSequences(jds).getSequencesInOrder(rjal);
 
             for (int h = 0; h < reps.length; h++)
             {
@@ -852,7 +986,7 @@ public class Jalview2XML
             pdb.setFile(matchedFile); // entry.getFile());
             if (pdbfiles == null)
             {
-              pdbfiles = new ArrayList<String>();
+              pdbfiles = new ArrayList<>();
             }
 
             if (!pdbfiles.contains(pdbId))
@@ -862,17 +996,16 @@ public class Jalview2XML
             }
           }
 
-          if (entry.getProperty() != null && !entry.getProperty().isEmpty())
+          Enumeration<String> props = entry.getProperties();
+          if (props.hasMoreElements())
           {
             PdbentryItem item = new PdbentryItem();
-            Hashtable properties = entry.getProperty();
-            Enumeration en2 = properties.keys();
-            while (en2.hasMoreElements())
+            while (props.hasMoreElements())
             {
               Property prop = new Property();
-              String key = en2.nextElement().toString();
+              String key = props.nextElement();
               prop.setName(key);
-              prop.setValue(properties.get(key).toString());
+              prop.setValue(entry.getProperty(key).toString());
               item.addProperty(prop);
             }
             pdb.addPdbentryItem(item);
@@ -892,7 +1025,8 @@ public class Jalview2XML
       jal = av.getAlignment();
     }
     // SAVE MAPPINGS
-    if (jal.getCodonFrames() != null)
+    // FOR DATASET
+    if (storeDS && jal.getCodonFrames() != null)
     {
       List<AlignedCodonFrame> jac = jal.getCodonFrames();
       for (AlignedCodonFrame acf : jac)
@@ -973,7 +1107,7 @@ public class Jalview2XML
               Tree tree = new Tree();
               tree.setTitle(tp.getTitle());
               tree.setCurrentTree((av.currentTree == tp.getTree()));
-              tree.setNewick(tp.getTree().toString());
+              tree.setNewick(tp.getTree().print());
               tree.setThreshold(tp.treeCanvas.threshold);
 
               tree.setFitToWindow(tp.fitToWindow.getState());
@@ -1001,7 +1135,7 @@ public class Jalview2XML
     /**
      * store forward refs from an annotationRow to any groups
      */
-    IdentityHashMap<SequenceGroup, String> groupRefs = new IdentityHashMap<SequenceGroup, String>();
+    IdentityHashMap<SequenceGroup, String> groupRefs = new IdentityHashMap<>();
     if (storeDS)
     {
       for (SequenceI sq : jal.getSequences())
@@ -1043,38 +1177,43 @@ public class Jalview2XML
           // group has references so set its ID field
           jGroup.setId(groupRefs.get(sg));
         }
-        if (sg.cs != null)
+        ColourSchemeI colourScheme = sg.getColourScheme();
+        if (colourScheme != null)
         {
-          if (sg.cs.conservationApplied())
+          ResidueShaderI groupColourScheme = sg
+                  .getGroupColourScheme();
+          if (groupColourScheme.conservationApplied())
           {
-            jGroup.setConsThreshold(sg.cs.getConservationInc());
+            jGroup.setConsThreshold(groupColourScheme.getConservationInc());
 
-            if (sg.cs instanceof jalview.schemes.UserColourScheme)
+            if (colourScheme instanceof jalview.schemes.UserColourScheme)
             {
-              jGroup.setColour(setUserColourScheme(sg.cs, userColours, jms));
+              jGroup.setColour(setUserColourScheme(colourScheme,
+                      userColours, jms));
             }
             else
             {
-              jGroup.setColour(ColourSchemeProperty.getColourName(sg.cs));
+              jGroup.setColour(colourScheme.getSchemeName());
             }
           }
-          else if (sg.cs instanceof jalview.schemes.AnnotationColourGradient)
+          else if (colourScheme instanceof jalview.schemes.AnnotationColourGradient)
           {
             jGroup.setColour("AnnotationColourGradient");
             jGroup.setAnnotationColours(constructAnnotationColours(
-                    (jalview.schemes.AnnotationColourGradient) sg.cs,
+                    (jalview.schemes.AnnotationColourGradient) colourScheme,
                     userColours, jms));
           }
-          else if (sg.cs instanceof jalview.schemes.UserColourScheme)
+          else if (colourScheme instanceof jalview.schemes.UserColourScheme)
           {
-            jGroup.setColour(setUserColourScheme(sg.cs, userColours, jms));
+            jGroup.setColour(setUserColourScheme(colourScheme,
+                    userColours, jms));
           }
           else
           {
-            jGroup.setColour(ColourSchemeProperty.getColourName(sg.cs));
+            jGroup.setColour(colourScheme.getSchemeName());
           }
 
-          jGroup.setPidThreshold(sg.cs.getThreshold());
+          jGroup.setPidThreshold(groupColourScheme.getThreshold());
         }
 
         jGroup.setOutlineColour(sg.getOutlineColour().getRGB());
@@ -1133,8 +1272,8 @@ public class Jalview2XML
       view.setWidth(size.width);
       view.setHeight(size.height);
 
-      view.setStartRes(av.startRes);
-      view.setStartSeq(av.startSeq);
+      view.setStartRes(vpRanges.getStartRes());
+      view.setStartSeq(vpRanges.getStartSeq());
 
       if (av.getGlobalColourScheme() instanceof jalview.schemes.UserColourScheme)
       {
@@ -1157,23 +1296,20 @@ public class Jalview2XML
                 .getGlobalColourScheme()));
       }
 
+      ResidueShaderI vcs = av.getResidueShading();
       ColourSchemeI cs = av.getGlobalColourScheme();
 
       if (cs != null)
       {
-        if (cs.conservationApplied())
+        if (vcs.conservationApplied())
         {
-          view.setConsThreshold(cs.getConservationInc());
+          view.setConsThreshold(vcs.getConservationInc());
           if (cs instanceof jalview.schemes.UserColourScheme)
           {
             view.setBgColour(setUserColourScheme(cs, userColours, jms));
           }
         }
-
-        if (cs instanceof ResidueColourScheme)
-        {
-          view.setPidThreshold(cs.getThreshold());
-        }
+        view.setPidThreshold(vcs.getThreshold());
       }
 
       view.setConservationSelected(av.getConservationSelected());
@@ -1213,14 +1349,13 @@ public class Jalview2XML
                 .getFeatureRenderer().getRenderOrder()
                 .toArray(new String[0]);
 
-        Vector<String> settingsAdded = new Vector<String>();
+        Vector<String> settingsAdded = new Vector<>();
         if (renderOrder != null)
         {
           for (String featureType : renderOrder)
           {
             FeatureColourI fcol = ap.getSeqPanel().seqCanvas
-                    .getFeatureRenderer()
-                    .getFeatureStyle(featureType);
+                    .getFeatureRenderer().getFeatureStyle(featureType);
             Setting setting = new Setting();
             setting.setType(featureType);
             if (!fcol.isSimpleColour())
@@ -1233,8 +1368,8 @@ public class Jalview2XML
               setting.setAutoScale(fcol.isAutoScaled());
               setting.setThreshold(fcol.getThreshold());
               // -1 = No threshold, 0 = Below, 1 = Above
-              setting.setThreshstate(fcol.isAboveThreshold() ? 1
-                      : (fcol.isBelowThreshold() ? 0 : -1));
+              setting.setThreshstate(fcol.isAboveThreshold() ? 1 : (fcol
+                      .isBelowThreshold() ? 0 : -1));
             }
             else
             {
@@ -1256,9 +1391,8 @@ public class Jalview2XML
 
         // is groups actually supposed to be a map here ?
         Iterator<String> en = ap.getSeqPanel().seqCanvas
-                .getFeatureRenderer()
-                .getFeatureGroups().iterator();
-        Vector<String> groupsAdded = new Vector<String>();
+                .getFeatureRenderer().getFeatureGroups().iterator();
+        Vector<String> groupsAdded = new Vector<>();
         while (en.hasNext())
         {
           String grp = en.next();
@@ -1279,18 +1413,18 @@ public class Jalview2XML
 
       if (av.hasHiddenColumns())
       {
-        if (av.getColumnSelection() == null
-                || av.getColumnSelection().getHiddenColumns() == null)
+        jalview.datamodel.HiddenColumns hidden = av.getAlignment()
+                .getHiddenColumns();
+        if (hidden == null)
         {
           warn("REPORT BUG: avoided null columnselection bug (DMAM reported). Please contact Jim about this.");
         }
         else
         {
-          for (int c = 0; c < av.getColumnSelection().getHiddenColumns()
-                  .size(); c++)
+          ArrayList<int[]> hiddenRegions = hidden
+                  .getHiddenColumnsCopyAsList();
+          for (int[] region : hiddenRegions)
           {
-            int[] region = av.getColumnSelection().getHiddenColumns()
-                    .get(c);
             HiddenColumns hc = new HiddenColumns();
             hc.setStart(region[0]);
             hc.setEnd(region[1]);
@@ -1579,6 +1713,15 @@ public class Jalview2XML
     return matchedFile;
   }
 
+  /**
+   * Populates the AnnotationColours xml for save. This captures the settings of
+   * the options in the 'Colour by Annotation' dialog.
+   * 
+   * @param acg
+   * @param userColours
+   * @param jms
+   * @return
+   */
   private AnnotationColours constructAnnotationColours(
           AnnotationColourGradient acg, List<UserColourScheme> userColours,
           JalviewModelSequence jms)
@@ -1586,16 +1729,16 @@ public class Jalview2XML
     AnnotationColours ac = new AnnotationColours();
     ac.setAboveThreshold(acg.getAboveThreshold());
     ac.setThreshold(acg.getAnnotationThreshold());
-    ac.setAnnotation(acg.getAnnotation());
-    if (acg.getBaseColour() instanceof jalview.schemes.UserColourScheme)
+    // 2.10.2 save annotationId (unique) not annotation label
+    ac.setAnnotation(acg.getAnnotation().annotationId);
+    if (acg.getBaseColour() instanceof UserColourScheme)
     {
       ac.setColourScheme(setUserColourScheme(acg.getBaseColour(),
               userColours, jms));
     }
     else
     {
-      ac.setColourScheme(ColourSchemeProperty.getColourName(acg
-              .getBaseColour()));
+      ac.setColourScheme(ColourSchemeProperty.getColourName(acg.getBaseColour()));
     }
 
     ac.setMaxColour(acg.getMaxColour().getRGB());
@@ -1940,16 +2083,17 @@ public class Jalview2XML
     if (jds.getDatasetSequence() != null)
     {
       vamsasSeq.setDsseqid(seqHash(jds.getDatasetSequence()));
-      if (jds.getDatasetSequence().getDBRefs() != null)
-      {
-        dbrefs = jds.getDatasetSequence().getDBRefs();
-      }
     }
     else
     {
-      vamsasSeq.setDsseqid(id); // so we can tell which sequences really are
+      // seqId==dsseqid so we can tell which sequences really are
       // dataset sequences only
+      vamsasSeq.setDsseqid(id);
       dbrefs = jds.getDBRefs();
+      if (parentseq == null)
+      {
+        parentseq = jds;
+      }
     }
     if (dbrefs != null)
     {
@@ -2001,38 +2145,32 @@ public class Jalview2XML
       if (jmp.getTo() != null)
       {
         MappingChoice mpc = new MappingChoice();
-        if (recurse
-                && (parentseq != jmp.getTo() || parentseq
-                        .getDatasetSequence() != jmp.getTo()))
+
+        // check/create ID for the sequence referenced by getTo()
+
+        String jmpid = "";
+        SequenceI ps = null;
+        if (parentseq != jmp.getTo()
+                && parentseq.getDatasetSequence() != jmp.getTo())
         {
-          mpc.setSequence(createVamsasSequence(false, seqHash(jmp.getTo()),
-                  jmp.getTo(), jds));
+          // chaining dbref rather than a handshaking one
+          jmpid = seqHash(ps = jmp.getTo());
         }
         else
         {
-          String jmpid = "";
-          SequenceI ps = null;
-          if (parentseq != jmp.getTo()
-                  && parentseq.getDatasetSequence() != jmp.getTo())
-          {
-            // chaining dbref rather than a handshaking one
-            jmpid = seqHash(ps = jmp.getTo());
-          }
-          else
-          {
-            jmpid = seqHash(ps = parentseq);
-          }
-          mpc.setDseqFor(jmpid);
-          if (!seqRefIds.containsKey(mpc.getDseqFor()))
-          {
-            jalview.bin.Cache.log.debug("creatign new DseqFor ID");
-            seqRefIds.put(mpc.getDseqFor(), ps);
-          }
-          else
-          {
-            jalview.bin.Cache.log.debug("reusing DseqFor ID");
-          }
+          jmpid = seqHash(ps = parentseq);
         }
+        mpc.setDseqFor(jmpid);
+        if (!seqRefIds.containsKey(mpc.getDseqFor()))
+        {
+          jalview.bin.Cache.log.debug("creatign new DseqFor ID");
+          seqRefIds.put(mpc.getDseqFor(), ps);
+        }
+        else
+        {
+          jalview.bin.Cache.log.debug("reusing DseqFor ID");
+        }
+
         mp.setMappingChoice(mpc);
       }
     }
@@ -2151,7 +2289,7 @@ public class Jalview2XML
     try
     {
       // create list to store references for any new Jmol viewers created
-      newStructureViewers = new Vector<JalviewStructureDisplayI>();
+      newStructureViewers = new Vector<>();
       // UNMARSHALLER SEEMS TO CLOSE JARINPUTSTREAM, MOST ANNOYING
       // Workaround is to make sure caller implements the JarInputStreamProvider
       // interface
@@ -2241,15 +2379,11 @@ public class Jalview2XML
     }
     if (seqRefIds == null)
     {
-      seqRefIds = new HashMap<String, SequenceI>();
-    }
-    if (frefedSequence == null)
-    {
-      frefedSequence = new Vector<Object[]>();
+      initSeqRefs();
     }
-
     AlignFrame af = null, _af = null;
-    Map<String, AlignFrame> gatherToThisFrame = new HashMap<String, AlignFrame>();
+    IdentityHashMap<AlignmentI, AlignmentI> importedDatasets = new IdentityHashMap<>();
+    Map<String, AlignFrame> gatherToThisFrame = new HashMap<>();
     final String file = jprovider.getFilename();
     try
     {
@@ -2276,13 +2410,24 @@ public class Jalview2XML
           if (true) // !skipViewport(object))
           {
             _af = loadFromObject(object, file, true, jprovider);
-            if (object.getJalviewModelSequence().getViewportCount() > 0)
+            if (_af != null
+                    && object.getJalviewModelSequence().getViewportCount() > 0)
             {
-              af = _af;
-              if (af.viewport.isGatherViewsHere())
+              if (af == null)
+              {
+                // store a reference to the first view
+                af = _af;
+              }
+              if (_af.viewport.isGatherViewsHere())
               {
-                gatherToThisFrame.put(af.viewport.getSequenceSetId(), af);
+                // if this is a gathered view, keep its reference since
+                // after gathering views, only this frame will remain
+                af = _af;
+                gatherToThisFrame.put(_af.viewport.getSequenceSetId(), _af);
               }
+              // Save dataset to register mappings once all resolved
+              importedDatasets.put(af.viewport.getAlignment().getDataset(),
+                      af.viewport.getAlignment().getDataset());
             }
           }
           entryCount++;
@@ -2338,11 +2483,6 @@ public class Jalview2XML
       e.printStackTrace();
     }
 
-    if (Desktop.instance != null)
-    {
-      Desktop.instance.stopLoading();
-    }
-
     /*
      * Regather multiple views (with the same sequence set id) to the frame (if
      * any) that is flagged as the one to gather to, i.e. convert them to tabbed
@@ -2356,11 +2496,24 @@ public class Jalview2XML
     }
 
     restoreSplitFrames();
-
+    for (AlignmentI ds : importedDatasets.keySet())
+    {
+      if (ds.getCodonFrames() != null)
+      {
+        StructureSelectionManager.getStructureSelectionManager(
+                Desktop.instance).registerMappings(ds.getCodonFrames());
+      }
+    }
     if (errorMessage != null)
     {
       reportErrors();
     }
+
+    if (Desktop.instance != null)
+    {
+      Desktop.instance.stopLoading();
+    }
+
     return af;
   }
 
@@ -2371,9 +2524,9 @@ public class Jalview2XML
    */
   protected void restoreSplitFrames()
   {
-    List<SplitFrame> gatherTo = new ArrayList<SplitFrame>();
-    List<AlignFrame> addedToSplitFrames = new ArrayList<AlignFrame>();
-    Map<String, AlignFrame> dna = new HashMap<String, AlignFrame>();
+    List<SplitFrame> gatherTo = new ArrayList<>();
+    List<AlignFrame> addedToSplitFrames = new ArrayList<>();
+    Map<String, AlignFrame> dna = new HashMap<>();
 
     /*
      * Identify the DNA alignments
@@ -2405,6 +2558,8 @@ public class Jalview2XML
           SplitFrame sf = createSplitFrame(dnaFrame, af);
           addedToSplitFrames.add(dnaFrame);
           addedToSplitFrames.add(af);
+          dnaFrame.setMenusForViewport();
+          af.setMenusForViewport();
           if (af.viewport.isGatherViewsHere())
           {
             gatherTo.add(sf);
@@ -2426,6 +2581,7 @@ public class Jalview2XML
         Viewport view = candidate.getKey();
         Desktop.addInternalFrame(af, view.getTitle(), view.getWidth(),
                 view.getHeight());
+        af.setMenusForViewport();
         System.err.println("Failed to restore view " + view.getTitle()
                 + " to split frame");
       }
@@ -2495,10 +2651,12 @@ public class Jalview2XML
           @Override
           public void run()
           {
-            JOptionPane.showInternalMessageDialog(Desktop.desktop,
-                    finalErrorMessage, "Error "
-                            + (saving ? "saving" : "loading")
-                            + " Jalview file", JOptionPane.WARNING_MESSAGE);
+            JvOptionPane
+                    .showInternalMessageDialog(Desktop.desktop,
+                            finalErrorMessage, "Error "
+                                    + (saving ? "saving" : "loading")
+                                    + " Jalview file",
+                            JvOptionPane.WARNING_MESSAGE);
           }
         });
       }
@@ -2510,7 +2668,7 @@ public class Jalview2XML
     errorMessage = null;
   }
 
-  Map<String, String> alreadyLoadedPDB = new HashMap<String, String>();
+  Map<String, String> alreadyLoadedPDB = new HashMap<>();
 
   /**
    * when set, local views will be updated from view stored in JalviewXML
@@ -2529,14 +2687,16 @@ public class Jalview2XML
    * @param pdbId
    * @return
    */
-  String loadPDBFile(jarInputStreamProvider jprovider, String pdbId)
+  String loadPDBFile(jarInputStreamProvider jprovider, String pdbId,
+          String origFile)
   {
     if (alreadyLoadedPDB.containsKey(pdbId))
     {
       return alreadyLoadedPDB.get(pdbId).toString();
     }
 
-    String tempFile = copyJarEntry(jprovider, pdbId, "jalview_pdb");
+    String tempFile = copyJarEntry(jprovider, pdbId, "jalview_pdb",
+            origFile);
     if (tempFile != null)
     {
       alreadyLoadedPDB.put(pdbId, tempFile);
@@ -2553,14 +2713,26 @@ public class Jalview2XML
    * @param prefix
    *          a prefix for the temporary file name, must be at least three
    *          characters long
+   * @param origFile
+   *          null or original file - so new file can be given the same suffix
+   *          as the old one
    * @return
    */
   protected String copyJarEntry(jarInputStreamProvider jprovider,
-          String jarEntryName, String prefix)
+          String jarEntryName, String prefix, String origFile)
   {
     BufferedReader in = null;
     PrintWriter out = null;
-
+    String suffix = ".tmp";
+    if (origFile == null)
+    {
+      origFile = jarEntryName;
+    }
+    int sfpos = origFile.lastIndexOf(".");
+    if (sfpos > -1 && sfpos < (origFile.length() - 3))
+    {
+      suffix = "." + origFile.substring(sfpos + 1);
+    }
     try
     {
       JarInputStream jin = jprovider.getJarInputStream();
@@ -2578,7 +2750,7 @@ public class Jalview2XML
       if (entry != null)
       {
         in = new BufferedReader(new InputStreamReader(jin, UTF_8));
-        File outFile = File.createTempFile(prefix, ".tmp");
+        File outFile = File.createTempFile(prefix, suffix);
         outFile.deleteOnExit();
         out = new PrintWriter(new FileOutputStream(outFile));
         String data;
@@ -2667,8 +2839,7 @@ public class Jalview2XML
 
     List<SequenceI> hiddenSeqs = null;
 
-
-    List<SequenceI> tmpseqs = new ArrayList<SequenceI>();
+    List<SequenceI> tmpseqs = new ArrayList<>();
 
     boolean multipleView = false;
     SequenceI referenceseqForView = null;
@@ -2681,10 +2852,38 @@ public class Jalview2XML
       SequenceI tmpSeq = seqRefIds.get(seqId);
       if (tmpSeq != null)
       {
+        if (!incompleteSeqs.containsKey(seqId))
+        {
+          // may not need this check, but keep it for at least 2.9,1 release
+          if (tmpSeq.getStart() != jseqs[i].getStart()
+                  || tmpSeq.getEnd() != jseqs[i].getEnd())
+          {
+            System.err
+                    .println("Warning JAL-2154 regression: updating start/end for sequence "
+                            + tmpSeq.toString() + " to " + jseqs[i]);
+          }
+        }
+        else
+        {
+          incompleteSeqs.remove(seqId);
+        }
+        if (vamsasSeq.length > vi && vamsasSeq[vi].getId().equals(seqId))
+        {
+          // most likely we are reading a dataset XML document so
+          // update from vamsasSeq section of XML for this sequence
+          tmpSeq.setName(vamsasSeq[vi].getName());
+          tmpSeq.setDescription(vamsasSeq[vi].getDescription());
+          tmpSeq.setSequence(vamsasSeq[vi].getSequence());
+          vi++;
+        }
+        else
+        {
+          // reading multiple views, so vamsasSeq set is a subset of JSeq
+          multipleView = true;
+        }
         tmpSeq.setStart(jseqs[i].getStart());
         tmpSeq.setEnd(jseqs[i].getEnd());
         tmpseqs.add(tmpSeq);
-        multipleView = true;
       }
       else
       {
@@ -2708,7 +2907,7 @@ public class Jalview2XML
       {
         if (hiddenSeqs == null)
         {
-          hiddenSeqs = new ArrayList<SequenceI>();
+          hiddenSeqs = new ArrayList<>();
         }
 
         hiddenSeqs.add(tmpSeq);
@@ -2721,7 +2920,39 @@ public class Jalview2XML
     SequenceI[] orderedSeqs = tmpseqs
             .toArray(new SequenceI[tmpseqs.size()]);
 
-    AlignmentI al = new Alignment(orderedSeqs);
+    AlignmentI al = null;
+    // so we must create or recover the dataset alignment before going further
+    // ///////////////////////////////
+    if (vamsasSet.getDatasetId() == null || vamsasSet.getDatasetId() == "")
+    {
+      // older jalview projects do not have a dataset - so creat alignment and
+      // dataset
+      al = new Alignment(orderedSeqs);
+      al.setDataset(null);
+    }
+    else
+    {
+      boolean isdsal = object.getJalviewModelSequence().getViewportCount() == 0;
+      if (isdsal)
+      {
+        // we are importing a dataset record, so
+        // recover reference to an alignment already materialsed as dataset
+        al = getDatasetFor(vamsasSet.getDatasetId());
+      }
+      if (al == null)
+      {
+        // materialse the alignment
+        al = new Alignment(orderedSeqs);
+      }
+      if (isdsal)
+      {
+        addDatasetRef(vamsasSet.getDatasetId(), al);
+      }
+
+      // finally, verify all data in vamsasSet is actually present in al
+      // passing on flag indicating if it is actually a stored dataset
+      recoverDatasetFor(vamsasSet, al, isdsal);
+    }
 
     if (referenceseqForView != null)
     {
@@ -2734,22 +2965,6 @@ public class Jalview2XML
       al.setProperty(ssp.getKey(), ssp.getValue());
     }
 
-    // /
-    // SequenceFeatures are added to the DatasetSequence,
-    // so we must create or recover the dataset before loading features
-    // ///////////////////////////////
-    if (vamsasSet.getDatasetId() == null || vamsasSet.getDatasetId() == "")
-    {
-      // older jalview projects do not have a dataset id.
-      al.setDataset(null);
-    }
-    else
-    {
-      // recover dataset - passing on flag indicating if this a 'viewless'
-      // sequence set (a.k.a. a stored dataset for the project)
-      recoverDatasetFor(vamsasSet, al, object.getJalviewModelSequence()
-              .getViewportCount() == 0);
-    }
     // ///////////////////////////////
 
     Hashtable pdbloaded = new Hashtable(); // TODO nothing writes to this??
@@ -2757,6 +2972,12 @@ public class Jalview2XML
     {
       // load sequence features, database references and any associated PDB
       // structures for the alignment
+      //
+      // prior to 2.10, this part would only be executed the first time a
+      // sequence was encountered, but not afterwards.
+      // now, for 2.10 projects, this is also done if the xml doc includes
+      // dataset sequences not actually present in any particular view.
+      //
       for (int i = 0; i < vamsasSeq.length; i++)
       {
         if (jseqs[i].getFeaturesCount() > 0)
@@ -2783,13 +3004,17 @@ public class Jalview2XML
               }
 
             }
-
-            al.getSequenceAt(i).getDatasetSequence().addSequenceFeature(sf);
+            // adds feature to datasequence's feature set (since Jalview 2.10)
+            al.getSequenceAt(i).addSequenceFeature(sf);
           }
         }
         if (vamsasSeq[i].getDBRefCount() > 0)
         {
-          addDBRefs(al.getSequenceAt(i).getDatasetSequence(), vamsasSeq[i]);
+          // adds dbrefs to datasequence's set (since Jalview 2.10)
+          addDBRefs(
+                  al.getSequenceAt(i).getDatasetSequence() == null ? al.getSequenceAt(i)
+                          : al.getSequenceAt(i).getDatasetSequence(),
+                  vamsasSeq[i]);
         }
         if (jseqs[i].getPdbidsCount() > 0)
         {
@@ -2800,29 +3025,49 @@ public class Jalview2XML
             entry.setId(ids[p].getId());
             if (ids[p].getType() != null)
             {
-              if (ids[p].getType().equalsIgnoreCase("PDB"))
+              if (PDBEntry.Type.getType(ids[p].getType()) != null)
               {
-                entry.setType(PDBEntry.Type.PDB);
+                entry.setType(PDBEntry.Type.getType(ids[p].getType()));
               }
               else
               {
                 entry.setType(PDBEntry.Type.FILE);
               }
             }
-            if (ids[p].getFile() != null)
+            // jprovider is null when executing 'New View'
+            if (ids[p].getFile() != null && jprovider != null)
             {
               if (!pdbloaded.containsKey(ids[p].getFile()))
               {
-                entry.setFile(loadPDBFile(jprovider, ids[p].getId()));
+                entry.setFile(loadPDBFile(jprovider, ids[p].getId(),
+                        ids[p].getFile()));
               }
               else
               {
                 entry.setFile(pdbloaded.get(ids[p].getId()).toString());
               }
             }
+            if (ids[p].getPdbentryItem() != null)
+            {
+              for (PdbentryItem item : ids[p].getPdbentryItem())
+              {
+                for (Property pr : item.getProperty())
+                {
+                  entry.setProperty(pr.getName(), pr.getValue());
+                }
+              }
+            }
             StructureSelectionManager.getStructureSelectionManager(
                     Desktop.instance).registerPDBEntry(entry);
-            al.getSequenceAt(i).getDatasetSequence().addPDBId(entry);
+            // adds PDBEntry to datasequence's set (since Jalview 2.10)
+            if (al.getSequenceAt(i).getDatasetSequence() != null)
+            {
+              al.getSequenceAt(i).getDatasetSequence().addPDBId(entry);
+            }
+            else
+            {
+              al.getSequenceAt(i).addPDBId(entry);
+            }
           }
         }
       }
@@ -2851,16 +3096,16 @@ public class Jalview2XML
             if (maps[m].getMapping() != null)
             {
               mapping = addMapping(maps[m].getMapping());
-            }
-            if (dnaseq != null && mapping.getTo() != null)
-            {
-              cf.addMap(dnaseq, mapping.getTo(), mapping.getMap());
-            }
-            else
-            {
-              // defer to later
-              frefedSequence.add(new Object[] { maps[m].getDnasq(), cf,
-                  mapping });
+              if (dnaseq != null && mapping.getTo() != null)
+              {
+                cf.addMap(dnaseq, mapping.getTo(), mapping.getMap());
+              }
+              else
+              {
+                // defer to later
+                frefedSequence.add(newAlcodMapRef(maps[m].getDnasq(), cf,
+                        mapping));
+              }
             }
           }
           al.addCodonFrame(cf);
@@ -2870,12 +3115,12 @@ public class Jalview2XML
 
     // ////////////////////////////////
     // LOAD ANNOTATIONS
-    List<JvAnnotRow> autoAlan = new ArrayList<JvAnnotRow>();
+    List<JvAnnotRow> autoAlan = new ArrayList<>();
 
     /*
      * store any annotations which forward reference a group's ID
      */
-    Map<String, List<AlignmentAnnotation>> groupAnnotRefs = new Hashtable<String, List<AlignmentAnnotation>>();
+    Map<String, List<AlignmentAnnotation>> groupAnnotRefs = new Hashtable<>();
 
     if (vamsasSet.getAnnotationCount() > 0)
     {
@@ -3030,7 +3275,7 @@ public class Jalview2XML
                   .get(an[i].getGroupRef());
           if (aal == null)
           {
-            aal = new ArrayList<jalview.datamodel.AlignmentAnnotation>();
+            aal = new ArrayList<>();
             groupAnnotRefs.put(an[i].getGroupRef(), aal);
           }
           aal.add(jaa);
@@ -3112,20 +3357,15 @@ public class Jalview2XML
                   && jGroup.getAnnotationColours() != null)
           {
             addAnnotSchemeGroup = true;
-            cs = null;
           }
           else
           {
-            cs = ColourSchemeProperty.getColour(al, jGroup.getColour());
-          }
-
-          if (cs != null)
-          {
-            cs.setThreshold(jGroup.getPidThreshold(), true);
+            cs = ColourSchemeProperty.getColourScheme(al, jGroup.getColour());
           }
         }
+        int pidThreshold = jGroup.getPidThreshold();
 
-        Vector<SequenceI> seqs = new Vector<SequenceI>();
+        Vector<SequenceI> seqs = new Vector<>();
 
         for (int s = 0; s < jGroup.getSeqCount(); s++)
         {
@@ -3146,7 +3386,8 @@ public class Jalview2XML
         SequenceGroup sg = new SequenceGroup(seqs, jGroup.getName(), cs,
                 jGroup.getDisplayBoxes(), jGroup.getDisplayText(),
                 jGroup.getColourText(), jGroup.getStart(), jGroup.getEnd());
-
+        sg.getGroupColourScheme().setThreshold(pidThreshold, true);
+        sg.getGroupColourScheme().setConservationInc(jGroup.getConsThreshold());
         sg.setOutlineColour(new java.awt.Color(jGroup.getOutlineColour()));
 
         sg.textColour = new java.awt.Color(jGroup.getTextCol1());
@@ -3173,9 +3414,8 @@ public class Jalview2XML
         }
         if (jGroup.getConsThreshold() != 0)
         {
-          jalview.analysis.Conservation c = new jalview.analysis.Conservation(
-                  "All", ResidueProperties.propHash, 3,
-                  sg.getSequences(null), 0, sg.getWidth() - 1);
+          Conservation c = new Conservation("All", sg.getSequences(null),
+                  0, sg.getWidth() - 1);
           c.calculate();
           c.verdict(false, 25);
           sg.cs.setConservation(c);
@@ -3213,8 +3453,8 @@ public class Jalview2XML
         if (addAnnotSchemeGroup)
         {
           // reconstruct the annotation colourscheme
-          sg.cs = constructAnnotationColour(jGroup.getAnnotationColours(),
-                  null, al, jms, false);
+          sg.setColourScheme(constructAnnotationColour(
+                  jGroup.getAnnotationColours(), null, al, jms, false));
         }
       }
     }
@@ -3364,7 +3604,7 @@ public class Jalview2XML
           String rnaTitle = ss.getTitle();
           String sessionState = ss.getViewerState();
           String tempStateFile = copyJarEntry(jprovider, sessionState,
-                  "varna");
+                  "varna", null);
           RnaModel rna = new RnaModel(rnaTitle, ann, seq, null, gapped);
           appVarna.addModelSession(rna, rnaTitle, tempStateFile);
         }
@@ -3440,7 +3680,7 @@ public class Jalview2XML
         TreePanel tp = (TreePanel) retrieveExistingObj(tree.getId());
         if (tp == null)
         {
-          tp = af.ShowNewickTree(
+          tp = af.showNewickTree(
                   new jalview.io.NewickFile(tree.getNewick()),
                   tree.getTitle(), tree.getWidth(), tree.getHeight(),
                   tree.getXpos(), tree.getYpos());
@@ -3518,7 +3758,7 @@ public class Jalview2XML
      * Run through all PDB ids on the alignment, and collect mappings between
      * distinct view ids and all sequences referring to that view.
      */
-    Map<String, StructureViewerModel> structureViewers = new LinkedHashMap<String, StructureViewerModel>();
+    Map<String, StructureViewerModel> structureViewers = new LinkedHashMap<>();
 
     for (int i = 0; i < jseqs.length; i++)
     {
@@ -3539,7 +3779,8 @@ public class Jalview2XML
             // Originally : ids[p].getFile()
             // : TODO: verify external PDB file recovery still works in normal
             // jalview project load
-            jpdb.setFile(loadPDBFile(jprovider, ids[p].getId()));
+            jpdb.setFile(loadPDBFile(jprovider, ids[p].getId(),
+                    ids[p].getFile()));
             jpdb.setId(ids[p].getId());
 
             int x = structureState.getXpos();
@@ -3550,7 +3791,8 @@ public class Jalview2XML
             // Probably don't need to do this anymore...
             // Desktop.desktop.getComponentAt(x, y);
             // TODO: NOW: check that this recovers the PDB file correctly.
-            String pdbFile = loadPDBFile(jprovider, ids[p].getId());
+            String pdbFile = loadPDBFile(jprovider, ids[p].getId(),
+                    ids[p].getFile());
             jalview.datamodel.SequenceI seq = seqRefIds.get(jseqs[i]
                     .getId() + "");
             if (sviewid == null)
@@ -3710,12 +3952,12 @@ public class Jalview2XML
      */
     String viewerJarEntryName = getViewerJarEntryName(data.getViewId());
     chimeraSessionFile = copyJarEntry(jprovider, viewerJarEntryName,
-            "chimera");
+            "chimera", null);
 
     Set<Entry<File, StructureData>> fileData = data.getFileData()
             .entrySet();
-    List<PDBEntry> pdbs = new ArrayList<PDBEntry>();
-    List<SequenceI[]> allseqs = new ArrayList<SequenceI[]>();
+    List<PDBEntry> pdbs = new ArrayList<>();
+    List<SequenceI[]> allseqs = new ArrayList<>();
     for (Entry<File, StructureData> pdb : fileData)
     {
       String filePath = pdb.getValue().getFilePath();
@@ -3771,9 +4013,9 @@ public class Jalview2XML
               getViewerJarEntryName(svattrib.getViewId()));
     }
 
-    List<String> pdbfilenames = new ArrayList<String>();
-    List<SequenceI[]> seqmaps = new ArrayList<SequenceI[]>();
-    List<String> pdbids = new ArrayList<String>();
+    List<String> pdbfilenames = new ArrayList<>();
+    List<SequenceI[]> seqmaps = new ArrayList<>();
+    List<String> pdbids = new ArrayList<>();
     StringBuilder newFileLoc = new StringBuilder(64);
     int cp = 0, ncp, ecp;
     Map<File, StructureData> oldFiles = svattrib.getFileData();
@@ -3791,6 +4033,11 @@ public class Jalview2XML
         // filename
         // translation differently.
         StructureData filedat = oldFiles.get(new File(oldfilenam));
+        if (filedat == null)
+        {
+          String reformatedOldFilename = oldfilenam.replaceAll("/", "\\\\");
+          filedat = oldFiles.get(new File(reformatedOldFilename));
+        }
         newFileLoc.append(Platform.escapeString(filedat.getFilePath()));
         pdbfilenames.add(filedat.getFilePath());
         pdbids.add(filedat.getPdbId());
@@ -3996,8 +4243,7 @@ public class Jalview2XML
       StructureData filedat = oldFiles.get(id);
       String pdbFile = filedat.getFilePath();
       SequenceI[] seq = filedat.getSeqList().toArray(new SequenceI[0]);
-      binding.getSsm().setMapping(seq, null, pdbFile,
-              jalview.io.AppletFormatAdapter.FILE);
+      binding.getSsm().setMapping(seq, null, pdbFile, DataSourceType.FILE);
       binding.addSequenceForStructFile(pdbFile, seq);
     }
     // and add the AlignmentPanel's reference to the view panel
@@ -4112,7 +4358,7 @@ public class Jalview2XML
     af = new AlignFrame(al, view.getWidth(), view.getHeight(),
             uniqueSeqSetId, viewId);
 
-    af.setFileName(file, "Jalview");
+    af.setFileName(file, FileFormat.Jalview);
 
     for (int i = 0; i < JSEQ.length; i++)
     {
@@ -4187,10 +4433,12 @@ public class Jalview2XML
 
     af.viewport.setShowAnnotation(view.getShowAnnotation());
     af.viewport.setAbovePIDThreshold(view.getPidSelected());
+    af.viewport.setThreshold(view.getPidThreshold());
 
     af.viewport.setColourText(view.getShowColourText());
 
     af.viewport.setConservationSelected(view.getConservationSelected());
+    af.viewport.setIncrement(view.getConsThreshold());
     af.viewport.setShowJVSuffix(view.getShowFullId());
     af.viewport.setRightAlignIds(view.getRightAlignIds());
     af.viewport.setFont(
@@ -4214,8 +4462,8 @@ public class Jalview2XML
     af.viewport.setThresholdTextColour(view.getTextColThreshold());
     af.viewport.setShowUnconserved(view.hasShowUnconserved() ? view
             .isShowUnconserved() : false);
-    af.viewport.setStartRes(view.getStartRes());
-    af.viewport.setStartSeq(view.getStartSeq());
+    af.viewport.getRanges().setStartRes(view.getStartRes());
+    // startSeq set in af.alignPanel.updateLayout below
     af.alignPanel.updateLayout();
     ColourSchemeI cs = null;
     // apply colourschemes
@@ -4235,22 +4483,21 @@ public class Jalview2XML
       }
       else
       {
-        cs = ColourSchemeProperty.getColour(al, view.getBgColour());
-      }
-
-      if (cs != null)
-      {
-        cs.setThreshold(view.getPidThreshold(), true);
-        cs.setConsensus(af.viewport.getSequenceConsensusHash());
+        cs = ColourSchemeProperty.getColourScheme(al, view.getBgColour());
       }
     }
 
     af.viewport.setGlobalColourScheme(cs);
+    af.viewport.getResidueShading().setThreshold(
+            view.getPidThreshold(), true);
+    af.viewport.getResidueShading().setConsensus(
+            af.viewport.getSequenceConsensusHash());
     af.viewport.setColourAppliesToAllGroups(false);
 
     if (view.getConservationSelected() && cs != null)
     {
-      cs.setConservationInc(view.getConsThreshold());
+      af.viewport.getResidueShading().setConservationInc(
+              view.getConsThreshold());
     }
 
     af.changeColour(cs);
@@ -4329,8 +4576,8 @@ public class Jalview2XML
       af.viewport.setFeaturesDisplayed(fdi = new FeaturesDisplayed());
       String[] renderOrder = new String[jms.getFeatureSettings()
               .getSettingCount()];
-      Map<String, FeatureColourI> featureColours = new Hashtable<String, FeatureColourI>();
-      Map<String, Float> featureOrder = new Hashtable<String, Float>();
+      Map<String, FeatureColourI> featureColours = new Hashtable<>();
+      Map<String, Float> featureOrder = new Hashtable<>();
 
       for (int fs = 0; fs < jms.getFeatureSettings().getSettingCount(); fs++)
       {
@@ -4389,7 +4636,7 @@ public class Jalview2XML
           fdi.setVisible(setting.getType());
         }
       }
-      Map<String, Boolean> fgtable = new Hashtable<String, Boolean>();
+      Map<String, Boolean> fgtable = new Hashtable<>();
       for (int gs = 0; gs < jms.getFeatureSettings().getGroupCount(); gs++)
       {
         Group grp = jms.getFeatureSettings().getGroup(gs);
@@ -4432,7 +4679,7 @@ public class Jalview2XML
       }
     }
     af.setMenusFromViewport(af.viewport);
-
+    af.setTitle(view.getTitle());
     // TODO: we don't need to do this if the viewport is aready visible.
     /*
      * Add the AlignFrame to the desktop (it may be 'gathered' later), unless it
@@ -4456,12 +4703,21 @@ public class Jalview2XML
     return af;
   }
 
+  /**
+   * Reads saved data to restore Colour by Annotation settings
+   * 
+   * @param viewAnnColour
+   * @param af
+   * @param al
+   * @param jms
+   * @param checkGroupAnnColour
+   * @return
+   */
   private ColourSchemeI constructAnnotationColour(
           AnnotationColours viewAnnColour, AlignFrame af, AlignmentI al,
           JalviewModelSequence jms, boolean checkGroupAnnColour)
   {
     boolean propagateAnnColour = false;
-    ColourSchemeI cs = null;
     AlignmentI annAlignment = af != null ? af.viewport.getAlignment() : al;
     if (checkGroupAnnColour && al.getGroups() != null
             && al.getGroups().size() > 0)
@@ -4469,113 +4725,92 @@ public class Jalview2XML
       // pre 2.8.1 behaviour
       // check to see if we should transfer annotation colours
       propagateAnnColour = true;
-      for (jalview.datamodel.SequenceGroup sg : al.getGroups())
+      for (SequenceGroup sg : al.getGroups())
       {
-        if (sg.cs instanceof AnnotationColourGradient)
+        if (sg.getColourScheme() instanceof AnnotationColourGradient)
         {
           propagateAnnColour = false;
         }
       }
     }
-    // int find annotation
-    if (annAlignment.getAlignmentAnnotation() != null)
+
+    /*
+     * 2.10.2- : saved annotationId is AlignmentAnnotation.annotationId
+     */
+    String annotationId = viewAnnColour.getAnnotation();
+    AlignmentAnnotation matchedAnnotation = annotationIds.get(annotationId);
+
+    /*
+     * pre 2.10.2: saved annotationId is AlignmentAnnotation.label
+     */
+    if (matchedAnnotation == null && annAlignment.getAlignmentAnnotation() != null)
     {
       for (int i = 0; i < annAlignment.getAlignmentAnnotation().length; i++)
       {
-        if (annAlignment.getAlignmentAnnotation()[i].label
-                .equals(viewAnnColour.getAnnotation()))
+        if (annotationId
+                .equals(annAlignment.getAlignmentAnnotation()[i].label))
         {
-          if (annAlignment.getAlignmentAnnotation()[i].getThreshold() == null)
-          {
-            annAlignment.getAlignmentAnnotation()[i]
-                    .setThreshold(new jalview.datamodel.GraphLine(
-                            viewAnnColour.getThreshold(), "Threshold",
-                            java.awt.Color.black)
-
-                    );
-          }
-
-          if (viewAnnColour.getColourScheme().equals("None"))
-          {
-            cs = new AnnotationColourGradient(
-                    annAlignment.getAlignmentAnnotation()[i],
-                    new java.awt.Color(viewAnnColour.getMinColour()),
-                    new java.awt.Color(viewAnnColour.getMaxColour()),
-                    viewAnnColour.getAboveThreshold());
-          }
-          else if (viewAnnColour.getColourScheme().startsWith("ucs"))
-          {
-            cs = new AnnotationColourGradient(
-                    annAlignment.getAlignmentAnnotation()[i],
-                    getUserColourScheme(jms,
-                            viewAnnColour.getColourScheme()),
-                    viewAnnColour.getAboveThreshold());
-          }
-          else
-          {
-            cs = new AnnotationColourGradient(
-                    annAlignment.getAlignmentAnnotation()[i],
-                    ColourSchemeProperty.getColour(al,
-                            viewAnnColour.getColourScheme()),
-                    viewAnnColour.getAboveThreshold());
-          }
-          if (viewAnnColour.hasPerSequence())
-          {
-            ((AnnotationColourGradient) cs).setSeqAssociated(viewAnnColour
-                    .isPerSequence());
-          }
-          if (viewAnnColour.hasPredefinedColours())
-          {
-            ((AnnotationColourGradient) cs)
-                    .setPredefinedColours(viewAnnColour
-                            .isPredefinedColours());
-          }
-          if (propagateAnnColour && al.getGroups() != null)
-          {
-            // Also use these settings for all the groups
-            for (int g = 0; g < al.getGroups().size(); g++)
-            {
-              jalview.datamodel.SequenceGroup sg = al.getGroups().get(g);
-
-              if (sg.cs == null)
-              {
-                continue;
-              }
+          matchedAnnotation = annAlignment.getAlignmentAnnotation()[i];
+          break;
+        }
+      }
+    }
+    if (matchedAnnotation == null)
+    {
+      System.err.println("Failed to match annotation colour scheme for "
+              + annotationId);
+      return null;
+    }
+    if (matchedAnnotation.getThreshold() == null)
+    {
+      matchedAnnotation.setThreshold(new GraphLine(viewAnnColour.getThreshold(),
+              "Threshold", Color.black));
+    }
 
-              /*
-               * if (viewAnnColour.getColourScheme().equals("None" )) { sg.cs =
-               * new AnnotationColourGradient(
-               * annAlignment.getAlignmentAnnotation()[i], new
-               * java.awt.Color(viewAnnColour. getMinColour()), new
-               * java.awt.Color(viewAnnColour. getMaxColour()),
-               * viewAnnColour.getAboveThreshold()); } else
-               */
-              {
-                sg.cs = new AnnotationColourGradient(
-                        annAlignment.getAlignmentAnnotation()[i], sg.cs,
-                        viewAnnColour.getAboveThreshold());
-                if (cs instanceof AnnotationColourGradient)
-                {
-                  if (viewAnnColour.hasPerSequence())
-                  {
-                    ((AnnotationColourGradient) cs)
-                            .setSeqAssociated(viewAnnColour.isPerSequence());
-                  }
-                  if (viewAnnColour.hasPredefinedColours())
-                  {
-                    ((AnnotationColourGradient) cs)
-                            .setPredefinedColours(viewAnnColour
-                                    .isPredefinedColours());
-                  }
-                }
-              }
+    AnnotationColourGradient cs = null;
+    if (viewAnnColour.getColourScheme().equals("None"))
+    {
+      cs = new AnnotationColourGradient(matchedAnnotation, new Color(
+              viewAnnColour.getMinColour()), new Color(
+              viewAnnColour.getMaxColour()),
+              viewAnnColour.getAboveThreshold());
+    }
+    else if (viewAnnColour.getColourScheme().startsWith("ucs"))
+    {
+      cs = new AnnotationColourGradient(matchedAnnotation, getUserColourScheme(
+              jms, viewAnnColour.getColourScheme()),
+              viewAnnColour.getAboveThreshold());
+    }
+    else
+    {
+      cs = new AnnotationColourGradient(matchedAnnotation,
+              ColourSchemeProperty.getColourScheme(al,
+                      viewAnnColour.getColourScheme()),
+              viewAnnColour.getAboveThreshold());
+    }
 
-            }
-          }
+    boolean perSequenceOnly = viewAnnColour.isPerSequence();
+    boolean useOriginalColours = viewAnnColour.isPredefinedColours();
+    cs.setSeqAssociated(perSequenceOnly);
+    cs.setPredefinedColours(useOriginalColours);
 
-          break;
+    if (propagateAnnColour && al.getGroups() != null)
+    {
+      // Also use these settings for all the groups
+      for (int g = 0; g < al.getGroups().size(); g++)
+      {
+        SequenceGroup sg = al.getGroups().get(g);
+        if (sg.getGroupColourScheme() == null)
+        {
+          continue;
         }
 
+        AnnotationColourGradient groupScheme = new AnnotationColourGradient(
+                matchedAnnotation, sg.getColourScheme(),
+                viewAnnColour.getAboveThreshold());
+        sg.setColourScheme(groupScheme);
+        groupScheme.setSeqAssociated(perSequenceOnly);
+        groupScheme.setPredefinedColours(useOriginalColours);
       }
     }
     return cs;
@@ -4594,7 +4829,7 @@ public class Jalview2XML
       String[] magicNames = new String[] { "Consensus", "Quality",
           "Conservation" };
       JvAnnotRow nullAnnot = new JvAnnotRow(-1, null);
-      Hashtable<String, JvAnnotRow> visan = new Hashtable<String, JvAnnotRow>();
+      Hashtable<String, JvAnnotRow> visan = new Hashtable<>();
       for (String nm : magicNames)
       {
         visan.put(nm, nullAnnot);
@@ -4606,11 +4841,11 @@ public class Jalview2XML
                         + auan.template.getCalcId()), auan);
       }
       int hSize = al.getAlignmentAnnotation().length;
-      List<JvAnnotRow> reorder = new ArrayList<JvAnnotRow>();
+      List<JvAnnotRow> reorder = new ArrayList<>();
       // work through any autoCalculated annotation already on the view
       // removing it if it should be placed in a different location on the
       // annotation panel.
-      List<String> remains = new ArrayList<String>(visan.keySet());
+      List<String> remains = new ArrayList<>(visan.keySet());
       for (int h = 0; h < hSize; h++)
       {
         jalview.datamodel.AlignmentAnnotation jalan = al
@@ -4750,7 +4985,7 @@ public class Jalview2XML
     for (int i = 0, iSize = vamsasSet.getSequenceCount(); i < iSize; i++)
     {
       Sequence vamsasSeq = vamsasSet.getSequence(i);
-      ensureJalviewDatasetSequence(vamsasSeq, ds, dseqs, ignoreUnrefed);
+      ensureJalviewDatasetSequence(vamsasSeq, ds, dseqs, ignoreUnrefed, i);
     }
     // create a new dataset
     if (ds == null)
@@ -4777,18 +5012,29 @@ public class Jalview2XML
    *          dataset alignment
    * @param dseqs
    *          vector to add new dataset sequence to
+   * @param ignoreUnrefed
+   *          - when true, don't create new sequences from vamsasSeq if it's id
+   *          doesn't already have an asssociated Jalview sequence.
+   * @param vseqpos
+   *          - used to reorder the sequence in the alignment according to the
+   *          vamsasSeq array ordering, to preserve ordering of dataset
    */
   private void ensureJalviewDatasetSequence(Sequence vamsasSeq,
-          AlignmentI ds, Vector dseqs, boolean ignoreUnrefed)
+          AlignmentI ds, Vector dseqs, boolean ignoreUnrefed, int vseqpos)
   {
     // JBP TODO: Check this is called for AlCodonFrames to support recovery of
     // xRef Codon Maps
     SequenceI sq = seqRefIds.get(vamsasSeq.getId());
+    boolean reorder = false;
     SequenceI dsq = null;
     if (sq != null && sq.getDatasetSequence() != null)
     {
       dsq = sq.getDatasetSequence();
     }
+    else
+    {
+      reorder = true;
+    }
     if (sq == null && ignoreUnrefed)
     {
       return;
@@ -4884,6 +5130,35 @@ public class Jalview2XML
         // + (post ? "appended" : ""));
       }
     }
+    else
+    {
+      // sequence refs are identical. We may need to update the existing dataset
+      // alignment with this one, though.
+      if (ds != null && dseqs == null)
+      {
+        int opos = ds.findIndex(dsq);
+        SequenceI tseq = null;
+        if (opos != -1 && vseqpos != opos)
+        {
+          // remove from old position
+          ds.deleteSequence(dsq);
+        }
+        if (vseqpos < ds.getHeight())
+        {
+          if (vseqpos != opos)
+          {
+            // save sequence at destination position
+            tseq = ds.getSequenceAt(vseqpos);
+            ds.replaceSequenceAt(vseqpos, dsq);
+            ds.addSequence(tseq);
+          }
+        }
+        else
+        {
+          ds.addSequence(dsq);
+        }
+      }
+    }
   }
 
   /*
@@ -4898,7 +5173,7 @@ public class Jalview2XML
   {
     if (datasetIds == null)
     {
-      datasetIds = new Hashtable<String, AlignmentI>();
+      datasetIds = new Hashtable<>();
       return null;
     }
     if (datasetIds.containsKey(datasetId))
@@ -4912,7 +5187,7 @@ public class Jalview2XML
   {
     if (datasetIds == null)
     {
-      datasetIds = new Hashtable<String, AlignmentI>();
+      datasetIds = new Hashtable<>();
     }
     datasetIds.put(datasetId, dataset);
   }
@@ -4935,7 +5210,7 @@ public class Jalview2XML
       // make a new datasetId and record it
       if (dataset2Ids == null)
       {
-        dataset2Ids = new IdentityHashMap<AlignmentI, String>();
+        dataset2Ids = new IdentityHashMap<>();
       }
       else
       {
@@ -5003,7 +5278,7 @@ public class Jalview2XML
         }
         else
         {
-          frefedSequence.add(new Object[] { dsfor, jmap });
+          frefedSequence.add(newMappingRef(dsfor, jmap));
         }
       }
       else
@@ -5041,6 +5316,7 @@ public class Jalview2XML
           djs.setEnd(jmap.getMap().getToHighest());
           djs.setVamsasId(uniqueSetSuffix + sqid);
           jmap.setTo(djs);
+          incompleteSeqs.put(sqid, djs);
           seqRefIds.put(sqid, djs);
 
         }
@@ -5213,11 +5489,11 @@ public class Jalview2XML
         // register sequence object so the XML parser can recover it.
         if (seqRefIds == null)
         {
-          seqRefIds = new HashMap<String, SequenceI>();
+          seqRefIds = new HashMap<>();
         }
         if (seqsToIds == null)
         {
-          seqsToIds = new IdentityHashMap<SequenceI, String>();
+          seqsToIds = new IdentityHashMap<>();
         }
         seqRefIds.put(jv2vobj.get(jvobj).toString(), (SequenceI) jvobj);
         seqsToIds.put((SequenceI) jvobj, id);