JAL-4305 Isolate and unify the Jalview object from all the gubbins in jalview.bin...
[jalview.git] / src / jalview / project / Jalview2XML.java
index e132bef..cc20fce 100644 (file)
@@ -44,6 +44,7 @@ import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.BitSet;
 import java.util.Collections;
 import java.util.Enumeration;
 import java.util.GregorianCalendar;
@@ -84,13 +85,18 @@ import jalview.api.analysis.ScoreModelI;
 import jalview.api.analysis.SimilarityParamsI;
 import jalview.api.structures.JalviewStructureDisplayI;
 import jalview.bin.Cache;
+import jalview.bin.Console;
+import jalview.bin.Jalview;
 import jalview.datamodel.AlignedCodonFrame;
 import jalview.datamodel.Alignment;
 import jalview.datamodel.AlignmentAnnotation;
 import jalview.datamodel.AlignmentI;
+import jalview.datamodel.ContactMatrix;
+import jalview.datamodel.ContactMatrixI;
 import jalview.datamodel.DBRefEntry;
 import jalview.datamodel.GeneLocus;
 import jalview.datamodel.GraphLine;
+import jalview.datamodel.GroupSet;
 import jalview.datamodel.PDBEntry;
 import jalview.datamodel.Point;
 import jalview.datamodel.RnaViewerModel;
@@ -111,6 +117,7 @@ import jalview.gui.AppVarna;
 import jalview.gui.Desktop;
 import jalview.gui.JvOptionPane;
 import jalview.gui.OOMWarning;
+import jalview.gui.OverviewPanel;
 import jalview.gui.PCAPanel;
 import jalview.gui.PaintRefresher;
 import jalview.gui.SplitFrame;
@@ -146,6 +153,8 @@ import jalview.viewmodel.ViewportRanges;
 import jalview.viewmodel.seqfeatures.FeatureRendererModel;
 import jalview.viewmodel.seqfeatures.FeatureRendererSettings;
 import jalview.viewmodel.seqfeatures.FeaturesDisplayed;
+import jalview.ws.datamodel.MappableContactMatrixI;
+import jalview.ws.datamodel.alphafold.PAEContactMatrix;
 import jalview.ws.jws2.Jws2Discoverer;
 import jalview.ws.jws2.dm.AAConSettings;
 import jalview.ws.jws2.jabaws2.Jws2Instance;
@@ -184,11 +193,14 @@ import jalview.xml.binding.jalview.JalviewModel.UserColours;
 import jalview.xml.binding.jalview.JalviewModel.Viewport;
 import jalview.xml.binding.jalview.JalviewModel.Viewport.CalcIdParam;
 import jalview.xml.binding.jalview.JalviewModel.Viewport.HiddenColumns;
+import jalview.xml.binding.jalview.JalviewModel.Viewport.Overview;
 import jalview.xml.binding.jalview.JalviewUserColours;
 import jalview.xml.binding.jalview.JalviewUserColours.Colour;
+import jalview.xml.binding.jalview.MapListType;
 import jalview.xml.binding.jalview.MapListType.MapListFrom;
 import jalview.xml.binding.jalview.MapListType.MapListTo;
 import jalview.xml.binding.jalview.Mapping;
+import jalview.xml.binding.jalview.MatrixType;
 import jalview.xml.binding.jalview.NoValueColour;
 import jalview.xml.binding.jalview.ObjectFactory;
 import jalview.xml.binding.jalview.PcaDataType;
@@ -228,6 +240,11 @@ public class Jalview2XML
   private static final String UTF_8 = "UTF-8";
 
   /**
+   * used in decision if quit confirmation should be issued
+   */
+  private static boolean stateSavedUpToDate = false;
+
+  /**
    * prefix for recovering datasets for alignments with multiple views where
    * non-existent dataset IDs were written for some views
    */
@@ -503,7 +520,7 @@ public class Jalview2XML
           }
         } catch (Exception x)
         {
-          System.err.println(
+          jalview.bin.Console.errPrintln(
                   "IMPLEMENTATION ERROR: Failed to resolve forward reference for sequence "
                           + ref.getSref());
           x.printStackTrace();
@@ -517,29 +534,30 @@ public class Jalview2XML
     }
     if (unresolved > 0)
     {
-      System.err.println("Jalview Project Import: There were " + unresolved
+      jalview.bin.Console.errPrintln("Jalview Project Import: There were "
+              + unresolved
               + " forward references left unresolved on the stack.");
     }
     if (failedtoresolve > 0)
     {
-      System.err.println("SERIOUS! " + failedtoresolve
+      jalview.bin.Console.errPrintln("SERIOUS! " + failedtoresolve
               + " resolvable forward references failed to resolve.");
     }
     if (incompleteSeqs != null && incompleteSeqs.size() > 0)
     {
-      System.err.println(
+      jalview.bin.Console.errPrintln(
               "Jalview Project Import: There are " + incompleteSeqs.size()
                       + " sequences which may have incomplete metadata.");
       if (incompleteSeqs.size() < 10)
       {
         for (SequenceI s : incompleteSeqs.values())
         {
-          System.err.println(s.toString());
+          jalview.bin.Console.errPrintln(s.toString());
         }
       }
       else
       {
-        System.err.println(
+        jalview.bin.Console.errPrintln(
                 "Too many to report. Skipping output of incomplete sequences.");
       }
     }
@@ -576,7 +594,7 @@ public class Jalview2XML
 
     } catch (Exception e)
     {
-      Cache.error("Couln't write Jalview state to " + statefile, e);
+      Console.error("Couln't write Jalview state to " + statefile, e);
       // TODO: inform user of the problem - they need to know if their data was
       // not saved !
       if (errorMessage == null)
@@ -613,7 +631,28 @@ public class Jalview2XML
    */
   public void saveState(JarOutputStream jout)
   {
-    AlignFrame[] frames = Desktop.getAlignFrames();
+    AlignFrame[] frames = Desktop.getDesktopAlignFrames();
+
+    setStateSavedUpToDate(true);
+
+    if (Cache.getDefault("DEBUG_DELAY_SAVE", false))
+    {
+      int n = debugDelaySave;
+      int i = 0;
+      while (i < n)
+      {
+        Console.debug("***** debugging save sleep " + i + "/" + n);
+        try
+        {
+          Thread.sleep(1000);
+        } catch (InterruptedException e)
+        {
+          // TODO Auto-generated catch block
+          e.printStackTrace();
+        }
+        i++;
+      }
+    }
 
     if (frames == null)
     {
@@ -762,6 +801,25 @@ public class Jalview2XML
       FileOutputStream fos = new FileOutputStream(
               doBackup ? backupfiles.getTempFilePath() : jarFile);
 
+      if (Cache.getDefault("DEBUG_DELAY_SAVE", false))
+      {
+        int n = debugDelaySave;
+        int i = 0;
+        while (i < n)
+        {
+          Console.debug("***** debugging save sleep " + i + "/" + n);
+          try
+          {
+            Thread.sleep(1000);
+          } catch (InterruptedException e)
+          {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+          }
+          i++;
+        }
+      }
+
       JarOutputStream jout = new JarOutputStream(fos);
       List<AlignFrame> frames = new ArrayList<>();
 
@@ -879,10 +937,9 @@ public class Jalview2XML
       object.setCreationDate(now);
     } catch (DatatypeConfigurationException e)
     {
-      System.err.println("error writing date: " + e.toString());
+      jalview.bin.Console.errPrintln("error writing date: " + e.toString());
     }
-    object.setVersion(
-            Cache.getDefault("VERSION", "Development Build"));
+    object.setVersion(Cache.getDefault("VERSION", "Development Build"));
 
     /**
      * rjal is full height alignment, jal is actual alignment with full metadata
@@ -947,14 +1004,14 @@ public class Jalview2XML
           // HAPPEN! (PF00072.15.stk does this)
           // JBPNote: Uncomment to debug writing out of files that do not read
           // back in due to ArrayOutOfBoundExceptions.
-          // System.err.println("vamsasSeq backref: "+id+"");
-          // System.err.println(jds.getName()+"
+          // jalview.bin.Console.errPrintln("vamsasSeq backref: "+id+"");
+          // jalview.bin.Console.errPrintln(jds.getName()+"
           // "+jds.getStart()+"-"+jds.getEnd()+" "+jds.getSequenceAsString());
-          // System.err.println("Hashcode: "+seqHash(jds));
+          // jalview.bin.Console.errPrintln("Hashcode: "+seqHash(jds));
           // SequenceI rsq = (SequenceI) seqRefIds.get(id + "");
-          // System.err.println(rsq.getName()+"
+          // jalview.bin.Console.errPrintln(rsq.getName()+"
           // "+rsq.getStart()+"-"+rsq.getEnd()+" "+rsq.getSequenceAsString());
-          // System.err.println("Hashcode: "+seqHash(rsq));
+          // jalview.bin.Console.errPrintln("Hashcode: "+seqHash(rsq));
         }
         else
         {
@@ -1083,38 +1140,55 @@ public class Jalview2XML
            * only view *should* be coped with sensibly.
            */
           // This must have been loaded, is it still visible?
-          JInternalFrame[] frames = Desktop.desktop.getAllFrames();
-          String matchedFile = null;
-          for (int f = frames.length - 1; f > -1; f--)
+          List<JalviewStructureDisplayI> viewFrames = new ArrayList<>();
+          if (Desktop.desktop != null)
           {
-            if (frames[f] instanceof StructureViewerBase)
+            JInternalFrame[] jifs = Desktop.desktop.getAllFrames();
+            if (jifs != null)
             {
-              StructureViewerBase viewFrame = (StructureViewerBase) frames[f];
-              matchedFile = saveStructureViewer(ap, jds, pdb, entry,
-                      viewIds, matchedFile, viewFrame);
-              /*
-               * Only store each structure viewer's state once in the project
-               * jar. First time through only (storeDS==false)
-               */
-              String viewId = viewFrame.getViewId();
-              String viewerType = viewFrame.getViewerType().toString();
-              if (!storeDS && !viewIds.contains(viewId))
+              for (JInternalFrame jif : jifs)
               {
-                viewIds.add(viewId);
-                File viewerState = viewFrame.saveSession();
-                if (viewerState != null)
+                if (jif instanceof JalviewStructureDisplayI)
                 {
-                  copyFileToJar(jout, viewerState.getPath(),
-                          getViewerJarEntryName(viewId), viewerType);
-                }
-                else
-                {
-                  Cache.error(
-                          "Failed to save viewer state for " + viewerType);
+                  viewFrames.add((JalviewStructureDisplayI) jif);
                 }
               }
             }
           }
+          else if (Jalview.isHeadlessMode()
+                  && Jalview.getInstance().getCommands() != null)
+          {
+            viewFrames.addAll(
+                    StructureViewerBase.getAllStructureViewerBases());
+          }
+
+          String matchedFile = null;
+          for (JalviewStructureDisplayI viewFrame : viewFrames)
+          {
+            matchedFile = saveStructureViewer(ap, jds, pdb, entry, viewIds,
+                    matchedFile, viewFrame);
+            /*
+             * Only store each structure viewer's state once in the project
+             * jar. First time through only (storeDS==false)
+             */
+            String viewId = viewFrame.getViewId();
+            String viewerType = viewFrame.getViewerType().toString();
+            if (!storeDS && !viewIds.contains(viewId))
+            {
+              viewIds.add(viewId);
+              File viewerState = viewFrame.saveSession();
+              if (viewerState != null)
+              {
+                copyFileToJar(jout, viewerState.getPath(),
+                        getViewerJarEntryName(viewId), viewerType);
+              }
+              else
+              {
+                Console.error(
+                        "Failed to save viewer state for " + viewerType);
+              }
+            }
+          }
 
           if (matchedFile != null || entry.getFile() != null)
           {
@@ -1272,6 +1346,13 @@ public class Jalview2XML
               tree.setLinkToAllViews(
                       tp.getTreeCanvas().isApplyToAllViews());
 
+              // columnWiseTree
+              if (tp.isColumnWise())
+              {
+                tree.setColumnWise(true);
+                String annId = tp.getAssocAnnotation().annotationId;
+                tree.setColumnReference(annId);
+              }
               // jms.addTree(tree);
               object.getTree().add(tree);
             }
@@ -1447,6 +1528,23 @@ public class Jalview2XML
       view.setStartRes(vpRanges.getStartRes());
       view.setStartSeq(vpRanges.getStartSeq());
 
+      OverviewPanel ov = ap.getOverviewPanel();
+      if (ov != null)
+      {
+        Overview overview = new Overview();
+        overview.setTitle(ov.getTitle());
+        Rectangle bounds = ov.getFrameBounds();
+        overview.setXpos(bounds.x);
+        overview.setYpos(bounds.y);
+        overview.setWidth(bounds.width);
+        overview.setHeight(bounds.height);
+        overview.setShowHidden(ov.isShowHiddenRegions());
+        overview.setGapColour(ov.getCanvas().getGapColour().getRGB());
+        overview.setResidueColour(
+                ov.getCanvas().getResidueColour().getRGB());
+        overview.setHiddenColour(ov.getCanvas().getHiddenColour().getRGB());
+        view.setOverview(overview);
+      }
       if (av.getGlobalColourScheme() instanceof jalview.schemes.UserColourScheme)
       {
         view.setBgColour(setUserColourScheme(av.getGlobalColourScheme(),
@@ -1487,6 +1585,8 @@ public class Jalview2XML
 
       view.setConservationSelected(av.getConservationSelected());
       view.setPidSelected(av.getAbovePIDThreshold());
+      view.setCharHeight(av.getCharHeight());
+      view.setCharWidth(av.getCharWidth());
       final Font font = av.getFont();
       view.setFontName(font.getName());
       view.setFontSize(font.getSize());
@@ -1632,7 +1732,8 @@ public class Jalview2XML
                 .getHiddenColumns();
         if (hidden == null)
         {
-          Cache.warn("REPORT BUG: avoided null columnselection bug (DMAM reported). Please contact Jim about this.");
+          Console.warn(
+                  "REPORT BUG: avoided null columnselection bug (DMAM reported). Please contact Jim about this.");
         }
         else
         {
@@ -1680,7 +1781,7 @@ public class Jalview2XML
       try
       {
         fileName = fileName.replace('\\', '/');
-        System.out.println("Writing jar entry " + fileName);
+        jalview.bin.Console.outPrintln("Writing jar entry " + fileName);
         JarEntry entry = new JarEntry(fileName);
         jout.putNextEntry(entry);
         PrintWriter pout = new PrintWriter(
@@ -1701,7 +1802,7 @@ public class Jalview2XML
       } catch (Exception ex)
       {
         // TODO: raise error in GUI if marshalling failed.
-        System.err.println("Error writing Jalview project");
+        jalview.bin.Console.errPrintln("Error writing Jalview project");
         ex.printStackTrace();
       }
     }
@@ -1801,7 +1902,7 @@ public class Jalview2XML
       object.getPcaViewer().add(viewer);
     } catch (Throwable t)
     {
-      Cache.error("Error saving PCA: " + t.getMessage());
+      Console.error("Error saving PCA: " + t.getMessage());
     }
   }
 
@@ -2015,7 +2116,7 @@ public class Jalview2XML
       File file = new File(infilePath);
       if (file.exists() && jout != null)
       {
-        System.out.println(
+        jalview.bin.Console.outPrintln(
                 "Writing jar entry " + jarEntryName + " (" + msg + ")");
         jout.putNextEntry(new JarEntry(jarEntryName));
         copyAll(is, jout);
@@ -2064,7 +2165,7 @@ public class Jalview2XML
    */
   protected String saveStructureViewer(AlignmentPanel ap, SequenceI jds,
           Pdbids pdb, PDBEntry entry, List<String> viewIds,
-          String matchedFile, StructureViewerBase viewFrame)
+          String matchedFile, JalviewStructureDisplayI viewFrame)
   {
     final AAStructureBindingModel bindingModel = viewFrame.getBinding();
 
@@ -2091,7 +2192,7 @@ public class Jalview2XML
       }
       else if (!matchedFile.equals(pdbentry.getFile()))
       {
-        Cache.warn(
+        Console.warn(
                 "Probably lost some PDB-Sequence mappings for this structure file (which apparently has same PDB Entry code): "
                         + pdbentry.getFile());
       }
@@ -2109,7 +2210,7 @@ public class Jalview2XML
         {
           StructureState state = new StructureState();
           state.setVisible(true);
-          state.setXpos(viewFrame.getX());
+          state.setXpos(viewFrame.getY());
           state.setYpos(viewFrame.getY());
           state.setWidth(viewFrame.getWidth());
           state.setHeight(viewFrame.getHeight());
@@ -2224,6 +2325,75 @@ public class Jalview2XML
           line.setColour(annotation.getThreshold().colour.getRGB());
           an.setThresholdLine(line);
         }
+        if (annotation.graph == AlignmentAnnotation.CONTACT_MAP)
+        {
+          if (annotation.sequenceRef.getContactMaps() != null)
+          {
+            ContactMatrixI cm = annotation.sequenceRef
+                    .getContactMatrixFor(annotation);
+            if (cm != null)
+            {
+              MatrixType xmlmat = new MatrixType();
+              xmlmat.setType(cm.getType());
+              xmlmat.setRows(BigInteger.valueOf(cm.getWidth()));
+              xmlmat.setCols(BigInteger.valueOf(cm.getHeight()));
+              // consider using an opaque to/from -> allow instance to control
+              // its representation ?
+              xmlmat.setElements(ContactMatrix.contactToFloatString(cm));
+              if (cm.hasGroups())
+              {
+                for (BitSet gp : cm.getGroups())
+                {
+                  xmlmat.getGroups().add(stringifyBitset(gp));
+                }
+              }
+              if (cm.hasTree())
+              {
+                // provenance object for tree ?
+                xmlmat.getNewick().add(cm.getNewick());
+                xmlmat.setTreeMethod(cm.getTreeMethod());
+              }
+              if (cm.hasCutHeight())
+              {
+                xmlmat.setCutHeight(cm.getCutHeight());
+              }
+              // set/get properties
+              if (cm instanceof MappableContactMatrixI)
+              {
+                jalview.util.MapList mlst = ((MappableContactMatrixI) cm)
+                        .getMapFor(annotation.sequenceRef);
+                if (mlst != null)
+                {
+                  MapListType mp = new MapListType();
+                  List<int[]> r = mlst.getFromRanges();
+                  for (int[] range : r)
+                  {
+                    MapListFrom mfrom = new MapListFrom();
+                    mfrom.setStart(range[0]);
+                    mfrom.setEnd(range[1]);
+                    // mp.addMapListFrom(mfrom);
+                    mp.getMapListFrom().add(mfrom);
+                  }
+                  r = mlst.getToRanges();
+                  for (int[] range : r)
+                  {
+                    MapListTo mto = new MapListTo();
+                    mto.setStart(range[0]);
+                    mto.setEnd(range[1]);
+                    // mp.addMapListTo(mto);
+                    mp.getMapListTo().add(mto);
+                  }
+                  mp.setMapFromUnit(
+                          BigInteger.valueOf(mlst.getFromRatio()));
+                  mp.setMapToUnit(BigInteger.valueOf(mlst.getToRatio()));
+                  xmlmat.setMapping(mp);
+                }
+              }
+              // and add to model
+              an.getContactmatrix().add(xmlmat);
+            }
+          }
+        }
       }
       else
       {
@@ -2254,10 +2424,9 @@ public class Jalview2XML
       {
         for (String pr : annotation.getProperties())
         {
-          jalview.xml.binding.jalview.Annotation.Property prop = new jalview.xml.binding.jalview.Annotation.Property();
+          jalview.xml.binding.jalview.Property prop = new jalview.xml.binding.jalview.Property();
           prop.setName(pr);
           prop.setValue(annotation.getProperty(pr));
-          // an.addProperty(prop);
           an.getProperty().add(prop);
         }
       }
@@ -2328,6 +2497,44 @@ public class Jalview2XML
 
   }
 
+  private String stringifyBitset(BitSet gp)
+  {
+    StringBuilder sb = new StringBuilder();
+    for (long val : gp.toLongArray())
+    {
+      if (sb.length() > 0)
+      {
+        sb.append(",");
+      }
+      sb.append(val);
+    }
+    return sb.toString();
+  }
+
+  private BitSet deStringifyBitset(String stringified)
+  {
+    if ("".equals(stringified) || stringified == null)
+    {
+      return new BitSet();
+    }
+    String[] longvals = stringified.split(",");
+    long[] newlongvals = new long[longvals.length];
+    for (int lv = 0; lv < longvals.length; lv++)
+    {
+      try
+      {
+        newlongvals[lv] = Long.valueOf(longvals[lv]);
+      } catch (Exception x)
+      {
+        errorMessage += "Couldn't destringify bitset from: '" + stringified
+                + "'";
+        newlongvals[lv] = 0;
+      }
+    }
+    return BitSet.valueOf(newlongvals);
+
+  }
+
   private CalcIdParam createCalcIdParam(String calcId, AlignViewport av)
   {
     AutoCalcSetting settings = av.getCalcIdSettingsFor(calcId);
@@ -2389,7 +2596,7 @@ public class Jalview2XML
                   calcIdParam.getParameters().replace("|\\n|", "\n"));
         } catch (IOException x)
         {
-          Cache.warn("Couldn't parse parameter data for "
+          Console.warn("Couldn't parse parameter data for "
                   + calcIdParam.getCalcId(), x);
           return false;
         }
@@ -2417,7 +2624,8 @@ public class Jalview2XML
       }
       else
       {
-        Cache.warn("Cannot resolve a service for the parameters used in this project. Try configuring a JABAWS server.");
+        Console.warn(
+                "Cannot resolve a service for the parameters used in this project. Try configuring a JABAWS server.");
         return false;
       }
     }
@@ -2461,7 +2669,8 @@ public class Jalview2XML
         return id.toString();
       }
       // give up and warn that something has gone wrong
-      Cache.warn("Cannot find ID for object in external mapping : " + jvobj);
+      Console.warn(
+              "Cannot find ID for object in external mapping : " + jvobj);
     }
     return altCode;
   }
@@ -2598,12 +2807,12 @@ public class Jalview2XML
         mp.setDseqFor(jmpid);
         if (!seqRefIds.containsKey(jmpid))
         {
-          Cache.debug("creatign new DseqFor ID");
+          Console.debug("creatign new DseqFor ID");
           seqRefIds.put(jmpid, ps);
         }
         else
         {
-          Cache.debug("reusing DseqFor ID");
+          Console.debug("reusing DseqFor ID");
         }
 
         // mp.setMappingChoice(mpc);
@@ -2769,7 +2978,8 @@ public class Jalview2XML
         });
       } catch (Exception x)
       {
-        System.err.println("Error loading alignment: " + x.getMessage());
+        jalview.bin.Console
+                .errPrintln("Error loading alignment: " + x.getMessage());
       }
     }
     return af;
@@ -2808,19 +3018,22 @@ public class Jalview2XML
         {
           if (bytes != null)
           {
-            // System.out.println("Jalview2XML: opening byte jarInputStream for
+            // jalview.bin.Console.outPrintln("Jalview2XML: opening byte
+            // jarInputStream for
             // bytes.length=" + bytes.length);
             return new JarInputStream(new ByteArrayInputStream(bytes));
           }
           if (_url != null)
           {
-            // System.out.println("Jalview2XML: opening url jarInputStream for "
+            // jalview.bin.Console.outPrintln("Jalview2XML: opening url
+            // jarInputStream for "
             // + _url);
             return new JarInputStream(_url.openStream());
           }
           else
           {
-            // System.out.println("Jalview2XML: opening file jarInputStream for
+            // jalview.bin.Console.outPrintln("Jalview2XML: opening file
+            // jarInputStream for
             // " + file);
             return new JarInputStream(new FileInputStream(file));
           }
@@ -2927,11 +3140,12 @@ public class Jalview2XML
     {
       ex.printStackTrace();
       errorMessage = "Couldn't locate Jalview XML file : " + file;
-      System.err.println(
+      jalview.bin.Console.errPrintln(
               "Exception whilst loading jalview XML file : " + ex + "\n");
     } catch (Exception ex)
     {
-      System.err.println("Parsing as Jalview Version 2 file failed.");
+      jalview.bin.Console
+              .errPrintln("Parsing as Jalview Version 2 file failed.");
       ex.printStackTrace(System.err);
       if (attemptversion1parse)
       {
@@ -2943,18 +3157,19 @@ public class Jalview2XML
       }
       if (af != null)
       {
-        System.out.println("Successfully loaded archive file");
+        jalview.bin.Console.outPrintln("Successfully loaded archive file");
         return af;
       }
       ex.printStackTrace();
 
-      System.err.println(
+      jalview.bin.Console.errPrintln(
               "Exception whilst loading jalview XML file : " + ex + "\n");
     } catch (OutOfMemoryError e)
     {
       // Don't use the OOM Window here
       errorMessage = "Out of memory loading jalview XML file";
-      System.err.println("Out of memory whilst loading jalview XML file");
+      jalview.bin.Console
+              .errPrintln("Out of memory whilst loading jalview XML file");
       e.printStackTrace();
     }
 
@@ -3058,8 +3273,8 @@ public class Jalview2XML
         Desktop.addInternalFrame(af, view.getTitle(),
                 safeInt(view.getWidth()), safeInt(view.getHeight()));
         af.setMenusForViewport();
-        System.err.println("Failed to restore view " + view.getTitle()
-                + " to split frame");
+        jalview.bin.Console.errPrintln("Failed to restore view "
+                + view.getTitle() + " to split frame");
       }
     }
 
@@ -3137,7 +3352,8 @@ public class Jalview2XML
       }
       else
       {
-        System.err.println("Problem loading Jalview file: " + errorMessage);
+        jalview.bin.Console.errPrintln(
+                "Problem loading Jalview file: " + errorMessage);
       }
     }
     errorMessage = null;
@@ -3229,7 +3445,8 @@ public class Jalview2XML
       }
       else
       {
-        Cache.warn("Couldn't find entry in Jalview Jar for " + jarEntryName);
+        Console.warn(
+                "Couldn't find entry in Jalview Jar for " + jarEntryName);
       }
     } catch (Exception ex)
     {
@@ -3334,7 +3551,7 @@ public class Jalview2XML
           if (tmpSeq.getStart() != jseq.getStart()
                   || tmpSeq.getEnd() != jseq.getEnd())
           {
-            System.err.println(String.format(
+            jalview.bin.Console.errPrintln(String.format(
                     "Warning JAL-2154 regression: updating start/end for sequence %s from %d/%d to %d/%d",
                     tmpSeq.getName(), tmpSeq.getStart(), tmpSeq.getEnd(),
                     jseq.getStart(), jseq.getEnd()));
@@ -3829,11 +4046,94 @@ public class Jalview2XML
         jaa.setCalcId(annotation.getCalcId());
         if (annotation.getProperty().size() > 0)
         {
-          for (Annotation.Property prop : annotation.getProperty())
+          for (jalview.xml.binding.jalview.Property prop : annotation
+                  .getProperty())
           {
             jaa.setProperty(prop.getName(), prop.getValue());
           }
         }
+        if (jaa.graph == AlignmentAnnotation.CONTACT_MAP)
+        {
+          if (annotation.getContactmatrix() != null
+                  && annotation.getContactmatrix().size() > 0)
+          {
+            for (MatrixType xmlmat : annotation.getContactmatrix())
+            {
+              if (PAEContactMatrix.PAEMATRIX.equals(xmlmat.getType()))
+              {
+                if (!xmlmat.getRows().equals(xmlmat.getCols()))
+                {
+                  Console.error("Can't handle non square PAE Matrices");
+                }
+                else
+                {
+                  float[][] elements = ContactMatrix
+                          .fromFloatStringToContacts(xmlmat.getElements(),
+                                  xmlmat.getCols().intValue(),
+                                  xmlmat.getRows().intValue());
+                  jalview.util.MapList mapping = null;
+                  if (xmlmat.getMapping() != null)
+                  {
+                    MapListType m = xmlmat.getMapping();
+                    // Mapping m = dr.getMapping();
+                    int fr[] = new int[m.getMapListFrom().size() * 2];
+                    Iterator<MapListFrom> from = m.getMapListFrom()
+                            .iterator();// enumerateMapListFrom();
+                    for (int _i = 0; from.hasNext(); _i += 2)
+                    {
+                      MapListFrom mf = from.next();
+                      fr[_i] = mf.getStart();
+                      fr[_i + 1] = mf.getEnd();
+                    }
+                    int fto[] = new int[m.getMapListTo().size() * 2];
+                    Iterator<MapListTo> to = m.getMapListTo().iterator();// enumerateMapListTo();
+                    for (int _i = 0; to.hasNext(); _i += 2)
+                    {
+                      MapListTo mf = to.next();
+                      fto[_i] = mf.getStart();
+                      fto[_i + 1] = mf.getEnd();
+                    }
+
+                    mapping = new jalview.util.MapList(fr, fto,
+                            m.getMapFromUnit().intValue(),
+                            m.getMapToUnit().intValue());
+                  }
+                  List<BitSet> newgroups = new ArrayList<BitSet>();
+                  if (xmlmat.getGroups().size() > 0)
+                  {
+                    for (String sgroup : xmlmat.getGroups())
+                    {
+                      newgroups.add(deStringifyBitset(sgroup));
+                    }
+                  }
+                  String nwk = xmlmat.getNewick().size() > 0
+                          ? xmlmat.getNewick().get(0)
+                          : null;
+                  if (xmlmat.getNewick().size() > 1)
+                  {
+                    Console.log.info(
+                            "Ignoring additional clusterings for contact matrix");
+                  }
+                  String treeMethod = xmlmat.getTreeMethod();
+                  double thresh = xmlmat.getCutHeight() != null
+                          ? xmlmat.getCutHeight()
+                          : 0;
+                  GroupSet grpset = new GroupSet();
+                  grpset.restoreGroups(newgroups, treeMethod, nwk, thresh);
+                  PAEContactMatrix newpae = new PAEContactMatrix(
+                          jaa.sequenceRef, mapping, elements, grpset);
+                  jaa.sequenceRef.addContactListFor(jaa, newpae);
+                }
+              }
+              else
+              {
+                Console.error("Ignoring CONTACT_MAP annotation with type "
+                        + xmlmat.getType());
+              }
+            }
+          }
+        }
+
         if (jaa.autoCalculated)
         {
           autoAlan.add(new JvAnnotRow(i, jaa));
@@ -3982,7 +4282,7 @@ public class Jalview2XML
       // XML.
       // and then recover its containing af to allow the settings to be applied.
       // TODO: fix for vamsas demo
-      System.err.println(
+      jalview.bin.Console.errPrintln(
               "About to recover a viewport for existing alignment: Sequence set ID is "
                       + uniqueSeqSetId);
       Object seqsetobj = retrieveExistingObj(uniqueSeqSetId);
@@ -3991,13 +4291,13 @@ public class Jalview2XML
         if (seqsetobj instanceof String)
         {
           uniqueSeqSetId = (String) seqsetobj;
-          System.err.println(
+          jalview.bin.Console.errPrintln(
                   "Recovered extant sequence set ID mapping for ID : New Sequence set ID is "
                           + uniqueSeqSetId);
         }
         else
         {
-          System.err.println(
+          jalview.bin.Console.errPrintln(
                   "Warning : Collision between sequence set ID string and existing jalview object mapping.");
         }
 
@@ -4044,7 +4344,7 @@ public class Jalview2XML
     }
 
     /*
-     * Load any trees, PDB structures and viewers
+     * Load any trees, PDB structures and viewers, Overview
      * 
      * Not done if flag is false (when this method is used for New View)
      */
@@ -4054,12 +4354,49 @@ public class Jalview2XML
       loadPCAViewers(jalviewModel, ap);
       loadPDBStructures(jprovider, jseqs, af, ap);
       loadRnaViewers(jprovider, jseqs, ap);
+      loadOverview(view, jalviewModel.getVersion(), af);
     }
     // and finally return.
     return af;
   }
 
   /**
+   * Load Overview window, restoring colours, 'show hidden regions' flag, title
+   * and geometry as saved
+   * 
+   * @param view
+   * @param af
+   */
+  protected void loadOverview(Viewport view, String version, AlignFrame af)
+  {
+    if (!isVersionStringLaterThan("2.11.3", version)
+            && view.getOverview() == null)
+    {
+      return;
+    }
+    /*
+     * first close any Overview that was opened automatically
+     * (if so configured in Preferences) so that the view is
+     * restored in the same state as saved
+     */
+    af.alignPanel.closeOverviewPanel();
+
+    Overview overview = view.getOverview();
+    if (overview != null)
+    {
+      OverviewPanel overviewPanel = af
+              .openOverviewPanel(overview.isShowHidden());
+      overviewPanel.setTitle(overview.getTitle());
+      overviewPanel.setFrameBounds(overview.getXpos(), overview.getYpos(),
+              overview.getWidth(), overview.getHeight());
+      Color gap = new Color(overview.getGapColour());
+      Color residue = new Color(overview.getResidueColour());
+      Color hidden = new Color(overview.getHiddenColour());
+      overviewPanel.getCanvas().setColours(gap, residue, hidden);
+    }
+  }
+
+  /**
    * Instantiate and link any saved RNA (Varna) viewers. The state of the Varna
    * panel is restored from separate jar entries, two (gapped and trimmed) per
    * sequence and secondary structure.
@@ -4178,10 +4515,28 @@ public class Jalview2XML
         TreePanel tp = (TreePanel) retrieveExistingObj(tree.getId());
         if (tp == null)
         {
-          tp = af.showNewickTree(new NewickFile(tree.getNewick()),
-                  tree.getTitle(), safeInt(tree.getWidth()),
-                  safeInt(tree.getHeight()), safeInt(tree.getXpos()),
-                  safeInt(tree.getYpos()));
+          if (tree.isColumnWise())
+          {
+            AlignmentAnnotation aa = annotationIds
+                    .get(tree.getColumnReference());
+            if (aa == null)
+            {
+              Console.warn(
+                      "Null alignment annotation when restoring columnwise tree");
+            }
+            tp = af.showColumnWiseTree(new NewickFile(tree.getNewick()), aa,
+                    tree.getTitle(), safeInt(tree.getWidth()),
+                    safeInt(tree.getHeight()), safeInt(tree.getXpos()),
+                    safeInt(tree.getYpos()));
+
+          }
+          else
+          {
+            tp = af.showNewickTree(new NewickFile(tree.getNewick()),
+                    tree.getTitle(), safeInt(tree.getWidth()),
+                    safeInt(tree.getHeight()), safeInt(tree.getXpos()),
+                    safeInt(tree.getYpos()));
+          }
           if (tree.getId() != null)
           {
             // perhaps bind the tree id to something ?
@@ -4204,8 +4559,9 @@ public class Jalview2XML
         tp.getTreeCanvas().setApplyToAllViews(tree.isLinkToAllViews());
         if (tp == null)
         {
-          Cache.warn("There was a problem recovering stored Newick tree: \n"
-                  + tree.getNewick());
+          Console.warn(
+                  "There was a problem recovering stored Newick tree: \n"
+                          + tree.getNewick());
           continue;
         }
 
@@ -4366,7 +4722,7 @@ public class Jalview2XML
             else
             {
               errorMessage = ("The Jmol views in this project were imported\nfrom an older version of Jalview.\nPlease review the sequence colour associations\nin the Colour by section of the Jmol View menu.\n\nIn the case of problems, see note at\nhttp://issues.jalview.org/browse/JAL-747");
-              Cache.warn(errorMessage);
+              Console.warn(errorMessage);
             }
           }
         }
@@ -4381,7 +4737,7 @@ public class Jalview2XML
         createOrLinkStructureViewer(entry, af, ap, jprovider);
       } catch (Exception e)
       {
-        System.err.println(
+        jalview.bin.Console.errPrintln(
                 "Error loading structure viewer: " + e.getMessage());
         // failed - try the next one
       }
@@ -4421,7 +4777,7 @@ public class Jalview2XML
     } catch (IllegalArgumentException | NullPointerException e)
     {
       // TODO JAL-3619 show error dialog / offer an alternative viewer
-      Cache.error("Invalid structure viewer type: " + type);
+      Console.error("Invalid structure viewer type: " + type);
     }
   }
 
@@ -4575,7 +4931,7 @@ public class Jalview2XML
    *          - minimum version we are comparing against
    * @param version
    *          - version of data being processsed
-   * @return
+   * @return true if version is equal to or later than supported
    */
   public static boolean isVersionStringLaterThan(String supported,
           String version)
@@ -4585,7 +4941,7 @@ public class Jalview2XML
             || version.equalsIgnoreCase("Test")
             || version.equalsIgnoreCase("AUTOMATED BUILD"))
     {
-      System.err.println("Assuming project file with "
+      jalview.bin.Console.errPrintln("Assuming project file with "
               + (version == null ? "null" : version)
               + " is compatible with Jalview version " + supported);
       return true;
@@ -4632,7 +4988,7 @@ public class Jalview2XML
     //
     // @Override
     // protected void processKeyEvent(java.awt.event.KeyEvent e) {
-    // System.out.println("Jalview2XML AF " + e);
+    // jalview.bin.Console.outPrintln("Jalview2XML AF " + e);
     // super.processKeyEvent(e);
     //
     // }
@@ -4720,9 +5076,15 @@ public class Jalview2XML
     viewport.setIncrement(safeInt(view.getConsThreshold()));
     viewport.setShowJVSuffix(safeBoolean(view.isShowFullId()));
     viewport.setRightAlignIds(safeBoolean(view.isRightAlignIds()));
-    viewport.setFont(new Font(view.getFontName(),
-            safeInt(view.getFontStyle()), safeInt(view.getFontSize())),
-            true);
+    viewport.setFont(
+            new Font(view.getFontName(), safeInt(view.getFontStyle()),
+                    safeInt(view.getFontSize())),
+            (view.getCharWidth() != null) ? false : true);
+    if (view.getCharWidth() != null)
+    {
+      viewport.setCharWidth(view.getCharWidth());
+      viewport.setCharHeight(view.getCharHeight());
+    }
     ViewStyleI vs = viewport.getViewStyle();
     vs.setScaleProteinAsCdna(view.isScaleProteinAsCdna());
     viewport.setViewStyle(vs);
@@ -4953,7 +5315,7 @@ public class Jalview2XML
           }
           else
           {
-            Cache.warn("Couldn't recover parameters for "
+            Console.warn("Couldn't recover parameters for "
                     + calcIdParam.getCalcId());
           }
         }
@@ -4981,6 +5343,7 @@ public class Jalview2XML
     {
       splitFrameCandidates.put(view, af);
     }
+
     return af;
   }
 
@@ -5040,11 +5403,15 @@ public class Jalview2XML
     }
     if (matchedAnnotation == null)
     {
-      System.err.println("Failed to match annotation colour scheme for "
-              + annotationId);
+      jalview.bin.Console
+              .errPrintln("Failed to match annotation colour scheme for "
+                      + annotationId);
       return null;
     }
-    if (matchedAnnotation.getThreshold() == null)
+    // belt-and-braces create a threshold line if the
+    // colourscheme needs one but the matchedAnnotation doesn't have one
+    if (safeInt(viewAnnColour.getAboveThreshold()) != 0
+            && matchedAnnotation.getThreshold() == null)
     {
       matchedAnnotation.setThreshold(
               new GraphLine(safeFloat(viewAnnColour.getThreshold()),
@@ -5229,7 +5596,7 @@ public class Jalview2XML
     String id = object.getViewport().get(0).getSequenceSetId();
     if (skipList.containsKey(id))
     {
-      Cache.debug("Skipping seuqence set id " + id);
+      Console.debug("Skipping seuqence set id " + id);
       return true;
     }
     return false;
@@ -5281,14 +5648,16 @@ public class Jalview2XML
       {
         if (ds != null && ds != seqSetDS)
         {
-          Cache.warn("JAL-3171 regression: Overwriting a dataset reference for an alignment"
-                  + " - CDS/Protein crossreference data may be lost");
+          Console.warn(
+                  "JAL-3171 regression: Overwriting a dataset reference for an alignment"
+                          + " - CDS/Protein crossreference data may be lost");
           if (xtant_ds != null)
           {
             // This can only happen if the unique sequence set ID was bound to a
             // dataset that did not contain any of the sequences in the view
             // currently being restored.
-            Cache.warn("JAL-3171 SERIOUS!  TOTAL CONFUSION - please consider contacting the Jalview Development team so they can investigate why your project caused this message to be displayed.");
+            Console.warn(
+                    "JAL-3171 SERIOUS!  TOTAL CONFUSION - please consider contacting the Jalview Development team so they can investigate why your project caused this message to be displayed.");
           }
         }
         ds = seqSetDS;
@@ -5313,7 +5682,7 @@ public class Jalview2XML
       SequenceI[] dsseqs = new SequenceI[dseqs.size()];
       dseqs.copyInto(dsseqs);
       ds = new jalview.datamodel.Alignment(dsseqs);
-      Cache.debug("Created new dataset " + vamsasSet.getDatasetId()
+      Console.debug("Created new dataset " + vamsasSet.getDatasetId()
               + " for alignment " + System.identityHashCode(al));
       addDatasetRef(vamsasSet.getDatasetId(), ds);
     }
@@ -5366,7 +5735,7 @@ public class Jalview2XML
       AlignmentI prevDS = seqToDataset.put(restoredSeq.getDsseqid(), ds);
       if (prevDS != null && prevDS != ds)
       {
-        Cache.warn("Dataset sequence appears in many datasets: "
+        Console.warn("Dataset sequence appears in many datasets: "
                 + restoredSeq.getDsseqid());
         // TODO: try to merge!
       }
@@ -5494,7 +5863,7 @@ public class Jalview2XML
         }
         // TODO: merges will never happen if we 'know' we have the real dataset
         // sequence - this should be detected when id==dssid
-        System.err.println(
+        jalview.bin.Console.errPrintln(
                 "DEBUG Notice:  Merged dataset sequence (if you see this often, post at http://issues.jalview.org/browse/JAL-1474)"); // ("
         // + (pre ? "prepended" : "") + " "
         // + (post ? "appended" : ""));
@@ -5572,7 +5941,8 @@ public class Jalview2XML
   {
     if (dataset.getDataset() != null)
     {
-      Cache.warn("Serious issue!  Dataset Object passed to getDatasetIdRef is not a Jalview DATASET alignment...");
+      Console.warn(
+              "Serious issue!  Dataset Object passed to getDatasetIdRef is not a Jalview DATASET alignment...");
     }
     String datasetId = makeHashCode(dataset, null);
     if (datasetId == null)
@@ -5686,7 +6056,7 @@ public class Jalview2XML
       }
       else
       {
-        System.err.println(
+        jalview.bin.Console.errPrintln(
                 "Warning - making up dataset sequence id for DbRef sequence map reference");
         sqid = ((Object) ms).toString(); // make up a new hascode for
         // undefined dataset sequence hash
@@ -5708,7 +6078,7 @@ public class Jalview2XML
         seqRefIds.put(sqid, djs);
 
       }
-      Cache.debug("about to recurse on addDBRefs.");
+      Console.debug("about to recurse on addDBRefs.");
       addDBRefs(djs, ms);
 
     }
@@ -5819,7 +6189,7 @@ public class Jalview2XML
         if (!jvann.annotationId.equals(anid))
         {
           // TODO verify that this is the correct behaviour
-          Cache.warn("Overriding Annotation ID for " + anid
+          Console.warn("Overriding Annotation ID for " + anid
                   + " from different id : " + jvann.annotationId);
           jvann.annotationId = anid;
         }
@@ -5834,7 +6204,7 @@ public class Jalview2XML
       }
       else
       {
-        Cache.debug("Ignoring " + jvobj.getClass() + " (ID = " + id);
+        Console.debug("Ignoring " + jvobj.getClass() + " (ID = " + id);
       }
     }
   }
@@ -5904,7 +6274,8 @@ public class Jalview2XML
       }
       else
       {
-        Cache.warn("Couldn't find entry in Jalview Jar for " + jarEntryName);
+        Console.warn(
+                "Couldn't find entry in Jalview Jar for " + jarEntryName);
       }
     } catch (Exception ex)
     {
@@ -6054,7 +6425,7 @@ public class Jalview2XML
       }
     } catch (Exception ex)
     {
-      Cache.error("Error loading PCA: " + ex.toString());
+      Console.error("Error loading PCA: " + ex.toString());
     }
   }
 
@@ -6114,7 +6485,7 @@ public class Jalview2XML
       });
     } catch (InvocationTargetException | InterruptedException ex)
     {
-      Cache.warn("Unexpected error when opening " + viewerType
+      Console.warn("Unexpected error when opening " + viewerType
               + " structure viewer", ex);
     }
   }
@@ -6221,7 +6592,7 @@ public class Jalview2XML
       }
     } catch (IOException e)
     {
-      Cache.error("Error restoring Jmol session: " + e.toString());
+      Console.error("Error restoring Jmol session: " + e.toString());
     }
     return null;
   }
@@ -6366,7 +6737,7 @@ public class Jalview2XML
     } catch (IllegalStateException e)
     {
       // mixing AND and OR conditions perhaps
-      System.err.println(
+      jalview.bin.Console.errPrintln(
               String.format("Error reading filter conditions for '%s': %s",
                       featureType, e.getMessage()));
       // return as much as was parsed up to the error
@@ -6447,7 +6818,8 @@ public class Jalview2XML
       }
       else
       {
-        System.err.println("Malformed compound filter condition");
+        jalview.bin.Console
+                .errPrintln("Malformed compound filter condition");
       }
     }
   }
@@ -6474,7 +6846,7 @@ public class Jalview2XML
         maxcol = new Color(Integer.parseInt(colourModel.getRGB(), 16));
       } catch (Exception e)
       {
-        Cache.warn("Couldn't parse out graduated feature color.", e);
+        Console.warn("Couldn't parse out graduated feature color.", e);
       }
 
       NoValueColour noCol = colourModel.getNoValueColour();
@@ -6528,4 +6900,44 @@ public class Jalview2XML
 
     return colour;
   }
+
+  public static void setStateSavedUpToDate(boolean s)
+  {
+    Console.debug("Setting overall stateSavedUpToDate to " + s);
+    stateSavedUpToDate = s;
+  }
+
+  public static boolean stateSavedUpToDate()
+  {
+    Console.debug("Returning overall stateSavedUpToDate value: "
+            + stateSavedUpToDate);
+    return stateSavedUpToDate;
+  }
+
+  public static boolean allSavedUpToDate()
+  {
+    if (stateSavedUpToDate()) // nothing happened since last project save
+      return true;
+
+    AlignFrame[] frames = Desktop.getDesktopAlignFrames();
+    if (frames != null)
+    {
+      for (int i = 0; i < frames.length; i++)
+      {
+        if (frames[i] == null)
+          continue;
+        if (!frames[i].getViewport().savedUpToDate())
+          return false; // at least one alignment is not individually saved
+      }
+    }
+    return true;
+  }
+
+  // used for debugging and tests
+  private static int debugDelaySave = 20;
+
+  public static void setDebugDelaySave(int n)
+  {
+    debugDelaySave = n;
+  }
 }