Merge branch 'Jalview-JS/jim/JAL-3253-JAL-3418' into Jalview-JS/JAL-3253-applet
[jalview.git] / test / jalview / io / CrossRef2xmlTests.java
index 36ea69e..daadfa5 100644 (file)
  */
 package jalview.io;
 
-import static org.testng.AssertJUnit.assertEquals;
-import static org.testng.AssertJUnit.assertFalse;
-import static org.testng.AssertJUnit.assertNotNull;
-import static org.testng.AssertJUnit.assertSame;
-import static org.testng.AssertJUnit.assertTrue;
-
-import jalview.api.AlignViewportI;
+import jalview.analysis.CrossRef;
 import jalview.api.AlignmentViewPanel;
-import jalview.api.ViewStyleI;
-import jalview.datamodel.AlignmentAnnotation;
+import jalview.datamodel.AlignedCodonFrame;
 import jalview.datamodel.AlignmentI;
-import jalview.datamodel.HiddenSequences;
-import jalview.datamodel.SequenceCollectionI;
-import jalview.datamodel.SequenceGroup;
+import jalview.datamodel.AlignmentTest;
 import jalview.datamodel.SequenceI;
 import jalview.gui.AlignFrame;
+import jalview.gui.CrossRefAction;
 import jalview.gui.Desktop;
-import jalview.gui.Jalview2XML;
-import jalview.schemes.AnnotationColourGradient;
-import jalview.schemes.ColourSchemeI;
-import jalview.structure.StructureImportSettings;
-import jalview.viewmodel.AlignmentViewport;
+import jalview.gui.JvOptionPane;
+import jalview.gui.SequenceFetcher;
+import jalview.project.Jalview2XML;
+import jalview.util.DBRefUtils;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 import org.testng.Assert;
-import org.testng.AssertJUnit;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
+import junit.extensions.PA;
+
 @Test(singleThreaded = true)
 public class CrossRef2xmlTests extends Jalview2xmlBase
 {
 
-  @Test(groups = { "Functional" })
-  public void testRNAStructureRecovery() throws Exception
+  @Override
+  @BeforeClass(alwaysRun = true)
+  public void setUpJvOptionPane()
   {
-    String inFile = "examples/RF00031_folded.stk";
-    String tfile = File.createTempFile("JalviewTest", ".jvp")
-            .getAbsolutePath();
-    AlignFrame af = new jalview.io.FileLoader().LoadFileWaitTillLoaded(
-            inFile, FormatAdapter.FILE);
-    assertTrue("Didn't read input file " + inFile, af != null);
-    int olddsann = countDsAnn(af.getViewport());
-    assertTrue("Didn't find any dataset annotations", olddsann > 0);
-    af.rnahelicesColour_actionPerformed(null);
-    assertTrue(
-            "Couldn't apply RNA helices colourscheme",
-            af.getViewport().getGlobalColourScheme() instanceof jalview.schemes.RNAHelicesColour);
-    assertTrue("Failed to store as a project.",
-            af.saveAlignment(tfile, "Jalview"));
-    af.closeMenuItem_actionPerformed(true);
-    af = null;
-    af = new jalview.io.FileLoader().LoadFileWaitTillLoaded(tfile,
-            FormatAdapter.FILE);
-    assertTrue("Failed to import new project", af != null);
-    int newdsann = countDsAnn(af.getViewport());
-    assertTrue(
-            "Differing numbers of dataset sequence annotation\nOriginally "
-                    + olddsann + " and now " + newdsann,
-            olddsann == newdsann);
-    System.out
-            .println("Read in same number of annotations as originally present ("
-                    + olddsann + ")");
-    assertTrue(
-            "RNA helices colourscheme was not applied on import.",
-            af.getViewport().getGlobalColourScheme() instanceof jalview.schemes.RNAHelicesColour);
+    JvOptionPane.setInteractiveMode(false);
+    JvOptionPane.setMockResponse(JvOptionPane.CANCEL_OPTION);
   }
 
-  @Test(groups = { "Functional" })
-  public void testTCoffeeScores() throws Exception
+  @DataProvider(name = "initialAccessions")
+  static Object[][] getAccessions()
   {
-    String inFile = "examples/uniref50.fa", inAnnot = "examples/uniref50.score_ascii";
-    String tfile = File.createTempFile("JalviewTest", ".jvp")
-            .getAbsolutePath();
-    AlignFrame af = new jalview.io.FileLoader().LoadFileWaitTillLoaded(
-            inFile, FormatAdapter.FILE);
-    assertTrue("Didn't read input file " + inFile, af != null);
-    af.loadJalviewDataFile(inAnnot, FormatAdapter.FILE, null, null);
-    assertTrue(
-            "Didn't set T-coffee colourscheme",
-            af.getViewport().getGlobalColourScheme().getClass()
-                    .equals(jalview.schemes.TCoffeeColourScheme.class));
-    assertTrue(
-            "Recognise T-Coffee score from string",
-            jalview.schemes.ColourSchemeProperty.getColour(af.getViewport()
-                    .getAlignment(),
-                    jalview.schemes.ColourSchemeProperty.getColourName(af
-                            .getViewport().getGlobalColourScheme())) != null);
-
-    assertTrue("Failed to store as a project.",
-            af.saveAlignment(tfile, "Jalview"));
-    af.closeMenuItem_actionPerformed(true);
-    af = null;
-    af = new jalview.io.FileLoader().LoadFileWaitTillLoaded(tfile,
-            FormatAdapter.FILE);
-    assertTrue("Failed to import new project", af != null);
-    assertTrue(
-            "Didn't set T-coffee colourscheme for imported project.",
-            af.getViewport().getGlobalColourScheme().getClass()
-                    .equals(jalview.schemes.TCoffeeColourScheme.class));
-    System.out
-            .println("T-Coffee score shading successfully recovered from project.");
+    return new String[][] { { "UNIPROT", "P00338" },
+        { "UNIPROT", "Q8Z9G6" },
+        { "ENSEMBLGENOMES", "CAD01290" } };
   }
 
-  @Test(groups = { "Functional" })
-  public void testColourByAnnotScores() throws Exception
+  /**
+   * test store and recovery of all reachable cross refs from all reachable
+   * crossrefs for one or more fetched db refs. Currently, this test has a known
+   * failure case.
+   * 
+   * @throws Exception
+   */
+  @Test(
+    groups =
+    { "Operational" },
+    dataProvider = "initialAccessions",
+    enabled = true)
+  public void testRetrieveAndShowCrossref(String forSource,
+          String forAccession) throws Exception
   {
-    String inFile = "examples/uniref50.fa", inAnnot = "examples/testdata/uniref50_iupred.jva";
-    String tfile = File.createTempFile("JalviewTest", ".jvp")
-            .getAbsolutePath();
-    AlignFrame af = new jalview.io.FileLoader().LoadFileWaitTillLoaded(
-            inFile, FormatAdapter.FILE);
-    assertTrue("Didn't read input file " + inFile, af != null);
-    af.loadJalviewDataFile(inAnnot, FormatAdapter.FILE, null, null);
-    AlignmentAnnotation[] aa = af.getViewport().getAlignment()
-            .getSequenceAt(0).getAnnotation("IUPredWS (Short)");
-    assertTrue(
-            "Didn't find any IUPred annotation to use to shade alignment.",
-            aa != null && aa.length > 0);
-    AnnotationColourGradient cs = new jalview.schemes.AnnotationColourGradient(
-            aa[0], null, AnnotationColourGradient.ABOVE_THRESHOLD);
-    AnnotationColourGradient gcs = new jalview.schemes.AnnotationColourGradient(
-            aa[0], null, AnnotationColourGradient.BELOW_THRESHOLD);
-    cs.setSeqAssociated(true);
-    gcs.setSeqAssociated(true);
-    af.changeColour(cs);
-    SequenceGroup sg = new SequenceGroup();
-    sg.setStartRes(57);
-    sg.setEndRes(92);
-    sg.cs = gcs;
-    af.getViewport().getAlignment().addGroup(sg);
-    sg.addSequence(af.getViewport().getAlignment().getSequenceAt(1), false);
-    sg.addSequence(af.getViewport().getAlignment().getSequenceAt(2), true);
-    af.alignPanel.alignmentChanged();
-    assertTrue("Failed to store as a project.",
-            af.saveAlignment(tfile, "Jalview"));
-    af.closeMenuItem_actionPerformed(true);
-    af = null;
-    af = new jalview.io.FileLoader().LoadFileWaitTillLoaded(tfile,
-            FormatAdapter.FILE);
-    assertTrue("Failed to import new project", af != null);
-
-    // check for group and alignment colourschemes
-
-    ColourSchemeI _rcs = af.getViewport().getGlobalColourScheme();
-    ColourSchemeI _rgcs = af.getViewport().getAlignment().getGroups()
-            .get(0).cs;
-    assertTrue("Didn't recover global colourscheme", _rcs != null);
-    assertTrue("Didn't recover annotation colour global scheme",
-            _rcs instanceof AnnotationColourGradient);
-    AnnotationColourGradient __rcs = (AnnotationColourGradient) _rcs;
-    assertTrue("Annotation colourscheme wasn't sequence associated",
-            __rcs.isSeqAssociated());
-
-    boolean diffseqcols = false, diffgseqcols = false;
-    SequenceI[] sqs = af.getViewport().getAlignment().getSequencesArray();
-    for (int p = 0, pSize = af.getViewport().getAlignment().getWidth(); p < pSize
-            && (!diffseqcols || !diffgseqcols); p++)
+
+    List<String> failedDBRetr = new ArrayList<>();
+    List<String> failedXrefMenuItems = new ArrayList<>();
+    List<String> failedProjectRecoveries = new ArrayList<>();
+    // only search for ensembl or Uniprot crossrefs
+    List<String> limit = Arrays
+            .asList(new String[]
+            { DBRefUtils.getCanonicalName("ENSEMBL"),
+                DBRefUtils.getCanonicalName("Uniprot") });
+    // for every set of db queries
+    // retrieve db query
+    // verify presence of expected xrefs
+    // show xrefs - verify expected type of frame is shown for each xref
+    // show xrefs again
+    // - verify original -> xref -> xref(original) recovers frame containing at
+    // least the first retrieved sequence
+    // store
+    // 1. whole project
+    // 2. individual frames
+    // 3. load each one back and verify
+    // . aligned sequences (.toString() )
+    // . xrefs (.toString() )
+    // . codonframes
+    //
+    //
+    Map<String, String> dbtoviewBit = new HashMap<>();
+    List<String> keyseq = new ArrayList<>();
+    Map<String, File> savedProjects = new HashMap<>();
+
+    // for (String[] did : new String[][] { { "UNIPROT", "P00338" } })
+    // {
+    // pass counters - 0 - first pass, 1 means retrieve project rather than
+    // perform action
+    int pass1 = 0, pass2 = 0, pass3 = 0;
+    // each do loop performs two iterations in the first outer loop pass, but
+    // only performs one iteration on the second outer loop
+    // ie. pass 1 = 0 {pass 2= 0 { pass 3 = 0,1 }, pass 2=1 { pass 3 = 0 }}, 1
+    // { pass 2 = 0 { pass 3 = 0 } }
+    do
     {
-      if (_rcs.findColour(sqs[0].getCharAt(p), p, sqs[0]) != _rcs
-              .findColour(sqs[5].getCharAt(p), p, sqs[5]))
+      String first = forSource + " " + forAccession;// did[0] + " " + did[1];
+      AlignFrame af = null;
+      boolean dna;
+      AlignmentI retral;
+      AlignmentI dataset;
+      SequenceI[] seqs;
+      List<String> ptypes = null;
+      if (pass1 == 0)
       {
-        diffseqcols = true;
+        // retrieve dbref
+
+        SequenceFetcher sf = new SequenceFetcher(Desktop.getInstance(),
+                forSource, forAccession);
+        sf.run();
+        AlignFrame[] afs = Desktop.getAlignFrames();
+        if (afs.length == 0)
+        {
+          failedDBRetr.add("Didn't retrieve " + first);
+          break;
+        }
+        keyseq.add(first);
+        af = afs[0];
+
+        // verify references for retrieved data
+        AlignmentTest.assertAlignmentDatasetRefs(
+                af.getViewport().getAlignment(), "Pass (" + pass1 + ","
+                        + pass2 + "," + pass3 + "): Fetch " + first + ":");
+        assertDatasetIsNormalisedKnownDefect(
+                af.getViewport().getAlignment(), "Pass (" + pass1 + ","
+                        + pass2 + "," + pass3 + "): Fetch " + first + ":");
+        dna = af.getViewport().getAlignment().isNucleotide();
+        retral = af.getViewport().getAlignment();
+        dataset = retral.getDataset();
+        seqs = retral.getSequencesArray();
+
       }
-    }
-    assertTrue("Got Different sequence colours", diffseqcols);
-    System.out
-            .println("Per sequence colourscheme (Background) successfully applied and recovered.");
-
-    assertTrue("Didn't recover group colourscheme", _rgcs != null);
-    assertTrue("Didn't recover annotation colour group colourscheme",
-            _rgcs instanceof AnnotationColourGradient);
-    __rcs = (AnnotationColourGradient) _rgcs;
-    assertTrue("Group Annotation colourscheme wasn't sequence associated",
-            __rcs.isSeqAssociated());
-
-    for (int p = 0, pSize = af.getViewport().getAlignment().getWidth(); p < pSize
-            && (!diffseqcols || !diffgseqcols); p++)
-    {
-      if (_rgcs.findColour(sqs[1].getCharAt(p), p, sqs[1]) != _rgcs
-              .findColour(sqs[2].getCharAt(p), p, sqs[2]))
+      else
       {
-        diffgseqcols = true;
+        Desktop.getInstance().closeAll_actionPerformed(null);
+        // recover stored project
+        af = new FileLoader(false).LoadFileWaitTillLoaded(
+                savedProjects.get(first).toString(), DataSourceType.FILE);
+        System.out.println("Recovered view for '" + first + "' from '"
+                + savedProjects.get(first).toString() + "'");
+        dna = af.getViewport().getAlignment().isNucleotide();
+        retral = af.getViewport().getAlignment();
+        dataset = retral.getDataset();
+        seqs = retral.getSequencesArray();
+
+        // verify references for recovered data
+        AlignmentTest.assertAlignmentDatasetRefs(
+                af.getViewport().getAlignment(),
+                "Pass (" + pass1 + "," + pass2 + "," + pass3 + "): Recover "
+                        + first + ":");
+        assertDatasetIsNormalisedKnownDefect(
+                af.getViewport().getAlignment(),
+                "Pass (" + pass1 + "," + pass2 + "," + pass3 + "): Recover "
+                        + first + ":");
+
       }
-    }
-    assertTrue("Got Different group sequence colours", diffgseqcols);
-    System.out
-            .println("Per sequence (Group) colourscheme successfully applied and recovered.");
-  }
 
-  @Test(groups = { "Functional" })
-  public void gatherViewsHere() throws Exception
-  {
-    int origCount = Desktop.getAlignFrames() == null ? 0 : Desktop
-            .getAlignFrames().length;
-    AlignFrame af = new jalview.io.FileLoader().LoadFileWaitTillLoaded(
-            "examples/exampleFile_2_7.jar", FormatAdapter.FILE);
-    assertTrue("Didn't read in the example file correctly.", af != null);
-    assertTrue("Didn't gather the views in the example file.",
-            Desktop.getAlignFrames().length == 1 + origCount);
+      // store project on first pass, compare next pass
+      stringify(dbtoviewBit, savedProjects, first, af.alignPanel);
 
-  }
+      ptypes = (seqs == null || seqs.length == 0) ? null
+              : new CrossRef(seqs, dataset)
+                      .findXrefSourcesForSequences(dna);
+      filterDbRefs(ptypes, limit);
 
-  @Test(groups = { "Functional" })
-  public void viewRefPdbAnnotation() throws Exception
-  {
-    // TODO: Make this pass without setting StructureParser.JALVIEW_PARSER
-    // StructureImportSettings
-    // .setDefaultPDBFileParser(StructureParser.JALVIEW_PARSER);
-    StructureImportSettings.setProcessSecondaryStructure(true);
-    StructureImportSettings.setVisibleChainAnnotation(true);
-    AlignFrame af = new jalview.io.FileLoader().LoadFileWaitTillLoaded(
-            "examples/exampleFile_2_7.jar", FormatAdapter.FILE);
-    assertTrue("Didn't read in the example file correctly.", af != null);
-    AlignmentViewPanel sps = null;
-    for (AlignmentViewPanel ap : af.alignPanel.alignFrame.getAlignPanels())
+      // start of pass2: retrieve each cross-ref for fetched or restored
+      // project.
+      do // first cross ref and recover crossref loop
+      {
+
+        for (String db : ptypes)
+        {
+          // counter for splitframe views retrieved via crossref
+          int firstcr_ap = 0;
+          // build next key so we an retrieve all views
+          String nextxref = first + " -> " + db + "{" + firstcr_ap + "}";
+          // perform crossref action, or retrieve stored project
+          List<AlignmentViewPanel> cra_views = new ArrayList<>();
+          CrossRefAction cra = null;
+
+          if (pass2 == 0)
+          { // retrieve and show cross-refs in this thread
+            cra = CrossRefAction.getHandlerFor(seqs, dna, db, af);
+            cra.run();
+            cra_views = (List<AlignmentViewPanel>) PA.getValue(cra,
+                    "xrefViews");
+            if (cra_views.size() == 0)
+            {
+              failedXrefMenuItems.add(
+                      "No crossrefs retrieved for " + first + " -> " + db);
+              continue;
+            }
+            assertNucleotide(cra_views.get(0),
+                    "Nucleotide panel included proteins for " + first
+                            + " -> " + db);
+            assertProtein(cra_views.get(1),
+                    "Protein panel included nucleotides for " + first
+                            + " -> " + db);
+          }
+          else
+          {
+            Desktop.getInstance().closeAll_actionPerformed(null);
+            pass3 = 0;
+            // recover stored project
+            File storedProject = savedProjects.get(nextxref);
+            if (storedProject == null)
+            {
+              failedProjectRecoveries
+                      .add("Failed to store a view for '" + nextxref + "'");
+              continue;
+            }
+
+            // recover stored project
+            AlignFrame af2 = new FileLoader(false).LoadFileWaitTillLoaded(
+                    savedProjects.get(nextxref).toString(),
+                    DataSourceType.FILE);
+            System.out
+                    .println("Recovered view for '" + nextxref + "' from '"
+                            + savedProjects.get(nextxref).toString() + "'");
+            // gymnastics to recover the alignPanel/Complementary alignPanel
+            if (af2.getViewport().isNucleotide())
+            {
+              // top view, then bottom
+              cra_views.add(af2.getViewport().getAlignPanel());
+              cra_views.add(((jalview.gui.AlignViewport) af2.getViewport()
+                      .getCodingComplement()).getAlignPanel());
+
+            }
+            else
+            {
+              // bottom view, then top
+              cra_views.add(((jalview.gui.AlignViewport) af2.getViewport()
+                      .getCodingComplement()).getAlignPanel());
+              cra_views.add(af2.getViewport().getAlignPanel());
+
+            }
+          }
+          HashMap<String, List<String>> xrptypes = new HashMap<>();
+          // first save/verify views.
+          for (AlignmentViewPanel avp : cra_views)
+          {
+            nextxref = first + " -> " + db + "{" + firstcr_ap++ + "}";
+            // verify references for this panel
+            AlignmentTest.assertAlignmentDatasetRefs(avp.getAlignment(),
+                    "Pass (" + pass1 + "," + pass2 + "," + pass3
+                            + "): before start of pass3: " + nextxref
+                            + ":");
+            assertDatasetIsNormalisedKnownDefect(avp.getAlignment(),
+                    "Pass (" + pass1 + "," + pass2 + "," + pass3
+                            + "): before start of pass3: " + nextxref
+                            + ":");
+
+            SequenceI[] xrseqs = avp.getAlignment().getSequencesArray();
+
+            List<String> _xrptypes = (seqs == null || seqs.length == 0)
+                    ? null
+                    : new CrossRef(xrseqs, dataset)
+                            .findXrefSourcesForSequences(
+                                    avp.getAlignViewport().isNucleotide());
+
+            stringify(dbtoviewBit, savedProjects, nextxref, avp);
+            xrptypes.put(nextxref, _xrptypes);
+
+          }
+
+          // now do the second xref pass starting from either saved or just
+          // recovered split pane, in sequence
+          do // retrieve second set of cross refs or recover and verify
+          {
+            firstcr_ap = 0;
+            for (AlignmentViewPanel avp : cra_views)
+            {
+              nextxref = first + " -> " + db + "{" + firstcr_ap++ + "}";
+              for (String xrefdb : xrptypes.get(nextxref))
+              {
+                List<AlignmentViewPanel> cra_views2 = new ArrayList<>();
+                int q = 0;
+                String nextnextxref = nextxref + " -> " + xrefdb + "{" + q
+                        + "}";
+
+                if (pass3 == 0)
+                {
+                  SequenceI[] xrseqs = avp.getAlignment()
+                          .getSequencesArray();
+                  AlignFrame nextaf = Desktop
+                          .getAlignFrameFor(avp.getAlignViewport());
+
+                  cra = CrossRefAction.getHandlerFor(xrseqs,
+                          avp.getAlignViewport().isNucleotide(), xrefdb,
+                          nextaf);
+                  cra.run();
+                  cra_views2 = (List<AlignmentViewPanel>) PA.getValue(cra,
+                          "xrefViews");
+                  if (cra_views2.size() == 0)
+                  {
+                    failedXrefMenuItems.add("No crossrefs retrieved for '"
+                            + nextxref + "' to " + xrefdb + " via '"
+                            + nextaf.getTitle() + "'");
+                    continue;
+                  }
+                  assertNucleotide(cra_views2.get(0),
+                          "Nucleotide panel included proteins for '"
+                                  + nextxref + "' to " + xrefdb + " via '"
+                                  + nextaf.getTitle() + "'");
+                  assertProtein(cra_views2.get(1),
+                          "Protein panel included nucleotides for '"
+                                  + nextxref + "' to " + xrefdb + " via '"
+                                  + nextaf.getTitle() + "'");
+
+                }
+                else
+                {
+                  Desktop.getInstance().closeAll_actionPerformed(null);
+                  // recover stored project
+                  File storedProject = savedProjects.get(nextnextxref);
+                  if (storedProject == null)
+                  {
+                    failedProjectRecoveries
+                            .add("Failed to store a view for '"
+                                    + nextnextxref + "'");
+                    continue;
+                  }
+                  AlignFrame af2 = new FileLoader(false)
+                          .LoadFileWaitTillLoaded(savedProjects
+                                  .get(nextnextxref).toString(),
+                                  DataSourceType.FILE);
+                  System.out
+                          .println("Recovered view for '" + nextnextxref
+                                  + "' from '" + savedProjects
+                                          .get(nextnextxref).toString()
+                                  + "'");
+                  // gymnastics to recover the alignPanel/Complementary
+                  // alignPanel
+                  if (af2.getViewport().isNucleotide())
+                  {
+                    // top view, then bottom
+                    cra_views2.add(af2.getViewport().getAlignPanel());
+                    cra_views2.add(((jalview.gui.AlignViewport) af2
+                            .getViewport().getCodingComplement())
+                                    .getAlignPanel());
+
+                  }
+                  else
+                  {
+                    // bottom view, then top
+                    cra_views2.add(((jalview.gui.AlignViewport) af2
+                            .getViewport().getCodingComplement())
+                                    .getAlignPanel());
+                    cra_views2.add(af2.getViewport().getAlignPanel());
+                  }
+                  Assert.assertEquals(cra_views2.size(), 2);
+                  Assert.assertNotNull(cra_views2.get(0));
+                  Assert.assertNotNull(cra_views2.get(1));
+                }
+
+                for (AlignmentViewPanel nextavp : cra_views2)
+                {
+                  nextnextxref = nextxref + " -> " + xrefdb + "{" + q++
+                          + "}";
+
+                  // verify references for this panel
+                  AlignmentTest.assertAlignmentDatasetRefs(
+                          nextavp.getAlignment(),
+                          "" + "Pass (" + pass1 + "," + pass2 + "): For "
+                                  + nextnextxref + ":");
+                  assertDatasetIsNormalisedKnownDefect(
+                          nextavp.getAlignment(),
+                          "" + "Pass (" + pass1 + "," + pass2 + "): For "
+                                  + nextnextxref + ":");
+
+                  stringify(dbtoviewBit, savedProjects, nextnextxref,
+                          nextavp);
+                  keyseq.add(nextnextxref);
+                }
+              } // end of loop around showing all xrefdb for crossrf2
+
+            } // end of loop around all viewpanels from crossrf1
+          } while (pass2 == 2 && pass3++ < 2);
+          // fetchdb->crossref1->crossref-2->verify for xrefs we
+          // either loop twice when pass2=0, or just once when pass2=1
+          // (recovered project from previous crossref)
+
+        } // end of loop over db-xrefs for crossref-2
+
+        // fetchdb-->crossref1
+        // for each xref we try to retrieve xref, store and verify when
+        // pass1=0, or just retrieve and verify when pass1=1
+      } while (pass1 == 1 && pass2++ < 2);
+      // fetchdb
+      // for each ref we
+      // loop twice: first, do the retrieve, second recover from saved project
+
+      // increment pass counters, so we repeat traversal starting from the
+      // oldest saved project first.
+      if (pass1 == 0)
+      {
+        // verify stored projects for first set of cross references
+        pass1 = 1;
+        // and verify cross-references retrieved from stored projects
+        pass2 = 0;
+        pass3 = 0;
+      }
+      else
+      {
+        pass1++;
+      }
+    } while (pass1 < 3);
+
+    if (failedXrefMenuItems.size() > 0)
     {
-      if ("Spinach Feredoxin Structure".equals(ap.getViewName()))
+      for (String s : failedXrefMenuItems)
       {
-        sps = ap;
-        break;
+        System.err.println(s);
       }
+      Assert.fail("Faulty xref menu (" + failedXrefMenuItems.size()
+              + " counts)");
     }
-    assertTrue("Couldn't find the structure view", sps != null);
-    SequenceI sq = sps.getAlignment().findName("1A70|");
-    AlignmentAnnotation refan = null;
-    for (AlignmentAnnotation ra : sps.getAlignment()
-            .getAlignmentAnnotation())
+    if (failedProjectRecoveries.size() > 0)
     {
-      if (ra.graph != 0)
+
+      for (String s : failedProjectRecoveries)
       {
-        refan = ra;
-        break;
+        System.err.println(s);
       }
+      Assert.fail(
+              "Didn't recover projects for some retrievals (did they retrieve ?) ("
+                      + failedProjectRecoveries.size() + " counts)");
     }
-    assertTrue("Annotation secondary structure not found.", refan != null);
-    assertTrue("Couldn't find 1a70 null chain", sq != null);
-    // compare the manually added temperature factor annotation
-    // to the track automatically transferred from the pdb structure on load
-    for (AlignmentAnnotation ala : sq.getDatasetSequence().getAnnotation())
+    if (failedDBRetr.size() > 0)
     {
-      AlignmentAnnotation alaa;
-      sq.addAlignmentAnnotation(alaa = new AlignmentAnnotation(ala));
-      alaa.adjustForAlignment();
-      if (ala.graph == refan.graph)
+      for (String s : failedProjectRecoveries)
       {
-        for (int p = 0; p < ala.annotations.length; p++)
-        {
-          sq.findPosition(p);
-          try
-          {
-            assertTrue(
-                    "Mismatch at alignment position " + p,
-                    (alaa.annotations[p] == null && refan.annotations[p] == null)
-                            || alaa.annotations[p].value == refan.annotations[p].value);
-          } catch (NullPointerException q)
-          {
-            Assert.fail("Mismatch of alignment annotations at position "
-                    + p + " Ref seq ann: " + refan.annotations[p]
-                    + " alignment " + alaa.annotations[p]);
-          }
-        }
+        System.err.println(s);
       }
+      Assert.fail("Didn't retrieve some db refs for checking cross-refs ("
+              + failedDBRetr.size() + " counts)");
     }
-
   }
 
-  @Test(groups = { "Functional" })
-  public void testCopyViewSettings() throws Exception
+  private void filterDbRefs(List<String> ptypes, List<String> limit)
   {
-    AlignFrame af = new jalview.io.FileLoader().LoadFileWaitTillLoaded(
-            "examples/exampleFile_2_7.jar", FormatAdapter.FILE);
-    assertTrue("Didn't read in the example file correctly.", af != null);
-    AlignmentViewPanel sps = null, groups = null;
-    for (AlignmentViewPanel ap : af.alignPanel.alignFrame.getAlignPanels())
+    if (limit != null)
     {
-      if ("Spinach Feredoxin Structure".equals(ap.getViewName()))
-      {
-        sps = ap;
-      }
-      if (ap.getViewName().contains("MAFFT"))
+      int p = 0;
+      while (ptypes.size() > p)
       {
-        groups = ap;
+        if (!limit.contains(ptypes.get(p)))
+        {
+          ptypes.remove(p);
+        }
+        else
+        {
+          p++;
+        }
       }
     }
-    assertTrue("Couldn't find the structure view", sps != null);
-    assertTrue("Couldn't find the MAFFT view", groups != null);
-
-    ViewStyleI structureStyle = sps.getAlignViewport().getViewStyle();
-    ViewStyleI groupStyle = groups.getAlignViewport().getViewStyle();
-    AssertJUnit.assertFalse(structureStyle.sameStyle(groupStyle));
-
-    groups.getAlignViewport().setViewStyle(structureStyle);
-    AssertJUnit.assertFalse(groupStyle.sameStyle(groups.getAlignViewport()
-            .getViewStyle()));
-    Assert.assertTrue(structureStyle.sameStyle(groups.getAlignViewport()
-            .getViewStyle()));
-
   }
 
   /**
-   * test store and recovery of expanded views
+   * wrapper to trap known defect for AH002001 testcase
    * 
-   * @throws Exception
+   * @param alignment
+   * @param string
    */
-  @Test(groups = { "Functional" }, enabled = true)
-  public void testStoreAndRecoverExpandedviews() throws Exception
+  private void assertDatasetIsNormalisedKnownDefect(AlignmentI al,
+          String message)
   {
-    Desktop.instance.closeAll_actionPerformed(null);
-
-    AlignFrame af = new jalview.io.FileLoader().LoadFileWaitTillLoaded(
-            "examples/exampleFile_2_7.jar", FormatAdapter.FILE);
-    assertTrue("Didn't read in the example file correctly.", af != null);
-    Assert.assertEquals(Desktop.getAlignFrames().length, 1);
-    String afid = af.getViewport().getSequenceSetId();
-
-    // check FileLoader returned a reference to the one alignFrame that is
-    // actually on the Desktop
-    assertTrue(
-            "Jalview2XML.loadAlignFrame() didn't return correct AlignFrame reference for multiple view window",
-            af == Desktop.getAlignFrameFor(af.getViewport()));
-
-    Desktop.explodeViews(af);
-
-    int oldviews = Desktop.getAlignFrames().length;
-    Assert.assertEquals(Desktop.getAlignFrames().length,
-            Desktop.getAlignmentPanels(afid).length);
-    File tfile = File.createTempFile("testStoreAndRecoverExpanded", ".jvp");
     try
     {
-      new Jalview2XML(false).saveState(tfile);
-    } catch (Error e)
+      AlignmentTest.assertDatasetIsNormalised(al, message);
+    } catch (AssertionError ae)
     {
-      Assert.fail("Didn't save the expanded view state", e);
-    } catch (Exception e)
-    {
-      Assert.fail("Didn't save the expanded view state", e);
-    }
-    Desktop.instance.closeAll_actionPerformed(null);
-    if (Desktop.getAlignFrames() != null)
-    {
-      Assert.assertEquals(Desktop.getAlignFrames().length, 0);
+      if (!ae.getMessage().endsWith("EMBL|AH002001"))
+      {
+        throw ae;
+      }
+      else
+      {
+        System.out.println("Ignored exception for known defect: JAL-2179 : "
+                + message);
+      }
+
     }
-    af = new jalview.io.FileLoader().LoadFileWaitTillLoaded(
-            tfile.getAbsolutePath(), FormatAdapter.FILE);
-    Assert.assertNotNull(af);
-    Assert.assertEquals(
-            Desktop.getAlignFrames().length,
-            Desktop.getAlignmentPanels(af.getViewport().getSequenceSetId()).length);
-    Assert.assertEquals(
-            oldviews,
-            Desktop.getAlignmentPanels(af.getViewport().getSequenceSetId()).length);
   }
 
-  /**
-   * Test save and reload of a project with a different representative sequence
-   * in each view.
-   * 
-   * @throws Exception
-   */
-  @Test(groups = { "Functional" })
-  public void testStoreAndRecoverReferenceSeqSettings() throws Exception
+  private void assertProtein(AlignmentViewPanel alignmentViewPanel,
+          String message)
   {
-    Desktop.instance.closeAll_actionPerformed(null);
-    AlignFrame af = new FileLoader().LoadFileWaitTillLoaded(
-            "examples/exampleFile_2_7.jar", FormatAdapter.FILE);
-    assertTrue("Didn't read in the example file correctly.", af != null);
-    String afid = af.getViewport().getSequenceSetId();
-
-    // remember reference sequence for each panel
-    Map<String, SequenceI> refseqs = new HashMap<String, SequenceI>();
-
-    /*
-     * mark sequence 2, 3, 4.. in panels 1, 2, 3...
-     * as reference sequence for itself and the preceding sequence
-     */
-    int n = 1;
-    for (AlignmentViewPanel ap : Desktop.getAlignmentPanels(afid))
-    {
-      AlignViewportI av = ap.getAlignViewport();
-      AlignmentI alignment = ap.getAlignment();
-      int repIndex = n % alignment.getHeight();
-      SequenceI rep = alignment.getSequenceAt(repIndex);
-      refseqs.put(ap.getViewName(), rep);
-
-      // code from mark/unmark sequence as reference in jalview.gui.PopupMenu
-      // todo refactor this to an alignment view controller
-      av.setDisplayReferenceSeq(true);
-      av.setColourByReferenceSeq(true);
-      av.getAlignment().setSeqrep(rep);
-
-      n++;
-    }
-    File tfile = File.createTempFile("testStoreAndRecoverReferenceSeq",
-            ".jvp");
-    try
-    {
-      new Jalview2XML(false).saveState(tfile);
-    } catch (Throwable e)
-    {
-      Assert.fail("Didn't save the expanded view state", e);
-    }
-    Desktop.instance.closeAll_actionPerformed(null);
-    if (Desktop.getAlignFrames() != null)
-    {
-      Assert.assertEquals(Desktop.getAlignFrames().length, 0);
-    }
-
-    af = new FileLoader().LoadFileWaitTillLoaded(tfile.getAbsolutePath(),
-            FormatAdapter.FILE);
-    afid = af.getViewport().getSequenceSetId();
+    assertType(true, alignmentViewPanel, message);
+  }
 
-    for (AlignmentViewPanel ap : Desktop.getAlignmentPanels(afid))
-    {
-      // check representative
-      AlignmentI alignment = ap.getAlignment();
-      SequenceI rep = alignment.getSeqrep();
-      Assert.assertNotNull(rep,
-              "Couldn't restore sequence representative from project");
-      // can't use a strong equals here, because by definition, the sequence IDs
-      // will be different.
-      // could set vamsas session save/restore flag to preserve IDs across
-      // load/saves.
-      Assert.assertEquals(refseqs.get(ap.getViewName()).toString(),
-              rep.toString(),
-              "Representative wasn't the same when recovered.");
-      Assert.assertTrue(ap.getAlignViewport().isDisplayReferenceSeq(),
-              "Display reference sequence view setting not set.");
-      Assert.assertTrue(ap.getAlignViewport().isColourByReferenceSeq(),
-              "Colour By Reference Seq view setting not set.");
-    }
+  private void assertNucleotide(AlignmentViewPanel alignmentViewPanel,
+          String message)
+  {
+    assertType(false, alignmentViewPanel, message);
   }
 
-  @Test(groups = { "Functional" })
-  public void testIsVersionStringLaterThan()
+  private void assertType(boolean expectProtein,
+          AlignmentViewPanel alignmentViewPanel, String message)
   {
-    /*
-     * No version / development / test / autobuild is leniently assumed to be
-     * compatible
-     */
-    assertTrue(Jalview2XML.isVersionStringLaterThan(null, null));
-    assertTrue(Jalview2XML.isVersionStringLaterThan("2.8.3", null));
-    assertTrue(Jalview2XML.isVersionStringLaterThan(null, "2.8.3"));
-    assertTrue(Jalview2XML.isVersionStringLaterThan(null,
-            "Development Build"));
-    assertTrue(Jalview2XML.isVersionStringLaterThan(null,
-            "DEVELOPMENT BUILD"));
-    assertTrue(Jalview2XML.isVersionStringLaterThan("2.8.3",
-            "Development Build"));
-    assertTrue(Jalview2XML.isVersionStringLaterThan(null, "Test"));
-    assertTrue(Jalview2XML.isVersionStringLaterThan(null, "TEST"));
-    assertTrue(Jalview2XML.isVersionStringLaterThan("2.8.3", "Test"));
-    assertTrue(Jalview2XML
-            .isVersionStringLaterThan(null, "Automated Build"));
-    assertTrue(Jalview2XML.isVersionStringLaterThan("2.8.3",
-            "Automated Build"));
-    assertTrue(Jalview2XML.isVersionStringLaterThan("2.8.3",
-            "AUTOMATED BUILD"));
-
-    /*
-     * same version returns true i.e. compatible
-     */
-    assertTrue(Jalview2XML.isVersionStringLaterThan("2.8", "2.8"));
-    assertTrue(Jalview2XML.isVersionStringLaterThan("2.8.3", "2.8.3"));
-    assertTrue(Jalview2XML.isVersionStringLaterThan("2.8.3b1", "2.8.3b1"));
-    assertTrue(Jalview2XML.isVersionStringLaterThan("2.8.3B1", "2.8.3b1"));
-    assertTrue(Jalview2XML.isVersionStringLaterThan("2.8.3b1", "2.8.3B1"));
-
-    /*
-     * later version returns true
-     */
-    assertTrue(Jalview2XML.isVersionStringLaterThan("2.8.3", "2.8.4"));
-    assertTrue(Jalview2XML.isVersionStringLaterThan("2.8.3", "2.9"));
-    assertTrue(Jalview2XML.isVersionStringLaterThan("2.8.3", "2.9.2"));
-    assertTrue(Jalview2XML.isVersionStringLaterThan("2.8", "2.8.3"));
-    assertTrue(Jalview2XML.isVersionStringLaterThan("2.8.3", "2.8.3b1"));
-
-    /*
-     * earlier version returns false
-     */
-    assertFalse(Jalview2XML.isVersionStringLaterThan("2.8.3", "2.8"));
-    assertFalse(Jalview2XML.isVersionStringLaterThan("2.8.4", "2.8.3"));
-    assertFalse(Jalview2XML.isVersionStringLaterThan("2.8.3b1", "2.8.3"));
-    assertFalse(Jalview2XML.isVersionStringLaterThan("2.8.3", "2.8.2b1"));
-    assertFalse(Jalview2XML.isVersionStringLaterThan("2.8.0b2", "2.8.0b1"));
+    List<SequenceI> nonType = new ArrayList<>();
+    for (SequenceI sq : alignmentViewPanel.getAlignViewport().getAlignment()
+            .getSequences())
+    {
+      if (sq.isProtein() != expectProtein)
+      {
+        nonType.add(sq);
+      }
+    }
+    if (nonType.size() > 0)
+    {
+      Assert.fail(message + " [ "
+              + (expectProtein ? "nucleotides were " : "proteins were ")
+              + nonType.toString() + " ]");
+    }
   }
 
   /**
-   * Test save and reload of a project with a different sequence group (and
-   * representative sequence) in each view.
+   * first time called, record strings derived from alignment and
+   * alignedcodonframes, and save view to a project file. Second time called,
+   * compare strings to existing ones. org.testng.Assert.assertTrue on
+   * stringmatch
    * 
-   * @throws Exception
+   * @param dbtoviewBit
+   *          map between xrefpath and view string
+   * @param savedProjects
+   *          - map from xrefpath to saved project filename (createTempFile)
+   * @param xrefpath
+   *          - xrefpath - unique ID for this context (composed of sequence of
+   *          db-fetch/cross-ref actions preceeding state)
+   * @param avp
+   *          - viewpanel to store (for viewpanels in splitframe, the same
+   *          project should be written for both panels, only one needs
+   *          recovering for comparison on the next stringify call, but each
+   *          viewpanel needs to be called with a distinct xrefpath to ensure
+   *          each one's strings are compared)
    */
-  @Test(groups = { "Functional" })
-  public void testStoreAndRecoverGroupRepSeqs() throws Exception
+  private void stringify(Map<String, String> dbtoviewBit,
+          Map<String, File> savedProjects, String xrefpath,
+          AlignmentViewPanel avp)
   {
-    Desktop.instance.closeAll_actionPerformed(null);
-    AlignFrame af = new FileLoader().LoadFileWaitTillLoaded(
-            "examples/uniref50.fa", FormatAdapter.FILE);
-    assertTrue("Didn't read in the example file correctly.", af != null);
-    String afid = af.getViewport().getSequenceSetId();
-    // make a second view of the alignment
-    af.newView_actionPerformed(null);
-
-    /*
-     * remember representative and hidden sequences marked 
-     * on each panel
-     */
-    Map<String, SequenceI> repSeqs = new HashMap<String, SequenceI>();
-    Map<String, List<String>> hiddenSeqNames = new HashMap<String, List<String>>();
-
-    /*
-     * mark sequence 2, 3, 4.. in panels 1, 2, 3...
-     * as reference sequence for itself and the preceding sequence
-     */
-    int n = 1;
-    for (AlignmentViewPanel ap : Desktop.getAlignmentPanels(afid))
+    if (savedProjects != null)
     {
-      AlignViewportI av = ap.getAlignViewport();
-      AlignmentI alignment = ap.getAlignment();
-      int repIndex = n % alignment.getHeight();
-      // ensure at least one preceding sequence i.e. index >= 1
-      repIndex = Math.max(repIndex, 1);
-      SequenceI repSeq = alignment.getSequenceAt(repIndex);
-      repSeqs.put(ap.getViewName(), repSeq);
-      List<String> hiddenNames = new ArrayList<String>();
-      hiddenSeqNames.put(ap.getViewName(), hiddenNames);
-
-      /*
-       * have rep sequence represent itself and the one before it
-       * this hides the group (except for the rep seq)
-       */
-      SequenceGroup sg = new SequenceGroup();
-      sg.addSequence(repSeq, false);
-      SequenceI precedingSeq = alignment.getSequenceAt(repIndex - 1);
-      sg.addSequence(precedingSeq, false);
-      sg.setSeqrep(repSeq);
-      assertTrue(sg.getSequences().contains(repSeq));
-      assertTrue(sg.getSequences().contains(precedingSeq));
-      av.setSelectionGroup(sg);
-      assertSame(repSeq, sg.getSeqrep());
-
-      /*
-       * represent group with sequence adds to a map of hidden rep sequences
-       * (it does not create a group on the alignment) 
-       */
-      ((AlignmentViewport) av).hideSequences(repSeq, true);
-      assertSame(repSeq, sg.getSeqrep());
-      assertTrue(sg.getSequences().contains(repSeq));
-      assertTrue(sg.getSequences().contains(precedingSeq));
-      assertTrue("alignment has groups", alignment.getGroups().isEmpty());
-      Map<SequenceI, SequenceCollectionI> hiddenRepSeqsMap = av
-              .getHiddenRepSequences();
-      assertNotNull(hiddenRepSeqsMap);
-      assertEquals(1, hiddenRepSeqsMap.size());
-      assertSame(sg, hiddenRepSeqsMap.get(repSeq));
-      assertTrue(alignment.getHiddenSequences().isHidden(precedingSeq));
-      assertFalse(alignment.getHiddenSequences().isHidden(repSeq));
-      hiddenNames.add(precedingSeq.getName());
-
-      n++;
+      if (savedProjects.get(xrefpath) == null)
+      {
+        // write a project file for this view. On the second pass, this will be
+        // recovered and cross-references verified
+        try
+        {
+          File prfile = File.createTempFile("crossRefTest", ".jvp");
+          AlignFrame af = Desktop.getAlignFrameFor(avp.getAlignViewport());
+          new Jalview2XML(false).saveAlignment(af, prfile.toString(),
+                  af.getTitle());
+          System.out.println("Written view from '" + xrefpath + "' as '"
+                  + prfile.getAbsolutePath() + "'");
+          savedProjects.put(xrefpath, prfile);
+        } catch (IOException q)
+        {
+          Assert.fail("Unexpected IO Exception", q);
+        }
+      }
+      else
+      {
+        System.out.println("Stringify check on view from '" + xrefpath
+                + "' [ possibly retrieved from '"
+                + savedProjects.get(xrefpath).getAbsolutePath() + "' ]");
+
+      }
     }
-    File tfile = File
-            .createTempFile("testStoreAndRecoverGroupReps", ".jvp");
-    try
-    {
-      new Jalview2XML(false).saveState(tfile);
-    } catch (Throwable e)
+
+    StringBuilder sbr = new StringBuilder();
+    sbr.append(avp.getAlignment().toString());
+    sbr.append("\n");
+    sbr.append("<End of alignment>");
+    sbr.append("\n");
+    sbr.append(avp.getAlignment().getDataset());
+    sbr.append("\n");
+    sbr.append("<End of dataset>");
+    sbr.append("\n");
+    int p = 0;
+    if (avp.getAlignment().getCodonFrames() != null)
     {
-      Assert.fail("Didn't save the expanded view state", e);
+      for (AlignedCodonFrame ac : avp.getAlignment().getCodonFrames())
+      {
+        sbr.append("<AlignedCodonFrame " + p++ + ">");
+        sbr.append("\n");
+        sbr.append(ac.toString());
+        sbr.append("\n");
+      }
     }
-    Desktop.instance.closeAll_actionPerformed(null);
-    if (Desktop.getAlignFrames() != null)
+    String dbt = dbtoviewBit.get(xrefpath);
+    if (dbt == null)
     {
-      Assert.assertEquals(Desktop.getAlignFrames().length, 0);
+      dbtoviewBit.put(xrefpath, sbr.toString());
     }
-
-    af = new FileLoader().LoadFileWaitTillLoaded(tfile.getAbsolutePath(),
-            FormatAdapter.FILE);
-    afid = af.getViewport().getSequenceSetId();
-
-    for (AlignmentViewPanel ap : Desktop.getAlignmentPanels(afid))
+    else
     {
-      String viewName = ap.getViewName();
-      AlignViewportI av = ap.getAlignViewport();
-      AlignmentI alignment = ap.getAlignment();
-      List<SequenceGroup> groups = alignment.getGroups();
-      assertNotNull(groups);
-      assertTrue("Alignment has groups", groups.isEmpty());
-      Map<SequenceI, SequenceCollectionI> hiddenRepSeqsMap = av
-              .getHiddenRepSequences();
-      assertNotNull("No hidden represented sequences", hiddenRepSeqsMap);
-      assertEquals(1, hiddenRepSeqsMap.size());
-      assertEquals(repSeqs.get(viewName).getDisplayId(true),
-              hiddenRepSeqsMap.keySet().iterator().next()
-                      .getDisplayId(true));
-
-      /*
-       * verify hidden sequences in restored panel
-       */
-      List<String> hidden = hiddenSeqNames.get(ap.getViewName());
-      HiddenSequences hs = alignment.getHiddenSequences();
-      assertEquals(
-              "wrong number of restored hidden sequences in "
-                      + ap.getViewName(), hidden.size(), hs.getSize());
+      Assert.assertEquals(sbr.toString(), dbt,
+              "stringify mismatch for " + xrefpath);
     }
   }
 }