test/jalview/io/CrossRef2xmlTests.java

   1 /*
   2  * Jalview - A Sequence Alignment Editor and Viewer ($$Version-Rel$$)
   3  * Copyright (C) $$Year-Rel$$ The Jalview Authors
   4  *
   5  * This file is part of Jalview.
   6  *
   7  * Jalview is free software: you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License
   9  * as published by the Free Software Foundation, either version 3
  10  * of the License, or (at your option) any later version.
  11  *
  12  * Jalview is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty
  14  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR
  15  * PURPOSE.  See the GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with Jalview.  If not, see <http://www.gnu.org/licenses/>.
  19  * The Jalview Authors are detailed in the 'AUTHORS' file.
  20  */
  21 package jalview.io;
  22
  23 import jalview.analysis.CrossRef;
  24 import jalview.api.AlignmentViewPanel;
  25 import jalview.datamodel.AlignedCodonFrame;
  26 import jalview.datamodel.AlignmentI;
  27 import jalview.datamodel.AlignmentTest;
  28 import jalview.datamodel.SequenceI;
  29 import jalview.gui.AlignFrame;
  30 import jalview.gui.CrossRefAction;
  31 import jalview.gui.Desktop;
  32 import jalview.gui.Jalview2XML;
  33 import jalview.gui.JvOptionPane;
  34 import jalview.util.DBRefUtils;
  35
  36 import java.io.File;
  37 import java.io.IOException;
  38 import java.util.ArrayList;
  39 import java.util.Arrays;
  40 import java.util.HashMap;
  41 import java.util.List;
  42 import java.util.Map;
  43
  44 import junit.extensions.PA;
  45
  46 import org.testng.Assert;
  47 import org.testng.annotations.BeforeClass;
  48 import org.testng.annotations.Test;
  49
  50 @Test(singleThreaded = true)
  51 public class CrossRef2xmlTests extends Jalview2xmlBase
  52 {
  53
  54   @Override
  55   @BeforeClass(alwaysRun = true)
  56   public void setUpJvOptionPane()
  57   {
  58     JvOptionPane.setInteractiveMode(false);
  59     JvOptionPane.setMockResponse(JvOptionPane.CANCEL_OPTION);
  60   }
  61
  62   /**
  63    * test store and recovery of all reachable cross refs from all reachable
  64    * crossrefs for one or more fetched db refs. Currently, this test has a known
  65    * failure case.
  66    *
  67    * @throws Exception
  68    */
  69   @Test(groups = { "Operational" }, enabled = true)
  70   public void testRetrieveAndShowCrossref() throws Exception
  71   {
  72
  73     List<String> failedDBRetr = new ArrayList<>();
  74     List<String> failedXrefMenuItems = new ArrayList<>();
  75     List<String> failedProjectRecoveries = new ArrayList<>();
  76     // only search for ensembl or Uniprot crossrefs
  77     List<String> limit=Arrays.asList(new String[] {
  78         DBRefUtils.getCanonicalName("ENSEMBL"),
  79         DBRefUtils.getCanonicalName("Uniprot")});
  80     // for every set of db queries
  81     // retrieve db query
  82     // verify presence of expected xrefs
  83     // show xrefs - verify expected type of frame is shown for each xref
  84     // show xrefs again
  85     // - verify original -> xref -> xref(original) recovers frame containing at
  86     // least the first retrieved sequence
  87     // store
  88     // 1. whole project
  89     // 2. individual frames
  90     // 3. load each one back and verify
  91     // . aligned sequences (.toString() )
  92     // . xrefs (.toString() )
  93     // . codonframes
  94     //
  95     //
  96     Map<String, String> dbtoviewBit = new HashMap<>();
  97     List<String> keyseq = new ArrayList<>();
  98     Map<String, File> savedProjects = new HashMap<>();
  99
 100     for (String[] did : new String[][] { { "UNIPROT", "P00338" } })
 101     {
 102       // pass counters - 0 - first pass, 1 means retrieve project rather than
 103       // perform action
 104       int pass1 = 0, pass2 = 0, pass3 = 0;
 105       // each do loop performs two iterations in the first outer loop pass, but
 106       // only performs one iteration on the second outer loop
 107       // ie. pass 1 = 0 {pass 2= 0 { pass 3 = 0,1 }, pass 2=1 { pass 3 = 0 }}, 1
 108       // { pass 2 = 0 { pass 3 = 0 } }
 109       do
 110       {
 111         String first = did[0] + " " + did[1];
 112         AlignFrame af = null;
 113         boolean dna;
 114         AlignmentI retral;
 115         AlignmentI dataset;
 116         SequenceI[] seqs;
 117         List<String> ptypes = null;
 118         if (pass1 == 0)
 119         {
 120           // retrieve dbref
 121
 122           List<AlignFrame> afs = jalview.gui.SequenceFetcher.fetchAndShow(
 123                   did[0], did[1]);
 124           if (afs.size() == 0)
 125           {
 126             failedDBRetr.add("Didn't retrieve " + first);
 127             break;
 128           }
 129           keyseq.add(first);
 130           af = afs.get(0);
 131
 132           // verify references for retrieved data
 133           AlignmentTest.assertAlignmentDatasetRefs(af.getViewport()
 134                   .getAlignment(), "Pass (" + pass1 + "," + pass2 + ","
 135                   + pass3 + "): Fetch " + first + ":");
 136           assertDatasetIsNormalisedKnownDefect(af.getViewport()
 137                   .getAlignment(), "Pass (" + pass1 + "," + pass2 + ","
 138                   + pass3 + "): Fetch " + first + ":");
 139           dna = af.getViewport().getAlignment().isNucleotide();
 140           retral = af.getViewport().getAlignment();
 141           dataset = retral.getDataset();
 142           seqs = retral.getSequencesArray();
 143
 144         }
 145         else
 146         {
 147           Desktop.instance.closeAll_actionPerformed(null);
 148           // recover stored project
 149           af = new FileLoader(false).LoadFileWaitTillLoaded(savedProjects
 150                   .get(first).toString(), DataSourceType.FILE);
 151           System.out.println("Recovered view for '" + first + "' from '"
 152                   + savedProjects.get(first).toString() + "'");
 153           dna = af.getViewport().getAlignment().isNucleotide();
 154           retral = af.getViewport().getAlignment();
 155           dataset = retral.getDataset();
 156           seqs = retral.getSequencesArray();
 157
 158           // verify references for recovered data
 159           AlignmentTest.assertAlignmentDatasetRefs(af.getViewport()
 160                   .getAlignment(), "Pass (" + pass1 + "," + pass2 + ","
 161                   + pass3 + "): Recover " + first + ":");
 162           assertDatasetIsNormalisedKnownDefect(af.getViewport()
 163                   .getAlignment(), "Pass (" + pass1 + "," + pass2 + ","
 164                   + pass3 + "): Recover " + first + ":");
 165
 166         }
 167
 168         // store project on first pass, compare next pass
 169         stringify(dbtoviewBit, savedProjects, first, af.alignPanel);
 170
 171         ptypes = (seqs == null || seqs.length == 0) ? null : new CrossRef(
 172                 seqs, dataset).findXrefSourcesForSequences(dna);
 173         filterDbRefs(ptypes, limit);
 174
 175         // start of pass2: retrieve each cross-ref for fetched or restored
 176         // project.
 177         do // first cross ref and recover crossref loop
 178         {
 179
 180           for (String db : ptypes)
 181           {
 182             // counter for splitframe views retrieved via crossref
 183             int firstcr_ap = 0;
 184             // build next key so we an retrieve all views
 185             String nextxref = first + " -> " + db + "{" + firstcr_ap + "}";
 186             // perform crossref action, or retrieve stored project
 187             List<AlignmentViewPanel> cra_views = new ArrayList<>();
 188             CrossRefAction cra = null;
 189
 190             if (pass2 == 0)
 191             { // retrieve and show cross-refs in this thread
 192               cra = CrossRefAction.getHandlerFor(seqs, dna, db, af);
 193               cra.run();
 194               cra_views = (List<AlignmentViewPanel>) PA.getValue(cra,
 195                       "xrefViews");
 196               if (cra_views.size() == 0)
 197               {
 198                 failedXrefMenuItems.add("No crossrefs retrieved for "
 199                         + first + " -> " + db);
 200                 continue;
 201               }
 202               assertNucleotide(cra_views.get(0),
 203                       "Nucleotide panel included proteins for " + first
 204                               + " -> " + db);
 205               assertProtein(cra_views.get(1),
 206                       "Protein panel included nucleotides for " + first
 207                               + " -> " + db);
 208             }
 209             else
 210             {
 211               Desktop.instance.closeAll_actionPerformed(null);
 212               pass3 = 0;
 213               // recover stored project
 214               File storedProject = savedProjects.get(nextxref);
 215               if (storedProject == null)
 216               {
 217                 failedProjectRecoveries.add("Failed to store a view for '"
 218                         + nextxref + "'");
 219                 continue;
 220               }
 221
 222               // recover stored project
 223               AlignFrame af2 = new FileLoader(false)
 224                       .LoadFileWaitTillLoaded(savedProjects.get(nextxref)
 225                               .toString(), DataSourceType.FILE);
 226               System.out.println("Recovered view for '" + nextxref
 227                       + "' from '" + savedProjects.get(nextxref).toString()
 228                       + "'");
 229               // gymnastics to recover the alignPanel/Complementary alignPanel
 230               if (af2.getViewport().isNucleotide())
 231               {
 232                 // top view, then bottom
 233                 cra_views.add(af2.getViewport().getAlignPanel());
 234                 cra_views.add(((jalview.gui.AlignViewport) af2
 235                         .getViewport().getCodingComplement())
 236                         .getAlignPanel());
 237
 238               }
 239               else
 240               {
 241                 // bottom view, then top
 242                 cra_views.add(((jalview.gui.AlignViewport) af2
 243                         .getViewport().getCodingComplement())
 244                         .getAlignPanel());
 245                 cra_views.add(af2.getViewport().getAlignPanel());
 246
 247               }
 248             }
 249             HashMap<String, List<String>> xrptypes = new HashMap<>();
 250             // first save/verify views.
 251             for (AlignmentViewPanel avp : cra_views)
 252             {
 253               nextxref = first + " -> " + db + "{" + firstcr_ap++ + "}";
 254               // verify references for this panel
 255               AlignmentTest.assertAlignmentDatasetRefs(avp.getAlignment(),
 256                       "Pass (" + pass1 + "," + pass2 + "," + pass3
 257                               + "): before start of pass3: " + nextxref
 258                               + ":");
 259               assertDatasetIsNormalisedKnownDefect(avp.getAlignment(),
 260                       "Pass (" + pass1 + "," + pass2 + "," + pass3
 261                               + "): before start of pass3: " + nextxref
 262                               + ":");
 263
 264               SequenceI[] xrseqs = avp.getAlignment().getSequencesArray();
 265
 266               List<String> _xrptypes = (seqs == null || seqs.length == 0) ? null
 267                       : new CrossRef(xrseqs, dataset)
 268                               .findXrefSourcesForSequences(avp
 269                                       .getAlignViewport().isNucleotide());
 270
 271               stringify(dbtoviewBit, savedProjects, nextxref, avp);
 272               xrptypes.put(nextxref, _xrptypes);
 273
 274             }
 275
 276             // now do the second xref pass starting from either saved or just
 277             // recovered split pane, in sequence
 278             do // retrieve second set of cross refs or recover and verify
 279             {
 280               firstcr_ap = 0;
 281               for (AlignmentViewPanel avp : cra_views)
 282               {
 283                 nextxref = first + " -> " + db + "{" + firstcr_ap++ + "}";
 284                 for (String xrefdb : xrptypes.get(nextxref))
 285                 {
 286                   List<AlignmentViewPanel> cra_views2 = new ArrayList<>();
 287                   int q = 0;
 288                   String nextnextxref = nextxref + " -> " + xrefdb + "{"
 289                           + q + "}";
 290
 291                   if (pass3 == 0)
 292                   {
 293                     SequenceI[] xrseqs = avp.getAlignment()
 294                             .getSequencesArray();
 295                     AlignFrame nextaf = Desktop.getAlignFrameFor(avp
 296                             .getAlignViewport());
 297
 298                     cra = CrossRefAction.getHandlerFor(xrseqs, avp
 299                             .getAlignViewport().isNucleotide(), xrefdb,
 300                             nextaf);
 301                     cra.run();
 302                     cra_views2 = (List<AlignmentViewPanel>) PA.getValue(
 303                             cra, "xrefViews");
 304                     if (cra_views2.size() == 0)
 305                     {
 306                       failedXrefMenuItems
 307                               .add("No crossrefs retrieved for '"
 308                                       + nextxref + "' to " + xrefdb
 309                                       + " via '" + nextaf.getTitle() + "'");
 310                       continue;
 311                     }
 312                     assertNucleotide(cra_views2.get(0),
 313                             "Nucleotide panel included proteins for '"
 314                                     + nextxref + "' to " + xrefdb
 315                                     + " via '" + nextaf.getTitle() + "'");
 316                     assertProtein(cra_views2.get(1),
 317                             "Protein panel included nucleotides for '"
 318                                     + nextxref + "' to " + xrefdb
 319                                     + " via '" + nextaf.getTitle() + "'");
 320
 321                   }
 322                   else
 323                   {
 324                     Desktop.instance.closeAll_actionPerformed(null);
 325                     // recover stored project
 326                     File storedProject = savedProjects.get(nextnextxref);
 327                     if (storedProject == null)
 328                     {
 329                       failedProjectRecoveries
 330                               .add("Failed to store a view for '"
 331                                       + nextnextxref + "'");
 332                       continue;
 333                     }
 334                     AlignFrame af2 = new FileLoader(false)
 335                             .LoadFileWaitTillLoaded(
 336                                     savedProjects.get(nextnextxref)
 337                                             .toString(),
 338                                     DataSourceType.FILE);
 339                     System.out.println("Recovered view for '"
 340                             + nextnextxref + "' from '"
 341                             + savedProjects.get(nextnextxref).toString()
 342                             + "'");
 343                     // gymnastics to recover the alignPanel/Complementary
 344                     // alignPanel
 345                     if (af2.getViewport().isNucleotide())
 346                     {
 347                       // top view, then bottom
 348                       cra_views2.add(af2.getViewport().getAlignPanel());
 349                       cra_views2.add(((jalview.gui.AlignViewport) af2
 350                               .getViewport().getCodingComplement())
 351                               .getAlignPanel());
 352
 353                     }
 354                     else
 355                     {
 356                       // bottom view, then top
 357                       cra_views2.add(((jalview.gui.AlignViewport) af2
 358                               .getViewport().getCodingComplement())
 359                               .getAlignPanel());
 360                       cra_views2.add(af2.getViewport().getAlignPanel());
 361                     }
 362                     Assert.assertEquals(cra_views2.size(), 2);
 363                     Assert.assertNotNull(cra_views2.get(0));
 364                     Assert.assertNotNull(cra_views2.get(1));
 365                   }
 366
 367                   for (AlignmentViewPanel nextavp : cra_views2)
 368                   {
 369                     nextnextxref = nextxref + " -> " + xrefdb + "{" + q++
 370                             + "}";
 371
 372                     // verify references for this panel
 373                     AlignmentTest.assertAlignmentDatasetRefs(
 374                             nextavp.getAlignment(), "" + "Pass (" + pass1
 375                                     + "," + pass2 + "): For "
 376                                     + nextnextxref + ":");
 377                     assertDatasetIsNormalisedKnownDefect(
 378                             nextavp.getAlignment(), "" + "Pass (" + pass1
 379                                     + "," + pass2 + "): For "
 380                                     + nextnextxref + ":");
 381
 382                     stringify(dbtoviewBit, savedProjects, nextnextxref,
 383                             nextavp);
 384                     keyseq.add(nextnextxref);
 385                   }
 386                 } // end of loop around showing all xrefdb for crossrf2
 387
 388               } // end of loop around all viewpanels from crossrf1
 389             } while (pass2 == 2 && pass3++ < 2);
 390             // fetchdb->crossref1->crossref-2->verify for xrefs we
 391             // either loop twice when pass2=0, or just once when pass2=1
 392             // (recovered project from previous crossref)
 393
 394           } // end of loop over db-xrefs for crossref-2
 395
 396           // fetchdb-->crossref1
 397           // for each xref we try to retrieve xref, store and verify when
 398           // pass1=0, or just retrieve and verify when pass1=1
 399         } while (pass1 == 1 && pass2++ < 2);
 400         // fetchdb
 401         // for each ref we
 402         // loop twice: first, do the retrieve, second recover from saved project
 403
 404         // increment pass counters, so we repeat traversal starting from the
 405         // oldest saved project first.
 406         if (pass1 == 0)
 407         {
 408           // verify stored projects for first set of cross references
 409           pass1 = 1;
 410           // and verify cross-references retrieved from stored projects
 411           pass2 = 0;
 412           pass3 = 0;
 413         }
 414         else
 415         {
 416           pass1++;
 417         }
 418       } while (pass1 < 3);
 419     }
 420     if (failedXrefMenuItems.size() > 0)
 421     {
 422       for (String s : failedXrefMenuItems)
 423       {
 424         System.err.println(s);
 425       }
 426       Assert.fail("Faulty xref menu (" + failedXrefMenuItems.size()
 427               + " counts)");
 428     }
 429     if (failedProjectRecoveries.size() > 0)
 430     {
 431
 432       for (String s : failedProjectRecoveries)
 433       {
 434         System.err.println(s);
 435       }
 436       Assert.fail("Didn't recover projects for some retrievals (did they retrieve ?) ("
 437               + failedProjectRecoveries.size() + " counts)");
 438     }
 439     if (failedDBRetr.size() > 0)
 440     {
 441       for (String s : failedProjectRecoveries)
 442       {
 443         System.err.println(s);
 444       }
 445       Assert.fail("Didn't retrieve some db refs for checking cross-refs ("
 446               + failedDBRetr.size() + " counts)");
 447     }
 448   }
 449
 450   private void filterDbRefs(List<String> ptypes, List<String> limit)
 451   {
 452     if (limit != null)
 453     {
 454       int p = 0;
 455       while (ptypes.size() > p)
 456       {
 457         if (!limit.contains(ptypes.get(p)))
 458         {
 459           ptypes.remove(p);
 460         }
 461         else
 462         {
 463           p++;
 464         }
 465       }
 466     }
 467   }
 468
 469   /**
 470    * wrapper to trap known defect for AH002001 testcase
 471    *
 472    * @param alignment
 473    * @param string
 474    */
 475   private void assertDatasetIsNormalisedKnownDefect(AlignmentI al,
 476           String message)
 477   {
 478     try
 479     {
 480       AlignmentTest.assertDatasetIsNormalised(al, message);
 481     } catch (AssertionError ae)
 482     {
 483       if (!ae.getMessage().endsWith("EMBL|AH002001"))
 484       {
 485         throw ae;
 486       }
 487       else
 488       {
 489         System.out
 490                 .println("Ignored exception for known defect: JAL-2179 : "
 491                         + message);
 492       }
 493
 494     }
 495   }
 496
 497   private void assertProtein(AlignmentViewPanel alignmentViewPanel,
 498           String message)
 499   {
 500     assertType(true, alignmentViewPanel, message);
 501   }
 502
 503   private void assertNucleotide(AlignmentViewPanel alignmentViewPanel,
 504           String message)
 505   {
 506     assertType(false, alignmentViewPanel, message);
 507   }
 508
 509   private void assertType(boolean expectProtein,
 510           AlignmentViewPanel alignmentViewPanel, String message)
 511   {
 512     List<SequenceI> nonType = new ArrayList<>();
 513     for (SequenceI sq : alignmentViewPanel.getAlignViewport()
 514             .getAlignment().getSequences())
 515     {
 516       if (sq.isProtein() != expectProtein)
 517       {
 518         nonType.add(sq);
 519       }
 520     }
 521     if (nonType.size() > 0)
 522     {
 523       Assert.fail(message + " [ "
 524               + (expectProtein ? "nucleotides were " : "proteins were ")
 525               + nonType.toString() + " ]");
 526     }
 527   }
 528
 529   /**
 530    * first time called, record strings derived from alignment and
 531    * alignedcodonframes, and save view to a project file. Second time called,
 532    * compare strings to existing ones. org.testng.Assert.assertTrue on
 533    * stringmatch
 534    *
 535    * @param dbtoviewBit
 536    *          map between xrefpath and view string
 537    * @param savedProjects
 538    *          - map from xrefpath to saved project filename (createTempFile)
 539    * @param xrefpath
 540    *          - xrefpath - unique ID for this context (composed of sequence of
 541    *          db-fetch/cross-ref actions preceeding state)
 542    * @param avp
 543    *          - viewpanel to store (for viewpanels in splitframe, the same
 544    *          project should be written for both panels, only one needs
 545    *          recovering for comparison on the next stringify call, but each
 546    *          viewpanel needs to be called with a distinct xrefpath to ensure
 547    *          each one's strings are compared)
 548    */
 549   private void stringify(Map<String, String> dbtoviewBit,
 550           Map<String, File> savedProjects, String xrefpath,
 551           AlignmentViewPanel avp)
 552   {
 553     if (savedProjects != null)
 554     {
 555       if (savedProjects.get(xrefpath) == null)
 556       {
 557         // write a project file for this view. On the second pass, this will be
 558         // recovered and cross-references verified
 559         try
 560         {
 561           File prfile = File.createTempFile("crossRefTest", ".jvp");
 562           AlignFrame af = Desktop.getAlignFrameFor(avp.getAlignViewport());
 563           new Jalview2XML(false).saveAlignment(af, prfile.toString(),
 564                   af.getTitle());
 565           System.out.println("Written view from '" + xrefpath + "' as '"
 566                   + prfile.getAbsolutePath() + "'");
 567           savedProjects.put(xrefpath, prfile);
 568         } catch (IOException q)
 569         {
 570           Assert.fail("Unexpected IO Exception", q);
 571         }
 572       }
 573       else
 574       {
 575         System.out.println("Stringify check on view from '" + xrefpath
 576                 + "' [ possibly retrieved from '"
 577                 + savedProjects.get(xrefpath).getAbsolutePath() + "' ]");
 578
 579       }
 580     }
 581
 582     StringBuilder sbr = new StringBuilder();
 583     sbr.append(avp.getAlignment().toString());
 584     sbr.append("\n");
 585     sbr.append("<End of alignment>");
 586     sbr.append("\n");
 587     sbr.append(avp.getAlignment().getDataset());
 588     sbr.append("\n");
 589     sbr.append("<End of dataset>");
 590     sbr.append("\n");
 591     int p = 0;
 592     if (avp.getAlignment().getCodonFrames() != null)
 593     {
 594       for (AlignedCodonFrame ac : avp.getAlignment().getCodonFrames())
 595       {
 596         sbr.append("<AlignedCodonFrame " + p++ + ">");
 597         sbr.append("\n");
 598         sbr.append(ac.toString());
 599         sbr.append("\n");
 600       }
 601     }
 602     String dbt = dbtoviewBit.get(xrefpath);
 603     if (dbt == null)
 604     {
 605       dbtoviewBit.put(xrefpath, sbr.toString());
 606     }
 607     else
 608     {
 609       Assert.assertEquals(sbr.toString(), dbt, "stringify mismatch for "
 610               + xrefpath);
 611     }
 612   }
 613 }