test/jalview/io/CrossRef2xmlTests.java

   1 /*
   2  * Jalview - A Sequence Alignment Editor and Viewer ($$Version-Rel$$)
   3  * Copyright (C) $$Year-Rel$$ The Jalview Authors
   4  *
   5  * This file is part of Jalview.
   6  *
   7  * Jalview is free software: you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License
   9  * as published by the Free Software Foundation, either version 3
  10  * of the License, or (at your option) any later version.
  11  *
  12  * Jalview is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty
  14  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR
  15  * PURPOSE.  See the GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with Jalview.  If not, see <http://www.gnu.org/licenses/>.
  19  * The Jalview Authors are detailed in the 'AUTHORS' file.
  20  */
  21 package jalview.io;
  22
  23 import jalview.analysis.CrossRef;
  24 import jalview.api.AlignmentViewPanel;
  25 import jalview.datamodel.AlignedCodonFrame;
  26 import jalview.datamodel.AlignmentI;
  27 import jalview.datamodel.AlignmentTest;
  28 import jalview.datamodel.SequenceI;
  29 import jalview.gui.AlignFrame;
  30 import jalview.gui.CrossRefAction;
  31 import jalview.gui.Desktop;
  32 import jalview.gui.Jalview2XML;
  33 import jalview.gui.JvOptionPane;
  34 import jalview.util.DBRefUtils;
  35
  36 import java.io.File;
  37 import java.io.IOException;
  38 import java.util.ArrayList;
  39 import java.util.Arrays;
  40 import java.util.HashMap;
  41 import java.util.List;
  42
  43 import org.testng.Assert;
  44 import org.testng.annotations.BeforeClass;
  45 import org.testng.annotations.DataProvider;
  46 import org.testng.annotations.Test;
  47
  48 @Test(singleThreaded = true)
  49 public class CrossRef2xmlTests extends Jalview2xmlBase
  50 {
  51
  52   @Override
  53   @BeforeClass(alwaysRun = true)
  54   public void setUpJvOptionPane()
  55   {
  56     JvOptionPane.setInteractiveMode(false);
  57     JvOptionPane.setMockResponse(JvOptionPane.CANCEL_OPTION);
  58   }
  59
  60   @DataProvider(name = "initialAccessions")
  61   static Object[][] getAccessions()
  62   {
  63     return new String[][] { { "UNIPROT", "P00338" },
  64         { "UNIPROT", "Q8Z9G6" },
  65         { "ENSEMBLGENOMES", "CAD01290" } };
  66   }
  67
  68   /**
  69    * test store and recovery of all reachable cross refs from all reachable
  70    * crossrefs for one or more fetched db refs. Currently, this test has a known
  71    * failure case.
  72    *
  73    * @throws Exception
  74    */
  75   @Test(
  76     groups =
  77     { "Operational" },
  78     dataProvider = "initialAccessions",
  79     enabled = true)
  80   public void testRetrieveAndShowCrossref(String forSource,
  81           String forAccession) throws Exception
  82   {
  83
  84     List<String> failedDBRetr = new ArrayList<>();
  85     List<String> failedXrefMenuItems = new ArrayList<>();
  86     List<String> failedProjectRecoveries = new ArrayList<>();
  87     // only search for ensembl or Uniprot crossrefs
  88     List<String> limit=Arrays.asList(new String[] {
  89         DBRefUtils.getCanonicalName("ENSEMBL"),
  90         DBRefUtils.getCanonicalName("Uniprot")});
  91     // for every set of db queries
  92     // retrieve db query
  93     // verify presence of expected xrefs
  94     // show xrefs - verify expected type of frame is shown for each xref
  95     // show xrefs again
  96     // - verify original -> xref -> xref(original) recovers frame containing at
  97     // least the first retrieved sequence
  98     // store
  99     // 1. whole project
 100     // 2. individual frames
 101     // 3. load each one back and verify
 102     // . aligned sequences (.toString() )
 103     // . xrefs (.toString() )
 104     // . codonframes
 105     //
 106     //
 107     HashMap<String, String> dbtoviewBit = new HashMap<>();
 108     List<String> keyseq = new ArrayList<>();
 109     HashMap<String, File> savedProjects = new HashMap<>();
 110
 111 //    for (String[] did : new String[][] { { "UNIPROT", "P00338" } })
 112 //    {
 113       // pass counters - 0 - first pass, 1 means retrieve project rather than
 114       // perform action
 115       int pass1 = 0, pass2 = 0, pass3 = 0;
 116       // each do loop performs two iterations in the first outer loop pass, but
 117       // only performs one iteration on the second outer loop
 118       // ie. pass 1 = 0 {pass 2= 0 { pass 3 = 0,1 }, pass 2=1 { pass 3 = 0 }}, 1
 119       // { pass 2 = 0 { pass 3 = 0 } }
 120       do
 121       {
 122         String first = forSource + " " + forAccession;//did[0] + " " + did[1];
 123         AlignFrame af = null;
 124         boolean dna;
 125         AlignmentI retral;
 126         AlignmentI dataset;
 127         SequenceI[] seqs;
 128         List<String> ptypes = null;
 129         if (pass1 == 0)
 130         {
 131           // retrieve dbref
 132
 133           List<AlignFrame> afs = jalview.gui.SequenceFetcher.fetchAndShow(
 134                 forSource, forAccession);
 135         // did[0], did[1]);
 136           if (afs.size() == 0)
 137           {
 138             failedDBRetr.add("Didn't retrieve " + first);
 139             break;
 140           }
 141           keyseq.add(first);
 142           af = afs.get(0);
 143
 144           // verify references for retrieved data
 145           AlignmentTest.assertAlignmentDatasetRefs(af.getViewport()
 146                   .getAlignment(), "Pass (" + pass1 + "," + pass2 + ","
 147                   + pass3 + "): Fetch " + first + ":");
 148           assertDatasetIsNormalisedKnownDefect(af.getViewport()
 149                   .getAlignment(), "Pass (" + pass1 + "," + pass2 + ","
 150                   + pass3 + "): Fetch " + first + ":");
 151           dna = af.getViewport().getAlignment().isNucleotide();
 152           retral = af.getViewport().getAlignment();
 153           dataset = retral.getDataset();
 154           seqs = retral.getSequencesArray();
 155
 156         }
 157         else
 158         {
 159           Desktop.instance.closeAll_actionPerformed(null);
 160           // recover stored project
 161           af = new FileLoader(false).LoadFileWaitTillLoaded(savedProjects
 162                   .get(first).toString(), DataSourceType.FILE);
 163           System.out.println("Recovered view for '" + first + "' from '"
 164                   + savedProjects.get(first).toString() + "'");
 165           dna = af.getViewport().getAlignment().isNucleotide();
 166           retral = af.getViewport().getAlignment();
 167           dataset = retral.getDataset();
 168           seqs = retral.getSequencesArray();
 169
 170           // verify references for recovered data
 171           AlignmentTest.assertAlignmentDatasetRefs(af.getViewport()
 172                   .getAlignment(), "Pass (" + pass1 + "," + pass2 + ","
 173                   + pass3 + "): Recover " + first + ":");
 174           assertDatasetIsNormalisedKnownDefect(af.getViewport()
 175                   .getAlignment(), "Pass (" + pass1 + "," + pass2 + ","
 176                   + pass3 + "): Recover " + first + ":");
 177
 178         }
 179
 180         // store project on first pass, compare next pass
 181         stringify(dbtoviewBit, savedProjects, first, af.alignPanel);
 182
 183         ptypes = (seqs == null || seqs.length == 0) ? null : new CrossRef(
 184                 seqs, dataset).findXrefSourcesForSequences(dna);
 185         filterDbRefs(ptypes, limit);
 186
 187         // start of pass2: retrieve each cross-ref for fetched or restored
 188         // project.
 189         do // first cross ref and recover crossref loop
 190         {
 191
 192           for (String db : ptypes)
 193           {
 194             // counter for splitframe views retrieved via crossref
 195             int firstcr_ap = 0;
 196             // build next key so we an retrieve all views
 197             String nextxref = first + " -> " + db + "{" + firstcr_ap + "}";
 198             // perform crossref action, or retrieve stored project
 199             List<AlignmentViewPanel> cra_views = new ArrayList<>();
 200             CrossRefAction cra = null;
 201
 202             if (pass2 == 0)
 203             { // retrieve and show cross-refs in this thread
 204               cra = new CrossRefAction(af, seqs, dna, db);
 205               cra.run();
 206               if (cra.getXrefViews().size() == 0)
 207               {
 208                 failedXrefMenuItems.add("No crossrefs retrieved for "
 209                         + first + " -> " + db);
 210                 continue;
 211               }
 212               cra_views = cra.getXrefViews();
 213               assertNucleotide(cra_views.get(0),
 214                       "Nucleotide panel included proteins for " + first
 215                               + " -> " + db);
 216               assertProtein(cra_views.get(1),
 217                       "Protein panel included nucleotides for " + first
 218                               + " -> " + db);
 219             }
 220             else
 221             {
 222               Desktop.instance.closeAll_actionPerformed(null);
 223               pass3 = 0;
 224               // recover stored project
 225               File storedProject = savedProjects.get(nextxref);
 226               if (storedProject == null)
 227               {
 228                 failedProjectRecoveries.add("Failed to store a view for '"
 229                         + nextxref + "'");
 230                 continue;
 231               }
 232
 233               // recover stored project
 234               AlignFrame af2 = new FileLoader(false)
 235                       .LoadFileWaitTillLoaded(savedProjects.get(nextxref)
 236                               .toString(), DataSourceType.FILE);
 237               System.out.println("Recovered view for '" + nextxref
 238                       + "' from '" + savedProjects.get(nextxref).toString()
 239                       + "'");
 240               // gymnastics to recover the alignPanel/Complementary alignPanel
 241               if (af2.getViewport().isNucleotide())
 242               {
 243                 // top view, then bottom
 244                 cra_views.add(af2.getViewport().getAlignPanel());
 245                 cra_views.add(((jalview.gui.AlignViewport) af2
 246                         .getViewport().getCodingComplement())
 247                         .getAlignPanel());
 248
 249               }
 250               else
 251               {
 252                 // bottom view, then top
 253                 cra_views.add(((jalview.gui.AlignViewport) af2
 254                         .getViewport().getCodingComplement())
 255                         .getAlignPanel());
 256                 cra_views.add(af2.getViewport().getAlignPanel());
 257
 258               }
 259             }
 260             HashMap<String, List<String>> xrptypes = new HashMap<>();
 261             // first save/verify views.
 262             for (AlignmentViewPanel avp : cra_views)
 263             {
 264               nextxref = first + " -> " + db + "{" + firstcr_ap++ + "}";
 265               // verify references for this panel
 266               AlignmentTest.assertAlignmentDatasetRefs(avp.getAlignment(),
 267                       "Pass (" + pass1 + "," + pass2 + "," + pass3
 268                               + "): before start of pass3: " + nextxref
 269                               + ":");
 270               assertDatasetIsNormalisedKnownDefect(avp.getAlignment(),
 271                       "Pass (" + pass1 + "," + pass2 + "," + pass3
 272                               + "): before start of pass3: " + nextxref
 273                               + ":");
 274
 275               SequenceI[] xrseqs = avp.getAlignment().getSequencesArray();
 276
 277               List<String> _xrptypes = (seqs == null || seqs.length == 0) ? null
 278                       : new CrossRef(xrseqs, dataset)
 279                               .findXrefSourcesForSequences(avp
 280                                       .getAlignViewport().isNucleotide());
 281
 282               stringify(dbtoviewBit, savedProjects, nextxref, avp);
 283               xrptypes.put(nextxref, _xrptypes);
 284
 285             }
 286
 287             // now do the second xref pass starting from either saved or just
 288             // recovered split pane, in sequence
 289             do // retrieve second set of cross refs or recover and verify
 290             {
 291               firstcr_ap = 0;
 292               for (AlignmentViewPanel avp : cra_views)
 293               {
 294                 nextxref = first + " -> " + db + "{" + firstcr_ap++ + "}";
 295                 for (String xrefdb : xrptypes.get(nextxref))
 296                 {
 297                   List<AlignmentViewPanel> cra_views2 = new ArrayList<>();
 298                   int q = 0;
 299                   String nextnextxref = nextxref + " -> " + xrefdb + "{"
 300                           + q + "}";
 301
 302                   if (pass3 == 0)
 303                   {
 304
 305                     SequenceI[] xrseqs = avp.getAlignment()
 306                             .getSequencesArray();
 307                     AlignFrame nextaf = Desktop.getAlignFrameFor(avp
 308                             .getAlignViewport());
 309
 310                     cra = new CrossRefAction(nextaf, xrseqs, avp
 311                             .getAlignViewport().isNucleotide(), xrefdb);
 312                     cra.run();
 313                     if (cra.getXrefViews().size() == 0)
 314                     {
 315                       failedXrefMenuItems
 316                               .add("No crossrefs retrieved for '"
 317                                       + nextxref + "' to " + xrefdb
 318                                       + " via '" + nextaf.getTitle() + "'");
 319                       continue;
 320                     }
 321                     cra_views2 = cra.getXrefViews();
 322                     assertNucleotide(cra_views2.get(0),
 323                             "Nucleotide panel included proteins for '"
 324                                     + nextxref + "' to " + xrefdb
 325                                     + " via '" + nextaf.getTitle() + "'");
 326                     assertProtein(cra_views2.get(1),
 327                             "Protein panel included nucleotides for '"
 328                                     + nextxref + "' to " + xrefdb
 329                                     + " via '" + nextaf.getTitle() + "'");
 330
 331                   }
 332                   else
 333                   {
 334                     Desktop.instance.closeAll_actionPerformed(null);
 335                     // recover stored project
 336                     File storedProject = savedProjects.get(nextnextxref);
 337                     if (storedProject == null)
 338                     {
 339                       failedProjectRecoveries
 340                               .add("Failed to store a view for '"
 341                                       + nextnextxref + "'");
 342                       continue;
 343                     }
 344                     AlignFrame af2 = new FileLoader(false)
 345                             .LoadFileWaitTillLoaded(
 346                                     savedProjects.get(nextnextxref)
 347                                             .toString(),
 348                                     DataSourceType.FILE);
 349                     System.out.println("Recovered view for '"
 350                             + nextnextxref + "' from '"
 351                             + savedProjects.get(nextnextxref).toString()
 352                             + "'");
 353                     // gymnastics to recover the alignPanel/Complementary
 354                     // alignPanel
 355                     if (af2.getViewport().isNucleotide())
 356                     {
 357                       // top view, then bottom
 358                       cra_views2.add(af2.getViewport().getAlignPanel());
 359                       cra_views2.add(((jalview.gui.AlignViewport) af2
 360                               .getViewport().getCodingComplement())
 361                               .getAlignPanel());
 362
 363                     }
 364                     else
 365                     {
 366                       // bottom view, then top
 367                       cra_views2.add(((jalview.gui.AlignViewport) af2
 368                               .getViewport().getCodingComplement())
 369                               .getAlignPanel());
 370                       cra_views2.add(af2.getViewport().getAlignPanel());
 371                     }
 372                     Assert.assertEquals(cra_views2.size(), 2);
 373                     Assert.assertNotNull(cra_views2.get(0));
 374                     Assert.assertNotNull(cra_views2.get(1));
 375                   }
 376
 377                   for (AlignmentViewPanel nextavp : cra_views2)
 378                   {
 379                     nextnextxref = nextxref + " -> " + xrefdb + "{" + q++
 380                             + "}";
 381
 382                     // verify references for this panel
 383                     AlignmentTest.assertAlignmentDatasetRefs(
 384                             nextavp.getAlignment(), "" + "Pass (" + pass1
 385                                     + "," + pass2 + "): For "
 386                                     + nextnextxref + ":");
 387                     assertDatasetIsNormalisedKnownDefect(
 388                             nextavp.getAlignment(), "" + "Pass (" + pass1
 389                                     + "," + pass2 + "): For "
 390                                     + nextnextxref + ":");
 391
 392                     stringify(dbtoviewBit, savedProjects, nextnextxref,
 393                             nextavp);
 394                     keyseq.add(nextnextxref);
 395                   }
 396                 } // end of loop around showing all xrefdb for crossrf2
 397
 398               } // end of loop around all viewpanels from crossrf1
 399             } while (pass2 == 2 && pass3++ < 2);
 400             // fetchdb->crossref1->crossref-2->verify for xrefs we
 401             // either loop twice when pass2=0, or just once when pass2=1
 402             // (recovered project from previous crossref)
 403
 404           } // end of loop over db-xrefs for crossref-2
 405
 406           // fetchdb-->crossref1
 407           // for each xref we try to retrieve xref, store and verify when
 408           // pass1=0, or just retrieve and verify when pass1=1
 409         } while (pass1 == 1 && pass2++ < 2);
 410         // fetchdb
 411         // for each ref we
 412         // loop twice: first, do the retrieve, second recover from saved project
 413
 414         // increment pass counters, so we repeat traversal starting from the
 415         // oldest saved project first.
 416         if (pass1 == 0)
 417         {
 418           // verify stored projects for first set of cross references
 419           pass1 = 1;
 420           // and verify cross-references retrieved from stored projects
 421           pass2 = 0;
 422           pass3 = 0;
 423         }
 424         else
 425         {
 426           pass1++;
 427         }
 428       } while (pass1 < 3);
 429
 430     if (failedXrefMenuItems.size() > 0)
 431     {
 432       for (String s : failedXrefMenuItems)
 433       {
 434         System.err.println(s);
 435       }
 436       Assert.fail("Faulty xref menu (" + failedXrefMenuItems.size()
 437               + " counts)");
 438     }
 439     if (failedProjectRecoveries.size() > 0)
 440     {
 441
 442       for (String s : failedProjectRecoveries)
 443       {
 444         System.err.println(s);
 445       }
 446       Assert.fail("Didn't recover projects for some retrievals (did they retrieve ?) ("
 447               + failedProjectRecoveries.size() + " counts)");
 448     }
 449     if (failedDBRetr.size() > 0)
 450     {
 451       for (String s : failedProjectRecoveries)
 452       {
 453         System.err.println(s);
 454       }
 455       Assert.fail("Didn't retrieve some db refs for checking cross-refs ("
 456               + failedDBRetr.size() + " counts)");
 457     }
 458   }
 459
 460   private void filterDbRefs(List<String> ptypes, List<String> limit)
 461   {
 462     if (limit != null)
 463     {
 464       int p = 0;
 465       while (ptypes.size() > p)
 466       {
 467         if (!limit.contains(ptypes.get(p)))
 468         {
 469           ptypes.remove(p);
 470         }
 471         else
 472         {
 473           p++;
 474         }
 475       }
 476     }
 477   }
 478
 479   /**
 480    * wrapper to trap known defect for AH002001 testcase
 481    *
 482    * @param alignment
 483    * @param string
 484    */
 485   private void assertDatasetIsNormalisedKnownDefect(AlignmentI al,
 486           String message)
 487   {
 488     try
 489     {
 490       AlignmentTest.assertDatasetIsNormalised(al, message);
 491     } catch (AssertionError ae)
 492     {
 493       if (!ae.getMessage().endsWith("EMBL|AH002001"))
 494       {
 495         throw ae;
 496       }
 497       else
 498       {
 499         System.out
 500                 .println("Ignored exception for known defect: JAL-2179 : "
 501                         + message);
 502       }
 503
 504     }
 505   }
 506
 507   private void assertProtein(AlignmentViewPanel alignmentViewPanel,
 508           String message)
 509   {
 510     assertType(true, alignmentViewPanel, message);
 511   }
 512
 513   private void assertNucleotide(AlignmentViewPanel alignmentViewPanel,
 514           String message)
 515   {
 516     assertType(false, alignmentViewPanel, message);
 517   }
 518
 519   private void assertType(boolean expectProtein,
 520           AlignmentViewPanel alignmentViewPanel, String message)
 521   {
 522     List<SequenceI> nonType = new ArrayList<>();
 523     for (SequenceI sq : alignmentViewPanel.getAlignViewport()
 524             .getAlignment().getSequences())
 525     {
 526       if (sq.isProtein() != expectProtein)
 527       {
 528         nonType.add(sq);
 529       }
 530     }
 531     if (nonType.size() > 0)
 532     {
 533       Assert.fail(message + " [ "
 534               + (expectProtein ? "nucleotides were " : "proteins were ")
 535               + nonType.toString() + " ]");
 536     }
 537   }
 538
 539   /**
 540    * first time called, record strings derived from alignment and
 541    * alignedcodonframes, and save view to a project file. Second time called,
 542    * compare strings to existing ones. org.testng.Assert.assertTrue on
 543    * stringmatch
 544    *
 545    * @param dbtoviewBit
 546    *          map between xrefpath and view string
 547    * @param savedProjects
 548    *          - map from xrefpath to saved project filename (createTempFile)
 549    * @param xrefpath
 550    *          - xrefpath - unique ID for this context (composed of sequence of
 551    *          db-fetch/cross-ref actions preceeding state)
 552    * @param avp
 553    *          - viewpanel to store (for viewpanels in splitframe, the same
 554    *          project should be written for both panels, only one needs
 555    *          recovering for comparison on the next stringify call, but each
 556    *          viewpanel needs to be called with a distinct xrefpath to ensure
 557    *          each one's strings are compared)
 558    */
 559   private void stringify(HashMap<String, String> dbtoviewBit,
 560           HashMap<String, File> savedProjects, String xrefpath,
 561           AlignmentViewPanel avp)
 562   {
 563     if (savedProjects != null)
 564     {
 565       if (savedProjects.get(xrefpath) == null)
 566       {
 567         // write a project file for this view. On the second pass, this will be
 568         // recovered and cross-references verified
 569         try
 570         {
 571           File prfile = File.createTempFile("crossRefTest", ".jvp");
 572           AlignFrame af = Desktop.getAlignFrameFor(avp.getAlignViewport());
 573           new Jalview2XML(false).saveAlignment(af, prfile.toString(),
 574                   af.getTitle());
 575           System.out.println("Written view from '" + xrefpath + "' as '"
 576                   + prfile.getAbsolutePath() + "'");
 577           savedProjects.put(xrefpath, prfile);
 578         } catch (IOException q)
 579         {
 580           Assert.fail("Unexpected IO Exception", q);
 581         }
 582       }
 583       else
 584       {
 585         System.out.println("Stringify check on view from '" + xrefpath
 586                 + "' [ possibly retrieved from '"
 587                 + savedProjects.get(xrefpath).getAbsolutePath() + "' ]");
 588
 589       }
 590     }
 591
 592     StringBuilder sbr = new StringBuilder();
 593     sbr.append(avp.getAlignment().toString());
 594     sbr.append("\n");
 595     sbr.append("<End of alignment>");
 596     sbr.append("\n");
 597     sbr.append(avp.getAlignment().getDataset());
 598     sbr.append("\n");
 599     sbr.append("<End of dataset>");
 600     sbr.append("\n");
 601     int p = 0;
 602     if (avp.getAlignment().getCodonFrames() != null)
 603     {
 604       for (AlignedCodonFrame ac : avp.getAlignment().getCodonFrames())
 605       {
 606         sbr.append("<AlignedCodonFrame " + p++ + ">");
 607         sbr.append("\n");
 608         sbr.append(ac.toString());
 609         sbr.append("\n");
 610       }
 611     }
 612     String dbt = dbtoviewBit.get(xrefpath);
 613     if (dbt == null)
 614     {
 615       dbtoviewBit.put(xrefpath, sbr.toString());
 616     }
 617     else
 618     {
 619       Assert.assertEquals(sbr.toString(), dbt, "stringify mismatch for "
 620               + xrefpath);
 621     }
 622   }
 623 }