test/jalview/io/CrossRef2xmlTests.java

   1 /*
   2  * Jalview - A Sequence Alignment Editor and Viewer ($$Version-Rel$$)
   3  * Copyright (C) $$Year-Rel$$ The Jalview Authors
   4  *
   5  * This file is part of Jalview.
   6  *
   7  * Jalview is free software: you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License
   9  * as published by the Free Software Foundation, either version 3
  10  * of the License, or (at your option) any later version.
  11  *
  12  * Jalview is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty
  14  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR
  15  * PURPOSE.  See the GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with Jalview.  If not, see <http://www.gnu.org/licenses/>.
  19  * The Jalview Authors are detailed in the 'AUTHORS' file.
  20  */
  21 package jalview.io;
  22
  23 import jalview.analysis.CrossRef;
  24 import jalview.api.AlignmentViewPanel;
  25 import jalview.datamodel.AlignedCodonFrame;
  26 import jalview.datamodel.AlignmentI;
  27 import jalview.datamodel.AlignmentTest;
  28 import jalview.datamodel.SequenceI;
  29 import jalview.gui.AlignFrame;
  30 import jalview.gui.CrossRefAction;
  31 import jalview.gui.Desktop;
  32 import jalview.gui.Jalview2XML;
  33 import jalview.gui.JvOptionPane;
  34 import jalview.util.DBRefUtils;
  35
  36 import java.io.File;
  37 import java.io.IOException;
  38 import java.util.ArrayList;
  39 import java.util.Arrays;
  40 import java.util.HashMap;
  41 import java.util.List;
  42
  43 import org.testng.Assert;
  44 import org.testng.annotations.BeforeClass;
  45 import org.testng.annotations.Test;
  46
  47 @Test(singleThreaded = true)
  48 public class CrossRef2xmlTests extends Jalview2xmlBase
  49 {
  50
  51   @Override
  52   @BeforeClass(alwaysRun = true)
  53   public void setUpJvOptionPane()
  54   {
  55     JvOptionPane.setInteractiveMode(false);
  56     JvOptionPane.setMockResponse(JvOptionPane.CANCEL_OPTION);
  57   }
  58
  59   /**
  60    * test store and recovery of all reachable cross refs from all reachable
  61    * crossrefs for one or more fetched db refs. Currently, this test has a known
  62    * failure case.
  63    *
  64    * @throws Exception
  65    */
  66   @Test(groups = { "Operational" }, enabled = true)
  67   public void testRetrieveAndShowCrossref() throws Exception
  68   {
  69
  70     List<String> failedDBRetr = new ArrayList<>();
  71     List<String> failedXrefMenuItems = new ArrayList<>();
  72     List<String> failedProjectRecoveries = new ArrayList<>();
  73     // only search for ensembl or Uniprot crossrefs
  74     List<String> limit=Arrays.asList(new String[] {
  75         DBRefUtils.getCanonicalName("ENSEMBL"),
  76         DBRefUtils.getCanonicalName("Uniprot")});
  77     // for every set of db queries
  78     // retrieve db query
  79     // verify presence of expected xrefs
  80     // show xrefs - verify expected type of frame is shown for each xref
  81     // show xrefs again
  82     // - verify original -> xref -> xref(original) recovers frame containing at
  83     // least the first retrieved sequence
  84     // store
  85     // 1. whole project
  86     // 2. individual frames
  87     // 3. load each one back and verify
  88     // . aligned sequences (.toString() )
  89     // . xrefs (.toString() )
  90     // . codonframes
  91     //
  92     //
  93     HashMap<String, String> dbtoviewBit = new HashMap<>();
  94     List<String> keyseq = new ArrayList<>();
  95     HashMap<String, File> savedProjects = new HashMap<>();
  96
  97     for (String[] did : new String[][] { { "UNIPROT", "P00338" } })
  98     {
  99       // pass counters - 0 - first pass, 1 means retrieve project rather than
 100       // perform action
 101       int pass1 = 0, pass2 = 0, pass3 = 0;
 102       // each do loop performs two iterations in the first outer loop pass, but
 103       // only performs one iteration on the second outer loop
 104       // ie. pass 1 = 0 {pass 2= 0 { pass 3 = 0,1 }, pass 2=1 { pass 3 = 0 }}, 1
 105       // { pass 2 = 0 { pass 3 = 0 } }
 106       do
 107       {
 108         String first = did[0] + " " + did[1];
 109         AlignFrame af = null;
 110         boolean dna;
 111         AlignmentI retral;
 112         AlignmentI dataset;
 113         SequenceI[] seqs;
 114         List<String> ptypes = null;
 115         if (pass1 == 0)
 116         {
 117           // retrieve dbref
 118
 119           List<AlignFrame> afs = jalview.gui.SequenceFetcher.fetchAndShow(
 120                   did[0], did[1]);
 121           if (afs.size() == 0)
 122           {
 123             failedDBRetr.add("Didn't retrieve " + first);
 124             break;
 125           }
 126           keyseq.add(first);
 127           af = afs.get(0);
 128
 129           // verify references for retrieved data
 130           AlignmentTest.assertAlignmentDatasetRefs(af.getViewport()
 131                   .getAlignment(), "Pass (" + pass1 + "," + pass2 + ","
 132                   + pass3 + "): Fetch " + first + ":");
 133           assertDatasetIsNormalisedKnownDefect(af.getViewport()
 134                   .getAlignment(), "Pass (" + pass1 + "," + pass2 + ","
 135                   + pass3 + "): Fetch " + first + ":");
 136           dna = af.getViewport().getAlignment().isNucleotide();
 137           retral = af.getViewport().getAlignment();
 138           dataset = retral.getDataset();
 139           seqs = retral.getSequencesArray();
 140
 141         }
 142         else
 143         {
 144           Desktop.instance.closeAll_actionPerformed(null);
 145           // recover stored project
 146           af = new FileLoader(false).LoadFileWaitTillLoaded(savedProjects
 147                   .get(first).toString(), DataSourceType.FILE);
 148           System.out.println("Recovered view for '" + first + "' from '"
 149                   + savedProjects.get(first).toString() + "'");
 150           dna = af.getViewport().getAlignment().isNucleotide();
 151           retral = af.getViewport().getAlignment();
 152           dataset = retral.getDataset();
 153           seqs = retral.getSequencesArray();
 154
 155           // verify references for recovered data
 156           AlignmentTest.assertAlignmentDatasetRefs(af.getViewport()
 157                   .getAlignment(), "Pass (" + pass1 + "," + pass2 + ","
 158                   + pass3 + "): Recover " + first + ":");
 159           assertDatasetIsNormalisedKnownDefect(af.getViewport()
 160                   .getAlignment(), "Pass (" + pass1 + "," + pass2 + ","
 161                   + pass3 + "): Recover " + first + ":");
 162
 163         }
 164
 165         // store project on first pass, compare next pass
 166         stringify(dbtoviewBit, savedProjects, first, af.alignPanel);
 167
 168         ptypes = (seqs == null || seqs.length == 0) ? null : new CrossRef(
 169                 seqs, dataset).findXrefSourcesForSequences(dna);
 170         filterDbRefs(ptypes, limit);
 171
 172         // start of pass2: retrieve each cross-ref for fetched or restored
 173         // project.
 174         do // first cross ref and recover crossref loop
 175         {
 176
 177           for (String db : ptypes)
 178           {
 179             // counter for splitframe views retrieved via crossref
 180             int firstcr_ap = 0;
 181             // build next key so we an retrieve all views
 182             String nextxref = first + " -> " + db + "{" + firstcr_ap + "}";
 183             // perform crossref action, or retrieve stored project
 184             List<AlignmentViewPanel> cra_views = new ArrayList<>();
 185             CrossRefAction cra = null;
 186
 187             if (pass2 == 0)
 188             { // retrieve and show cross-refs in this thread
 189               cra = new CrossRefAction(af, seqs, dna, db);
 190               cra.run();
 191               if (cra.getXrefViews().size() == 0)
 192               {
 193                 failedXrefMenuItems.add("No crossrefs retrieved for "
 194                         + first + " -> " + db);
 195                 continue;
 196               }
 197               cra_views = cra.getXrefViews();
 198               assertNucleotide(cra_views.get(0),
 199                       "Nucleotide panel included proteins for " + first
 200                               + " -> " + db);
 201               assertProtein(cra_views.get(1),
 202                       "Protein panel included nucleotides for " + first
 203                               + " -> " + db);
 204             }
 205             else
 206             {
 207               Desktop.instance.closeAll_actionPerformed(null);
 208               pass3 = 0;
 209               // recover stored project
 210               File storedProject = savedProjects.get(nextxref);
 211               if (storedProject == null)
 212               {
 213                 failedProjectRecoveries.add("Failed to store a view for '"
 214                         + nextxref + "'");
 215                 continue;
 216               }
 217
 218               // recover stored project
 219               AlignFrame af2 = new FileLoader(false)
 220                       .LoadFileWaitTillLoaded(savedProjects.get(nextxref)
 221                               .toString(), DataSourceType.FILE);
 222               System.out.println("Recovered view for '" + nextxref
 223                       + "' from '" + savedProjects.get(nextxref).toString()
 224                       + "'");
 225               // gymnastics to recover the alignPanel/Complementary alignPanel
 226               if (af2.getViewport().isNucleotide())
 227               {
 228                 // top view, then bottom
 229                 cra_views.add(af2.getViewport().getAlignPanel());
 230                 cra_views.add(((jalview.gui.AlignViewport) af2
 231                         .getViewport().getCodingComplement())
 232                         .getAlignPanel());
 233
 234               }
 235               else
 236               {
 237                 // bottom view, then top
 238                 cra_views.add(((jalview.gui.AlignViewport) af2
 239                         .getViewport().getCodingComplement())
 240                         .getAlignPanel());
 241                 cra_views.add(af2.getViewport().getAlignPanel());
 242
 243               }
 244             }
 245             HashMap<String, List<String>> xrptypes = new HashMap<>();
 246             // first save/verify views.
 247             for (AlignmentViewPanel avp : cra_views)
 248             {
 249               nextxref = first + " -> " + db + "{" + firstcr_ap++ + "}";
 250               // verify references for this panel
 251               AlignmentTest.assertAlignmentDatasetRefs(avp.getAlignment(),
 252                       "Pass (" + pass1 + "," + pass2 + "," + pass3
 253                               + "): before start of pass3: " + nextxref
 254                               + ":");
 255               assertDatasetIsNormalisedKnownDefect(avp.getAlignment(),
 256                       "Pass (" + pass1 + "," + pass2 + "," + pass3
 257                               + "): before start of pass3: " + nextxref
 258                               + ":");
 259
 260               SequenceI[] xrseqs = avp.getAlignment().getSequencesArray();
 261
 262               List<String> _xrptypes = (seqs == null || seqs.length == 0) ? null
 263                       : new CrossRef(xrseqs, dataset)
 264                               .findXrefSourcesForSequences(avp
 265                                       .getAlignViewport().isNucleotide());
 266
 267               stringify(dbtoviewBit, savedProjects, nextxref, avp);
 268               xrptypes.put(nextxref, _xrptypes);
 269
 270             }
 271
 272             // now do the second xref pass starting from either saved or just
 273             // recovered split pane, in sequence
 274             do // retrieve second set of cross refs or recover and verify
 275             {
 276               firstcr_ap = 0;
 277               for (AlignmentViewPanel avp : cra_views)
 278               {
 279                 nextxref = first + " -> " + db + "{" + firstcr_ap++ + "}";
 280                 for (String xrefdb : xrptypes.get(nextxref))
 281                 {
 282                   List<AlignmentViewPanel> cra_views2 = new ArrayList<>();
 283                   int q = 0;
 284                   String nextnextxref = nextxref + " -> " + xrefdb + "{"
 285                           + q + "}";
 286
 287                   if (pass3 == 0)
 288                   {
 289
 290                     SequenceI[] xrseqs = avp.getAlignment()
 291                             .getSequencesArray();
 292                     AlignFrame nextaf = Desktop.getAlignFrameFor(avp
 293                             .getAlignViewport());
 294
 295                     cra = new CrossRefAction(nextaf, xrseqs, avp
 296                             .getAlignViewport().isNucleotide(), xrefdb);
 297                     cra.run();
 298                     if (cra.getXrefViews().size() == 0)
 299                     {
 300                       failedXrefMenuItems
 301                               .add("No crossrefs retrieved for '"
 302                                       + nextxref + "' to " + xrefdb
 303                                       + " via '" + nextaf.getTitle() + "'");
 304                       continue;
 305                     }
 306                     cra_views2 = cra.getXrefViews();
 307                     assertNucleotide(cra_views2.get(0),
 308                             "Nucleotide panel included proteins for '"
 309                                     + nextxref + "' to " + xrefdb
 310                                     + " via '" + nextaf.getTitle() + "'");
 311                     assertProtein(cra_views2.get(1),
 312                             "Protein panel included nucleotides for '"
 313                                     + nextxref + "' to " + xrefdb
 314                                     + " via '" + nextaf.getTitle() + "'");
 315
 316                   }
 317                   else
 318                   {
 319                     Desktop.instance.closeAll_actionPerformed(null);
 320                     // recover stored project
 321                     File storedProject = savedProjects.get(nextnextxref);
 322                     if (storedProject == null)
 323                     {
 324                       failedProjectRecoveries
 325                               .add("Failed to store a view for '"
 326                                       + nextnextxref + "'");
 327                       continue;
 328                     }
 329                     AlignFrame af2 = new FileLoader(false)
 330                             .LoadFileWaitTillLoaded(
 331                                     savedProjects.get(nextnextxref)
 332                                             .toString(),
 333                                     DataSourceType.FILE);
 334                     System.out.println("Recovered view for '"
 335                             + nextnextxref + "' from '"
 336                             + savedProjects.get(nextnextxref).toString()
 337                             + "'");
 338                     // gymnastics to recover the alignPanel/Complementary
 339                     // alignPanel
 340                     if (af2.getViewport().isNucleotide())
 341                     {
 342                       // top view, then bottom
 343                       cra_views2.add(af2.getViewport().getAlignPanel());
 344                       cra_views2.add(((jalview.gui.AlignViewport) af2
 345                               .getViewport().getCodingComplement())
 346                               .getAlignPanel());
 347
 348                     }
 349                     else
 350                     {
 351                       // bottom view, then top
 352                       cra_views2.add(((jalview.gui.AlignViewport) af2
 353                               .getViewport().getCodingComplement())
 354                               .getAlignPanel());
 355                       cra_views2.add(af2.getViewport().getAlignPanel());
 356                     }
 357                     Assert.assertEquals(cra_views2.size(), 2);
 358                     Assert.assertNotNull(cra_views2.get(0));
 359                     Assert.assertNotNull(cra_views2.get(1));
 360                   }
 361
 362                   for (AlignmentViewPanel nextavp : cra_views2)
 363                   {
 364                     nextnextxref = nextxref + " -> " + xrefdb + "{" + q++
 365                             + "}";
 366
 367                     // verify references for this panel
 368                     AlignmentTest.assertAlignmentDatasetRefs(
 369                             nextavp.getAlignment(), "" + "Pass (" + pass1
 370                                     + "," + pass2 + "): For "
 371                                     + nextnextxref + ":");
 372                     assertDatasetIsNormalisedKnownDefect(
 373                             nextavp.getAlignment(), "" + "Pass (" + pass1
 374                                     + "," + pass2 + "): For "
 375                                     + nextnextxref + ":");
 376
 377                     stringify(dbtoviewBit, savedProjects, nextnextxref,
 378                             nextavp);
 379                     keyseq.add(nextnextxref);
 380                   }
 381                 } // end of loop around showing all xrefdb for crossrf2
 382
 383               } // end of loop around all viewpanels from crossrf1
 384             } while (pass2 == 2 && pass3++ < 2);
 385             // fetchdb->crossref1->crossref-2->verify for xrefs we
 386             // either loop twice when pass2=0, or just once when pass2=1
 387             // (recovered project from previous crossref)
 388
 389           } // end of loop over db-xrefs for crossref-2
 390
 391           // fetchdb-->crossref1
 392           // for each xref we try to retrieve xref, store and verify when
 393           // pass1=0, or just retrieve and verify when pass1=1
 394         } while (pass1 == 1 && pass2++ < 2);
 395         // fetchdb
 396         // for each ref we
 397         // loop twice: first, do the retrieve, second recover from saved project
 398
 399         // increment pass counters, so we repeat traversal starting from the
 400         // oldest saved project first.
 401         if (pass1 == 0)
 402         {
 403           // verify stored projects for first set of cross references
 404           pass1 = 1;
 405           // and verify cross-references retrieved from stored projects
 406           pass2 = 0;
 407           pass3 = 0;
 408         }
 409         else
 410         {
 411           pass1++;
 412         }
 413       } while (pass1 < 3);
 414     }
 415     if (failedXrefMenuItems.size() > 0)
 416     {
 417       for (String s : failedXrefMenuItems)
 418       {
 419         System.err.println(s);
 420       }
 421       Assert.fail("Faulty xref menu (" + failedXrefMenuItems.size()
 422               + " counts)");
 423     }
 424     if (failedProjectRecoveries.size() > 0)
 425     {
 426
 427       for (String s : failedProjectRecoveries)
 428       {
 429         System.err.println(s);
 430       }
 431       Assert.fail("Didn't recover projects for some retrievals (did they retrieve ?) ("
 432               + failedProjectRecoveries.size() + " counts)");
 433     }
 434     if (failedDBRetr.size() > 0)
 435     {
 436       for (String s : failedProjectRecoveries)
 437       {
 438         System.err.println(s);
 439       }
 440       Assert.fail("Didn't retrieve some db refs for checking cross-refs ("
 441               + failedDBRetr.size() + " counts)");
 442     }
 443   }
 444
 445   private void filterDbRefs(List<String> ptypes, List<String> limit)
 446   {
 447     if (limit != null)
 448     {
 449       int p = 0;
 450       while (ptypes.size() > p)
 451       {
 452         if (!limit.contains(ptypes.get(p)))
 453         {
 454           ptypes.remove(p);
 455         }
 456         else
 457         {
 458           p++;
 459         }
 460       }
 461     }
 462   }
 463
 464   /**
 465    * wrapper to trap known defect for AH002001 testcase
 466    *
 467    * @param alignment
 468    * @param string
 469    */
 470   private void assertDatasetIsNormalisedKnownDefect(AlignmentI al,
 471           String message)
 472   {
 473     try
 474     {
 475       AlignmentTest.assertDatasetIsNormalised(al, message);
 476     } catch (AssertionError ae)
 477     {
 478       if (!ae.getMessage().endsWith("EMBL|AH002001"))
 479       {
 480         throw ae;
 481       }
 482       else
 483       {
 484         System.out
 485                 .println("Ignored exception for known defect: JAL-2179 : "
 486                         + message);
 487       }
 488
 489     }
 490   }
 491
 492   private void assertProtein(AlignmentViewPanel alignmentViewPanel,
 493           String message)
 494   {
 495     assertType(true, alignmentViewPanel, message);
 496   }
 497
 498   private void assertNucleotide(AlignmentViewPanel alignmentViewPanel,
 499           String message)
 500   {
 501     assertType(false, alignmentViewPanel, message);
 502   }
 503
 504   private void assertType(boolean expectProtein,
 505           AlignmentViewPanel alignmentViewPanel, String message)
 506   {
 507     List<SequenceI> nonType = new ArrayList<>();
 508     for (SequenceI sq : alignmentViewPanel.getAlignViewport()
 509             .getAlignment().getSequences())
 510     {
 511       if (sq.isProtein() != expectProtein)
 512       {
 513         nonType.add(sq);
 514       }
 515     }
 516     if (nonType.size() > 0)
 517     {
 518       Assert.fail(message + " [ "
 519               + (expectProtein ? "nucleotides were " : "proteins were ")
 520               + nonType.toString() + " ]");
 521     }
 522   }
 523
 524   /**
 525    * first time called, record strings derived from alignment and
 526    * alignedcodonframes, and save view to a project file. Second time called,
 527    * compare strings to existing ones. org.testng.Assert.assertTrue on
 528    * stringmatch
 529    *
 530    * @param dbtoviewBit
 531    *          map between xrefpath and view string
 532    * @param savedProjects
 533    *          - map from xrefpath to saved project filename (createTempFile)
 534    * @param xrefpath
 535    *          - xrefpath - unique ID for this context (composed of sequence of
 536    *          db-fetch/cross-ref actions preceeding state)
 537    * @param avp
 538    *          - viewpanel to store (for viewpanels in splitframe, the same
 539    *          project should be written for both panels, only one needs
 540    *          recovering for comparison on the next stringify call, but each
 541    *          viewpanel needs to be called with a distinct xrefpath to ensure
 542    *          each one's strings are compared)
 543    */
 544   private void stringify(HashMap<String, String> dbtoviewBit,
 545           HashMap<String, File> savedProjects, String xrefpath,
 546           AlignmentViewPanel avp)
 547   {
 548     if (savedProjects != null)
 549     {
 550       if (savedProjects.get(xrefpath) == null)
 551       {
 552         // write a project file for this view. On the second pass, this will be
 553         // recovered and cross-references verified
 554         try
 555         {
 556           File prfile = File.createTempFile("crossRefTest", ".jvp");
 557           AlignFrame af = Desktop.getAlignFrameFor(avp.getAlignViewport());
 558           new Jalview2XML(false).saveAlignment(af, prfile.toString(),
 559                   af.getTitle());
 560           System.out.println("Written view from '" + xrefpath + "' as '"
 561                   + prfile.getAbsolutePath() + "'");
 562           savedProjects.put(xrefpath, prfile);
 563         } catch (IOException q)
 564         {
 565           Assert.fail("Unexpected IO Exception", q);
 566         }
 567       }
 568       else
 569       {
 570         System.out.println("Stringify check on view from '" + xrefpath
 571                 + "' [ possibly retrieved from '"
 572                 + savedProjects.get(xrefpath).getAbsolutePath() + "' ]");
 573
 574       }
 575     }
 576
 577     StringBuilder sbr = new StringBuilder();
 578     sbr.append(avp.getAlignment().toString());
 579     sbr.append("\n");
 580     sbr.append("<End of alignment>");
 581     sbr.append("\n");
 582     sbr.append(avp.getAlignment().getDataset());
 583     sbr.append("\n");
 584     sbr.append("<End of dataset>");
 585     sbr.append("\n");
 586     int p = 0;
 587     if (avp.getAlignment().getCodonFrames() != null)
 588     {
 589       for (AlignedCodonFrame ac : avp.getAlignment().getCodonFrames())
 590       {
 591         sbr.append("<AlignedCodonFrame " + p++ + ">");
 592         sbr.append("\n");
 593         sbr.append(ac.toString());
 594         sbr.append("\n");
 595       }
 596     }
 597     String dbt = dbtoviewBit.get(xrefpath);
 598     if (dbt == null)
 599     {
 600       dbtoviewBit.put(xrefpath, sbr.toString());
 601     }
 602     else
 603     {
 604       Assert.assertEquals(sbr.toString(), dbt, "stringify mismatch for "
 605               + xrefpath);
 606     }
 607   }
 608 }