test/jalview/io/CrossRef2xmlTests.java

   1 /*
   2  * Jalview - A Sequence Alignment Editor and Viewer ($$Version-Rel$$)
   3  * Copyright (C) $$Year-Rel$$ The Jalview Authors
   4  *
   5  * This file is part of Jalview.
   6  *
   7  * Jalview is free software: you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License
   9  * as published by the Free Software Foundation, either version 3
  10  * of the License, or (at your option) any later version.
  11  *
  12  * Jalview is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty
  14  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR
  15  * PURPOSE.  See the GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with Jalview.  If not, see <http://www.gnu.org/licenses/>.
  19  * The Jalview Authors are detailed in the 'AUTHORS' file.
  20  */
  21 package jalview.io;
  22
  23 import jalview.analysis.CrossRef;
  24 import jalview.api.AlignmentViewPanel;
  25 import jalview.datamodel.AlignedCodonFrame;
  26 import jalview.datamodel.AlignmentI;
  27 import jalview.datamodel.AlignmentTest;
  28 import jalview.datamodel.SequenceI;
  29 import jalview.gui.AlignFrame;
  30 import jalview.gui.CrossRefAction;
  31 import jalview.gui.Desktop;
  32 import jalview.gui.Jalview2XML;
  33 import jalview.gui.JvOptionPane;
  34 import jalview.util.DBRefUtils;
  35
  36 import java.io.File;
  37 import java.io.IOException;
  38 import java.util.ArrayList;
  39 import java.util.Arrays;
  40 import java.util.HashMap;
  41 import java.util.List;
  42 import java.util.Map;
  43
  44 import junit.extensions.PA;
  45
  46 import org.testng.Assert;
  47 import org.testng.annotations.BeforeClass;
  48 import org.testng.annotations.DataProvider;
  49 import org.testng.annotations.Test;
  50
  51 @Test(singleThreaded = true)
  52 public class CrossRef2xmlTests extends Jalview2xmlBase
  53 {
  54
  55   @Override
  56   @BeforeClass(alwaysRun = true)
  57   public void setUpJvOptionPane()
  58   {
  59     JvOptionPane.setInteractiveMode(false);
  60     JvOptionPane.setMockResponse(JvOptionPane.CANCEL_OPTION);
  61   }
  62
  63   @DataProvider(name = "initialAccessions")
  64   static Object[][] getAccessions()
  65   {
  66     return new String[][] { { "UNIPROT", "P00338" },
  67         { "UNIPROT", "Q8Z9G6" },
  68         { "ENSEMBLGENOMES", "CAD01290" } };
  69   }
  70
  71   /**
  72    * test store and recovery of all reachable cross refs from all reachable
  73    * crossrefs for one or more fetched db refs. Currently, this test has a known
  74    * failure case.
  75    *
  76    * @throws Exception
  77    */
  78   @Test(
  79     groups =
  80     { "Operational" },
  81     dataProvider = "initialAccessions",
  82     enabled = true)
  83   public void testRetrieveAndShowCrossref(String forSource,
  84           String forAccession) throws Exception
  85   {
  86
  87     List<String> failedDBRetr = new ArrayList<>();
  88     List<String> failedXrefMenuItems = new ArrayList<>();
  89     List<String> failedProjectRecoveries = new ArrayList<>();
  90     // only search for ensembl or Uniprot crossrefs
  91     List<String> limit=Arrays.asList(new String[] {
  92         DBRefUtils.getCanonicalName("ENSEMBL"),
  93         DBRefUtils.getCanonicalName("Uniprot")});
  94     // for every set of db queries
  95     // retrieve db query
  96     // verify presence of expected xrefs
  97     // show xrefs - verify expected type of frame is shown for each xref
  98     // show xrefs again
  99     // - verify original -> xref -> xref(original) recovers frame containing at
 100     // least the first retrieved sequence
 101     // store
 102     // 1. whole project
 103     // 2. individual frames
 104     // 3. load each one back and verify
 105     // . aligned sequences (.toString() )
 106     // . xrefs (.toString() )
 107     // . codonframes
 108     //
 109     //
 110     Map<String, String> dbtoviewBit = new HashMap<>();
 111     List<String> keyseq = new ArrayList<>();
 112     Map<String, File> savedProjects = new HashMap<>();
 113
 114 //    for (String[] did : new String[][] { { "UNIPROT", "P00338" } })
 115 //    {
 116       // pass counters - 0 - first pass, 1 means retrieve project rather than
 117       // perform action
 118       int pass1 = 0, pass2 = 0, pass3 = 0;
 119       // each do loop performs two iterations in the first outer loop pass, but
 120       // only performs one iteration on the second outer loop
 121       // ie. pass 1 = 0 {pass 2= 0 { pass 3 = 0,1 }, pass 2=1 { pass 3 = 0 }}, 1
 122       // { pass 2 = 0 { pass 3 = 0 } }
 123       do
 124       {
 125         String first = forSource + " " + forAccession;//did[0] + " " + did[1];
 126         AlignFrame af = null;
 127         boolean dna;
 128         AlignmentI retral;
 129         AlignmentI dataset;
 130         SequenceI[] seqs;
 131         List<String> ptypes = null;
 132         if (pass1 == 0)
 133         {
 134           // retrieve dbref
 135
 136           List<AlignFrame> afs = jalview.gui.SequenceFetcher.fetchAndShow(
 137                 forSource, forAccession);
 138         // did[0], did[1]);
 139           if (afs.size() == 0)
 140           {
 141             failedDBRetr.add("Didn't retrieve " + first);
 142             break;
 143           }
 144           keyseq.add(first);
 145           af = afs.get(0);
 146
 147           // verify references for retrieved data
 148           AlignmentTest.assertAlignmentDatasetRefs(af.getViewport()
 149                   .getAlignment(), "Pass (" + pass1 + "," + pass2 + ","
 150                   + pass3 + "): Fetch " + first + ":");
 151           assertDatasetIsNormalisedKnownDefect(af.getViewport()
 152                   .getAlignment(), "Pass (" + pass1 + "," + pass2 + ","
 153                   + pass3 + "): Fetch " + first + ":");
 154           dna = af.getViewport().getAlignment().isNucleotide();
 155           retral = af.getViewport().getAlignment();
 156           dataset = retral.getDataset();
 157           seqs = retral.getSequencesArray();
 158
 159         }
 160         else
 161         {
 162           Desktop.instance.closeAll_actionPerformed(null);
 163           // recover stored project
 164           af = new FileLoader(false).LoadFileWaitTillLoaded(savedProjects
 165                   .get(first).toString(), DataSourceType.FILE);
 166           System.out.println("Recovered view for '" + first + "' from '"
 167                   + savedProjects.get(first).toString() + "'");
 168           dna = af.getViewport().getAlignment().isNucleotide();
 169           retral = af.getViewport().getAlignment();
 170           dataset = retral.getDataset();
 171           seqs = retral.getSequencesArray();
 172
 173           // verify references for recovered data
 174           AlignmentTest.assertAlignmentDatasetRefs(af.getViewport()
 175                   .getAlignment(), "Pass (" + pass1 + "," + pass2 + ","
 176                   + pass3 + "): Recover " + first + ":");
 177           assertDatasetIsNormalisedKnownDefect(af.getViewport()
 178                   .getAlignment(), "Pass (" + pass1 + "," + pass2 + ","
 179                   + pass3 + "): Recover " + first + ":");
 180
 181         }
 182
 183         // store project on first pass, compare next pass
 184         stringify(dbtoviewBit, savedProjects, first, af.alignPanel);
 185
 186         ptypes = (seqs == null || seqs.length == 0) ? null : new CrossRef(
 187                 seqs, dataset).findXrefSourcesForSequences(dna);
 188         filterDbRefs(ptypes, limit);
 189
 190         // start of pass2: retrieve each cross-ref for fetched or restored
 191         // project.
 192         do // first cross ref and recover crossref loop
 193         {
 194
 195           for (String db : ptypes)
 196           {
 197             // counter for splitframe views retrieved via crossref
 198             int firstcr_ap = 0;
 199             // build next key so we an retrieve all views
 200             String nextxref = first + " -> " + db + "{" + firstcr_ap + "}";
 201             // perform crossref action, or retrieve stored project
 202             List<AlignmentViewPanel> cra_views = new ArrayList<>();
 203             CrossRefAction cra = null;
 204
 205             if (pass2 == 0)
 206             { // retrieve and show cross-refs in this thread
 207               cra = CrossRefAction.getHandlerFor(seqs, dna, db, af);
 208               cra.run();
 209               cra_views = (List<AlignmentViewPanel>) PA.getValue(cra,
 210                       "xrefViews");
 211               if (cra_views.size() == 0)
 212               {
 213                 failedXrefMenuItems.add("No crossrefs retrieved for "
 214                         + first + " -> " + db);
 215                 continue;
 216               }
 217               assertNucleotide(cra_views.get(0),
 218                       "Nucleotide panel included proteins for " + first
 219                               + " -> " + db);
 220               assertProtein(cra_views.get(1),
 221                       "Protein panel included nucleotides for " + first
 222                               + " -> " + db);
 223             }
 224             else
 225             {
 226               Desktop.instance.closeAll_actionPerformed(null);
 227               pass3 = 0;
 228               // recover stored project
 229               File storedProject = savedProjects.get(nextxref);
 230               if (storedProject == null)
 231               {
 232                 failedProjectRecoveries.add("Failed to store a view for '"
 233                         + nextxref + "'");
 234                 continue;
 235               }
 236
 237               // recover stored project
 238               AlignFrame af2 = new FileLoader(false)
 239                       .LoadFileWaitTillLoaded(savedProjects.get(nextxref)
 240                               .toString(), DataSourceType.FILE);
 241               System.out.println("Recovered view for '" + nextxref
 242                       + "' from '" + savedProjects.get(nextxref).toString()
 243                       + "'");
 244               // gymnastics to recover the alignPanel/Complementary alignPanel
 245               if (af2.getViewport().isNucleotide())
 246               {
 247                 // top view, then bottom
 248                 cra_views.add(af2.getViewport().getAlignPanel());
 249                 cra_views.add(((jalview.gui.AlignViewport) af2
 250                         .getViewport().getCodingComplement())
 251                         .getAlignPanel());
 252
 253               }
 254               else
 255               {
 256                 // bottom view, then top
 257                 cra_views.add(((jalview.gui.AlignViewport) af2
 258                         .getViewport().getCodingComplement())
 259                         .getAlignPanel());
 260                 cra_views.add(af2.getViewport().getAlignPanel());
 261
 262               }
 263             }
 264             HashMap<String, List<String>> xrptypes = new HashMap<>();
 265             // first save/verify views.
 266             for (AlignmentViewPanel avp : cra_views)
 267             {
 268               nextxref = first + " -> " + db + "{" + firstcr_ap++ + "}";
 269               // verify references for this panel
 270               AlignmentTest.assertAlignmentDatasetRefs(avp.getAlignment(),
 271                       "Pass (" + pass1 + "," + pass2 + "," + pass3
 272                               + "): before start of pass3: " + nextxref
 273                               + ":");
 274               assertDatasetIsNormalisedKnownDefect(avp.getAlignment(),
 275                       "Pass (" + pass1 + "," + pass2 + "," + pass3
 276                               + "): before start of pass3: " + nextxref
 277                               + ":");
 278
 279               SequenceI[] xrseqs = avp.getAlignment().getSequencesArray();
 280
 281               List<String> _xrptypes = (seqs == null || seqs.length == 0) ? null
 282                       : new CrossRef(xrseqs, dataset)
 283                               .findXrefSourcesForSequences(avp
 284                                       .getAlignViewport().isNucleotide());
 285
 286               stringify(dbtoviewBit, savedProjects, nextxref, avp);
 287               xrptypes.put(nextxref, _xrptypes);
 288
 289             }
 290
 291             // now do the second xref pass starting from either saved or just
 292             // recovered split pane, in sequence
 293             do // retrieve second set of cross refs or recover and verify
 294             {
 295               firstcr_ap = 0;
 296               for (AlignmentViewPanel avp : cra_views)
 297               {
 298                 nextxref = first + " -> " + db + "{" + firstcr_ap++ + "}";
 299                 for (String xrefdb : xrptypes.get(nextxref))
 300                 {
 301                   List<AlignmentViewPanel> cra_views2 = new ArrayList<>();
 302                   int q = 0;
 303                   String nextnextxref = nextxref + " -> " + xrefdb + "{"
 304                           + q + "}";
 305
 306                   if (pass3 == 0)
 307                   {
 308                     SequenceI[] xrseqs = avp.getAlignment()
 309                             .getSequencesArray();
 310                     AlignFrame nextaf = Desktop.getAlignFrameFor(avp
 311                             .getAlignViewport());
 312
 313                     cra = CrossRefAction.getHandlerFor(xrseqs, avp
 314                             .getAlignViewport().isNucleotide(), xrefdb,
 315                             nextaf);
 316                     cra.run();
 317                     cra_views2 = (List<AlignmentViewPanel>) PA.getValue(
 318                             cra, "xrefViews");
 319                     if (cra_views2.size() == 0)
 320                     {
 321                       failedXrefMenuItems
 322                               .add("No crossrefs retrieved for '"
 323                                       + nextxref + "' to " + xrefdb
 324                                       + " via '" + nextaf.getTitle() + "'");
 325                       continue;
 326                     }
 327                     assertNucleotide(cra_views2.get(0),
 328                             "Nucleotide panel included proteins for '"
 329                                     + nextxref + "' to " + xrefdb
 330                                     + " via '" + nextaf.getTitle() + "'");
 331                     assertProtein(cra_views2.get(1),
 332                             "Protein panel included nucleotides for '"
 333                                     + nextxref + "' to " + xrefdb
 334                                     + " via '" + nextaf.getTitle() + "'");
 335
 336                   }
 337                   else
 338                   {
 339                     Desktop.instance.closeAll_actionPerformed(null);
 340                     // recover stored project
 341                     File storedProject = savedProjects.get(nextnextxref);
 342                     if (storedProject == null)
 343                     {
 344                       failedProjectRecoveries
 345                               .add("Failed to store a view for '"
 346                                       + nextnextxref + "'");
 347                       continue;
 348                     }
 349                     AlignFrame af2 = new FileLoader(false)
 350                             .LoadFileWaitTillLoaded(
 351                                     savedProjects.get(nextnextxref)
 352                                             .toString(),
 353                                     DataSourceType.FILE);
 354                     System.out.println("Recovered view for '"
 355                             + nextnextxref + "' from '"
 356                             + savedProjects.get(nextnextxref).toString()
 357                             + "'");
 358                     // gymnastics to recover the alignPanel/Complementary
 359                     // alignPanel
 360                     if (af2.getViewport().isNucleotide())
 361                     {
 362                       // top view, then bottom
 363                       cra_views2.add(af2.getViewport().getAlignPanel());
 364                       cra_views2.add(((jalview.gui.AlignViewport) af2
 365                               .getViewport().getCodingComplement())
 366                               .getAlignPanel());
 367
 368                     }
 369                     else
 370                     {
 371                       // bottom view, then top
 372                       cra_views2.add(((jalview.gui.AlignViewport) af2
 373                               .getViewport().getCodingComplement())
 374                               .getAlignPanel());
 375                       cra_views2.add(af2.getViewport().getAlignPanel());
 376                     }
 377                     Assert.assertEquals(cra_views2.size(), 2);
 378                     Assert.assertNotNull(cra_views2.get(0));
 379                     Assert.assertNotNull(cra_views2.get(1));
 380                   }
 381
 382                   for (AlignmentViewPanel nextavp : cra_views2)
 383                   {
 384                     nextnextxref = nextxref + " -> " + xrefdb + "{" + q++
 385                             + "}";
 386
 387                     // verify references for this panel
 388                     AlignmentTest.assertAlignmentDatasetRefs(
 389                             nextavp.getAlignment(), "" + "Pass (" + pass1
 390                                     + "," + pass2 + "): For "
 391                                     + nextnextxref + ":");
 392                     assertDatasetIsNormalisedKnownDefect(
 393                             nextavp.getAlignment(), "" + "Pass (" + pass1
 394                                     + "," + pass2 + "): For "
 395                                     + nextnextxref + ":");
 396
 397                     stringify(dbtoviewBit, savedProjects, nextnextxref,
 398                             nextavp);
 399                     keyseq.add(nextnextxref);
 400                   }
 401                 } // end of loop around showing all xrefdb for crossrf2
 402
 403               } // end of loop around all viewpanels from crossrf1
 404             } while (pass2 == 2 && pass3++ < 2);
 405             // fetchdb->crossref1->crossref-2->verify for xrefs we
 406             // either loop twice when pass2=0, or just once when pass2=1
 407             // (recovered project from previous crossref)
 408
 409           } // end of loop over db-xrefs for crossref-2
 410
 411           // fetchdb-->crossref1
 412           // for each xref we try to retrieve xref, store and verify when
 413           // pass1=0, or just retrieve and verify when pass1=1
 414         } while (pass1 == 1 && pass2++ < 2);
 415         // fetchdb
 416         // for each ref we
 417         // loop twice: first, do the retrieve, second recover from saved project
 418
 419         // increment pass counters, so we repeat traversal starting from the
 420         // oldest saved project first.
 421         if (pass1 == 0)
 422         {
 423           // verify stored projects for first set of cross references
 424           pass1 = 1;
 425           // and verify cross-references retrieved from stored projects
 426           pass2 = 0;
 427           pass3 = 0;
 428         }
 429         else
 430         {
 431           pass1++;
 432         }
 433       } while (pass1 < 3);
 434
 435     if (failedXrefMenuItems.size() > 0)
 436     {
 437       for (String s : failedXrefMenuItems)
 438       {
 439         System.err.println(s);
 440       }
 441       Assert.fail("Faulty xref menu (" + failedXrefMenuItems.size()
 442               + " counts)");
 443     }
 444     if (failedProjectRecoveries.size() > 0)
 445     {
 446
 447       for (String s : failedProjectRecoveries)
 448       {
 449         System.err.println(s);
 450       }
 451       Assert.fail("Didn't recover projects for some retrievals (did they retrieve ?) ("
 452               + failedProjectRecoveries.size() + " counts)");
 453     }
 454     if (failedDBRetr.size() > 0)
 455     {
 456       for (String s : failedProjectRecoveries)
 457       {
 458         System.err.println(s);
 459       }
 460       Assert.fail("Didn't retrieve some db refs for checking cross-refs ("
 461               + failedDBRetr.size() + " counts)");
 462     }
 463   }
 464
 465   private void filterDbRefs(List<String> ptypes, List<String> limit)
 466   {
 467     if (limit != null)
 468     {
 469       int p = 0;
 470       while (ptypes.size() > p)
 471       {
 472         if (!limit.contains(ptypes.get(p)))
 473         {
 474           ptypes.remove(p);
 475         }
 476         else
 477         {
 478           p++;
 479         }
 480       }
 481     }
 482   }
 483
 484   /**
 485    * wrapper to trap known defect for AH002001 testcase
 486    *
 487    * @param alignment
 488    * @param string
 489    */
 490   private void assertDatasetIsNormalisedKnownDefect(AlignmentI al,
 491           String message)
 492   {
 493     try
 494     {
 495       AlignmentTest.assertDatasetIsNormalised(al, message);
 496     } catch (AssertionError ae)
 497     {
 498       if (!ae.getMessage().endsWith("EMBL|AH002001"))
 499       {
 500         throw ae;
 501       }
 502       else
 503       {
 504         System.out
 505                 .println("Ignored exception for known defect: JAL-2179 : "
 506                         + message);
 507       }
 508
 509     }
 510   }
 511
 512   private void assertProtein(AlignmentViewPanel alignmentViewPanel,
 513           String message)
 514   {
 515     assertType(true, alignmentViewPanel, message);
 516   }
 517
 518   private void assertNucleotide(AlignmentViewPanel alignmentViewPanel,
 519           String message)
 520   {
 521     assertType(false, alignmentViewPanel, message);
 522   }
 523
 524   private void assertType(boolean expectProtein,
 525           AlignmentViewPanel alignmentViewPanel, String message)
 526   {
 527     List<SequenceI> nonType = new ArrayList<>();
 528     for (SequenceI sq : alignmentViewPanel.getAlignViewport()
 529             .getAlignment().getSequences())
 530     {
 531       if (sq.isProtein() != expectProtein)
 532       {
 533         nonType.add(sq);
 534       }
 535     }
 536     if (nonType.size() > 0)
 537     {
 538       Assert.fail(message + " [ "
 539               + (expectProtein ? "nucleotides were " : "proteins were ")
 540               + nonType.toString() + " ]");
 541     }
 542   }
 543
 544   /**
 545    * first time called, record strings derived from alignment and
 546    * alignedcodonframes, and save view to a project file. Second time called,
 547    * compare strings to existing ones. org.testng.Assert.assertTrue on
 548    * stringmatch
 549    *
 550    * @param dbtoviewBit
 551    *          map between xrefpath and view string
 552    * @param savedProjects
 553    *          - map from xrefpath to saved project filename (createTempFile)
 554    * @param xrefpath
 555    *          - xrefpath - unique ID for this context (composed of sequence of
 556    *          db-fetch/cross-ref actions preceeding state)
 557    * @param avp
 558    *          - viewpanel to store (for viewpanels in splitframe, the same
 559    *          project should be written for both panels, only one needs
 560    *          recovering for comparison on the next stringify call, but each
 561    *          viewpanel needs to be called with a distinct xrefpath to ensure
 562    *          each one's strings are compared)
 563    */
 564   private void stringify(Map<String, String> dbtoviewBit,
 565           Map<String, File> savedProjects, String xrefpath,
 566           AlignmentViewPanel avp)
 567   {
 568     if (savedProjects != null)
 569     {
 570       if (savedProjects.get(xrefpath) == null)
 571       {
 572         // write a project file for this view. On the second pass, this will be
 573         // recovered and cross-references verified
 574         try
 575         {
 576           File prfile = File.createTempFile("crossRefTest", ".jvp");
 577           AlignFrame af = Desktop.getAlignFrameFor(avp.getAlignViewport());
 578           new Jalview2XML(false).saveAlignment(af, prfile.toString(),
 579                   af.getTitle());
 580           System.out.println("Written view from '" + xrefpath + "' as '"
 581                   + prfile.getAbsolutePath() + "'");
 582           savedProjects.put(xrefpath, prfile);
 583         } catch (IOException q)
 584         {
 585           Assert.fail("Unexpected IO Exception", q);
 586         }
 587       }
 588       else
 589       {
 590         System.out.println("Stringify check on view from '" + xrefpath
 591                 + "' [ possibly retrieved from '"
 592                 + savedProjects.get(xrefpath).getAbsolutePath() + "' ]");
 593
 594       }
 595     }
 596
 597     StringBuilder sbr = new StringBuilder();
 598     sbr.append(avp.getAlignment().toString());
 599     sbr.append("\n");
 600     sbr.append("<End of alignment>");
 601     sbr.append("\n");
 602     sbr.append(avp.getAlignment().getDataset());
 603     sbr.append("\n");
 604     sbr.append("<End of dataset>");
 605     sbr.append("\n");
 606     int p = 0;
 607     if (avp.getAlignment().getCodonFrames() != null)
 608     {
 609       for (AlignedCodonFrame ac : avp.getAlignment().getCodonFrames())
 610       {
 611         sbr.append("<AlignedCodonFrame " + p++ + ">");
 612         sbr.append("\n");
 613         sbr.append(ac.toString());
 614         sbr.append("\n");
 615       }
 616     }
 617     String dbt = dbtoviewBit.get(xrefpath);
 618     if (dbt == null)
 619     {
 620       dbtoviewBit.put(xrefpath, sbr.toString());
 621     }
 622     else
 623     {
 624       Assert.assertEquals(sbr.toString(), dbt, "stringify mismatch for "
 625               + xrefpath);
 626     }
 627   }
 628 }