JAL-1988 JAL-3130 New jlinked JRE in jalview_java2 with modules. Still needs jaxws...
[jalview.git] / test / jalview / io / CrossRef2xmlTests.java
1 /*
2  * Jalview - A Sequence Alignment Editor and Viewer ($$Version-Rel$$)
3  * Copyright (C) $$Year-Rel$$ The Jalview Authors
4  * 
5  * This file is part of Jalview.
6  * 
7  * Jalview is free software: you can redistribute it and/or
8  * modify it under the terms of the GNU General Public License 
9  * as published by the Free Software Foundation, either version 3
10  * of the License, or (at your option) any later version.
11  *  
12  * Jalview is distributed in the hope that it will be useful, but 
13  * WITHOUT ANY WARRANTY; without even the implied warranty 
14  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR 
15  * PURPOSE.  See the GNU General Public License for more details.
16  * 
17  * You should have received a copy of the GNU General Public License
18  * along with Jalview.  If not, see <http://www.gnu.org/licenses/>.
19  * The Jalview Authors are detailed in the 'AUTHORS' file.
20  */
21 package jalview.io;
22
23 import jalview.analysis.CrossRef;
24 import jalview.api.AlignmentViewPanel;
25 import jalview.datamodel.AlignedCodonFrame;
26 import jalview.datamodel.AlignmentI;
27 import jalview.datamodel.AlignmentTest;
28 import jalview.datamodel.SequenceI;
29 import jalview.gui.AlignFrame;
30 import jalview.gui.CrossRefAction;
31 import jalview.gui.Desktop;
32 import jalview.gui.Jalview2XML;
33 import jalview.gui.JvOptionPane;
34 import jalview.util.DBRefUtils;
35
36 import java.io.File;
37 import java.io.IOException;
38 import java.util.ArrayList;
39 import java.util.Arrays;
40 import java.util.HashMap;
41 import java.util.List;
42 import java.util.Map;
43
44 import junit.extensions.PA;
45
46 import org.testng.Assert;
47 import org.testng.annotations.BeforeClass;
48 import org.testng.annotations.DataProvider;
49 import org.testng.annotations.Test;
50
51 @Test(singleThreaded = true)
52 public class CrossRef2xmlTests extends Jalview2xmlBase
53 {
54
55   @Override
56   @BeforeClass(alwaysRun = true)
57   public void setUpJvOptionPane()
58   {
59     JvOptionPane.setInteractiveMode(false);
60     JvOptionPane.setMockResponse(JvOptionPane.CANCEL_OPTION);
61   }
62
63   @DataProvider(name = "initialAccessions")
64   static Object[][] getAccessions()
65   {
66     return new String[][] { { "UNIPROT", "P00338" },
67         { "UNIPROT", "Q8Z9G6" },
68         { "ENSEMBLGENOMES", "CAD01290" } };
69   }
70
71   /**
72    * test store and recovery of all reachable cross refs from all reachable
73    * crossrefs for one or more fetched db refs. Currently, this test has a known
74    * failure case.
75    * 
76    * @throws Exception
77    */
78   @Test(
79     groups =
80     { "Operational" },
81     dataProvider = "initialAccessions",
82     enabled = true)
83   public void testRetrieveAndShowCrossref(String forSource,
84           String forAccession) throws Exception
85   {
86
87     List<String> failedDBRetr = new ArrayList<>();
88     List<String> failedXrefMenuItems = new ArrayList<>();
89     List<String> failedProjectRecoveries = new ArrayList<>();
90     // only search for ensembl or Uniprot crossrefs
91     List<String> limit=Arrays.asList(new String[] {
92         DBRefUtils.getCanonicalName("ENSEMBL"), 
93         DBRefUtils.getCanonicalName("Uniprot")});
94     // for every set of db queries
95     // retrieve db query
96     // verify presence of expected xrefs
97     // show xrefs - verify expected type of frame is shown for each xref
98     // show xrefs again
99     // - verify original -> xref -> xref(original) recovers frame containing at
100     // least the first retrieved sequence
101     // store
102     // 1. whole project
103     // 2. individual frames
104     // 3. load each one back and verify
105     // . aligned sequences (.toString() )
106     // . xrefs (.toString() )
107     // . codonframes
108     //
109     //
110     Map<String, String> dbtoviewBit = new HashMap<>();
111     List<String> keyseq = new ArrayList<>();
112     Map<String, File> savedProjects = new HashMap<>();
113
114 //    for (String[] did : new String[][] { { "UNIPROT", "P00338" } })
115 //    {
116       // pass counters - 0 - first pass, 1 means retrieve project rather than
117       // perform action
118       int pass1 = 0, pass2 = 0, pass3 = 0;
119       // each do loop performs two iterations in the first outer loop pass, but
120       // only performs one iteration on the second outer loop
121       // ie. pass 1 = 0 {pass 2= 0 { pass 3 = 0,1 }, pass 2=1 { pass 3 = 0 }}, 1
122       // { pass 2 = 0 { pass 3 = 0 } }
123       do
124       {
125         String first = forSource + " " + forAccession;//did[0] + " " + did[1];
126         AlignFrame af = null;
127         boolean dna;
128         AlignmentI retral;
129         AlignmentI dataset;
130         SequenceI[] seqs;
131         List<String> ptypes = null;
132         if (pass1 == 0)
133         {
134           // retrieve dbref
135
136           List<AlignFrame> afs = jalview.gui.SequenceFetcher.fetchAndShow(
137                 forSource, forAccession);
138         // did[0], did[1]);
139           if (afs.size() == 0)
140           {
141             failedDBRetr.add("Didn't retrieve " + first);
142             break;
143           }
144           keyseq.add(first);
145           af = afs.get(0);
146
147           // verify references for retrieved data
148           AlignmentTest.assertAlignmentDatasetRefs(af.getViewport()
149                   .getAlignment(), "Pass (" + pass1 + "," + pass2 + ","
150                   + pass3 + "): Fetch " + first + ":");
151           assertDatasetIsNormalisedKnownDefect(af.getViewport()
152                   .getAlignment(), "Pass (" + pass1 + "," + pass2 + ","
153                   + pass3 + "): Fetch " + first + ":");
154           dna = af.getViewport().getAlignment().isNucleotide();
155           retral = af.getViewport().getAlignment();
156           dataset = retral.getDataset();
157           seqs = retral.getSequencesArray();
158
159         }
160         else
161         {
162           Desktop.instance.closeAll_actionPerformed(null);
163           // recover stored project
164           af = new FileLoader(false).LoadFileWaitTillLoaded(savedProjects
165                   .get(first).toString(), DataSourceType.FILE);
166           System.out.println("Recovered view for '" + first + "' from '"
167                   + savedProjects.get(first).toString() + "'");
168           dna = af.getViewport().getAlignment().isNucleotide();
169           retral = af.getViewport().getAlignment();
170           dataset = retral.getDataset();
171           seqs = retral.getSequencesArray();
172
173           // verify references for recovered data
174           AlignmentTest.assertAlignmentDatasetRefs(af.getViewport()
175                   .getAlignment(), "Pass (" + pass1 + "," + pass2 + ","
176                   + pass3 + "): Recover " + first + ":");
177           assertDatasetIsNormalisedKnownDefect(af.getViewport()
178                   .getAlignment(), "Pass (" + pass1 + "," + pass2 + ","
179                   + pass3 + "): Recover " + first + ":");
180
181         }
182
183         // store project on first pass, compare next pass
184         stringify(dbtoviewBit, savedProjects, first, af.alignPanel);
185
186         ptypes = (seqs == null || seqs.length == 0) ? null : new CrossRef(
187                 seqs, dataset).findXrefSourcesForSequences(dna);
188         filterDbRefs(ptypes, limit);
189         
190         // start of pass2: retrieve each cross-ref for fetched or restored
191         // project.
192         do // first cross ref and recover crossref loop
193         {
194
195           for (String db : ptypes)
196           {
197             // counter for splitframe views retrieved via crossref
198             int firstcr_ap = 0;
199             // build next key so we an retrieve all views
200             String nextxref = first + " -> " + db + "{" + firstcr_ap + "}";
201             // perform crossref action, or retrieve stored project
202             List<AlignmentViewPanel> cra_views = new ArrayList<>();
203             CrossRefAction cra = null;
204
205             if (pass2 == 0)
206             { // retrieve and show cross-refs in this thread
207               cra = CrossRefAction.getHandlerFor(seqs, dna, db, af);
208               cra.run();
209               cra_views = (List<AlignmentViewPanel>) PA.getValue(cra,
210                       "xrefViews");
211               if (cra_views.size() == 0)
212               {
213                 failedXrefMenuItems.add("No crossrefs retrieved for "
214                         + first + " -> " + db);
215                 continue;
216               }
217               assertNucleotide(cra_views.get(0),
218                       "Nucleotide panel included proteins for " + first
219                               + " -> " + db);
220               assertProtein(cra_views.get(1),
221                       "Protein panel included nucleotides for " + first
222                               + " -> " + db);
223             }
224             else
225             {
226               Desktop.instance.closeAll_actionPerformed(null);
227               pass3 = 0;
228               // recover stored project
229               File storedProject = savedProjects.get(nextxref);
230               if (storedProject == null)
231               {
232                 failedProjectRecoveries.add("Failed to store a view for '"
233                         + nextxref + "'");
234                 continue;
235               }
236
237               // recover stored project
238               AlignFrame af2 = new FileLoader(false)
239                       .LoadFileWaitTillLoaded(savedProjects.get(nextxref)
240                               .toString(), DataSourceType.FILE);
241               System.out.println("Recovered view for '" + nextxref
242                       + "' from '" + savedProjects.get(nextxref).toString()
243                       + "'");
244               // gymnastics to recover the alignPanel/Complementary alignPanel
245               if (af2.getViewport().isNucleotide())
246               {
247                 // top view, then bottom
248                 cra_views.add(af2.getViewport().getAlignPanel());
249                 cra_views.add(((jalview.gui.AlignViewport) af2
250                         .getViewport().getCodingComplement())
251                         .getAlignPanel());
252
253               }
254               else
255               {
256                 // bottom view, then top
257                 cra_views.add(((jalview.gui.AlignViewport) af2
258                         .getViewport().getCodingComplement())
259                         .getAlignPanel());
260                 cra_views.add(af2.getViewport().getAlignPanel());
261
262               }
263             }
264             HashMap<String, List<String>> xrptypes = new HashMap<>();
265             // first save/verify views.
266             for (AlignmentViewPanel avp : cra_views)
267             {
268               nextxref = first + " -> " + db + "{" + firstcr_ap++ + "}";
269               // verify references for this panel
270               AlignmentTest.assertAlignmentDatasetRefs(avp.getAlignment(),
271                       "Pass (" + pass1 + "," + pass2 + "," + pass3
272                               + "): before start of pass3: " + nextxref
273                               + ":");
274               assertDatasetIsNormalisedKnownDefect(avp.getAlignment(),
275                       "Pass (" + pass1 + "," + pass2 + "," + pass3
276                               + "): before start of pass3: " + nextxref
277                               + ":");
278
279               SequenceI[] xrseqs = avp.getAlignment().getSequencesArray();
280
281               List<String> _xrptypes = (seqs == null || seqs.length == 0) ? null
282                       : new CrossRef(xrseqs, dataset)
283                               .findXrefSourcesForSequences(avp
284                                       .getAlignViewport().isNucleotide());
285
286               stringify(dbtoviewBit, savedProjects, nextxref, avp);
287               xrptypes.put(nextxref, _xrptypes);
288
289             }
290
291             // now do the second xref pass starting from either saved or just
292             // recovered split pane, in sequence
293             do // retrieve second set of cross refs or recover and verify
294             {
295               firstcr_ap = 0;
296               for (AlignmentViewPanel avp : cra_views)
297               {
298                 nextxref = first + " -> " + db + "{" + firstcr_ap++ + "}";
299                 for (String xrefdb : xrptypes.get(nextxref))
300                 {
301                   List<AlignmentViewPanel> cra_views2 = new ArrayList<>();
302                   int q = 0;
303                   String nextnextxref = nextxref + " -> " + xrefdb + "{"
304                           + q + "}";
305
306                   if (pass3 == 0)
307                   {
308                     SequenceI[] xrseqs = avp.getAlignment()
309                             .getSequencesArray();
310                     AlignFrame nextaf = Desktop.getAlignFrameFor(avp
311                             .getAlignViewport());
312
313                     cra = CrossRefAction.getHandlerFor(xrseqs, avp
314                             .getAlignViewport().isNucleotide(), xrefdb,
315                             nextaf);
316                     cra.run();
317                     cra_views2 = (List<AlignmentViewPanel>) PA.getValue(
318                             cra, "xrefViews");
319                     if (cra_views2.size() == 0)
320                     {
321                       failedXrefMenuItems
322                               .add("No crossrefs retrieved for '"
323                                       + nextxref + "' to " + xrefdb
324                                       + " via '" + nextaf.getTitle() + "'");
325                       continue;
326                     }
327                     assertNucleotide(cra_views2.get(0),
328                             "Nucleotide panel included proteins for '"
329                                     + nextxref + "' to " + xrefdb
330                                     + " via '" + nextaf.getTitle() + "'");
331                     assertProtein(cra_views2.get(1),
332                             "Protein panel included nucleotides for '"
333                                     + nextxref + "' to " + xrefdb
334                                     + " via '" + nextaf.getTitle() + "'");
335
336                   }
337                   else
338                   {
339                     Desktop.instance.closeAll_actionPerformed(null);
340                     // recover stored project
341                     File storedProject = savedProjects.get(nextnextxref);
342                     if (storedProject == null)
343                     {
344                       failedProjectRecoveries
345                               .add("Failed to store a view for '"
346                                       + nextnextxref + "'");
347                       continue;
348                     }
349                     AlignFrame af2 = new FileLoader(false)
350                             .LoadFileWaitTillLoaded(
351                                     savedProjects.get(nextnextxref)
352                                             .toString(),
353                                     DataSourceType.FILE);
354                     System.out.println("Recovered view for '"
355                             + nextnextxref + "' from '"
356                             + savedProjects.get(nextnextxref).toString()
357                             + "'");
358                     // gymnastics to recover the alignPanel/Complementary
359                     // alignPanel
360                     if (af2.getViewport().isNucleotide())
361                     {
362                       // top view, then bottom
363                       cra_views2.add(af2.getViewport().getAlignPanel());
364                       cra_views2.add(((jalview.gui.AlignViewport) af2
365                               .getViewport().getCodingComplement())
366                               .getAlignPanel());
367
368                     }
369                     else
370                     {
371                       // bottom view, then top
372                       cra_views2.add(((jalview.gui.AlignViewport) af2
373                               .getViewport().getCodingComplement())
374                               .getAlignPanel());
375                       cra_views2.add(af2.getViewport().getAlignPanel());
376                     }
377                     Assert.assertEquals(cra_views2.size(), 2);
378                     Assert.assertNotNull(cra_views2.get(0));
379                     Assert.assertNotNull(cra_views2.get(1));
380                   }
381
382                   for (AlignmentViewPanel nextavp : cra_views2)
383                   {
384                     nextnextxref = nextxref + " -> " + xrefdb + "{" + q++
385                             + "}";
386
387                     // verify references for this panel
388                     AlignmentTest.assertAlignmentDatasetRefs(
389                             nextavp.getAlignment(), "" + "Pass (" + pass1
390                                     + "," + pass2 + "): For "
391                                     + nextnextxref + ":");
392                     assertDatasetIsNormalisedKnownDefect(
393                             nextavp.getAlignment(), "" + "Pass (" + pass1
394                                     + "," + pass2 + "): For "
395                                     + nextnextxref + ":");
396
397                     stringify(dbtoviewBit, savedProjects, nextnextxref,
398                             nextavp);
399                     keyseq.add(nextnextxref);
400                   }
401                 } // end of loop around showing all xrefdb for crossrf2
402
403               } // end of loop around all viewpanels from crossrf1
404             } while (pass2 == 2 && pass3++ < 2);
405             // fetchdb->crossref1->crossref-2->verify for xrefs we
406             // either loop twice when pass2=0, or just once when pass2=1
407             // (recovered project from previous crossref)
408
409           } // end of loop over db-xrefs for crossref-2
410
411           // fetchdb-->crossref1
412           // for each xref we try to retrieve xref, store and verify when
413           // pass1=0, or just retrieve and verify when pass1=1
414         } while (pass1 == 1 && pass2++ < 2);
415         // fetchdb
416         // for each ref we
417         // loop twice: first, do the retrieve, second recover from saved project
418
419         // increment pass counters, so we repeat traversal starting from the
420         // oldest saved project first.
421         if (pass1 == 0)
422         {
423           // verify stored projects for first set of cross references
424           pass1 = 1;
425           // and verify cross-references retrieved from stored projects
426           pass2 = 0;
427           pass3 = 0;
428         }
429         else
430         {
431           pass1++;
432         }
433       } while (pass1 < 3);
434
435     if (failedXrefMenuItems.size() > 0)
436     {
437       for (String s : failedXrefMenuItems)
438       {
439         System.err.println(s);
440       }
441       Assert.fail("Faulty xref menu (" + failedXrefMenuItems.size()
442               + " counts)");
443     }
444     if (failedProjectRecoveries.size() > 0)
445     {
446
447       for (String s : failedProjectRecoveries)
448       {
449         System.err.println(s);
450       }
451       Assert.fail("Didn't recover projects for some retrievals (did they retrieve ?) ("
452               + failedProjectRecoveries.size() + " counts)");
453     }
454     if (failedDBRetr.size() > 0)
455     {
456       for (String s : failedProjectRecoveries)
457       {
458         System.err.println(s);
459       }
460       Assert.fail("Didn't retrieve some db refs for checking cross-refs ("
461               + failedDBRetr.size() + " counts)");
462     }
463   }
464
465   private void filterDbRefs(List<String> ptypes, List<String> limit)
466   {
467     if (limit != null)
468     {
469       int p = 0;
470       while (ptypes.size() > p)
471       {
472         if (!limit.contains(ptypes.get(p)))
473         {
474           ptypes.remove(p);
475         }
476         else
477         {
478           p++;
479         }
480       }
481     }
482   }
483
484   /**
485    * wrapper to trap known defect for AH002001 testcase
486    * 
487    * @param alignment
488    * @param string
489    */
490   private void assertDatasetIsNormalisedKnownDefect(AlignmentI al,
491           String message)
492   {
493     try
494     {
495       AlignmentTest.assertDatasetIsNormalised(al, message);
496     } catch (AssertionError ae)
497     {
498       if (!ae.getMessage().endsWith("EMBL|AH002001"))
499       {
500         throw ae;
501       }
502       else
503       {
504         System.out
505                 .println("Ignored exception for known defect: JAL-2179 : "
506                         + message);
507       }
508
509     }
510   }
511
512   private void assertProtein(AlignmentViewPanel alignmentViewPanel,
513           String message)
514   {
515     assertType(true, alignmentViewPanel, message);
516   }
517
518   private void assertNucleotide(AlignmentViewPanel alignmentViewPanel,
519           String message)
520   {
521     assertType(false, alignmentViewPanel, message);
522   }
523
524   private void assertType(boolean expectProtein,
525           AlignmentViewPanel alignmentViewPanel, String message)
526   {
527     List<SequenceI> nonType = new ArrayList<>();
528     for (SequenceI sq : alignmentViewPanel.getAlignViewport()
529             .getAlignment().getSequences())
530     {
531       if (sq.isProtein() != expectProtein)
532       {
533         nonType.add(sq);
534       }
535     }
536     if (nonType.size() > 0)
537     {
538       Assert.fail(message + " [ "
539               + (expectProtein ? "nucleotides were " : "proteins were ")
540               + nonType.toString() + " ]");
541     }
542   }
543
544   /**
545    * first time called, record strings derived from alignment and
546    * alignedcodonframes, and save view to a project file. Second time called,
547    * compare strings to existing ones. org.testng.Assert.assertTrue on
548    * stringmatch
549    * 
550    * @param dbtoviewBit
551    *          map between xrefpath and view string
552    * @param savedProjects
553    *          - map from xrefpath to saved project filename (createTempFile)
554    * @param xrefpath
555    *          - xrefpath - unique ID for this context (composed of sequence of
556    *          db-fetch/cross-ref actions preceeding state)
557    * @param avp
558    *          - viewpanel to store (for viewpanels in splitframe, the same
559    *          project should be written for both panels, only one needs
560    *          recovering for comparison on the next stringify call, but each
561    *          viewpanel needs to be called with a distinct xrefpath to ensure
562    *          each one's strings are compared)
563    */
564   private void stringify(Map<String, String> dbtoviewBit,
565           Map<String, File> savedProjects, String xrefpath,
566           AlignmentViewPanel avp)
567   {
568     if (savedProjects != null)
569     {
570       if (savedProjects.get(xrefpath) == null)
571       {
572         // write a project file for this view. On the second pass, this will be
573         // recovered and cross-references verified
574         try
575         {
576           File prfile = File.createTempFile("crossRefTest", ".jvp");
577           AlignFrame af = Desktop.getAlignFrameFor(avp.getAlignViewport());
578           new Jalview2XML(false).saveAlignment(af, prfile.toString(),
579                   af.getTitle());
580           System.out.println("Written view from '" + xrefpath + "' as '"
581                   + prfile.getAbsolutePath() + "'");
582           savedProjects.put(xrefpath, prfile);
583         } catch (IOException q)
584         {
585           Assert.fail("Unexpected IO Exception", q);
586         }
587       }
588       else
589       {
590         System.out.println("Stringify check on view from '" + xrefpath
591                 + "' [ possibly retrieved from '"
592                 + savedProjects.get(xrefpath).getAbsolutePath() + "' ]");
593
594       }
595     }
596
597     StringBuilder sbr = new StringBuilder();
598     sbr.append(avp.getAlignment().toString());
599     sbr.append("\n");
600     sbr.append("<End of alignment>");
601     sbr.append("\n");
602     sbr.append(avp.getAlignment().getDataset());
603     sbr.append("\n");
604     sbr.append("<End of dataset>");
605     sbr.append("\n");
606     int p = 0;
607     if (avp.getAlignment().getCodonFrames() != null)
608     {
609       for (AlignedCodonFrame ac : avp.getAlignment().getCodonFrames())
610       {
611         sbr.append("<AlignedCodonFrame " + p++ + ">");
612         sbr.append("\n");
613         sbr.append(ac.toString());
614         sbr.append("\n");
615       }
616     }
617     String dbt = dbtoviewBit.get(xrefpath);
618     if (dbt == null)
619     {
620       dbtoviewBit.put(xrefpath, sbr.toString());
621     }
622     else
623     {
624       Assert.assertEquals(sbr.toString(), dbt, "stringify mismatch for "
625               + xrefpath);
626     }
627   }
628 }