Fix core WST file
[jabaws.git] / testsrc / compbio / runner / msa / ClustalWTester.java
1 /* Copyright (c) 2009 Peter Troshin\r
2  *  \r
3  *  JAva Bioinformatics Analysis Web Services (JABAWS) @version: 1.0     \r
4  * \r
5  *  This library is free software; you can redistribute it and/or modify it under the terms of the\r
6  *  Apache License version 2 as published by the Apache Software Foundation\r
7  * \r
8  *  This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without\r
9  *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Apache \r
10  *  License for more details.\r
11  * \r
12  *  A copy of the license is in apache_license.txt. It is also available here:\r
13  * @see: http://www.apache.org/licenses/LICENSE-2.0.txt\r
14  * \r
15  * Any republication or derived work distributed in source code form\r
16  * must include this copyright and license notice.\r
17  */\r
18 \r
19 package compbio.runner.msa;\r
20 \r
21 import static org.testng.Assert.assertEquals;\r
22 import static org.testng.Assert.assertFalse;\r
23 import static org.testng.Assert.assertNotNull;\r
24 import static org.testng.Assert.assertTrue;\r
25 import static org.testng.Assert.fail;\r
26 \r
27 import java.io.File;\r
28 import java.io.FileInputStream;\r
29 import java.io.FileNotFoundException;\r
30 import java.io.IOException;\r
31 import java.text.ParseException;\r
32 import java.util.ArrayList;\r
33 import java.util.Collections;\r
34 import java.util.List;\r
35 \r
36 import javax.xml.bind.JAXBException;\r
37 import javax.xml.bind.ValidationException;\r
38 \r
39 import org.ggf.drmaa.DrmaaException;\r
40 import org.ggf.drmaa.JobInfo;\r
41 import org.testng.annotations.Test;\r
42 \r
43 import compbio.data.sequence.FastaSequence;\r
44 import compbio.engine.AsyncExecutor;\r
45 import compbio.engine.Configurator;\r
46 import compbio.engine.FilePuller;\r
47 import compbio.engine.SyncExecutor;\r
48 import compbio.engine.client.ConfExecutable;\r
49 import compbio.engine.client.ConfiguredExecutable;\r
50 import compbio.engine.client.Executable;\r
51 import compbio.engine.client.RunConfiguration;\r
52 import compbio.engine.client.Executable.ExecProvider;\r
53 import compbio.engine.cluster.drmaa.ClusterUtil;\r
54 import compbio.engine.cluster.drmaa.JobRunner;\r
55 import compbio.engine.cluster.drmaa.StatisticManager;\r
56 import compbio.engine.conf.RunnerConfigMarshaller;\r
57 import compbio.engine.local.AsyncLocalRunner;\r
58 import compbio.engine.local.LocalExecutorService;\r
59 import compbio.engine.local.LocalRunner;\r
60 import compbio.metadata.AllTestSuit;\r
61 import compbio.metadata.ChunkHolder;\r
62 import compbio.metadata.JobExecutionException;\r
63 import compbio.metadata.JobStatus;\r
64 import compbio.metadata.JobSubmissionException;\r
65 import compbio.metadata.LimitsManager;\r
66 import compbio.metadata.PresetManager;\r
67 import compbio.metadata.ResultNotAvailableException;\r
68 import compbio.metadata.RunnerConfig;\r
69 import compbio.runner.OptionCombinator;\r
70 import compbio.runner.Util;\r
71 import compbio.runner.msa.ClustalW;\r
72 import compbio.util.FileWatcher;\r
73 import compbio.util.SysPrefs;\r
74 \r
75 public class ClustalWTester {\r
76 \r
77         static final String clustalConfigFile = AllTestSuit.TEST_DATA_PATH\r
78                         + "ClustalParameters.xml";\r
79         public static String test_outfile = "TO1381.clustal.out";\r
80         public static String cluster_test_outfile = "TO1381.clustal.cluster.out";\r
81 \r
82         @Test(groups = { AllTestSuit.test_group_cluster,\r
83                         AllTestSuit.test_group_runner })\r
84         public void testRunOnCluster() {\r
85                 ClustalW clustal = new ClustalW();\r
86                 assertFalse(SysPrefs.isWindows,\r
87                                 "Cluster execution can only be in unix environment");\r
88                 clustal.setInput(AllTestSuit.test_input)\r
89                                 .setOutput(cluster_test_outfile);\r
90 \r
91                 try {\r
92 \r
93                         ConfiguredExecutable<ClustalW> confClustal = Configurator\r
94                                         .configureExecutable(clustal);\r
95                         JobRunner runner = JobRunner.getInstance(confClustal);\r
96                         // ClusterSession csession = JobRunner.getSession();\r
97                         assertNotNull(runner);\r
98                         runner.executeJob();\r
99                         // assertNotNull("JobId is null", jobId1);\r
100                         JobStatus status = runner.getJobStatus();\r
101                         assertTrue(status == JobStatus.PENDING\r
102                                         || status == JobStatus.RUNNING);\r
103                         JobInfo info = runner.getJobInfo();\r
104                         assertNotNull(info);\r
105                         StatisticManager sm = new StatisticManager(info);\r
106                         assertNotNull(sm);\r
107                         try {\r
108                                 String exits = sm.getExitStatus();\r
109                                 assertNotNull("Exit status is null", exits);\r
110                                 // cut 4 trailing zeros from the number\r
111                                 int exitsInt = ClusterUtil.CLUSTER_STAT_IN_SEC.parse(exits)\r
112                                                 .intValue();\r
113                                 assertEquals(0, exitsInt);\r
114                                 System.out.println(sm.getAllStats());\r
115 \r
116                         } catch (ParseException e) {\r
117                                 e.printStackTrace();\r
118                                 fail("Parse Exception: " + e.getMessage());\r
119                         }\r
120                         // At present the task directory could not be completely removed\r
121                         // @see JobRunner.cleanup()\r
122                         assertFalse(runner.cleanup(),\r
123                                         "Could not remove some files whilst cleaning up ");\r
124                         assertTrue(sm.hasExited());\r
125                         assertFalse(sm.wasAborted());\r
126                         assertFalse(sm.hasDump());\r
127                         assertFalse(sm.hasSignaled());\r
128 \r
129                 } catch (JobSubmissionException e) {\r
130                         e.printStackTrace();\r
131                         fail("DrmaaException caught:" + e.getMessage());\r
132                 } catch (JobExecutionException e) {\r
133                         e.printStackTrace();\r
134                         fail("DrmaaException caught:" + e.getMessage());\r
135                 } catch (DrmaaException e) {\r
136                         e.printStackTrace();\r
137                         fail("DrmaaException caught:" + e.getMessage());\r
138                 }\r
139         }\r
140 \r
141         @Test(groups = { AllTestSuit.test_group_runner })\r
142         public void testRunLocally() {\r
143                 ClustalW clustal = new ClustalW();\r
144                 clustal.setInput(AllTestSuit.test_input).setOutput(test_outfile);\r
145                 try {\r
146 \r
147                         // For local execution use relavive\r
148                         ConfiguredExecutable<ClustalW> confClustal = Configurator\r
149                                         .configureExecutable(clustal, Executable.ExecProvider.Local);\r
150                         LocalRunner lr = new LocalRunner(confClustal);\r
151                         lr.executeJob();\r
152                         confClustal = (ConfiguredExecutable<ClustalW>) lr.waitForResult();\r
153                         assertNotNull(confClustal.getResults());\r
154                 } catch (JobSubmissionException e) {\r
155                         e.printStackTrace();\r
156                         fail(e.getLocalizedMessage());\r
157                 } catch (JobExecutionException e) {\r
158                         e.printStackTrace();\r
159                         fail(e.getLocalizedMessage());\r
160                 } catch (ResultNotAvailableException e) {\r
161                         e.printStackTrace();\r
162                         fail(e.getLocalizedMessage());\r
163                 }\r
164         }\r
165 \r
166         @Test(groups = { AllTestSuit.test_group_runner })\r
167         public void testRunWithMatrix() {\r
168                 ClustalW clustal = new ClustalW();\r
169                 clustal.setInput(AllTestSuit.test_input).setOutput(test_outfile);\r
170                 clustal.setParameter("-matrix=BLOSUM62");\r
171                 try {\r
172 \r
173                         // For local execution use relavive\r
174                         ConfiguredExecutable<ClustalW> confClustal = Configurator\r
175                                         .configureExecutable(clustal, Executable.ExecProvider.Local);\r
176                         LocalRunner lr = new LocalRunner(confClustal);\r
177                         lr.executeJob();\r
178                         confClustal = (ConfiguredExecutable<ClustalW>) lr.waitForResult();\r
179                         assertNotNull(confClustal.getResults());\r
180                 } catch (JobSubmissionException e) {\r
181                         e.printStackTrace();\r
182                         fail(e.getLocalizedMessage());\r
183                 } catch (JobExecutionException e) {\r
184                         e.printStackTrace();\r
185                         fail(e.getLocalizedMessage());\r
186                 } catch (ResultNotAvailableException e) {\r
187                         e.printStackTrace();\r
188                         fail(e.getLocalizedMessage());\r
189                 }\r
190         }\r
191 \r
192         @Test(groups = { AllTestSuit.test_group_runner })\r
193         public void testConfigurationLoading() {\r
194                 try {\r
195                         RunnerConfig<ClustalW> clustalConfig = ConfExecutable\r
196                                         .getRunnerOptions(ClustalW.class);\r
197                         assertNotNull(clustalConfig);\r
198                         assertTrue(clustalConfig.getArguments().size() > 0);\r
199 \r
200                         PresetManager<ClustalW> clustalPresets = ConfExecutable\r
201                                         .getRunnerPresets(ClustalW.class);\r
202                         assertNotNull(clustalPresets);\r
203                         assertTrue(clustalPresets.getPresets().size() > 0);\r
204                         clustalPresets.validate(clustalConfig);\r
205 \r
206                         LimitsManager<ClustalW> clustalLimits = ConfExecutable\r
207                                         .getRunnerLimits(ClustalW.class);\r
208                         assertNotNull(clustalLimits);\r
209                         assertTrue(clustalLimits.getLimits().size() > 0);\r
210                         clustalLimits.validate(clustalPresets);\r
211 \r
212                 } catch (FileNotFoundException e) {\r
213                         e.printStackTrace();\r
214                         fail(e.getLocalizedMessage());\r
215                 } catch (IOException e) {\r
216                         e.printStackTrace();\r
217                         fail(e.getLocalizedMessage());\r
218                 } catch (ValidationException e) {\r
219                         e.printStackTrace();\r
220                         fail(e.getLocalizedMessage());\r
221                 }\r
222         }\r
223 \r
224         @Test(groups = { AllTestSuit.test_group_runner })\r
225         public void testOptionsLocally() {\r
226                 try {\r
227 \r
228                         RunnerConfigMarshaller<ClustalW> clustalmarsh = new RunnerConfigMarshaller<ClustalW>(\r
229                                         RunnerConfig.class);\r
230 \r
231                         RunnerConfig<ClustalW> clustalConfig = clustalmarsh.read(\r
232                                         new FileInputStream(new File(clustalConfigFile)),\r
233                                         RunnerConfig.class);\r
234 \r
235                         OptionCombinator clustalOpc = new OptionCombinator(clustalConfig);\r
236                         List<String> options = clustalOpc.getOptionsAtRandom();\r
237                         for (int i = 0; i < options.size(); i++) {\r
238                                 System.out.println("Using options: " + options);\r
239                                 ClustalW clustal = new ClustalW();\r
240                                 clustal.setInput(AllTestSuit.test_input)\r
241                                                 .setOutput(test_outfile);\r
242 \r
243                                 // For local execution use relavive\r
244                                 ConfiguredExecutable<ClustalW> confClustal = Configurator\r
245                                                 .configureExecutable(clustal, ExecProvider.Local);\r
246 \r
247                                 // Add options to the executable\r
248                                 confClustal.addParameters(options);\r
249 \r
250                                 LocalRunner lr = new LocalRunner(confClustal);\r
251                                 lr.executeJob();\r
252                                 confClustal = (ConfiguredExecutable<ClustalW>) lr\r
253                                                 .waitForResult();\r
254                                 assertNotNull(confClustal.getResults());\r
255                                 Collections.shuffle(options);\r
256                         }\r
257 \r
258                 } catch (JobSubmissionException e) {\r
259                         e.printStackTrace();\r
260                         fail(e.getLocalizedMessage());\r
261                 } catch (JobExecutionException e) {\r
262                         e.printStackTrace();\r
263                         fail(e.getLocalizedMessage());\r
264                 } catch (JAXBException e) {\r
265                         e.printStackTrace();\r
266                         fail(e.getLocalizedMessage());\r
267                 } catch (ResultNotAvailableException e) {\r
268                         e.printStackTrace();\r
269                         fail(e.getLocalizedMessage());\r
270                 } catch (FileNotFoundException e) {\r
271                         e.printStackTrace();\r
272                         fail(e.getLocalizedMessage());\r
273                 }\r
274         }\r
275 \r
276         public static final void main(String[] args) throws JobSubmissionException,\r
277                         JobExecutionException, InterruptedException {\r
278                 ClustalW clustal = new ClustalW();\r
279                 clustal.setInput(AllTestSuit.test_input).setOutput(test_outfile);\r
280                 // For local execution use relavive\r
281                 ConfiguredExecutable<ClustalW> confClustal = Configurator\r
282                                 .configureExecutable(clustal);\r
283                 AsyncExecutor lr = new AsyncLocalRunner();\r
284                 lr.submitJob(confClustal);\r
285                 Thread.sleep(3000);\r
286                 LocalExecutorService.shutDown();\r
287         }\r
288 \r
289         @Test(enabled = false)\r
290         public void testAddParameters() {\r
291                 ArrayList<FastaSequence> seqs = new ArrayList<FastaSequence>();\r
292                 FastaSequence fs = new FastaSequence("tests1",\r
293                                 "aqtctcatcatctcatctgcccccgggttatgagtagtacgcatctacg");\r
294                 FastaSequence fs2 = new FastaSequence("tests2",\r
295                                 "aqtctcatcatctcatctgcccccgggttatgagtagtacgcatctacg");\r
296                 FastaSequence fs3 = new FastaSequence("tests3",\r
297                                 "aqtctcatcatctcatctgcccccgggttatgagtagtacgcatctacg");\r
298                 seqs.add(fs);\r
299                 seqs.add(fs2);\r
300                 seqs.add(fs3);\r
301                 ClustalW cl = new ClustalW();\r
302                 cl.setInput("input.txt").setOutput("output.txt");\r
303                 ConfiguredExecutable<ClustalW> confClustal;\r
304                 try {\r
305                         confClustal = Configurator.configureExecutable(cl);\r
306                         Util.writeInput(seqs, confClustal);\r
307 \r
308                         LocalRunner lr = new LocalRunner(confClustal);\r
309                         lr.executeJob();\r
310                         confClustal = (ConfiguredExecutable<ClustalW>) lr.waitForResult();\r
311                         assertNotNull(confClustal.getResults());\r
312 \r
313                         assertTrue(confClustal.saveRunConfiguration());\r
314                         ConfiguredExecutable<ClustalW> cexec = (ConfiguredExecutable<ClustalW>) confClustal\r
315                                         .loadRunConfiguration(new FileInputStream(new File(\r
316                                                         confClustal.getWorkDirectory(),\r
317                                                         RunConfiguration.rconfigFile)));\r
318                         assertNotNull(cexec);\r
319 \r
320                         lr = new LocalRunner(cexec);\r
321                         lr.executeJob();\r
322                         confClustal = (ConfiguredExecutable<ClustalW>) lr.waitForResult();\r
323                         assertNotNull(confClustal.getResults());\r
324 \r
325                         System.out.println("CE:" + cexec);\r
326                 } catch (JobSubmissionException e) {\r
327                         e.printStackTrace();\r
328                         fail(e.getMessage());\r
329                 } catch (FileNotFoundException e) {\r
330                         e.printStackTrace();\r
331                         fail(e.getMessage());\r
332                 } catch (JobExecutionException e) {\r
333                         e.printStackTrace();\r
334                         fail(e.getMessage());\r
335                 } catch (ResultNotAvailableException e) {\r
336                         e.printStackTrace();\r
337                         fail(e.getMessage());\r
338                 } catch (IOException e) {\r
339                         e.printStackTrace();\r
340                         fail(e.getMessage());\r
341                 }\r
342         }\r
343 \r
344         @Test(groups = { AllTestSuit.test_group_runner })\r
345         public void testPersistance() {\r
346                 try {\r
347                         ClustalW clustal = new ClustalW();\r
348                         clustal.setError("errrr.txt").setInput(AllTestSuit.test_input)\r
349                                         .setOutput("outtt.txt");\r
350                         assertEquals(clustal.getInput(), AllTestSuit.test_input);\r
351                         assertEquals(clustal.getError(), "errrr.txt");\r
352                         assertEquals(clustal.getOutput(), "outtt.txt");\r
353                         ConfiguredExecutable<ClustalW> cClustal = Configurator\r
354                                         .configureExecutable(clustal, Executable.ExecProvider.Local);\r
355 \r
356                         SyncExecutor sexec = Configurator.getSyncEngine(cClustal);\r
357                         sexec.executeJob();\r
358                         cClustal = (ConfiguredExecutable<ClustalW>) sexec.waitForResult();\r
359                         assertNotNull(cClustal.getResults());\r
360                         // Save run configuration\r
361                         assertTrue(cClustal.saveRunConfiguration());\r
362 \r
363                         // See if loaded configuration is the same as saved\r
364                         RunConfiguration loadedRun = RunConfiguration\r
365                                         .load(new FileInputStream(new File(cClustal\r
366                                                         .getWorkDirectory(), RunConfiguration.rconfigFile)));\r
367                         assertTrue(((ConfExecutable<ClustalW>) cClustal)\r
368                                         .getRunConfiguration().equals(loadedRun));\r
369                         // Load run configuration as ConfExecutable\r
370                         ConfiguredExecutable<ClustalW> resurrectedCclustal = (ConfiguredExecutable<ClustalW>) cClustal\r
371                                         .loadRunConfiguration(new FileInputStream(new File(cClustal\r
372                                                         .getWorkDirectory(), RunConfiguration.rconfigFile)));\r
373                         assertNotNull(resurrectedCclustal);\r
374                         // See in details whether executables are the same\r
375                         assertEquals(resurrectedCclustal.getExecutable(), clustal);\r
376 \r
377                         // Finally rerun the job in the new task directory\r
378                         ConfiguredExecutable<ClustalW> resclustal = Configurator\r
379                                         .configureExecutable(resurrectedCclustal.getExecutable(),\r
380                                                         Executable.ExecProvider.Local);\r
381 \r
382                         sexec = Configurator.getSyncEngine(resclustal,\r
383                                         Executable.ExecProvider.Local);\r
384                         sexec.executeJob();\r
385                         cClustal = (ConfiguredExecutable<ClustalW>) sexec.waitForResult();\r
386                         assertNotNull(cClustal.getResults());\r
387 \r
388                 } catch (JobSubmissionException e) {\r
389                         e.printStackTrace();\r
390                         fail(e.getMessage());\r
391                 } catch (JobExecutionException e) {\r
392                         e.printStackTrace();\r
393                         fail(e.getMessage());\r
394                 } catch (FileNotFoundException e) {\r
395                         e.printStackTrace();\r
396                         fail(e.getMessage());\r
397                 } catch (IOException e) {\r
398                         e.printStackTrace();\r
399                         fail(e.getMessage());\r
400                 } catch (ResultNotAvailableException e) {\r
401                         e.printStackTrace();\r
402                         fail(e.getMessage());\r
403                 }\r
404         }\r
405 \r
406         @Test(groups = { AllTestSuit.test_group_runner })\r
407         public void readStatistics() {\r
408                 try {\r
409                         ClustalW clustal = new ClustalW().setInput(AllTestSuit.test_input)\r
410                                         .setOutput(test_outfile);\r
411                         ConfiguredExecutable<ClustalW> confClustal = Configurator\r
412                                         .configureExecutable(clustal, Executable.ExecProvider.Local);\r
413 \r
414                         AsyncExecutor sexec = Configurator.getAsyncEngine(confClustal);\r
415                         String jobId = sexec.submitJob(confClustal);\r
416                         FilePuller fw = FilePuller.newFilePuller(confClustal\r
417                                         .getWorkDirectory()\r
418                                         + File.separator + ClustalW.getStatFile(),\r
419                                         FileWatcher.MIN_CHUNK_SIZE_BYTES);\r
420                         int count = 0;\r
421                         long position = 0;\r
422                         fw.waitForFile(4);\r
423                         while (!(sexec.getJobStatus(jobId) == JobStatus.FINISHED || sexec\r
424                                         .getJobStatus(jobId) == JobStatus.FAILED || sexec\r
425                                         .getJobStatus(jobId) == JobStatus.UNDEFINED)\r
426                                         || fw.hasMoreData()) {\r
427                                 ChunkHolder ch = fw.pull(position);\r
428                                 String chunk = ch.getChunk();\r
429                                 position = ch.getNextPosition();\r
430                                 System.out.print(chunk);\r
431                                 count++;\r
432                         }\r
433                         assertTrue(count > 1);\r
434                         ConfiguredExecutable<?> al = sexec.getResults(jobId);\r
435                         assertNotNull(al.getResults());\r
436                 } catch (JobSubmissionException e) {\r
437                         e.printStackTrace();\r
438                         fail(e.getMessage());\r
439                 } catch (ResultNotAvailableException e) {\r
440                         e.printStackTrace();\r
441                         fail(e.getMessage());\r
442                 } catch (IOException e) {\r
443                         e.printStackTrace();\r
444                         fail(e.getMessage());\r
445                 }\r
446         }\r
447 \r
448         @Test(groups = { AllTestSuit.test_group_cluster,\r
449                         AllTestSuit.test_group_runner })\r
450         public void readStatisticsClusterExecution() {\r
451                 try {\r
452                         ClustalW clustal = new ClustalW().setInput(AllTestSuit.test_input)\r
453                                         .setOutput(test_outfile);\r
454                         ConfiguredExecutable<ClustalW> confClustal = Configurator\r
455                                         .configureExecutable(clustal,\r
456                                                         Executable.ExecProvider.Cluster);\r
457 \r
458                         AsyncExecutor sexec = Configurator.getAsyncEngine(confClustal);\r
459                         String jobId = sexec.submitJob(confClustal);\r
460                         FilePuller fw = FilePuller.newFilePuller(confClustal\r
461                                         .getWorkDirectory()\r
462                                         + File.separator + ClustalW.getStatFile(),\r
463                                         FileWatcher.MIN_CHUNK_SIZE_BYTES);\r
464                         int count = 0;\r
465                         long position = 0;\r
466                         fw.waitForFile(200);\r
467                         /* Under certain circumstances DRMAA could report the status wrongly thus this loop never ends \r
468                          * TODO deal with this! \r
469                          * */\r
470                         while (!(sexec.getJobStatus(jobId) == JobStatus.FINISHED || sexec\r
471                                         .getJobStatus(jobId) == JobStatus.FAILED )\r
472                                         || fw.hasMoreData()) {\r
473                                 ChunkHolder ch = fw.pull(position);\r
474                                 String chunk = ch.getChunk();\r
475                                 position = ch.getNextPosition();\r
476                                 System.out.print(chunk);\r
477                                 count++;\r
478                                 if(sexec.getJobStatus(jobId) == JobStatus.UNDEFINED ) {\r
479                                         System.out.println("DRMAA reported wrong status for job + " + jobId +" continue anyway!");\r
480                                         break; \r
481                                 }\r
482                         }\r
483                         assertTrue(count > 1);\r
484                         ConfiguredExecutable<?> al = sexec.getResults(jobId);\r
485                         assertNotNull(al.getResults());\r
486                 } catch (JobSubmissionException e) {\r
487                         e.printStackTrace();\r
488                         fail(e.getMessage());\r
489                 } catch (ResultNotAvailableException e) {\r
490                         e.printStackTrace();\r
491                         fail(e.getMessage());\r
492                 } catch (IOException e) {\r
493                         e.printStackTrace();\r
494                         fail(e.getMessage());\r
495                 }\r
496         }\r
497 }\r