Refactoring (renaming) 2 classes: AsyncJobRunner.java -> AsyncClusterRunner.java...
[jabaws.git] / testsrc / compbio / runner / msa / ClustalWTester.java
1 /* Copyright (c) 2009 Peter Troshin\r
2  * Copyright (c) 2013 Alexander Sherstnev\r
3  *  \r
4  *  JAva Bioinformatics Analysis Web Services (JABAWS)\r
5  *  @version: 2.5     \r
6  * \r
7  *  This library is free software; you can redistribute it and/or modify it under the terms of the\r
8  *  Apache License version 2 as published by the Apache Software Foundation\r
9  * \r
10  *  This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without\r
11  *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Apache \r
12  *  License for more details.\r
13  * \r
14  *  A copy of the license is in apache_license.txt. It is also available here:\r
15  * @see: http://www.apache.org/licenses/LICENSE-2.0.txt\r
16  * \r
17  * Any republication or derived work distributed in source code form\r
18  * must include this copyright and license notice.\r
19  */\r
20 \r
21 package compbio.runner.msa;\r
22 \r
23 import static org.testng.Assert.assertEquals;\r
24 import static org.testng.Assert.assertFalse;\r
25 import static org.testng.Assert.assertNotNull;\r
26 import static org.testng.Assert.assertTrue;\r
27 import static org.testng.Assert.fail;\r
28 \r
29 import java.io.File;\r
30 import java.io.FileInputStream;\r
31 import java.io.FileNotFoundException;\r
32 import java.io.IOException;\r
33 import java.text.ParseException;\r
34 import java.util.ArrayList;\r
35 import java.util.Collections;\r
36 import java.util.List;\r
37 \r
38 import javax.xml.bind.JAXBException;\r
39 import javax.xml.bind.ValidationException;\r
40 \r
41 import org.ggf.drmaa.DrmaaException;\r
42 import org.ggf.drmaa.JobInfo;\r
43 import org.testng.annotations.Test;\r
44 \r
45 import compbio.data.sequence.FastaSequence;\r
46 import compbio.engine.AsyncExecutor;\r
47 import compbio.engine.Configurator;\r
48 import compbio.engine.FilePuller;\r
49 import compbio.engine.SyncExecutor;\r
50 import compbio.engine.client.ConfExecutable;\r
51 import compbio.engine.client.ConfiguredExecutable;\r
52 import compbio.engine.client.Executable;\r
53 import compbio.engine.client.RunConfiguration;\r
54 import compbio.engine.client.Executable.ExecProvider;\r
55 import compbio.engine.cluster.drmaa.ClusterEngineUtil;\r
56 import compbio.engine.cluster.drmaa.ClusterRunner;\r
57 import compbio.engine.cluster.drmaa.StatisticManager;\r
58 import compbio.engine.conf.RunnerConfigMarshaller;\r
59 import compbio.engine.local.AsyncLocalRunner;\r
60 import compbio.engine.local.LocalExecutorService;\r
61 import compbio.engine.local.LocalRunner;\r
62 import compbio.metadata.AllTestSuit;\r
63 import compbio.metadata.ChunkHolder;\r
64 import compbio.metadata.JobExecutionException;\r
65 import compbio.metadata.JobStatus;\r
66 import compbio.metadata.JobSubmissionException;\r
67 import compbio.metadata.LimitsManager;\r
68 import compbio.metadata.PresetManager;\r
69 import compbio.metadata.ResultNotAvailableException;\r
70 import compbio.metadata.RunnerConfig;\r
71 import compbio.runner.OptionCombinator;\r
72 import compbio.runner.RunnerUtil;\r
73 import compbio.runner.msa.ClustalW;\r
74 import compbio.util.FileWatcher;\r
75 import compbio.util.SysPrefs;\r
76 \r
77 public class ClustalWTester {\r
78 \r
79         static final String clustalConfigFile = AllTestSuit.TEST_DATA_PATH + "ClustalParameters.xml";\r
80         public static String test_outfile = "TO1381.clustal.out";\r
81         public static String cluster_test_outfile = "TO1381.clustal.cluster.out";\r
82 \r
83         @Test(groups = { AllTestSuit.test_group_runner })\r
84         public void RunLocally() {\r
85                 ClustalW clustal = new ClustalW();\r
86                 clustal.setInput(AllTestSuit.test_input).setOutput(test_outfile);\r
87                 try {\r
88 \r
89                         // For local execution use relavive\r
90                         ConfiguredExecutable<ClustalW> confClustal = Configurator.configureExecutable(clustal, Executable.ExecProvider.Local);\r
91                         LocalRunner lr = new LocalRunner(confClustal);\r
92                         lr.executeJob();\r
93                         confClustal = (ConfiguredExecutable<ClustalW>) lr.waitForResult();\r
94                         assertNotNull(confClustal.getResults());\r
95                 } catch (JobSubmissionException e) {\r
96                         e.printStackTrace();\r
97                         fail(e.getLocalizedMessage());\r
98                 } catch (JobExecutionException e) {\r
99                         e.printStackTrace();\r
100                         fail(e.getLocalizedMessage());\r
101                 } catch (ResultNotAvailableException e) {\r
102                         e.printStackTrace();\r
103                         fail(e.getLocalizedMessage());\r
104                 }\r
105         }\r
106 \r
107         @Test(groups = { AllTestSuit.test_group_runner })\r
108         public void RunWithMatrix() {\r
109                 ClustalW clustal = new ClustalW();\r
110                 clustal.setInput(AllTestSuit.test_input).setOutput(test_outfile);\r
111                 clustal.setParameter("-matrix=BLOSUM62");\r
112                 try {\r
113 \r
114                         // For local execution use relavive\r
115                         ConfiguredExecutable<ClustalW> confClustal = Configurator\r
116                                         .configureExecutable(clustal, Executable.ExecProvider.Local);\r
117                         LocalRunner lr = new LocalRunner(confClustal);\r
118                         lr.executeJob();\r
119                         confClustal = (ConfiguredExecutable<ClustalW>) lr.waitForResult();\r
120                         assertNotNull(confClustal.getResults());\r
121                 } catch (JobSubmissionException e) {\r
122                         e.printStackTrace();\r
123                         fail(e.getLocalizedMessage());\r
124                 } catch (JobExecutionException e) {\r
125                         e.printStackTrace();\r
126                         fail(e.getLocalizedMessage());\r
127                 } catch (ResultNotAvailableException e) {\r
128                         e.printStackTrace();\r
129                         fail(e.getLocalizedMessage());\r
130                 }\r
131         }\r
132 \r
133         @Test(groups = {AllTestSuit.test_group_runner})\r
134         public void ConfigurationLoading() {\r
135                 try {\r
136                         RunnerConfig<ClustalW> clustalConfig = ConfExecutable.getRunnerOptions(ClustalW.class);\r
137                         assertNotNull(clustalConfig);\r
138                         assertTrue(clustalConfig.getArguments().size() > 0);\r
139 \r
140                         PresetManager<ClustalW> clustalPresets = ConfExecutable.getRunnerPresets(ClustalW.class);\r
141                         assertNotNull(clustalPresets);\r
142                         assertTrue(clustalPresets.getPresets().size() > 0);\r
143                         clustalPresets.validate(clustalConfig);\r
144 \r
145                         LimitsManager<ClustalW> clustalLimits = ConfExecutable.getRunnerLimits(ClustalW.class);\r
146                         assertNotNull(clustalLimits);\r
147                         assertTrue(clustalLimits.getLimits().size() > 0);\r
148                         clustalLimits.validate(clustalPresets);\r
149                 } catch (FileNotFoundException e) {\r
150                         e.printStackTrace();\r
151                         fail(e.getLocalizedMessage());\r
152                 } catch (IOException e) {\r
153                         e.printStackTrace();\r
154                         fail(e.getLocalizedMessage());\r
155                 } catch (ValidationException e) {\r
156                         e.printStackTrace();\r
157                         fail(e.getLocalizedMessage());\r
158                 }\r
159         }\r
160 \r
161         @Test(groups = { AllTestSuit.test_group_runner })\r
162         public void OptionsLocally() {\r
163                 try {\r
164                         RunnerConfigMarshaller<ClustalW> clustalmarsh = new RunnerConfigMarshaller<ClustalW>(RunnerConfig.class);\r
165                         RunnerConfig<ClustalW> clustalConfig = clustalmarsh.read(new FileInputStream(new File(clustalConfigFile)),RunnerConfig.class);\r
166 \r
167                         OptionCombinator clustalOpc = new OptionCombinator(clustalConfig);\r
168                         List<String> options = clustalOpc.getOptionsAtRandom();\r
169                         for (int i = 0; i < options.size(); i++) {\r
170                                 System.out.println("Using options: " + options);\r
171                                 ClustalW clustal = new ClustalW();\r
172                                 clustal.setInput(AllTestSuit.test_input).setOutput(test_outfile);\r
173 \r
174                                 // For local execution use relavive\r
175                                 ConfiguredExecutable<ClustalW> confClustal = Configurator.configureExecutable(clustal, ExecProvider.Local);\r
176 \r
177                                 // Add options to the executable\r
178                                 confClustal.addParameters(options);\r
179 \r
180                                 LocalRunner lr = new LocalRunner(confClustal);\r
181                                 lr.executeJob();\r
182                                 confClustal = (ConfiguredExecutable<ClustalW>) lr.waitForResult();\r
183                                 assertNotNull(confClustal.getResults());\r
184                                 Collections.shuffle(options);\r
185                         }\r
186                 } catch (JobSubmissionException e) {\r
187                         e.printStackTrace();\r
188                         fail(e.getLocalizedMessage());\r
189                 } catch (JobExecutionException e) {\r
190                         e.printStackTrace();\r
191                         fail(e.getLocalizedMessage());\r
192                 } catch (JAXBException e) {\r
193                         e.printStackTrace();\r
194                         fail(e.getLocalizedMessage());\r
195                 } catch (ResultNotAvailableException e) {\r
196                         e.printStackTrace();\r
197                         fail(e.getLocalizedMessage());\r
198                 } catch (FileNotFoundException e) {\r
199                         e.printStackTrace();\r
200                         fail(e.getLocalizedMessage());\r
201                 }\r
202         }\r
203 \r
204         public static final void main(String[] args)\r
205                         throws JobSubmissionException, JobExecutionException, InterruptedException {\r
206                 ClustalW clustal = new ClustalW();\r
207                 clustal.setInput(AllTestSuit.test_input).setOutput(test_outfile);\r
208                 // For local execution use relavive\r
209                 ConfiguredExecutable<ClustalW> confClustal = Configurator.configureExecutable(clustal);\r
210                 AsyncExecutor lr = new AsyncLocalRunner();\r
211                 lr.submitJob(confClustal);\r
212                 Thread.sleep(3000);\r
213                 LocalExecutorService.shutDown();\r
214         }\r
215 \r
216         // disabled\r
217         @Test(enabled = false)\r
218         public void AddParameters() {\r
219                 ArrayList<FastaSequence> seqs = new ArrayList<FastaSequence>();\r
220                 FastaSequence fs = new FastaSequence("tests1", "aqtctcatcatctcatctgcccccgggttatgagtagtacgcatctacg");\r
221                 FastaSequence fs2 = new FastaSequence("tests2", "aqtctcatcatctcatctgcccccgggttatgagtagtacgcatctacg");\r
222                 FastaSequence fs3 = new FastaSequence("tests3", "aqtctcatcatctcatctgcccccgggttatgagtagtacgcatctacg");\r
223                 seqs.add(fs);\r
224                 seqs.add(fs2);\r
225                 seqs.add(fs3);\r
226                 ClustalW cl = new ClustalW();\r
227                 cl.setInput("input.txt").setOutput("output.txt");\r
228                 ConfiguredExecutable<ClustalW> confClustal;\r
229                 try {\r
230                         confClustal = Configurator.configureExecutable(cl);\r
231                         RunnerUtil.writeInput(seqs, confClustal);\r
232 \r
233                         LocalRunner lr = new LocalRunner(confClustal);\r
234                         lr.executeJob();\r
235                         confClustal = (ConfiguredExecutable<ClustalW>) lr.waitForResult();\r
236                         assertNotNull(confClustal.getResults());\r
237 \r
238                         assertTrue(confClustal.saveRunConfiguration());\r
239                         ConfiguredExecutable<ClustalW> cexec = (ConfiguredExecutable<ClustalW>) confClustal\r
240                                         .loadRunConfiguration(new FileInputStream(new File(\r
241                                                         confClustal.getWorkDirectory(),\r
242                                                         RunConfiguration.rconfigFile)));\r
243                         assertNotNull(cexec);\r
244 \r
245                         lr = new LocalRunner(cexec);\r
246                         lr.executeJob();\r
247                         confClustal = (ConfiguredExecutable<ClustalW>) lr.waitForResult();\r
248                         assertNotNull(confClustal.getResults());\r
249 \r
250                         System.out.println("CE:" + cexec);\r
251                 } catch (JobSubmissionException e) {\r
252                         e.printStackTrace();\r
253                         fail(e.getMessage());\r
254                 } catch (FileNotFoundException e) {\r
255                         e.printStackTrace();\r
256                         fail(e.getMessage());\r
257                 } catch (JobExecutionException e) {\r
258                         e.printStackTrace();\r
259                         fail(e.getMessage());\r
260                 } catch (ResultNotAvailableException e) {\r
261                         e.printStackTrace();\r
262                         fail(e.getMessage());\r
263                 } catch (IOException e) {\r
264                         e.printStackTrace();\r
265                         fail(e.getMessage());\r
266                 }\r
267         }\r
268 \r
269         @Test(groups = { AllTestSuit.test_group_runner })\r
270         public void Persistance() {\r
271                 try {\r
272                         ClustalW clustal = new ClustalW();\r
273                         clustal.setError("errrr.txt");\r
274                         clustal.setInput(AllTestSuit.test_input);\r
275                         clustal.setOutput("outtt.txt");\r
276                         assertEquals(clustal.getInput(), AllTestSuit.test_input);\r
277                         assertEquals(clustal.getError(), "errrr.txt");\r
278                         assertEquals(clustal.getOutput(), "outtt.txt");\r
279                         ConfiguredExecutable<ClustalW> cClustal = Configurator.configureExecutable(clustal, Executable.ExecProvider.Local);\r
280 \r
281                         SyncExecutor sexec = Configurator.getSyncEngine(cClustal);\r
282                         sexec.executeJob();\r
283                         cClustal = (ConfiguredExecutable<ClustalW>) sexec.waitForResult();\r
284                         assertNotNull(cClustal.getResults());\r
285                         // Save run configuration\r
286                         assertTrue(cClustal.saveRunConfiguration());\r
287 \r
288                         // See if loaded configuration is the same as saved\r
289                         RunConfiguration loadedRun = RunConfiguration\r
290                                         .load(new FileInputStream(new File(cClustal\r
291                                                         .getWorkDirectory(), RunConfiguration.rconfigFile)));\r
292                         assertTrue(((ConfExecutable<ClustalW>) cClustal)\r
293                                         .getRunConfiguration().equals(loadedRun));\r
294                         // Load run configuration as ConfExecutable\r
295                         ConfiguredExecutable<ClustalW> resurrectedCclustal = (ConfiguredExecutable<ClustalW>) cClustal\r
296                                         .loadRunConfiguration(new FileInputStream(new File(cClustal\r
297                                                         .getWorkDirectory(), RunConfiguration.rconfigFile)));\r
298                         assertNotNull(resurrectedCclustal);\r
299                         // See in details whether executables are the same\r
300                         assertEquals(resurrectedCclustal.getExecutable(), clustal);\r
301 \r
302                         // Finally rerun the job in the new task directory\r
303                         ConfiguredExecutable<ClustalW> resclustal = Configurator\r
304                                         .configureExecutable(resurrectedCclustal.getExecutable(),\r
305                                                         Executable.ExecProvider.Local);\r
306 \r
307                         sexec = Configurator.getSyncEngine(resclustal, Executable.ExecProvider.Local);\r
308                         sexec.executeJob();\r
309                         cClustal = (ConfiguredExecutable<ClustalW>) sexec.waitForResult();\r
310                         assertNotNull(cClustal.getResults());\r
311                 } catch (JobSubmissionException e) {\r
312                         e.printStackTrace();\r
313                         fail(e.getMessage());\r
314                 } catch (JobExecutionException e) {\r
315                         e.printStackTrace();\r
316                         fail(e.getMessage());\r
317                 } catch (FileNotFoundException e) {\r
318                         e.printStackTrace();\r
319                         fail(e.getMessage());\r
320                 } catch (IOException e) {\r
321                         e.printStackTrace();\r
322                         fail(e.getMessage());\r
323                 } catch (ResultNotAvailableException e) {\r
324                         e.printStackTrace();\r
325                         fail(e.getMessage());\r
326                 }\r
327         }\r
328 \r
329         @Test(groups = { AllTestSuit.test_group_runner })\r
330         public void readStatistics() {\r
331                 try {\r
332                         ClustalW clustal = new ClustalW().setInput(AllTestSuit.test_input).setOutput(test_outfile);\r
333                         ConfiguredExecutable<ClustalW> confClustal = Configurator.configureExecutable(clustal, Executable.ExecProvider.Local);\r
334 \r
335                         AsyncExecutor sexec = Configurator.getAsyncEngine(confClustal);\r
336                         String jobId = sexec.submitJob(confClustal);\r
337                         String file = confClustal.getWorkDirectory() + File.separator + ClustalW.getStatFile();\r
338                         FilePuller fw = FilePuller.newFilePuller(file, FileWatcher.MIN_CHUNK_SIZE_BYTES);\r
339                         int count = 0;\r
340                         long position = 0;\r
341                         fw.waitForFile(4);\r
342                         while (!(sexec.getJobStatus(jobId) == JobStatus.FINISHED || sexec\r
343                                         .getJobStatus(jobId) == JobStatus.FAILED || sexec\r
344                                         .getJobStatus(jobId) == JobStatus.UNDEFINED)\r
345                                         || fw.hasMoreData()) {\r
346                                 ChunkHolder ch = fw.pull(position);\r
347                                 String chunk = ch.getChunk();\r
348                                 position = ch.getNextPosition();\r
349                                 System.out.print(chunk);\r
350                                 count++;\r
351                         }\r
352                         assertTrue(count > 1);\r
353                         ConfiguredExecutable<?> al = sexec.getResults(jobId);\r
354                         assertNotNull(al.getResults());\r
355                 } catch (JobSubmissionException e) {\r
356                         e.printStackTrace();\r
357                         fail(e.getMessage());\r
358                 } catch (ResultNotAvailableException e) {\r
359                         e.printStackTrace();\r
360                         fail(e.getMessage());\r
361                 } catch (IOException e) {\r
362                         e.printStackTrace();\r
363                         fail(e.getMessage());\r
364                 }\r
365         }\r
366 \r
367         @Test(groups = { AllTestSuit.test_group_cluster, AllTestSuit.test_group_runner })\r
368         public void RunOnCluster() {\r
369                 ClustalW clustal = new ClustalW();\r
370                 assertFalse(SysPrefs.isWindows, "Cluster execution can only be in unix environment");\r
371                 clustal.setInput(AllTestSuit.test_input).setOutput(cluster_test_outfile);\r
372 \r
373                 try {\r
374                         ConfiguredExecutable<ClustalW> confClustal = Configurator.configureExecutable(clustal);\r
375                         ClusterRunner runner = ClusterRunner.getInstance(confClustal);\r
376                         // ClusterSession csession = JobRunner.getSession();\r
377                         assertNotNull(runner);\r
378                         runner.executeJob();\r
379                         // assertNotNull("JobId is null", jobId1);\r
380                         JobStatus status = runner.getJobStatus();\r
381                         assertTrue(status == JobStatus.PENDING || status == JobStatus.RUNNING);\r
382                         JobInfo info = runner.getJobInfo();\r
383                         assertNotNull(info);\r
384                         StatisticManager sm = new StatisticManager(info);\r
385                         assertNotNull(sm);\r
386                         try {\r
387                                 String exits = sm.getExitStatus();\r
388                                 assertNotNull("Exit status is null", exits);\r
389                                 // cut 4 trailing zeros from the number\r
390                                 int exitsInt = ClusterEngineUtil.CLUSTER_STAT_IN_SEC.parse(exits).intValue();\r
391                                 assertEquals(0, exitsInt);\r
392                                 System.out.println(sm.getAllStats());\r
393                         } catch (ParseException e) {\r
394                                 e.printStackTrace();\r
395                                 fail("Parse Exception: " + e.getMessage());\r
396                         }\r
397                         // At present the task directory could not be completely removed\r
398                         // @see JobRunner.cleanup()\r
399                         assertFalse(runner.cleanup(), "Could not remove some files whilst cleaning up ");\r
400                         assertTrue(sm.hasExited());\r
401                         assertFalse(sm.wasAborted());\r
402                         assertFalse(sm.hasDump());\r
403                         assertFalse(sm.hasSignaled());\r
404                 } catch (JobSubmissionException e) {\r
405                         e.printStackTrace();\r
406                         fail("DrmaaException caught:" + e.getMessage());\r
407                 } catch (JobExecutionException e) {\r
408                         e.printStackTrace();\r
409                         fail("DrmaaException caught:" + e.getMessage());\r
410                 } catch (DrmaaException e) {\r
411                         e.printStackTrace();\r
412                         fail("DrmaaException caught:" + e.getMessage());\r
413                 }\r
414         }\r
415 \r
416         @Test(groups = { AllTestSuit.test_group_cluster, AllTestSuit.test_group_runner })\r
417         public void readStatisticsClusterExecution() {\r
418                 try {\r
419                         ClustalW clustal = new ClustalW();\r
420                         clustal.setInput(AllTestSuit.test_input);\r
421                         clustal.setOutput(test_outfile);\r
422                         ConfiguredExecutable<ClustalW> confClustal = Configurator.configureExecutable(clustal, Executable.ExecProvider.Cluster);\r
423 \r
424                         AsyncExecutor sexec = Configurator.getAsyncEngine(confClustal);\r
425                         String jobId = sexec.submitJob(confClustal);\r
426                         String file = confClustal.getWorkDirectory() + File.separator + ClustalW.getStatFile();\r
427                         FilePuller fw = FilePuller.newFilePuller(file, FileWatcher.MIN_CHUNK_SIZE_BYTES);\r
428                         int count = 0;\r
429                         long position = 0;\r
430                         fw.waitForFile(200);\r
431                         /* Under certain circumstances DRMAA could report the status wrongly thus this loop never ends \r
432                          * TODO deal with this! \r
433                          * */\r
434                         while (!(sexec.getJobStatus(jobId) == JobStatus.FINISHED || sexec\r
435                                         .getJobStatus(jobId) == JobStatus.FAILED )\r
436                                         || fw.hasMoreData()) {\r
437                                 ChunkHolder ch = fw.pull(position);\r
438                                 String chunk = ch.getChunk();\r
439                                 position = ch.getNextPosition();\r
440                                 System.out.print(chunk);\r
441                                 count++;\r
442                                 if(sexec.getJobStatus(jobId) == JobStatus.UNDEFINED ) {\r
443                                         System.out.println("DRMAA reported wrong status for job + " + jobId +" continue anyway!");\r
444                                         break; \r
445                                 }\r
446                         }\r
447                         assertTrue(count > 1);\r
448                         ConfiguredExecutable<?> al = sexec.getResults(jobId);\r
449                         assertNotNull(al.getResults());\r
450                 } catch (JobSubmissionException e) {\r
451                         e.printStackTrace();\r
452                         fail(e.getMessage());\r
453                 } catch (ResultNotAvailableException e) {\r
454                         e.printStackTrace();\r
455                         fail(e.getMessage());\r
456                 } catch (IOException e) {\r
457                         e.printStackTrace();\r
458                         fail(e.getMessage());\r
459                 }\r
460         }\r
461 }\r