1 /* Copyright (c) 2009 Peter Troshin
\r
3 * JAva Bioinformatics Analysis Web Services (JABAWS) @version: 1.0
\r
5 * This library is free software; you can redistribute it and/or modify it under the terms of the
\r
6 * Apache License version 2 as published by the Apache Software Foundation
\r
8 * This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
\r
9 * even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Apache
\r
10 * License for more details.
\r
12 * A copy of the license is in apache_license.txt. It is also available here:
\r
13 * @see: http://www.apache.org/licenses/LICENSE-2.0.txt
\r
15 * Any republication or derived work distributed in source code form
\r
16 * must include this copyright and license notice.
\r
19 package compbio.runner.msa;
\r
21 import static org.testng.Assert.assertEquals;
\r
22 import static org.testng.Assert.assertFalse;
\r
23 import static org.testng.Assert.assertNotNull;
\r
24 import static org.testng.Assert.assertTrue;
\r
25 import static org.testng.Assert.fail;
\r
27 import java.io.File;
\r
28 import java.io.FileInputStream;
\r
29 import java.io.FileNotFoundException;
\r
30 import java.io.IOException;
\r
31 import java.text.ParseException;
\r
32 import java.util.ArrayList;
\r
33 import java.util.Collections;
\r
34 import java.util.List;
\r
36 import javax.xml.bind.JAXBException;
\r
37 import javax.xml.bind.ValidationException;
\r
39 import org.ggf.drmaa.DrmaaException;
\r
40 import org.ggf.drmaa.JobInfo;
\r
41 import org.testng.annotations.Test;
\r
43 import compbio.data.sequence.FastaSequence;
\r
44 import compbio.engine.AsyncExecutor;
\r
45 import compbio.engine.Configurator;
\r
46 import compbio.engine.FilePuller;
\r
47 import compbio.engine.SyncExecutor;
\r
48 import compbio.engine.client.ConfExecutable;
\r
49 import compbio.engine.client.ConfiguredExecutable;
\r
50 import compbio.engine.client.Executable;
\r
51 import compbio.engine.client.RunConfiguration;
\r
52 import compbio.engine.client.Executable.ExecProvider;
\r
53 import compbio.engine.cluster.drmaa.ClusterUtil;
\r
54 import compbio.engine.cluster.drmaa.JobRunner;
\r
55 import compbio.engine.cluster.drmaa.StatisticManager;
\r
56 import compbio.engine.conf.RunnerConfigMarshaller;
\r
57 import compbio.engine.local.AsyncLocalRunner;
\r
58 import compbio.engine.local.LocalExecutorService;
\r
59 import compbio.engine.local.LocalRunner;
\r
60 import compbio.metadata.AllTestSuit;
\r
61 import compbio.metadata.ChunkHolder;
\r
62 import compbio.metadata.JobExecutionException;
\r
63 import compbio.metadata.JobStatus;
\r
64 import compbio.metadata.JobSubmissionException;
\r
65 import compbio.metadata.LimitsManager;
\r
66 import compbio.metadata.PresetManager;
\r
67 import compbio.metadata.ResultNotAvailableException;
\r
68 import compbio.metadata.RunnerConfig;
\r
69 import compbio.runner.OptionCombinator;
\r
70 import compbio.runner.Util;
\r
71 import compbio.runner.msa.ClustalW;
\r
72 import compbio.util.FileWatcher;
\r
73 import compbio.util.SysPrefs;
\r
75 public class ClustalWTester {
\r
77 static final String clustalConfigFile = AllTestSuit.TEST_DATA_PATH
\r
78 + "ClustalParameters.xml";
\r
79 public static String test_outfile = "TO1381.clustal.out";
\r
80 public static String cluster_test_outfile = "TO1381.clustal.cluster.out";
\r
82 @Test(groups = { AllTestSuit.test_group_cluster,
\r
83 AllTestSuit.test_group_runner })
\r
84 public void testRunOnCluster() {
\r
85 ClustalW clustal = new ClustalW();
\r
86 assertFalse(SysPrefs.isWindows,
\r
87 "Cluster execution can only be in unix environment");
\r
88 clustal.setInput(AllTestSuit.test_input)
\r
89 .setOutput(cluster_test_outfile);
\r
93 ConfiguredExecutable<ClustalW> confClustal = Configurator
\r
94 .configureExecutable(clustal);
\r
95 JobRunner runner = JobRunner.getInstance(confClustal);
\r
96 // ClusterSession csession = JobRunner.getSession();
\r
97 assertNotNull(runner);
\r
98 runner.executeJob();
\r
99 // assertNotNull("JobId is null", jobId1);
\r
100 JobStatus status = runner.getJobStatus();
\r
101 assertTrue(status == JobStatus.PENDING
\r
102 || status == JobStatus.RUNNING);
\r
103 JobInfo info = runner.getJobInfo();
\r
104 assertNotNull(info);
\r
105 StatisticManager sm = new StatisticManager(info);
\r
108 String exits = sm.getExitStatus();
\r
109 assertNotNull("Exit status is null", exits);
\r
110 // cut 4 trailing zeros from the number
\r
111 int exitsInt = ClusterUtil.CLUSTER_STAT_IN_SEC.parse(exits)
\r
113 assertEquals(0, exitsInt);
\r
114 System.out.println(sm.getAllStats());
\r
116 } catch (ParseException e) {
\r
117 e.printStackTrace();
\r
118 fail("Parse Exception: " + e.getMessage());
\r
120 // At present the task directory could not be completely removed
\r
121 // @see JobRunner.cleanup()
\r
122 assertFalse(runner.cleanup(),
\r
123 "Could not remove some files whilst cleaning up ");
\r
124 assertTrue(sm.hasExited());
\r
125 assertFalse(sm.wasAborted());
\r
126 assertFalse(sm.hasDump());
\r
127 assertFalse(sm.hasSignaled());
\r
129 } catch (JobSubmissionException e) {
\r
130 e.printStackTrace();
\r
131 fail("DrmaaException caught:" + e.getMessage());
\r
132 } catch (JobExecutionException e) {
\r
133 e.printStackTrace();
\r
134 fail("DrmaaException caught:" + e.getMessage());
\r
135 } catch (DrmaaException e) {
\r
136 e.printStackTrace();
\r
137 fail("DrmaaException caught:" + e.getMessage());
\r
141 @Test(groups = { AllTestSuit.test_group_runner })
\r
142 public void testRunLocally() {
\r
143 ClustalW clustal = new ClustalW();
\r
144 clustal.setInput(AllTestSuit.test_input).setOutput(test_outfile);
\r
147 // For local execution use relavive
\r
148 ConfiguredExecutable<ClustalW> confClustal = Configurator
\r
149 .configureExecutable(clustal, Executable.ExecProvider.Local);
\r
150 LocalRunner lr = new LocalRunner(confClustal);
\r
152 confClustal = (ConfiguredExecutable<ClustalW>) lr.waitForResult();
\r
153 assertNotNull(confClustal.getResults());
\r
154 } catch (JobSubmissionException e) {
\r
155 e.printStackTrace();
\r
156 fail(e.getLocalizedMessage());
\r
157 } catch (JobExecutionException e) {
\r
158 e.printStackTrace();
\r
159 fail(e.getLocalizedMessage());
\r
160 } catch (ResultNotAvailableException e) {
\r
161 e.printStackTrace();
\r
162 fail(e.getLocalizedMessage());
\r
166 @Test(groups = { AllTestSuit.test_group_runner })
\r
167 public void testRunWithMatrix() {
\r
168 ClustalW clustal = new ClustalW();
\r
169 clustal.setInput(AllTestSuit.test_input).setOutput(test_outfile);
\r
170 clustal.setParameter("-matrix=BLOSUM62");
\r
173 // For local execution use relavive
\r
174 ConfiguredExecutable<ClustalW> confClustal = Configurator
\r
175 .configureExecutable(clustal, Executable.ExecProvider.Local);
\r
176 LocalRunner lr = new LocalRunner(confClustal);
\r
178 confClustal = (ConfiguredExecutable<ClustalW>) lr.waitForResult();
\r
179 assertNotNull(confClustal.getResults());
\r
180 } catch (JobSubmissionException e) {
\r
181 e.printStackTrace();
\r
182 fail(e.getLocalizedMessage());
\r
183 } catch (JobExecutionException e) {
\r
184 e.printStackTrace();
\r
185 fail(e.getLocalizedMessage());
\r
186 } catch (ResultNotAvailableException e) {
\r
187 e.printStackTrace();
\r
188 fail(e.getLocalizedMessage());
\r
192 @Test(groups = { AllTestSuit.test_group_runner })
\r
193 public void testConfigurationLoading() {
\r
195 RunnerConfig<ClustalW> clustalConfig = ConfExecutable
\r
196 .getRunnerOptions(ClustalW.class);
\r
197 assertNotNull(clustalConfig);
\r
198 assertTrue(clustalConfig.getArguments().size() > 0);
\r
200 PresetManager<ClustalW> clustalPresets = ConfExecutable
\r
201 .getRunnerPresets(ClustalW.class);
\r
202 assertNotNull(clustalPresets);
\r
203 assertTrue(clustalPresets.getPresets().size() > 0);
\r
204 clustalPresets.validate(clustalConfig);
\r
206 LimitsManager<ClustalW> clustalLimits = ConfExecutable
\r
207 .getRunnerLimits(ClustalW.class);
\r
208 assertNotNull(clustalLimits);
\r
209 assertTrue(clustalLimits.getLimits().size() > 0);
\r
210 clustalLimits.validate(clustalPresets);
\r
212 } catch (FileNotFoundException e) {
\r
213 e.printStackTrace();
\r
214 fail(e.getLocalizedMessage());
\r
215 } catch (IOException e) {
\r
216 e.printStackTrace();
\r
217 fail(e.getLocalizedMessage());
\r
218 } catch (ValidationException e) {
\r
219 e.printStackTrace();
\r
220 fail(e.getLocalizedMessage());
\r
224 @Test(groups = { AllTestSuit.test_group_runner })
\r
225 public void testOptionsLocally() {
\r
228 RunnerConfigMarshaller<ClustalW> clustalmarsh = new RunnerConfigMarshaller<ClustalW>(
\r
229 RunnerConfig.class);
\r
231 RunnerConfig<ClustalW> clustalConfig = clustalmarsh.read(
\r
232 new FileInputStream(new File(clustalConfigFile)),
\r
233 RunnerConfig.class);
\r
235 OptionCombinator clustalOpc = new OptionCombinator(clustalConfig);
\r
236 List<String> options = clustalOpc.getOptionsAtRandom();
\r
237 for (int i = 0; i < options.size(); i++) {
\r
238 System.out.println("Using options: " + options);
\r
239 ClustalW clustal = new ClustalW();
\r
240 clustal.setInput(AllTestSuit.test_input)
\r
241 .setOutput(test_outfile);
\r
243 // For local execution use relavive
\r
244 ConfiguredExecutable<ClustalW> confClustal = Configurator
\r
245 .configureExecutable(clustal, ExecProvider.Local);
\r
247 // Add options to the executable
\r
248 confClustal.addParameters(options);
\r
250 LocalRunner lr = new LocalRunner(confClustal);
\r
252 confClustal = (ConfiguredExecutable<ClustalW>) lr
\r
254 assertNotNull(confClustal.getResults());
\r
255 Collections.shuffle(options);
\r
258 } catch (JobSubmissionException e) {
\r
259 e.printStackTrace();
\r
260 fail(e.getLocalizedMessage());
\r
261 } catch (JobExecutionException e) {
\r
262 e.printStackTrace();
\r
263 fail(e.getLocalizedMessage());
\r
264 } catch (JAXBException e) {
\r
265 e.printStackTrace();
\r
266 fail(e.getLocalizedMessage());
\r
267 } catch (ResultNotAvailableException e) {
\r
268 e.printStackTrace();
\r
269 fail(e.getLocalizedMessage());
\r
270 } catch (FileNotFoundException e) {
\r
271 e.printStackTrace();
\r
272 fail(e.getLocalizedMessage());
\r
276 public static final void main(String[] args) throws JobSubmissionException,
\r
277 JobExecutionException, InterruptedException {
\r
278 ClustalW clustal = new ClustalW();
\r
279 clustal.setInput(AllTestSuit.test_input).setOutput(test_outfile);
\r
280 // For local execution use relavive
\r
281 ConfiguredExecutable<ClustalW> confClustal = Configurator
\r
282 .configureExecutable(clustal);
\r
283 AsyncExecutor lr = new AsyncLocalRunner();
\r
284 lr.submitJob(confClustal);
\r
285 Thread.sleep(3000);
\r
286 LocalExecutorService.shutDown();
\r
289 @Test(enabled = false)
\r
290 public void testAddParameters() {
\r
291 ArrayList<FastaSequence> seqs = new ArrayList<FastaSequence>();
\r
292 FastaSequence fs = new FastaSequence("tests1",
\r
293 "aqtctcatcatctcatctgcccccgggttatgagtagtacgcatctacg");
\r
294 FastaSequence fs2 = new FastaSequence("tests2",
\r
295 "aqtctcatcatctcatctgcccccgggttatgagtagtacgcatctacg");
\r
296 FastaSequence fs3 = new FastaSequence("tests3",
\r
297 "aqtctcatcatctcatctgcccccgggttatgagtagtacgcatctacg");
\r
301 ClustalW cl = new ClustalW();
\r
302 cl.setInput("input.txt").setOutput("output.txt");
\r
303 ConfiguredExecutable<ClustalW> confClustal;
\r
305 confClustal = Configurator.configureExecutable(cl);
\r
306 Util.writeInput(seqs, confClustal);
\r
308 LocalRunner lr = new LocalRunner(confClustal);
\r
310 confClustal = (ConfiguredExecutable<ClustalW>) lr.waitForResult();
\r
311 assertNotNull(confClustal.getResults());
\r
313 assertTrue(confClustal.saveRunConfiguration());
\r
314 ConfiguredExecutable<ClustalW> cexec = (ConfiguredExecutable<ClustalW>) confClustal
\r
315 .loadRunConfiguration(new FileInputStream(new File(
\r
316 confClustal.getWorkDirectory(),
\r
317 RunConfiguration.rconfigFile)));
\r
318 assertNotNull(cexec);
\r
320 lr = new LocalRunner(cexec);
\r
322 confClustal = (ConfiguredExecutable<ClustalW>) lr.waitForResult();
\r
323 assertNotNull(confClustal.getResults());
\r
325 System.out.println("CE:" + cexec);
\r
326 } catch (JobSubmissionException e) {
\r
327 e.printStackTrace();
\r
328 fail(e.getMessage());
\r
329 } catch (FileNotFoundException e) {
\r
330 e.printStackTrace();
\r
331 fail(e.getMessage());
\r
332 } catch (JobExecutionException e) {
\r
333 e.printStackTrace();
\r
334 fail(e.getMessage());
\r
335 } catch (ResultNotAvailableException e) {
\r
336 e.printStackTrace();
\r
337 fail(e.getMessage());
\r
338 } catch (IOException e) {
\r
339 e.printStackTrace();
\r
340 fail(e.getMessage());
\r
344 @Test(groups = { AllTestSuit.test_group_runner })
\r
345 public void testPersistance() {
\r
347 ClustalW clustal = new ClustalW();
\r
348 clustal.setError("errrr.txt").setInput(AllTestSuit.test_input)
\r
349 .setOutput("outtt.txt");
\r
350 assertEquals(clustal.getInput(), AllTestSuit.test_input);
\r
351 assertEquals(clustal.getError(), "errrr.txt");
\r
352 assertEquals(clustal.getOutput(), "outtt.txt");
\r
353 ConfiguredExecutable<ClustalW> cClustal = Configurator
\r
354 .configureExecutable(clustal, Executable.ExecProvider.Local);
\r
356 SyncExecutor sexec = Configurator.getSyncEngine(cClustal);
\r
357 sexec.executeJob();
\r
358 cClustal = (ConfiguredExecutable<ClustalW>) sexec.waitForResult();
\r
359 assertNotNull(cClustal.getResults());
\r
360 // Save run configuration
\r
361 assertTrue(cClustal.saveRunConfiguration());
\r
363 // See if loaded configuration is the same as saved
\r
364 RunConfiguration loadedRun = RunConfiguration
\r
365 .load(new FileInputStream(new File(cClustal
\r
366 .getWorkDirectory(), RunConfiguration.rconfigFile)));
\r
367 assertTrue(((ConfExecutable<ClustalW>) cClustal)
\r
368 .getRunConfiguration().equals(loadedRun));
\r
369 // Load run configuration as ConfExecutable
\r
370 ConfiguredExecutable<ClustalW> resurrectedCclustal = (ConfiguredExecutable<ClustalW>) cClustal
\r
371 .loadRunConfiguration(new FileInputStream(new File(cClustal
\r
372 .getWorkDirectory(), RunConfiguration.rconfigFile)));
\r
373 assertNotNull(resurrectedCclustal);
\r
374 // See in details whether executables are the same
\r
375 assertEquals(resurrectedCclustal.getExecutable(), clustal);
\r
377 // Finally rerun the job in the new task directory
\r
378 ConfiguredExecutable<ClustalW> resclustal = Configurator
\r
379 .configureExecutable(resurrectedCclustal.getExecutable(),
\r
380 Executable.ExecProvider.Local);
\r
382 sexec = Configurator.getSyncEngine(resclustal,
\r
383 Executable.ExecProvider.Local);
\r
384 sexec.executeJob();
\r
385 cClustal = (ConfiguredExecutable<ClustalW>) sexec.waitForResult();
\r
386 assertNotNull(cClustal.getResults());
\r
388 } catch (JobSubmissionException e) {
\r
389 e.printStackTrace();
\r
390 fail(e.getMessage());
\r
391 } catch (JobExecutionException e) {
\r
392 e.printStackTrace();
\r
393 fail(e.getMessage());
\r
394 } catch (FileNotFoundException e) {
\r
395 e.printStackTrace();
\r
396 fail(e.getMessage());
\r
397 } catch (IOException e) {
\r
398 e.printStackTrace();
\r
399 fail(e.getMessage());
\r
400 } catch (ResultNotAvailableException e) {
\r
401 e.printStackTrace();
\r
402 fail(e.getMessage());
\r
406 @Test(groups = { AllTestSuit.test_group_runner })
\r
407 public void readStatistics() {
\r
409 ClustalW clustal = new ClustalW().setInput(AllTestSuit.test_input)
\r
410 .setOutput(test_outfile);
\r
411 ConfiguredExecutable<ClustalW> confClustal = Configurator
\r
412 .configureExecutable(clustal, Executable.ExecProvider.Local);
\r
414 AsyncExecutor sexec = Configurator.getAsyncEngine(confClustal);
\r
415 String jobId = sexec.submitJob(confClustal);
\r
416 FilePuller fw = FilePuller.newFilePuller(confClustal
\r
417 .getWorkDirectory()
\r
418 + File.separator + ClustalW.getStatFile(),
\r
419 FileWatcher.MIN_CHUNK_SIZE_BYTES);
\r
423 while (!(sexec.getJobStatus(jobId) == JobStatus.FINISHED || sexec
\r
424 .getJobStatus(jobId) == JobStatus.FAILED || sexec
\r
425 .getJobStatus(jobId) == JobStatus.UNDEFINED)
\r
426 || fw.hasMoreData()) {
\r
427 ChunkHolder ch = fw.pull(position);
\r
428 String chunk = ch.getChunk();
\r
429 position = ch.getNextPosition();
\r
430 System.out.print(chunk);
\r
433 assertTrue(count > 1);
\r
434 ConfiguredExecutable<?> al = sexec.getResults(jobId);
\r
435 assertNotNull(al.getResults());
\r
436 } catch (JobSubmissionException e) {
\r
437 e.printStackTrace();
\r
438 fail(e.getMessage());
\r
439 } catch (ResultNotAvailableException e) {
\r
440 e.printStackTrace();
\r
441 fail(e.getMessage());
\r
442 } catch (IOException e) {
\r
443 e.printStackTrace();
\r
444 fail(e.getMessage());
\r
448 @Test(groups = { AllTestSuit.test_group_cluster,
\r
449 AllTestSuit.test_group_runner })
\r
450 public void readStatisticsClusterExecution() {
\r
452 ClustalW clustal = new ClustalW().setInput(AllTestSuit.test_input)
\r
453 .setOutput(test_outfile);
\r
454 ConfiguredExecutable<ClustalW> confClustal = Configurator
\r
455 .configureExecutable(clustal,
\r
456 Executable.ExecProvider.Cluster);
\r
458 AsyncExecutor sexec = Configurator.getAsyncEngine(confClustal);
\r
459 String jobId = sexec.submitJob(confClustal);
\r
460 FilePuller fw = FilePuller.newFilePuller(confClustal
\r
461 .getWorkDirectory()
\r
462 + File.separator + ClustalW.getStatFile(),
\r
463 FileWatcher.MIN_CHUNK_SIZE_BYTES);
\r
466 fw.waitForFile(200);
\r
467 /* Under certain circumstances DRMAA could report the status wrongly thus this loop never ends
\r
468 * TODO deal with this!
\r
470 while (!(sexec.getJobStatus(jobId) == JobStatus.FINISHED || sexec
\r
471 .getJobStatus(jobId) == JobStatus.FAILED )
\r
472 || fw.hasMoreData()) {
\r
473 ChunkHolder ch = fw.pull(position);
\r
474 String chunk = ch.getChunk();
\r
475 position = ch.getNextPosition();
\r
476 System.out.print(chunk);
\r
478 if(sexec.getJobStatus(jobId) == JobStatus.UNDEFINED ) {
\r
479 System.out.println("DRMAA reported wrong status for job + " + jobId +" continue anyway!");
\r
483 assertTrue(count > 1);
\r
484 ConfiguredExecutable<?> al = sexec.getResults(jobId);
\r
485 assertNotNull(al.getResults());
\r
486 } catch (JobSubmissionException e) {
\r
487 e.printStackTrace();
\r
488 fail(e.getMessage());
\r
489 } catch (ResultNotAvailableException e) {
\r
490 e.printStackTrace();
\r
491 fail(e.getMessage());
\r
492 } catch (IOException e) {
\r
493 e.printStackTrace();
\r
494 fail(e.getMessage());
\r