1 /* Copyright (c) 2009 Peter Troshin
\r
2 * Copyright (c) 2013 Alexander Sherstnev
\r
4 * JAva Bioinformatics Analysis Web Services (JABAWS)
\r
7 * This library is free software; you can redistribute it and/or modify it under the terms of the
\r
8 * Apache License version 2 as published by the Apache Software Foundation
\r
10 * This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
\r
11 * even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Apache
\r
12 * License for more details.
\r
14 * A copy of the license is in apache_license.txt. It is also available here:
\r
15 * @see: http://www.apache.org/licenses/LICENSE-2.0.txt
\r
17 * Any republication or derived work distributed in source code form
\r
18 * must include this copyright and license notice.
\r
21 package compbio.runner.msa;
\r
23 import static org.testng.Assert.assertEquals;
\r
24 import static org.testng.Assert.assertFalse;
\r
25 import static org.testng.Assert.assertNotNull;
\r
26 import static org.testng.Assert.assertTrue;
\r
27 import static org.testng.Assert.fail;
\r
29 import java.io.File;
\r
30 import java.io.FileInputStream;
\r
31 import java.io.FileNotFoundException;
\r
32 import java.io.IOException;
\r
33 import java.text.ParseException;
\r
34 import java.util.Collections;
\r
35 import java.util.List;
\r
37 import javax.xml.bind.JAXBException;
\r
39 import org.ggf.drmaa.DrmaaException;
\r
40 import org.ggf.drmaa.JobInfo;
\r
41 import org.testng.annotations.Test;
\r
43 import compbio.engine.AsyncExecutor;
\r
44 import compbio.engine.Configurator;
\r
45 import compbio.engine.FilePuller;
\r
46 import compbio.engine.SyncExecutor;
\r
47 import compbio.engine.client.Util;
\r
48 import compbio.engine.client.ConfExecutable;
\r
49 import compbio.engine.client.ConfiguredExecutable;
\r
50 import compbio.engine.client.Executable;
\r
51 import compbio.engine.client.Executable.ExecProvider;
\r
52 import compbio.engine.client.RunConfiguration;
\r
53 import compbio.engine.cluster.drmaa.ClusterUtil;
\r
54 import compbio.engine.cluster.drmaa.JobRunner;
\r
55 import compbio.engine.cluster.drmaa.StatisticManager;
\r
56 import compbio.engine.conf.RunnerConfigMarshaller;
\r
57 import compbio.engine.local.AsyncLocalRunner;
\r
58 import compbio.engine.local.LocalExecutorService;
\r
59 import compbio.engine.local.LocalRunner;
\r
60 import compbio.metadata.AllTestSuit;
\r
61 import compbio.metadata.ChunkHolder;
\r
62 import compbio.metadata.JobExecutionException;
\r
63 import compbio.metadata.JobStatus;
\r
64 import compbio.metadata.JobSubmissionException;
\r
65 import compbio.metadata.LimitsManager;
\r
66 import compbio.metadata.ResultNotAvailableException;
\r
67 import compbio.metadata.RunnerConfig;
\r
68 import compbio.runner.OptionCombinator;
\r
69 import compbio.util.FileWatcher;
\r
70 import compbio.util.SysPrefs;
\r
72 public class ClustalOTester {
\r
74 static final String clustalConfigFile = AllTestSuit.TEST_DATA_PATH + "ClustaloParameters.xml";
\r
75 public static String test_outfile = "TO1381.clustalo.out";
\r
76 public static String cluster_test_outfile = "TO1381.clustalo.cluster.out";
\r
78 @Test(groups = {AllTestSuit.test_group_runner})
\r
79 public void RunLocally() {
\r
80 ClustalO clustal = new ClustalO();
\r
81 clustal.setInput(AllTestSuit.test_input).setOutput(test_outfile);
\r
83 // For local execution use relavive
\r
84 ConfiguredExecutable<ClustalO> confClustal = Configurator.configureExecutable(clustal, Executable.ExecProvider.Local);
\r
85 LocalRunner lr = new LocalRunner(confClustal);
\r
87 confClustal = (ConfiguredExecutable<ClustalO>) lr.waitForResult();
\r
88 assertNotNull(confClustal.getResults());
\r
89 } catch (JobSubmissionException e) {
\r
90 e.printStackTrace();
\r
91 fail(e.getLocalizedMessage());
\r
92 } catch (JobExecutionException e) {
\r
93 e.printStackTrace();
\r
94 fail(e.getLocalizedMessage());
\r
95 } catch (ResultNotAvailableException e) {
\r
96 e.printStackTrace();
\r
97 fail(e.getLocalizedMessage());
\r
101 @Test(groups = {AllTestSuit.test_group_runner})
\r
102 public void ConfigurationLoading() {
\r
104 RunnerConfig<ClustalO> clustalConfig = ConfExecutable.getRunnerOptions(ClustalO.class);
\r
105 assertNotNull(clustalConfig);
\r
106 assertTrue(clustalConfig.getArguments().size() > 0);
\r
108 LimitsManager<ClustalO> clustalLimits = ConfExecutable.getRunnerLimits(ClustalO.class);
\r
109 assertNotNull(clustalLimits);
\r
110 assertTrue(clustalLimits.getLimits().size() > 0);
\r
111 } catch (FileNotFoundException e) {
\r
112 e.printStackTrace();
\r
113 fail(e.getLocalizedMessage());
\r
114 } catch (IOException e) {
\r
115 e.printStackTrace();
\r
116 fail(e.getLocalizedMessage());
\r
120 @Test(groups = {AllTestSuit.test_group_runner})
\r
121 public void OptionsLocally() {
\r
123 RunnerConfigMarshaller<ClustalO> clustalmarsh = new RunnerConfigMarshaller<ClustalO>(RunnerConfig.class);
\r
124 RunnerConfig<ClustalO> clustalConfig = clustalmarsh.read(new FileInputStream(new File(clustalConfigFile)), RunnerConfig.class);
\r
126 OptionCombinator clustalOpc = new OptionCombinator(clustalConfig);
\r
127 List<String> options = clustalOpc.getOptionsAtRandom();
\r
128 for (int i = 0; i < options.size(); i++) {
\r
129 System.out.println("Using options: " + options);
\r
130 ClustalO clustal = new ClustalO();
\r
131 clustal.setInput(AllTestSuit.test_input);
\r
132 clustal.setOutput(test_outfile);
\r
134 // For local execution use relavive
\r
135 ConfiguredExecutable<ClustalO> confClustal = Configurator.configureExecutable(clustal, ExecProvider.Local);
\r
137 // Add options to the executable
\r
138 confClustal.addParameters(options);
\r
140 LocalRunner lr = new LocalRunner(confClustal);
\r
142 confClustal = (ConfiguredExecutable<ClustalO>) lr.waitForResult();
\r
143 assertNotNull(confClustal.getResults());
\r
144 Collections.shuffle(options);
\r
147 } catch (JobSubmissionException e) {
\r
148 e.printStackTrace();
\r
149 fail(e.getLocalizedMessage());
\r
150 } catch (JobExecutionException e) {
\r
151 e.printStackTrace();
\r
152 fail(e.getLocalizedMessage());
\r
153 } catch (JAXBException e) {
\r
154 e.printStackTrace();
\r
155 fail(e.getLocalizedMessage());
\r
156 } catch (ResultNotAvailableException e) {
\r
157 e.printStackTrace();
\r
158 fail(e.getLocalizedMessage());
\r
159 } catch (FileNotFoundException e) {
\r
160 e.printStackTrace();
\r
161 fail(e.getLocalizedMessage());
\r
165 public static final void main(String[] args)
\r
166 throws JobSubmissionException, JobExecutionException, InterruptedException {
\r
167 ClustalO clustal = new ClustalO();
\r
168 clustal.setInput(AllTestSuit.test_input).setOutput(test_outfile);
\r
169 // For local execution use relavive
\r
170 ConfiguredExecutable<ClustalO> confClustal = Configurator.configureExecutable(clustal);
\r
171 AsyncExecutor lr = new AsyncLocalRunner();
\r
172 lr.submitJob(confClustal);
\r
173 Thread.sleep(3000);
\r
174 LocalExecutorService.shutDown();
\r
177 @Test(groups = {AllTestSuit.test_group_runner})
\r
178 public void Persistance() {
\r
180 ClustalO clustal = new ClustalO();
\r
181 clustal.setError("errrr.txt");
\r
182 clustal.setInput(AllTestSuit.test_input);
\r
183 clustal.setOutput("outtt.txt");
\r
184 assertEquals(clustal.getInput(), AllTestSuit.test_input);
\r
185 assertEquals(clustal.getError(), "errrr.txt");
\r
186 assertEquals(clustal.getOutput(), "outtt.txt");
\r
187 ConfiguredExecutable<ClustalO> cClustal = Configurator.configureExecutable(clustal, Executable.ExecProvider.Local);
\r
189 SyncExecutor sexec = Configurator.getSyncEngine(cClustal);
\r
190 sexec.executeJob();
\r
191 cClustal = (ConfiguredExecutable<ClustalO>) sexec.waitForResult();
\r
192 assertNotNull(cClustal.getResults());
\r
193 // Save run configuration
\r
194 assertTrue(cClustal.saveRunConfiguration());
\r
196 // See if loaded configuration is the same as saved
\r
197 RunConfiguration loadedRun = RunConfiguration.load(new FileInputStream(new File(cClustal.getWorkDirectory(), RunConfiguration.rconfigFile)));
\r
198 assertTrue(((ConfExecutable<ClustalO>) cClustal)
\r
199 .getRunConfiguration().equals(loadedRun));
\r
200 // Load run configuration as ConfExecutable
\r
201 ConfiguredExecutable<ClustalO> resurrectedCclustal = (ConfiguredExecutable<ClustalO>) cClustal
\r
202 .loadRunConfiguration(new FileInputStream(new File(cClustal
\r
203 .getWorkDirectory(), RunConfiguration.rconfigFile)));
\r
204 assertNotNull(resurrectedCclustal);
\r
205 // See in details whether executables are the same
\r
206 assertEquals(resurrectedCclustal.getExecutable(), clustal);
\r
208 // Finally rerun the job in the new task directory
\r
209 ConfiguredExecutable<ClustalO> resclustal = Configurator
\r
210 .configureExecutable(resurrectedCclustal.getExecutable(),
\r
211 Executable.ExecProvider.Local);
\r
213 sexec = Configurator.getSyncEngine(resclustal, Executable.ExecProvider.Local);
\r
214 sexec.executeJob();
\r
215 cClustal = (ConfiguredExecutable<ClustalO>) sexec.waitForResult();
\r
216 assertNotNull(cClustal.getResults());
\r
218 } catch (JobSubmissionException e) {
\r
219 e.printStackTrace();
\r
220 fail(e.getMessage());
\r
221 } catch (JobExecutionException e) {
\r
222 e.printStackTrace();
\r
223 fail(e.getMessage());
\r
224 } catch (FileNotFoundException e) {
\r
225 e.printStackTrace();
\r
226 fail(e.getMessage());
\r
227 } catch (IOException e) {
\r
228 e.printStackTrace();
\r
229 fail(e.getMessage());
\r
230 } catch (ResultNotAvailableException e) {
\r
231 e.printStackTrace();
\r
232 fail(e.getMessage());
\r
236 @Test(groups = {AllTestSuit.test_group_runner})
\r
237 public void readStatistics()
\r
238 throws InterruptedException {
\r
240 ClustalO al = new ClustalO();
\r
241 al.setInput(AllTestSuit.test_input);
\r
242 al.setOutput(test_outfile);
\r
243 ConfiguredExecutable<ClustalO> confal = Configurator.configureExecutable(al, Executable.ExecProvider.Local);
\r
245 AsyncExecutor sexec = Configurator.getAsyncEngine(confal);
\r
246 String jobId = sexec.submitJob(confal);
\r
247 String file = Util.getFullPath(confal.getWorkDirectory(), ClustalW.getStatFile());
\r
248 FilePuller fw = FilePuller.newFilePuller(file, FileWatcher.MIN_CHUNK_SIZE_BYTES);
\r
252 JobStatus status = sexec.getJobStatus(jobId);
\r
253 while (status != JobStatus.FINISHED || fw.hasMoreData()) {
\r
254 if (fw.hasMoreData()) {
\r
255 ChunkHolder ch = fw.pull(position);
\r
256 String chunk = ch.getChunk();
\r
257 position = ch.getNextPosition();
\r
260 if ((status == JobStatus.UNDEFINED || status == JobStatus.FAILED)) {
\r
261 fail("job " + jobId +" failed!");
\r
265 status = sexec.getJobStatus(jobId);
\r
266 System.out.println("CLustalO: Job status = " + status + ", file status = " + fw.hasMoreData());
\r
268 assertTrue(count > 1);
\r
269 ConfiguredExecutable<?> al2 = sexec.getResults(jobId);
\r
270 assertNotNull(al2.getResults());
\r
271 } catch (JobSubmissionException e) {
\r
272 e.printStackTrace();
\r
273 fail(e.getMessage());
\r
274 } catch (ResultNotAvailableException e) {
\r
275 e.printStackTrace();
\r
276 fail(e.getMessage());
\r
277 } catch (IOException e) {
\r
278 e.printStackTrace();
\r
279 fail(e.getMessage());
\r
283 @Test(groups = {AllTestSuit.test_group_cluster, AllTestSuit.test_group_runner})
\r
284 public void RunOnCluster() {
\r
285 ClustalO clustal = new ClustalO();
\r
286 assertFalse(SysPrefs.isWindows, "Cluster execution can only be in unix environment");
\r
287 clustal.setInput(AllTestSuit.test_input).setOutput(cluster_test_outfile);
\r
289 ConfiguredExecutable<ClustalO> confClustal = Configurator.configureExecutable(clustal);
\r
290 JobRunner runner = JobRunner.getInstance(confClustal);
\r
291 // ClusterSession csession = JobRunner.getSession();
\r
292 assertNotNull(runner);
\r
293 runner.executeJob();
\r
294 // assertNotNull("JobId is null", jobId1);
\r
295 JobStatus status = runner.getJobStatus();
\r
296 assertTrue(status == JobStatus.PENDING || status == JobStatus.RUNNING);
\r
297 JobInfo info = runner.getJobInfo();
\r
298 assertNotNull(info);
\r
299 StatisticManager sm = new StatisticManager(info);
\r
302 String exits = sm.getExitStatus();
\r
303 assertNotNull("Exit status is null", exits);
\r
304 // cut 4 trailing zeros from the number
\r
305 int exitsInt = ClusterUtil.CLUSTER_STAT_IN_SEC.parse(exits).intValue();
\r
306 assertEquals(0, exitsInt);
\r
307 System.out.println(sm.getAllStats());
\r
308 } catch (ParseException e) {
\r
309 e.printStackTrace();
\r
310 fail("Parse Exception: " + e.getMessage());
\r
312 // At present the task directory could not be completely removed
\r
313 // @see JobRunner.cleanup()
\r
314 assertFalse(runner.cleanup(), "Could not remove some files whilst cleaning up ");
\r
315 assertTrue(sm.hasExited());
\r
316 assertFalse(sm.wasAborted());
\r
317 assertFalse(sm.hasDump());
\r
318 assertFalse(sm.hasSignaled());
\r
319 } catch (JobSubmissionException e) {
\r
320 e.printStackTrace();
\r
321 fail("DrmaaException caught:" + e.getMessage());
\r
322 } catch (JobExecutionException e) {
\r
323 e.printStackTrace();
\r
324 fail("DrmaaException caught:" + e.getMessage());
\r
325 } catch (DrmaaException e) {
\r
326 e.printStackTrace();
\r
327 fail("DrmaaException caught:" + e.getMessage());
\r
331 @Test(groups = {AllTestSuit.test_group_cluster, AllTestSuit.test_group_runner})
\r
332 public void readStatisticsClusterExecution() {
\r
334 ClustalO clustal = new ClustalO().setInput(AllTestSuit.test_input).setOutput(test_outfile);
\r
335 ConfiguredExecutable<ClustalO> confClustal = Configurator.configureExecutable(clustal, Executable.ExecProvider.Cluster);
\r
337 AsyncExecutor sexec = Configurator.getAsyncEngine(confClustal);
\r
338 String jobId = sexec.submitJob(confClustal);
\r
339 String file = Util.getFullPath(confClustal.getWorkDirectory(), ClustalW.getStatFile());
\r
340 FilePuller fw = FilePuller.newFilePuller(file, FileWatcher.MIN_CHUNK_SIZE_BYTES);
\r
343 int maxloopcount = 108000; // max waiting time = 6h*60m*60s/0.2(one loop sleep)
\r
344 fw.waitForFile(200);
\r
346 * Under certain circumstances DRMAA could report the status wrongly
\r
347 * thus this loop never ends. maxloopcount ensures hard stop of the loop...
\r
349 while (!( sexec.getJobStatus(jobId) == JobStatus.FINISHED || sexec.getJobStatus(jobId) == JobStatus.FAILED)
\r
350 || count < maxloopcount || fw.hasMoreData()) {
\r
351 ChunkHolder ch = fw.pull(position);
\r
352 String chunk = ch.getChunk();
\r
353 position = ch.getNextPosition();
\r
354 System.out.print(chunk);
\r
357 if (sexec.getJobStatus(jobId) == JobStatus.UNDEFINED) {
\r
358 System.out.println("DRMAA reported wrong status for job + " + jobId + " continue anyway!");
\r
362 assertTrue(count > 1);
\r
363 ConfiguredExecutable<?> al = sexec.getResults(jobId);
\r
364 assertNotNull(al.getResults());
\r
365 } catch (JobSubmissionException e) {
\r
366 e.printStackTrace();
\r
367 fail(e.getMessage());
\r
368 } catch (ResultNotAvailableException e) {
\r
369 e.printStackTrace();
\r
370 fail(e.getMessage());
\r
371 } catch (IOException e) {
\r
372 e.printStackTrace();
\r
373 fail(e.getMessage());
\r
374 } catch (InterruptedException e) {
\r
375 e.printStackTrace();
\r
376 fail(e.getMessage());
\r