2 * Copyright (c) 2010 Peter Troshin JAva Bioinformatics Analysis Web Services
\r
3 * (JABAWS) @version: 2.0
\r
5 * This library is free software; you can redistribute it and/or modify it under
\r
6 * the terms of the Apache License version 2 as published
\r
7 * by the Apache Software Foundation This library is distributed in the hope
\r
8 * that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
\r
9 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
\r
10 * Apache License for more details. A copy of the license is in
\r
11 * apache_license.txt. It is also available here:
\r
13 * @see: http://www.apache.org/licenses/LICENSE-2.0.txt
\r
15 * Any republication or derived work distributed in source code form must include
\r
16 * this copyright and license notice.
\r
18 package compbio.runner.conservation;
\r
20 import static org.testng.Assert.assertEquals;
\r
21 import static org.testng.Assert.assertFalse;
\r
22 import static org.testng.Assert.assertNotNull;
\r
23 import static org.testng.Assert.assertTrue;
\r
24 import static org.testng.Assert.fail;
\r
26 import java.io.File;
\r
27 import java.io.FileInputStream;
\r
28 import java.io.FileNotFoundException;
\r
29 import java.io.IOException;
\r
30 import java.text.ParseException;
\r
31 import java.util.Arrays;
\r
32 import java.util.HashSet;
\r
34 import javax.xml.bind.ValidationException;
\r
36 import org.ggf.drmaa.DrmaaException;
\r
37 import org.ggf.drmaa.JobInfo;
\r
38 import org.testng.annotations.BeforeMethod;
\r
39 import org.testng.annotations.Test;
\r
41 import compbio.data.sequence.Score;
\r
42 import compbio.data.sequence.ScoreManager;
\r
43 import compbio.engine.AsyncExecutor;
\r
44 import compbio.engine.Configurator;
\r
45 import compbio.engine.FilePuller;
\r
46 import compbio.engine.SyncExecutor;
\r
47 import compbio.engine.client.ConfExecutable;
\r
48 import compbio.engine.client.ConfiguredExecutable;
\r
49 import compbio.engine.client.Executable;
\r
50 import compbio.engine.client.RunConfiguration;
\r
51 import compbio.engine.cluster.drmaa.ClusterUtil;
\r
52 import compbio.engine.cluster.drmaa.JobRunner;
\r
53 import compbio.engine.cluster.drmaa.StatisticManager;
\r
54 import compbio.engine.local.LocalRunner;
\r
55 import compbio.metadata.ChunkHolder;
\r
56 import compbio.metadata.JobExecutionException;
\r
57 import compbio.metadata.JobStatus;
\r
58 import compbio.metadata.JobSubmissionException;
\r
59 import compbio.metadata.LimitsManager;
\r
60 import compbio.metadata.Preset;
\r
61 import compbio.metadata.PresetManager;
\r
62 import compbio.metadata.ResultNotAvailableException;
\r
63 import compbio.metadata.RunnerConfig;
\r
64 import compbio.runner.Util;
\r
65 import compbio.util.FileWatcher;
\r
66 import compbio.util.SysPrefs;
\r
68 public class AAConTester {
\r
70 public static final String CURRENT_DIRECTORY = SysPrefs
\r
71 .getCurrentDirectory() + File.separator;
\r
73 public static String test_outfile = "TO1381.aacon.out"; // "/homes/pvtroshin/TO1381.clustal.cluster.out
\r
74 public static String test_alignment_input = CURRENT_DIRECTORY + "testsrc"
\r
75 + File.separator + "testdata" + File.separator + "TO1381.fasta.aln";
\r
76 private AACon aacon;
\r
78 @BeforeMethod(alwaysRun = true)
\r
80 aacon = new AACon();
\r
81 aacon.setInput(test_alignment_input).setOutput(test_outfile);
\r
85 public void testRunOnCluster() {
\r
86 assertFalse(SysPrefs.isWindows,
\r
87 "Cluster execution can only be in unix environment");
\r
89 ConfiguredExecutable<AACon> confAAcon = Configurator
\r
90 .configureExecutable(aacon, Executable.ExecProvider.Cluster);
\r
91 JobRunner runner = JobRunner.getInstance(confAAcon);
\r
93 assertNotNull(runner, "Runner is NULL");
\r
94 runner.executeJob();
\r
95 // assertNotNull("JobId is null", jobId1);
\r
96 JobStatus status = runner.getJobStatus();
\r
97 assertTrue(status == JobStatus.PENDING
\r
98 || status == JobStatus.RUNNING,
\r
99 "Status of the process is wrong!");
\r
100 JobInfo info = runner.getJobInfo();
\r
101 assertNotNull(info, "JobInfo is null");
\r
102 StatisticManager sm = new StatisticManager(info);
\r
103 assertNotNull(sm, "Statictic manager is null");
\r
106 String exits = sm.getExitStatus();
\r
107 assertNotNull("Exit status is null", exits);
\r
108 // cut 4 trailing zeros from the number
\r
109 int exitsInt = ClusterUtil.CLUSTER_STAT_IN_SEC.parse(exits)
\r
111 assertEquals(0, exitsInt);
\r
112 System.out.println(sm.getAllStats());
\r
114 } catch (ParseException e) {
\r
115 e.printStackTrace();
\r
116 fail("Parse Exception: " + e.getMessage());
\r
118 // assertFalse(runner.cleanup());
\r
119 assertTrue(sm.hasExited());
\r
120 assertFalse(sm.wasAborted());
\r
121 assertFalse(sm.hasDump());
\r
122 assertFalse(sm.hasSignaled());
\r
124 } catch (JobSubmissionException e) {
\r
125 e.printStackTrace();
\r
126 fail("DrmaaException caught:" + e.getMessage());
\r
127 } catch (JobExecutionException e) {
\r
128 e.printStackTrace();
\r
129 fail("DrmaaException caught:" + e.getMessage());
\r
130 } catch (DrmaaException e) {
\r
131 e.printStackTrace();
\r
132 fail("DrmaaException caught:" + e.getMessage());
\r
137 * This tests fails from time to time depending on the cluster load or some
\r
138 * other factors. Any client code has to adjust for this issue
\r
141 public void testRunOnClusterAsync() {
\r
142 assertFalse(SysPrefs.isWindows,
\r
143 "Cluster execution can only be in unix environment");
\r
145 ConfiguredExecutable<AACon> confAAcon = Configurator
\r
146 .configureExecutable(aacon, Executable.ExecProvider.Cluster);
\r
147 AsyncExecutor aengine = Configurator.getAsyncEngine(confAAcon);
\r
148 String jobId = aengine.submitJob(confAAcon);
\r
149 assertNotNull(jobId, "Runner is NULL");
\r
150 // let drmaa to start
\r
152 JobStatus status = aengine.getJobStatus(jobId);
\r
153 while (status != JobStatus.FINISHED) {
\r
154 System.out.println("Job Status: " + status);
\r
155 Thread.sleep(1000);
\r
156 status = aengine.getJobStatus(jobId);
\r
157 ConfiguredExecutable<AACon> result = (ConfiguredExecutable<AACon>) aengine
\r
158 .getResults(jobId);
\r
159 assertNotNull(result);
\r
160 System.out.println("RES:" + result);
\r
161 // Some times the job could be removed from the cluster
\r
163 // before it has been reported to finish. Make sure
\r
164 // to stop waiting in such case
\r
165 if (status == JobStatus.UNDEFINED) {
\r
169 } catch (JobSubmissionException e) {
\r
170 e.printStackTrace();
\r
171 fail("DrmaaException caught:" + e.getMessage());
\r
172 } catch (InterruptedException e) {
\r
173 e.printStackTrace();
\r
174 fail(e.getMessage());
\r
175 } catch (ResultNotAvailableException e) {
\r
176 e.printStackTrace();
\r
177 fail(e.getMessage());
\r
182 public void testRunLocally() {
\r
184 ConfiguredExecutable<AACon> confAAcon = Configurator
\r
185 .configureExecutable(aacon, Executable.ExecProvider.Local);
\r
187 // For local execution use relative
\r
188 LocalRunner lr = new LocalRunner(confAAcon);
\r
190 ConfiguredExecutable<?> al1 = lr.waitForResult();
\r
191 assertNotNull(al1.getResults());
\r
192 ScoreManager annotations = confAAcon.getResults();
\r
193 assertNotNull(annotations);
\r
194 assertEquals(annotations.asSet().size(), 18);
\r
195 assertEquals(al1.getResults(), annotations);
\r
196 } catch (JobSubmissionException e) {
\r
197 e.printStackTrace();
\r
198 fail(e.getLocalizedMessage());
\r
199 } catch (ResultNotAvailableException e) {
\r
200 e.printStackTrace();
\r
201 fail(e.getLocalizedMessage());
\r
202 } catch (JobExecutionException e) {
\r
203 e.printStackTrace();
\r
204 fail(e.getLocalizedMessage());
\r
209 public void testRunLocallyWithPreset() {
\r
211 PresetManager<AACon> aaconPresets = Util.getPresets(AACon.class);
\r
212 assert aaconPresets != null;
\r
213 ConfiguredExecutable<AACon> confAAcon = Configurator
\r
214 .configureExecutable(aacon, Executable.ExecProvider.Local);
\r
215 Preset<AACon> quick = aaconPresets
\r
216 .getPresetByName("Quick conservation");
\r
217 confAAcon.addParameters(quick.getOptions());
\r
218 // For local execution use relative
\r
219 LocalRunner lr = new LocalRunner(confAAcon);
\r
221 ConfiguredExecutable<?> al1 = lr.waitForResult();
\r
222 assertNotNull(al1.getResults());
\r
223 ScoreManager annotations = confAAcon.getResults();
\r
224 assertNotNull(annotations);
\r
225 assertEquals(annotations.asSet().size(), 13);
\r
226 assertEquals(al1.getResults(), annotations);
\r
227 } catch (JobSubmissionException e) {
\r
228 e.printStackTrace();
\r
229 fail(e.getLocalizedMessage());
\r
230 } catch (ResultNotAvailableException e) {
\r
231 e.printStackTrace();
\r
232 fail(e.getLocalizedMessage());
\r
233 } catch (JobExecutionException e) {
\r
234 e.printStackTrace();
\r
235 fail(e.getLocalizedMessage());
\r
240 public void testRunLocallyOnTwoCpu() {
\r
242 aacon = new AACon();
\r
243 aacon.setInput(test_alignment_input).setOutput(test_outfile);
\r
246 ConfiguredExecutable<AACon> confAAcon = Configurator
\r
247 .configureExecutable(aacon, Executable.ExecProvider.Local);
\r
248 confAAcon.addParameters(Arrays.asList("-m=KABAT,JORES"));
\r
249 confAAcon.addParameters(Arrays.asList("-m=TAYLOR_GAPS"));
\r
250 // For local execution use relative
\r
251 LocalRunner lr = new LocalRunner(confAAcon);
\r
254 ConfiguredExecutable<?> al1 = lr.waitForResult();
\r
255 assertNotNull(al1.getResults());
\r
256 ScoreManager annotations = confAAcon.getResults();
\r
257 assertNotNull(annotations);
\r
258 assertEquals(annotations.asSet().size(), 3);
\r
259 assertEquals(al1.getResults(), annotations);
\r
261 } catch (JobSubmissionException e) {
\r
262 e.printStackTrace();
\r
263 fail(e.getLocalizedMessage());
\r
264 } catch (ResultNotAvailableException e) {
\r
265 e.printStackTrace();
\r
266 fail(e.getLocalizedMessage());
\r
267 } catch (JobExecutionException e) {
\r
268 e.printStackTrace();
\r
269 fail(e.getLocalizedMessage());
\r
273 public void readStatistics() {
\r
275 ConfiguredExecutable<AACon> confAAcon = Configurator
\r
276 .configureExecutable(aacon, Executable.ExecProvider.Local);
\r
277 // For local execution use relative
\r
279 AsyncExecutor sexec = Configurator.getAsyncEngine(confAAcon);
\r
280 String jobId = sexec.submitJob(confAAcon);
\r
281 FilePuller fw = FilePuller.newFilePuller(
\r
282 confAAcon.getWorkDirectory() + File.separator
\r
283 + AACon.getStatFile(),
\r
284 FileWatcher.MIN_CHUNK_SIZE_BYTES);
\r
288 JobStatus status = sexec.getJobStatus(jobId);
\r
290 ChunkHolder ch = fw.pull(position);
\r
291 String chunk = ch.getChunk();
\r
292 position = ch.getNextPosition();
\r
293 // System.out.println(chunk);
\r
295 // Make sure the loop is terminated if the job fails
\r
296 if ((status == JobStatus.UNDEFINED || status == JobStatus.FAILED)) {
\r
297 fail("job failed!");
\r
301 status = sexec.getJobStatus(jobId);
\r
302 } while (status != JobStatus.FINISHED || fw.hasMoreData());
\r
304 assertTrue(count >= 1);
\r
305 ConfiguredExecutable<?> al = sexec.getResults(jobId);
\r
306 assertNotNull(al.getResults());
\r
307 } catch (JobSubmissionException e) {
\r
308 e.printStackTrace();
\r
309 fail(e.getMessage());
\r
310 } catch (ResultNotAvailableException e) {
\r
311 e.printStackTrace();
\r
312 fail(e.getMessage());
\r
313 } catch (IOException e) {
\r
314 e.printStackTrace();
\r
315 fail(e.getMessage());
\r
316 } catch (InterruptedException e) {
\r
317 e.printStackTrace();
\r
318 fail(e.getMessage());
\r
323 public void testPersistance() {
\r
325 AACon aacon = new AACon();
\r
326 aacon.setError("errrr.txt").setInput(test_alignment_input)
\r
327 .setOutput("outtt.txt");
\r
328 assertEquals(aacon.getInput(), test_alignment_input);
\r
329 assertEquals(aacon.getError(), "errrr.txt");
\r
330 assertEquals(aacon.getOutput(), "outtt.txt");
\r
331 ConfiguredExecutable<AACon> cAAcon = Configurator
\r
332 .configureExecutable(aacon, Executable.ExecProvider.Local);
\r
334 SyncExecutor sexec = Configurator.getSyncEngine(cAAcon);
\r
335 sexec.executeJob();
\r
336 ConfiguredExecutable<?> al = sexec.waitForResult();
\r
337 assertNotNull(al.getResults());
\r
338 // Save run configuration
\r
339 assertTrue(cAAcon.saveRunConfiguration());
\r
341 // See if loaded configuration is the same as saved
\r
342 RunConfiguration loadedRun = RunConfiguration
\r
343 .load(new FileInputStream(new File(cAAcon
\r
344 .getWorkDirectory(), RunConfiguration.rconfigFile)));
\r
346 ((ConfExecutable<AACon>) cAAcon).getRunConfiguration(),
\r
348 // Load run configuration as ConfExecutable
\r
349 ConfiguredExecutable<AACon> resurrectedCAAcon = (ConfiguredExecutable<AACon>) cAAcon
\r
350 .loadRunConfiguration(new FileInputStream(new File(cAAcon
\r
351 .getWorkDirectory(), RunConfiguration.rconfigFile)));
\r
352 assertNotNull(resurrectedCAAcon);
\r
353 assertEquals(resurrectedCAAcon.getExecutable().getInput(),
\r
354 test_alignment_input);
\r
355 assertEquals(resurrectedCAAcon.getExecutable().getError(),
\r
357 assertEquals(resurrectedCAAcon.getExecutable().getOutput(),
\r
359 // See in details whether executables are the same
\r
360 assertEquals(resurrectedCAAcon.getExecutable(), aacon);
\r
362 ConfiguredExecutable<AACon> resAAcon = Configurator
\r
363 .configureExecutable(resurrectedCAAcon.getExecutable(),
\r
364 Executable.ExecProvider.Local);
\r
366 sexec = Configurator.getSyncEngine(resAAcon,
\r
367 Executable.ExecProvider.Local);
\r
368 sexec.executeJob();
\r
369 al = sexec.waitForResult();
\r
372 } catch (JobSubmissionException e) {
\r
373 e.printStackTrace();
\r
374 fail(e.getMessage());
\r
375 } catch (JobExecutionException e) {
\r
376 e.printStackTrace();
\r
377 fail(e.getMessage());
\r
378 } catch (FileNotFoundException e) {
\r
379 e.printStackTrace();
\r
380 fail(e.getMessage());
\r
381 } catch (IOException e) {
\r
382 e.printStackTrace();
\r
383 fail(e.getMessage());
\r
384 } catch (ResultNotAvailableException e) {
\r
385 e.printStackTrace();
\r
386 fail(e.getMessage());
\r
391 public void testConfigurationLoading() {
\r
393 RunnerConfig<AACon> aaconConfig = ConfExecutable
\r
394 .getRunnerOptions(AACon.class);
\r
395 assertNotNull(aaconConfig);
\r
396 assertTrue(aaconConfig.getArguments().size() > 0);
\r
398 PresetManager<AACon> aaconPresets = ConfExecutable
\r
399 .getRunnerPresets(AACon.class);
\r
400 assertNotNull(aaconPresets);
\r
402 LimitsManager<AACon> jronnLimits = ConfExecutable
\r
403 .getRunnerLimits(AACon.class);
\r
404 assertNotNull(jronnLimits);
\r
405 assertTrue(jronnLimits.getLimits().size() > 0);
\r
406 jronnLimits.validate(aaconPresets);
\r
408 } catch (FileNotFoundException e) {
\r
409 e.printStackTrace();
\r
410 fail(e.getLocalizedMessage());
\r
411 } catch (IOException e) {
\r
412 e.printStackTrace();
\r
413 fail(e.getLocalizedMessage());
\r
414 } catch (ValidationException e) {
\r
415 e.printStackTrace();
\r
416 fail(e.getLocalizedMessage());
\r