2 * Copyright (c) 2010 Peter Troshin JAva Bioinformatics Analysis Web Services
\r
3 * (JABAWS) @version: 2.0
\r
5 * This library is free software; you can redistribute it and/or modify it under
\r
6 * the terms of the Apache License version 2 as published
\r
7 * by the Apache Software Foundation This library is distributed in the hope
\r
8 * that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
\r
9 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
\r
10 * Apache License for more details. A copy of the license is in
\r
11 * apache_license.txt. It is also available here:
\r
13 * @see: http://www.apache.org/licenses/LICENSE-2.0.txt
\r
15 * Any republication or derived work distributed in source code form must include
\r
16 * this copyright and license notice.
\r
18 package compbio.runner.conservation;
\r
20 import static org.testng.Assert.assertEquals;
\r
21 import static org.testng.Assert.assertFalse;
\r
22 import static org.testng.Assert.assertNotNull;
\r
23 import static org.testng.Assert.assertTrue;
\r
24 import static org.testng.Assert.fail;
\r
26 import java.io.File;
\r
27 import java.io.FileInputStream;
\r
28 import java.io.FileNotFoundException;
\r
29 import java.io.IOException;
\r
30 import java.text.ParseException;
\r
31 import java.util.Arrays;
\r
32 import java.util.HashSet;
\r
34 import javax.xml.bind.ValidationException;
\r
36 import org.ggf.drmaa.DrmaaException;
\r
37 import org.ggf.drmaa.JobInfo;
\r
38 import org.testng.annotations.BeforeMethod;
\r
39 import org.testng.annotations.Test;
\r
41 import compbio.data.sequence.Score;
\r
42 import compbio.engine.AsyncExecutor;
\r
43 import compbio.engine.Configurator;
\r
44 import compbio.engine.FilePuller;
\r
45 import compbio.engine.SyncExecutor;
\r
46 import compbio.engine.client.ConfExecutable;
\r
47 import compbio.engine.client.ConfiguredExecutable;
\r
48 import compbio.engine.client.Executable;
\r
49 import compbio.engine.client.RunConfiguration;
\r
50 import compbio.engine.cluster.drmaa.ClusterUtil;
\r
51 import compbio.engine.cluster.drmaa.JobRunner;
\r
52 import compbio.engine.cluster.drmaa.StatisticManager;
\r
53 import compbio.engine.local.LocalRunner;
\r
54 import compbio.metadata.ChunkHolder;
\r
55 import compbio.metadata.JobExecutionException;
\r
56 import compbio.metadata.JobStatus;
\r
57 import compbio.metadata.JobSubmissionException;
\r
58 import compbio.metadata.LimitsManager;
\r
59 import compbio.metadata.Preset;
\r
60 import compbio.metadata.PresetManager;
\r
61 import compbio.metadata.ResultNotAvailableException;
\r
62 import compbio.metadata.RunnerConfig;
\r
63 import compbio.runner.Util;
\r
64 import compbio.util.FileWatcher;
\r
65 import compbio.util.SysPrefs;
\r
67 public class AAConTester {
\r
69 public static final String CURRENT_DIRECTORY = SysPrefs
\r
70 .getCurrentDirectory() + File.separator;
\r
72 public static String test_outfile = "TO1381.aacon.out"; // "/homes/pvtroshin/TO1381.clustal.cluster.out
\r
73 public static String test_alignment_input = CURRENT_DIRECTORY + "testsrc"
\r
74 + File.separator + "testdata" + File.separator + "TO1381.fasta.aln";
\r
75 private AACon aacon;
\r
77 @BeforeMethod(alwaysRun = true)
\r
79 aacon = new AACon();
\r
80 aacon.setInput(test_alignment_input).setOutput(test_outfile);
\r
84 public void testRunOnCluster() {
\r
85 assertFalse(SysPrefs.isWindows,
\r
86 "Cluster execution can only be in unix environment");
\r
88 ConfiguredExecutable<AACon> confAAcon = Configurator
\r
89 .configureExecutable(aacon, Executable.ExecProvider.Cluster);
\r
90 JobRunner runner = JobRunner.getInstance(confAAcon);
\r
92 assertNotNull(runner, "Runner is NULL");
\r
93 runner.executeJob();
\r
94 // assertNotNull("JobId is null", jobId1);
\r
95 JobStatus status = runner.getJobStatus();
\r
96 assertTrue(status == JobStatus.PENDING
\r
97 || status == JobStatus.RUNNING,
\r
98 "Status of the process is wrong!");
\r
99 JobInfo info = runner.getJobInfo();
\r
100 assertNotNull(info, "JobInfo is null");
\r
101 StatisticManager sm = new StatisticManager(info);
\r
102 assertNotNull(sm, "Statictic manager is null");
\r
105 String exits = sm.getExitStatus();
\r
106 assertNotNull("Exit status is null", exits);
\r
107 // cut 4 trailing zeros from the number
\r
108 int exitsInt = ClusterUtil.CLUSTER_STAT_IN_SEC.parse(exits)
\r
110 assertEquals(0, exitsInt);
\r
111 System.out.println(sm.getAllStats());
\r
113 } catch (ParseException e) {
\r
114 e.printStackTrace();
\r
115 fail("Parse Exception: " + e.getMessage());
\r
117 // assertFalse(runner.cleanup());
\r
118 assertTrue(sm.hasExited());
\r
119 assertFalse(sm.wasAborted());
\r
120 assertFalse(sm.hasDump());
\r
121 assertFalse(sm.hasSignaled());
\r
123 } catch (JobSubmissionException e) {
\r
124 e.printStackTrace();
\r
125 fail("DrmaaException caught:" + e.getMessage());
\r
126 } catch (JobExecutionException e) {
\r
127 e.printStackTrace();
\r
128 fail("DrmaaException caught:" + e.getMessage());
\r
129 } catch (DrmaaException e) {
\r
130 e.printStackTrace();
\r
131 fail("DrmaaException caught:" + e.getMessage());
\r
136 * This tests fails from time to time depending on the cluster load or some
\r
137 * other factors. Any client code has to adjust for this issue
\r
140 public void testRunOnClusterAsync() {
\r
141 assertFalse(SysPrefs.isWindows,
\r
142 "Cluster execution can only be in unix environment");
\r
144 ConfiguredExecutable<AACon> confAAcon = Configurator
\r
145 .configureExecutable(aacon, Executable.ExecProvider.Cluster);
\r
146 AsyncExecutor aengine = Configurator.getAsyncEngine(confAAcon);
\r
147 String jobId = aengine.submitJob(confAAcon);
\r
148 assertNotNull(jobId, "Runner is NULL");
\r
149 // let drmaa to start
\r
151 JobStatus status = aengine.getJobStatus(jobId);
\r
152 while (status != JobStatus.FINISHED) {
\r
153 System.out.println("Job Status: " + status);
\r
154 Thread.sleep(1000);
\r
155 status = aengine.getJobStatus(jobId);
\r
156 ConfiguredExecutable<AACon> result = (ConfiguredExecutable<AACon>) aengine
\r
157 .getResults(jobId);
\r
158 assertNotNull(result);
\r
159 System.out.println("RES:" + result);
\r
160 // Some times the job could be removed from the cluster
\r
162 // before it has been reported to finish. Make sure
\r
163 // to stop waiting in such case
\r
164 if (status == JobStatus.UNDEFINED) {
\r
168 } catch (JobSubmissionException e) {
\r
169 e.printStackTrace();
\r
170 fail("DrmaaException caught:" + e.getMessage());
\r
171 } catch (InterruptedException e) {
\r
172 e.printStackTrace();
\r
173 fail(e.getMessage());
\r
174 } catch (ResultNotAvailableException e) {
\r
175 e.printStackTrace();
\r
176 fail(e.getMessage());
\r
181 public void testRunLocally() {
\r
183 ConfiguredExecutable<AACon> confAAcon = Configurator
\r
184 .configureExecutable(aacon, Executable.ExecProvider.Local);
\r
186 // For local execution use relative
\r
187 LocalRunner lr = new LocalRunner(confAAcon);
\r
189 ConfiguredExecutable<?> al1 = lr.waitForResult();
\r
190 assertNotNull(al1.getResults());
\r
191 HashSet<Score> annotations = confAAcon.getResults();
\r
192 assertNotNull(annotations);
\r
193 assertEquals(annotations.size(), 1);
\r
194 assertEquals(al1.getResults(), annotations);
\r
195 } catch (JobSubmissionException e) {
\r
196 e.printStackTrace();
\r
197 fail(e.getLocalizedMessage());
\r
198 } catch (ResultNotAvailableException e) {
\r
199 e.printStackTrace();
\r
200 fail(e.getLocalizedMessage());
\r
201 } catch (JobExecutionException e) {
\r
202 e.printStackTrace();
\r
203 fail(e.getLocalizedMessage());
\r
208 public void testRunLocallyWithPreset() {
\r
210 PresetManager<AACon> aaconPresets = Util.getPresets(AACon.class);
\r
211 assert aaconPresets != null;
\r
212 ConfiguredExecutable<AACon> confAAcon = Configurator
\r
213 .configureExecutable(aacon, Executable.ExecProvider.Local);
\r
214 Preset<AACon> quick = aaconPresets
\r
215 .getPresetByName("Quick conservation");
\r
216 confAAcon.addParameters(quick.getOptions());
\r
217 // For local execution use relative
\r
218 LocalRunner lr = new LocalRunner(confAAcon);
\r
220 ConfiguredExecutable<?> al1 = lr.waitForResult();
\r
221 assertNotNull(al1.getResults());
\r
222 HashSet<Score> annotations = confAAcon.getResults();
\r
223 assertNotNull(annotations);
\r
224 assertEquals(annotations.size(), 13);
\r
225 assertEquals(al1.getResults(), annotations);
\r
226 } catch (JobSubmissionException e) {
\r
227 e.printStackTrace();
\r
228 fail(e.getLocalizedMessage());
\r
229 } catch (ResultNotAvailableException e) {
\r
230 e.printStackTrace();
\r
231 fail(e.getLocalizedMessage());
\r
232 } catch (JobExecutionException e) {
\r
233 e.printStackTrace();
\r
234 fail(e.getLocalizedMessage());
\r
239 public void testRunLocallyOnTwoCpu() {
\r
241 aacon = new AACon();
\r
242 aacon.setInput(test_alignment_input).setOutput(test_outfile);
\r
245 ConfiguredExecutable<AACon> confAAcon = Configurator
\r
246 .configureExecutable(aacon, Executable.ExecProvider.Local);
\r
247 confAAcon.addParameters(Arrays.asList("-m=KABAT,JORES"));
\r
248 confAAcon.addParameters(Arrays.asList("-m=TAYLOR_GAPS"));
\r
249 // For local execution use relative
\r
250 LocalRunner lr = new LocalRunner(confAAcon);
\r
253 ConfiguredExecutable<?> al1 = lr.waitForResult();
\r
254 assertNotNull(al1.getResults());
\r
255 HashSet<Score> annotations = confAAcon.getResults();
\r
256 assertNotNull(annotations);
\r
257 assertEquals(annotations.size(), 3);
\r
258 assertEquals(al1.getResults(), annotations);
\r
260 } catch (JobSubmissionException e) {
\r
261 e.printStackTrace();
\r
262 fail(e.getLocalizedMessage());
\r
263 } catch (ResultNotAvailableException e) {
\r
264 e.printStackTrace();
\r
265 fail(e.getLocalizedMessage());
\r
266 } catch (JobExecutionException e) {
\r
267 e.printStackTrace();
\r
268 fail(e.getLocalizedMessage());
\r
272 public void readStatistics() {
\r
274 ConfiguredExecutable<AACon> confAAcon = Configurator
\r
275 .configureExecutable(aacon, Executable.ExecProvider.Local);
\r
276 // For local execution use relative
\r
278 AsyncExecutor sexec = Configurator.getAsyncEngine(confAAcon);
\r
279 String jobId = sexec.submitJob(confAAcon);
\r
280 FilePuller fw = FilePuller.newFilePuller(
\r
281 confAAcon.getWorkDirectory() + File.separator
\r
282 + AACon.getStatFile(),
\r
283 FileWatcher.MIN_CHUNK_SIZE_BYTES);
\r
287 JobStatus status = sexec.getJobStatus(jobId);
\r
289 ChunkHolder ch = fw.pull(position);
\r
290 String chunk = ch.getChunk();
\r
291 position = ch.getNextPosition();
\r
292 // System.out.println(chunk);
\r
294 // Make sure the loop is terminated if the job fails
\r
295 if ((status == JobStatus.UNDEFINED || status == JobStatus.FAILED)) {
\r
296 fail("job failed!");
\r
300 status = sexec.getJobStatus(jobId);
\r
301 } while (status != JobStatus.FINISHED || fw.hasMoreData());
\r
303 assertTrue(count >= 1);
\r
304 ConfiguredExecutable<?> al = sexec.getResults(jobId);
\r
305 assertNotNull(al.getResults());
\r
306 } catch (JobSubmissionException e) {
\r
307 e.printStackTrace();
\r
308 fail(e.getMessage());
\r
309 } catch (ResultNotAvailableException e) {
\r
310 e.printStackTrace();
\r
311 fail(e.getMessage());
\r
312 } catch (IOException e) {
\r
313 e.printStackTrace();
\r
314 fail(e.getMessage());
\r
315 } catch (InterruptedException e) {
\r
316 e.printStackTrace();
\r
317 fail(e.getMessage());
\r
322 public void testPersistance() {
\r
324 AACon aacon = new AACon();
\r
325 aacon.setError("errrr.txt").setInput(test_alignment_input)
\r
326 .setOutput("outtt.txt");
\r
327 assertEquals(aacon.getInput(), test_alignment_input);
\r
328 assertEquals(aacon.getError(), "errrr.txt");
\r
329 assertEquals(aacon.getOutput(), "outtt.txt");
\r
330 ConfiguredExecutable<AACon> cAAcon = Configurator
\r
331 .configureExecutable(aacon, Executable.ExecProvider.Local);
\r
333 SyncExecutor sexec = Configurator.getSyncEngine(cAAcon);
\r
334 sexec.executeJob();
\r
335 ConfiguredExecutable<?> al = sexec.waitForResult();
\r
336 assertNotNull(al.getResults());
\r
337 // Save run configuration
\r
338 assertTrue(cAAcon.saveRunConfiguration());
\r
340 // See if loaded configuration is the same as saved
\r
341 RunConfiguration loadedRun = RunConfiguration
\r
342 .load(new FileInputStream(new File(cAAcon
\r
343 .getWorkDirectory(), RunConfiguration.rconfigFile)));
\r
345 ((ConfExecutable<AACon>) cAAcon).getRunConfiguration(),
\r
347 // Load run configuration as ConfExecutable
\r
348 ConfiguredExecutable<AACon> resurrectedCAAcon = (ConfiguredExecutable<AACon>) cAAcon
\r
349 .loadRunConfiguration(new FileInputStream(new File(cAAcon
\r
350 .getWorkDirectory(), RunConfiguration.rconfigFile)));
\r
351 assertNotNull(resurrectedCAAcon);
\r
352 assertEquals(resurrectedCAAcon.getExecutable().getInput(),
\r
353 test_alignment_input);
\r
354 assertEquals(resurrectedCAAcon.getExecutable().getError(),
\r
356 assertEquals(resurrectedCAAcon.getExecutable().getOutput(),
\r
358 // See in details whether executables are the same
\r
359 assertEquals(resurrectedCAAcon.getExecutable(), aacon);
\r
361 ConfiguredExecutable<AACon> resAAcon = Configurator
\r
362 .configureExecutable(resurrectedCAAcon.getExecutable(),
\r
363 Executable.ExecProvider.Local);
\r
365 sexec = Configurator.getSyncEngine(resAAcon,
\r
366 Executable.ExecProvider.Local);
\r
367 sexec.executeJob();
\r
368 al = sexec.waitForResult();
\r
371 } catch (JobSubmissionException e) {
\r
372 e.printStackTrace();
\r
373 fail(e.getMessage());
\r
374 } catch (JobExecutionException e) {
\r
375 e.printStackTrace();
\r
376 fail(e.getMessage());
\r
377 } catch (FileNotFoundException e) {
\r
378 e.printStackTrace();
\r
379 fail(e.getMessage());
\r
380 } catch (IOException e) {
\r
381 e.printStackTrace();
\r
382 fail(e.getMessage());
\r
383 } catch (ResultNotAvailableException e) {
\r
384 e.printStackTrace();
\r
385 fail(e.getMessage());
\r
390 public void testConfigurationLoading() {
\r
392 RunnerConfig<AACon> aaconConfig = ConfExecutable
\r
393 .getRunnerOptions(AACon.class);
\r
394 assertNotNull(aaconConfig);
\r
395 assertTrue(aaconConfig.getArguments().size() > 0);
\r
397 PresetManager<AACon> aaconPresets = ConfExecutable
\r
398 .getRunnerPresets(AACon.class);
\r
399 assertNotNull(aaconPresets);
\r
401 LimitsManager<AACon> jronnLimits = ConfExecutable
\r
402 .getRunnerLimits(AACon.class);
\r
403 assertNotNull(jronnLimits);
\r
404 assertTrue(jronnLimits.getLimits().size() > 0);
\r
405 jronnLimits.validate(aaconPresets);
\r
407 } catch (FileNotFoundException e) {
\r
408 e.printStackTrace();
\r
409 fail(e.getLocalizedMessage());
\r
410 } catch (IOException e) {
\r
411 e.printStackTrace();
\r
412 fail(e.getLocalizedMessage());
\r
413 } catch (ValidationException e) {
\r
414 e.printStackTrace();
\r
415 fail(e.getLocalizedMessage());
\r