testsrc/compbio/runner/conservation/AAConTester.java

   1 /*\r
   2  * Copyright (c) 2010 Peter Troshin JAva Bioinformatics Analysis Web Services\r
   3  * (JABAWS) @version: 2.0 \r
   4  * \r
   5  * This library is free software; you can redistribute it and/or modify it under \r
   6  * the terms of the Apache License version 2 as published\r
   7  * by the Apache Software Foundation This library is distributed in the hope\r
   8  * that it will be useful, but WITHOUT ANY WARRANTY; without even the implied\r
   9  * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\r
  10  * Apache License for more details. A copy of the license is in\r
  11  * apache_license.txt. It is also available here:\r
  12  * \r
  13  * @see: http://www.apache.org/licenses/LICENSE-2.0.txt \r
  14  * \r
  15  * Any republication or derived work distributed in source code form must include \r
  16  * this copyright and license notice.\r
  17  */\r
  18 package compbio.runner.conservation;\r
  19 \r
  20 import static org.testng.Assert.assertEquals;\r
  21 import static org.testng.Assert.assertFalse;\r
  22 import static org.testng.Assert.assertNotNull;\r
  23 import static org.testng.Assert.assertTrue;\r
  24 import static org.testng.Assert.fail;\r
  25 \r
  26 import java.io.File;\r
  27 import java.io.FileInputStream;\r
  28 import java.io.FileNotFoundException;\r
  29 import java.io.IOException;\r
  30 import java.text.ParseException;\r
  31 import java.util.Arrays;\r
  32 import java.util.HashSet;\r
  33 \r
  34 import javax.xml.bind.ValidationException;\r
  35 \r
  36 import org.ggf.drmaa.DrmaaException;\r
  37 import org.ggf.drmaa.JobInfo;\r
  38 import org.testng.annotations.BeforeMethod;\r
  39 import org.testng.annotations.Test;\r
  40 \r
  41 import compbio.data.sequence.Score;\r
  42 import compbio.data.sequence.ScoreManager;\r
  43 import compbio.engine.AsyncExecutor;\r
  44 import compbio.engine.Configurator;\r
  45 import compbio.engine.FilePuller;\r
  46 import compbio.engine.SyncExecutor;\r
  47 import compbio.engine.client.ConfExecutable;\r
  48 import compbio.engine.client.ConfiguredExecutable;\r
  49 import compbio.engine.client.Executable;\r
  50 import compbio.engine.client.RunConfiguration;\r
  51 import compbio.engine.cluster.drmaa.ClusterUtil;\r
  52 import compbio.engine.cluster.drmaa.JobRunner;\r
  53 import compbio.engine.cluster.drmaa.StatisticManager;\r
  54 import compbio.engine.local.LocalRunner;\r
  55 import compbio.metadata.AllTestSuit;\r
  56 import compbio.metadata.ChunkHolder;\r
  57 import compbio.metadata.JobExecutionException;\r
  58 import compbio.metadata.JobStatus;\r
  59 import compbio.metadata.JobSubmissionException;\r
  60 import compbio.metadata.LimitsManager;\r
  61 import compbio.metadata.Preset;\r
  62 import compbio.metadata.PresetManager;\r
  63 import compbio.metadata.ResultNotAvailableException;\r
  64 import compbio.metadata.RunnerConfig;\r
  65 import compbio.runner.Util;\r
  66 import compbio.util.FileWatcher;\r
  67 import compbio.util.SysPrefs;\r
  68 \r
  69 public class AAConTester {\r
  70 \r
  71         public static final String CURRENT_DIRECTORY = SysPrefs\r
  72                         .getCurrentDirectory() + File.separator;\r
  73 \r
  74         public static String test_outfile = "TO1381.aacon.out"; // "/homes/pvtroshin/TO1381.clustal.cluster.out\r
  75         public static String test_alignment_input = CURRENT_DIRECTORY + "testsrc"\r
  76                         + File.separator + "testdata" + File.separator + "TO1381.fasta.aln";\r
  77         private AACon aacon;\r
  78 \r
  79         @BeforeMethod(alwaysRun = true)\r
  80         void init() {\r
  81                 aacon = new AACon();\r
  82                 aacon.setInput(test_alignment_input).setOutput(test_outfile);\r
  83         }\r
  84 \r
  85         @Test(groups = {AllTestSuit.test_group_runner})\r
  86         public void testRunOnCluster() {\r
  87                 assertFalse(SysPrefs.isWindows,\r
  88                                 "Cluster execution can only be in unix environment");\r
  89                 try {\r
  90                         ConfiguredExecutable<AACon> confAAcon = Configurator\r
  91                                         .configureExecutable(aacon, Executable.ExecProvider.Cluster);\r
  92                         JobRunner runner = JobRunner.getInstance(confAAcon);\r
  93 \r
  94                         assertNotNull(runner, "Runner is NULL");\r
  95                         runner.executeJob();\r
  96                         // assertNotNull("JobId is null", jobId1);\r
  97                         JobStatus status = runner.getJobStatus();\r
  98                         assertTrue(status == JobStatus.PENDING\r
  99                                         || status == JobStatus.RUNNING,\r
 100                                         "Status of the process is wrong!");\r
 101                         JobInfo info = runner.getJobInfo();\r
 102                         assertNotNull(info, "JobInfo is null");\r
 103                         StatisticManager sm = new StatisticManager(info);\r
 104                         assertNotNull(sm, "Statictic manager is null");\r
 105                         try {\r
 106 \r
 107                                 String exits = sm.getExitStatus();\r
 108                                 assertNotNull("Exit status is null", exits);\r
 109                                 // cut 4 trailing zeros from the number\r
 110                                 int exitsInt = ClusterUtil.CLUSTER_STAT_IN_SEC.parse(exits)\r
 111                                                 .intValue();\r
 112                                 assertEquals(0, exitsInt);\r
 113                                 System.out.println(sm.getAllStats());\r
 114 \r
 115                         } catch (ParseException e) {\r
 116                                 e.printStackTrace();\r
 117                                 fail("Parse Exception: " + e.getMessage());\r
 118                         }\r
 119                         // assertFalse(runner.cleanup());\r
 120                         assertTrue(sm.hasExited());\r
 121                         assertFalse(sm.wasAborted());\r
 122                         assertFalse(sm.hasDump());\r
 123                         assertFalse(sm.hasSignaled());\r
 124 \r
 125                 } catch (JobSubmissionException e) {\r
 126                         e.printStackTrace();\r
 127                         fail("DrmaaException caught:" + e.getMessage());\r
 128                 } catch (JobExecutionException e) {\r
 129                         e.printStackTrace();\r
 130                         fail("DrmaaException caught:" + e.getMessage());\r
 131                 } catch (DrmaaException e) {\r
 132                         e.printStackTrace();\r
 133                         fail("DrmaaException caught:" + e.getMessage());\r
 134                 }\r
 135         }\r
 136 \r
 137         /**\r
 138          * This tests fails from time to time depending on the cluster load or some\r
 139          * other factors. Any client code has to adjust for this issue\r
 140          */\r
 141         @Test(groups = {AllTestSuit.test_group_runner, AllTestSuit.test_group_cluster})\r
 142         public void testRunOnClusterAsync() {\r
 143                 assertFalse(SysPrefs.isWindows,\r
 144                                 "Cluster execution can only be in unix environment");\r
 145                 try {\r
 146                         ConfiguredExecutable<AACon> confAAcon = Configurator\r
 147                                         .configureExecutable(aacon, Executable.ExecProvider.Cluster);\r
 148                         AsyncExecutor aengine = Configurator.getAsyncEngine(confAAcon);\r
 149                         String jobId = aengine.submitJob(confAAcon);\r
 150                         assertNotNull(jobId, "Runner is NULL");\r
 151                         // let drmaa to start\r
 152                         Thread.sleep(500);\r
 153                         JobStatus status = aengine.getJobStatus(jobId);\r
 154                         while (status != JobStatus.FINISHED) {\r
 155                                 System.out.println("Job Status: " + status);\r
 156                                 Thread.sleep(1000);\r
 157                                 status = aengine.getJobStatus(jobId);\r
 158                                 ConfiguredExecutable<AACon> result = (ConfiguredExecutable<AACon>) aengine\r
 159                                                 .getResults(jobId);\r
 160                                 assertNotNull(result);\r
 161                                 System.out.println("RES:" + result);\r
 162                                 // Some times the job could be removed from the cluster\r
 163                                 // accounting\r
 164                                 // before it has been reported to finish. Make sure\r
 165                                 // to stop waiting in such case\r
 166                                 if (status == JobStatus.UNDEFINED) {\r
 167                                         break;\r
 168                                 }\r
 169                         }\r
 170                 } catch (JobSubmissionException e) {\r
 171                         e.printStackTrace();\r
 172                         fail("DrmaaException caught:" + e.getMessage());\r
 173                 } catch (InterruptedException e) {\r
 174                         e.printStackTrace();\r
 175                         fail(e.getMessage());\r
 176                 } catch (ResultNotAvailableException e) {\r
 177                         e.printStackTrace();\r
 178                         fail(e.getMessage());\r
 179                 }\r
 180         }\r
 181 \r
 182         @Test(groups = {AllTestSuit.test_group_runner})\r
 183         public void testRunLocally() {\r
 184                 try {\r
 185                         ConfiguredExecutable<AACon> confAAcon = Configurator\r
 186                                         .configureExecutable(aacon, Executable.ExecProvider.Local);\r
 187 \r
 188                         // For local execution use relative\r
 189                         LocalRunner lr = new LocalRunner(confAAcon);\r
 190                         lr.executeJob();\r
 191                         ConfiguredExecutable<?> al1 = lr.waitForResult();\r
 192                         assertNotNull(al1.getResults());\r
 193                         ScoreManager annotations = confAAcon.getResults();\r
 194                         assertNotNull(annotations);\r
 195                         assertEquals(annotations.asSet().size(), 18);\r
 196                         assertEquals(al1.getResults(), annotations);\r
 197                 } catch (JobSubmissionException e) {\r
 198                         e.printStackTrace();\r
 199                         fail(e.getLocalizedMessage());\r
 200                 } catch (ResultNotAvailableException e) {\r
 201                         e.printStackTrace();\r
 202                         fail(e.getLocalizedMessage());\r
 203                 } catch (JobExecutionException e) {\r
 204                         e.printStackTrace();\r
 205                         fail(e.getLocalizedMessage());\r
 206                 }\r
 207         }\r
 208 \r
 209         @Test(groups = {AllTestSuit.test_group_runner})\r
 210         public void testRunLocallyWithPreset() {\r
 211                 try {\r
 212                         PresetManager<AACon> aaconPresets = Util.getPresets(AACon.class);\r
 213                         assert aaconPresets != null;\r
 214                         ConfiguredExecutable<AACon> confAAcon = Configurator\r
 215                                         .configureExecutable(aacon, Executable.ExecProvider.Local);\r
 216                         Preset<AACon> quick = aaconPresets\r
 217                                         .getPresetByName("Quick conservation");\r
 218                         confAAcon.addParameters(quick.getOptions());\r
 219                         // For local execution use relative\r
 220                         LocalRunner lr = new LocalRunner(confAAcon);\r
 221                         lr.executeJob();\r
 222                         ConfiguredExecutable<?> al1 = lr.waitForResult();\r
 223                         assertNotNull(al1.getResults());\r
 224                         ScoreManager annotations = confAAcon.getResults();\r
 225                         assertNotNull(annotations);\r
 226                         assertEquals(annotations.asSet().size(), 13);\r
 227                         assertEquals(al1.getResults(), annotations);\r
 228                 } catch (JobSubmissionException e) {\r
 229                         e.printStackTrace();\r
 230                         fail(e.getLocalizedMessage());\r
 231                 } catch (ResultNotAvailableException e) {\r
 232                         e.printStackTrace();\r
 233                         fail(e.getLocalizedMessage());\r
 234                 } catch (JobExecutionException e) {\r
 235                         e.printStackTrace();\r
 236                         fail(e.getLocalizedMessage());\r
 237                 }\r
 238         }\r
 239 \r
 240         @Test(groups = {AllTestSuit.test_group_runner})\r
 241         public void testRunLocallyOnTwoCpu() {\r
 242                 try {\r
 243                         aacon = new AACon();\r
 244                         aacon.setInput(test_alignment_input).setOutput(test_outfile);\r
 245 \r
 246                         aacon.setNCore(2);\r
 247                         ConfiguredExecutable<AACon> confAAcon = Configurator\r
 248                                         .configureExecutable(aacon, Executable.ExecProvider.Local);\r
 249                         confAAcon.addParameters(Arrays.asList("-m=KABAT,JORES"));\r
 250                         confAAcon.addParameters(Arrays.asList("-m=TAYLOR_GAPS"));\r
 251                         // For local execution use relative\r
 252                         LocalRunner lr = new LocalRunner(confAAcon);\r
 253 \r
 254                         lr.executeJob();\r
 255                         ConfiguredExecutable<?> al1 = lr.waitForResult();\r
 256                         assertNotNull(al1.getResults());\r
 257                         ScoreManager annotations = confAAcon.getResults();\r
 258                         assertNotNull(annotations);\r
 259                         assertEquals(annotations.asSet().size(), 3);\r
 260                         assertEquals(al1.getResults(), annotations);\r
 261 \r
 262                 } catch (JobSubmissionException e) {\r
 263                         e.printStackTrace();\r
 264                         fail(e.getLocalizedMessage());\r
 265                 } catch (ResultNotAvailableException e) {\r
 266                         e.printStackTrace();\r
 267                         fail(e.getLocalizedMessage());\r
 268                 } catch (JobExecutionException e) {\r
 269                         e.printStackTrace();\r
 270                         fail(e.getLocalizedMessage());\r
 271                 }\r
 272         }\r
 273         \r
 274         @Test(groups = {AllTestSuit.test_group_runner})\r
 275         public void readStatistics() {\r
 276                 try {\r
 277                         ConfiguredExecutable<AACon> confAAcon = Configurator\r
 278                                         .configureExecutable(aacon, Executable.ExecProvider.Local);\r
 279                         // For local execution use relative\r
 280 \r
 281                         AsyncExecutor sexec = Configurator.getAsyncEngine(confAAcon);\r
 282                         String jobId = sexec.submitJob(confAAcon);\r
 283                         FilePuller fw = FilePuller.newFilePuller(\r
 284                                         confAAcon.getWorkDirectory() + File.separator\r
 285                                                         + AACon.getStatFile(),\r
 286                                         FileWatcher.MIN_CHUNK_SIZE_BYTES);\r
 287                         int count = 0;\r
 288                         long position = 0;\r
 289                         fw.waitForFile(2);\r
 290                         JobStatus status = sexec.getJobStatus(jobId);\r
 291                         do {\r
 292                                 ChunkHolder ch = fw.pull(position);\r
 293                                 String chunk = ch.getChunk();\r
 294                                 position = ch.getNextPosition();\r
 295                                 // System.out.println(chunk);\r
 296                                 count++;\r
 297                                 // Make sure the loop is terminated if the job fails\r
 298                                 if ((status == JobStatus.UNDEFINED || status == JobStatus.FAILED)) {\r
 299                                         fail("job failed!");\r
 300                                         break;\r
 301                                 }\r
 302                                 Thread.sleep(300);\r
 303                                 status = sexec.getJobStatus(jobId);\r
 304                         } while (status != JobStatus.FINISHED || fw.hasMoreData());\r
 305 \r
 306                         assertTrue(count >= 1);\r
 307                         ConfiguredExecutable<?> al = sexec.getResults(jobId);\r
 308                         assertNotNull(al.getResults());\r
 309                 } catch (JobSubmissionException e) {\r
 310                         e.printStackTrace();\r
 311                         fail(e.getMessage());\r
 312                 } catch (ResultNotAvailableException e) {\r
 313                         e.printStackTrace();\r
 314                         fail(e.getMessage());\r
 315                 } catch (IOException e) {\r
 316                         e.printStackTrace();\r
 317                         fail(e.getMessage());\r
 318                 } catch (InterruptedException e) {\r
 319                         e.printStackTrace();\r
 320                         fail(e.getMessage());\r
 321                 }\r
 322         }\r
 323 \r
 324         @Test(groups = {AllTestSuit.test_group_runner})\r
 325         public void testPersistance() {\r
 326                 try {\r
 327                         AACon aacon = new AACon();\r
 328                         aacon.setError("errrr.txt").setInput(test_alignment_input)\r
 329                                         .setOutput("outtt.txt");\r
 330                         assertEquals(aacon.getInput(), test_alignment_input);\r
 331                         assertEquals(aacon.getError(), "errrr.txt");\r
 332                         assertEquals(aacon.getOutput(), "outtt.txt");\r
 333                         ConfiguredExecutable<AACon> cAAcon = Configurator\r
 334                                         .configureExecutable(aacon, Executable.ExecProvider.Local);\r
 335 \r
 336                         SyncExecutor sexec = Configurator.getSyncEngine(cAAcon);\r
 337                         sexec.executeJob();\r
 338                         ConfiguredExecutable<?> al = sexec.waitForResult();\r
 339                         assertNotNull(al.getResults());\r
 340                         // Save run configuration\r
 341                         assertTrue(cAAcon.saveRunConfiguration());\r
 342 \r
 343                         // See if loaded configuration is the same as saved\r
 344                         RunConfiguration loadedRun = RunConfiguration\r
 345                                         .load(new FileInputStream(new File(cAAcon\r
 346                                                         .getWorkDirectory(), RunConfiguration.rconfigFile)));\r
 347                         assertEquals(\r
 348                                         ((ConfExecutable<AACon>) cAAcon).getRunConfiguration(),\r
 349                                         loadedRun);\r
 350                         // Load run configuration as ConfExecutable\r
 351                         ConfiguredExecutable<AACon> resurrectedCAAcon = (ConfiguredExecutable<AACon>) cAAcon\r
 352                                         .loadRunConfiguration(new FileInputStream(new File(cAAcon\r
 353                                                         .getWorkDirectory(), RunConfiguration.rconfigFile)));\r
 354                         assertNotNull(resurrectedCAAcon);\r
 355                         assertEquals(resurrectedCAAcon.getExecutable().getInput(),\r
 356                                         test_alignment_input);\r
 357                         assertEquals(resurrectedCAAcon.getExecutable().getError(),\r
 358                                         "errrr.txt");\r
 359                         assertEquals(resurrectedCAAcon.getExecutable().getOutput(),\r
 360                                         "outtt.txt");\r
 361                         // See in details whether executables are the same\r
 362                         assertEquals(resurrectedCAAcon.getExecutable(), aacon);\r
 363 \r
 364                         ConfiguredExecutable<AACon> resAAcon = Configurator\r
 365                                         .configureExecutable(resurrectedCAAcon.getExecutable(),\r
 366                                                         Executable.ExecProvider.Local);\r
 367 \r
 368                         sexec = Configurator.getSyncEngine(resAAcon,\r
 369                                         Executable.ExecProvider.Local);\r
 370                         sexec.executeJob();\r
 371                         al = sexec.waitForResult();\r
 372                         assertNotNull(al);\r
 373 \r
 374                 } catch (JobSubmissionException e) {\r
 375                         e.printStackTrace();\r
 376                         fail(e.getMessage());\r
 377                 } catch (JobExecutionException e) {\r
 378                         e.printStackTrace();\r
 379                         fail(e.getMessage());\r
 380                 } catch (FileNotFoundException e) {\r
 381                         e.printStackTrace();\r
 382                         fail(e.getMessage());\r
 383                 } catch (IOException e) {\r
 384                         e.printStackTrace();\r
 385                         fail(e.getMessage());\r
 386                 } catch (ResultNotAvailableException e) {\r
 387                         e.printStackTrace();\r
 388                         fail(e.getMessage());\r
 389                 }\r
 390         }\r
 391 \r
 392         @Test(groups = {AllTestSuit.test_group_runner})\r
 393         public void testConfigurationLoading() {\r
 394                 try {\r
 395                         RunnerConfig<AACon> aaconConfig = ConfExecutable\r
 396                                         .getRunnerOptions(AACon.class);\r
 397                         assertNotNull(aaconConfig);\r
 398                         assertTrue(aaconConfig.getArguments().size() > 0);\r
 399 \r
 400                         PresetManager<AACon> aaconPresets = ConfExecutable\r
 401                                         .getRunnerPresets(AACon.class);\r
 402                         assertNotNull(aaconPresets);\r
 403 \r
 404                         LimitsManager<AACon> jronnLimits = ConfExecutable\r
 405                                         .getRunnerLimits(AACon.class);\r
 406                         assertNotNull(jronnLimits);\r
 407                         assertTrue(jronnLimits.getLimits().size() > 0);\r
 408                         jronnLimits.validate(aaconPresets);\r
 409 \r
 410                 } catch (FileNotFoundException e) {\r
 411                         e.printStackTrace();\r
 412                         fail(e.getLocalizedMessage());\r
 413                 } catch (IOException e) {\r
 414                         e.printStackTrace();\r
 415                         fail(e.getLocalizedMessage());\r
 416                 } catch (ValidationException e) {\r
 417                         e.printStackTrace();\r
 418                         fail(e.getLocalizedMessage());\r
 419                 }\r
 420         }\r
 421 \r
 422 }\r