engine/compbio/engine/client/Util.java

   1 /* Copyright (c) 2009 Peter Troshin\r
   2  *  \r
   3  *  JAva Bioinformatics Analysis Web Services (JABAWS) @version: 1.0     \r
   4  * \r
   5  *  This library is free software; you can redistribute it and/or modify it under the terms of the\r
   6  *  Apache License version 2 as published by the Apache Software Foundation\r
   7  * \r
   8  *  This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without\r
   9  *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Apache \r
  10  *  License for more details.\r
  11  * \r
  12  *  A copy of the license is in apache_license.txt. It is also available here:\r
  13  * @see: http://www.apache.org/licenses/LICENSE-2.0.txt\r
  14  * \r
  15  * Any republication or derived work distributed in source code form\r
  16  * must include this copyright and license notice.\r
  17  */\r
  18 \r
  19 package compbio.engine.client;\r
  20 \r
  21 import java.io.File;\r
  22 import java.io.FileInputStream;\r
  23 import java.io.FileNotFoundException;\r
  24 import java.io.FileWriter;\r
  25 import java.io.IOException;\r
  26 import java.security.InvalidParameterException;\r
  27 import java.util.Map;\r
  28 \r
  29 import org.apache.log4j.Logger;\r
  30 \r
  31 import compbio.engine.client.Executable.ExecProvider;\r
  32 import compbio.engine.conf.DirectoryManager;\r
  33 import compbio.engine.conf.PropertyHelperManager;\r
  34 import compbio.metadata.JobStatus;\r
  35 import compbio.metadata.LimitsManager;\r
  36 import compbio.metadata.ResultNotAvailableException;\r
  37 import compbio.util.FileUtil;\r
  38 import compbio.util.PropertyHelper;\r
  39 import compbio.util.SysPrefs;\r
  40 \r
  41 public final class Util {\r
  42 \r
  43         private static final PropertyHelper ph = PropertyHelperManager\r
  44                         .getPropertyHelper();\r
  45 \r
  46         private static final Logger log = Logger.getLogger(Util.class);\r
  47 \r
  48         public static boolean isValidJobId(final String key) {\r
  49                 if (compbio.util.Util.isEmpty(key)) {\r
  50                         return false;\r
  51                 }\r
  52                 int delIdx = key.indexOf(DirectoryManager.DELIM);\r
  53                 if (delIdx < 0) {\r
  54                         return false;\r
  55                 }\r
  56                 String id = key.substring(delIdx + DirectoryManager.DELIM.length());\r
  57                 try {\r
  58                         Long.parseLong(id);\r
  59                 } catch (NumberFormatException e) {\r
  60                         log.debug("Invalid key! " + e.getLocalizedMessage());\r
  61                         return false;\r
  62                 }\r
  63                 return true;\r
  64         }\r
  65 \r
  66         public static void writeStatFile(String workDirectory,\r
  67                         String fileAndEventName) {\r
  68                 // never override old stat files!\r
  69                 // Work directory could be null for cancelled or incomplete jobs, just\r
  70                 // ignore\r
  71                 if (!compbio.util.Util.isEmpty(workDirectory)) {\r
  72                         writeFile(workDirectory, fileAndEventName,\r
  73                                         new Long(System.currentTimeMillis()).toString(), false);\r
  74                 }\r
  75         }\r
  76 \r
  77         public static void writeFile(String workDirectory, String fileAndEventName,\r
  78                         String content, boolean override) {\r
  79                 File file = null;\r
  80                 if (compbio.util.Util.isEmpty(workDirectory)) {\r
  81                         log.debug("Calling compbio.engine.Util.writeFile() with not work directory."\r
  82                                         + " Skipping writing statistics!");\r
  83                         return;\r
  84                 }\r
  85                 assert !compbio.util.Util.isEmpty(content) : "Content expected!";\r
  86                 FileWriter writer = null;\r
  87                 try {\r
  88                         file = new File(workDirectory, fileAndEventName);\r
  89                         // Do not override existing files unless asked to do so !\r
  90                         if (file.exists() && !override) {\r
  91                                 return;\r
  92                         }\r
  93                         writer = new FileWriter(file);\r
  94                         writer.write(content);\r
  95                         writer.close();\r
  96                         log.debug("File " + fileAndEventName + " with content: " + content\r
  97                                         + " has been recorder successfully! ");\r
  98                 } catch (IOException e) {\r
  99                         log.error("Could not record the " + fileAndEventName + " file in "\r
 100                                         + workDirectory + " for local execution! Ignoring... "\r
 101                                         + e.getMessage());\r
 102                 } finally {\r
 103                         FileUtil.closeSilently(log, writer);\r
 104                 }\r
 105         }\r
 106 \r
 107         public static final boolean writeMarker(String workDirectory,\r
 108                         JobStatus fileType) {\r
 109                 if (fileType == null) {\r
 110                         throw new NullPointerException("MarkerType must be provided!");\r
 111                 }\r
 112                 if (fileType == fileType.FINISHED || fileType == fileType.STARTED) {\r
 113                         throw new IllegalArgumentException(\r
 114                                         "Please use Util.writeStatFile(workDirectory, fileAndEventName) to record FINISHED and STARTED statuses!");\r
 115                 }\r
 116                 if (!PathValidator.isValidDirectory(workDirectory)) {\r
 117                         // This is OK as some task could be cancelled even before they\r
 118                         // started\r
 119                         log.warn("Attempting to write " + fileType\r
 120                                         + " marker in the work directory " + workDirectory\r
 121                                         + " is not provided or does not exist!");\r
 122                         return false;\r
 123                 }\r
 124                 try {\r
 125                         File sfile = new File(workDirectory, fileType.toString());\r
 126                         if (!sfile.exists()) {\r
 127                                 return sfile.createNewFile();\r
 128                         }\r
 129                 } catch (IOException e) {\r
 130                         log.error(\r
 131                                         "Could not record stat marker file " + fileType\r
 132                                                         + " into the directory " + workDirectory + " ! "\r
 133                                                         + e.getMessage(), e.getCause());\r
 134                 }\r
 135                 return false;\r
 136         }\r
 137 \r
 138         public static boolean isMarked(String workDirectory, JobStatus marker) {\r
 139                 if (!PathValidator.isValidDirectory(workDirectory)) {\r
 140                         throw new NullPointerException("Work directory " + workDirectory\r
 141                                         + " is not provided or does not exist!");\r
 142                 }\r
 143                 return new File(workDirectory, marker.toString()).exists();\r
 144         }\r
 145 \r
 146         public static Map<String, String> mergeEnvVariables(\r
 147                         final Map<String, String> sysEnvTobeModified,\r
 148                         final Map<String, String> variables) {\r
 149                 if (variables.containsKey(EnvVariableProcessor.PATH)) {\r
 150                         String propPath = variables.get(EnvVariableProcessor.PATH);\r
 151                         String sysPATH = sysEnvTobeModified.get(EnvVariableProcessor.PATH);\r
 152                         String syspath = sysEnvTobeModified.get(EnvVariableProcessor.PATH\r
 153                                         .toLowerCase());\r
 154                         // This version appears surprisingly often on windows machines\r
 155                         boolean added = false;\r
 156                         String sysPath = sysEnvTobeModified.get("Path");\r
 157                         if (sysPATH != null) {\r
 158                                 sysEnvTobeModified.put(EnvVariableProcessor.PATH, sysPATH\r
 159                                                 + File.pathSeparator + propPath);\r
 160                                 added = true;\r
 161                         }\r
 162                         if (syspath != null) {\r
 163                                 sysEnvTobeModified.put(EnvVariableProcessor.PATH.toLowerCase(),\r
 164                                                 syspath + File.pathSeparator + propPath);\r
 165                                 added = true;\r
 166                         }\r
 167                         if (sysPath != null) {\r
 168                                 sysEnvTobeModified.put("Path", sysPath + File.pathSeparator\r
 169                                                 + propPath);\r
 170                                 added = true;\r
 171                         }\r
 172                         // If not path variable is found, then add it\r
 173                         if (!added) {\r
 174                                 sysEnvTobeModified.put(EnvVariableProcessor.PATH, propPath);\r
 175                         }\r
 176                         variables.remove(EnvVariableProcessor.PATH);\r
 177                 }\r
 178                 sysEnvTobeModified.putAll(variables);\r
 179                 return sysEnvTobeModified;\r
 180         }\r
 181 \r
 182         public static String convertToAbsolute(String relativePath) {\r
 183                 // If specified path is relative, than make it absolute\r
 184                 String absolute = relativePath;\r
 185                 if (!PathValidator.isAbsolutePath(relativePath)) {\r
 186                         absolute = PropertyHelperManager.getLocalPath() + relativePath;\r
 187                         Util.log.trace("Changing local path in enviromental variable to absolute: FROM "\r
 188                                         + relativePath + " TO " + absolute);\r
 189                 }\r
 190                 return absolute;\r
 191         }\r
 192 \r
 193         public static String getExecProperty(String propertySpec, Executable<?> exec) {\r
 194                 assert !compbio.util.Util.isEmpty(propertySpec);\r
 195                 assert exec != null;\r
 196                 return Util.getExecProperty(propertySpec, exec.getClass());\r
 197         }\r
 198 \r
 199         public static String getExecProperty(String propertySpec, Class<?> clazz) {\r
 200                 assert !compbio.util.Util.isEmpty(propertySpec);\r
 201                 assert clazz != null;\r
 202                 String property = clazz.getSimpleName().toLowerCase() + "."\r
 203                                 + propertySpec.toLowerCase();\r
 204                 log.trace("Processing property: " + property);\r
 205                 return ph.getProperty(property);\r
 206         }\r
 207 \r
 208         public static String getFullPath(String workDirectory, String fileName) {\r
 209                 assert !compbio.util.Util.isEmpty(fileName) : "Filename must be provided! ";\r
 210                 assert !compbio.util.Util.isEmpty(workDirectory) : "Workdirectory must be provided! ";\r
 211                 return workDirectory + File.separator + fileName;\r
 212         }\r
 213 \r
 214         public static String getCommand(ExecProvider provider, Class<?> clazz) {\r
 215                 if (provider == ExecProvider.Any) {\r
 216                         throw new IllegalArgumentException(\r
 217                                         "A particular execution environment must be chosen");\r
 218                 }\r
 219                 String execCommandName = clazz.getSimpleName().toLowerCase();\r
 220                 String bin = "";\r
 221                 if (provider == ExecProvider.Local) {\r
 222                         if (SysPrefs.isWindows) {\r
 223                                 bin = ph.getProperty("local." + execCommandName\r
 224                                                 + ".bin.windows");\r
 225                         } else {\r
 226                                 bin = ph.getProperty("local." + execCommandName + ".bin");\r
 227                         }\r
 228                         // For executable Jar files the location of Java executable is not\r
 229                         // required for local execution. If it is not provided, JABAWS will\r
 230                         // attempt to use Java from JAVA_HOME env variable\r
 231                         if (isJavaLibrary(clazz)) {\r
 232                                 if (compbio.util.Util.isEmpty(bin)) {\r
 233                                         bin = getJava();\r
 234                                 }\r
 235                         }\r
 236                         // If path to executable defined in the properties is not absolute,\r
 237                         // then make it so\r
 238                         // as setting working directory of ProcessBuilder will make it\r
 239                         // impossible\r
 240                         // to find an executable otherwise\r
 241                         if (!compbio.util.Util.isEmpty(bin)\r
 242                                         && !PathValidator.isAbsolutePath(bin)) {\r
 243                                 bin = bin.trim();\r
 244                                 if (bin.equalsIgnoreCase("java")\r
 245                                                 || bin.equalsIgnoreCase("java.exe")) {\r
 246                                         // do not make path absolute to the java executable if\r
 247                                         // relative path is provided. Java executable is not a part\r
 248                                         // of JABAWS distribution!\r
 249                                 } else {\r
 250                                         bin = PropertyHelperManager.getLocalPath() + bin;\r
 251                                 }\r
 252                         }\r
 253                 } else {\r
 254                         bin = ph.getProperty("cluster." + execCommandName + ".bin");\r
 255                 }\r
 256                 log.debug("Using executable: " + bin);\r
 257                 return bin; // File.separator\r
 258         }\r
 259         /**\r
 260          * Returns true of executableName.jar.file property has some value in the\r
 261          * Executable.properties file, false otherwise.\r
 262          * \r
 263          * @param clazz\r
 264          * @return\r
 265          */\r
 266         public static boolean isJavaLibrary(Class<?> clazz) {\r
 267                 String execCommandName = clazz.getSimpleName().toLowerCase();\r
 268                 String java_lib = ph.getProperty(execCommandName + ".jar.file");\r
 269                 return !compbio.util.Util.isEmpty(java_lib);\r
 270         }\r
 271 \r
 272         /**\r
 273          * Returns the absolute path to the Java executable from JAVA_HOME\r
 274          * \r
 275          * @return\r
 276          */\r
 277         public static String getJava() {\r
 278                 String javahome = System.getProperty("java.home");\r
 279                 if (compbio.util.Util.isEmpty(javahome)) {\r
 280                         javahome = System.getenv("JAVA_HOME");\r
 281                 }\r
 282                 if (compbio.util.Util.isEmpty(javahome)) {\r
 283                         log.warn("Cannot find Java in java.home system property "\r
 284                                         + "or JAVA_HOME environment variable! ");\r
 285                         return null;\r
 286                 }\r
 287                 File jh = new File(javahome);\r
 288                 if (jh.exists() && jh.isDirectory()) {\r
 289                         String java = javahome + File.separator + "bin" + File.separator\r
 290                                         + "java";\r
 291                         if (SysPrefs.isWindows) {\r
 292                                 java += ".exe";\r
 293                         }\r
 294                         File jexe = new File(java);\r
 295                         if (jexe.exists() && jexe.isFile() && jexe.canExecute()) {\r
 296                                 log.info("Using Java from: " + jexe.getAbsolutePath());\r
 297                                 return jexe.getAbsolutePath();\r
 298                         } else {\r
 299                                 log.warn("Cannot find java executable in the JAVA_HOME!");\r
 300                         }\r
 301                 } else {\r
 302                         log.warn("JAVA_HOME does not seems to point to a valid directory! Value: "\r
 303                                         + javahome);\r
 304                 }\r
 305                 return null;\r
 306         }\r
 307 \r
 308         public static ExecProvider getSupportedRuntimes(Class<?> clazz) {\r
 309                 boolean localRuntimeSupport = false;\r
 310                 boolean clusterRuntimeSupport = false;\r
 311                 String executableName = clazz.getSimpleName().toLowerCase();\r
 312                 String localRuntime1 = ph.getProperty("local." + executableName\r
 313                                 + ".bin.windows");\r
 314                 String localRuntime2 = ph.getProperty("local." + executableName\r
 315                                 + ".bin");\r
 316                 if (!compbio.util.Util.isEmpty(localRuntime1)\r
 317                                 || !compbio.util.Util.isEmpty(localRuntime2)) {\r
 318                         localRuntimeSupport = true;\r
 319                 } else {\r
 320                         localRuntimeSupport = isJavaLibrary(clazz) && getJava() != null;\r
 321                 }\r
 322 \r
 323                 String clusterRuntime = ph.getProperty("cluster." + executableName\r
 324                                 + ".bin");\r
 325                 if (!compbio.util.Util.isEmpty(clusterRuntime)) {\r
 326                         clusterRuntimeSupport = true;\r
 327                 }\r
 328                 if (localRuntimeSupport && clusterRuntimeSupport) {\r
 329                         return ExecProvider.Any;\r
 330                 } else if (localRuntimeSupport) {\r
 331                         return ExecProvider.Local;\r
 332                 } else if (clusterRuntimeSupport) {\r
 333                         return ExecProvider.Cluster;\r
 334                 }\r
 335                 // Means executable cannot be executed -> is improperly configured\r
 336                 // should be ignored\r
 337                 throw new InvalidParameterException(\r
 338                                 "Executable is not provided for any runtime environments");\r
 339         }\r
 340         public static ConfiguredExecutable<?> loadExecutable(String taskId)\r
 341                         throws ResultNotAvailableException {\r
 342                 String workDir = compbio.engine.Configurator.getWorkDirectory(taskId);\r
 343                 // The results for this job has been collected once, or the JVM may\r
 344                 // have been restarted,\r
 345                 // so that the job is not in the job list\r
 346                 // ->load a ConfiguredExercutable from saved run and return it\r
 347                 FileInputStream fileInStream = null;\r
 348                 ConfiguredExecutable<?> exec = null;\r
 349                 try {\r
 350                         fileInStream = new FileInputStream(workDir + File.separator\r
 351                                         + RunConfiguration.rconfigFile);\r
 352                         RunConfiguration rconf = RunConfiguration.load(fileInStream);\r
 353                         exec = ConfExecutable.newConfExecutable(rconf);\r
 354                         fileInStream.close();\r
 355                 } catch (FileNotFoundException e) {\r
 356                         log.error(\r
 357                                         "Could not find run configuration to load!"\r
 358                                                         + e.getLocalizedMessage(), e.getCause());\r
 359                         throw new ResultNotAvailableException(\r
 360                                         "Could not find run configuration to load!"\r
 361                                                         + e.getMessage(), e.getCause());\r
 362                 } catch (IOException e) {\r
 363                         log.error(\r
 364                                         "IO Exception while reading run configuration file!"\r
 365                                                         + e.getLocalizedMessage(), e.getCause());\r
 366                         throw new ResultNotAvailableException(\r
 367                                         "Could not load run configuration!" + e.getMessage(),\r
 368                                         e.getCause());\r
 369                 } finally {\r
 370                         FileUtil.closeSilently(log, fileInStream);\r
 371                 }\r
 372                 return exec;\r
 373         }\r
 374 \r
 375         /**\r
 376          * For now just assume that all parameters which came in needs setting it\r
 377          * will be a client responsibility to prepare RunnerConfig object then\r
 378          * \r
 379          * @param rconfig\r
 380          * @return\r
 381          * \r
 382          *         public static List<String> toOptionString(RunnerConfig<?>\r
 383          *         rconfig) { String option = ""; List<String> options = new\r
 384          *         ArrayList<String>(); for (Parameter<?> par :\r
 385          *         rconfig.getParameters()) { if (par.getPossibleValues().isEmpty())\r
 386          *         { option = par.getOptionName(); } else { option =\r
 387          *         par.getOptionName() + "=" + par.getPossibleValues().get(0); } //\r
 388          *         separate options options.add(option); } return options; }\r
 389          */\r
 390 \r
 391         public static <T> LimitsManager<T> getLimits(Class<T> clazz) {\r
 392                 LimitsManager<T> limits = null;\r
 393                 try {\r
 394                         limits = ConfExecutable.getRunnerLimits(clazz);\r
 395                 } catch (FileNotFoundException e) {\r
 396                         Util.log.warn("No limits are found for " + clazz + " executable! "\r
 397                                         + e.getLocalizedMessage(), e.getCause());\r
 398                         // its ok, limit may not be initialized\r
 399                 } catch (IOException e) {\r
 400                         Util.log.warn("IO exception while attempting to read limits for "\r
 401                                         + clazz + " executable! " + e.getLocalizedMessage(),\r
 402                                         e.getCause());\r
 403                 }\r
 404                 return limits;\r
 405         }\r
 406 \r
 407 }\r