engine/compbio/engine/client/Util.java

   1 /* Copyright (c) 2009 Peter Troshin\r
   2  *  \r
   3  *  JAva Bioinformatics Analysis Web Services (JABAWS) @version: 1.0     \r
   4  * \r
   5  *  This library is free software; you can redistribute it and/or modify it under the terms of the\r
   6  *  Apache License version 2 as published by the Apache Software Foundation\r
   7  * \r
   8  *  This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without\r
   9  *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Apache \r
  10  *  License for more details.\r
  11  * \r
  12  *  A copy of the license is in apache_license.txt. It is also available here:\r
  13  * @see: http://www.apache.org/licenses/LICENSE-2.0.txt\r
  14  * \r
  15  * Any republication or derived work distributed in source code form\r
  16  * must include this copyright and license notice.\r
  17  */\r
  18 \r
  19 package compbio.engine.client;\r
  20 \r
  21 import java.io.File;\r
  22 import java.io.FileInputStream;\r
  23 import java.io.FileNotFoundException;\r
  24 import java.io.FileWriter;\r
  25 import java.io.IOException;\r
  26 import java.security.InvalidParameterException;\r
  27 import java.util.Map;\r
  28 \r
  29 import org.apache.log4j.Logger;\r
  30 \r
  31 import compbio.engine.client.Executable.ExecProvider;\r
  32 import compbio.engine.conf.DirectoryManager;\r
  33 import compbio.engine.conf.PropertyHelperManager;\r
  34 import compbio.metadata.JobStatus;\r
  35 import compbio.metadata.LimitsManager;\r
  36 import compbio.metadata.ResultNotAvailableException;\r
  37 import compbio.util.FileUtil;\r
  38 import compbio.util.PropertyHelper;\r
  39 import compbio.util.SysPrefs;\r
  40 \r
  41 public final class Util {\r
  42 \r
  43         private static final PropertyHelper ph = PropertyHelperManager\r
  44                         .getPropertyHelper();\r
  45 \r
  46         private static final Logger log = Logger.getLogger(Util.class);\r
  47 \r
  48         public static boolean isValidJobId(final String key) {\r
  49                 if (compbio.util.Util.isEmpty(key)) {\r
  50                         return false;\r
  51                 }\r
  52                 int delIdx = key.indexOf(DirectoryManager.DELIM);\r
  53                 if (delIdx < 0) {\r
  54                         return false;\r
  55                 }\r
  56                 String id = key.substring(delIdx + DirectoryManager.DELIM.length());\r
  57                 try {\r
  58                         Long.parseLong(id);\r
  59                 } catch (NumberFormatException e) {\r
  60                         log.debug("Invalid key! " + e.getLocalizedMessage());\r
  61                         return false;\r
  62                 }\r
  63                 return true;\r
  64         }\r
  65 \r
  66         public static void writeStatFile(String workDirectory,\r
  67                         String fileAndEventName) {\r
  68                 // never override old stat files!\r
  69                 // Work directory could be null for cancelled or incomplete jobs, just\r
  70                 // ignore\r
  71                 if (!compbio.util.Util.isEmpty(workDirectory)) {\r
  72                         writeFile(workDirectory, fileAndEventName, new Long(System\r
  73                                         .currentTimeMillis()).toString(), false);\r
  74                 }\r
  75         }\r
  76 \r
  77         public static void writeFile(String workDirectory, String fileAndEventName,\r
  78                         String content, boolean override) {\r
  79                 File file = null;\r
  80                 if (compbio.util.Util.isEmpty(workDirectory)) {\r
  81                         log\r
  82                                         .debug("Calling compbio.engine.Util.writeFile() with not work directory."\r
  83                                                         + " Skipping writing statistics!");\r
  84                         return;\r
  85                 }\r
  86                 assert !compbio.util.Util.isEmpty(content) : "Content expected!";\r
  87                 FileWriter writer = null;\r
  88                 try {\r
  89                         file = new File(workDirectory, fileAndEventName);\r
  90                         // Do not override existing files unless asked to do so !\r
  91                         if (file.exists() && !override) {\r
  92                                 return;\r
  93                         }\r
  94                         writer = new FileWriter(file);\r
  95                         writer.write(content);\r
  96                         writer.close();\r
  97                         log.debug("File " + fileAndEventName + " with content: " + content\r
  98                                         + " has been recorder successfully! ");\r
  99                 } catch (IOException e) {\r
 100                         log.error("Could not record the " + fileAndEventName + " file in "\r
 101                                         + workDirectory + " for local execution! Ignoring... "\r
 102                                         + e.getMessage());\r
 103                 } finally {\r
 104                         FileUtil.closeSilently(log, writer);\r
 105                 }\r
 106         }\r
 107 \r
 108         public static final boolean writeMarker(String workDirectory,\r
 109                         JobStatus fileType) {\r
 110                 if (fileType == null) {\r
 111                         throw new NullPointerException("MarkerType must be provided!");\r
 112                 }\r
 113                 if (fileType == fileType.FINISHED || fileType == fileType.STARTED) {\r
 114                         throw new IllegalArgumentException(\r
 115                                         "Please use Util.writeStatFile(workDirectory, fileAndEventName) to record FINISHED and STARTED statuses!");\r
 116                 }\r
 117                 if (!PathValidator.isValidDirectory(workDirectory)) {\r
 118                         // This is OK as some task could be cancelled even before they\r
 119                         // started\r
 120                         log.warn("Attempting to write " + fileType\r
 121                                         + " marker in the work directory " + workDirectory\r
 122                                         + " is not provided or does not exist!");\r
 123                         return false;\r
 124                 }\r
 125                 try {\r
 126                         File sfile = new File(workDirectory, fileType.toString());\r
 127                         if (!sfile.exists()) {\r
 128                                 return sfile.createNewFile();\r
 129                         }\r
 130                 } catch (IOException e) {\r
 131                         log.error("Could not record stat marker file " + fileType\r
 132                                         + " into the directory " + workDirectory + " ! "\r
 133                                         + e.getMessage(), e.getCause());\r
 134                 }\r
 135                 return false;\r
 136         }\r
 137 \r
 138         public static boolean isMarked(String workDirectory, JobStatus marker) {\r
 139                 if (!PathValidator.isValidDirectory(workDirectory)) {\r
 140                         throw new NullPointerException("Work directory " + workDirectory\r
 141                                         + " is not provided or does not exist!");\r
 142                 }\r
 143                 return new File(workDirectory, marker.toString()).exists();\r
 144         }\r
 145 \r
 146         public static Map<String, String> mergeEnvVariables(\r
 147                         final Map<String, String> sysEnvTobeModified,\r
 148                         final Map<String, String> variables) {\r
 149                 if (variables.containsKey(EnvVariableProcessor.PATH)) {\r
 150                         String propPath = variables.get(EnvVariableProcessor.PATH);\r
 151                         String sysPATH = sysEnvTobeModified.get(EnvVariableProcessor.PATH);\r
 152                         String syspath = sysEnvTobeModified.get(EnvVariableProcessor.PATH\r
 153                                         .toLowerCase());\r
 154                         // This version appears surprisingly often on windows machines\r
 155                         boolean added = false;\r
 156                         String sysPath = sysEnvTobeModified.get("Path");\r
 157                         if (sysPATH != null) {\r
 158                                 sysEnvTobeModified.put(EnvVariableProcessor.PATH, sysPATH\r
 159                                                 + File.pathSeparator + propPath);\r
 160                                 added = true;\r
 161                         }\r
 162                         if (syspath != null) {\r
 163                                 sysEnvTobeModified.put(EnvVariableProcessor.PATH.toLowerCase(),\r
 164                                                 syspath + File.pathSeparator + propPath);\r
 165                                 added = true;\r
 166                         }\r
 167                         if (sysPath != null) {\r
 168                                 sysEnvTobeModified.put("Path", sysPath + File.pathSeparator\r
 169                                                 + propPath);\r
 170                                 added = true;\r
 171                         }\r
 172                         // If not path variable is found, then add it\r
 173                         if (!added) {\r
 174                                 sysEnvTobeModified.put(EnvVariableProcessor.PATH, propPath);\r
 175                         }\r
 176                         variables.remove(EnvVariableProcessor.PATH);\r
 177                 }\r
 178                 sysEnvTobeModified.putAll(variables);\r
 179                 return sysEnvTobeModified;\r
 180         }\r
 181 \r
 182         public static String convertToAbsolute(String relativePath) {\r
 183                 // If specified path is relative, than make it absolute\r
 184                 String absolute = relativePath;\r
 185                 if (!PathValidator.isAbsolutePath(relativePath)) {\r
 186                         absolute = PropertyHelperManager.getLocalPath() + relativePath;\r
 187                         Util.log\r
 188                                         .trace("Changing local path in enviromental variable to absolute: FROM "\r
 189                                                         + relativePath + " TO " + absolute);\r
 190                 }\r
 191                 return absolute;\r
 192         }\r
 193 \r
 194         public static String getExecProperty(String propertySpec, Executable<?> exec) {\r
 195                 assert !compbio.util.Util.isEmpty(propertySpec);\r
 196                 assert exec != null;\r
 197                 return Util.getExecProperty(propertySpec, exec.getClass());\r
 198         }\r
 199 \r
 200         public static String getExecProperty(String propertySpec, Class<?> clazz) {\r
 201                 assert !compbio.util.Util.isEmpty(propertySpec);\r
 202                 assert clazz != null;\r
 203                 String property = clazz.getSimpleName().toLowerCase() + "."\r
 204                                 + propertySpec.toLowerCase();\r
 205                 log.trace("Processing property: " + property);\r
 206                 return ph.getProperty(property);\r
 207         }\r
 208 \r
 209         public static String getFullPath(String workDirectory, String fileName) {\r
 210                 assert !compbio.util.Util.isEmpty(fileName) : "Filename must be provided! ";\r
 211                 assert !compbio.util.Util.isEmpty(workDirectory) : "Workdirectory must be provided! ";\r
 212                 return workDirectory + File.separator + fileName;\r
 213         }\r
 214 \r
 215         public static String getCommand(ExecProvider provider, Class<?> clazz) {\r
 216                 if (provider == ExecProvider.Any) {\r
 217                         throw new IllegalArgumentException(\r
 218                                         "A particular execution environment must be chosen");\r
 219                 }\r
 220                 String execCommandName = clazz.getSimpleName().toLowerCase();\r
 221                 String bin = "";\r
 222                 if (provider == ExecProvider.Local) {\r
 223                         if (SysPrefs.isWindows) {\r
 224                                 bin = ph.getProperty("local." + execCommandName\r
 225                                                 + ".bin.windows");\r
 226                         } else {\r
 227                                 bin = ph.getProperty("local." + execCommandName + ".bin");\r
 228                         }\r
 229                         // If path to executable defined in the properties is not absolute,\r
 230                         // then make it so\r
 231                         // as setting working directory of ProcessBuilder will make it\r
 232                         // impossible\r
 233                         // to find an executable otherwise\r
 234                         if (!compbio.util.Util.isEmpty(bin)\r
 235                                         && !PathValidator.isAbsolutePath(bin)) {\r
 236                                 bin = PropertyHelperManager.getLocalPath() + bin;\r
 237                         }\r
 238                 } else {\r
 239                         bin = ph.getProperty("cluster." + execCommandName + ".bin");\r
 240                 }\r
 241                 return bin; // File.separator\r
 242         }\r
 243 \r
 244         public static ExecProvider getSupportedRuntimes(Class<?> clazz) {\r
 245                 boolean localRuntimeSupport = false;\r
 246                 boolean clusterRuntimeSupport = false;\r
 247                 String executableName = clazz.getSimpleName().toLowerCase();\r
 248                 String localRuntime1 = ph.getProperty("local." + executableName\r
 249                                 + ".bin.windows");\r
 250                 String localRuntime2 = ph.getProperty("local." + executableName\r
 251                                 + ".bin");\r
 252                 if (!compbio.util.Util.isEmpty(localRuntime1)\r
 253                                 || !compbio.util.Util.isEmpty(localRuntime2)) {\r
 254                         localRuntimeSupport = true;\r
 255                 }\r
 256                 String clusterRuntime = ph.getProperty("cluster." + executableName\r
 257                                 + ".bin");\r
 258                 if (!compbio.util.Util.isEmpty(clusterRuntime)) {\r
 259                         clusterRuntimeSupport = true;\r
 260                 }\r
 261                 if (localRuntimeSupport && clusterRuntimeSupport) {\r
 262                         return ExecProvider.Any;\r
 263                 } else if (localRuntimeSupport) {\r
 264                         return ExecProvider.Local;\r
 265                 } else if (clusterRuntimeSupport) {\r
 266                         return ExecProvider.Cluster;\r
 267                 }\r
 268                 // Means executable cannot be executed -> is improperly configured\r
 269                 // should be ignored\r
 270                 throw new InvalidParameterException(\r
 271                                 "Executable is not provided for any runtime environments");\r
 272         }\r
 273 \r
 274         public static ConfiguredExecutable<?> loadExecutable(String taskId)\r
 275                         throws ResultNotAvailableException {\r
 276                 String workDir = compbio.engine.Configurator.getWorkDirectory(taskId);\r
 277                 // The results for this job has been collected once, or the JVM may\r
 278                 // have been restarted,\r
 279                 // so that the job is not in the job list\r
 280                 // ->load a ConfiguredExercutable from saved run and return it\r
 281                 FileInputStream fileInStream = null;\r
 282                 ConfiguredExecutable<?> exec = null;\r
 283                 try {\r
 284                         fileInStream = new FileInputStream(workDir + File.separator\r
 285                                         + RunConfiguration.rconfigFile);\r
 286                         RunConfiguration rconf = RunConfiguration.load(fileInStream);\r
 287                         exec = ConfExecutable.newConfExecutable(rconf);\r
 288                         fileInStream.close();\r
 289                 } catch (FileNotFoundException e) {\r
 290                         log.error("Could not find run configuration to load!"\r
 291                                         + e.getLocalizedMessage(), e.getCause());\r
 292                         throw new ResultNotAvailableException(\r
 293                                         "Could not find run configuration to load!"\r
 294                                                         + e.getMessage(), e.getCause());\r
 295                 } catch (IOException e) {\r
 296                         log.error("IO Exception while reading run configuration file!"\r
 297                                         + e.getLocalizedMessage(), e.getCause());\r
 298                         throw new ResultNotAvailableException(\r
 299                                         "Could not load run configuration!" + e.getMessage(), e\r
 300                                                         .getCause());\r
 301                 } finally {\r
 302                         FileUtil.closeSilently(log, fileInStream);\r
 303                 }\r
 304                 return exec;\r
 305         }\r
 306 \r
 307         /**\r
 308          * For now just assume that all parameters which came in needs setting it\r
 309          * will be a client responsibility to prepare RunnerConfig object then\r
 310          * \r
 311          * @param rconfig\r
 312          * @return\r
 313          * \r
 314          *         public static List<String> toOptionString(RunnerConfig<?>\r
 315          *         rconfig) { String option = ""; List<String> options = new\r
 316          *         ArrayList<String>(); for (Parameter<?> par :\r
 317          *         rconfig.getParameters()) { if (par.getPossibleValues().isEmpty())\r
 318          *         { option = par.getOptionName(); } else { option =\r
 319          *         par.getOptionName() + "=" + par.getPossibleValues().get(0); } //\r
 320          *         separate options options.add(option); } return options; }\r
 321          */\r
 322         \r
 323         public static <T> LimitsManager<T> getLimits(Class<Executable<T>> clazz) {\r
 324                 LimitsManager<T> limits = null;\r
 325                 try {\r
 326                         limits = ConfExecutable.getRunnerLimits(clazz);\r
 327                 } catch (FileNotFoundException e) {\r
 328                         Util.log.warn(\r
 329                                         "No limits are found for " + clazz + " executable! "\r
 330                                                         + e.getLocalizedMessage(), e.getCause());\r
 331                         // its ok, limit may not be initialized\r
 332                 } catch (IOException e) {\r
 333                         Util.log.warn("IO exception while attempting to read limits for "\r
 334                                         + clazz + " executable! " + e.getLocalizedMessage(),\r
 335                                         e.getCause());\r
 336                 }\r
 337                 return limits;\r
 338         }\r
 339 \r
 340 }\r