From dd7d61d4081ea595dc9a9befe8bd5fb6d2453f01 Mon Sep 17 00:00:00 2001 From: Sasha Date: Fri, 14 Jun 2013 12:03:42 +0100 Subject: [PATCH] Code for cleaning up old job directories --- engine/compbio/engine/Cleaner.java | 6 +- .../compbio/engine/local/LocalExecutorService.java | 3 +- .../collector/ExecutionStatCollectorTester.java | 2 +- webservices/compbio/stat/collector/DirCleaner.java | 19 +- .../compbio/stat/collector/JobDirectory.java | 197 ++++++++++++++++++++ webservices/compbio/ws/server/MainManager.java | 128 +++++++++++++ 6 files changed, 345 insertions(+), 10 deletions(-) create mode 100644 webservices/compbio/stat/collector/JobDirectory.java create mode 100644 webservices/compbio/ws/server/MainManager.java diff --git a/engine/compbio/engine/Cleaner.java b/engine/compbio/engine/Cleaner.java index ce9c774..31c97a8 100644 --- a/engine/compbio/engine/Cleaner.java +++ b/engine/compbio/engine/Cleaner.java @@ -22,11 +22,9 @@ import java.util.List; import org.apache.log4j.Logger; -import compbio.engine.conf.PropertyHelperManager; import compbio.engine.client.ConfiguredExecutable; import compbio.engine.client.PathValidator; import compbio.engine.local.ExecutableWrapper; -import compbio.util.PropertyHelper; //@Deprecated @@ -112,8 +110,12 @@ public class Cleaner { } else { if (f.delete()) { deletedCount++; + log.debug("file " + f.getName() + " removed"); + } else { + log.debug("file " + f.getName() + " is not removed"); } } + } rootdir.delete(); return deletedCount == files.length; diff --git a/engine/compbio/engine/local/LocalExecutorService.java b/engine/compbio/engine/local/LocalExecutorService.java index cbed001..49e506c 100644 --- a/engine/compbio/engine/local/LocalExecutorService.java +++ b/engine/compbio/engine/local/LocalExecutorService.java @@ -32,8 +32,7 @@ import compbio.util.Util; public final class LocalExecutorService extends ThreadPoolExecutor { - private final static Logger log = Logger - .getLogger(LocalExecutorService.class); + private final static Logger log = Logger.getLogger(LocalExecutorService.class); private final static String threadNumPropName = "engine.local.thread.number"; private static LocalExecutorService INSTANCE = null; diff --git a/testsrc/compbio/stat/collector/ExecutionStatCollectorTester.java b/testsrc/compbio/stat/collector/ExecutionStatCollectorTester.java index 1489dcd..b99b36c 100644 --- a/testsrc/compbio/stat/collector/ExecutionStatCollectorTester.java +++ b/testsrc/compbio/stat/collector/ExecutionStatCollectorTester.java @@ -15,7 +15,7 @@ import org.testng.Assert; import org.testng.annotations.Test; import compbio.metadata.AllTestSuit; -import compbio.stat.collector.ExecutionStatCollector.JobDirectory; +import compbio.stat.collector.JobDirectory; public class ExecutionStatCollectorTester { diff --git a/webservices/compbio/stat/collector/DirCleaner.java b/webservices/compbio/stat/collector/DirCleaner.java index d10461e..0171128 100644 --- a/webservices/compbio/stat/collector/DirCleaner.java +++ b/webservices/compbio/stat/collector/DirCleaner.java @@ -95,7 +95,9 @@ public class DirCleaner implements Runnable { } boolean livesOverLifeSpan(JobDirectory jd) { - return ((System.currentTimeMillis() - jd.jobdir.lastModified()) / (1000 * 60 * 60)) > LifeSpanInHours; + long LifeTime = (System.currentTimeMillis() - jd.jobdir.lastModified()) / (1000 * 60 * 60); + log.debug("lifetime = " + LifeTime + ", lifespan = " + LifeSpanInHours); + return LifeTime > LifeSpanInHours; } static FileFilter directories = new FileFilter() { @@ -111,11 +113,18 @@ public class DirCleaner implements Runnable { for (File dir : dirs) { // Do not look at dirs with unfinished jobs JobDirectory jd = new JobDirectory(dir); - if (hasCompleted(jd) && livesOverLifeSpan(jd)) { - Cleaner.deleteDirectory(workDirectory.getAbsolutePath() + File.separator + dir.getName()); - log.debug("Directory " + dir.getName() + " is deleted in doCleaning"); + Date d = new Date (dir.lastModified()); + log.debug("Directory " + dir.getName() + " has timestamp: " + d); + // TODO. removed hasCompeted. Maybe it needs to be restored... + // if (hasCompleted(jd) && livesOverLifeSpan(jd)) { + if (livesOverLifeSpan(jd)) { + if (Cleaner.deleteDirectory(workDirectory.getAbsolutePath() + File.separator + dir.getName())) { + log.error("Directory " + dir.getName() + " failed to deleted..."); + } else { + log.debug("Directory " + dir.getName() + " is deleted"); + } } else { - log.debug("Directory " + dir.getName() + " is too new and kept in doCleaning"); + log.debug("Directory " + dir.getName() + " is too new and kept"); } } } diff --git a/webservices/compbio/stat/collector/JobDirectory.java b/webservices/compbio/stat/collector/JobDirectory.java new file mode 100644 index 0000000..51c0e6e --- /dev/null +++ b/webservices/compbio/stat/collector/JobDirectory.java @@ -0,0 +1,197 @@ +/* Copyright (c) 2013 Alexander Sherstnev + * + * JAva Bioinformatics Analysis Web Services (JABAWS) @version: 2.0 + * + * This library is free software; you can redistribute it and/or modify it under the terms of the + * Apache License version 2 as published by the Apache Software Foundation + * + * This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without + * even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Apache + * License for more details. + * + * A copy of the license is in apache_license.txt. It is also available here: + * @see: http://www.apache.org/licenses/LICENSE-2.0.txt + * + * Any republication or derived work distributed in source code form + * must include this copyright and license notice. + */ +package compbio.stat.collector; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.log4j.Logger; + +import compbio.engine.client.Executable; +import compbio.engine.client.SkeletalExecutable; +import compbio.metadata.JobStatus; +import compbio.util.FileUtil; +import compbio.ws.client.Services; +import compbio.ws.client.ServicesUtil; + +/** + * + * @author Alexander Sherstnev + * + */ +public class JobDirectory { + + static final int UNDEFINED = -1; + + private static final Logger log = Logger.getLogger(JobDirectory.class); + + File jobdir; + Map files = new HashMap(); + + JobDirectory(File directory) { + this.jobdir = directory; + for (File f : jobdir.listFiles()) { + files.put(f.getName(), f); + } + } + + boolean hasStatus(JobStatus status) { + return files.containsKey(status.toString()); + } + + boolean isCollected() { + return hasStatus(JobStatus.COLLECTED); + } + + boolean isCancelled() { + return hasStatus(JobStatus.CANCELLED); + } + + long getStartTime() { + long starttime = UNDEFINED; + File startfile = files.get(JobStatus.STARTED.toString()); + if (startfile == null) { + startfile = files.get(JobStatus.SUBMITTED.toString()); + } + try { + if (startfile != null) { + String start = FileUtil.readFileToString(startfile); + starttime = Long.parseLong(start.trim()); + } + } catch (IOException ignore) { + log.warn("IOException while reading STARTED status file! Ignoring...", ignore); + // fall back + starttime = startfile.lastModified(); + } catch (NumberFormatException ignore) { + log.warn("NumberFormatException while reading STARTED status file! Ignoring...", ignore); + // fall back + starttime = startfile.lastModified(); + } + return starttime; + } + + String getClusterJobID() { + String clustjobId = ""; + File jobid = files.get("JOBID"); + try { + if (jobid != null) { + clustjobId = FileUtil.readFileToString(jobid); + } + } catch (IOException ioe) { + log.error( + "IO Exception while reading the content of JOBID file for job " + + jobid, ioe); + } + return clustjobId.trim(); + } + + long getFinishedTime() { + long ftime = UNDEFINED; + File finished = files.get(JobStatus.FINISHED.toString()); + if (finished != null) { + try { + if (finished != null) { + String start = FileUtil.readFileToString(finished); + ftime = Long.parseLong(start.trim()); + } + } catch (IOException ignore) { + log.warn( + "IOException while reading FINISHED status file! Ignoring...", + ignore); + // fall back + ftime = finished.lastModified(); + } catch (NumberFormatException ignore) { + log.warn( + "NumberFormatException while reading FINISHED status file! Ignoring...", + ignore); + // fall back + ftime = finished.lastModified(); + } + } + return ftime; + } + + private Services getService() { + return ServicesUtil.getServiceByJobDirectory(jobdir); + } + + long getResultSize() { + Class> name = ServicesUtil + .getRunnerByJobDirectory(jobdir); + + File f = null; + if (name.getSimpleName().equalsIgnoreCase("IUPred")) { + f = files.get("out.glob"); + if (f == null) + f = files.get("out.short"); + if (f == null) + f = files.get("out.long"); + } else { + f = files.get(SkeletalExecutable.OUTPUT); + } + if (f != null) { + return f.length(); + } + return UNDEFINED; + } + + long getInputSize() { + Class> name = ServicesUtil + .getRunnerByJobDirectory(jobdir); + + File input = files.get(SkeletalExecutable.INPUT); + if (input != null) { + return input.length(); + } + return UNDEFINED; + } + + JobStat getJobStat() { + return JobStat.newInstance(getService(), getClusterJobID(), + jobdir.getName(), getStartTime(), getFinishedTime(), + getInputSize(), getResultSize(), isCancelled(), + isCollected()); + } + + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((jobdir == null) ? 0 : jobdir.hashCode()); + return result; + } + + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + JobDirectory other = (JobDirectory) obj; + if (jobdir == null) { + if (other.jobdir != null) + return false; + } else if (!jobdir.equals(other.jobdir)) + return false; + return true; + } +} + diff --git a/webservices/compbio/ws/server/MainManager.java b/webservices/compbio/ws/server/MainManager.java new file mode 100644 index 0000000..54f1daf --- /dev/null +++ b/webservices/compbio/ws/server/MainManager.java @@ -0,0 +1,128 @@ +/* Copyright (c) 2011 Peter Troshin + * + * JAva Bioinformatics Analysis Web Services (JABAWS) @version: 2.0 + * + * This library is free software; you can redistribute it and/or modify it under the terms of the + * Apache License version 2 as published by the Apache Software Foundation + * + * This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without + * even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Apache + * License for more details. + * + * A copy of the license is in apache_license.txt. It is also available here: + * @see: http://www.apache.org/licenses/LICENSE-2.0.txt + * + * Any republication or derived work distributed in source code form + * must include this copyright and license notice. + */ +package compbio.ws.server; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; + +import org.apache.log4j.Logger; + +import compbio.stat.collector.DirCleaner; +import compbio.stat.collector.StatDB; +import compbio.engine.conf.PropertyHelperManager; +import compbio.engine.local.ExecutableWrapper; +import compbio.engine.local.LocalExecutorService; +import compbio.util.PropertyHelper; +import compbio.util.Util; + +/** + * Two tasks: + * 1. Switch off engines if JABAWS web application is un-deployed, or web server is shutdown + * 2. delete old job directories + * + * @author Peter Troshin + * @author Alexander Sherstnev + * @version 2.0 + */ +public class MainManager implements ServletContextListener { + + private final Logger log = Logger.getLogger(MainManager.class); + static PropertyHelper ph = PropertyHelperManager.getPropertyHelper(); + + private ScheduledFuture localcl; + private ScheduledFuture clustercl; + private ScheduledExecutorService executor; + + @Override + public void contextDestroyed(ServletContextEvent ignored) { + // stop cleaning job directories +// try { + if (null != localcl) { + localcl.cancel(true); + } + if (null != clustercl) { + clustercl.cancel(true); + } + //executor.shutdown(); + //executor.awaitTermination(3, TimeUnit.SECONDS); +// } catch (InterruptedException e) { +// log.warn(e.getMessage(), e); +// } + // Shutdown local engine + log.info("JABAWS context is destroyed. Shutting down engines..."); + LocalExecutorService.shutDown(); + log.info("Local engine is shutdown OK"); + ExecutableWrapper.shutdownService(); + log.info("Individual executables stream engine is shutdown OK"); + } + + @Override + public void contextInitialized(ServletContextEvent arg0) { + log.info("Initializing directory cleaners"); + executor = Executors.newScheduledThreadPool(2); + + // configure cluster cleaner + String clusterWorkDir = getClusterJobDir(); + int clusterDirLifespan = PropertyHelperManager.getIntProperty(ph.getProperty("cluster.jobdir.maxlifespan")); + int clusterCleaningRate = PropertyHelperManager.getIntProperty(ph.getProperty("cluster.jobdir.cleaning.frequency")); + boolean cleanClasterDir = PropertyHelperManager.getBooleanProperty(ph.getProperty("cluster.stat.collector.enable")); + + if (0 < clusterDirLifespan && cleanClasterDir) { + DirCleaner clusterDirCleaner = new DirCleaner( clusterWorkDir, clusterDirLifespan); + clustercl = executor.scheduleAtFixedRate(clusterDirCleaner, 1, clusterCleaningRate, TimeUnit.MINUTES); + log.info("Cleaning local job directory every " + clusterCleaningRate + " minutes"); + } else { + log.info("Cluster job directory cleaner is disabled. "); + } + + // configure local cleaner + String localWorkDir = compbio.engine.client.Util.convertToAbsolute(getLocalJobDir()); + int localDirLiveSpan = PropertyHelperManager.getIntProperty(ph.getProperty("local.jobdir.maxlifespan")); + int localCleaningRate = PropertyHelperManager.getIntProperty(ph.getProperty("local.jobdir.cleaning.frequency")); + boolean cleanLocalDir = PropertyHelperManager.getBooleanProperty(ph.getProperty("local.stat.collector.enable")); + + if (0 < localDirLiveSpan && cleanLocalDir) { + DirCleaner localDirCleaner = new DirCleaner( localWorkDir, localDirLiveSpan); + localcl = executor.scheduleAtFixedRate(localDirCleaner, 1, localCleaningRate, TimeUnit.MINUTES); + log.info("Cleaning local job directory every " + localCleaningRate + " minutes"); + } else { + log.info("Local job directory cleaner is disabled. "); + } + } + + static String getClusterJobDir() { + String ln = ph.getProperty("cluster.tmp.directory"); + if (null != ln ) { + ln = ln.trim(); + } + return ln; + } + + static String getLocalJobDir() { + String ln = ph.getProperty("local.tmp.directory"); + if (null != ln ) { + ln = ln.trim(); + } + return ln; + } +} -- 1.7.10.2