1 package compbio.ws.execstat;
\r
4 import java.io.FileFilter;
\r
5 import java.io.FileWriter;
\r
6 import java.io.IOException;
\r
7 import java.sql.SQLException;
\r
8 import java.text.SimpleDateFormat;
\r
9 import java.util.ArrayList;
\r
10 import java.util.HashMap;
\r
11 import java.util.HashSet;
\r
12 import java.util.List;
\r
13 import java.util.Map;
\r
15 import org.apache.log4j.Logger;
\r
17 import compbio.engine.client.ConfExecutable;
\r
18 import compbio.engine.conf.PropertyHelperManager;
\r
19 import compbio.metadata.JobStatus;
\r
20 import compbio.util.FileUtil;
\r
21 import compbio.util.PropertyHelper;
\r
22 import compbio.ws.client.Services;
\r
25 * Number of runs of each WS = number of folders with name
\r
27 * Number of successful runs = all runs with no result file
\r
29 * Per period of time = limit per file creating time Runtime (avg/max) =
\r
31 * started time - finished time
\r
33 * Task & result size = result.size
\r
35 * Abandoned runs - not collected runs
\r
37 * Cancelled runs - cancelled
\r
39 * Cluster vs local runs
\r
41 * Reasons for failure = look in the err out?
\r
44 * Metadata required:
\r
46 * work directory for local and cluster tasks = from Helper or cmd parameter. WS
\r
47 * names - enumeration. Status file names and content.
\r
52 public class ExecutionStatCollector {
\r
54 static final int UNDEFINED = -1;
\r
56 private static final Logger log = Logger
\r
57 .getLogger(ExecutionStatCollector.class);
\r
59 static SimpleDateFormat DF = new SimpleDateFormat("dd/MM/yyyy hh:mm:ss");
\r
61 static PropertyHelper ph = PropertyHelperManager.getPropertyHelper();
\r
63 static String getClusterJobDir() {
\r
64 String clusterdir = ph.getProperty("cluster.tmp.directory");
\r
65 if (clusterdir != null) {
\r
71 static void updateTime(File statFile) throws IOException {
\r
72 long lastMod = statFile.lastModified();
\r
73 FileWriter fw = new FileWriter(statFile);
\r
74 fw.write(new Long(lastMod).toString());
\r
78 static String getLocalJobDir() {
\r
79 String locdir = ph.getProperty("local.tmp.directory");
\r
80 if (locdir != null) {
\r
89 * @throws IOException
\r
90 * @throws SQLException
\r
92 public static void main(String[] args) throws IOException, SQLException {
\r
94 // updateTime(new File(
\r
95 // "D:\\workspace\\JABA2\\jobsout\\AACon#170462904473672\\STARTED"));
\r
97 String workDir = PropertyHelperManager.getLocalPath()
\r
98 + getLocalJobDir().trim();
\r
99 System.out.println(workDir);
\r
100 File[] files = FileUtil.getFiles("H:/www-jws2/job_dir/local_jobsout",
\r
102 List<StatProcessor.JobStat> stats = new ArrayList<StatProcessor.JobStat>();
\r
103 for (File file : files) {
\r
104 JobDirectory jd = new JobDirectory(file);
\r
105 stats.add(jd.getJobStat());
\r
106 // System.out.println(jd.getJobStat().getJobReportTabulated());
\r
108 StatProcessor sp = new StatProcessor(stats);
\r
109 System.out.println(sp.reportStat());
\r
110 System.out.println();
\r
111 System.out.println("!!!!!!!!!!!!!!!!!!");
\r
112 System.out.println();
\r
113 System.out.println(sp.getSingleWSStat(Services.TcoffeeWS).reportStat());
\r
115 StatDB.insertData(new HashSet<StatProcessor.JobStat>(sp
\r
116 .getSingleWSStat(Services.TcoffeeWS).stats));
\r
119 static FileFilter directories = new FileFilter() {
\r
121 public boolean accept(File pathname) {
\r
122 return pathname.isDirectory();
\r
126 static class JobDirectory {
\r
129 Map<String, File> files = new HashMap<String, File>();
\r
131 public JobDirectory(File directory) {
\r
132 this.jobdir = directory;
\r
133 for (File f : jobdir.listFiles()) {
\r
134 files.put(f.getName(), f);
\r
138 public boolean hasStatus(JobStatus status) {
\r
139 return files.containsKey(status.toString());
\r
142 boolean isCollected() {
\r
143 return hasStatus(JobStatus.COLLECTED);
\r
146 boolean isCancelled() {
\r
147 return hasStatus(JobStatus.CANCELLED);
\r
150 long getStartTime() {
\r
151 long starttime = UNDEFINED;
\r
152 File startfile = files.get(JobStatus.STARTED.toString());
\r
153 if (startfile == null) {
\r
154 startfile = files.get(JobStatus.SUBMITTED.toString());
\r
156 if (startfile != null) {
\r
157 starttime = startfile.lastModified();
\r
159 * String start = FileUtil.readFileToString(startfile);
\r
160 * starttime = Long.parseLong(start.trim());
\r
166 String getClusterJobID() {
\r
167 String clustjobId = "";
\r
168 File jobid = files.get("JOBID");
\r
170 if (jobid != null) {
\r
171 clustjobId = FileUtil.readFileToString(jobid);
\r
173 } catch (IOException ioe) {
\r
174 ioe.printStackTrace();
\r
177 return clustjobId.trim();
\r
180 long getFinishedTime() {
\r
181 long ftime = UNDEFINED;
\r
182 File finished = files.get(JobStatus.FINISHED.toString());
\r
183 if (finished != null) {
\r
184 ftime = finished.lastModified();
\r
186 * String start = FileUtil.readFileToString(finished); ftime =
\r
187 * Long.parseLong(start.trim());
\r
189 // System.out.println("f " + ftime);
\r
192 * } catch (IOException e) { log.log(Level.WARN,
\r
193 * "Cannot parse finished time: " + e.getMessage(), e); } catch
\r
194 * (NumberFormatException e) { log.log(Level.WARN,
\r
195 * "Cannot parse finished time: " + e.getMessage(), e); }
\r
200 String getWSName() {
\r
201 String name = jobdir.getName().split("#")[0];
\r
202 if (name.startsWith(ConfExecutable.CLUSTER_TASK_ID_PREFIX)) {
\r
203 assert ConfExecutable.CLUSTER_TASK_ID_PREFIX.length() == 1;
\r
204 name = name.substring(1);
\r
206 if (name.startsWith("ClustalW")) {
\r
207 name = name.trim().substring(name.length() - 1);
\r
212 Services getService() {
\r
213 return Services.getService(getWSName() + "WS");
\r
215 // Mafft, Muscle, Tcoffee, Clustal task:fasta.in result:fasta.out
\r
216 // Probcons task:fasta.in result:alignment.out
\r
218 * TODO replace with Universal names for WS!
\r
220 long getResultSize() {
\r
221 String name = getWSName();
\r
223 if (name.equalsIgnoreCase("Probcons")) {
\r
224 f = files.get("alignment.out");
\r
226 f = files.get("fasta.out");
\r
233 long getInputSize() {
\r
234 File input = files.get("fasta.in");
\r
235 if (input != null) {
\r
236 return input.length();
\r
241 StatProcessor.JobStat getJobStat() {
\r
242 return new StatProcessor.JobStat(getService(), getClusterJobID(),
\r
243 jobdir.getName(), getStartTime(), getFinishedTime(),
\r
244 getInputSize(), getResultSize(), isCollected(),
\r
249 public int hashCode() {
\r
250 final int prime = 31;
\r
252 result = prime * result
\r
253 + ((jobdir == null) ? 0 : jobdir.hashCode());
\r
258 public boolean equals(Object obj) {
\r
263 if (getClass() != obj.getClass())
\r
265 JobDirectory other = (JobDirectory) obj;
\r
266 if (jobdir == null) {
\r
267 if (other.jobdir != null)
\r
269 } else if (!jobdir.equals(other.jobdir))
\r