From e016e819d1f9d851aa28a273414b4432c2137858 Mon Sep 17 00:00:00 2001 From: Mateusz Warowny Date: Thu, 18 Nov 2021 21:10:19 +0100 Subject: [PATCH] JAL-3878 Add slivka's implementations of web service and discoverer. --- src/jalview/ws2/slivka/SlivkaWSDiscoverer.java | 210 ++++++++++++++++ src/jalview/ws2/slivka/SlivkaWebService.java | 304 ++++++++++++++++++++++++ 2 files changed, 514 insertions(+) create mode 100644 src/jalview/ws2/slivka/SlivkaWSDiscoverer.java create mode 100644 src/jalview/ws2/slivka/SlivkaWebService.java diff --git a/src/jalview/ws2/slivka/SlivkaWSDiscoverer.java b/src/jalview/ws2/slivka/SlivkaWSDiscoverer.java new file mode 100644 index 0000000..802254b --- /dev/null +++ b/src/jalview/ws2/slivka/SlivkaWSDiscoverer.java @@ -0,0 +1,210 @@ +package jalview.ws2.slivka; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.*; +import java.util.concurrent.*; + +import jalview.bin.Cache; +import jalview.ws2.*; +import jalview.ws2.operations.*; +import uk.ac.dundee.compbio.slivkaclient.SlivkaClient; +import uk.ac.dundee.compbio.slivkaclient.SlivkaService; + +public class SlivkaWSDiscoverer implements WebServiceDiscovererI +{ + private static final String SLIVKA_HOST_URLS = "SLIVKSHOSTURLS"; + + private static final String DEFAULT_URL = "https://www.compbio.dundee.ac.uk/slivka/"; + + private static SlivkaWSDiscoverer instance = null; + + private List operations = List.of(); + + private SlivkaWSDiscoverer() + { + } + + public static SlivkaWSDiscoverer getInstance() + { + if (instance == null) + { + instance = new SlivkaWSDiscoverer(); + } + return instance; + } + + @Override + public List getUrls() + { + String surls = Cache.getDefault(SLIVKA_HOST_URLS, DEFAULT_URL); + String urls[] = surls.split(","); + ArrayList valid = new ArrayList<>(urls.length); + for (String url : urls) + { + try + { + new URL(url); + valid.add(url); + } catch (MalformedURLException e) + { + Cache.log.warn("Problem whilst trying to make a URL from '" + + Objects.toString(url, "") + "'. " + + "This was probably due to malformed comma-separated-list " + + "in the " + SLIVKA_HOST_URLS + + " entry of ${HOME}/.jalview_properties"); + Cache.log.debug("Exception occurred while reading url list", e); + } + } + return valid; + } + + @Override + public void setUrls(List wsUrls) + { + if (wsUrls != null && !wsUrls.isEmpty()) + { + Cache.setProperty(SLIVKA_HOST_URLS, String.join(",", wsUrls)); + } + else + { + Cache.removeProperty(SLIVKA_HOST_URLS); + } + } + + @Override + public boolean testUrl(URL url) + { + return getStatusForUrl(url.toString()) == STATUS_OK; + } + + @Override + public int getStatusForUrl(String url) + { + try + { + List services = new SlivkaClient(url).getServices(); + return services.isEmpty() ? STATUS_NO_SERVICES : STATUS_OK; + } catch (IOException e) + { + Cache.log.error("Slivka could not retrieve services list from " + url, + e); + return STATUS_INVALID; + } + } + + @Override + public List getOperations() + { + return Collections.unmodifiableList(operations); + } + + @Override + public boolean hasServices() + { + return !isRunning() && operations.size() > 0; + } + + public boolean isRunning() + { + for (Future task : discoveryTasks) + { + if (!task.isDone()) + { + return true; + } + } + return false; + } + + public boolean isDone() + { + return !isRunning() && discoveryTasks.size() > 0; + } + + private Vector> discoveryTasks = new Vector<>(); + + @Override + public CompletableFuture startDiscoverer() + { + CompletableFuture task = CompletableFuture + .supplyAsync(() -> { + reloadServices(); + return SlivkaWSDiscoverer.this; + }); + task.thenRun(() -> fireOperationsChanged(getOperations())); + discoveryTasks.add(task); + return task; + } + + private List reloadServices() + { + Cache.log.info("Reloading Slivka services"); + fireOperationsChanged(Collections.emptyList()); + ArrayList allOperations= new ArrayList<>(); + for (String url : getUrls()) + { + SlivkaClient client = new SlivkaClient(url); + List services; + try + { + services = client.getServices(); + } catch (IOException e) + { + Cache.log.error("Unable to fetch services from " + url, e); + continue; + } + for (SlivkaService service : services) + { + SlivkaWebService webService = new SlivkaWebService(client, service); + AbstractOperation op = null; + for (String classifier : service.classifiers) + { + String[] path = classifier.split("\\s*::\\s*"); + if (path.length >= 3 && path[0].toLowerCase().equals("operation") + && path[1].toLowerCase().equals("analysis")) + { + switch (path[path.length - 1].toLowerCase()) + { + case "rna secondary structure prediction": + op = new OperationStub(webService, "Secondary Structure Prediction"); + op.setInteractive(true); + op.setAlignmentAnalysis(true); + op.setProteinOperation(false); + break; + case "sequence alignment analysis (conservation)": + op = new OperationStub(webService, "Conservation"); + op.setAlignmentAnalysis(true); + op.setInteractive(true); + break; + case "protein sequence analysis": + op = new OperationStub(webService, "Protein Disorder"); + break; + case "multiple sequence alignment": + op = new OperationStub(webService, "Alignment"); + break; + } + if (op != null) + { + break; + } + } + } + if (op != null) { + allOperations.add(op); + } + } + } + this.operations = allOperations; + Cache.log.info("Reloading slivka services finished"); + return allOperations; + } + + @Override + public String getErrorMessages() + { + return ""; + } + +} diff --git a/src/jalview/ws2/slivka/SlivkaWebService.java b/src/jalview/ws2/slivka/SlivkaWebService.java new file mode 100644 index 0000000..a2d7f20 --- /dev/null +++ b/src/jalview/ws2/slivka/SlivkaWebService.java @@ -0,0 +1,304 @@ +package jalview.ws2.slivka; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.EnumMap; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import jalview.api.AlignViewportI; +import jalview.bin.Cache; +import jalview.datamodel.Alignment; +import jalview.datamodel.AlignmentAnnotation; +import jalview.datamodel.AlignmentI; +import jalview.datamodel.SequenceI; +import jalview.io.AnnotationFile; +import jalview.io.DataSourceType; +import jalview.io.FeaturesFile; +import jalview.io.FileFormat; +import jalview.io.FileFormatI; +import jalview.io.FormatAdapter; +import jalview.io.JPredFile; +import jalview.ws.gui.WsJob; +import jalview.ws.params.ArgumentI; +import jalview.ws.params.ParamDatastoreI; +import jalview.ws.params.WsParamSetI; +import jalview.ws.slivkaws.SlivkaDatastore; +import jalview.ws2.WebServiceI; +import jalview.ws2.operations.Operation; +import jalview.ws2.WSJob; +import jalview.ws2.WSJobStatus; +import javajs.http.ClientProtocolException; +import uk.ac.dundee.compbio.slivkaclient.Job; +import uk.ac.dundee.compbio.slivkaclient.Parameter; +import uk.ac.dundee.compbio.slivkaclient.RemoteFile; +import uk.ac.dundee.compbio.slivkaclient.SlivkaClient; +import uk.ac.dundee.compbio.slivkaclient.SlivkaService; + +public class SlivkaWebService implements WebServiceI +{ + protected final SlivkaClient client; + + protected final SlivkaService service; + + protected ParamDatastoreI store; + + protected static final EnumMap statusMap = new EnumMap<>( + Job.Status.class); + { + statusMap.put(Job.Status.PENDING, WSJobStatus.SUBMITTED); + statusMap.put(Job.Status.REJECTED, WSJobStatus.INVALID); + statusMap.put(Job.Status.ACCEPTED, WSJobStatus.QUEUED); + statusMap.put(Job.Status.QUEUED, WSJobStatus.QUEUED); + statusMap.put(Job.Status.RUNNING, WSJobStatus.RUNNING); + statusMap.put(Job.Status.COMPLETED, WSJobStatus.FINISHED); + statusMap.put(Job.Status.INTERRUPTED, WSJobStatus.CANCELLED); + statusMap.put(Job.Status.DELETED, WSJobStatus.CANCELLED); + statusMap.put(Job.Status.FAILED, WSJobStatus.FAILED); + statusMap.put(Job.Status.ERROR, WSJobStatus.SERVER_ERROR); + statusMap.put(Job.Status.UNKNOWN, WSJobStatus.UNKNOWN); + } + + public SlivkaWebService(SlivkaClient client, SlivkaService service) + { + this.client = client; + this.service = service; + } + + @Override + public String getHostName() + { + return client.getUrl().toString(); + } + + @Override + public String getProviderName() + { + return "slivka"; + } + + @Override + public String getName() + { + return service.getName(); + } + + @Override + public String getDescription() + { + return service.getDescription(); + } + + @Override + public boolean hasParameters() + { + return getParamStore().getServiceParameters().size() > 0; + } + + @Override + public ParamDatastoreI getParamStore() + { + if (store == null) + { + store = new SlivkaDatastore(service); + } + return store; + } + + @Override + public String submit(List sequences, List args) + throws IOException + { + var request = new uk.ac.dundee.compbio.slivkaclient.JobRequest(); + for (Parameter param : service.getParameters()) + { + if (param instanceof Parameter.FileParameter) + { + // if finds a file input, gives it sequences stream + Parameter.FileParameter fileParam = (Parameter.FileParameter) param; + FileFormat format; + switch (fileParam.getMediaType()) + { + case "application/pfam": + format = FileFormat.Pfam; + break; + case "application/stockholm": + format = FileFormat.Stockholm; + break; + case "application/clustal": + format = FileFormat.Clustal; + break; + case "application/fasta": + default: + format = FileFormat.Fasta; + break; + } + InputStream stream = new ByteArrayInputStream(format.getWriter(null) + .print(sequences.toArray(new SequenceI[0]), false) + .getBytes()); + request.addFile(param.getId(), stream); + } + } + if (args != null) + { + for (ArgumentI arg : args) + { + // multiple choice field names are name$number to avoid duplications + // the number is stripped here + String paramId = arg.getName().split("\\$", 2)[0]; + Parameter param = service.getParameter(paramId); + if (param instanceof Parameter.FlagParameter) + { + if (arg.getValue() != null && !arg.getValue().isBlank()) + request.addData(paramId, true); + else + request.addData(paramId, false); + } + else + { + request.addData(paramId, arg.getValue()); + } + } + } + var job = service.submitJob(request); + return job.getId(); + } + + @Override + public void updateProgress(WSJob job) throws IOException + { + var slivkaJob = client.getJob(job.getJobId()); + job.setStatus(statusMap.get(slivkaJob.getStatus())); + Collection files = slivkaJob.getResults(); + for (RemoteFile f : files) + { + if (f.getLabel().equals("log")) + { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + f.writeTo(stream); + job.setLog(stream.toString("UTF-8")); + } + else if (f.getLabel().equals("error-log")) + { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + f.writeTo(stream); + job.setErrorLog(stream.toString("UTF-8")); + } + } + } + + @Override + public void cancel(WSJob job) throws IOException + { + job.setStatus(WSJobStatus.CANCELLED); + Cache.log.warn("Slivka does not support job cancellation yet."); + } + + @Override + public boolean handleSubmissionError(WSJob job, Exception ex) + { + if (ex instanceof ClientProtocolException) + { + Cache.log.error("Job submission failed due to exception.", ex); + return true; + } + return false; + } + + @Override + public boolean handleCollectionError(WSJob job, Exception ex) + { + // TODO Auto-generated method stub + return false; + } + + public AlignmentI getAlignment(WSJob job, List dataset, + AlignViewportI viewport) throws IOException + { + Collection files; + var slivkaJob = client.getJob(job.getJobId()); + files = slivkaJob.getResults(); + for (RemoteFile f : files) + { + if (f.getMediaType().equals("application/clustal")) + { + return new FormatAdapter().readFile(f.getContentUrl().toString(), + DataSourceType.URL, FileFormat.Clustal); + } + else if (f.getMediaType().equals("application/fasta")) + { + return new FormatAdapter().readFile(f.getContentUrl().toString(), + DataSourceType.URL, FileFormat.Fasta); + } + } + return null; + } + + public FeaturesFile getFeaturesFile(WSJob job, + List dataset, AlignViewportI viewport) throws IOException + { + var slivkaJob = client.getJob(job.getJobId()); + Collection files = slivkaJob.getResults(); + for (RemoteFile f : files) + { + if (f.getMediaType().equals("application/jalview-features")) + { + return new FeaturesFile(f.getContentUrl().toString(), DataSourceType.URL); + } + } + return null; + } + + public List getAnnotations(WSJob job, + List dataset, AlignViewportI viewport) throws IOException + { + var slivkaJob = client.getJob(job.getJobId()); + Collection files = slivkaJob.getResults(); + for (RemoteFile f : files) + { + if (f.getMediaType().equals("application/jalview-annotations")) + { + Alignment aln = new Alignment(dataset.toArray(new SequenceI[0])); + AnnotationFile af = new AnnotationFile(); + boolean valid = af.readAnnotationFileWithCalcId(aln, service.getId(), + f.getContentUrl().toString(), DataSourceType.URL); + if (valid) + { + return Arrays.asList(aln.getAlignmentAnnotation()); + } + else + { + throw new IOException("Unable to read annotations from file " + + f.getContentUrl().toString()); + } + } + } + return null; + } + + public JPredFile getPrediction(WSJob job, List dataset, + AlignViewportI viewport) throws IOException + { + Collection files = client.getJob(job.getJobId()).getResults(); + for (RemoteFile f : files) + { + if (f.getLabel().equals("concise")) + { + return new JPredFile(f.getContentUrl(), DataSourceType.URL); + } + } + return null; + } + + @Override + public String toString() + { + return String.format("SlivkaWebService[%s]", getName()); + } +} -- 1.7.10.2