From 84d733e133ea14346163daa3d55078dacb65594d Mon Sep 17 00:00:00 2001 From: Mateusz Warowny Date: Mon, 20 Sep 2021 18:29:16 +0200 Subject: [PATCH] JAL-3878 Implement slivka web service and discoverer. --- src/jalview/ws2/slivka/SlivkaWS.java | 175 ------------------- src/jalview/ws2/slivka/SlivkaWSDiscoverer.java | 175 +++++++++++++++++++ src/jalview/ws2/slivka/SlivkaWebService.java | 215 ++++++++++++++++++++++++ 3 files changed, 390 insertions(+), 175 deletions(-) delete mode 100644 src/jalview/ws2/slivka/SlivkaWS.java create mode 100644 src/jalview/ws2/slivka/SlivkaWSDiscoverer.java create mode 100644 src/jalview/ws2/slivka/SlivkaWebService.java diff --git a/src/jalview/ws2/slivka/SlivkaWS.java b/src/jalview/ws2/slivka/SlivkaWS.java deleted file mode 100644 index 3fbd8b7..0000000 --- a/src/jalview/ws2/slivka/SlivkaWS.java +++ /dev/null @@ -1,175 +0,0 @@ -package jalview.ws2.slivka; - -import java.io.IOException; -import java.util.Arrays; -import java.util.EnumMap; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import jalview.datamodel.SequenceI; -import jalview.ws.gui.WsJob; -import jalview.ws.params.ArgumentI; -import jalview.ws.params.ParamDatastoreI; -import jalview.ws.params.WsParamSetI; -import jalview.ws.slivkaws.SlivkaDatastore; -import jalview.ws2.JalviewWebServiceI; -import jalview.ws2.WSJobID; -import jalview.ws2.WSJobTrackerI; -import uk.ac.dundee.compbio.slivkaclient.JobState; -import uk.ac.dundee.compbio.slivkaclient.SlivkaClient; -import uk.ac.dundee.compbio.slivkaclient.SlivkaService; - -public abstract class SlivkaWS implements JalviewWebServiceI -{ - protected final SlivkaClient client; - protected final SlivkaService service; - protected SlivkaDatastore store = null; - protected final String name; - protected final String description; - protected final String operation; - protected int type = 0; - - protected static final EnumMap stateMap = new EnumMap<>(JobState.class); - { - stateMap.put(JobState.PENDING, WsJob.JobState.QUEUED); - stateMap.put(JobState.REJECTED, WsJob.JobState.INVALID); - stateMap.put(JobState.ACCEPTED, WsJob.JobState.QUEUED); - stateMap.put(JobState.QUEUED, WsJob.JobState.QUEUED); - stateMap.put(JobState.RUNNING, WsJob.JobState.RUNNING); - stateMap.put(JobState.COMPLETED, WsJob.JobState.FINISHED); - stateMap.put(JobState.INTERRUPTED, WsJob.JobState.CANCELLED); - stateMap.put(JobState.DELETED, WsJob.JobState.CANCELLED); - stateMap.put(JobState.FAILED, WsJob.JobState.FAILED); - stateMap.put(JobState.ERROR, WsJob.JobState.SERVERERROR); - stateMap.put(JobState.UNKNOWN, WsJob.JobState.UNKNOWN); - } - protected final Set failedStates = new HashSet<>(Arrays.asList( - WsJob.JobState.INVALID, WsJob.JobState.BROKEN, WsJob.JobState.FAILED, - WsJob.JobState.SERVERERROR, WsJob.JobState.CANCELLED - )); - - public SlivkaWS(SlivkaClient client, SlivkaService service, String operation) { - this.client = client; - this.service = service; - this.operation = operation; - this.name = service.getName(); - this.description = ""; - } - - @Override - public String getHostName() - { - return client.getUrl().toString(); - } - - @Override - public String getName() - { - // TODO Auto-generated method stub - return null; - } - - @Override - public String getDescription() - { - // TODO Auto-generated method stub - return null; - } - - @Override - public String getOperationType() - { - // TODO Auto-generated method stub - return null; - } - - @Override - public int getTypeFlags() - { - // TODO Auto-generated method stub - return 0; - } - - @Override - public boolean canSubmitGaps() - { - // TODO Auto-generated method stub - return false; - } - - @Override - public int getMinSequences() - { - // TODO Auto-generated method stub - return 0; - } - - @Override - public int getMaxSequences() - { - // TODO Auto-generated method stub - return 0; - } - - @Override - public boolean hasParameters() - { - // TODO Auto-generated method stub - return false; - } - - @Override - public ParamDatastoreI getParamStore() - { - // TODO Auto-generated method stub - return null; - } - - @Override - public WSJobID submit(List sequences, WsParamSetI preset, - List parameters) throws IOException - { - // TODO Auto-generated method stub - return null; - } - - @Override - public void updateProgress(WSJobID id, WSJobTrackerI tracker) - throws IOException - { - // TODO Auto-generated method stub - - } - - @Override - public R getResult(WSJobID id) throws IOException - { - // TODO Auto-generated method stub - return null; - } - - @Override - public void cancel(WSJobID id) throws IOException - { - // TODO Auto-generated method stub - - } - - @Override - public boolean handleSubmissionError(WSJobID id, Throwable th, - WSJobTrackerI tracker) - { - // TODO Auto-generated method stub - return false; - } - - @Override - public boolean handleCollectionError(WSJobID id, Throwable th, - WSJobTrackerI tracker) - { - // TODO Auto-generated method stub - return false; - } - -} diff --git a/src/jalview/ws2/slivka/SlivkaWSDiscoverer.java b/src/jalview/ws2/slivka/SlivkaWSDiscoverer.java new file mode 100644 index 0000000..fe5c040 --- /dev/null +++ b/src/jalview/ws2/slivka/SlivkaWSDiscoverer.java @@ -0,0 +1,175 @@ +package jalview.ws2.slivka; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.*; +import java.util.concurrent.*; + +import jalview.bin.Cache; +import jalview.ws2.*; +import jalview.ws2.operations.AlignmentOperation; +import jalview.ws2.operations.Operation; +import uk.ac.dundee.compbio.slivkaclient.SlivkaClient; +import uk.ac.dundee.compbio.slivkaclient.SlivkaService; + +public class SlivkaWSDiscoverer implements WebServiceDiscoverer +{ + private static final String SLIVKA_HOST_URLS = "SLIVKSHOSTURLS"; + + private static final String DEFAULT_URL = "https://www.compbio.dundee.ac.uk/slivka/"; + + private static SlivkaWSDiscoverer instance = null; + + private List services = List.of(); + + private SlivkaWSDiscoverer() + { + } + + public static SlivkaWSDiscoverer getInstance() + { + if (instance == null) + { + instance = new SlivkaWSDiscoverer(); + } + return instance; + } + + @Override + public List getUrls() { + String surls = Cache.getDefault(SLIVKA_HOST_URLS, DEFAULT_URL); + String urls[] = surls.split(","); + ArrayList valid = new ArrayList<>(urls.length); + for (String url : urls) + { + try + { + new URL(url); + valid.add(url); + } + catch (MalformedURLException e) + { + Cache.log.warn("Problem whilst trying to make a URL from '" + + Objects.toString(url, "") + "'. " + + "This was probably due to malformed comma-separated-list " + + "in the " + SLIVKA_HOST_URLS + " entry of ${HOME}/.jalview_properties"); + Cache.log.debug("Exception occurred while reading url list", e); + } + } + return valid; + } + + @Override + public void setUrls(List wsUrls) { + if (wsUrls != null && !wsUrls.isEmpty()) + { + Cache.setProperty(SLIVKA_HOST_URLS, String.join(",", wsUrls)); + } + else { + Cache.removeProperty(SLIVKA_HOST_URLS); + } + } + + @Override + public boolean testUrl(URL url) { + return getStatusForUrl(url.toString()) == STATUS_OK; + } + + @Override + public int getStatusForUrl(String url) { + try + { + List services = new SlivkaClient(url).getServices(); + return services.isEmpty() ? STATUS_NO_SERVICES : STATUS_OK; + } + catch (IOException e) + { + Cache.log.error("Slivka could not retrieve services list from " + url, e); + return STATUS_INVALID; + } + } + + public List getServices() { + return Collections.unmodifiableList(services); + } + + public boolean hasServices() { + return !isRunning() && services.size() > 0; + } + + public boolean isRunning() { + for (Future task : discoveryTasks) { + if (!task.isDone()) { + return false; + } + } + return true; + } + + public boolean isDone() { + return !isRunning() && discoveryTasks.size() > 0; + } + + private Vector> discoveryTasks = new Vector<>(); + + @Override + public CompletableFuture startDiscoverer() + { + CompletableFuture task = CompletableFuture + .supplyAsync(() -> { + reloadServices(); + return SlivkaWSDiscoverer.this; + }); + discoveryTasks.add(task); + return task; + } + + private List reloadServices() + { + Cache.log.info("Reloading Slivka services"); + fireServicesChanged(Collections.emptyList()); + ArrayList allServices = new ArrayList<>(); + for (String url : getUrls()) { + SlivkaClient client = new SlivkaClient(url); + List services; + try { + services = client.getServices(); + } catch (IOException e) { + Cache.log.error("Unable to fetch services from " + url, e); + continue; + } + for (SlivkaService service : services) { + SlivkaWebService instance = new SlivkaWebService(client, service, service.getName()); + for (String classifier : service.classifiers) + { + String[] path = classifier.split("\\s*::\\s*"); + if (path.length >= 3 && path[0].toLowerCase().equals("operation") + && path[1].toLowerCase().equals("analysis")) + { + Operation op = null; + switch (path[path.length - 1].toLowerCase()) { + case "multiple sequence alignment": + op = new AlignmentOperation(instance, instance::getAlignment); + } + if (op != null) + instance.addOperation(op); + } + } + if (instance.operations.size() > 0) { + allServices.add(instance); + } + } + } + this.services = allServices; + Cache.log.info("Reloading slivka services finished"); + fireServicesChanged(getServices()); + return allServices; + } + + @Override + public String getErrorMessages() { + return ""; + } + +} diff --git a/src/jalview/ws2/slivka/SlivkaWebService.java b/src/jalview/ws2/slivka/SlivkaWebService.java new file mode 100644 index 0000000..a3aa226 --- /dev/null +++ b/src/jalview/ws2/slivka/SlivkaWebService.java @@ -0,0 +1,215 @@ +package jalview.ws2.slivka; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.EnumMap; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import jalview.bin.Cache; +import jalview.datamodel.AlignmentI; +import jalview.datamodel.SequenceI; +import jalview.io.DataSourceType; +import jalview.io.FileFormat; +import jalview.io.FileFormatI; +import jalview.io.FormatAdapter; +import jalview.ws.gui.WsJob; +import jalview.ws.params.ArgumentI; +import jalview.ws.params.ParamDatastoreI; +import jalview.ws.params.WsParamSetI; +import jalview.ws.slivkaws.SlivkaDatastore; +import jalview.ws2.WebServiceI; +import jalview.ws2.operations.Operation; +import jalview.ws2.ResultSupplier; +import jalview.ws2.WSJob; +import jalview.ws2.WSJobStatus; +import javajs.http.ClientProtocolException; +import uk.ac.dundee.compbio.slivkaclient.Job; +import uk.ac.dundee.compbio.slivkaclient.Parameter; +import uk.ac.dundee.compbio.slivkaclient.RemoteFile; +import uk.ac.dundee.compbio.slivkaclient.SlivkaClient; +import uk.ac.dundee.compbio.slivkaclient.SlivkaService; + +public class SlivkaWebService implements WebServiceI +{ + protected final SlivkaClient client; + protected final SlivkaService service; + protected SlivkaDatastore store = null; + protected final String operation; + protected final ArrayList operations = new ArrayList<>(); + protected int typeFlags = 0; + + protected static final EnumMap stateMap = new EnumMap<>(Job.Status.class); + { + stateMap.put(Job.Status.PENDING, WsJob.JobState.QUEUED); + stateMap.put(Job.Status.REJECTED, WsJob.JobState.INVALID); + stateMap.put(Job.Status.ACCEPTED, WsJob.JobState.QUEUED); + stateMap.put(Job.Status.QUEUED, WsJob.JobState.QUEUED); + stateMap.put(Job.Status.RUNNING, WsJob.JobState.RUNNING); + stateMap.put(Job.Status.COMPLETED, WsJob.JobState.FINISHED); + stateMap.put(Job.Status.INTERRUPTED, WsJob.JobState.CANCELLED); + stateMap.put(Job.Status.DELETED, WsJob.JobState.CANCELLED); + stateMap.put(Job.Status.FAILED, WsJob.JobState.FAILED); + stateMap.put(Job.Status.ERROR, WsJob.JobState.SERVERERROR); + stateMap.put(Job.Status.UNKNOWN, WsJob.JobState.UNKNOWN); + } + protected final Set failedStates = new HashSet<>(Arrays.asList( + WsJob.JobState.INVALID, WsJob.JobState.BROKEN, WsJob.JobState.FAILED, + WsJob.JobState.SERVERERROR, WsJob.JobState.CANCELLED + )); + + public SlivkaWebService(SlivkaClient client, SlivkaService service, String operation) { + this.client = client; + this.service = service; + this.operation = operation; + } + + @Override + public String getHostName() { return client.getUrl().toString(); } + + @Override + public String getName() { return service.getName(); } + + @Override + public String getDescription() { return service.getDescription(); } + + @Override + public String getOperationType() { return operation; } + + @Override + public List getOperations() { + return operations; + } + + void addOperation(Operation operation) { + operations.add(operation); + } + + void removeOperation(Operation operation) { + operations.remove(operation); + } + + @Override + public boolean hasParameters() { + return getParamStore().getServiceParameters().size() > 0; + } + + @Override + public ParamDatastoreI getParamStore() { + if (store == null) { + store = new SlivkaDatastore(service); + } + return store; + } + + @Override + public WSJob submit(List sequences, List args) throws IOException + { + var request = new uk.ac.dundee.compbio.slivkaclient.JobRequest(); + for (Parameter param : service.getParameters()) { + if (param instanceof Parameter.FileParameter) { + // if finds a file input, gives it sequences stream + Parameter.FileParameter fileParam = (Parameter.FileParameter) param; + FileFormat format; + switch (fileParam.getMediaType()) { + case "application/pfam": + format = FileFormat.Pfam; + break; + case "application/stockholm": + format = FileFormat.Stockholm; + break; + case "application/clustal": + format = FileFormat.Clustal; + break; + case "application/fasta": + default: + format = FileFormat.Fasta; + break; + } + InputStream stream = new ByteArrayInputStream( + format.getWriter(null) + .print(sequences.toArray(new SequenceI[0]), false) + .getBytes()); + request.addFile(param.getId(), stream); + } + } + if (args != null) { + for (ArgumentI arg : args) { + // multiple choice field names are name$number to avoid duplications + // the number is stripped here + String paramId = arg.getName().split("\\$", 2)[0]; + Parameter param = service.getParameter(paramId); + if (param instanceof Parameter.FlagParameter) { + if (arg.getValue() != null && !arg.getValue().isBlank()) + request.addData(paramId, true); + else + request.addData(paramId, false); + } + else + { + request.addData(paramId, arg.getValue()); + } + } + } + var job = service.submitJob(request); + return new WSJob("slivka", getName(), job.getId(), getHostName()); + } + + @Override + public void updateProgress(WSJob job) + throws IOException + { + // TODO Auto-generated method stub + + } + + @Override + public void cancel(WSJob job) throws IOException + { + Cache.log.warn("Slivka does not support job cancellation yet."); + } + + @Override + public boolean handleSubmissionError(WSJob job, Exception ex) + { + if (ex instanceof ClientProtocolException) { + Cache.log.error("Job submission failed due to exception.", ex); + return true; + } + return false; + } + + @Override + public boolean handleCollectionError(WSJob job, Exception ex) + { + // TODO Auto-generated method stub + return false; + } + + public AlignmentI getAlignment(WSJob job) throws IOException { + Collection files; + var slivkaJob = client.getJob(job.getJobID()); + files = slivkaJob.getResults(); + for (RemoteFile f : files) { + if (f.getMediaType().equals("application/clustal")) { + return new FormatAdapter().readFile( + f.getContentUrl().toString(), DataSourceType.URL, FileFormat.Clustal); + } + else if (f.getMediaType().equals("application/fasta")) { + return new FormatAdapter().readFile( + f.getContentUrl().toString(), DataSourceType.URL, FileFormat.Fasta); + } + } + return null; + } + + @Override + public String toString() { + return String.format("SlivkaWebService[%s]", getName()); + } +} -- 1.7.10.2