From: Mateusz Warowny <mmzwarowny@dundee.ac.uk>
Date: Thu, 18 Nov 2021 20:10:19 +0000 (+0100)
Subject: JAL-3878 Add slivka's implementations of web service and discoverer.
X-Git-Url: http://source.jalview.org/gitweb/?a=commitdiff_plain;h=e016e819d1f9d851aa28a273414b4432c2137858;p=jalview.git

JAL-3878 Add slivka's implementations of web service and discoverer.
---

diff --git a/src/jalview/ws2/slivka/SlivkaWSDiscoverer.java b/src/jalview/ws2/slivka/SlivkaWSDiscoverer.java
new file mode 100644
index 0000000..802254b
--- /dev/null
+++ b/src/jalview/ws2/slivka/SlivkaWSDiscoverer.java
@@ -0,0 +1,210 @@
+package jalview.ws2.slivka;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.*;
+import java.util.concurrent.*;
+
+import jalview.bin.Cache;
+import jalview.ws2.*;
+import jalview.ws2.operations.*;
+import uk.ac.dundee.compbio.slivkaclient.SlivkaClient;
+import uk.ac.dundee.compbio.slivkaclient.SlivkaService;
+
+public class SlivkaWSDiscoverer implements WebServiceDiscovererI
+{
+  private static final String SLIVKA_HOST_URLS = "SLIVKSHOSTURLS";
+
+  private static final String DEFAULT_URL = "https://www.compbio.dundee.ac.uk/slivka/";
+
+  private static SlivkaWSDiscoverer instance = null;
+
+  private List<Operation> operations = List.of();
+
+  private SlivkaWSDiscoverer()
+  {
+  }
+
+  public static SlivkaWSDiscoverer getInstance()
+  {
+    if (instance == null)
+    {
+      instance = new SlivkaWSDiscoverer();
+    }
+    return instance;
+  }
+
+  @Override
+  public List<String> getUrls()
+  {
+    String surls = Cache.getDefault(SLIVKA_HOST_URLS, DEFAULT_URL);
+    String urls[] = surls.split(",");
+    ArrayList<String> valid = new ArrayList<>(urls.length);
+    for (String url : urls)
+    {
+      try
+      {
+        new URL(url);
+        valid.add(url);
+      } catch (MalformedURLException e)
+      {
+        Cache.log.warn("Problem whilst trying to make a URL from '"
+                + Objects.toString(url, "<null>") + "'. "
+                + "This was probably due to malformed comma-separated-list "
+                + "in the " + SLIVKA_HOST_URLS
+                + " entry of ${HOME}/.jalview_properties");
+        Cache.log.debug("Exception occurred while reading url list", e);
+      }
+    }
+    return valid;
+  }
+
+  @Override
+  public void setUrls(List<String> wsUrls)
+  {
+    if (wsUrls != null && !wsUrls.isEmpty())
+    {
+      Cache.setProperty(SLIVKA_HOST_URLS, String.join(",", wsUrls));
+    }
+    else
+    {
+      Cache.removeProperty(SLIVKA_HOST_URLS);
+    }
+  }
+
+  @Override
+  public boolean testUrl(URL url)
+  {
+    return getStatusForUrl(url.toString()) == STATUS_OK;
+  }
+
+  @Override
+  public int getStatusForUrl(String url)
+  {
+    try
+    {
+      List<?> services = new SlivkaClient(url).getServices();
+      return services.isEmpty() ? STATUS_NO_SERVICES : STATUS_OK;
+    } catch (IOException e)
+    {
+      Cache.log.error("Slivka could not retrieve services list from " + url,
+              e);
+      return STATUS_INVALID;
+    }
+  }
+
+  @Override
+  public List<Operation> getOperations()
+  {
+    return Collections.unmodifiableList(operations);
+  }
+
+  @Override
+  public boolean hasServices()
+  {
+    return !isRunning() && operations.size() > 0;
+  }
+
+  public boolean isRunning()
+  {
+    for (Future<?> task : discoveryTasks)
+    {
+      if (!task.isDone())
+      {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  public boolean isDone()
+  {
+    return !isRunning() && discoveryTasks.size() > 0;
+  }
+
+  private Vector<Future<?>> discoveryTasks = new Vector<>();
+
+  @Override
+  public CompletableFuture<WebServiceDiscovererI> startDiscoverer()
+  {
+    CompletableFuture<WebServiceDiscovererI> task = CompletableFuture
+            .supplyAsync(() -> {
+              reloadServices();
+              return SlivkaWSDiscoverer.this;
+            });
+    task.thenRun(() -> fireOperationsChanged(getOperations()));
+    discoveryTasks.add(task);
+    return task;
+  }
+
+  private List<Operation> reloadServices()
+  {
+    Cache.log.info("Reloading Slivka services");
+    fireOperationsChanged(Collections.emptyList());
+    ArrayList<Operation> allOperations= new ArrayList<>();
+    for (String url : getUrls())
+    {
+      SlivkaClient client = new SlivkaClient(url);
+      List<SlivkaService> services;
+      try
+      {
+        services = client.getServices();
+      } catch (IOException e)
+      {
+        Cache.log.error("Unable to fetch services from " + url, e);
+        continue;
+      }
+      for (SlivkaService service : services)
+      {
+        SlivkaWebService webService = new SlivkaWebService(client, service);
+        AbstractOperation op = null;
+        for (String classifier : service.classifiers)
+        {
+          String[] path = classifier.split("\\s*::\\s*");
+          if (path.length >= 3 && path[0].toLowerCase().equals("operation")
+                  && path[1].toLowerCase().equals("analysis"))
+          {
+            switch (path[path.length - 1].toLowerCase())
+            {
+            case "rna secondary structure prediction":
+              op = new OperationStub(webService, "Secondary Structure Prediction");
+              op.setInteractive(true);
+              op.setAlignmentAnalysis(true);
+              op.setProteinOperation(false);
+              break;
+            case "sequence alignment analysis (conservation)":
+              op = new OperationStub(webService, "Conservation");
+              op.setAlignmentAnalysis(true);
+              op.setInteractive(true);
+              break;
+            case "protein sequence analysis":
+              op = new OperationStub(webService, "Protein Disorder");
+              break;
+            case "multiple sequence alignment":
+              op = new OperationStub(webService, "Alignment");
+              break;
+            }
+            if (op != null)
+            {
+              break;
+            }
+          }
+        }
+        if (op != null) {
+          allOperations.add(op);
+        }
+      }
+    }
+    this.operations = allOperations;
+    Cache.log.info("Reloading slivka services finished");
+    return allOperations;
+  }
+
+  @Override
+  public String getErrorMessages()
+  {
+    return "";
+  }
+
+}
diff --git a/src/jalview/ws2/slivka/SlivkaWebService.java b/src/jalview/ws2/slivka/SlivkaWebService.java
new file mode 100644
index 0000000..a2d7f20
--- /dev/null
+++ b/src/jalview/ws2/slivka/SlivkaWebService.java
@@ -0,0 +1,304 @@
+package jalview.ws2.slivka;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.EnumMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import jalview.api.AlignViewportI;
+import jalview.bin.Cache;
+import jalview.datamodel.Alignment;
+import jalview.datamodel.AlignmentAnnotation;
+import jalview.datamodel.AlignmentI;
+import jalview.datamodel.SequenceI;
+import jalview.io.AnnotationFile;
+import jalview.io.DataSourceType;
+import jalview.io.FeaturesFile;
+import jalview.io.FileFormat;
+import jalview.io.FileFormatI;
+import jalview.io.FormatAdapter;
+import jalview.io.JPredFile;
+import jalview.ws.gui.WsJob;
+import jalview.ws.params.ArgumentI;
+import jalview.ws.params.ParamDatastoreI;
+import jalview.ws.params.WsParamSetI;
+import jalview.ws.slivkaws.SlivkaDatastore;
+import jalview.ws2.WebServiceI;
+import jalview.ws2.operations.Operation;
+import jalview.ws2.WSJob;
+import jalview.ws2.WSJobStatus;
+import javajs.http.ClientProtocolException;
+import uk.ac.dundee.compbio.slivkaclient.Job;
+import uk.ac.dundee.compbio.slivkaclient.Parameter;
+import uk.ac.dundee.compbio.slivkaclient.RemoteFile;
+import uk.ac.dundee.compbio.slivkaclient.SlivkaClient;
+import uk.ac.dundee.compbio.slivkaclient.SlivkaService;
+
+public class SlivkaWebService implements WebServiceI
+{
+  protected final SlivkaClient client;
+
+  protected final SlivkaService service;
+  
+  protected ParamDatastoreI store;
+
+  protected static final EnumMap<Job.Status, WSJobStatus> statusMap = new EnumMap<>(
+          Job.Status.class);
+  {
+    statusMap.put(Job.Status.PENDING, WSJobStatus.SUBMITTED);
+    statusMap.put(Job.Status.REJECTED, WSJobStatus.INVALID);
+    statusMap.put(Job.Status.ACCEPTED, WSJobStatus.QUEUED);
+    statusMap.put(Job.Status.QUEUED, WSJobStatus.QUEUED);
+    statusMap.put(Job.Status.RUNNING, WSJobStatus.RUNNING);
+    statusMap.put(Job.Status.COMPLETED, WSJobStatus.FINISHED);
+    statusMap.put(Job.Status.INTERRUPTED, WSJobStatus.CANCELLED);
+    statusMap.put(Job.Status.DELETED, WSJobStatus.CANCELLED);
+    statusMap.put(Job.Status.FAILED, WSJobStatus.FAILED);
+    statusMap.put(Job.Status.ERROR, WSJobStatus.SERVER_ERROR);
+    statusMap.put(Job.Status.UNKNOWN, WSJobStatus.UNKNOWN);
+  }
+
+  public SlivkaWebService(SlivkaClient client, SlivkaService service)
+  {
+    this.client = client;
+    this.service = service;
+  }
+
+  @Override
+  public String getHostName()
+  {
+    return client.getUrl().toString();
+  }
+
+  @Override
+  public String getProviderName()
+  {
+    return "slivka";
+  }
+
+  @Override
+  public String getName()
+  {
+    return service.getName();
+  }
+
+  @Override
+  public String getDescription()
+  {
+    return service.getDescription();
+  }
+
+  @Override
+  public boolean hasParameters()
+  {
+    return getParamStore().getServiceParameters().size() > 0;
+  }
+
+  @Override
+  public ParamDatastoreI getParamStore()
+  {
+    if (store == null)
+    {
+      store = new SlivkaDatastore(service);
+    }
+    return store;
+  }
+
+  @Override
+  public String submit(List<SequenceI> sequences, List<ArgumentI> args)
+          throws IOException
+  {
+    var request = new uk.ac.dundee.compbio.slivkaclient.JobRequest();
+    for (Parameter param : service.getParameters())
+    {
+      if (param instanceof Parameter.FileParameter)
+      {
+        // if finds a file input, gives it sequences stream
+        Parameter.FileParameter fileParam = (Parameter.FileParameter) param;
+        FileFormat format;
+        switch (fileParam.getMediaType())
+        {
+        case "application/pfam":
+          format = FileFormat.Pfam;
+          break;
+        case "application/stockholm":
+          format = FileFormat.Stockholm;
+          break;
+        case "application/clustal":
+          format = FileFormat.Clustal;
+          break;
+        case "application/fasta":
+        default:
+          format = FileFormat.Fasta;
+          break;
+        }
+        InputStream stream = new ByteArrayInputStream(format.getWriter(null)
+                .print(sequences.toArray(new SequenceI[0]), false)
+                .getBytes());
+        request.addFile(param.getId(), stream);
+      }
+    }
+    if (args != null)
+    {
+      for (ArgumentI arg : args)
+      {
+        // multiple choice field names are name$number to avoid duplications
+        // the number is stripped here
+        String paramId = arg.getName().split("\\$", 2)[0];
+        Parameter param = service.getParameter(paramId);
+        if (param instanceof Parameter.FlagParameter)
+        {
+          if (arg.getValue() != null && !arg.getValue().isBlank())
+            request.addData(paramId, true);
+          else
+            request.addData(paramId, false);
+        }
+        else
+        {
+          request.addData(paramId, arg.getValue());
+        }
+      }
+    }
+    var job = service.submitJob(request);
+    return job.getId();
+  }
+
+  @Override
+  public void updateProgress(WSJob job) throws IOException
+  {
+    var slivkaJob = client.getJob(job.getJobId());
+    job.setStatus(statusMap.get(slivkaJob.getStatus()));
+    Collection<RemoteFile> files = slivkaJob.getResults();
+    for (RemoteFile f : files)
+    {
+      if (f.getLabel().equals("log"))
+      {
+        ByteArrayOutputStream stream = new ByteArrayOutputStream();
+        f.writeTo(stream);
+        job.setLog(stream.toString("UTF-8"));
+      }
+      else if (f.getLabel().equals("error-log"))
+      {
+        ByteArrayOutputStream stream = new ByteArrayOutputStream();
+        f.writeTo(stream);
+        job.setErrorLog(stream.toString("UTF-8"));
+      }
+    }
+  }
+
+  @Override
+  public void cancel(WSJob job) throws IOException
+  {
+    job.setStatus(WSJobStatus.CANCELLED);
+    Cache.log.warn("Slivka does not support job cancellation yet.");
+  }
+
+  @Override
+  public boolean handleSubmissionError(WSJob job, Exception ex)
+  {
+    if (ex instanceof ClientProtocolException)
+    {
+      Cache.log.error("Job submission failed due to exception.", ex);
+      return true;
+    }
+    return false;
+  }
+
+  @Override
+  public boolean handleCollectionError(WSJob job, Exception ex)
+  {
+    // TODO Auto-generated method stub
+    return false;
+  }
+
+  public AlignmentI getAlignment(WSJob job, List<SequenceI> dataset,
+      AlignViewportI viewport) throws IOException
+  {
+    Collection<RemoteFile> files;
+    var slivkaJob = client.getJob(job.getJobId());
+    files = slivkaJob.getResults();
+    for (RemoteFile f : files)
+    {
+      if (f.getMediaType().equals("application/clustal"))
+      {
+        return new FormatAdapter().readFile(f.getContentUrl().toString(),
+                DataSourceType.URL, FileFormat.Clustal);
+      }
+      else if (f.getMediaType().equals("application/fasta"))
+      {
+        return new FormatAdapter().readFile(f.getContentUrl().toString(),
+                DataSourceType.URL, FileFormat.Fasta);
+      }
+    }
+    return null;
+  }
+
+  public FeaturesFile getFeaturesFile(WSJob job,
+      List<SequenceI> dataset, AlignViewportI viewport) throws IOException
+  {
+    var slivkaJob = client.getJob(job.getJobId());
+    Collection<RemoteFile> files = slivkaJob.getResults();
+    for (RemoteFile f : files)
+    {
+      if (f.getMediaType().equals("application/jalview-features"))
+      {
+        return new FeaturesFile(f.getContentUrl().toString(), DataSourceType.URL);
+      }
+    }
+    return null;
+  }
+
+  public List<AlignmentAnnotation> getAnnotations(WSJob job,
+      List<SequenceI> dataset, AlignViewportI viewport) throws IOException
+  {
+    var slivkaJob = client.getJob(job.getJobId());
+    Collection<RemoteFile> files = slivkaJob.getResults();
+    for (RemoteFile f : files)
+    {
+      if (f.getMediaType().equals("application/jalview-annotations"))
+      {
+        Alignment aln = new Alignment(dataset.toArray(new SequenceI[0]));
+        AnnotationFile af = new AnnotationFile();
+        boolean valid = af.readAnnotationFileWithCalcId(aln, service.getId(),
+            f.getContentUrl().toString(), DataSourceType.URL);
+        if (valid)
+        {
+          return Arrays.asList(aln.getAlignmentAnnotation());
+        }
+        else
+        {
+          throw new IOException("Unable to read annotations from file " +
+              f.getContentUrl().toString());
+        }
+      }
+    }
+    return null;
+  }
+
+  public JPredFile getPrediction(WSJob job, List<SequenceI> dataset,
+      AlignViewportI viewport) throws IOException
+  {
+    Collection<RemoteFile> files = client.getJob(job.getJobId()).getResults();
+    for (RemoteFile f : files)
+    {
+      if (f.getLabel().equals("concise"))
+      {
+        return new JPredFile(f.getContentUrl(), DataSourceType.URL);
+      }
+    }
+    return null;
+  }
+
+  @Override
+  public String toString()
+  {
+    return String.format("SlivkaWebService[%s]", getName());
+  }
+}