From: Mateusz Warowny <mmzwarowny@dundee.ac.uk>
Date: Wed, 28 Jul 2021 10:36:30 +0000 (+0200)
Subject: Merge branch 'improvement/JAL-3848_slivka_0.8' into alpha/JAL-3066_Jalview_212_slivka... 
X-Git-Url: http://source.jalview.org/gitweb/?p=jalview.git;a=commitdiff_plain;h=871ad3a9474c5f4fbbe896a8a0551d0a6250cb77

Merge branch 'improvement/JAL-3848_slivka_0.8' into alpha/JAL-3066_Jalview_212_slivka-integration
---

871ad3a9474c5f4fbbe896a8a0551d0a6250cb77
diff --cc j11lib/slivka-client.jar
index 68cc932,9cb3a61..49ab4fc
Binary files differ
diff --cc src/jalview/ws/jws2/JPredThread.java
index d10ca8a,0000000..67f44fa
mode 100644,000000..100644
--- a/src/jalview/ws/jws2/JPredThread.java
+++ b/src/jalview/ws/jws2/JPredThread.java
@@@ -1,406 -1,0 +1,405 @@@
 +package jalview.ws.jws2;
 +
 +import static java.lang.String.format;
 +
 +import java.util.Hashtable;
 +import java.util.List;
 +
 +import jalview.analysis.SeqsetUtils;
 +import jalview.bin.Cache;
 +import jalview.commands.RemoveGapsCommand;
 +import jalview.datamodel.Alignment;
 +import jalview.datamodel.AlignmentAnnotation;
 +import jalview.datamodel.AlignmentI;
 +import jalview.datamodel.AlignmentView;
 +import jalview.datamodel.HiddenColumns;
 +import jalview.datamodel.SequenceI;
 +import jalview.gui.AlignFrame;
 +import jalview.gui.Desktop;
 +import jalview.gui.WebserviceInfo;
- import jalview.io.JPredFile;
 +import jalview.io.JnetAnnotationMaker;
 +import jalview.util.MessageManager;
 +import jalview.ws.AWSThread;
 +import jalview.ws.AWsJob;
 +import jalview.ws.JobStateSummary;
 +import jalview.ws.WSClientI;
 +import jalview.ws.api.CancellableI;
 +import jalview.ws.api.JPredServiceI;
 +import jalview.ws.gui.WsJob;
 +import jalview.ws.gui.WsJob.JobState;
 +
 +
 +public class JPredThread extends AWSThread implements WSClientI
 +{
 +
 +  private static class JPredJob extends WsJob
 +  {
 +    private final Hashtable<?, ?> sequenceInfo;
 +    private final List<SequenceI> msf;
 +    private final int[] delMap;
 +    private AlignmentI alignment = null;
 +    private HiddenColumns hiddenCols = null;
 +
 +    private JPredJob(Hashtable<?, ?> sequenceInfo, SequenceI[] msf, int[] delMap)
 +    {
 +      this.sequenceInfo = sequenceInfo;
 +      this.msf = List.of(msf);
 +      this.delMap = delMap;
 +    }
 +
 +    @Override
 +    public boolean hasValidInput()
 +    {
 +      return true;
 +    }
 +
 +    @Override
 +    public boolean hasResults()
 +    {
 +      return (isSubjobComplete() && alignment != null);
 +    }
 +
 +    public boolean isMSA()
 +    {
 +      return msf.size() > 1;
 +    }
 +  }
 +
 +
 +  private JPredServiceI server;
 +  private String title;
 +  private Hashtable<?, ?> sequenceInfo;
 +  private SequenceI[] msf;
 +  private int[] delMap;
 +
 +  public JPredThread(WebserviceInfo wsInfo, String title,
 +      JPredServiceI server, Hashtable<?, ?> sequenceInfo,
 +      SequenceI[] msf, int[] delMap, AlignmentView view, AlignFrame frame,
 +      String wsURL)
 +  {
 +    super(frame, wsInfo, view, wsURL);
 +    this.server = server;
 +    this.title = title;
 +    this.sequenceInfo = sequenceInfo;
 +    this.msf = msf;
 +    this.delMap = delMap;
 +    JPredJob job = new JPredJob(sequenceInfo, msf, delMap);
 +    this.jobs = new JPredJob[] { job };
 +  }
 +
 +  @Override
 +  public boolean isCancellable()
 +  {
 +    return server instanceof CancellableI;
 +  }
 +
 +  @Override
 +  public boolean canMergeResults()
 +  {
 +    return false;
 +  }
 +
 +  @Override
 +  public void cancelJob()
 +  {
 +    // TODO Auto-generated method stub
 +
 +  }
 +
 +  @Override
 +  public void pollJob(AWsJob job_) throws Exception
 +  {
 +    var job = (JPredJob) job_;
 +    server.updateStatus(job);
 +    server.updateJobProgress(job);
 +  }
 +
 +  @Override
 +  public void StartJob(AWsJob job_)
 +  {
 +    if (!(job_ instanceof JPredJob))
 +      throw new RuntimeException("Invalid job type");
 +    var job = (JPredJob) job_;
 +    if (job.isSubmitted())
 +    {
 +      return;
 +    }
 +    try {
 +      try
 +      {
 +        var jobHandle = server.predict(job.msf, job.isMSA());
 +        if (jobHandle != null)
 +          job.setJobHandle(jobHandle);
 +      }
 +      catch (Throwable th) {
 +        if (!server.handleSubmitError(th, job, wsInfo)) {
 +          throw th;
 +        }
 +      }
 +      if (job.getJobId() != null) {
 +        job.setSubmitted(true);
 +        job.setSubjobComplete(false);
 +        return;
 +      }
 +      else {
 +        throw new Exception(MessageManager.formatMessage(
 +                "exception.web_service_returned_null_try_later",
 +                new String[]
 +                { WsUrl }));
 +      }
 +    }
 +    catch (Throwable th)
 +    {
 +      // For unexpected errors
 +      System.err.println(WebServiceName
 +              + "Client: Failed to submit the sequences for alignment (probably a server side problem)\n"
 +              + "When contacting Server:" + WsUrl + "\n");
 +      th.printStackTrace(System.err);
 +      wsInfo.setStatus(WebserviceInfo.STATE_STOPPED_SERVERERROR);
 +      wsInfo.setStatus(job.getJobnum(),
 +              WebserviceInfo.STATE_STOPPED_SERVERERROR);
 +
 +    }
 +    finally
 +    {
 +      if (!job.isSubmitted())
 +      {
 +        job.setAllowedServerExceptions(0);
 +        wsInfo.appendProgressText(job.getJobnum(), MessageManager.getString(
 +                "info.failed_to_submit_sequences_for_alignment"));
 +      }
 +    }
 +  }
 +
 +  @Override
 +  public void parseResult()
 +  {
 +    long progbar = (long) (Math.random() * ~(1L << 63));
 +    wsInfo.setProgressBar(
 +        MessageManager.getString("status.collecting_job_results"), progbar);
 +    int results = 0;
 +    var finalState = new JobStateSummary();
 +    try
 +    {
 +      for (int i = 0; i < jobs.length; i++) {
 +        final var job = (JPredJob) jobs[i];
 +        finalState.updateJobPanelState(wsInfo, OutputHeader, job);
 +        if (job.isFinished()) {
 +          try {
 +            server.updateJobProgress(job);
 +          }
 +          catch (Exception e) {
 +            Cache.log.warn(format(
 +                "Exception when retrieving remaining Job progress data " +
 +                "for job %s on server %s", job.getJobId(), WsUrl));
 +            e.printStackTrace();
 +          }
 +          // removed the waiting loop
 +          Cache.log.debug(format("Job Execution file for job: %s " +
 +              "on server %s%n%s", job.getJobId(), WsUrl, job.getStatus()));
 +          try {
 +            prepareJobResult(job);
 +          }
 +          catch (Exception e) {
 +            if (!server.handleCollectionException(e, job, wsInfo)) {
 +              Cache.log.error("Could not get alignment for job.", e);
 +              job.setState(JobState.SERVERERROR);
 +            }
 +          }
 +        }
 +        finalState.updateJobPanelState(wsInfo, OutputHeader, job);
 +        if (job.isSubmitted() && job.isSubjobComplete() && job.hasResults()) {
 +          results++;
 +        }
 +      }
 +    }
 +    catch (Exception e) {
 +      Cache.log.error(
 +          "Unexpected exception when processing results for " + title, e);
 +      wsInfo.setStatus(WebserviceInfo.STATE_STOPPED_ERROR);
 +    }
 +    if (results > 0) {
 +      wsInfo.showResultsNewFrame.addActionListener(
 +          (evt) -> displayResults(true));
 +      wsInfo.mergeResults.addActionListener(
 +          (evt) -> displayResults(false));
 +      wsInfo.setResultsReady();
 +    }
 +    else {
 +      wsInfo.setStatus(WebserviceInfo.STATE_STOPPED_ERROR);
 +      wsInfo.appendInfoText("No jobs ran.");
 +      wsInfo.setFinishedNoResults();
 +    }
 +    updateGlobalStatus(finalState);
 +    wsInfo.removeProgressBar(progbar);
 +  }
 +
 +  static final int msaIndex = 0;
 +
 +  private void prepareJobResult(JPredJob job) throws Exception
 +  {
 +    HiddenColumns hiddenCols = null;
 +    int firstSeq = -1;
 +    AlignmentI alignment;
 +    var prediction = server.getPrediction(job.getJobHandle());
 +    var preds = prediction.getSeqsAsArray();
 +
 +    if (job.msf.size() > 1)
 +    {
 +      if (job.delMap != null)
 +      {
 +        Object[] alandcolsel = input
 +                .getAlignmentAndHiddenColumns(getGapChar());
 +        alignment = new Alignment((SequenceI[]) alandcolsel[0]);
 +        hiddenCols = (HiddenColumns) alandcolsel[1];
 +      }
 +      else
 +      {
 +        alignment = server.getAlignment(job.getJobHandle());
 +        var seqs = new SequenceI[alignment.getHeight()];
 +        for (int i = 0; i < alignment.getHeight(); i++)
 +        {
 +          seqs[i] = alignment.getSequenceAt(i);
 +        }
 +        if (!SeqsetUtils.deuniquify(sequenceInfo, seqs))
 +        {
 +          throw (new Exception(MessageManager.getString(
 +                  "exception.couldnt_recover_sequence_properties_for_alignment")));
 +        }
 +      }
 +      firstSeq = 0;
 +      if (currentView.getDataset() != null)
 +      {
 +        alignment.setDataset(currentView.getDataset());
 +      }
 +      else
 +      {
 +        alignment.setDataset(null);
 +      }
 +      JnetAnnotationMaker.add_annotation(prediction, alignment, firstSeq, false,
 +              job.delMap);
 +    }
 +    else
 +    {
 +      alignment = new Alignment(preds);
 +      firstSeq = prediction.getQuerySeqPosition();
 +      if (job.delMap != null)
 +      {
 +        Object[] alanndcolsel = input.getAlignmentAndHiddenColumns(getGapChar());
 +        SequenceI[] seqs = (SequenceI[]) alanndcolsel[0];
 +        new RemoveGapsCommand(MessageManager.getString("label.remove_gaps"),
 +                new SequenceI[] {seqs[msaIndex]}, currentView);
 +        SequenceI profileSeq = alignment.getSequenceAt(firstSeq);
 +        profileSeq.setSequence(seqs[msaIndex].getSequenceAsString());
 +      }
 +      if (!SeqsetUtils.SeqCharacterUnhash(
 +              alignment.getSequenceAt(firstSeq), sequenceInfo))
 +      {
 +        throw new Exception(MessageManager.getString(
 +                "exception.couldnt_recover_sequence_props_for_jnet_query"));
 +      }
 +      alignment.setDataset(currentView.getDataset());
 +      JnetAnnotationMaker.add_annotation(prediction, alignment, firstSeq, true,
 +              job.delMap);
 +      SequenceI profileSeq = alignment.getSequenceAt(0);
 +      alignToProfileSeq(alignment, profileSeq);
 +      if (job.delMap != null)
 +      {
 +        hiddenCols = alignment.propagateInsertions(profileSeq, input);
 +      }
 +    }
 +
 +    for (var annot : alignment.getAlignmentAnnotation())
 +    {
 +      if (annot.sequenceRef != null)
 +      {
 +        replaceAnnotationOnAlignmentWith(annot, annot.label,
 +                "jalview.ws.JPred", annot.sequenceRef);
 +      }
 +    }
 +    job.alignment = alignment;
 +    job.hiddenCols = hiddenCols;
 +  }
 +
 +  private static void replaceAnnotationOnAlignmentWith(
 +          AlignmentAnnotation newAnnot, String typeName, String calcId,
 +          SequenceI aSeq)
 +  {
 +    SequenceI dsseq = aSeq.getDatasetSequence();
 +    while (dsseq.getDatasetSequence() != null)
 +    {
 +      dsseq = dsseq.getDatasetSequence();
 +    }
 +    // look for same annotation on dataset and lift this one over
 +    List<AlignmentAnnotation> dsan = dsseq.getAlignmentAnnotations(calcId,
 +            typeName);
 +    if (dsan != null && dsan.size() > 0)
 +    {
 +      for (AlignmentAnnotation dssan : dsan)
 +      {
 +        dsseq.removeAlignmentAnnotation(dssan);
 +      }
 +    }
 +    AlignmentAnnotation dssan = new AlignmentAnnotation(newAnnot);
 +    dsseq.addAlignmentAnnotation(dssan);
 +    dssan.adjustForAlignment();
 +  }
 +
 +  private static void alignToProfileSeq(AlignmentI al, SequenceI profileseq)
 +  {
 +    char gc = al.getGapCharacter();
 +    int[] gapMap = profileseq.gapMap();
 +    // insert gaps into profile
 +    for (int lp = 0, r = 0; r < gapMap.length; r++)
 +    {
 +      if (gapMap[r] - lp > 1)
 +      {
 +        StringBuffer sb = new StringBuffer();
 +        for (int s = 0, ns = gapMap[r] - lp; s < ns; s++)
 +        {
 +          sb.append(gc);
 +        }
 +        for (int s = 1, ns = al.getHeight(); s < ns; s++)
 +        {
 +          String sq = al.getSequenceAt(s).getSequenceAsString();
 +          int diff = gapMap[r] - sq.length();
 +          if (diff > 0)
 +          {
 +            // pad gaps
 +            sq = sq + sb;
 +            while ((diff = gapMap[r] - sq.length()) > 0)
 +            {
 +              sq = sq + ((diff >= sb.length()) ? sb.toString()
 +                      : sb.substring(0, diff));
 +            }
 +            al.getSequenceAt(s).setSequence(sq);
 +          }
 +          else
 +          {
 +            al.getSequenceAt(s).setSequence(sq.substring(0, gapMap[r])
 +                    + sb.toString() + sq.substring(gapMap[r]));
 +          }
 +        }
 +      }
 +      lp = gapMap[r];
 +    }
 +  }
 +
 +  private void displayResults(boolean newWindow)
 +  {
 +    if (jobs == null || jobs.length == 0)
 +    {
 +      return;
 +    }
 +    var job = (JPredJob) jobs[0];
 +    if (job.hasResults() && newWindow)
 +    {
 +      job.alignment.setSeqrep(job.alignment.getSequenceAt(0));
 +      AlignFrame frame = new AlignFrame(job.alignment, job.hiddenCols,
 +              AlignFrame.DEFAULT_WIDTH, AlignFrame.DEFAULT_HEIGHT);
 +      Desktop.addInternalFrame(frame, title, AlignFrame.DEFAULT_WIDTH,
 +              AlignFrame.DEFAULT_HEIGHT);
 +    }
 +  }
 +
 +}
diff --cc src/jalview/ws/slivkaws/SlivkaJPredServiceInstance.java
index 462be28,0000000..3958198
mode 100644,000000..100644
--- a/src/jalview/ws/slivkaws/SlivkaJPredServiceInstance.java
+++ b/src/jalview/ws/slivkaws/SlivkaJPredServiceInstance.java
@@@ -1,96 -1,0 +1,98 @@@
 +package jalview.ws.slivkaws;
 +
 +import java.io.IOError;
 +import java.io.IOException;
++import java.util.Collection;
 +import java.util.List;
 +
 +import jalview.datamodel.AlignmentI;
 +import jalview.datamodel.SequenceI;
 +import jalview.io.DataSourceType;
 +import jalview.io.JPredFile;
 +import jalview.ws.api.JPredServiceI;
 +import jalview.ws.api.JobId;
 +import jalview.ws.params.ArgumentI;
 +import uk.ac.dundee.compbio.slivkaclient.RemoteFile;
 +import uk.ac.dundee.compbio.slivkaclient.SlivkaClient;
 +import uk.ac.dundee.compbio.slivkaclient.SlivkaService;
 +
 +public class SlivkaJPredServiceInstance extends SlivkaWSInstance
 +    implements JPredServiceI
 +{
 +
 +  private class InputFormatParameter implements ArgumentI
 +  {
 +    String value = "";
 +
 +    @Override
 +    public String getName()
 +    {
 +      return "format";
 +    }
 +
 +    @Override
 +    public String getValue()
 +    {
 +      return value;
 +    }
 +
 +    @Override
 +    public void setValue(String selectedItem)
 +    {
 +      value = selectedItem;
 +    }
 +  }
 +
 +
 +  public SlivkaJPredServiceInstance(SlivkaClient client,
 +      SlivkaService service, String action)
 +  {
 +    super(client, service, action);
 +    style = ServiceClient.JPREDWSCLIENT;
 +  }
 +
 +  @Override
 +  public JobId predict(List<SequenceI> sequences, boolean msa) throws Throwable
 +  {
 +    // Hack allowing to send both single and msa jobs
 +    // until msa and single sequence services are separated.
 +    var arg = new InputFormatParameter();
 +    arg.setValue(msa ? "fasta" : "seq");
 +    return super.submit(sequences, null, List.of(arg));
 +  }
 +
 +  @Override
 +  public AlignmentI getAlignment(JobId jobId) throws Exception
 +  {
-     List<RemoteFile> files;
++    Collection<RemoteFile> files;
 +    try {
-       files = client.getJobResults(jobId.getJobId());
++      var job = client.getJob(jobId.getJobId());
++      files = job.getResults();
 +      for (RemoteFile f : files) {
 +        var alignment = readAlignment(f);
 +        if (alignment != null)
 +        {
 +          return alignment;
 +        }
 +      }
 +    }
 +    catch (IOException e) {
 +      throw new IOError(e);
 +    }
 +    return null;
 +  }
 +
 +  @Override
 +  public JPredFile getPrediction(JobId jobId) throws Exception
 +  {
-     List<RemoteFile> files = client.getJobResults(jobId.getJobId());
++    Collection<RemoteFile> files = client.getJob(jobId.getJobId()).getResults();
 +    for (RemoteFile f : files)
 +    {
 +      if (f.getLabel().equals("concise"))
 +      {
-         return new JPredFile(f.getURL(), DataSourceType.URL);
++        return new JPredFile(f.getContentUrl(), DataSourceType.URL);
 +      }
 +    }
 +    return null;
 +  }
 +}
diff --cc src/jalview/ws/slivkaws/SlivkaMsaServiceInstance.java
index 8d6332c,03dda3f..b992fbe
--- a/src/jalview/ws/slivkaws/SlivkaMsaServiceInstance.java
+++ b/src/jalview/ws/slivkaws/SlivkaMsaServiceInstance.java
@@@ -11,8 -15,10 +15,9 @@@ import jalview.ws.params.WsParamSetI
  import java.io.IOError;
  import java.io.IOException;
  import java.rmi.ServerError;
+ import java.util.Collection;
  import java.util.List;
  
 -import compbio.data.msa.Category;
  import uk.ac.dundee.compbio.slivkaclient.RemoteFile;
  import uk.ac.dundee.compbio.slivkaclient.SlivkaClient;
  import uk.ac.dundee.compbio.slivkaclient.SlivkaService;
diff --cc src/jalview/ws/slivkaws/SlivkaWSDiscoverer.java
index d290024,accb40d..50fa2d6
--- a/src/jalview/ws/slivkaws/SlivkaWSDiscoverer.java
+++ b/src/jalview/ws/slivkaws/SlivkaWSDiscoverer.java
@@@ -4,6 -4,6 +4,8 @@@ import jalview.bin.Cache
  import jalview.ws.ServiceChangeListener;
  import jalview.ws.WSDiscovererI;
  import jalview.ws.api.ServiceWithParameters;
++import javajs.http.HttpClientFactory;
++
  import java.io.IOException;
  import java.net.MalformedURLException;
  import java.net.URL;
diff --cc src/jalview/ws/slivkaws/SlivkaWSInstance.java
index fa54cf0,0c71c35..80e928a
--- a/src/jalview/ws/slivkaws/SlivkaWSInstance.java
+++ b/src/jalview/ws/slivkaws/SlivkaWSInstance.java
@@@ -1,21 -1,7 +1,20 @@@
  package jalview.ws.slivkaws;
  
 +import java.io.ByteArrayInputStream;
 +import java.io.ByteArrayOutputStream;
 +import java.io.IOError;
 +import java.io.IOException;
 +import java.io.InputStream;
 +import java.util.Arrays;
 +import java.util.EnumMap;
 +import java.util.HashSet;
 +import java.util.List;
- import java.util.Optional;
 +import java.util.Set;
 +
 +import jalview.datamodel.AlignmentI;
  import jalview.datamodel.SequenceI;
  import jalview.gui.WebserviceInfo;
 +import jalview.io.DataSourceType;
  import jalview.io.FileFormat;
  import jalview.io.FormatAdapter;
  import jalview.ws.api.JalviewServiceEndpointProviderI;
@@@ -27,16 -13,26 +26,15 @@@ import jalview.ws.params.ArgumentI
  import jalview.ws.params.ParamDatastoreI;
  import jalview.ws.params.ParamManager;
  import jalview.ws.params.WsParamSetI;
- import uk.ac.dundee.compbio.slivkaclient.FieldType;
- import uk.ac.dundee.compbio.slivkaclient.FileField;
- import uk.ac.dundee.compbio.slivkaclient.FormField;
- import uk.ac.dundee.compbio.slivkaclient.FormValidationException;
- import uk.ac.dundee.compbio.slivkaclient.JobState;
+ import javajs.http.ClientProtocolException;
+ 
 -import java.io.ByteArrayInputStream;
 -import java.io.ByteArrayOutputStream;
 -import java.io.IOError;
 -import java.io.IOException;
 -import java.io.InputStream;
 -import java.util.Arrays;
+ import java.util.Collection;
 -import java.util.EnumMap;
 -import java.util.HashSet;
 -import java.util.List;
 -import java.util.Set;
 -
+ import uk.ac.dundee.compbio.slivkaclient.Job;
+ import uk.ac.dundee.compbio.slivkaclient.JobRequest;
+ import uk.ac.dundee.compbio.slivkaclient.Parameter;
  import uk.ac.dundee.compbio.slivkaclient.RemoteFile;
  import uk.ac.dundee.compbio.slivkaclient.SlivkaClient;
- import uk.ac.dundee.compbio.slivkaclient.SlivkaForm;
  import uk.ac.dundee.compbio.slivkaclient.SlivkaService;
- import uk.ac.dundee.compbio.slivkaclient.ValidationException;
  
  public abstract class SlivkaWSInstance extends ServiceWithParameters
      implements JalviewServiceEndpointProviderI, JalviewWebServiceI
@@@ -261,19 -254,5 +256,19 @@@
      }
      return store;
    }
 +  
 +  public static AlignmentI readAlignment(RemoteFile f) throws IOException
 +  {
-     final var mimetype = f.getMimeType();
++    final var mimetype = f.getMediaType();
 +    FileFormat format;
 +    if (mimetype.equals("application/clustal"))
 +      format = FileFormat.Clustal;
 +    else if (mimetype.equals("application/fasta"))
 +      format = FileFormat.Fasta;
 +    else
 +      return null;
-     return new FormatAdapter().readFile(f.getURL().toString(),
++    return new FormatAdapter().readFile(f.getContentUrl().toString(),
 +        DataSourceType.URL, format);
 +  }
  
  }