From 6fad8c3fe5fc07de005c19cded2df58dc34c3e03 Mon Sep 17 00:00:00 2001 From: Mateusz Warowny Date: Thu, 25 Nov 2021 15:50:45 +0100 Subject: [PATCH] JAL-3878 Add jpred operation and worker to the services. --- src/jalview/ws2/gui/JPredMenuBuilder.java | 157 +++++++++++ src/jalview/ws2/operations/AbstractWorker.java | 2 +- src/jalview/ws2/operations/JPredOperation.java | 35 +++ src/jalview/ws2/operations/JPredWorker.java | 334 ++++++++++++++++++++++++ src/jalview/ws2/slivka/SlivkaWSDiscoverer.java | 22 +- src/jalview/ws2/slivka/SlivkaWebService.java | 38 +-- 6 files changed, 570 insertions(+), 18 deletions(-) create mode 100644 src/jalview/ws2/gui/JPredMenuBuilder.java create mode 100644 src/jalview/ws2/operations/JPredOperation.java create mode 100644 src/jalview/ws2/operations/JPredWorker.java diff --git a/src/jalview/ws2/gui/JPredMenuBuilder.java b/src/jalview/ws2/gui/JPredMenuBuilder.java new file mode 100644 index 0000000..7059c5f --- /dev/null +++ b/src/jalview/ws2/gui/JPredMenuBuilder.java @@ -0,0 +1,157 @@ +package jalview.ws2.gui; + +import static java.lang.String.format; + +import java.util.Objects; +import java.util.function.Consumer; + +import javax.swing.JMenu; +import javax.swing.JMenuItem; + +import jalview.datamodel.Alignment; +import jalview.gui.AlignFrame; +import jalview.gui.Desktop; +import jalview.gui.WebserviceInfo; +import jalview.util.MessageManager; +import jalview.ws2.WSJob; +import jalview.ws2.operations.JPredOperation; +import jalview.ws2.operations.JPredWorker; +import jalview.ws2.operations.WebServiceWorkerI; +import jalview.ws2.operations.WebServiceWorkerListener; +import jalview.ws2.operations.JPredWorker.PredictionResult; + +public class JPredMenuBuilder implements MenuEntryProviderI +{ + private JPredOperation operation; + + public JPredMenuBuilder(JPredOperation operation) + { + this.operation = operation; + } + + public void buildMenu(JMenu menu, AlignFrame frame) + { + final JMenuItem mi = new JMenuItem(operation.getName()); + mi.setToolTipText(operation.getHostName()); + mi.addActionListener((event) -> { + String panelInfo = String.format("%s using service hosted at %s%n%s", + operation.getName(), operation.getHostName(), + Objects.requireNonNullElse(operation.getDescription(), "")); + var wsInfo = new WebserviceInfo(operation.getName(), panelInfo, false); + + var alignView = frame.gatherSeqOrMsaForSecStrPrediction(); + var worker = new JPredWorker(operation, alignView, + frame.getCurrentView()); + + var jpu = new JPredProgressUpdater(worker, wsInfo, frame); + worker.setResultConsumer(jpu); + worker.addListener(jpu); + + frame.getViewport().getWSExecutor().submit(worker); + }); + menu.add(mi); + } +} + +class JPredProgressUpdater + implements WebServiceWorkerListener, Consumer +{ + WebServiceWorkerI worker; + + WebserviceInfo wsInfo; + + AlignFrame frame; + + private final WebServiceInfoUpdater wsInfoUpdater; + + JPredProgressUpdater(WebServiceWorkerI worker, WebserviceInfo wsInfo, + AlignFrame frame) + { + this.worker = worker; + this.wsInfo = wsInfo; + this.frame = frame; + this.wsInfoUpdater = new WebServiceInfoUpdater(worker, wsInfo); + } + + @Override + public void workerStarted(WebServiceWorkerI source) + { + wsInfo.setVisible(true); + } + + @Override + public void workerNotStarted(WebServiceWorkerI source) + { + wsInfo.setVisible(true); + wsInfo.setStatus(WebserviceInfo.STATE_STOPPED_SERVERERROR); + wsInfo.setStatus(0, WebserviceInfo.STATE_STOPPED_SERVERERROR); + wsInfo.appendProgressText(0, MessageManager.getString( + "info.failed_to_submit_sequences_for_alignment")); + } + + @Override + public void jobCreated(WebServiceWorkerI source, WSJob job) + { + wsInfo.addJobPane(); + job.addPropertyChangeListener(wsInfoUpdater); + } + + @Override + public void pollException(WebServiceWorkerI source, WSJob job, Exception e) + { + wsInfo.appendProgressText(job.getJobNum(), + MessageManager.formatMessage("info.server_exception", + source.getOperation().getName(), e.getMessage())); + } + + @Override + public void workerCompleting(WebServiceWorkerI source) + { + wsInfo.setProgressBar( + MessageManager.getString("status.collecting_job_results"), + worker.getUID()); + } + + @Override + public void workerCompleted(WebServiceWorkerI source) + { + wsInfo.removeProgressBar(worker.getUID()); + } + + @Override + public void accept(PredictionResult result) + { + if (result != null) + { + wsInfo.showResultsNewFrame.addActionListener( + (evt) -> displayResults(result, true)); + wsInfo.mergeResults.addActionListener( + (evt) -> displayResults(result, false)); + wsInfo.setResultsReady(); + } + else + { + wsInfo.setStatus(WebserviceInfo.STATE_STOPPED_ERROR); + wsInfo.appendInfoText("No jobs ran."); + wsInfo.setFinishedNoResults(); + } + } + + private void displayResults(PredictionResult result, boolean newWindow) + { + if (newWindow) + { + Alignment alignment = new Alignment(result.getAlignment()); + alignment.setSeqrep(alignment.getSequenceAt(0)); + for (var annotation : result.getAlignment().getAlignmentAnnotation()) + { + alignment.addAnnotation(annotation); + } + AlignFrame frame = new AlignFrame(alignment, result.getHiddenCols(), + AlignFrame.DEFAULT_WIDTH, AlignFrame.DEFAULT_HEIGHT); + Desktop.addInternalFrame(frame, frame.getTitle(), AlignFrame.DEFAULT_WIDTH, + AlignFrame.DEFAULT_HEIGHT); + } + } + +} \ No newline at end of file diff --git a/src/jalview/ws2/operations/AbstractWorker.java b/src/jalview/ws2/operations/AbstractWorker.java index 5186849..56d1af7 100644 --- a/src/jalview/ws2/operations/AbstractWorker.java +++ b/src/jalview/ws2/operations/AbstractWorker.java @@ -29,7 +29,7 @@ public abstract class AbstractWorker implements WebServiceWorkerI private Map exceptionCount = new HashMap<>(); - private static final int MAX_RETRY = 5; + protected static final int MAX_RETRY = 5; public boolean poll() { diff --git a/src/jalview/ws2/operations/JPredOperation.java b/src/jalview/ws2/operations/JPredOperation.java new file mode 100644 index 0000000..1a2226f --- /dev/null +++ b/src/jalview/ws2/operations/JPredOperation.java @@ -0,0 +1,35 @@ +package jalview.ws2.operations; + +import java.io.IOException; + +import jalview.datamodel.AlignmentI; +import jalview.io.JPredFile; +import jalview.ws2.WSJob; +import jalview.ws2.WebServiceI; +import jalview.ws2.gui.JPredMenuBuilder; +import jalview.ws2.gui.MenuEntryProviderI; + +public class JPredOperation extends AbstractOperation +{ + public static interface PredictionResultSupplier + { + public AlignmentI getAlignment(WSJob job) throws IOException; + + public JPredFile getPrediction(WSJob job) throws IOException; + } + + PredictionResultSupplier predictionSupplier; + + public JPredOperation(WebServiceI service, String typeName, + PredictionResultSupplier predictionSupplier) + { + super(service, typeName); + this.predictionSupplier = predictionSupplier; + } + + @Override + public MenuEntryProviderI getMenuBuilder() + { + return new JPredMenuBuilder(this); + } +} diff --git a/src/jalview/ws2/operations/JPredWorker.java b/src/jalview/ws2/operations/JPredWorker.java new file mode 100644 index 0000000..6cb8a2e --- /dev/null +++ b/src/jalview/ws2/operations/JPredWorker.java @@ -0,0 +1,334 @@ +package jalview.ws2.operations; + +import static java.lang.String.format; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + +import jalview.analysis.SeqsetUtils; +import jalview.analysis.SeqsetUtils.SequenceInfo; +import jalview.api.AlignViewportI; +import jalview.bin.Cache; +import jalview.commands.RemoveGapsCommand; +import jalview.datamodel.Alignment; +import jalview.datamodel.AlignmentAnnotation; +import jalview.datamodel.AlignmentI; +import jalview.datamodel.AlignmentView; +import jalview.datamodel.HiddenColumns; +import jalview.datamodel.SeqCigar; +import jalview.datamodel.SequenceI; +import jalview.gui.AlignFrame; +import jalview.io.JnetAnnotationMaker; +import jalview.util.MessageManager; +import jalview.ws.params.ArgumentI; +import jalview.ws2.WSJob; +import jalview.ws2.WSJobStatus; +import jalview.ws2.operations.AlignmentWorker.AlignmentJob; + +public class JPredWorker extends AbstractPollableWorker +{ + + private class InputFormatParameter implements ArgumentI + { + String value = ""; + + @Override + public String getName() + { + return "format"; + } + + @Override + public String getValue() + { + return value; + } + + @Override + public void setValue(String selectedItem) + { + value = selectedItem; + } + } + + private static class JobInput + { + List msf; + + int[] delMap; + + Map sequenceInfo; + } + + public class JPredJob extends WSJob + { + List msf; + + int[] delMap; + + Map sequenceInfo; + + private JPredJob() + { + super(operation.service.getProviderName(), operation.getName(), + operation.getHostName()); + } + + private void setInput(JobInput input) + { + msf = input.msf; + delMap = input.delMap; + sequenceInfo = input.sequenceInfo; + } + } + + public class PredictionResult + { + AlignmentI alignment; + + HiddenColumns hiddenCols; + + int firstSeq; + + public AlignmentI getAlignment() + { + return alignment; + } + + public HiddenColumns getHiddenCols() + { + return hiddenCols; + } + } + + private JPredOperation operation; + + private Consumer resultConsumer; + + private AlignmentView view; + + private WSJobList jobs = new WSJobList<>(); + + private JPredJob job; + + private char gapChar; + + AlignmentI currentView; + + public JPredWorker(JPredOperation operation, AlignmentView alignView, + AlignViewportI viewport) + { + this.operation = operation; + this.view = alignView; + this.gapChar = viewport.getGapCharacter(); + this.currentView = viewport.getAlignment(); + } + + @Override + public Operation getOperation() + { + return operation; + } + + @Override + public WSJobList getJobs() + { + return jobs; + } + + public void setResultConsumer(Consumer consumer) + { + this.resultConsumer = consumer; + } + + @Override + public void start() throws IOException + { + var input = prepareInputData(view, true); + job = new JPredJob(); + job.setInput(input); + jobs.add(job); + listeners.fireJobCreated(job); + + var formatArg = new InputFormatParameter(); + formatArg.setValue(input.msf.size() > 1 ? "fasta" : "seq"); + List args = List.of(formatArg); + int exceptionCount = MAX_RETRY; + String jobId = null; + do + { + try + { + jobId = operation.getWebService().submit(job.msf, args); + } catch (IOException e) + { + Cache.log.warn(format("%s failed to submit sequences to the server %s.", + operation.getName(), operation.getHostName()), e); + exceptionCount--; + } + } while (jobId == null && exceptionCount > 0); + if (jobId != null) + { + job.setJobId(jobId); + job.setStatus(WSJobStatus.SUBMITTED); + listeners.fireWorkerStarted(); + } + else + { + job.setStatus(WSJobStatus.SERVER_ERROR); + listeners.fireWorkerNotStarted(); + } + } + + private static JobInput prepareInputData(AlignmentView view, boolean viewOnly) + { + SeqCigar[] msf = view.getSequences(); + SequenceI seq = msf[0].getSeq('-'); + int[] delMap = null; + if (viewOnly) + delMap = view.getVisibleContigMapFor(seq.gapMap()); + SequenceI[] aln = new SequenceI[msf.length]; + for (int i = 0; i < msf.length; i++) + aln[i] = msf[i].getSeq('-'); + var sequenceInfo = msf.length > 1 ? SeqsetUtils.uniquify(aln, true) + : Map.of("Sequence", SeqsetUtils.SeqCharacterHash(seq)); + if (viewOnly) + { + // Remove hidden regions from sequence objects. + String seqs[] = view.getSequenceStrings('-'); + for (int i = 0; i < msf.length; i++) + aln[i].setSequence(seqs[i]); + seq.setSequence(seqs[0]); + } + var input = new JobInput(); + input.msf = List.of(aln); + input.delMap = delMap; + input.sequenceInfo = sequenceInfo; + return input; + } + + @Override + public void done() + { + listeners.fireWorkerCompleting(); + PredictionResult result = null; + try + { + result = (job.msf.size() > 1) + ? prepareMultipleSequenceResult(job) + : prepareSingleSequenceResult(job); + } catch (Exception e) + { + Cache.log.error("Couldn't retrieve results for job.", e); + job.setStatus(WSJobStatus.SERVER_ERROR); + } + if (result != null) + { + for (var annot : result.alignment.getAlignmentAnnotation()) + { + if (annot.sequenceRef != null) + { + replaceAnnotationOnAlignmentWith(annot, annot.label, + getClass().getName(), annot.sequenceRef); + } + } + } + resultConsumer.accept(result); + listeners.fireWorkerCompleted(); + } + + private PredictionResult prepareMultipleSequenceResult(JPredJob job) + throws Exception + { + AlignmentI alignment; + HiddenColumns hiddenCols = null; + var prediction = operation.predictionSupplier.getPrediction(job); + if (job.delMap != null) + { + Object[] alandcolsel = view.getAlignmentAndHiddenColumns(gapChar); + alignment = new Alignment((SequenceI[]) alandcolsel[0]); + hiddenCols = (HiddenColumns) alandcolsel[1]; + } + else + { + alignment = operation.predictionSupplier.getAlignment(job); + var seqs = new SequenceI[alignment.getHeight()]; + for (int i = 0; i < alignment.getHeight(); i++) + { + seqs[i] = alignment.getSequenceAt(i); + } + SeqsetUtils.deuniquify(job.sequenceInfo, seqs); + } + int firstSeq = 0; + alignment.setDataset(currentView.getDataset()); + JnetAnnotationMaker.add_annotation(prediction, alignment, firstSeq, false, + job.delMap); + var result = new PredictionResult(); + result.alignment = alignment; + result.hiddenCols = hiddenCols; + result.firstSeq = firstSeq; + return result; + } + + static final int msaIndex = 0; + + private PredictionResult prepareSingleSequenceResult(JPredJob job) + throws Exception + { + var prediction = operation.predictionSupplier.getPrediction(job); + AlignmentI alignment = new Alignment(prediction.getSeqsAsArray()); + HiddenColumns hiddenCols = null; + int firstSeq = prediction.getQuerySeqPosition(); + if (job.delMap != null) + { + Object[] alanndcolsel = view.getAlignmentAndHiddenColumns(gapChar); + SequenceI[] seqs = (SequenceI[]) alanndcolsel[0]; + new RemoveGapsCommand(MessageManager.getString("label.remove_gaps"), + new SequenceI[] + { seqs[msaIndex] }, currentView); + SequenceI profileSeq = alignment.getSequenceAt(firstSeq); + profileSeq.setSequence(seqs[msaIndex].getSequenceAsString()); + } + SeqsetUtils.SeqCharacterUnhash(alignment.getSequenceAt(firstSeq), + job.sequenceInfo.get("Sequence")); + alignment.setDataset(currentView.getDataset()); + JnetAnnotationMaker.add_annotation(prediction, alignment, firstSeq, true, + job.delMap); + SequenceI profileSeq = alignment.getSequenceAt(0); + if (job.delMap != null) + { + hiddenCols = alignment.propagateInsertions(profileSeq, view); + } + var result = new PredictionResult(); + result.alignment = alignment; + result.hiddenCols = hiddenCols; + result.firstSeq = firstSeq; + return result; + } + + private static void replaceAnnotationOnAlignmentWith( + AlignmentAnnotation newAnnot, String typeName, String calcId, + SequenceI aSeq) + { + SequenceI dsseq = aSeq.getDatasetSequence(); + while (dsseq.getDatasetSequence() != null) + { + dsseq = dsseq.getDatasetSequence(); + } + // look for same annotation on dataset and lift this one over + List dsan = dsseq.getAlignmentAnnotations(calcId, + typeName); + if (dsan != null && dsan.size() > 0) + { + for (AlignmentAnnotation dssan : dsan) + { + dsseq.removeAlignmentAnnotation(dssan); + } + } + AlignmentAnnotation dssan = new AlignmentAnnotation(newAnnot); + dsseq.addAlignmentAnnotation(dssan); + dssan.adjustForAlignment(); + } + +} diff --git a/src/jalview/ws2/slivka/SlivkaWSDiscoverer.java b/src/jalview/ws2/slivka/SlivkaWSDiscoverer.java index cda6702..4cbd1a2 100644 --- a/src/jalview/ws2/slivka/SlivkaWSDiscoverer.java +++ b/src/jalview/ws2/slivka/SlivkaWSDiscoverer.java @@ -7,6 +7,8 @@ import java.util.*; import java.util.concurrent.*; import jalview.bin.Cache; +import jalview.datamodel.AlignmentI; +import jalview.io.JPredFile; import jalview.ws2.*; import jalview.ws2.operations.*; import uk.ac.dundee.compbio.slivkaclient.SlivkaClient; @@ -157,7 +159,7 @@ public class SlivkaWSDiscoverer implements WebServiceDiscovererI } for (SlivkaService service : services) { - SlivkaWebService webService = new SlivkaWebService(client, service); + final SlivkaWebService webService = new SlivkaWebService(client, service); AbstractOperation op = null; for (String classifier : service.classifiers) { @@ -184,6 +186,24 @@ public class SlivkaWSDiscoverer implements WebServiceDiscovererI op = new AnnotationOperation(webService, "Protein Disorder", webService::attachAnnotations); break; + case "protein secondary structure prediction": + var predictionSupplier = new JPredOperation.PredictionResultSupplier() + { + @Override + public JPredFile getPrediction(WSJob job) throws IOException + { + return webService.getPrediction(job); + } + + @Override + public AlignmentI getAlignment(WSJob job) throws IOException + { + return webService.getAlignment(job); + } + }; + op = new JPredOperation(webService, + "Secondary Structure Prediction", predictionSupplier); + break; case "multiple sequence alignment": op = new AlignmentOperation(webService, webService::getAlignment); break; diff --git a/src/jalview/ws2/slivka/SlivkaWebService.java b/src/jalview/ws2/slivka/SlivkaWebService.java index 9078013..e2873c1 100644 --- a/src/jalview/ws2/slivka/SlivkaWebService.java +++ b/src/jalview/ws2/slivka/SlivkaWebService.java @@ -1,20 +1,17 @@ package jalview.ws2.slivka; import static java.lang.String.format; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.EnumMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; -import jalview.api.AlignViewportI; import jalview.api.FeatureColourI; import jalview.bin.Cache; import jalview.datamodel.Alignment; @@ -26,18 +23,14 @@ import jalview.io.AnnotationFile; import jalview.io.DataSourceType; import jalview.io.FeaturesFile; import jalview.io.FileFormat; -import jalview.io.FileFormatI; import jalview.io.FormatAdapter; import jalview.io.JPredFile; -import jalview.ws.gui.WsJob; import jalview.ws.params.ArgumentI; import jalview.ws.params.ParamDatastoreI; -import jalview.ws.params.WsParamSetI; import jalview.ws.slivkaws.SlivkaDatastore; -import jalview.ws2.WebServiceI; -import jalview.ws2.operations.Operation; import jalview.ws2.WSJob; import jalview.ws2.WSJobStatus; +import jalview.ws2.WebServiceI; import javajs.http.ClientProtocolException; import uk.ac.dundee.compbio.slivkaclient.Job; import uk.ac.dundee.compbio.slivkaclient.Parameter; @@ -50,11 +43,11 @@ public class SlivkaWebService implements WebServiceI protected final SlivkaClient client; protected final SlivkaService service; - + protected ParamDatastoreI store; protected static final EnumMap statusMap = new EnumMap<>( - Job.Status.class); + Job.Status.class); { statusMap.put(Job.Status.PENDING, WSJobStatus.SUBMITTED); statusMap.put(Job.Status.REJECTED, WSJobStatus.INVALID); @@ -117,7 +110,7 @@ public class SlivkaWebService implements WebServiceI @Override public String submit(List sequences, List args) - throws IOException + throws IOException { var request = new uk.ac.dundee.compbio.slivkaclient.JobRequest(); for (Parameter param : service.getParameters()) @@ -144,8 +137,8 @@ public class SlivkaWebService implements WebServiceI break; } InputStream stream = new ByteArrayInputStream(format.getWriter(null) - .print(sequences.toArray(new SequenceI[0]), false) - .getBytes()); + .print(sequences.toArray(new SequenceI[0]), false) + .getBytes()); request.addFile(param.getId(), stream); } } @@ -232,12 +225,12 @@ public class SlivkaWebService implements WebServiceI if (f.getMediaType().equals("application/clustal")) { return new FormatAdapter().readFile(f.getContentUrl().toString(), - DataSourceType.URL, FileFormat.Clustal); + DataSourceType.URL, FileFormat.Clustal); } else if (f.getMediaType().equals("application/fasta")) { return new FormatAdapter().readFile(f.getContentUrl().toString(), - DataSourceType.URL, FileFormat.Fasta); + DataSourceType.URL, FileFormat.Fasta); } } return null; @@ -288,6 +281,19 @@ public class SlivkaWebService implements WebServiceI return Arrays.asList(aln.getAlignmentAnnotation()); } + public JPredFile getPrediction(WSJob job) throws IOException + { + Collection files = client.getJob(job.getJobId()).getResults(); + for (RemoteFile f : files) + { + if (f.getLabel().equals("concise")) + { + return new JPredFile(f.getContentUrl(), DataSourceType.URL); + } + } + return null; + } + @Override public String toString() { -- 1.7.10.2