Adding SwingJS interface and utility classes
[jalview.git] / src / swingjs / api / js / HTML5Video.java
1 package swingjs.api.js;
2
3 import java.awt.Container;
4 import java.awt.Dimension;
5 import java.awt.Frame;
6 import java.awt.Graphics;
7 import java.awt.event.ActionEvent;
8 import java.awt.event.ActionListener;
9 import java.awt.image.BufferedImage;
10 import java.io.File;
11 import java.net.URL;
12 import java.nio.file.Files;
13 import java.util.ArrayList;
14 import java.util.function.Function;
15
16 import javax.swing.BoxLayout;
17 import javax.swing.ImageIcon;
18 import javax.swing.JButton;
19 import javax.swing.JDialog;
20 import javax.swing.JLabel;
21 import javax.swing.JPanel;
22
23 import swingjs.api.JSUtilI;
24
25 /**
26  * A full-service interface for HTML5 video element interaction. Allows setting
27  * and getting HTML5 video element properties. ActionListeners can be set to
28  * listen for JavaScript events associated with a video element.
29  * 
30  * Video is added using a JavaScript-only two-parameter constructor for
31  * ImageIcon with "jsvideo" as the description, allowing for video construction
32  * from byte[], File, or URL.
33  * 
34  * After adding the ImageIcon to a JLabel, calling
35  * jlabel.getClientProperty("jsvideo") returns an HTML5 object of type
36  * HTML5Video (the <video> tag), which has the full suite of HTML5 video
37  * element properties, methods, and events.
38  * 
39  * Access to event listeners is via the method addActionListener, below, which
40  * return an ActionEvent that has as its source both the video element source as
41  * well as the original JavaScript event as an Object[] { jsvideo, event }. The
42  * id of this ActionEvent is 12345, and its command is the name of the event,
43  * for example, "canplay" or "canplaythrough".
44  * 
45  * See https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement for
46  * details.
47  * 
48  * @author hansonr
49  *
50  */
51 public interface HTML5Video extends DOMNode {
52
53         public interface Promise {
54
55         }
56
57         final static String[] eventTypes = new String[] { "audioprocess", // The input buffer of a ScriptProcessorNode is
58                                                                                                                                                 // ready to be processed.
59                         "canplay", // The browser can play the media, but estimates that not enough data has been
60                                                 // loaded to play the media up to its end without having to stop for further
61                                                 // buffering of content.
62                         "canplaythrough", // The browser estimates it can play the media up to its end without stopping
63                                                                 // for content buffering.
64                         "complete", // The rendering of an OfflineAudioContext is terminated.
65                         "durationchange", // The duration attribute has been updated.
66                         "emptied", // The media has become empty; for example, this event is sent if the media has
67                                                 // already been loaded (or partially loaded), and the load() method is called to
68                                                 // reload it.
69                         "ended", // Playback has stopped because the end of the media was reached.
70                         "loadeddata", // The first frame of the media has finished loading.
71                         "loadedmetadata", // The metadata has been loaded.
72                         "pause", // Playback has been paused.
73                         "play", // Playback has begun.
74                         "playing", // Playback is ready to start after having been paused or delayed due to lack of
75                                                 // data.
76                         "progress", // Fired periodically as the browser loads a resource.
77                         "ratechange", // The playback rate has changed.
78                         "seeked", // A seek operation completed.
79                         "seeking", // A seek operation began.
80                         "stalled", // The user agent is trying to fetch media data, but data is unexpectedly not
81                                                 // forthcoming.
82                         "suspend", // Media data loading has been suspended.
83                         "timeupdate", // The time indicated by the currentTimeattribute has been updated.
84                         "volumechange", // The volume has changed.
85                         "waiting", // Playback has stopped because of a temporary lack of data
86         };
87
88         // direct methods
89
90         public void addTextTrack() throws Throwable;
91
92         public Object captureStream() throws Throwable;
93
94         public String canPlayType(String mediaType) throws Throwable;
95
96         public void fastSeek(double time) throws Throwable;
97
98         public void load() throws Throwable;
99
100         public void mozCaptureStream() throws Throwable;
101
102         public void mozCaptureStreamUntilEnded() throws Throwable;
103
104         public void mozGetMetadata() throws Throwable;
105
106         public void pause() throws Throwable;
107
108         public Promise play() throws Throwable;
109
110         public Promise seekToNextFrame() throws Throwable;
111
112         public Promise setMediaKeys(Object mediaKeys) throws Throwable;
113
114         public Promise setSinkId(String id) throws Throwable;
115
116         // convenience methods
117
118         public static double getDuration(HTML5Video v) {
119                 return /** @j2sNative v.duration || */
120                 0;
121         }
122
123         public static double setCurrentTime(HTML5Video v, double time) {
124                 return /** @j2sNative v.currentTime = time|| */
125                 0;
126         }
127
128         public static double getCurrentTime(HTML5Video v) {
129                 return /** @j2sNative v.currentTime|| */
130                 0;
131         }
132
133         public static Dimension getSize(HTML5Video v) {
134                 return new Dimension(/** @j2sNative v.videoWidth || */
135                                 0, /** @j2sNative v.videoHeight|| */
136                                 0);
137         }
138
139         /**
140          * 
141          * Create a BufferedIfmage from the current frame. The image will be of type
142          * swingjs.api.JSUtilI.TYPE_4BYTE_HTML5, matching the data buffer of HTML5
143          * images.
144          * 
145          * @param v
146          * @param imageType  if Integer.MIN_VALUE, swingjs.api.JSUtilI.TYPE_4BYTE_HTML5
147          * @return
148          */
149         public static BufferedImage getImage(HTML5Video v, int imageType) {
150                 Dimension d = HTML5Video.getSize(v);
151                 BufferedImage image = (BufferedImage) HTML5Video.getProperty(v, "_image");
152                 if (image == null || image.getWidth() != d.width || image.getHeight() != d.height) {
153                         image = new BufferedImage(d.width, d.height, imageType == Integer.MIN_VALUE ? JSUtilI.TYPE_4BYTE_HTML5 : imageType);
154                         HTML5Video.setProperty(v, "_image", image);
155                 }
156                 HTML5Canvas.setImageNode(v, image);
157                 return image;
158         }
159
160         // property setting and getting
161
162         /**
163          * Set a property of the the HTML5 video element using jsvideo[key] = value.
164          * Numbers and Booleans will be unboxed.
165          * 
166          * @param jsvideo the HTML5 video element
167          * @param key
168          * @param value
169          */
170         public static void setProperty(HTML5Video jsvideo, String key, Object value) {
171                 if (value instanceof Number) {
172                         /** @j2sNative jsvideo[key] = +value; */
173                 } else if (value instanceof Boolean) {
174                         /** @j2sNative jsvideo[key] = !!+value */
175                 } else {
176                         /** @j2sNative jsvideo[key] = value; */
177                 }
178         }
179
180         /**
181          * Get a property using jsvideo[key], boxing number as Double and boolean as
182          * Boolean.
183          * 
184          * @param jsvideo the HTML5 video element
185          * 
186          * @param key
187          * @return value or value boxed as Double or Boolean
188          */
189         @SuppressWarnings("unused")
190         public static Object getProperty(HTML5Video jsvideo, String key) {
191                 Object val = (/** @j2sNative 1? jsvideo[key] : */
192                 null);
193                 if (val == null)
194                         return null;
195                 switch (/** @j2sNative typeof val || */
196                 "") {
197                 case "number":
198                         return Double.valueOf(/** @j2sNative val || */
199                                         0);
200                 case "boolean":
201                         return Boolean.valueOf(/** @j2sNative val || */
202                                         false);
203                 default:
204                         return val;
205                 }
206         }
207
208         // event action
209
210         /**
211          * Add an ActionListener for the designated events. When an event is fired,
212          * 
213          * @param jsvideo  the HTML5 video element
214          * @param listener
215          * @param events   array of events to listen to or null to listen on all video
216          *                 element event types
217          * @return an array of event/listener pairs that can be used for removal.
218          */
219         public static Object[] addActionListener(HTML5Video jsvideo, ActionListener listener, String... events) {
220                 if (events == null || events.length == 0)
221                         events = eventTypes;
222                 @SuppressWarnings("unused")
223                 Function<Object, Void> f = new Function<Object, Void>() {
224
225                         @Override
226                         public Void apply(Object jsevent) {
227                                 String name = (/** @j2sNative jsevent.type || */
228                                 "?");
229                                 System.out.println("HTML5Video " + name);
230                                 ActionEvent e = new ActionEvent(new Object[] { jsvideo, jsevent }, 12345, name,
231                                                 System.currentTimeMillis(), 0);
232                                 listener.actionPerformed(e);
233                                 return null;
234                         }
235                 };
236                 ArrayList<Object> listeners = new ArrayList<>();
237                 for (int i = 0; i < events.length; i++) {
238                         Object func = /**
239                                                          * @j2sNative function(event){f.apply$O.apply(f, [event])} ||
240                                                          */
241                                         null;
242                         listeners.add(events[i]);
243                         listeners.add(func);
244                         if (jsvideo != null)
245                                 jsvideo.addEventListener(events[i], func);
246
247                 }
248                 return listeners.toArray(new Object[listeners.size()]);
249         }
250
251         /**
252          * Remove action listener
253          * 
254          * @param jsvideo   the HTML5 video element
255          * @param listeners an array of event/listener pairs created by
256          *                  addActionListener
257          */
258         public static void removeActionListener(HTML5Video jsvideo, Object[] listeners) {
259                 if (listeners == null) {
260                         for (int i = 0; i < eventTypes.length; i++) {
261                                 jsvideo.removeEventListener(eventTypes[i]);
262                         }
263                 }
264                 
265                 for (int i = 0; i < listeners.length; i += 2) {
266                         String event = (String) listeners[i];
267                         Object listener = listeners[i + 1];
268                         jsvideo.removeEventListener(event, listener);
269                 }
270         }
271
272         /**
273          * Create an ImageIcon which, when placed in a JLabel, displays the video.
274          * 
275          * @param source
276          * @return
277          */
278         public static ImageIcon createIcon(Object source) {
279                 try {
280                         if (source instanceof URL) {
281                                 return new ImageIcon((URL) source, "jsvideo");
282                         } else if (source instanceof byte[]) {
283                                 return new ImageIcon((byte[]) source, "jsvideo");
284                         } else if (source instanceof File) {
285                                 return new ImageIcon(Files.readAllBytes(((File) source).toPath()));
286                         } else {
287                                 return new ImageIcon(Files.readAllBytes(new File(source.toString()).toPath()));
288                         }
289                 } catch (Throwable t) {
290                         return null;
291                 }
292         }
293
294         /**
295          * Create a label that, when shown, displays the video.
296          * 
297          * @param source
298          * @return
299          */
300         public static JLabel createLabel(Object source) {
301                 ImageIcon icon = (source instanceof ImageIcon ? (ImageIcon) source : createIcon(source));
302                 return (icon == null ? null : new JLabel(icon));
303         }
304
305         /**
306          * Create a dialog that includes rudimentary controls. Optional maxWidth allows image downscaling by factors of two.
307          * 
308          * @param parent
309          * @param source 
310          * @param maxWidth
311          * @return
312          */
313         public static JDialog createDialog(Frame parent, Object source, int maxWidth, Function<HTML5Video, Void> whenReady) {
314                 JDialog dialog = new JDialog(parent);
315                 Container p = dialog.getContentPane();
316                 p.setLayout(new BoxLayout(p, BoxLayout.Y_AXIS));
317                 JLabel label = (source instanceof JLabel ? (JLabel) source : createLabel(source));
318                 label.setAlignmentX(0.5f);
319                 // not in Java! dialog.putClientProperty("jsvideo", label);
320                 p.add(label);
321                 label.setVisible(false);
322                 p.add(getControls(label));
323                 dialog.setModal(false);
324                 dialog.pack();
325                 dialog.setVisible(true);
326                 dialog.setVisible(false);
327                 HTML5Video jsvideo = (HTML5Video) label.getClientProperty("jsvideo");
328                 HTML5Video.addActionListener(jsvideo, new ActionListener() {
329
330                         @Override
331                         public void actionPerformed(ActionEvent e) {
332                                 if (label.getClientProperty("jsvideo.size") != null)
333                                         return;
334                                 Dimension dim = HTML5Video.getSize(jsvideo);
335                                 while (dim.width > maxWidth) {
336                                         dim.width /= 2;
337                                         dim.height /= 2;
338                                 }
339                                 label.putClientProperty("jsvideo.size", dim);
340                                 label.setPreferredSize(dim);
341                                 label.setVisible(true);
342 //                              label.invalidate();
343                                 dialog.pack();
344 //                              dialog.setVisible(false);
345                                 if (whenReady != null)
346                                         whenReady.apply(jsvideo);
347                         }
348                         
349                 }, "canplaythrough");
350                 HTML5Video.setCurrentTime(jsvideo,  0);
351                 return dialog;
352         }
353
354         static JPanel getControls(JLabel label) {
355
356                 JPanel controls = new JPanel();
357                 controls.setAlignmentX(0.5f);
358                 JButton btn = new JButton("play");
359                 btn.addActionListener(new ActionListener() {
360
361                         @Override
362                         public void actionPerformed(ActionEvent e) {
363                                 try {
364                                         ((HTML5Video) label.getClientProperty("jsvideo")).play();
365                                 } catch (Throwable e1) {
366                                         e1.printStackTrace();
367                                 }
368                         }
369
370                 });
371                 controls.add(btn);
372
373                 btn = new JButton("pause");
374                 btn.addActionListener(new ActionListener() {
375
376                         @Override
377                         public void actionPerformed(ActionEvent e) {
378                                 try {
379                                         ((HTML5Video) label.getClientProperty("jsvideo")).pause();
380                                 } catch (Throwable e1) {
381                                         e1.printStackTrace();
382                                 }
383                         }
384
385                 });
386                 controls.add(btn);
387                 
388                 btn = new JButton("reset");
389                 btn.addActionListener(new ActionListener() {
390
391                         @Override
392                         public void actionPerformed(ActionEvent e) {
393                                 HTML5Video.setCurrentTime((HTML5Video) label.getClientProperty("jsvideo"), 0);
394                         }
395
396                 });
397                 controls.add(btn);
398
399                 return controls;
400         }
401
402         /**
403          * Advance to the next frame, using seekToNextFrame() if available, or using the time difference supplied.
404          * 
405          * @param jsvideo
406          * @param dt  seconds to advance if seekToNextFrame() is not available
407          * @return true if can use seekToNextFrame()
408          * 
409          */
410         public static boolean nextFrame(HTML5Video jsvideo, double dt) {
411                 Boolean canSeek = (Boolean) getProperty(jsvideo,"_canseek");
412                 if (canSeek == null) {
413                         setProperty(jsvideo, "_canseek", canSeek = Boolean.valueOf(getProperty(jsvideo, "seekToNextFrame") != null));
414                 }
415                 try {                   
416                         if (canSeek) {
417                                 jsvideo.seekToNextFrame();
418                         } else {
419                                 HTML5Video.setCurrentTime(jsvideo, HTML5Video.getCurrentTime(jsvideo) + dt);                                            
420                         }
421                 } catch (Throwable e1) {
422                 }
423                 return canSeek.booleanValue();
424         }
425
426         public static int getFrameCount(HTML5Video jsvideo) {
427                 return (int) (getDuration(jsvideo) / 0.033334);
428         }
429
430 // HTMLMediaElement properties
431
432 //      audioTracks
433 //      autoplay
434 //      buffered Read only
435 //      controller
436 //      controls
437 //      controlsList Read only
438 //      crossOrigin
439 //      currentSrc Read only
440 //      currentTime
441 //      defaultMuted
442 //      defaultPlaybackRate
443 //      disableRemotePlayback
444 //      duration Read only
445 //      ended Read only
446 //      error Read only
447 //      loop
448 //      mediaGroup
449 //      mediaKeys Read only
450 //      mozAudioCaptured Read only
451 //      mozFragmentEnd
452 //      mozFrameBufferLength
453 //      mozSampleRate Read only
454 //      muted
455 //      networkState Read only
456 //      paused Read only
457 //      playbackRate
458 //      played Read only
459 //      preload
460 //      preservesPitch
461 //      readyState Read only
462 //      seekable Read only
463 //      seeking Read only
464 //      sinkId Read only
465 //      src
466 //      srcObject
467 //      textTracks Read only
468 //      videoTracks Read only
469 //      volume
470 //      initialTime Read only
471 //      mozChannels Read only
472
473 }