Merge pull request biigle#823 from ToukL/jump-frame-by-frame

Implements 'Jump frame by frame' in video

resources/assets/js/videos/components/settingsTab.vue

@@ -6,6 +6,12 @@ export default {
     components: {
         powerToggle: PowerToggle,
     },
+    props: {
+        supportsJumpByFrame: {
+            type: Boolean,
+            default: false,
+        },
+    },
     data() {
         return {
             restoreKeys: [
@@ -15,6 +21,7 @@ export default {
                 'showLabelTooltip',
                 'showMousePosition',
                 'showProgressIndicator',
+                'enableJumpByFrame',
                 'jumpStep',
                 'muteVideo'
             ],
@@ -26,9 +33,15 @@ export default {
             playbackRate: 1.0,
             jumpStep: 5.0,
             showProgressIndicator: true,
+            enableJumpByFrame: false,
             muteVideo: true,
         };
     },
+    computed: {
+        jumpByFrameNotSupported() {
+            return !this.supportsJumpByFrame;
+        },
+    },
     methods: {
         handleShowMinimap() {
             this.showMinimap = true;
@@ -54,6 +67,12 @@ export default {
         handleHideProgressIndicator() {
             this.showProgressIndicator = false;
         },
+        handleEnableJumpByFrame() {
+            this.enableJumpByFrame = true;
+        },
+        handleDisableJumpByFrame() {
+            this.enableJumpByFrame = false;
+        },
         handleMuteVideo() {
             this.muteVideo = true;
         },
@@ -101,6 +120,10 @@ export default {
             this.$emit('update', 'showProgressIndicator', show);
             Settings.set('showProgressIndicator', show);
         },
+        enableJumpByFrame(show) {
+            this.$emit('update', 'enableJumpByFrame', show);
+            Settings.set('enableJumpByFrame', show);
+        },
         muteVideo(show) {
             this.$emit('update', 'muteVideo', show);
             Settings.set('muteVideo', show);

resources/assets/js/videos/components/videoScreen.vue

@@ -11,20 +11,33 @@
             :position="mousePosition"
             ></label-tooltip>
         <div class="controls">
-            <div class="btn-group">
+            <div v-if="showPrevNext" class="btn-group">
                 <control-button
-                    v-if="showPrevNext"
                     icon="fa-step-backward"
-                    title="Previous video 𝗟𝗲𝗳𝘁 𝗮𝗿𝗿𝗼𝘄"
+                    :title="enableJumpByFrame ? 'Previous video 𝗦𝗵𝗶𝗳𝘁+𝗟𝗲𝗳𝘁 𝗮𝗿𝗿𝗼𝘄' : 'Previous video 𝗟𝗲𝗳𝘁 𝗮𝗿𝗿𝗼𝘄'"
                     @click="emitPrevious"
                     ></control-button>
+                <control-button
+                    icon="fa-step-forward"
+                    :title="enableJumpByFrame ? 'Next video 𝗦𝗵𝗶𝗳𝘁+𝗥𝗶𝗴𝗵𝘁 𝗮𝗿𝗿𝗼𝘄' : 'Next video 𝗥𝗶𝗴𝗵𝘁 𝗮𝗿𝗿𝗼𝘄'"
+                    @click="emitNext"
+                    ></control-button>
+            </div>
+            <div class="btn-group">
                 <control-button
                     v-if="jumpStep!=0"
                     :disabled="seeking"
                     icon="fa-backward"
                     :title="jumpBackwardMessage"
                     @click="jumpBackward"
                     ></control-button>
+                <control-button
+                    v-if="enableJumpByFrame"
+                    :disabled="seeking"
+                    icon="fa-caret-square-left"
+                    title="Previous frame 𝗟𝗲𝗳𝘁 𝗮𝗿𝗿𝗼𝘄"
+                    v-on:click="emitPreviousFrame"
+                    ></control-button>
                 <control-button
                     v-if="playing"
                     icon="fa-pause"
@@ -39,19 +52,20 @@
                     :disabled="hasError"
                     @click="play"
                     ></control-button>
+                <control-button
+                    v-if="enableJumpByFrame"
+                    :disabled="seeking"
+                    icon="fa-caret-square-right"
+                    title="Next frame 𝗥𝗶𝗴𝗵𝘁 𝗮𝗿𝗿𝗼𝘄"
+                    v-on:click="emitNextFrame"
+                    ></control-button>
                 <control-button
                     v-if="jumpStep!=0"
                     :disabled="seeking"
                     icon="fa-forward"
                     :title="jumpForwardMessage"
                     @click="jumpForward"
                     ></control-button>
-                <control-button
-                     v-if="showPrevNext"
-                    icon="fa-step-forward"
-                    title="Next video 𝗥𝗶𝗴𝗵𝘁 𝗮𝗿𝗿𝗼𝘄"
-                    @click="emitNext"
-                    ></control-button>
             </div>
             <div v-if="canAdd" class="btn-group">
                 <control-button
@@ -374,7 +388,11 @@ export default {
         reachedTrackedAnnotationLimit: {
             type: Boolean,
             default: false,
-        }
+        },
+        enableJumpByFrame: {
+            type: Boolean,
+            default: false,
+        },
     },
     data() {
         return {
@@ -537,6 +555,24 @@ export default {
             this.setPaused(true);
             this.resetInteractionMode();
         },
+        adaptKeyboardShortcuts() {
+            if(this.enableJumpByFrame) {
+                Keyboard.off('ArrowRight', this.emitNext, 0, this.listenerSet);
+                Keyboard.off('ArrowLeft', this.emitPrevious, 0, this.listenerSet);
+                Keyboard.on('Shift+ArrowRight', this.emitNext, 0, this.listenerSet);
+                Keyboard.on('Shift+ArrowLeft', this.emitPrevious, 0, this.listenerSet);
+                Keyboard.on('ArrowRight', this.emitNextFrame, 0, this.listenerSet);
+                Keyboard.on('ArrowLeft', this.emitPreviousFrame, 0, this.listenerSet);
+            }
+            else {
+                Keyboard.off('Shift+ArrowRight', this.emitNext, 0, this.listenerSet);
+                Keyboard.off('Shift+ArrowLeft', this.emitPrevious, 0, this.listenerSet);
+                Keyboard.off('ArrowRight', this.emitNextFrame, 0, this.listenerSet);
+                Keyboard.off('ArrowLeft', this.emitPreviousFrame, 0, this.listenerSet);
+                Keyboard.on('ArrowRight', this.emitNext, 0, this.listenerSet);
+                Keyboard.on('ArrowLeft', this.emitPrevious, 0, this.listenerSet);
+            }
+        }
     },
     watch: {
         selectedAnnotations(annotations) {
@@ -564,6 +600,9 @@ export default {
         heightOffset() {
             this.updateSize();
         },
+        enableJumpByFrame() {
+            this.adaptKeyboardShortcuts();
+        },
     },
     created() {
         this.$once('map-ready', this.initLayersAndInteractions);
@@ -573,9 +612,8 @@ export default {
         this.map.on('pointermove', this.updateMousePosition);
         this.map.on('moveend', this.emitMoveend);
 
+        this.adaptKeyboardShortcuts();
         Keyboard.on('Escape', this.resetInteractionMode, 0, this.listenerSet);
-        Keyboard.on('ArrowRight', this.emitNext, 0, this.listenerSet);
-        Keyboard.on('ArrowLeft', this.emitPrevious, 0, this.listenerSet);
         Keyboard.on('Control+ArrowRight', this.jumpForward, 0, this.listenerSet);
         Keyboard.on('Control+ArrowLeft', this.jumpBackward, 0, this.listenerSet);
     },

resources/assets/js/videos/components/videoScreen/videoPlayback.vue

@@ -24,6 +24,8 @@ export default {
             // Allow a maximum of 100x magnification. More cannot be represented in the
             // URL parameters.
             minResolution: 0.01,
+            // parameter tracking seeking state specific for frame jump, needed because looking for seeking directly leads to error
+            seekingFrame: this.seeking,
         };
     },
     methods: {
@@ -153,6 +155,79 @@ export default {
         handleSeeked() {
             this.renderVideo(true);
         },
+        // 5 next methods are a workaround to get previous and next frames, adapted from here: https://github.com/angrycoding/requestVideoFrameCallback-prev-next/tree/main
+        async emitPreviousFrame() {
+            if(this.video.currentTime == 0 || this.seekingFrame) return;
+            this.$emit('start-seeking');
+            this.seekingFrame = true;
+            await this.showPreviousFrame();
+            this.seekingFrame = false;
+        },
+        async emitNextFrame() {
+            if(this.video.duration - this.video.currentTime == 0 || this.seekingFrame) return;
+            this.$emit('start-seeking');
+            this.seekingFrame = true;
+            await this.showNextFrame();
+            this.seekingFrame = false;
+        },
+        frameInfoCallback() {
+            let promise = new Vue.Promise((resolve) => {
+                this.video.requestVideoFrameCallback((now, metadata) => {
+                    resolve(metadata);
+                })
+            })
+            return promise;
+        },
+        async showPreviousFrame() {
+            try {
+                // force rerender adapting step on begining or end of video
+                let step = 1;
+                if (this.video.currentTime < 1) {
+                   step = this.video.currentTime;
+                }
+                if (this.video.duration - this.video.currentTime < 1) {
+                   step = this.video.duration - this.video.currentTime;
+                }
+                this.video.currentTime += step;
+                this.video.currentTime -= step;
+
+                // get current frame time
+                const firstMetadata = await this.frameInfoCallback();
+                for (;;) {
+                    // now adjust video's current time until actual frame time changes
+                    this.video.currentTime -= 0.01;
+                    // check that we are not at first frame, otherwise we'll end up in infinte loop
+                    if (this.video.currentTime == 0) break;
+                    const metadata = await this.frameInfoCallback();
+                    if (metadata.mediaTime !== firstMetadata.mediaTime) break;
+                }
+            } catch(e) {console.error(e)}
+        },
+        async showNextFrame() {
+            try {
+                // force rerender adapting step on begining or end of video
+                let step = 1;
+                if (this.video.currentTime < 1) {
+                   step = this.video.currentTime;
+                }
+                if (this.video.duration - this.video.currentTime < 1) {
+                   step = this.video.duration - this.video.currentTime;
+                }
+                this.video.currentTime += step;
+                this.video.currentTime -= step;
+
+                // get current frame time
+                const firstMetadata = await this.frameInfoCallback();
+                for (;;) {
+                    // now adjust video's current time until actual frame time changes
+                    this.video.currentTime += 0.01;
+                    // check that we are not at last frame, otherwise we'll end up in infinte loop
+                    if (this.video.duration - this.video.currentTime == 0) break;
+                    const metadata = await this.frameInfoCallback();
+                    if (metadata.mediaTime !== firstMetadata.mediaTime) break;
+                }
+            } catch(e) {console.error(e)}
+        },
         // Methods to jump back and forward in video. Step is given by parameter jumpStep.
         jumpBackward() {
             if (this.video.currentTime > 0 && this.jumpStep > 0) {

resources/assets/js/videos/stores/settings.js

@@ -7,6 +7,7 @@ let defaults = {
     showLabelTooltip: false,
     showMousePosition: false,
     showProgressIndicator: true,
+    enableJumpByFrame: false,
     jumpStep: 5.0,
     muteVideo: true,
 };

resources/assets/js/videos/videoContainer.vue

@@ -69,6 +69,7 @@ export default {
                 playbackRate: 1.0,
                 jumpStep: 5.0,
                 showProgressIndicator: true,
+                enableJumpByFrame: false,
                 muteVideo: true,
             },
             openTab: '',
@@ -95,6 +96,7 @@ export default {
             attachingLabel: false,
             swappingLabel: false,
             disableJobTracking: false,
+            supportsJumpByFrame: false,
         };
     },
     computed: {
@@ -203,6 +205,9 @@ export default {
 
             return promise;
         },
+        startSeeking() {
+            this.seeking = true;
+        },
         selectAnnotation(annotation, time, shift) {
             if (this.attachingLabel) {
                 this.attachAnnotationLabel(annotation);
@@ -725,6 +730,10 @@ export default {
         if (this.canEdit) {
             this.initializeEcho();
         }
+
+        if ("requestVideoFrameCallback" in HTMLVideoElement.prototype) {
+            this.supportsJumpByFrame = true;
+        }
     },
     mounted() {
         // Wait for the sub-components to register their event listeners before

resources/views/manual/tutorials/videos/shortcuts.blade.php

@@ -14,8 +14,7 @@
                     <th>Key</th>
                     <th>Function</th>
                 </tr>
-            </thead>
-            <tbody>
+            </tbody>
                 <tr>
                     <td><kbd>Arrow left</kbd></td>
                     <td>Previous video</td>
@@ -119,6 +118,37 @@
             </tbody>
         </table>
 
+        <p>
+            <a name="jump-by-frame"></a>When <a href="{{route('manual-tutorials', ['videos', 'sidebar'])}}#jump-by-frame">jump by frame</a> is enabled:
+        </p>
+
+        <table class="table">
+            <thead>
+                <tr>
+                    <th>Key</th>
+                    <th>Function</th>
+                </tr>
+            </thead>
+            <tbody>
+                <tr>
+                    <td><kbd>Arrow left</kbd></td>
+                    <td>Previous frame</td>
+                </tr>
+                <tr>
+                    <td><kbd>Arrow right</kbd></td>
+                    <td>Next frame</td>
+                </tr>
+                <tr>
+                    <td><kbd>Shift</kbd>+<kbd>Arrow left</kbd></td>
+                    <td>Previous video</td>
+                </tr>
+                <tr>
+                    <td><kbd>Shift</kbd>+<kbd>Arrow right</kbd></td>
+                    <td>Next video</td>
+                </tr>
+            </tbody>
+        </table>
+
         <p>
             When any of the rectangle, line string or polygon annotation tools are activated:
         </p>

resources/views/manual/tutorials/videos/sidebar.blade.php

@@ -76,6 +76,10 @@
             The mouse position switch controls the display of an additional map overlay that shows the current position of the cursor on the video in pixels.
         </p>
 
+        <p>
+            <a name="jump-by-frame"></a>The jump by frame switch allows you to navigate frame by frame (forward and backward) in the video. Note that this is an experimental feature as it is only available in Chrome and may not always give the right frame, so please use it with caution. When the feature is enabled, the <button class="btn btn-default btn-xs"><i class="fa fa-caret-square-left"></i></button> and <button class="btn btn-default btn-xs"><i class="fa fa-caret-square-right"></i></button> control buttons will appear in the tool bar at the bottom of the video. Use these to jump to the previous/next frame. Also, the <a href="{{route('manual-tutorials', ['videos', 'shortcuts'])}}#jump-by-frame">keyboard shortcuts</a> are updated.
+        </p>
+
         <p>
             The mute video switch enables or disables the audio track of the video.
         </p>

resources/views/videos/show/content.blade.php

@@ -39,6 +39,7 @@
       :show-label-tooltip="settings.showLabelTooltip"
       :show-minimap="settings.showMinimap"
       :show-mouse-position="settings.showMousePosition"
+      :enable-jump-by-frame="settings.enableJumpByFrame"
       :video="video"
       :height-offset="screenHeightOffset"
       :show-prev-next="hasSiblingVideos"
@@ -62,6 +63,7 @@
       v-on:attaching-active="handleAttachingLabelActive"
       v-on:swapping-active="handleSwappingLabelActive"
       v-on:seek="seek"
+      v-on:start-seeking="startSeeking"
       v-on:is-invalid-shape="handleInvalidShape"
       ></video-screen>
 <video-timeline