Read hover and click values at the frame on screen

cortex/tests/test_webgl_headless.py

@@ -313,6 +313,39 @@ def test_overlay_visibility_changes_image(tmp_path):
 # ---------------------------------------------------------------------------
 
 
+@pytest.mark.parametrize("dtype_name", ["Vertex", "Volume"])
+def test_picked_value_follows_movie_frame(dtype_name):
+    """The click readout shows the value at the frame on screen.
+
+    Regression test for #636: the readout always indexed frame 0, so after
+    advancing a movie it kept showing the first frame's values.
+    """
+    from cortex.webgl.serve import JSProxy
+
+    nframes = 3
+    frames = np.arange(nframes, dtype=np.float32)
+    if dtype_name == "Vertex":
+        data = np.repeat(frames[:, None], nverts, axis=1)
+        view = cortex.Vertex(data, subj, vmin=0, vmax=nframes - 1)
+    else:
+        data = np.broadcast_to(frames[:, None, None, None], (nframes, *volshape))
+        view = cortex.Volume(data.copy(), subj, xfmname, vmin=0, vmax=nframes - 1)
+
+    with cortex.export.headless_viewer(view, viewer_params={}) as handle:
+        readout = JSProxy(handle.send, "window.picked_value")
+        # Every vertex or voxel in frame k holds k, so any point on the brain works.
+        for frame in [0, 2, 1]:
+            handle.setFrame(frame)
+            time.sleep(0.5)
+            handle.pick({"x": 640, "y": 360})
+            time.sleep(0.5)
+            text = readout.attrs["textContent"][1]
+            assert text, "the pick at the canvas center missed the brain"
+            # Volume floats are decoded through a 2D canvas, whose premultiplied
+            # alpha rounds their low bytes: 1.0 reads back as 1.015625.
+            assert float(text) == pytest.approx(frame, abs=0.05), (frame, text)
+
+
 def test_vertex_no_nan_renders_data(tmp_path):
     """A NaN-free Vertex must render visibly, not fall through to transparent.
 

cortex/webgl/resources/js/dataset.js

@@ -290,6 +290,12 @@ var dataset = (function(module) {
             this.alphaData.setFilter(this.filter);
         this.setFrame(0);
     };
+    // Index of the frame on screen, as setFrame computes it. The shaders blend
+    // toward the next frame by framemix, but the NaN mask comes from this one,
+    // so value readouts report this frame's sample.
+    module.DataView.prototype.frameIndex = function() {
+        return Math.floor(((this.frame + this.delay) * this.rate).mod(this.frames));
+    };
     module.DataView.prototype.setFrame = function(time) {
         this.frame = time;
         var frame = ((time + this.delay) * this.rate).mod(this.frames);

cortex/webgl/resources/js/mriview.js

@@ -48,13 +48,15 @@ var mriview = (function(module) {
         var subject = dataview.data[0].subject;
         var indexMap = subjects[subject].hemis[coords.hemi].indexMap;
         var vertex = indexMap[coords.vertex];
-        return dataview.data.map(function (d) {
+        // One value per channel (1 for 1D, 2 for 2D), at the frame on screen
+        return module.frameValues(dataview, function (d, frame) {
+            if (d.verts[frame] === undefined) return undefined;
             // NaN was replaced by 0 in the GPU buffer (dataset.js); report
             // NaN from the mask instead of a fake 0.
             if (d.nanmasks !== undefined && d.nanmasks.length > 0 &&
-                d.nanmasks[0][hemiIdx].array[vertex] < 0.5)
+                d.nanmasks[frame.mod(d.nanmasks.length)][hemiIdx].array[vertex] < 0.5)
                 return NaN;
-            return d.verts[0][hemiIdx].array[vertex];
+            return d.verts[frame][hemiIdx].array[vertex];
         });
     };
 
@@ -78,6 +80,20 @@ var mriview = (function(module) {
         return true;
     };
 
+    // Values at the frame the viewer is showing, one per dim, read from each
+    // data object by `read(data, frame)`. Returns null while that frame's
+    // buffer is still loading: movie mosaics arrive one frame at a time.
+    module.frameValues = function (dataview, read) {
+        var frame = dataview.frameIndex();
+        var values = [];
+        for (var i = 0; i < dataview.data.length; i++) {
+            var v = read(dataview.data[i], frame);
+            if (v === undefined) return null;
+            values.push(v);
+        }
+        return values;
+    };
+
     module.Viewer = function(figure) {
         jsplot.Axes.call(this, figure);
 
@@ -841,8 +857,8 @@ var mriview = (function(module) {
                     }
                     let mouse_index = this.getMouseIndex(event)
                     if (mouse_index !== -1) {
-                        values = this.active.data.map(function (d) {
-                            return d.textures[0].image.data[mouse_index]
+                        values = module.frameValues(this.active, function (d, frame) {
+                            return d.textures[frame] && d.textures[frame].image.data[mouse_index]
                         })
                     }
                 }
@@ -1167,8 +1183,8 @@ var mriview = (function(module) {
                 }
                 let mouse_index = this.xyxToI(coords.voxel.x, coords.voxel.y, coords.voxel.z)
                 if (mouse_index !== -1) {
-                    values = this.active.data.map(function (d) {
-                        return d.textures[0].image.data[mouse_index]
+                    values = module.frameValues(this.active, function (d, frame) {
+                        return d.textures[frame] && d.textures[frame].image.data[mouse_index]
                     })
                 }
             }