feat: camera follow for scene objects (declarative SystemCameraCfg.follow + SimRenderer.set_camera_view)

motrix_env_core/src/motrix_env_core/config/scene/base.py

@@ -191,12 +191,20 @@ def validate(self) -> None:
 
 @configclass
 class SystemCameraCfg:
-    """System camera settings used by interactive viewing and video recording."""
+    """System camera settings used by interactive viewing and video recording.
+
+    ``follow`` names a scene object (a ``BodyCfg``/``RobotCfg`` field name in
+    ``SceneCfg.objs``) whose root link the camera tracks: backends refresh the
+    view's lookat from live state on every rendered frame while
+    ``distance`` / ``elevation`` / ``azimuth`` keep their configured values.
+    ``lookat`` is ignored while following.
+    """
 
     lookat: Vec3 | None = None
     distance: float = 2.0
     elevation: float = -20.0
     azimuth: float = 90.0
+    follow: str | None = None
 
     def validate(self) -> None:
         optional_vec("scene.system_camera.lookat", self.lookat, 3)

motrix_env_core/src/motrix_env_core/config/scene/validation.py

@@ -9,7 +9,7 @@
     SkyboxCfg,
     TextureCfg,
 )
-from motrix_env_core.config.scene.base import SceneAssetCfg, SceneCfg, SceneVisualCfg
+from motrix_env_core.config.scene.base import BodyCfg, SceneAssetCfg, SceneCfg, SceneVisualCfg
 from motrix_env_core.config.scene.geometry import GeomCfg, HFieldTerrainCfg
 
 
@@ -46,11 +46,14 @@ def validate_scene_cfg(scene: SceneCfg) -> None:
                 raise ValueError(f"Material asset {name!r} must reference a TextureCfg, got {asset.texture!r}")
 
     names: set[str] = set()
+    body_names: set[str] = set()
     for name, obj in scene.iter_objs():
         obj.validate(name)
         if name in names:
             raise ValueError(f"SceneCfg object names must be unique, got duplicate {name!r}")
         names.add(name)
+        if isinstance(obj, BodyCfg):
+            body_names.add(name)
 
         if isinstance(obj, GeomCfg) and obj.material is not None:
             material = assets.get(obj.material)
@@ -65,6 +68,13 @@ def validate_scene_cfg(scene: SceneCfg) -> None:
                     f"got {obj.hfield!r}"
                 )
 
+    follow = scene.system_camera.follow
+    if follow is not None and follow not in body_names:
+        raise ValueError(
+            f"scene.system_camera.follow must name a body object in the scene, got {follow!r}; "
+            f"available body objects: {sorted(body_names)}."
+        )
+
     sensor_names: set[str] = set()
     for name, sensor in scene.iter_sensors():
         sensor.validate(name)

motrix_env_core/src/motrix_env_core/sim/backend.py

@@ -80,6 +80,15 @@ class SimRenderer(abc.ABC):
     are no pixels to return.
     """
 
+    def set_camera_view(self, lookat: Sequence[float], distance: float, elevation: float, azimuth: float) -> None:
+        """Update the system camera view at runtime (windowed and headless).
+
+        Pure value method for scripts that drive the camera themselves (custom
+        follow, smoothing, view switching): no camera object crosses the
+        backend boundary. Optional capability.
+        """
+        raise NotImplementedError(f"{type(self).__name__} does not support runtime camera view updates")
+
     @abc.abstractmethod
     def render(self) -> None:
         """Present one frame from the current simulator state (sync + viewer input)."""

motrix_env_motrixsim/src/motrix_env_motrixsim/renderer.py

@@ -34,6 +34,7 @@ def __init__(
         num_envs: int,
         render_spacing: float,
         system_camera: SystemCameraCfg,
+        follow_link: str | None = None,
     ):
         self._data_source = data_source
         self._headless = config.headless
@@ -55,24 +56,64 @@ def __init__(
             render_settings=_render_settings(),
         )
         # The view is fixed at construction: config camera settings override
-        # the scene's system-camera defaults in both modes.
+        # the scene's system-camera defaults in both modes. A follow target
+        # replaces the lookat with the tracked link's live position on every
+        # rendered frame (env row 0 plus its render offset).
+        self._camera_distance = config.camera_distance if config.camera_distance is not None else system_camera.distance
+        self._camera_elevation = (
+            config.camera_elevation if config.camera_elevation is not None else system_camera.elevation
+        )
+        self._camera_azimuth = config.camera_azimuth if config.camera_azimuth is not None else system_camera.azimuth
         _set_system_camera_view(
             self._render,
             offsets,
             config.camera_lookat if config.camera_lookat is not None else system_camera.lookat,
-            config.camera_distance if config.camera_distance is not None else system_camera.distance,
-            config.camera_elevation if config.camera_elevation is not None else system_camera.elevation,
-            config.camera_azimuth if config.camera_azimuth is not None else system_camera.azimuth,
+            self._camera_distance,
+            self._camera_elevation,
+            self._camera_azimuth,
+        )
+        self._follow_offset = [float(v) for v in offsets[0]]
+        self._follow_program = (
+            model.compile_query({"follow": mtx.query.LinkPosition([follow_link])}).allocate(data_source())
+            if follow_link is not None
+            else None
         )
         self._sync_render_data = True
         self._render.system_camera.active = True
 
+    def set_camera_view(self, lookat: Sequence[float], distance: float, elevation: float, azimuth: float) -> None:
+        """Update the system camera view (windowed and headless)."""
+        lookat = np.asarray(lookat, dtype=np.float64).reshape(-1)
+        if lookat.shape != (3,):
+            raise ValueError(f"lookat must contain 3 values, got {lookat!r}")
+        self._render.system_camera.set_view(
+            [float(v) for v in lookat],
+            float(distance),
+            float(elevation),
+            float(azimuth),
+        )
+
+    def _update_follow_view(self, data: mtx.SceneData) -> None:
+        position = np.asarray(self._follow_program.execute(data).values()[0])[0, 0]
+        self._render.system_camera.set_view(
+            [float(position[i]) + self._follow_offset[i] for i in range(3)],
+            self._camera_distance,
+            self._camera_elevation,
+            self._camera_azimuth,
+        )
+
     def render(self) -> None:
         if self._headless:
-            self._render.sync(data=self._data_source())
+            data = self._data_source()
+            if self._follow_program is not None:
+                self._update_follow_view(data)
+            self._render.sync(data=data)
             return
         if self._sync_render_data:
-            self._render.sync(data=self._data_source())
+            data = self._data_source()
+            if self._follow_program is not None:
+                self._update_follow_view(data)
+            self._render.sync(data=data)
         else:
             self._render.sync(data=None)
         if self._render.input.is_key_just_pressed("space"):
@@ -84,6 +125,8 @@ def capture(self) -> np.ndarray:
             raise NotImplementedError(
                 "Windowed renderers set no system render target; pass headless=True to capture frames."
             )
+        if self._follow_program is not None:
+            self._update_follow_view(self._data_source())
         # The capture request rides this frame's sync to the renderer; its map
         # callback only fires on a *later* submit's maintenance (issue #37), so
         # a blocking sync drains the service and guarantees the pixels exist

motrix_env_motrixsim/src/motrix_env_motrixsim/runtime.py

@@ -343,13 +343,17 @@ def create_renderer(
         render_spacing: float,
         system_camera: SystemCameraCfg,
     ) -> SimRenderer:
+        follow_link = None
+        if system_camera.follow is not None:
+            follow_link = self._scene.objs[system_camera.follow].resolved_base_link_name
         return MotrixSimRenderer(
             self._model,
             lambda: self._data,
             config,
             num_envs=num_envs,
             render_spacing=render_spacing,
             system_camera=system_camera,
+            follow_link=follow_link,
         )
 
     def step(self, substeps: int) -> None:

motrix_env_motrixsim/tests/test_renderer.py

@@ -136,3 +136,108 @@ def test_headless_renderer_configures_camera_resolution_and_captures(monkeypatch
     render_app.sync.reset_mock()
     renderer.render()
     assert render_app.sync.call_args.kwargs == {"data": data}
+
+
+def _follow_model(positions: np.ndarray) -> Mock:
+    """Model mock whose one-field link-position query reports ``positions`` per row."""
+    model = Mock()
+    program = Mock()
+    program.execute.return_value = program
+    program.values.return_value = [np.asarray(positions, dtype=np.float32).reshape(len(positions), 1, 3)]
+    plan = Mock()
+    plan.allocate.return_value = program
+    model.compile_query.return_value = plan
+    return model
+
+
+def test_set_camera_view_updates_system_camera(monkeypatch):
+    render_app = MagicMock()
+    monkeypatch.setattr(motrixsim_renderer, "RenderApp", lambda headless=False, fps=None: render_app)
+
+    renderer = motrixsim_renderer.MotrixSimRenderer(
+        object(),
+        lambda: object(),
+        RenderConfig(),
+        num_envs=1,
+        render_spacing=1.0,
+        system_camera=_system_camera(),
+    )
+
+    renderer.set_camera_view((1.0, 2.0, 3.0), 4.0, -15.0, 45.0)
+    render_app.system_camera.set_view.assert_called_with([1.0, 2.0, 3.0], 4.0, -15.0, 45.0)
+
+    with pytest.raises(ValueError, match="lookat must contain 3 values"):
+        renderer.set_camera_view((1.0, 2.0), 4.0, -15.0, 45.0)
+
+
+def test_follow_refreshes_lookat_from_tracked_link_each_frame(monkeypatch):
+    render_app = MagicMock()
+    monkeypatch.setattr(motrixsim_renderer, "RenderApp", lambda headless=False, fps=None: render_app)
+    positions = np.array([[1.0, 2.0, 0.8], [9.0, 9.0, 9.0]])
+    model = _follow_model(positions)
+    data = object()
+
+    renderer = motrixsim_renderer.MotrixSimRenderer(
+        model,
+        lambda: data,
+        RenderConfig(),
+        num_envs=2,
+        render_spacing=2.0,
+        system_camera=SystemCameraCfg(distance=5.0, elevation=-30.0, azimuth=10.0, follow="robot"),
+        follow_link="pelvis",
+    )
+
+    model.compile_query.assert_called_once()
+    render_app.system_camera.set_view.reset_mock()
+    renderer.render()
+    # The camera tracks env row 0 of the followed link; distance/elevation/azimuth
+    # keep their configured values (env 0's render offset is the origin).
+    assert render_app.system_camera.set_view.call_count == 1
+    assert render_app.system_camera.set_view.call_args.args[0] == pytest.approx([1.0, 2.0, 0.8])
+    assert render_app.system_camera.set_view.call_args.args[1:] == (5.0, -30.0, 10.0)
+    render_app.sync.assert_called_once_with(data=data)
+
+
+def test_headless_capture_refreshes_follow_view_before_sync(monkeypatch):
+    render_app = MagicMock()
+    monkeypatch.setattr(motrixsim_renderer, "RenderApp", MagicMock(return_value=render_app))
+    image = MagicMock()
+    image.pixels = np.full((4, 6, 3), 255, dtype=np.uint8)
+    render_app.system_camera.capture.return_value.take_image.return_value = image
+    model = _follow_model(np.array([[0.5, -0.5, 1.0]]))
+    data = object()
+    config = RenderConfig(headless=True, path=Path("/tmp/video.mp4"), fps=20, num_frames=10)
+
+    renderer = motrixsim_renderer.MotrixSimRenderer(
+        model,
+        lambda: data,
+        config,
+        num_envs=1,
+        render_spacing=1.0,
+        system_camera=SystemCameraCfg(follow="robot"),
+        follow_link="pelvis",
+    )
+
+    render_app.system_camera.set_view.reset_mock()
+    frame = renderer.capture()
+    assert frame.shape == (4, 6, 3)
+    render_app.system_camera.set_view.assert_called_once()
+    assert render_app.system_camera.set_view.call_args.args[0] == [0.5, -0.5, 1.0]
+
+
+def test_interactive_renderer_without_follow_keeps_static_view(monkeypatch):
+    render_app = MagicMock()
+    monkeypatch.setattr(motrixsim_renderer, "RenderApp", lambda headless=False, fps=None: render_app)
+
+    renderer = motrixsim_renderer.MotrixSimRenderer(
+        object(),
+        lambda: object(),
+        RenderConfig(),
+        num_envs=1,
+        render_spacing=1.0,
+        system_camera=_system_camera(),
+    )
+
+    render_app.system_camera.set_view.reset_mock()
+    renderer.render()
+    render_app.system_camera.set_view.assert_not_called()

motrix_envs/tests/test_scene_cfg.py

@@ -1209,3 +1209,28 @@ def test_direct_env_loads_model_from_scene_cfg():
     assert model.options.timestep == pytest.approx(0.005)
     assert model.options.max_iterations == 3
     assert model.options.solver_tolerance == pytest.approx(1e-4)
+
+
+def test_system_camera_follow_must_name_a_body_object():
+    @configclass
+    class FollowSceneObjsCfg(SceneObjsCfg):
+        floor: FlatTerrainCfg = FlatTerrainCfg()
+        cartpole: RobotCfg = RobotCfg(
+            model=MjcfFileCfg(file=_CARTPOLE_XML),
+            base_link_name="cart",
+        )
+
+    def scene_with_follow(target: str | None) -> SceneCfg:
+        return SceneCfg(
+            objs=FollowSceneObjsCfg(),
+            system_camera=SystemCameraCfg(follow=target),
+        )
+
+    # A declared body object resolves; validation passes.
+    validate_scene_cfg(scene_with_follow("cartpole"))
+
+    with pytest.raises(ValueError, match="system_camera.follow must name a body object"):
+        validate_scene_cfg(scene_with_follow("missing"))
+    with pytest.raises(ValueError, match="system_camera.follow must name a body object"):
+        # Declared but not a body: terrain objects cannot be followed.
+        validate_scene_cfg(scene_with_follow("floor"))