From 7945469afcf43d048d9c60d979ae44415deb21e0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 3 Aug 2026 12:27:38 +0000 Subject: [PATCH 06/10] feat(ui): render the 3-D scene nodes from interpolated positions (kills jitter) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F1 (real-time FPV milestone), UI half. The Filament scene drew device NODES from the raw ~10 Hz fused snapshot, so they jittered/jumped. Route them through the same entity-interpolated source the aim overlay already uses. - HudViewModel.renderNodePositions(): interpolated world positions from the client-local NetcodeSession.renderPositions(), converted to the scene's self-relative frame (subtract the live self world pose — the same pure translation syntheticTargets uses). Pull-based, main-thread, like aimTracks. - MeshSceneView: each SphereNode captures its own ref; onFrame slews the retained nodes to their interpolated positions imperatively — the same no-recomposition path the camera uses (per-frame recomposition churned Filament native resources). Nodes seed from the interpolated map so a 10 Hz recompose never resets to the raw spot; overlay edges + selection ring track the smooth positions too. No session ⇒ empty map ⇒ identical prior behaviour. - MeshSceneScreen / AppShell: thread the renderPositions provider through. Pairs with the engine half (renderPositions, prior commit). androidApp is not compilable in the headless env — delivered as a reviewed patch. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01WppuiKZt4CuQxX4N7k6SVR --- .../main/java/com/aether/mofe/ui/AppShell.kt | 1 + .../aether/mofe/ui/scene/MeshSceneScreen.kt | 2 + .../com/aether/mofe/ui/scene/MeshSceneView.kt | 66 +++++++++++++++---- .../com/aether/mofe/viewmodel/HudViewModel.kt | 17 +++++ 4 files changed, 74 insertions(+), 12 deletions(-) diff --git a/androidApp/src/main/java/com/aether/mofe/ui/AppShell.kt b/androidApp/src/main/java/com/aether/mofe/ui/AppShell.kt index 97ef13f..abe7316 100644 --- a/androidApp/src/main/java/com/aether/mofe/ui/AppShell.kt +++ b/androidApp/src/main/java/com/aether/mofe/ui/AppShell.kt @@ -220,6 +220,7 @@ fun AppShell(app: AetherApp, hud: HudViewModel) { }, facing = { hud.latestFacing() }, tracks = { cone -> hud.aimTracks(cone) }, + renderPositions = { hud.renderNodePositions() }, modifier = Modifier.fillMaxSize(), ) } diff --git a/androidApp/src/main/java/com/aether/mofe/ui/scene/MeshSceneScreen.kt b/androidApp/src/main/java/com/aether/mofe/ui/scene/MeshSceneScreen.kt index 512a6c9..78745a1 100644 --- a/androidApp/src/main/java/com/aether/mofe/ui/scene/MeshSceneScreen.kt +++ b/androidApp/src/main/java/com/aether/mofe/ui/scene/MeshSceneScreen.kt @@ -88,6 +88,7 @@ fun MeshSceneScreen( onToggleLayer: (SceneLayer, Boolean) -> Unit = { _, _ -> }, facing: () -> Vector3D? = { null }, tracks: (Double) -> List = { emptyList() }, + renderPositions: () -> Map = { emptyMap() }, modifier: Modifier = Modifier, ) { var selectedId by remember { mutableStateOf(null) } @@ -108,6 +109,7 @@ fun MeshSceneScreen( showPredicates = layers.predicates, firstPerson = firstPerson, facing = facing, + renderPositions = renderPositions, recenterSignal = recenterSignal, modifier = Modifier.fillMaxSize(), ) diff --git a/androidApp/src/main/java/com/aether/mofe/ui/scene/MeshSceneView.kt b/androidApp/src/main/java/com/aether/mofe/ui/scene/MeshSceneView.kt index 170e4e2..68733dd 100644 --- a/androidApp/src/main/java/com/aether/mofe/ui/scene/MeshSceneView.kt +++ b/androidApp/src/main/java/com/aether/mofe/ui/scene/MeshSceneView.kt @@ -4,6 +4,7 @@ import androidx.compose.foundation.Canvas import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.key import androidx.compose.runtime.mutableStateOf @@ -26,6 +27,7 @@ import io.github.sceneview.collision.Vector3 import io.github.sceneview.gesture.CameraGestureDetector import io.github.sceneview.math.Position import io.github.sceneview.node.CameraNode +import io.github.sceneview.node.Node import io.github.sceneview.rememberCameraManipulator import io.github.sceneview.rememberCameraNode import io.github.sceneview.rememberEngine @@ -61,6 +63,14 @@ import kotlin.math.sqrt * material instances are native resources, and churning them per frame was crashing the render * thread under sustained pinch/rotate. * + * Smooth positions (F1): the node WORLD positions no longer come from the raw ~10 Hz [snapshot] (which + * jitters/jumps). Each SphereNode captures its own reference in `apply`, and the frame callback slews + * those retained nodes to the entity-interpolated [renderPositions] every frame — the same imperative, + * no-recomposition path the camera uses, so smoothing costs nothing extra on the render thread. The + * [snapshot] still supplies the roster, roles and predicate geometry; only the device POSITIONS are + * interpolated. With no session feeding [renderPositions] the map is empty and the nodes keep their + * raw snapshot positions — identical to the prior behaviour. + * * The snapshot keeps only SELF in `anchors`; every other device — including the constellation * anchors — arrives in `meshPoints`, coloured as an ANCHOR when its id is in [anchorIds]. A peer * that shares an id already drawn in the anchors pass is skipped, so self can never double-draw as @@ -90,6 +100,7 @@ fun MeshSceneView( showPredicates: Boolean = true, firstPerson: Boolean = false, facing: () -> Vector3D? = { null }, + renderPositions: () -> Map = { emptyMap() }, recenterSignal: Int = 0, modifier: Modifier = Modifier, ) { @@ -153,9 +164,23 @@ fun MeshSceneView( // scene. Empty in first-person. val overlay = remember { mutableStateOf(SceneOverlay()) } - // One combined frame callback. First-person seats the eye at self and aims down the lens vector; - // orbit projects the edges + floor grid to screen space for the 2D overlay (zero scene geometry). + // Retained scene-node references, keyed by device id, captured as each SphereNode is created (see + // the `apply` blocks below). The frame callback pushes the interpolated [renderPositions] onto + // them IMPERATIVELY — exactly like the camera is driven — so the nodes track the smooth mesh at + // frame rate WITHOUT recomposing the scene per frame (which churned native resources; see the + // class note). Absent a session the map stays unused and nodes keep their raw-snapshot positions. + val liveNodes = remember { mutableMapOf() } + + // One combined frame callback. First it slews the retained nodes to their entity-interpolated + // positions (F1: kills the raw ~10 Hz jitter). First-person then seats the eye at self and aims + // down the lens vector; orbit projects the edges + floor grid to screen space for the 2D overlay. val onFrame: (Long) -> Unit = { _ -> + // Interpolated, self-relative device positions for THIS frame, shared by the nodes and the + // overlay so edges/selection stay glued to where the smooth nodes actually are. + val live = renderPositions() + if (live.isNotEmpty()) { + liveNodes.forEach { (id, node) -> live[id]?.let { node.position = it.toFilament() } } + } if (firstPerson) { val f = facing() if (f != null) { @@ -165,14 +190,14 @@ fun MeshSceneView( // FPV honors grid + predicates, but node-to-node EDGES are forced off (they'd clutter // the lens). The front-cull needs the lens direction as "forward" — the orbit heuristic // (−camera) is degenerate here because the eye sits at the origin. - overlay.value = buildOverlay(cameraNode, snapshot, anchorIds, showGrid, + overlay.value = buildOverlay(cameraNode, snapshot, live, anchorIds, showGrid, showEdges = false, showPredicates, forward = lens, dropGridBelowEye = true, selectedId = selectedId) } else { overlay.value = SceneOverlay() } } else { - overlay.value = buildOverlay(cameraNode, snapshot, anchorIds, showGrid, showEdges, showPredicates, + overlay.value = buildOverlay(cameraNode, snapshot, live, anchorIds, showGrid, showEdges, showPredicates, selectedId = selectedId) } } @@ -193,6 +218,10 @@ fun MeshSceneView( ) { // NODES layer — the constellation spheres (self + peers), gated by the Layers control. if (showNodes) { + // Interpolated positions to SEED each node at composition (~10 Hz), so a recomposition never + // resets it to the raw jittery spot between the 60 Hz onFrame slews. Empty ⇒ raw snapshot, + // i.e. exactly the prior behaviour when no session feeds [renderPositions]. + val seed = renderPositions() // SELF is the only entry in `anchors`; render it in the self colour (pink) — but never in // first-person, where the eye sits inside self and the sphere would fill the lens. snapshot.anchors.forEach { anchor -> @@ -204,13 +233,18 @@ fun MeshSceneView( // Constant radius — selection is shown as a 2D overlay ring instead. Resizing // the sphere on select/deselect rebuilt its geometry and flashed it flat. radius = 0.13f, - position = anchor.spatial.position.toFilament(), + position = (seed[anchor.id] ?: anchor.spatial.position).toFilament(), materialInstance = material, apply = { name = anchor.id + // Retain the ref so onFrame can slew it to the interpolated position. + liveNodes[anchor.id] = this onSingleTapConfirmed = { onNodeSelected(anchor.id); true } }, ) + // Drop the retained ref when this node leaves composition (peer left, or self on + // the FPV switch) so onFrame never slews a destroyed Filament node. + DisposableEffect(anchor.id) { onDispose { liveNodes.remove(anchor.id) } } } } } @@ -225,13 +259,15 @@ fun MeshSceneView( val base = if (role == NodeRole.Anchor) 0.12f else 0.09f SphereNode( radius = base, - position = point.spatial.position.toFilament(), + position = (seed[point.id] ?: point.spatial.position).toFilament(), materialInstance = material, apply = { name = point.id + liveNodes[point.id] = this onSingleTapConfirmed = { onNodeSelected(point.id); true } }, ) + DisposableEffect(point.id) { onDispose { liveNodes.remove(point.id) } } } } } @@ -297,6 +333,7 @@ private class SceneOverlay( private fun buildOverlay( cameraNode: CameraNode, snapshot: MeshSnapshot, + live: Map, anchorIds: Set, showGrid: Boolean, showEdges: Boolean, @@ -323,6 +360,9 @@ private fun buildOverlay( val s = cameraNode.worldToScreenPoint(Vector3(p.x, p.y, p.z)) return Offset(s.x / vw, s.y / vh) } + // A device's Filament position, preferring its interpolated spot so the edges + selection ring + // stay glued to where the smooth nodes actually are; falls back to the raw snapshot position. + fun devPos(id: String, fallback: Vector3D): Position = (live[id] ?: fallback).toFilament() // Grid (toggleable): a square that reaches just past the farthest node, on the y = 0 plane. val grid = ArrayList>() @@ -354,15 +394,15 @@ private fun buildOverlay( } } - // Edges (toggleable): self → each anchor. + // Edges (toggleable): self → each anchor. Both endpoints prefer the interpolated node position. val edges = ArrayList>() if (showEdges) { - val selfPos = snapshot.anchors.firstOrNull { it.isLocalDevice }?.spatial?.position?.toFilament() - val selfScreen = selfPos?.let { project(it) } + val self = snapshot.anchors.firstOrNull { it.isLocalDevice } + val selfScreen = self?.let { project(devPos(it.id, it.spatial.position)) } if (selfScreen != null) { snapshot.meshPoints.forEach { point -> if (point.id in anchorIds && (point.positionSolved || point.bearingKnown)) { - project(point.spatial.position.toFilament())?.let { edges.add(selfScreen to it) } + project(devPos(point.id, point.spatial.position))?.let { edges.add(selfScreen to it) } } } } @@ -410,9 +450,11 @@ private fun buildOverlay( } } // Selected node → a screen-space ring drawn in the Canvas (instead of resizing the sphere). + // Prefer the interpolated position so the ring tracks the smooth node, not the raw snapshot spot. val selPos = selectedId?.let { id -> - (snapshot.anchors.firstOrNull { it.id == id }?.spatial?.position - ?: snapshot.meshPoints.firstOrNull { it.id == id }?.spatial?.position)?.toFilament() + val fallback = snapshot.anchors.firstOrNull { it.id == id }?.spatial?.position + ?: snapshot.meshPoints.firstOrNull { it.id == id }?.spatial?.position + fallback?.let { devPos(id, it) } } SceneOverlay(grid, edges, predicates, selPos?.let { project(it) }) }.getOrDefault(SceneOverlay()) diff --git a/androidApp/src/main/java/com/aether/mofe/viewmodel/HudViewModel.kt b/androidApp/src/main/java/com/aether/mofe/viewmodel/HudViewModel.kt index 66d0b27..17c72a8 100644 --- a/androidApp/src/main/java/com/aether/mofe/viewmodel/HudViewModel.kt +++ b/androidApp/src/main/java/com/aether/mofe/viewmodel/HudViewModel.kt @@ -160,6 +160,23 @@ class HudViewModel(application: Application) : AndroidViewModel(application) { return aim.tracks(observer, facing, up, Timestamp(latestMeshMicros), coneRadians) } + /** + * Entity-interpolated, self-relative render POSITIONS for the 3-D scene NODES — the smooth + * source that replaces the raw ~10 Hz snapshot positions the scene draws today (F1). Same + * client-local [netcode] session and interp delay as [aimTracks], so the nodes and the aim + * markers agree. [NetcodeSession.renderPositions] are WORLD-frame; the scene is self-relative + * (self at the snapshot origin), so subtract the live self world position — the identical pure + * translation [syntheticTargets] uses (no rotation: the snapshot frame is world-translated, not + * world-rotated). Pull-based like [latestFacing]/[aimTracks] and read on the same (main) thread, + * so it shares their session access safely; empty until a self pose exists. Keyed by device-id + * string, the identity the scene nodes carry. + */ + fun renderNodePositions(): Map { + val selfWorld = app.mofeEngineHost.latestSelfPose()?.position ?: return emptyMap() + return netcode.renderPositions(Timestamp(latestMeshMicros)) + .entries.associate { (id, p) -> id.value to (p - selfWorld) } + } + /** * Synthetic target points to render in the 3D viewport, expressed relative to self * (the snapshot frame, self at origin) so the viewport can draw a device→target aim -- 2.43.0