From 59f227c8bd4dd0b460c5574e1d78025aefccae5c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 2 Aug 2026 17:40:05 +0000 Subject: [PATCH] =?UTF-8?q?feat(ui):=20wire=20MeshAimTracker=20into=20the?= =?UTF-8?q?=20FPV=20scene=20=E2=80=94=20per-object=20aim=20markers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the aim-tracker vertical across the app's layers. The engine tracker (MeshAimTracker) + its pan-around validation already landed on the branch; this feeds it live and renders it: - HudViewModel: a client-local NetcodeSession fed from mofeEngineHost.fusedStates (the engine's own fused world poses — no peer transport, which is the separate mesh-netcode go-live), plus MeshAimTracker over it and a pull-based aimTracks() mirroring latestFacing(). Observer position is the world-frame self pose, so a track's (0,0) reticle offset coincides with the crosshair by construction. - MeshSceneScreen: an AimMarkersOverlay drawn in FPV — each Track's reticleOffset (right,up) mapped to the screen with the same centre+focal convention as the FPV projector (no camera projection needed: reticleOffset IS the projector's normalized coord). onReticle -> filled/locked, else a tracking ring. Repaints on the frame clock. Gated by a new SceneLayer.AimTracks (opt-in), persisted in AppSettingsStore and threaded through AppShell like the other layers. The engine/common layer this builds on is already build-verified (:common:jvmTest green). This UI wiring is written against the existing scene/host patterns but was NOT compiled here (the Android Gradle plugin is not resolvable in this environment), so it is intended for the paused UI/UX agent to compile and visually tune on re-sync (see the off-centre-focal note in AimMarkersOverlay). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01WppuiKZt4CuQxX4N7k6SVR --- .../aether/mofe/platform/AppSettingsStore.kt | 8 +++ .../main/java/com/aether/mofe/ui/AppShell.kt | 4 ++ .../aether/mofe/ui/scene/MeshSceneScreen.kt | 60 ++++++++++++++++++- .../com/aether/mofe/viewmodel/HudViewModel.kt | 41 +++++++++++++ 4 files changed, 112 insertions(+), 1 deletion(-) diff --git a/androidApp/src/main/java/com/aether/mofe/platform/AppSettingsStore.kt b/androidApp/src/main/java/com/aether/mofe/platform/AppSettingsStore.kt index 06d2b8d..5931084 100644 --- a/androidApp/src/main/java/com/aether/mofe/platform/AppSettingsStore.kt +++ b/androidApp/src/main/java/com/aether/mofe/platform/AppSettingsStore.kt @@ -94,6 +94,14 @@ class AppSettingsStore(context: Context) { prefs.edit { putBoolean("scene_show_legend", on) } } + // First-person per-object aim markers (OFF by default — opt-in overlay, FPV only). + private val _showAimTracks = MutableStateFlow(prefs.getBoolean("scene_show_aim_tracks", false)) + val showAimTracks: StateFlow = _showAimTracks.asStateFlow() + fun setShowAimTracks(on: Boolean) { + _showAimTracks.value = on + prefs.edit { putBoolean("scene_show_aim_tracks", on) } + } + // ── Startup behavior (persisted; survive app restart / update push) ────────── // Volatile session state — the selected mesh and whether it was uplinking — is // restored on next launch per these toggles, so the user doesn't re-select and diff --git a/androidApp/src/main/java/com/aether/mofe/ui/AppShell.kt b/androidApp/src/main/java/com/aether/mofe/ui/AppShell.kt index 8e868b0..0ef43a8 100644 --- a/androidApp/src/main/java/com/aether/mofe/ui/AppShell.kt +++ b/androidApp/src/main/java/com/aether/mofe/ui/AppShell.kt @@ -101,6 +101,7 @@ fun AppShell(app: AetherApp, hud: HudViewModel) { val showNodes by app.appSettingsStore.showNodes.collectAsState() val showPredicates by app.appSettingsStore.showPredicates.collectAsState() val showLegend by app.appSettingsStore.showLegend.collectAsState() + val showAimTracks by app.appSettingsStore.showAimTracks.collectAsState() val view = LocalView.current DisposableEffect(keepScreenOn) { view.keepScreenOn = keepScreenOn @@ -198,6 +199,7 @@ fun AppShell(app: AetherApp, hud: HudViewModel) { nodes = showNodes, predicates = showPredicates, legend = showLegend, + aimTracks = showAimTracks, ), onToggleLayer = { layer, on -> when (layer) { @@ -206,9 +208,11 @@ fun AppShell(app: AetherApp, hud: HudViewModel) { SceneLayer.Nodes -> app.appSettingsStore.setShowNodes(on) SceneLayer.Predicates -> app.appSettingsStore.setShowPredicates(on) SceneLayer.Legend -> app.appSettingsStore.setShowLegend(on) + SceneLayer.AimTracks -> app.appSettingsStore.setShowAimTracks(on) } }, facing = { hud.latestFacing() }, + tracks = { cone -> hud.aimTracks(cone) }, modifier = Modifier.fillMaxSize(), ) } diff --git a/androidApp/src/main/java/com/aether/mofe/ui/scene/MeshSceneScreen.kt b/androidApp/src/main/java/com/aether/mofe/ui/scene/MeshSceneScreen.kt index 919b21a..0da98ab 100644 --- a/androidApp/src/main/java/com/aether/mofe/ui/scene/MeshSceneScreen.kt +++ b/androidApp/src/main/java/com/aether/mofe/ui/scene/MeshSceneScreen.kt @@ -22,10 +22,12 @@ import androidx.compose.material3.Icon import androidx.compose.material3.Switch import androidx.compose.material3.Text import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.setValue +import androidx.compose.runtime.withFrameNanos import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip @@ -36,6 +38,7 @@ import androidx.compose.ui.text.font.FontFamily import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp +import com.aether.mofe.engine.netcode.MeshAimTracker import com.aether.mofe.model.MeshOperatingMode import com.aether.mofe.model.Vector3D import com.aether.mofe.model.mesh.MeshSnapshot @@ -51,7 +54,7 @@ import com.aether.mofe.ui.theme.AetherColors import com.aether.mofe.ui.theme.AetherShapes /** The toggleable scene layers, surfaced in the in-scene Layers control (bottom-right). */ -enum class SceneLayer { Grid, Connections, Nodes, Predicates, Legend } +enum class SceneLayer { Grid, Connections, Nodes, Predicates, Legend, AimTracks } /** On/off state per [SceneLayer]; defaults mirror AppSettingsStore's scene prefs. */ data class SceneLayers( @@ -60,6 +63,7 @@ data class SceneLayers( val nodes: Boolean = true, val predicates: Boolean = true, val legend: Boolean = true, + val aimTracks: Boolean = false, ) /** @@ -84,6 +88,7 @@ fun MeshSceneScreen( layers: SceneLayers = SceneLayers(), onToggleLayer: (SceneLayer, Boolean) -> Unit = { _, _ -> }, facing: () -> Vector3D? = { null }, + tracks: (Double) -> List = { emptyList() }, modifier: Modifier = Modifier, ) { var selectedId by remember { mutableStateOf(null) } @@ -108,6 +113,17 @@ fun MeshSceneScreen( modifier = Modifier.fillMaxSize(), ) + // First-person per-object aim markers (opt-in layer): every mesh object placed on the + // reticle from its interpolated pose, so the aim reads at a glance. Drawn under the reticle + // so the crosshair stays crisp; (0,0) == on-aim, coinciding with it by construction. + if (firstPerson && layers.aimTracks) { + AimMarkersOverlay( + tracks = tracks, + coneRadians = 3.0 * kotlin.math.PI / 180.0, + modifier = Modifier.fillMaxSize(), + ) + } + // First-person aim reticle — a small centred crosshair, so the phone's pointing direction // reads at a glance. Kept tiny so it never blocks scene gestures elsewhere. if (firstPerson) { @@ -280,6 +296,7 @@ private fun LayersPanel(layers: SceneLayers, onToggle: (SceneLayer, Boolean) -> LayerRow(Icons.Filled.ScatterPlot, "Nodes", layers.nodes) { onToggle(SceneLayer.Nodes, it) } LayerRow(Icons.Filled.Category, "Predicates", layers.predicates) { onToggle(SceneLayer.Predicates, it) } LayerRow(Icons.Filled.Label, "Legend", layers.legend) { onToggle(SceneLayer.Legend, it) } + LayerRow(Icons.Filled.CenterFocusStrong, "Aim tracks", layers.aimTracks) { onToggle(SceneLayer.AimTracks, it) } } } @@ -316,3 +333,44 @@ private fun FpvReticle(modifier: Modifier = Modifier) { drawCircle(color = AetherColors.Accent, radius = w, center = Offset(cx, cy)) } } + +/** + * First-person per-object aim markers. Each [MeshAimTracker.Track] carries a reticleOffset + * (right, up) that is (0,0) — the crosshair — exactly when the phone is aimed at that object, and + * grows off-axis. Map it to the screen with the same centre + focal convention as the FPV + * projector, so a marker slides onto [FpvReticle] precisely when its object is on-aim. onReticle + * objects render filled (locked); the rest as a ring (tracking). Repaints on the frame clock so + * markers follow the ~60 Hz hand motion, not the ~10 Hz solve. + * + * NOTE (UI): focal = min(w,h)*0.9 matches the hand-rolled FPV projector (Mesh3DCanvas). For + * pixel-exact OFF-centre alignment with the Filament FPV camera, set focal = (h/2)/tan(vFov/2) + * from that camera; the on-aim (0,0) placement is exact regardless. + */ +@Composable +private fun AimMarkersOverlay( + tracks: (Double) -> List, + coneRadians: Double, + modifier: Modifier = Modifier, +) { + val frame = remember { mutableStateOf(0L) } + LaunchedEffect(Unit) { while (true) { withFrameNanos { frame.value = it } } } + Canvas(modifier) { + frame.value // frame-clock read → redraw each frame + val cx = size.width / 2f + val cy = size.height / 2f + val focal = kotlin.math.min(size.width, size.height) * 0.9f + for (t in tracks(coneRadians)) { + val off = t.reticleOffset ?: continue // behind the horizon — nothing to place + val x = cx + off.first.toFloat() * focal + val y = cy - off.second.toFloat() * focal // screen-y is down + if (t.onReticle) { + drawCircle(AetherColors.Accent, radius = 7.dp.toPx(), center = Offset(x, y)) + } else { + drawCircle( + AetherColors.Accent.copy(alpha = 0.55f), radius = 5.dp.toPx(), + center = Offset(x, y), style = Stroke(width = 2.dp.toPx()), + ) + } + } + } +} diff --git a/androidApp/src/main/java/com/aether/mofe/viewmodel/HudViewModel.kt b/androidApp/src/main/java/com/aether/mofe/viewmodel/HudViewModel.kt index ef1a38a..66d0b27 100644 --- a/androidApp/src/main/java/com/aether/mofe/viewmodel/HudViewModel.kt +++ b/androidApp/src/main/java/com/aether/mofe/viewmodel/HudViewModel.kt @@ -5,6 +5,11 @@ import androidx.lifecycle.AndroidViewModel import androidx.lifecycle.viewModelScope import com.aether.mofe.AetherApp import com.aether.mofe.data.* +import com.aether.mofe.engine.netcode.MeshAimTracker +import com.aether.mofe.engine.netcode.NetcodeSession +import com.aether.mofe.engine.netcode.toTemporalPose +import com.aether.mofe.model.DeviceId +import com.aether.mofe.model.Timestamp import com.aether.mofe.model.Vector3D import com.aether.mofe.model.mesh.ImuSample import com.aether.mofe.model.mesh.MeshPredicate @@ -55,6 +60,16 @@ class HudViewModel(application: Application) : AndroidViewModel(application) { private val _mesh = MutableStateFlow(MeshSnapshot()) val mesh: StateFlow = _mesh.asStateFlow() + // ── Client-local aim tracker ────────────────────────────────────────────── + // A per-client NetcodeSession fed from the engine's own fused world poses (no peer transport — + // that is the separate mesh-netcode go-live). MeshAimTracker then places every mesh object on + // the reticle from entity-interpolated poses, so the aim reads self-consistent regardless of + // absolute-heading accuracy. Local single-observer ⇒ the mesh clock is identity, so each + // FusedState's own timestamp is its mesh time. + private val netcode = NetcodeSession(baseInterpDelayMicros = 60_000L) + private val aim = MeshAimTracker(netcode) + @Volatile private var latestMeshMicros: Long = 0L + init { viewModelScope.launch { combine(repo.snapshot, repo.sharedState) { snapshot, shared -> @@ -67,6 +82,16 @@ class HudViewModel(application: Application) : AndroidViewModel(application) { _recentEvents.update { (listOf(PredicateEventLog(e.timestampMs, e.message)) + it).take(30) } } } + // Feed the aim tracker from the engine's fused world poses (~10 Hz solve). + viewModelScope.launch { + app.mofeEngineHost.fusedStates.collect { states -> + for ((id, state) in states) { + netcode.recordSample(DeviceId(id), state.timestamp, state.toTemporalPose()) + val us = state.timestamp.microseconds + if (us > latestMeshMicros) latestMeshMicros = us + } + } + } startImuUpdates() } @@ -119,6 +144,22 @@ class HudViewModel(application: Application) : AndroidViewModel(application) { return if (m > 1e-6f) Vector3D(fx / m, fy / m, fz / m) else Vector3D(0.0, 1.0, 0.0) } + /** + * Per-object aim tracks for the first-person reticle: every mesh object placed on the + * observer's facing frame from its entity-interpolated pose ([MeshAimTracker]). Pull-based, + * mirroring [latestFacing]; empty until a self pose + facing exist. Observer position is the + * engine's WORLD-frame self pose (the same frame as the recorded fused poses), so a track's + * (0,0) reticle offset coincides with the crosshair by construction. [coneRadians] is the + * on-reticle "locked" half-angle. Queried at the freshest fused mesh time, so renderPose + * interpolates one interp-delay behind the latest solve. + */ + fun aimTracks(coneRadians: Double): List { + val observer = app.mofeEngineHost.latestSelfPose()?.position ?: return emptyList() + val facing = latestFacing() ?: return emptyList() + val up = latestTopEdge() ?: Vector3D(0.0, 1.0, 0.0) + return aim.tracks(observer, facing, up, Timestamp(latestMeshMicros), coneRadians) + } + /** * Synthetic target points to render in the 3D viewport, expressed relative to self * (the snapshot frame, self at origin) so the viewport can draw a device→target aim -- 2.43.0