From 50e891ecdb8609066a01566694bbed3ece2546a3 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 23:53:08 +0000 Subject: [PATCH] =?UTF-8?q?feat(android):=20black-box=20telemetry=20screen?= =?UTF-8?q?=20overlay=20(#72)=20=E2=80=94=20raw=20IMU=20+=20sync=20for=20s?= =?UTF-8?q?creen-record=20capture?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Debug-gated full-screen overlay that turns the phone screen into a machine-readable data channel, so a plain screen recording of a field walk carries what the on-device MofePipelineRecorder NDJSON does NOT: the RAW IMU buffer, plus a per-frame tEngineMicros sync key (the same engine clock the recorder stamps) + monotonic frameIdx for offline alignment, and the fused self pose as a cheap cross-check. - MofeEngineHost: hoist the engine AndroidPlatformClock to a field and expose engineNowMicros() so the overlay reads the SAME timeline the recorder stamps (MofePipelineRecorder t = clock.now()). - HudViewModel: tee the existing imuFlow collector into a quantized BbImu ring buffer (second SensorManager listener; the engine IMU path is untouched); nextBlackBoxLuma() drains it per frame, builds a BlackBoxState (ranges empty — NDJSON is authoritative there), and encodes to an 80x130 @repeat=5 luma grid. IMU block is capped to the frame's byte budget so encodeToLuma never overflows. - BlackBoxOverlay: full-screen Canvas that draws the 0/255 grid inside a thin quiet border with 3 solid corner fiducials (TL/TR/BL), redrawn per display frame via withFrameNanos (mirrors Mesh3DCanvas). - Gating mirrors the recorder exactly: the same mofe-trace.on sentinel in AetherApp arms BOTH; overlay injected in AppShell's root Box. Default OFF. Reuses committed :common BlackBoxCodec/BlackBoxSchema — no :common change. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01WppuiKZt4CuQxX4N7k6SVR --- .../main/java/com/aether/mofe/AetherApp.kt | 12 ++ .../aether/mofe/platform/MofeEngineHost.kt | 15 ++- .../main/java/com/aether/mofe/ui/AppShell.kt | 9 ++ .../com/aether/mofe/ui/BlackBoxOverlay.kt | 104 ++++++++++++++++++ .../com/aether/mofe/viewmodel/HudViewModel.kt | 101 ++++++++++++++++- 5 files changed, 239 insertions(+), 2 deletions(-) create mode 100644 androidApp/src/main/java/com/aether/mofe/ui/BlackBoxOverlay.kt diff --git a/androidApp/src/main/java/com/aether/mofe/AetherApp.kt b/androidApp/src/main/java/com/aether/mofe/AetherApp.kt index c69e8f0..5fa2845 100644 --- a/androidApp/src/main/java/com/aether/mofe/AetherApp.kt +++ b/androidApp/src/main/java/com/aether/mofe/AetherApp.kt @@ -53,6 +53,13 @@ class AetherApp : Application() { lateinit var mofeEngineHost: MofeEngineHost private set + /** Debug black-box telemetry overlay opt-in (#72). Set true ONLY when the `mofe-trace.on` + * sentinel is present — the SAME gate that starts the NDJSON pipeline recorder — so a single + * opt-in turns BOTH on together for a screen-record capture. Read by the UI (AppShell) to draw + * [com.aether.mofe.ui.BlackBoxOverlay]. Default OFF (absent in normal installs). */ + @Volatile var blackBoxOverlayEnabled: Boolean = false + private set + /** Offline → server fold for admin role decisions. Inject into * MemberRolesViewModel so an approve/deny made while disconnected from the * server is signed + queued for the leader uplink to carry on reconnect. */ @@ -139,6 +146,11 @@ class AetherApp : Application() { val dir = getExternalFilesDir("mofe-trace") ?: filesDir val path = mofeEngineHost.startPipelineRecording(dir, selfId, android.os.Build.MODEL ?: "device") android.util.Log.i("AetherApp", "MOFE pipeline recording ENABLED → $path") + // The SAME sentinel also arms the on-screen black-box telemetry overlay (#72), so a + // screen recording of this capture carries the raw IMU buffer + engine-clock sync key + // the NDJSON trace does NOT log — decodable offline and aligned to the trace by tEngineMicros. + blackBoxOverlayEnabled = true + android.util.Log.i("AetherApp", "MOFE black-box overlay ENABLED (screen-record capture)") } } diff --git a/androidApp/src/main/java/com/aether/mofe/platform/MofeEngineHost.kt b/androidApp/src/main/java/com/aether/mofe/platform/MofeEngineHost.kt index 952db4e..c9663be 100644 --- a/androidApp/src/main/java/com/aether/mofe/platform/MofeEngineHost.kt +++ b/androidApp/src/main/java/com/aether/mofe/platform/MofeEngineHost.kt @@ -74,6 +74,13 @@ class MofeEngineHost( ) private val mutex = Mutex() + // The engine's monotonic clock. HOISTED to a field (was inline in the builder below) so the + // black-box diagnostic overlay can read the SAME timeline the engine stamps into the pipeline + // recorder — MofePipelineRecorder's `t` values are `clock.now().microseconds` (see + // MultiObserverFusionEngine.processRanging). Embedding that shared key per overlay frame is + // what lets an offline decoder align a screen recording to the on-device NDJSON trace (#72). + private val engineClock = AndroidPlatformClock() + // SINGLE-THREADED engine dispatcher: MofeRuntime feeds the engine from several // collectors (local UWB ranging, peer-anchor observations, IMU, maintenance). The // engine is NOT concurrency-safe (its ObservationCollector mutates a LinkedHashMap @@ -222,7 +229,7 @@ class MofeEngineHost( val mgr = frameManager ?: CoordinateFrameManager(MultilaterationSolver()).also { frameManager = it } val eng = engine ?: MofeBuilder() .frameManager(mgr) - .clock(AndroidPlatformClock()) + .clock(engineClock) .telemetryEmitter(HubForwardingTelemetryEmitter()) // Real-device retention window: intermittent per-link UWB means the 4 // anchors rarely all range a target within 500 ms (the sim default), so @@ -303,6 +310,12 @@ class MofeEngineHost( @Volatile private var _selfPose: FusedState? = null fun latestSelfPose(): FusedState? = _selfPose + /** Engine clock "now" in microseconds — the SAME monotonic timeline MofePipelineRecorder + * stamps its `t` values with (both come from [engineClock].now()). The black-box overlay + * embeds this as each frame's `tEngineMicros` sync key, so an offline decoder can align a + * screen recording to the NDJSON trace. Read lock-free (the clock is monotonic + stateless). */ + fun engineNowMicros(): Long = engineClock.now().microseconds + // ── F2: view-layer self-yaw drift reconcile ──────────────────────────────── // A magnetometer-less phone has no absolute yaw reference, so its gyro-integrated heading // drifts and the FPV view slides under motion. This walks the drift out SLOWLY at the VIEW diff --git a/androidApp/src/main/java/com/aether/mofe/ui/AppShell.kt b/androidApp/src/main/java/com/aether/mofe/ui/AppShell.kt index 2430ec0..cddb701 100644 --- a/androidApp/src/main/java/com/aether/mofe/ui/AppShell.kt +++ b/androidApp/src/main/java/com/aether/mofe/ui/AppShell.kt @@ -301,6 +301,15 @@ fun AppShell(app: AetherApp, hud: HudViewModel) { .align(Alignment.BottomCenter) .padding(bottom = 24.dp, start = 8.dp, end = 8.dp), ) + + // Debug black-box telemetry overlay (#72): a full-screen, machine-readable data grid + // (raw IMU buffer + engine-clock sync key + frameIdx + fused pose) for SCREEN-RECORD + // capture. Gated by the SAME `mofe-trace.on` sentinel that starts the NDJSON pipeline + // recorder, so one opt-in arms both together; default OFF in normal installs. Drawn last + // so it covers the scene (an opaque data channel) while the recording runs. + if (app.blackBoxOverlayEnabled) { + BlackBoxOverlay(viewModel = hud, modifier = Modifier.fillMaxSize()) + } } } } diff --git a/androidApp/src/main/java/com/aether/mofe/ui/BlackBoxOverlay.kt b/androidApp/src/main/java/com/aether/mofe/ui/BlackBoxOverlay.kt new file mode 100644 index 0000000..1289e95 --- /dev/null +++ b/androidApp/src/main/java/com/aether/mofe/ui/BlackBoxOverlay.kt @@ -0,0 +1,104 @@ +package com.aether.mofe.ui + +import androidx.compose.foundation.Canvas +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.runtime.withFrameNanos +import androidx.compose.ui.Modifier +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.geometry.Size +import androidx.compose.ui.graphics.Color +import com.aether.mofe.viewmodel.HudViewModel + +/** + * Debug-gated, full-screen "black-box" telemetry overlay (#72) — turns the phone screen into a + * machine-readable data channel so a plain SCREEN RECORDING of a field walk carries MORE than the + * on-device NDJSON trace: notably the RAW IMU buffer (which [com.aether.mofe.platform.MofePipelineRecorder] + * does not log), plus a per-frame `tEngineMicros` sync key (the SAME engine clock the recorder stamps) + * and a monotonic `frameIdx`, so an offline decoder can align the video to the NDJSON. The fused self + * pose rides along as a cheap cross-check. + * + * Not human-readable by design — its only consumer is the agent's offline decoder ([BlackBoxCodec]). + * It is drawn LAST over the whole app (opaque), effectively replacing the visible scene while a + * capture runs, so no scene pixels bleed into the data cells. Gated by the SAME `mofe-trace.on` + * sentinel that starts the NDJSON recorder (see [HudViewModel.blackBoxOverlayEnabled]); default OFF. + * + * Drawing (matches the transport tuning in [com.aether.mofe.diag.BlackBoxCodec]): + * - The [HudViewModel.BB_COLS]×[HudViewModel.BB_ROWS] luma grid fills the screen inside a thin + * uniform-black QUIET border, each cell a solid black(0)/white(255) block (~13–18 px @1080p). + * - Three solid white CORNER FIDUCIALS (top-left, top-right, bottom-left — bottom-right left empty + * so orientation is unambiguous) sit in the quiet margin, one fiducial-gap outside the data grid, + * so the decoder can register the grid and sample each cell centre. + * - Redrawn every DISPLAY frame via [withFrameNanos] (the app's redraw convention, cf. Mesh3DCanvas), + * which is also when the frame is built — so `frameIdx` advances and the IMU ring drains per frame. + */ +@Composable +fun BlackBoxOverlay(viewModel: HudViewModel, modifier: Modifier = Modifier) { + val cols = HudViewModel.BB_COLS + val rows = HudViewModel.BB_ROWS + + // Rebuild + encode one frame per display frame; the luma state write drives the Canvas redraw. + var luma by remember { mutableStateOf(IntArray(cols * rows)) } + LaunchedEffect(Unit) { + while (true) { + withFrameNanos { /* pace to the display; the work is the encode below */ } + luma = viewModel.nextBlackBoxLuma() + } + } + + Canvas(modifier = modifier.fillMaxSize()) { + val w = size.width + val h = size.height + // Quiet border: a uniform black margin so the grid never touches the screen edge (the decoder + // needs a clean boundary). ~4% of the short side leaves room for the corner fiducials too. + val margin = minOf(w, h) * 0.04f + + // Solid black background = the quiet zone AND every 0-cell (we only paint the white cells). + drawRect(Color.Black, topLeft = Offset.Zero, size = Size(w, h)) + + val gridLeft = margin + val gridTop = margin + val gridW = w - 2f * margin + val gridH = h - 2f * margin + if (gridW <= 0f || gridH <= 0f) return@Canvas + val cellW = gridW / cols + val cellH = gridH / rows + + // Data cells: paint only the white (255) cells; 0-cells are the black background already. + val g = luma + if (g.size == cols * rows) { + var r = 0 + while (r < rows) { + val top = gridTop + r * cellH + val base = r * cols + var c = 0 + while (c < cols) { + if (g[base + c] != 0) { + drawRect( + Color.White, + topLeft = Offset(gridLeft + c * cellW, top), + size = Size(cellW, cellH), + ) + } + c++ + } + r++ + } + } + + // Three corner fiducials as solid white squares in the quiet margin, one gap outside the grid. + val fid = margin * 0.72f + val gap = margin * 0.14f + // Top-left (its bottom-right corner points at the grid's top-left corner): + drawRect(Color.White, topLeft = Offset(gridLeft - gap - fid, gridTop - gap - fid), size = Size(fid, fid)) + // Top-right (bottom-left corner points at the grid's top-right corner): + drawRect(Color.White, topLeft = Offset(gridLeft + gridW + gap, gridTop - gap - fid), size = Size(fid, fid)) + // Bottom-left (top-right corner points at the grid's bottom-left corner): + drawRect(Color.White, topLeft = Offset(gridLeft - gap - fid, gridTop + gridH + gap), size = Size(fid, fid)) + } +} diff --git a/androidApp/src/main/java/com/aether/mofe/viewmodel/HudViewModel.kt b/androidApp/src/main/java/com/aether/mofe/viewmodel/HudViewModel.kt index fea7616..3645aab 100644 --- a/androidApp/src/main/java/com/aether/mofe/viewmodel/HudViewModel.kt +++ b/androidApp/src/main/java/com/aether/mofe/viewmodel/HudViewModel.kt @@ -5,6 +5,10 @@ import androidx.lifecycle.AndroidViewModel import androidx.lifecycle.viewModelScope import com.aether.mofe.AetherApp import com.aether.mofe.data.* +import com.aether.mofe.diag.BbImu +import com.aether.mofe.diag.BbPose +import com.aether.mofe.diag.BlackBoxCodec +import com.aether.mofe.diag.BlackBoxState import com.aether.mofe.engine.netcode.MeshAimTracker import com.aether.mofe.engine.netcode.NetcodeSession import com.aether.mofe.engine.netcode.RenderClock @@ -22,6 +26,7 @@ import com.aether.mofe.platform.band.BandPredicateRecord import kotlinx.coroutines.flow.* import kotlinx.coroutines.launch import java.util.UUID +import kotlin.math.roundToInt class HudViewModel(application: Application) : AndroidViewModel(application) { @@ -37,6 +42,22 @@ class HudViewModel(application: Application) : AndroidViewModel(application) { private val _imuSample = MutableStateFlow(null) val imuSample: StateFlow = _imuSample.asStateFlow() + // ── Black-box telemetry overlay (#72, debug screen-record capture) ───────────────────────── + // A raw-IMU ring buffer TEED off the SAME [imuFlow] the HUD already collects (a second + // SensorManager listener on the same physical sensors — it never touches the engine's own + // IMU→EKF path). Drained once per rendered overlay frame into a BlackBoxState the overlay + // encodes and draws, so a plain screen recording carries the raw IMU stream the NDJSON + // recorder does NOT log, keyed to the engine clock for offline alignment. See BlackBoxOverlay. + private val bbImuBuf = ArrayDeque() // guarded by bbImuLock + private val bbImuLock = Any() + private var bbLastImuMicros = 0L // inter-sample dt source (persists across frames) + private var bbFrameIdx = 0 // monotonic render-frame index + + /** Mirrors the pipeline-recorder opt-in: true only when the `mofe-trace.on` sentinel is present + * (see [AetherApp.blackBoxOverlayEnabled]), so a capture turns on BOTH the NDJSON recorder and + * this overlay together. Default OFF. */ + val blackBoxOverlayEnabled: Boolean get() = app.blackBoxOverlayEnabled + /** Live self-attitude "spin" diagnostic (see MofeEngineHost.SelfAttitudeDiag), * surfaced for the on-screen debug overlay so a screenshot captures the exact * yaw / drift-rate / rest-gate / gyro-bias state at a moment. */ @@ -113,10 +134,75 @@ class HudViewModel(application: Application) : AndroidViewModel(application) { viewModelScope.launch { imuService.imuFlow() .catch { /* sensor absent — handled via hasGyroscope */ } - .collect { sample -> _imuSample.value = sample } + .collect { sample -> + _imuSample.value = sample + // Tee into the black-box ring buffer only when the overlay is armed, so normal + // installs (sentinel absent) pay nothing beyond the existing _imuSample update. + if (app.blackBoxOverlayEnabled) captureBlackBoxImu(sample) + } + } + } + + /** Append one raw IMU sample to the black-box ring buffer, quantized to [BbImu] scales. + * `dtMicros` is the gap since the PREVIOUS sample (continuous across frame boundaries, so the + * offline decoder can reconstruct the true sample cadence). Runs on the IMU collector coroutine. */ + private fun captureBlackBoxImu(s: ImuSample) { + val tMicros = s.timestamp.microseconds + val dt = if (bbLastImuMicros == 0L) 0 else (tMicros - bbLastImuMicros).coerceIn(0L, 65_535L).toInt() + bbLastImuMicros = tMicros + val a = s.acceleration; val g = s.angularVelocity + val sample = BbImu( + dtMicros = dt, + axMilli = qMilli(a.x), ayMilli = qMilli(a.y), azMilli = qMilli(a.z), // m/s²·1000 + gxMilli = qMilli(g.x), gyMilli = qMilli(g.y), gzMilli = qMilli(g.z), // rad/s·1000 + ) + synchronized(bbImuLock) { + bbImuBuf.addLast(sample) + // Bound memory if a drain is delayed (or the overlay armed but not yet drawing): keep newest. + while (bbImuBuf.size > BB_IMU_BUFFER_CAP) bbImuBuf.removeFirst() } } + /** + * Build + encode ONE black-box telemetry frame for the overlay to draw. Carries what the NDJSON + * trace does NOT: the raw-IMU buffer drained since the previous frame, plus the engine-clock sync + * key ([MofeEngineHost.engineNowMicros] — the SAME timeline MofePipelineRecorder stamps), a + * monotonic `frameIdx`, and the fused self pose (cheap cross-check). Ranges are left EMPTY — the + * NDJSON trace is authoritative for ranges/solve. Returns a row-major 0/255 luma grid + * ([BB_COLS]×[BB_ROWS]) encoded at [BB_REPEAT]-way redundancy. Called once per rendered frame. + */ + fun nextBlackBoxLuma(): IntArray { + val idx = bbFrameIdx++ + val tEngine = app.mofeEngineHost.engineNowMicros() + val pose = app.mofeEngineHost.latestSelfPose() + ?.takeIf { it.position.x.isFinite() && it.position.y.isFinite() && it.position.z.isFinite() } + ?.let { st -> + val q = st.orientation + BbPose( + xMm = qMilli(st.position.x), yMm = qMilli(st.position.y), zMm = qMilli(st.position.z), + qwE4 = qE4(q.w), qxE4 = qE4(q.x), qyE4 = qE4(q.y), qzE4 = qE4(q.z), + ) + } + val drained = synchronized(bbImuLock) { val copy = ArrayList(bbImuBuf); bbImuBuf.clear(); copy } + // Cap the IMU block to what one frame's payload can hold (keeping the NEWEST samples, closest + // to this frame's tEngineMicros) so BlackBoxCodec.encodeToLuma never exceeds grid capacity. At + // normal frame rates (IMU ~100–200 Hz, video ~30–60 fps ⇒ a handful/frame) nothing is dropped. + val cap = BlackBoxCodec.capacityBytes(BB_COLS, BB_ROWS, BB_REPEAT) + val fixed = BB_HEADER_BYTES + 1 + (if (pose != null) BB_POSE_BYTES else 0) // +1 = imuCount byte + val maxImu = ((cap - fixed) / BB_IMU_BYTES).coerceAtLeast(0) + val imu = if (drained.size > maxImu) drained.subList(drained.size - maxImu, drained.size) else drained + val payload = BlackBoxState.encode( + BlackBoxState(frameIdx = idx, tEngineMicros = tEngine, ranges = emptyList(), imu = imu, pose = pose) + ) + return BlackBoxCodec.encodeToLuma(payload, BB_COLS, BB_ROWS, BB_REPEAT) + } + + /** m/s² or rad/s → ·1000 i16 (BbImu), or m → mm i16 (BbPose position); NaN/∞ → 0, clamped to i16. */ + private fun qMilli(v: Double): Int = if (!v.isFinite()) 0 else (v * 1000.0).roundToInt().coerceIn(-32768, 32767) + + /** Unit-quaternion component → ·10000 i16 (BbPose); NaN/∞ → 0, clamped to i16. */ + private fun qE4(v: Double): Int = if (!v.isFinite()) 0 else (v * 10000.0).roundToInt().coerceIn(-32768, 32767) + fun selectNode(node: SelectedNode?) { _selectedNode.value = node } fun clearSelection() { _selectedNode.value = null } @@ -266,6 +352,19 @@ class HudViewModel(application: Application) : AndroidViewModel(application) { val current = predicates.value.find { it.id == predicateId } ?: return repo.publishPredicate(current.copy(isActive = !current.isActive).toBandRecord()) } + + companion object { + // Field black-box grid: 80×130 cells @5-way repetition ECC (~251 B/frame). Read by + // BlackBoxOverlay to lay out the drawn grid; MUST stay in sync with the encode call above. + const val BB_COLS = 80 + const val BB_REPEAT = 5 + const val BB_ROWS = 130 + private const val BB_IMU_BUFFER_CAP = 256 // ring cap between drains (memory bound) + // BlackBoxSchema wire sizes (big-endian) for the per-frame IMU capacity budget: + private const val BB_HEADER_BYTES = 15 // schemaVer1 + frameIdx4 + tEngineMicros8 + flags1 + rangeCount1 + private const val BB_POSE_BYTES = 14 // i16 × 7 (xyz + wxyz) + private const val BB_IMU_BYTES = 14 // u16 dt + i16 × 6 (accel xyz + gyro xyz) + } } data class PredicateEventLog(val timestampMs: Long, val message: String) -- 2.43.0