From 9d0f124b7de8c087310b5be2e69388e27e337bb5 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 23:16:23 +0000 Subject: [PATCH 1/2] =?UTF-8?q?feat(diag):=20screen=20black-box=20codec=20?= =?UTF-8?q?=E2=80=94=20turn=20a=20screen=20recording=20into=20machine-read?= =?UTF-8?q?able=20telemetry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BlackBoxCodec + BlackBoxSchema (pure :common, jvmTest-verified): encode the engine's live per-frame state (observer->anchor ranges+RSSI, a BUFFER of raw IMU samples since the last frame, fused pose+quaternion) into a luma-only binary cell grid an androidApp overlay draws on-screen. A plain screen recording of a field walk then decodes offline into complete, frame-synchronized telemetry — notably the RAW IMU the ndjson never logs, so IMU-odometry observer constraints (#69b) need no on-device recorder change. Transport is tuned for video: luma-only cells, large blocks, interleaved-repetition ECC (copies spread far apart so a smashed-macroblock region hits <=1 copy) + CRC32 that FAILS CLOSED (a corrupt frame decodes to null, never a wrong record). BlackBoxCodecTest (5) proves clean round-trip, recovery through contrast-squeeze+noise+band-wipe+salt, and fail-closed under catastrophic loss. The real-device screen-record round-trip is the remaining at-risk step (androidApp overlay + a decode script). --- .../com/aether/mofe/diag/BlackBoxCodec.kt | 127 +++++++++++++++++ .../com/aether/mofe/diag/BlackBoxSchema.kt | 129 ++++++++++++++++++ .../com/aether/mofe/diag/BlackBoxCodecTest.kt | 90 ++++++++++++ 3 files changed, 346 insertions(+) create mode 100644 common/src/commonMain/kotlin/com/aether/mofe/diag/BlackBoxCodec.kt create mode 100644 common/src/commonMain/kotlin/com/aether/mofe/diag/BlackBoxSchema.kt create mode 100644 common/src/commonTest/kotlin/com/aether/mofe/diag/BlackBoxCodecTest.kt diff --git a/common/src/commonMain/kotlin/com/aether/mofe/diag/BlackBoxCodec.kt b/common/src/commonMain/kotlin/com/aether/mofe/diag/BlackBoxCodec.kt new file mode 100644 index 0000000..2bb34ba --- /dev/null +++ b/common/src/commonMain/kotlin/com/aether/mofe/diag/BlackBoxCodec.kt @@ -0,0 +1,127 @@ +package com.aether.mofe.diag + +/** + * MOFE screen "black-box" codec — turns the phone screen into a robust, machine-readable data channel so a + * plain SCREEN RECORDING of a field walk becomes a self-contained, frame-synchronized telemetry capture + * (observer→anchor ranges + RAW IMU buffer + fused pose + constellation), decodable OFFLINE even through a + * lossy H.264 recording. Not human-readable by design — its only consumer is the agent's decoder. + * + * Why this exists: a screen recording is the cheapest artifact a field operator can produce (no adb, no file + * pull), and it is perfectly time-synchronized to what the user saw. By overlaying an encoded data grid we + * make that recording carry MORE than the on-device NDJSON trace — notably the raw IMU stream (which the + * `MofePipelineRecorder` does not log), so IMU-odometry observer constraints (#69b) need no separate patch. + * + * Transport design (tuned for video compression, not print): + * - **Luma-only** binary cells (0 or 255). Video is chroma-subsampled + block-quantized; luminance extremes + * survive best, so we never rely on colour. + * - **Large cells** (the androidApp overlay draws each cell as a ~13–16 px block @1080p) so the CELL CENTRE + * the decoder samples stays clean while compression only bleeds the borders. + * - **Interleaved repetition ECC**: each payload bit is written to `repeat` cells spread far apart in the + * grid (copy c at bit i → cell i + c·usableBits). A smashed macroblock region is contiguous, so as long as + * the band is thinner than the copy spacing it corrupts at most one copy of any bit → majority vote + * recovers it. CRC32 rejects anything unrecoverable (fail-closed: a bad frame decodes to null, never to a + * wrong-but-plausible record). The API default is `repeat=3`; a field screen-capture layout uses `repeat=5` + * on an ~80×130 grid (copies ~26 rows apart, ~251 B/frame) to survive a smashed band **plus** salt noise. + * + * This file is pure `commonMain` (jvmTest-verifiable). The androidApp overlay calls [encodeToLuma] and draws + * the returned grid on a Canvas; the agent calls [decodeFromLuma] on sampled video frames. The state⇄bytes + * schema is [BlackBoxSchema]. + */ +object BlackBoxCodec { + const val MAGIC0 = 0xA5 + const val MAGIC1 = 0x3C + const val VERSION = 1 + private const val HEADER = 5 // magic(2) + ver(1) + len(2) + private const val TRAILER = 4 // crc32(4) + const val OVERHEAD = HEADER + TRAILER + + /** Payload bytes that fit in a [cols]×[rows] grid at [repeat]-way redundancy (0 if the grid is too small). */ + fun capacityBytes(cols: Int, rows: Int, repeat: Int = 3): Int { + val usableBits = (cols * rows) / repeat + val bytes = usableBits / 8 - OVERHEAD + return if (bytes < 0) 0 else bytes + } + + /** Wrap a payload into a framed byte record: MAGIC(2) VER(1) LEN(2, u16) PAYLOAD CRC32(4 over VER..PAYLOAD). */ + fun frame(payload: ByteArray): ByteArray { + require(payload.size <= 0xFFFF) { "payload too large: ${payload.size}" } + val out = ByteArray(HEADER + payload.size + TRAILER) + out[0] = MAGIC0.toByte(); out[1] = MAGIC1.toByte(); out[2] = VERSION.toByte() + out[3] = ((payload.size ushr 8) and 0xFF).toByte(); out[4] = (payload.size and 0xFF).toByte() + payload.copyInto(out, HEADER) + val crc = crc32(out, 2, HEADER + payload.size) + writeU32(out, HEADER + payload.size, crc) + return out + } + + /** Inverse of [frame]; returns the payload, or null if magic / length / CRC do not check out. */ + fun deframe(bytes: ByteArray): ByteArray? { + if (bytes.size < OVERHEAD) return null + if ((bytes[0].toInt() and 0xFF) != MAGIC0 || (bytes[1].toInt() and 0xFF) != MAGIC1) return null + val len = ((bytes[3].toInt() and 0xFF) shl 8) or (bytes[4].toInt() and 0xFF) + if (HEADER + len + TRAILER > bytes.size) return null + val crcCalc = crc32(bytes, 2, HEADER + len) + val crcRead = readU32(bytes, HEADER + len) + if (crcCalc != crcRead) return null + return bytes.copyOfRange(HEADER, HEADER + len) + } + + /** Encode [payload] into a row-major luma grid (each entry 0 or 255), length cols*rows. */ + fun encodeToLuma(payload: ByteArray, cols: Int, rows: Int, repeat: Int = 3): IntArray { + val f = frame(payload) + val total = cols * rows + val usableBits = total / repeat + require(f.size * 8 <= usableBits) { + "payload+overhead ${f.size}B needs ${f.size * 8} bits > $usableBits usable (cap ${capacityBytes(cols, rows, repeat)}B)" + } + val luma = IntArray(total) // default 0 → padding bits read as 0 + val bitCount = f.size * 8 + for (i in 0 until bitCount) { + val bit = (f[i / 8].toInt() ushr (7 - (i % 8))) and 1 + if (bit == 1) { + var c = 0 + while (c < repeat) { luma[i + c * usableBits] = 255; c++ } + } + } + return luma + } + + /** Decode a row-major luma grid (values 0..255) back to the payload, or null if unrecoverable. */ + fun decodeFromLuma(luma: IntArray, cols: Int, rows: Int, repeat: Int = 3): ByteArray? { + val total = cols * rows + if (luma.size != total) return null + val usableBits = total / repeat + // adaptive threshold: midpoint of observed luma range (robust to global brightness/contrast shift). + var mn = 255; var mx = 0 + for (v in luma) { if (v < mn) mn = v; if (v > mx) mx = v } + val thr = (mn + mx) / 2 + val nbytes = usableBits / 8 + val bytes = ByteArray(nbytes) + for (i in 0 until nbytes * 8) { + var ones = 0 + var c = 0 + while (c < repeat) { if (luma[i + c * usableBits] > thr) ones++; c++ } + if (ones * 2 >= repeat) bytes[i / 8] = (bytes[i / 8].toInt() or (1 shl (7 - (i % 8)))).toByte() + } + return deframe(bytes) + } + + // ── helpers ────────────────────────────────────────────────────────────────────────── + fun crc32(data: ByteArray, from: Int = 0, to: Int = data.size): Int { + var crc = 0.inv() + for (i in from until to) { + crc = crc xor (data[i].toInt() and 0xFF) + var k = 0 + while (k < 8) { crc = (crc ushr 1) xor (0xEDB88320.toInt() and -(crc and 1)); k++ } + } + return crc.inv() + } + + private fun writeU32(b: ByteArray, off: Int, v: Int) { + b[off] = ((v ushr 24) and 0xFF).toByte(); b[off + 1] = ((v ushr 16) and 0xFF).toByte() + b[off + 2] = ((v ushr 8) and 0xFF).toByte(); b[off + 3] = (v and 0xFF).toByte() + } + private fun readU32(b: ByteArray, off: Int): Int = + ((b[off].toInt() and 0xFF) shl 24) or ((b[off + 1].toInt() and 0xFF) shl 16) or + ((b[off + 2].toInt() and 0xFF) shl 8) or (b[off + 3].toInt() and 0xFF) +} diff --git a/common/src/commonMain/kotlin/com/aether/mofe/diag/BlackBoxSchema.kt b/common/src/commonMain/kotlin/com/aether/mofe/diag/BlackBoxSchema.kt new file mode 100644 index 0000000..6d82b70 --- /dev/null +++ b/common/src/commonMain/kotlin/com/aether/mofe/diag/BlackBoxSchema.kt @@ -0,0 +1,129 @@ +package com.aether.mofe.diag + +/** + * Compact, versioned binary schema for one [BlackBoxCodec] frame — the exact wire format the androidApp + * diagnostic overlay emits per rendered frame and the agent decodes offline from a screen recording. + * + * All fields are pre-quantized INTEGERS with fixed scales (documented per field) so encode⇄decode is EXACT + * and unit-testable without float tolerance; the androidApp side does the SI→int quantization when it taps + * engine state. Layout (big-endian): + * + * u8 schemaVer + * u32 frameIdx monotonic render-frame index (detect dropped video frames) + * u64 tEngineMicros engine clock at emit (odometry integration + ndjson cross-ref) + * u8 flags bit0 = pose present + * u8 rangeCount ; rangeCount × { u24 tag, u16 distanceMm, u8 sq(0..255), i8 rssiDbm, u16 azCentiDeg, i16 elCentiDeg } + * u8 imuCount ; imuCount × { u16 dtMicros, i16 ax,ay,az (m/s²·1000), i16 gx,gy,gz (rad/s·1000) } + * [pose] i16 x,y,z (mm) i16 qw,qx,qy,qz (unit-quat ·10000) + * + * The IMU block is a BUFFER of every raw sample since the previous rendered frame (video is ~30–60 fps but + * the IMU is ~100–200 Hz), so no IMU sample is lost to the frame rate — this is what lets a screen recording + * alone supply IMU-odometry observer constraints (#69b) with no on-device NDJSON change. + */ +data class BbRange( + /** device-tag suffix, 6 hex chars (e.g. "85b92d"); carried as u24. */ val anchorTag: String, + val distanceMm: Int, val sq: Int, val rssiDbm: Int, val azCentiDeg: Int, val elCentiDeg: Int, +) + +/** One raw IMU sample. accel = m/s²·1000, gyro = rad/s·1000, dt = µs since the previous sample. */ +data class BbImu( + val dtMicros: Int, + val axMilli: Int, val ayMilli: Int, val azMilli: Int, + val gxMilli: Int, val gyMilli: Int, val gzMilli: Int, +) + +/** Fused self pose. position = mm in the frame; quaternion = unit-quat components ·10000. */ +data class BbPose( + val xMm: Int, val yMm: Int, val zMm: Int, + val qwE4: Int, val qxE4: Int, val qyE4: Int, val qzE4: Int, +) + +data class BlackBoxState( + val frameIdx: Int, + val tEngineMicros: Long, + val ranges: List, + val imu: List, + val pose: BbPose?, +) { + companion object { + const val SCHEMA_VERSION = 1 + + fun encode(s: BlackBoxState): ByteArray { + val w = ByteWriter() + w.u8(SCHEMA_VERSION) + w.u32(s.frameIdx) + w.u64(s.tEngineMicros) + w.u8(if (s.pose != null) 1 else 0) + require(s.ranges.size <= 255 && s.imu.size <= 255) { "too many ranges/imu samples in one frame" } + w.u8(s.ranges.size) + for (r in s.ranges) { + w.u24(r.anchorTag.takeLast(6).toIntOrNull(16) ?: 0) + w.u16(r.distanceMm); w.u8(r.sq and 0xFF); w.i8(r.rssiDbm) + w.u16(r.azCentiDeg); w.i16(r.elCentiDeg) + } + w.u8(s.imu.size) + for (m in s.imu) { + w.u16(m.dtMicros) + w.i16(m.axMilli); w.i16(m.ayMilli); w.i16(m.azMilli) + w.i16(m.gxMilli); w.i16(m.gyMilli); w.i16(m.gzMilli) + } + s.pose?.let { p -> + w.i16(p.xMm); w.i16(p.yMm); w.i16(p.zMm) + w.i16(p.qwE4); w.i16(p.qxE4); w.i16(p.qyE4); w.i16(p.qzE4) + } + return w.toByteArray() + } + + /** Inverse of [encode]; returns null if the version is unknown or the buffer is truncated/malformed. */ + fun decode(bytes: ByteArray): BlackBoxState? { + return try { + val r = ByteReader(bytes) + if (r.u8() != SCHEMA_VERSION) return null + val frameIdx = r.u32() + val t = r.u64() + val flags = r.u8() + val nR = r.u8() + val ranges = ArrayList(nR) + repeat(nR) { + val tag = r.u24().toString(16).padStart(6, '0') + ranges.add(BbRange(tag, r.u16(), r.u8(), r.i8(), r.u16(), r.i16())) + } + val nI = r.u8() + val imu = ArrayList(nI) + repeat(nI) { imu.add(BbImu(r.u16(), r.i16(), r.i16(), r.i16(), r.i16(), r.i16(), r.i16())) } + val pose = if (flags and 1 != 0) { + BbPose(r.i16(), r.i16(), r.i16(), r.i16(), r.i16(), r.i16(), r.i16()) + } else null + BlackBoxState(frameIdx, t, ranges, imu, pose) + } catch (e: IndexOutOfBoundsException) { + null + } + } + } +} + +/** Minimal big-endian byte writer (diagnostic path — clarity over allocation). */ +class ByteWriter { + private val b = ArrayList(256) + fun u8(v: Int) { b.add((v and 0xFF).toByte()) } + fun i8(v: Int) { b.add((v and 0xFF).toByte()) } + fun u16(v: Int) { u8(v ushr 8); u8(v) } + fun i16(v: Int) { u16(v and 0xFFFF) } + fun u24(v: Int) { u8(v ushr 16); u8(v ushr 8); u8(v) } + fun u32(v: Int) { u8(v ushr 24); u8(v ushr 16); u8(v ushr 8); u8(v) } + fun u64(v: Long) { for (s in 56 downTo 0 step 8) u8((v ushr s).toInt()) } + fun toByteArray(): ByteArray = b.toByteArray() +} + +/** Minimal big-endian byte reader; throws [IndexOutOfBoundsException] past the end (caught by [BlackBoxState.decode]). */ +class ByteReader(private val b: ByteArray) { + private var p = 0 + private fun next(): Int = (b[p++].toInt() and 0xFF) + fun u8(): Int = next() + fun i8(): Int = next().let { if (it >= 0x80) it - 0x100 else it } + fun u16(): Int = (next() shl 8) or next() + fun i16(): Int = u16().let { if (it >= 0x8000) it - 0x10000 else it } + fun u24(): Int = (next() shl 16) or (next() shl 8) or next() + fun u32(): Int = (next() shl 24) or (next() shl 16) or (next() shl 8) or next() + fun u64(): Long { var v = 0L; repeat(8) { v = (v shl 8) or next().toLong() }; return v } +} diff --git a/common/src/commonTest/kotlin/com/aether/mofe/diag/BlackBoxCodecTest.kt b/common/src/commonTest/kotlin/com/aether/mofe/diag/BlackBoxCodecTest.kt new file mode 100644 index 0000000..31a1415 --- /dev/null +++ b/common/src/commonTest/kotlin/com/aether/mofe/diag/BlackBoxCodecTest.kt @@ -0,0 +1,90 @@ +package com.aether.mofe.diag + +import kotlin.random.Random +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * Proves the screen "black-box" channel: a per-frame telemetry record survives a lossy screen recording and + * fails CLOSED (never a wrong-but-plausible record) under catastrophic damage. Degradation is a fair proxy + * for what H.264 does to the CELL CENTRE the decoder samples: global contrast squeeze, additive noise, a + * smashed-macroblock BAND, and salt. (Inter-cell bleed is not modelled because the overlay draws each cell as + * a ~13–16 px block, so the centre stays clean — the real risk is regional loss, which the spread repetition + * and CRC defend against.) + * + * ECC sizing is honest, not seed-tuned: at 5× interleaved repetition on an 80×130 grid the copies of any bit + * are ~26 rows apart, so a smashed band < 26 rows tall kills ≤1 of 5 copies, and even a further ~1 % salt + * rarely kills a 3rd — majority holds. A band + 3 % salt would kill a 2nd copy of enough band-hit bits to + * defeat 3× repetition, which is exactly why this uses 5×. The real-device screen-record round-trip is the one + * thing this can't prove; that's the at-risk step (see the androidApp overlay + decode script). + */ +class BlackBoxCodecTest { + private val cols = 80; private val rows = 130; private val repeat = 5 // copies ~26 rows apart; ~251 B capacity + + private fun sampleState() = BlackBoxState( + frameIdx = 12345, + tEngineMicros = 1_700_000_123_456L, + ranges = listOf( + BbRange("264b1b", distanceMm = 1234, sq = 200, rssiDbm = -72, azCentiDeg = 4510, elCentiDeg = -320), + BbRange("a6080f", distanceMm = 2860, sq = 180, rssiDbm = -80, azCentiDeg = 9000, elCentiDeg = 15), + BbRange("85b92d", distanceMm = 3697, sq = 90, rssiDbm = -91, azCentiDeg = 27010, elCentiDeg = 220), + BbRange("c77cc1", distanceMm = 2540, sq = 150, rssiDbm = -77, azCentiDeg = 18000, elCentiDeg = -900), + ), + imu = (0 until 7).map { i -> + BbImu(5000 + i, axMilli = 120 - i, ayMilli = -30 + i, azMilli = 9810, gxMilli = 5 * i, gyMilli = -4 * i, gzMilli = 2 * i) + }, + pose = BbPose(xMm = 812, yMm = -1503, zMm = 1100, qwE4 = 9998, qxE4 = 100, qyE4 = -50, qzE4 = 30), + ) + + private fun encodeFrame(s: BlackBoxState) = BlackBoxCodec.encodeToLuma(BlackBoxState.encode(s), cols, rows, repeat) + private fun decodeFrame(luma: IntArray): BlackBoxState? = + BlackBoxCodec.decodeFromLuma(luma, cols, rows, repeat)?.let { BlackBoxState.decode(it) } + + private fun wipeRows(luma: IntArray, r0: Int, r1: Int, value: Int) { + for (r in r0 until r1) for (c in 0 until cols) luma[r * cols + c] = value + } + + @Test + fun clean_round_trip_reconstructs_state_exactly() { + val s = sampleState() + assertEquals(s, decodeFrame(encodeFrame(s)), "a clean screen-grab must round-trip bit-exact") + } + + @Test + fun survives_realistic_video_degradation() { + val s = sampleState() + val luma = encodeFrame(s) + val rnd = Random(0xBEEF) + for (i in luma.indices) luma[i] = 40 + luma[i] * (210 - 40) / 255 // (a) global contrast squeeze + for (i in luma.indices) luma[i] = (luma[i] + rnd.nextInt(-45, 46)).coerceIn(0, 255) // (b) additive noise + wipeRows(luma, 34, 52, value = 90) // (c) smashed band 18 rows (< 26 spacing) + repeat(luma.size / 100) { luma[rnd.nextInt(luma.size)] = rnd.nextInt(256) } // (d) ~1 % salt + assertEquals(s, decodeFrame(luma), "5× interleaved-repetition ECC + CRC must recover through squeeze+noise+band+salt") + } + + @Test + fun contiguous_region_loss_survives_because_copies_are_spread() { + val s = sampleState() + val luma = encodeFrame(s) + wipeRows(luma, 30, 52, value = 0) // 22-row contiguous kill, below the ~26-row copy spacing → recoverable + assertEquals(s, decodeFrame(luma), "a contiguous region below the copy spacing must not defeat the frame") + } + + @Test + fun catastrophic_damage_fails_closed_not_wrong() { + val s = sampleState() + val luma = encodeFrame(s) + wipeRows(luma, 8, 122, value = 0) // kills ≥3 of 5 copies for every bit → majority wrong → CRC rejects + assertNull(BlackBoxCodec.decodeFromLuma(luma, cols, rows, repeat), "catastrophic loss must fail closed (null), never a wrong record") + } + + @Test + fun capacity_is_reported_and_oversize_rejected() { + val cap = BlackBoxCodec.capacityBytes(cols, rows, repeat) + assertTrue(cap in 220..300, "expected ~251 B capacity for ${cols}x$rows @${repeat}x, got $cap") + val threw = try { BlackBoxCodec.encodeToLuma(ByteArray(cap + 50), cols, rows, repeat); false } catch (e: IllegalArgumentException) { true } + assertTrue(threw, "an over-capacity payload must be rejected, not silently truncated") + } +} -- 2.43.0