feat(update): SD-card firmware update + X3 bootloader compatibility (#1786)

src/network/FirmwareFlasher.cpp

@@ -0,0 +1,311 @@
+#include "FirmwareFlasher.h"
+
+#include <Arduino.h>
+#include <HalStorage.h>
+#include <Logging.h>
+#include <esp_ota_ops.h>
+#include <esp_partition.h>
+#include <mbedtls/sha256.h>
+#include <spi_flash_mmap.h>
+
+#include <algorithm>
+#include <cstring>
+#include <memory>
+
+#include "OtaBootSwitch.h"
+
+namespace firmware_flash {
+
+namespace {
+constexpr uint8_t ESP_IMAGE_MAGIC = 0xE9;
+constexpr size_t MIN_FIRMWARE_SIZE = 64 * 1024;
+constexpr size_t SEC = SPI_FLASH_SEC_SIZE;  // 4 KiB
+constexpr size_t BLK = 64 * 1024;           // 64 KiB block-erase granularity
+constexpr size_t CHUNK = 4096;
+constexpr size_t SHA_TRAILER = 32;
+constexpr uint8_t CHECKSUM_SEED = 0xEF;
+constexpr size_t HEADER_SIZE = 24;
+constexpr size_t SEG_HEADER_SIZE = 8;
+}  // namespace
+
+const char* resultName(Result r) {
+  switch (r) {
+    case Result::OK:
+      return "OK";
+    case Result::OPEN_FAIL:
+      return "OPEN_FAIL";
+    case Result::TOO_SMALL:
+      return "TOO_SMALL";
+    case Result::TOO_LARGE:
+      return "TOO_LARGE";
+    case Result::BAD_MAGIC:
+      return "BAD_MAGIC";
+    case Result::BAD_SEGMENTS:
+      return "BAD_SEGMENTS";
+    case Result::BAD_CHECKSUM:
+      return "BAD_CHECKSUM";
+    case Result::BAD_SHA:
+      return "BAD_SHA";
+    case Result::BAD_SIZE:
+      return "BAD_SIZE";
+    case Result::NO_PARTITION:
+      return "NO_PARTITION";
+    case Result::OOM:
+      return "OOM";
+    case Result::READ_FAIL:
+      return "READ_FAIL";
+    case Result::ERASE_FAIL:
+      return "ERASE_FAIL";
+    case Result::WRITE_FAIL:
+      return "WRITE_FAIL";
+    case Result::OTADATA_FAIL:
+      return "OTADATA_FAIL";
+  }
+  return "?";
+}
+
+namespace {
+// Stream `length` bytes from `file` starting at the current read offset, feeding them through
+// both the XOR-checksum and SHA256 accumulators. Used by validateImageFile so the whole image
+// is verified end-to-end without holding it in RAM (ESP32-C3 only has ~380 KB).
+Result feedHashAndChecksum(HalFile& file, size_t length, uint8_t* xorAccum, mbedtls_sha256_context* sha, uint8_t* buf) {
+  size_t remaining = length;
+  while (remaining > 0) {
+    const size_t want = std::min<size_t>(CHUNK, remaining);
+    const int got = file.read(buf, want);
+    if (got <= 0 || static_cast<size_t>(got) != want) return Result::READ_FAIL;
+    if (sha) mbedtls_sha256_update(sha, buf, want);
+    if (xorAccum) {
+      uint8_t acc = *xorAccum;
+      for (size_t i = 0; i < want; i++) acc ^= buf[i];
+      *xorAccum = acc;
+    }
+    remaining -= want;
+  }
+  return Result::OK;
+}
+}  // namespace
+
+Result validateImageFile(const char* sdPath, size_t partitionSize) {
+  HalFile file;
+  if (!Storage.openFileForRead("FLASH", sdPath, file) || !file) {
+    LOG_ERR("FLASH", "validate: open failed: %s", sdPath);
+    return Result::OPEN_FAIL;
+  }
+
+  const size_t fileSize = file.fileSize();
+  if (fileSize < MIN_FIRMWARE_SIZE) {
+    LOG_ERR("FLASH", "validate: too small: %u", static_cast<unsigned>(fileSize));
+    file.close();
+    return Result::TOO_SMALL;
+  }
+  if (partitionSize > 0 && fileSize > partitionSize) {
+    LOG_ERR("FLASH", "validate: too large: %u > %u", static_cast<unsigned>(fileSize),
+            static_cast<unsigned>(partitionSize));
+    file.close();
+    return Result::TOO_LARGE;
+  }
+
+  uint8_t header[HEADER_SIZE];
+  if (file.read(header, HEADER_SIZE) != static_cast<int>(HEADER_SIZE)) {
+    LOG_ERR("FLASH", "validate: header read failed");
+    file.close();
+    return Result::READ_FAIL;
+  }
+  if (header[0] != ESP_IMAGE_MAGIC) {
+    LOG_ERR("FLASH", "validate: bad magic 0x%02X", header[0]);
+    file.close();
+    return Result::BAD_MAGIC;
+  }
+  const uint8_t segCount = header[1];
+  const bool hashAppended = header[23] != 0;
+
+  auto buf = std::unique_ptr<uint8_t[]>(new (std::nothrow) uint8_t[CHUNK]);
+  if (!buf) {
+    file.close();
+    return Result::OOM;
+  }
+
+  mbedtls_sha256_context shaCtx;
+  mbedtls_sha256_init(&shaCtx);
+  mbedtls_sha256_starts(&shaCtx, /*is224=*/0);
+  mbedtls_sha256_update(&shaCtx, header, HEADER_SIZE);
+
+  uint8_t xorAccum = CHECKSUM_SEED;
+  size_t pos = HEADER_SIZE;
+
+  for (uint8_t i = 0; i < segCount; i++) {
+    if (pos + SEG_HEADER_SIZE > fileSize) {
+      LOG_ERR("FLASH", "validate: seg %u header overruns EOF at %u", i, static_cast<unsigned>(pos));
+      mbedtls_sha256_free(&shaCtx);
+      file.close();
+      return Result::BAD_SEGMENTS;
+    }
+    uint8_t segHdr[SEG_HEADER_SIZE];
+    if (file.read(segHdr, SEG_HEADER_SIZE) != static_cast<int>(SEG_HEADER_SIZE)) {
+      mbedtls_sha256_free(&shaCtx);
+      file.close();
+      return Result::READ_FAIL;
+    }
+    mbedtls_sha256_update(&shaCtx, segHdr, SEG_HEADER_SIZE);
+    pos += SEG_HEADER_SIZE;
+
+    uint32_t dataLen;
+    std::memcpy(&dataLen, segHdr + 4, sizeof(dataLen));
+    if (pos + dataLen > fileSize) {
+      LOG_ERR("FLASH", "validate: seg %u data overruns EOF (%u + %u > %u)", i, static_cast<unsigned>(pos),
+              static_cast<unsigned>(dataLen), static_cast<unsigned>(fileSize));
+      mbedtls_sha256_free(&shaCtx);
+      file.close();
+      return Result::BAD_SEGMENTS;
+    }
+
+    const Result feedRes = feedHashAndChecksum(file, dataLen, &xorAccum, &shaCtx, buf.get());
+    if (feedRes != Result::OK) {
+      mbedtls_sha256_free(&shaCtx);
+      file.close();
+      return feedRes;
+    }
+    pos += dataLen;
+  }
+
+  // pad_end is the 16-byte aligned offset at which the checksum byte sits at pad_end - 1.
+  const size_t padEnd = (pos + 16) & ~static_cast<size_t>(15);
+  const size_t expectedTotal = padEnd + (hashAppended ? SHA_TRAILER : 0);
+  if (expectedTotal != fileSize) {
+    LOG_ERR("FLASH", "validate: size mismatch body+pad=%u sha=%u expected=%u actual=%u", static_cast<unsigned>(padEnd),
+            static_cast<unsigned>(hashAppended ? SHA_TRAILER : 0), static_cast<unsigned>(expectedTotal),
+            static_cast<unsigned>(fileSize));
+    mbedtls_sha256_free(&shaCtx);
+    file.close();
+    return Result::BAD_SIZE;
+  }
+
+  // Read the padding bytes (which include the stored checksum at the last byte) into the SHA stream.
+  const size_t padLen = padEnd - pos;
+  uint8_t padBuf[16];
+  if (padLen > sizeof(padBuf)) {
+    mbedtls_sha256_free(&shaCtx);
+    file.close();
+    return Result::BAD_SIZE;
+  }
+  if (padLen > 0 && file.read(padBuf, padLen) != static_cast<int>(padLen)) {
+    mbedtls_sha256_free(&shaCtx);
+    file.close();
+    return Result::READ_FAIL;
+  }
+  mbedtls_sha256_update(&shaCtx, padBuf, padLen);
+
+  const uint8_t storedChecksum = padBuf[padLen - 1];
+  if ((xorAccum & 0xFF) != storedChecksum) {
+    LOG_ERR("FLASH", "validate: checksum mismatch computed=0x%02X stored=0x%02X", xorAccum, storedChecksum);
+    mbedtls_sha256_free(&shaCtx);
+    file.close();
+    return Result::BAD_CHECKSUM;
+  }
+
+  if (hashAppended) {
+    uint8_t computed[SHA_TRAILER];
+    mbedtls_sha256_finish(&shaCtx, computed);
+    uint8_t stored[SHA_TRAILER];
+    if (file.read(stored, SHA_TRAILER) != static_cast<int>(SHA_TRAILER)) {
+      mbedtls_sha256_free(&shaCtx);
+      file.close();
+      return Result::READ_FAIL;
+    }
+    if (std::memcmp(computed, stored, SHA_TRAILER) != 0) {
+      LOG_ERR("FLASH", "validate: SHA256 mismatch");
+      mbedtls_sha256_free(&shaCtx);
+      file.close();
+      return Result::BAD_SHA;
+    }
+  }
+
+  mbedtls_sha256_free(&shaCtx);
+  file.close();
+  return Result::OK;
+}
+
+Result flashFromSdPath(const char* sdPath, ProgressCb onProgress, void* ctx, bool alreadyValidated) {
+  // Resolve destination first so we can size-check during validation. The full image-integrity
+  // pass below verifies header, segment table, XOR checksum and SHA256 trailer end-to-end before
+  // we touch otadata, so a truncated/corrupted .bin can never become the next boot target.
+  const esp_partition_t* dest = esp_ota_get_next_update_partition(nullptr);
+  if (!dest) {
+    LOG_ERR("FLASH", "no next-update partition");
+    return Result::NO_PARTITION;
+  }
+
+  // When the caller already ran validateImageFile() against this same partition
+  // size (e.g. SdFirmwareUpdateActivity validates before the confirmation
+  // prompt), skip the redundant integrity scan. We still keep the partition
+  // lookup so the rest of the flashing path stays unchanged.
+  if (!alreadyValidated) {
+    const Result validateRes = validateImageFile(sdPath, dest->size);
+    if (validateRes != Result::OK) {
+      LOG_ERR("FLASH", "image validation failed: %s", resultName(validateRes));
+      return validateRes;
+    }
+  }
+
+  HalFile file;
+  if (!Storage.openFileForRead("FLASH", sdPath, file) || !file) {
+    LOG_ERR("FLASH", "open failed: %s", sdPath);
+    return Result::OPEN_FAIL;
+  }
+
+  const size_t firmwareSize = file.fileSize();
+  LOG_INF("FLASH", "src=%s size=%u dest=%s @0x%x partsize=%u", sdPath, static_cast<unsigned>(firmwareSize), dest->label,
+          static_cast<unsigned>(dest->address), static_cast<unsigned>(dest->size));
+
+  auto buffer = std::unique_ptr<uint8_t[]>(new (std::nothrow) uint8_t[CHUNK]);
+  if (!buffer) {
+    LOG_ERR("FLASH", "OOM");
+    file.close();
+    return Result::OOM;
+  }
+
+  // Interleave erase + write so the progress bar advances 0→100% smoothly
+  // rather than stalling for several seconds during a single up-front erase.
+  size_t streamPos = 0;
+  size_t erasedUpto = 0;
+  while (streamPos < firmwareSize) {
+    if (streamPos >= erasedUpto) {
+      size_t eraseLen = std::min<size_t>(BLK, dest->size - streamPos);
+      eraseLen = (eraseLen + SEC - 1) & ~(SEC - 1);
+      eraseLen = std::min<size_t>(eraseLen, dest->size - streamPos);
+      if (esp_partition_erase_range(dest, streamPos, eraseLen) != ESP_OK) {
+        LOG_ERR("FLASH", "erase @%u (len=%u) failed", static_cast<unsigned>(streamPos),
+                static_cast<unsigned>(eraseLen));
+        file.close();
+        return Result::ERASE_FAIL;
+      }
+      erasedUpto = streamPos + eraseLen;
+    }
+
+    const size_t want = std::min<size_t>(CHUNK, firmwareSize - streamPos);
+    const int read = file.read(buffer.get(), want);
+    if (read <= 0 || static_cast<size_t>(read) != want) {
+      LOG_ERR("FLASH", "read @%u: got=%d want=%u", static_cast<unsigned>(streamPos), read, static_cast<unsigned>(want));
+      file.close();
+      return Result::READ_FAIL;
+    }
+    if (esp_partition_write(dest, streamPos, buffer.get(), want) != ESP_OK) {
+      LOG_ERR("FLASH", "write @%u failed", static_cast<unsigned>(streamPos));
+      file.close();
+      return Result::WRITE_FAIL;
+    }
+    streamPos += want;
+    if (onProgress) onProgress(streamPos, firmwareSize, ctx);
+    delay(1);
+  }
+  file.close();
+
+  if (!ota_boot::switchTo(dest)) {
+    LOG_ERR("FLASH", "otadata switch failed");
+    return Result::OTADATA_FAIL;
+  }
+  return Result::OK;
+}
+
+}  // namespace firmware_flash

src/network/FirmwareFlasher.h

@@ -0,0 +1,63 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+// Flash a firmware image from an SD-card path into the next OTA app
+// partition, then switch otadata so the X3/X4 stock bootloader picks it up
+// on next boot. Mirrors the web flasher: raw esp_partition_erase_range +
+// esp_partition_write + ota_boot::switchTo (no Arduino Update class, no
+// esp_image_verify — those reject our patched image on X4 silicon).
+//
+// Both the SD update activity and the OTA path land here. OTA first
+// downloads the firmware to an SD-card cache file, then calls this.
+
+namespace firmware_flash {
+
+enum class Result {
+  OK,
+  OPEN_FAIL,
+  TOO_SMALL,
+  TOO_LARGE,
+  BAD_MAGIC,
+  BAD_SEGMENTS,  // segment table malformed or runs past EOF
+  BAD_CHECKSUM,  // ESP image XOR checksum mismatch
+  BAD_SHA,       // SHA256 trailer mismatch (hash_appended images)
+  BAD_SIZE,      // body+pad+sha length doesn't match file size
+  NO_PARTITION,
+  OOM,
+  READ_FAIL,
+  ERASE_FAIL,
+  WRITE_FAIL,
+  OTADATA_FAIL,
+};
+
+// Progress callback: called after every chunk write. `written`/`total` are bytes.
+using ProgressCb = void (*)(size_t written, size_t total, void* ctx);
+
+// Open `sdPath`, validate it looks like an ESP32 image, then stream it into the
+// next OTA app partition with interleaved 64 KiB erase + sector writes. On
+// success switches otadata via ota_boot::switchTo. Caller is responsible for
+// ESP.restart() afterwards.
+//
+// `alreadyValidated` lets callers that have just run `validateImageFile()`
+// themselves (e.g. SdFirmwareUpdateActivity, which validates before showing
+// the user the confirmation prompt) skip the redundant second pass. Defaults
+// to false so callers without prior validation (any future entry point) keep
+// the defense-in-depth check.
+Result flashFromSdPath(const char* sdPath, ProgressCb onProgress, void* ctx, bool alreadyValidated = false);
+
+// Full-image integrity check that mirrors the bootloader's verification:
+// header magic, segment table walk, XOR checksum, and SHA256 trailer (when
+// hash_appended == 1). Run this before flashing a candidate firmware so a
+// truncated/corrupted .bin never reaches otadata.
+//
+// `partitionSize` is the size of the destination OTA partition; pass 0 to
+// skip the size-fits-partition check (e.g. when validating ahead of partition
+// lookup). Streams the file in CHUNK-sized reads; the file is rewound on
+// success so the caller can immediately reread it for flashing.
+Result validateImageFile(const char* sdPath, size_t partitionSize);
+
+const char* resultName(Result r);
+
+}  // namespace firmware_flash

src/network/OtaBootSwitch.cpp

@@ -0,0 +1,86 @@
+#include "OtaBootSwitch.h"
+
+#include <Logging.h>
+#include <esp_rom_crc.h>
+#include <spi_flash_mmap.h>
+#include <string.h>
+
+namespace ota_boot {
+
+uint32_t computeSeqCrc(uint32_t seq) {
+  return esp_rom_crc32_le(UINT32_MAX, reinterpret_cast<const uint8_t*>(&seq), kOtaSeqCrcLen);
+}
+
+bool switchTo(const esp_partition_t* dest) {
+  if (!dest) return false;
+
+  const esp_partition_t* otadata =
+      esp_partition_find_first(ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_OTA, nullptr);
+  if (!otadata) {
+    LOG_ERR("BOOT", "otadata partition not found");
+    return false;
+  }
+  if (otadata->size < 2 * SPI_FLASH_SEC_SIZE) {
+    LOG_ERR("BOOT", "otadata too small: %u", static_cast<unsigned>(otadata->size));
+    return false;
+  }
+
+  SelectEntry slots[2] = {};
+  if (esp_partition_read(otadata, 0, &slots[0], sizeof(SelectEntry)) != ESP_OK ||
+      esp_partition_read(otadata, SPI_FLASH_SEC_SIZE, &slots[1], sizeof(SelectEntry)) != ESP_OK) {
+    LOG_ERR("BOOT", "otadata read failed");
+    return false;
+  }
+
+  // Pick the slot with valid CRC and highest seq, ignoring INVALID/ABORTED.
+  int activeIdx = -1;
+  uint32_t activeSeq = 0;
+  for (int i = 0; i < 2; ++i) {
+    if (slots[i].ota_seq == 0xFFFFFFFFu) continue;
+    if (slots[i].crc != computeSeqCrc(slots[i].ota_seq)) continue;
+    if (slots[i].ota_state == kOtaImgInvalid || slots[i].ota_state == kOtaImgAborted) continue;
+    if (activeIdx < 0 || slots[i].ota_seq > activeSeq) {
+      activeIdx = i;
+      activeSeq = slots[i].ota_seq;
+    }
+  }
+  LOG_INF("BOOT", "otadata: active slot=%d seq=%u", activeIdx, static_cast<unsigned>(activeSeq));
+
+  // ota_seq encoding: (seq - 1) % NUM_OTA_PARTITIONS picks the partition.
+  const uint32_t destOtaIdx =
+      static_cast<uint32_t>(dest->subtype) - static_cast<uint32_t>(ESP_PARTITION_SUBTYPE_APP_OTA_0);
+  if (destOtaIdx > 15) {
+    LOG_ERR("BOOT", "dest is not an OTA app partition (subtype=0x%02X)", dest->subtype);
+    return false;
+  }
+
+  // Find smallest seq > activeSeq such that (seq-1) % 2 == destOtaIdx,
+  // assuming 2 OTA partitions (matches our partitions.csv with ota_0 + ota_1).
+  uint32_t newSeq = activeSeq + 1;
+  while (((newSeq - 1u) % 2u) != (destOtaIdx % 2u)) ++newSeq;
+
+  SelectEntry next = {};
+  next.ota_seq = newSeq;
+  memset(next.seq_label, 0xFF, sizeof(next.seq_label));
+  next.ota_state = kOtaImgNew;
+  next.crc = computeSeqCrc(next.ota_seq);
+
+  // Write to the OTHER slot (so the bootloader sees a higher seq there).
+  const int targetSlot = (activeIdx == 0) ? 1 : 0;
+  const size_t targetOff = static_cast<size_t>(targetSlot) * SPI_FLASH_SEC_SIZE;
+
+  if (esp_partition_erase_range(otadata, targetOff, SPI_FLASH_SEC_SIZE) != ESP_OK) {
+    LOG_ERR("BOOT", "otadata erase failed (slot=%d)", targetSlot);
+    return false;
+  }
+  if (esp_partition_write(otadata, targetOff, &next, sizeof(next)) != ESP_OK) {
+    LOG_ERR("BOOT", "otadata write failed (slot=%d)", targetSlot);
+    return false;
+  }
+
+  LOG_INF("BOOT", "otadata: wrote slot=%d seq=%u crc=0x%08x -> %s", targetSlot, static_cast<unsigned>(newSeq),
+          static_cast<unsigned>(next.crc), dest->label);
+  return true;
+}
+
+}  // namespace ota_boot

src/network/OtaBootSwitch.h

@@ -0,0 +1,43 @@
+#pragma once
+
+#include <esp_partition.h>
+
+#include <cstddef>
+#include <cstdint>
+
+// X4 (and X3) factory bootloaders accept our patch_firmware_image.py-patched
+// firmware.bin (web flasher proves this), but the running ESP-IDF's
+// esp_image_verify rejects with bogus efuse-blk-rev errors. Both SD-card and
+// OTA update paths bypass that runtime check by writing the OTA app partition
+// raw and updating otadata directly — same scheme as the web flasher
+// (crosspoint-reader-docs/src/lib/flasher/OtaPartition.ts).
+//
+// Layout reference: esp_flash_partitions.h. CRC covers ota_seq (4 bytes) only.
+
+namespace ota_boot {
+
+struct __attribute__((packed)) SelectEntry {
+  uint32_t ota_seq;
+  uint8_t seq_label[20];
+  uint32_t ota_state;
+  uint32_t crc;
+};
+static_assert(sizeof(SelectEntry) == 32, "SelectEntry must be 32 bytes");
+
+constexpr uint32_t kOtaImgNew = 0;      // ESP_OTA_IMG_NEW
+constexpr uint32_t kOtaImgInvalid = 3;  // ESP_OTA_IMG_INVALID
+constexpr uint32_t kOtaImgAborted = 4;  // ESP_OTA_IMG_ABORTED
+constexpr size_t kOtaSeqCrcLen = 4;
+
+// CRC32-LE over the 4-byte ota_seq, init UINT32_MAX. Matches IDF and web flasher.
+uint32_t computeSeqCrc(uint32_t seq);
+
+// Switch the bootloader's selected app partition to `dest` by writing a fresh
+// otadata entry into the inactive otadata slot. Bypasses esp_ota_set_boot_partition's
+// esp_image_verify call. The bytes in `dest` must already be a valid app image
+// (e.g. patch_firmware_image.py output) — caller is responsible for that.
+//
+// Returns true on success.
+bool switchTo(const esp_partition_t* dest);
+
+}  // namespace ota_boot