fix: Read GH release JSON as stream in OTA updater (#1810)
## Summary The GitHub recent release API returns lots of data, including the full release notes. For 1.2.0, that data is 30,530 bytes. The existing OtaUpdater HTTP handler and single-buffer JSON parsing can't reliably handle that much data in the constrained ESP32 environment. This change does a few things: 1. Adds a very simple lib/JsonParser/StreamingJsonParser.cpp with SAX-style callbacks to read JSON data incrementally. 2. Adds a very simple lib/JsonParser/ReleaseJsonParser.cpp building on StreamingJsonParser, which specifically parses the GitHub release JSON for the release version, URL, and size. 3. Updates OtaUpdater.cpp to use an instance of ReleaseJsonParser to incrementally parse the large response it may receive from GitHub. Building from this commit while overriding my local version to 1.1.9, I was able to run the OTA update process ~5 times in a row successfully. Fixes #1561 (second part, after #1805). --- ### AI Usage While CrossPoint doesn't have restrictions on AI tools in contributing, please be transparent about their usage as it helps set the right context for reviewers. Did you use AI tools to help write this code? _**YES**_
This commit is contained in:
Zach Nelson
@@ -0,0 +1,208 @@
|
||||
#include "ReleaseJsonParser.h"
|
||||
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
|
||||
namespace {
|
||||
|
||||
void safeCopy(char* dst, size_t dstSize, const char* src, size_t srcLen) {
|
||||
size_t n = srcLen < dstSize - 1 ? srcLen : dstSize - 1;
|
||||
memcpy(dst, src, n);
|
||||
dst[n] = '\0';
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
ReleaseJsonParser::ReleaseJsonParser()
|
||||
: parser(JsonCallbacks{this, sOnKey, sOnString, sOnNumber, sOnBool, sOnNull, sOnObjectStart, sOnObjectEnd,
|
||||
sOnArrayStart, sOnArrayEnd}) {
|
||||
reset();
|
||||
}
|
||||
|
||||
void ReleaseJsonParser::reset() {
|
||||
parser.reset();
|
||||
position = Position::TOP_LEVEL;
|
||||
lastKey = LastKey::NONE;
|
||||
depth = 0;
|
||||
assetDepth = 0;
|
||||
tagName[0] = '\0';
|
||||
firmwareUrl[0] = '\0';
|
||||
firmwareSize = 0;
|
||||
tagFound = false;
|
||||
firmwareFound = false;
|
||||
currentAssetName[0] = '\0';
|
||||
currentAssetUrl[0] = '\0';
|
||||
currentAssetSize = 0;
|
||||
}
|
||||
|
||||
void ReleaseJsonParser::feed(const char* data, size_t len) { parser.feed(data, len); }
|
||||
|
||||
bool ReleaseJsonParser::foundTag() const { return tagFound; }
|
||||
bool ReleaseJsonParser::foundFirmware() const { return firmwareFound; }
|
||||
const char* ReleaseJsonParser::getTagName() const { return tagName; }
|
||||
const char* ReleaseJsonParser::getFirmwareUrl() const { return firmwareUrl; }
|
||||
size_t ReleaseJsonParser::getFirmwareSize() const { return firmwareSize; }
|
||||
|
||||
void ReleaseJsonParser::commitAsset() {
|
||||
if (strcmp(currentAssetName, "firmware.bin") == 0) {
|
||||
memcpy(firmwareUrl, currentAssetUrl, sizeof(firmwareUrl));
|
||||
firmwareSize = currentAssetSize;
|
||||
firmwareFound = true;
|
||||
}
|
||||
currentAssetName[0] = '\0';
|
||||
currentAssetUrl[0] = '\0';
|
||||
currentAssetSize = 0;
|
||||
}
|
||||
|
||||
// -- SAX callbacks (static trampolines) -------------------------------------
|
||||
|
||||
void ReleaseJsonParser::sOnKey(void* ctx, const char* key, size_t len) {
|
||||
auto* self = static_cast<ReleaseJsonParser*>(ctx);
|
||||
|
||||
switch (self->position) {
|
||||
case Position::TOP_LEVEL:
|
||||
if (self->depth == 1) {
|
||||
if (len == 8 && memcmp(key, "tag_name", 8) == 0)
|
||||
self->lastKey = LastKey::TAG_NAME;
|
||||
else if (len == 6 && memcmp(key, "assets", 6) == 0)
|
||||
self->lastKey = LastKey::ASSETS;
|
||||
else
|
||||
self->lastKey = LastKey::NONE;
|
||||
}
|
||||
break;
|
||||
case Position::IN_ASSET_OBJECT:
|
||||
if (self->assetDepth == 1) {
|
||||
if (len == 4 && memcmp(key, "name", 4) == 0)
|
||||
self->lastKey = LastKey::ASSET_NAME;
|
||||
else if (len == 20 && memcmp(key, "browser_download_url", 20) == 0)
|
||||
self->lastKey = LastKey::ASSET_URL;
|
||||
else if (len == 4 && memcmp(key, "size", 4) == 0)
|
||||
self->lastKey = LastKey::ASSET_SIZE;
|
||||
else
|
||||
self->lastKey = LastKey::NONE;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void ReleaseJsonParser::sOnString(void* ctx, const char* value, size_t len) {
|
||||
auto* self = static_cast<ReleaseJsonParser*>(ctx);
|
||||
|
||||
switch (self->lastKey) {
|
||||
case LastKey::TAG_NAME:
|
||||
if (self->position == Position::TOP_LEVEL && self->depth == 1) {
|
||||
safeCopy(self->tagName, sizeof(self->tagName), value, len);
|
||||
self->tagFound = true;
|
||||
}
|
||||
break;
|
||||
case LastKey::ASSET_NAME:
|
||||
if (self->position == Position::IN_ASSET_OBJECT && self->assetDepth == 1)
|
||||
safeCopy(self->currentAssetName, sizeof(self->currentAssetName), value, len);
|
||||
break;
|
||||
case LastKey::ASSET_URL:
|
||||
if (self->position == Position::IN_ASSET_OBJECT && self->assetDepth == 1)
|
||||
safeCopy(self->currentAssetUrl, sizeof(self->currentAssetUrl), value, len);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
self->lastKey = LastKey::NONE;
|
||||
}
|
||||
|
||||
void ReleaseJsonParser::sOnNumber(void* ctx, const char* value, size_t /*len*/) {
|
||||
auto* self = static_cast<ReleaseJsonParser*>(ctx);
|
||||
|
||||
if (self->lastKey == LastKey::ASSET_SIZE && self->position == Position::IN_ASSET_OBJECT && self->assetDepth == 1) {
|
||||
self->currentAssetSize = static_cast<size_t>(strtoul(value, nullptr, 10));
|
||||
}
|
||||
self->lastKey = LastKey::NONE;
|
||||
}
|
||||
|
||||
void ReleaseJsonParser::sOnBool(void* ctx, bool /*value*/) {
|
||||
static_cast<ReleaseJsonParser*>(ctx)->lastKey = LastKey::NONE;
|
||||
}
|
||||
|
||||
void ReleaseJsonParser::sOnNull(void* ctx) { static_cast<ReleaseJsonParser*>(ctx)->lastKey = LastKey::NONE; }
|
||||
|
||||
void ReleaseJsonParser::sOnObjectStart(void* ctx) {
|
||||
auto* self = static_cast<ReleaseJsonParser*>(ctx);
|
||||
|
||||
switch (self->position) {
|
||||
case Position::TOP_LEVEL:
|
||||
self->depth++;
|
||||
self->lastKey = LastKey::NONE;
|
||||
break;
|
||||
case Position::IN_ASSETS_ARRAY:
|
||||
self->position = Position::IN_ASSET_OBJECT;
|
||||
self->assetDepth = 1;
|
||||
self->currentAssetName[0] = '\0';
|
||||
self->currentAssetUrl[0] = '\0';
|
||||
self->currentAssetSize = 0;
|
||||
self->lastKey = LastKey::NONE;
|
||||
break;
|
||||
case Position::IN_ASSET_OBJECT:
|
||||
self->assetDepth++;
|
||||
self->lastKey = LastKey::NONE;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void ReleaseJsonParser::sOnObjectEnd(void* ctx) {
|
||||
auto* self = static_cast<ReleaseJsonParser*>(ctx);
|
||||
|
||||
switch (self->position) {
|
||||
case Position::TOP_LEVEL:
|
||||
if (self->depth > 0) self->depth--;
|
||||
break;
|
||||
case Position::IN_ASSET_OBJECT:
|
||||
self->assetDepth--;
|
||||
if (self->assetDepth == 0) {
|
||||
self->commitAsset();
|
||||
self->position = Position::IN_ASSETS_ARRAY;
|
||||
}
|
||||
self->lastKey = LastKey::NONE;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void ReleaseJsonParser::sOnArrayStart(void* ctx) {
|
||||
auto* self = static_cast<ReleaseJsonParser*>(ctx);
|
||||
|
||||
switch (self->position) {
|
||||
case Position::TOP_LEVEL:
|
||||
if (self->lastKey == LastKey::ASSETS && self->depth == 1) {
|
||||
self->position = Position::IN_ASSETS_ARRAY;
|
||||
} else {
|
||||
self->depth++;
|
||||
}
|
||||
self->lastKey = LastKey::NONE;
|
||||
break;
|
||||
case Position::IN_ASSET_OBJECT:
|
||||
self->assetDepth++;
|
||||
self->lastKey = LastKey::NONE;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void ReleaseJsonParser::sOnArrayEnd(void* ctx) {
|
||||
auto* self = static_cast<ReleaseJsonParser*>(ctx);
|
||||
|
||||
switch (self->position) {
|
||||
case Position::TOP_LEVEL:
|
||||
if (self->depth > 0) self->depth--;
|
||||
break;
|
||||
case Position::IN_ASSETS_ARRAY:
|
||||
self->position = Position::TOP_LEVEL;
|
||||
break;
|
||||
case Position::IN_ASSET_OBJECT:
|
||||
self->assetDepth--;
|
||||
self->lastKey = LastKey::NONE;
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
|
||||
#include "StreamingJsonParser.h"
|
||||
|
||||
class ReleaseJsonParser {
|
||||
public:
|
||||
ReleaseJsonParser();
|
||||
|
||||
ReleaseJsonParser(const ReleaseJsonParser&) = delete;
|
||||
ReleaseJsonParser& operator=(const ReleaseJsonParser&) = delete;
|
||||
|
||||
void reset();
|
||||
void feed(const char* data, size_t len);
|
||||
|
||||
bool foundTag() const;
|
||||
bool foundFirmware() const;
|
||||
const char* getTagName() const;
|
||||
const char* getFirmwareUrl() const;
|
||||
size_t getFirmwareSize() const;
|
||||
|
||||
private:
|
||||
enum class Position : uint8_t {
|
||||
TOP_LEVEL,
|
||||
IN_ASSETS_ARRAY,
|
||||
IN_ASSET_OBJECT,
|
||||
};
|
||||
|
||||
enum class LastKey : uint8_t {
|
||||
NONE,
|
||||
TAG_NAME,
|
||||
ASSETS,
|
||||
ASSET_NAME,
|
||||
ASSET_URL,
|
||||
ASSET_SIZE,
|
||||
};
|
||||
|
||||
static void sOnKey(void* ctx, const char* key, size_t len);
|
||||
static void sOnString(void* ctx, const char* value, size_t len);
|
||||
static void sOnNumber(void* ctx, const char* value, size_t len);
|
||||
static void sOnBool(void* ctx, bool value);
|
||||
static void sOnNull(void* ctx);
|
||||
static void sOnObjectStart(void* ctx);
|
||||
static void sOnObjectEnd(void* ctx);
|
||||
static void sOnArrayStart(void* ctx);
|
||||
static void sOnArrayEnd(void* ctx);
|
||||
|
||||
void commitAsset();
|
||||
|
||||
StreamingJsonParser parser;
|
||||
|
||||
Position position;
|
||||
LastKey lastKey;
|
||||
uint8_t depth;
|
||||
uint8_t assetDepth;
|
||||
|
||||
char tagName[32];
|
||||
char firmwareUrl[512];
|
||||
size_t firmwareSize;
|
||||
bool tagFound;
|
||||
bool firmwareFound;
|
||||
|
||||
char currentAssetName[32];
|
||||
char currentAssetUrl[512];
|
||||
size_t currentAssetSize;
|
||||
};
|
||||
@@ -0,0 +1,249 @@
|
||||
#include "StreamingJsonParser.h"
|
||||
|
||||
#include <cstring>
|
||||
|
||||
StreamingJsonParser::StreamingJsonParser(const JsonCallbacks& callbacks) : cb(callbacks) { reset(); }
|
||||
|
||||
void StreamingJsonParser::reset() {
|
||||
tokenLen = 0;
|
||||
state = State::SCANNING;
|
||||
expectingValue = false;
|
||||
escaped = false;
|
||||
tokenOverflow = false;
|
||||
error = false;
|
||||
nestingDepth = 0;
|
||||
literalLen = 0;
|
||||
literalPos = 0;
|
||||
}
|
||||
|
||||
void StreamingJsonParser::feed(const char* data, size_t len) {
|
||||
for (size_t i = 0; i < len && !error; ++i) {
|
||||
char c = data[i];
|
||||
switch (state) {
|
||||
case State::SCANNING:
|
||||
handleScanning(c);
|
||||
break;
|
||||
case State::IN_STRING_KEY:
|
||||
case State::IN_STRING_VALUE:
|
||||
handleStringChar(c);
|
||||
break;
|
||||
case State::IN_NUMBER:
|
||||
handleNumber(c);
|
||||
break;
|
||||
case State::IN_LITERAL:
|
||||
handleLiteral(c);
|
||||
break;
|
||||
case State::SKIP_STRING:
|
||||
handleSkipString(c);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void StreamingJsonParser::handleScanning(char c) {
|
||||
switch (c) {
|
||||
case '"':
|
||||
tokenLen = 0;
|
||||
tokenOverflow = false;
|
||||
if (expectingValue || inArray()) {
|
||||
state = State::IN_STRING_VALUE;
|
||||
} else {
|
||||
state = State::IN_STRING_KEY;
|
||||
}
|
||||
break;
|
||||
case '{':
|
||||
if (nestingDepth < MAX_NESTING) {
|
||||
nestingStack[nestingDepth++] = Container::OBJECT;
|
||||
} else {
|
||||
error = true;
|
||||
return;
|
||||
}
|
||||
if (cb.onObjectStart) cb.onObjectStart(cb.ctx);
|
||||
expectingValue = false;
|
||||
break;
|
||||
case '}':
|
||||
if (cb.onObjectEnd) cb.onObjectEnd(cb.ctx);
|
||||
if (nestingDepth > 0) --nestingDepth;
|
||||
expectingValue = false;
|
||||
break;
|
||||
case '[':
|
||||
if (nestingDepth < MAX_NESTING) {
|
||||
nestingStack[nestingDepth++] = Container::ARRAY;
|
||||
} else {
|
||||
error = true;
|
||||
return;
|
||||
}
|
||||
if (cb.onArrayStart) cb.onArrayStart(cb.ctx);
|
||||
expectingValue = false;
|
||||
break;
|
||||
case ']':
|
||||
if (cb.onArrayEnd) cb.onArrayEnd(cb.ctx);
|
||||
if (nestingDepth > 0) --nestingDepth;
|
||||
expectingValue = false;
|
||||
break;
|
||||
case ':':
|
||||
expectingValue = true;
|
||||
break;
|
||||
case ',':
|
||||
expectingValue = false;
|
||||
break;
|
||||
case 't':
|
||||
if (expectingValue || inArray()) {
|
||||
memcpy(literalExpected, "true", 4);
|
||||
literalLen = 4;
|
||||
literalPos = 1;
|
||||
state = State::IN_LITERAL;
|
||||
}
|
||||
break;
|
||||
case 'f':
|
||||
if (expectingValue || inArray()) {
|
||||
memcpy(literalExpected, "false", 5);
|
||||
literalLen = 5;
|
||||
literalPos = 1;
|
||||
state = State::IN_LITERAL;
|
||||
}
|
||||
break;
|
||||
case 'n':
|
||||
if (expectingValue || inArray()) {
|
||||
memcpy(literalExpected, "null", 4);
|
||||
literalLen = 4;
|
||||
literalPos = 1;
|
||||
state = State::IN_LITERAL;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
if ((expectingValue || inArray()) && (c == '-' || (c >= '0' && c <= '9'))) {
|
||||
tokenLen = 0;
|
||||
tokenOverflow = false;
|
||||
appendToken(c);
|
||||
state = State::IN_NUMBER;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void StreamingJsonParser::handleStringChar(char c) {
|
||||
if (escaped) {
|
||||
escaped = false;
|
||||
switch (c) {
|
||||
case '"':
|
||||
case '\\':
|
||||
case '/':
|
||||
appendToken(c);
|
||||
break;
|
||||
case 'b':
|
||||
appendToken('\b');
|
||||
break;
|
||||
case 'f':
|
||||
appendToken('\f');
|
||||
break;
|
||||
case 'n':
|
||||
appendToken('\n');
|
||||
break;
|
||||
case 'r':
|
||||
appendToken('\r');
|
||||
break;
|
||||
case 't':
|
||||
appendToken('\t');
|
||||
break;
|
||||
case 'u':
|
||||
// Pass \uXXXX through as literal characters -- we don't decode
|
||||
// Unicode escapes since our use case only needs ASCII field matching.
|
||||
appendToken('\\');
|
||||
appendToken('u');
|
||||
break;
|
||||
default:
|
||||
appendToken('\\');
|
||||
appendToken(c);
|
||||
break;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (c == '\\') {
|
||||
escaped = true;
|
||||
return;
|
||||
}
|
||||
|
||||
if (c == '"') {
|
||||
emitToken();
|
||||
return;
|
||||
}
|
||||
|
||||
appendToken(c);
|
||||
}
|
||||
|
||||
void StreamingJsonParser::handleNumber(char c) {
|
||||
if ((c >= '0' && c <= '9') || c == '.' || c == '-' || c == '+' || c == 'e' || c == 'E') {
|
||||
appendToken(c);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!tokenOverflow && cb.onNumber) {
|
||||
tokenBuf[tokenLen] = '\0';
|
||||
cb.onNumber(cb.ctx, tokenBuf, tokenLen);
|
||||
}
|
||||
state = State::SCANNING;
|
||||
expectingValue = false;
|
||||
|
||||
handleScanning(c);
|
||||
}
|
||||
|
||||
void StreamingJsonParser::handleLiteral(char c) {
|
||||
if (c == literalExpected[literalPos]) {
|
||||
++literalPos;
|
||||
if (literalPos == literalLen) {
|
||||
if (literalExpected[0] == 't') {
|
||||
if (cb.onBool) cb.onBool(cb.ctx, true);
|
||||
} else if (literalExpected[0] == 'f') {
|
||||
if (cb.onBool) cb.onBool(cb.ctx, false);
|
||||
} else {
|
||||
if (cb.onNull) cb.onNull(cb.ctx);
|
||||
}
|
||||
state = State::SCANNING;
|
||||
expectingValue = false;
|
||||
}
|
||||
} else {
|
||||
error = true;
|
||||
}
|
||||
}
|
||||
|
||||
void StreamingJsonParser::handleSkipString(char c) {
|
||||
if (escaped) {
|
||||
escaped = false;
|
||||
return;
|
||||
}
|
||||
if (c == '\\') {
|
||||
escaped = true;
|
||||
return;
|
||||
}
|
||||
if (c == '"') {
|
||||
state = State::SCANNING;
|
||||
expectingValue = false;
|
||||
}
|
||||
}
|
||||
|
||||
void StreamingJsonParser::appendToken(char c) {
|
||||
if (tokenLen < TOKEN_BUF_SIZE - 1) {
|
||||
tokenBuf[tokenLen++] = c;
|
||||
} else {
|
||||
tokenOverflow = true;
|
||||
}
|
||||
}
|
||||
|
||||
void StreamingJsonParser::emitToken() {
|
||||
if (state == State::IN_STRING_KEY) {
|
||||
if (!tokenOverflow && cb.onKey) {
|
||||
tokenBuf[tokenLen] = '\0';
|
||||
cb.onKey(cb.ctx, tokenBuf, tokenLen);
|
||||
}
|
||||
state = State::SCANNING;
|
||||
} else {
|
||||
if (!tokenOverflow && cb.onString) {
|
||||
tokenBuf[tokenLen] = '\0';
|
||||
cb.onString(cb.ctx, tokenBuf, tokenLen);
|
||||
}
|
||||
state = State::SCANNING;
|
||||
expectingValue = false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
|
||||
struct JsonCallbacks {
|
||||
void* ctx;
|
||||
void (*onKey)(void* ctx, const char* key, size_t len);
|
||||
void (*onString)(void* ctx, const char* value, size_t len);
|
||||
void (*onNumber)(void* ctx, const char* value, size_t len);
|
||||
void (*onBool)(void* ctx, bool value);
|
||||
void (*onNull)(void* ctx);
|
||||
void (*onObjectStart)(void* ctx);
|
||||
void (*onObjectEnd)(void* ctx);
|
||||
void (*onArrayStart)(void* ctx);
|
||||
void (*onArrayEnd)(void* ctx);
|
||||
};
|
||||
|
||||
class StreamingJsonParser {
|
||||
public:
|
||||
static constexpr size_t TOKEN_BUF_SIZE = 512;
|
||||
static constexpr size_t MAX_NESTING = 32;
|
||||
|
||||
explicit StreamingJsonParser(const JsonCallbacks& callbacks);
|
||||
|
||||
void reset();
|
||||
void feed(const char* data, size_t len);
|
||||
|
||||
bool hasError() const { return error; }
|
||||
|
||||
private:
|
||||
enum class State : uint8_t {
|
||||
SCANNING,
|
||||
IN_STRING_KEY,
|
||||
IN_STRING_VALUE,
|
||||
IN_NUMBER,
|
||||
IN_LITERAL,
|
||||
SKIP_STRING,
|
||||
};
|
||||
|
||||
enum class Container : uint8_t {
|
||||
NONE,
|
||||
OBJECT,
|
||||
ARRAY,
|
||||
};
|
||||
|
||||
void handleScanning(char c);
|
||||
void handleStringChar(char c);
|
||||
void handleNumber(char c);
|
||||
void handleLiteral(char c);
|
||||
void handleSkipString(char c);
|
||||
|
||||
void appendToken(char c);
|
||||
void emitToken();
|
||||
|
||||
bool inArray() const { return nestingDepth > 0 && nestingStack[nestingDepth - 1] == Container::ARRAY; }
|
||||
|
||||
JsonCallbacks cb;
|
||||
char tokenBuf[TOKEN_BUF_SIZE];
|
||||
size_t tokenLen;
|
||||
State state;
|
||||
bool expectingValue;
|
||||
bool escaped;
|
||||
bool tokenOverflow;
|
||||
bool error;
|
||||
|
||||
Container nestingStack[MAX_NESTING];
|
||||
uint8_t nestingDepth;
|
||||
|
||||
char literalExpected[6];
|
||||
uint8_t literalLen;
|
||||
uint8_t literalPos;
|
||||
};
|
||||
Reference in New Issue
Block a user