eric/Crosspoint
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Read GH release JSON as stream in OTA updater (#1810)

Browse Source 
## Summary

The GitHub recent release API returns lots of data, including the full
release notes. For 1.2.0, that data is 30,530 bytes. The existing
OtaUpdater HTTP handler and single-buffer JSON parsing can't reliably
handle that much data in the constrained ESP32 environment.

This change does a few things:
1. Adds a very simple lib/JsonParser/StreamingJsonParser.cpp with
SAX-style callbacks to read JSON data incrementally.
2. Adds a very simple lib/JsonParser/ReleaseJsonParser.cpp building on
StreamingJsonParser, which specifically parses the GitHub release JSON
for the release version, URL, and size.
3. Updates OtaUpdater.cpp to use an instance of ReleaseJsonParser to
incrementally parse the large response it may receive from GitHub.

Building from this commit while overriding my local version to 1.1.9, I
was able to run the OTA update process ~5 times in a row successfully.

Fixes #1561 (second part, after #1805).

---

### AI Usage

While CrossPoint doesn't have restrictions on AI tools in contributing,
please be transparent about their usage as it
helps set the right context for reviewers.

Did you use AI tools to help write this code? _**YES**_
This commit is contained in:
Zach Nelson
2026-05-03 20:48:06 -05:00
committed by GitHub
parent 22701ccf18
commit aa7a31b3db
9 changed files with 2257 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/network/OtaUpdater.cpp
+32 -92
View File
@@ -1,91 +1,48 @@
#include "OtaUpdater.h"
#include <ArduinoJson.h>
#include <Logging.h>
#include "esp_http_client.h"
#include "esp_https_ota.h"
#include "esp_wifi.h"
#include <ReleaseJsonParser.h>
#include <esp_crt_bundle.h>
#include <esp_http_client.h>
#include <esp_https_ota.h>
#include <esp_wifi.h>
namespace {
constexpr char latestReleaseUrl[] = "https://api.github.com/repos/crosspoint-reader/crosspoint-reader/releases/latest";
/* This is buffer and size holder to keep upcoming data from latestReleaseUrl */
char* local_buf;
int output_len;
/*
* When esp_crt_bundle.h included, it is pointing wrong header file
* which is something under WifiClientSecure because of our framework based on arduno platform.
* To manage this obstacle, don't include anything, just extern and it will point correct one.
*/
extern "C" {
extern esp_err_t esp_crt_bundle_attach(void* conf);
}
esp_err_t http_client_set_header_cb(esp_http_client_handle_t http_client) {
return esp_http_client_set_header(http_client, "User-Agent", "CrossPoint-ESP32-" CROSSPOINT_VERSION);
}
size_t totalBytesReceived = 0;
esp_err_t event_handler(esp_http_client_event_t* event) {
/* We do interested in only HTTP_EVENT_ON_DATA event only */
if (event->event_id != HTTP_EVENT_ON_DATA) return ESP_OK;
if (!esp_http_client_is_chunked_response(event->client)) {
int content_len = esp_http_client_get_content_length(event->client);
int copy_len = 0;
if (local_buf == NULL) {
/* local_buf life span is tracked by caller checkForUpdate */
local_buf = static_cast<char*>(calloc(content_len + 1, sizeof(char)));
output_len = 0;
if (local_buf == NULL) {
LOG_ERR("OTA", "HTTP Client Out of Memory Failed, Allocation %d", content_len);
return ESP_ERR_NO_MEM;
}
}
copy_len = min(event->data_len, (content_len - output_len));
if (copy_len) {
memcpy(local_buf + output_len, event->data, copy_len);
}
output_len += copy_len;
} else {
/* Code might be hits here, It happened once (for version checking) but I need more logs to handle that */
int chunked_len;
esp_http_client_get_chunk_length(event->client, &chunked_len);
LOG_DBG("OTA", "esp_http_client_is_chunked_response failed, chunked_len: %d", chunked_len);
}
totalBytesReceived += event->data_len;
LOG_DBG("OTA", "HTTP chunk: %d bytes (total: %zu)", event->data_len, totalBytesReceived);
auto* parser = static_cast<ReleaseJsonParser*>(event->user_data);
parser->feed(static_cast<const char*>(event->data), event->data_len);
return ESP_OK;
} /* event_handler */
} /* namespace */
}
} // namespace
OtaUpdater::OtaUpdaterError OtaUpdater::checkForUpdate() {
JsonDocument filter;
esp_err_t esp_err;
JsonDocument doc;
ReleaseJsonParser releaseParser;
esp_http_client_config_t client_config = {
.url = latestReleaseUrl,
.event_handler = event_handler,
/* Default HTTP client buffer size 512 byte only */
.buffer_size = 8192,
.buffer_size_tx = 8192,
.user_data = &releaseParser,
.skip_cert_common_name_check = true,
.crt_bundle_attach = esp_crt_bundle_attach,
.keep_alive_enable = true,
};
/* To track life time of local_buf, dtor will be called on exit from that function */
struct localBufCleaner {
char** bufPtr;
~localBufCleaner() {
if (*bufPtr) {
free(*bufPtr);
*bufPtr = NULL;
}
}
} localBufCleaner = {&local_buf};
totalBytesReceived = 0;
LOG_DBG("OTA", "Checking for update (current: %s)", CROSSPOINT_VERSION);
esp_http_client_handle_t client_handle = esp_http_client_init(&client_config);
if (!client_handle) {
@@ -107,51 +64,34 @@ OtaUpdater::OtaUpdaterError OtaUpdater::checkForUpdate() {
return HTTP_ERROR;
}
/* esp_http_client_close will be called inside cleanup as well*/
esp_err = esp_http_client_cleanup(client_handle);
if (esp_err != ESP_OK) {
LOG_ERR("OTA", "esp_http_client_cleanup Failed : %s", esp_err_to_name(esp_err));
return INTERNAL_UPDATE_ERROR;
}
filter["tag_name"] = true;
filter["assets"][0]["name"] = true;
filter["assets"][0]["browser_download_url"] = true;
filter["assets"][0]["size"] = true;
const DeserializationError error = deserializeJson(doc, local_buf, DeserializationOption::Filter(filter));
if (error) {
LOG_ERR("OTA", "JSON parse failed: %s", error.c_str());
LOG_DBG("OTA", "Response received: %zu bytes total", totalBytesReceived);
LOG_DBG("OTA", "Parser results: tag=%s firmware=%s", releaseParser.foundTag() ? "yes" : "no",
releaseParser.foundFirmware() ? "yes" : "no");
if (!releaseParser.foundTag()) {
LOG_ERR("OTA", "No tag_name in release JSON");
return JSON_PARSE_ERROR;
}
if (!doc["tag_name"].is<std::string>()) {
LOG_ERR("OTA", "No tag_name found");
return JSON_PARSE_ERROR;
}
if (!doc["assets"].is<JsonArray>()) {
LOG_ERR("OTA", "No assets found");
return JSON_PARSE_ERROR;
}
latestVersion = doc["tag_name"].as<std::string>();
for (int i = 0; i < doc["assets"].size(); i++) {
if (doc["assets"][i]["name"] == "firmware.bin") {
otaUrl = doc["assets"][i]["browser_download_url"].as<std::string>();
otaSize = doc["assets"][i]["size"].as<size_t>();
totalSize = otaSize;
updateAvailable = true;
break;
}
}
if (!updateAvailable) {
if (!releaseParser.foundFirmware()) {
LOG_ERR("OTA", "No firmware.bin asset found");
return NO_UPDATE;
}
LOG_DBG("OTA", "Found update: %s", latestVersion.c_str());
latestVersion = releaseParser.getTagName();
otaUrl = releaseParser.getFirmwareUrl();
otaSize = releaseParser.getFirmwareSize();
totalSize = otaSize;
updateAvailable = true;
LOG_DBG("OTA", "Found update: tag=%s size=%zu", latestVersion.c_str(), otaSize);
LOG_DBG("OTA", "Firmware URL: %s", otaUrl.c_str());
return OK;
}