From cd215519c017d499e4fa4a99ea3c849a84ab2b1a Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Wed, 4 Jun 2025 15:19:02 +0300 Subject: [PATCH 1/4] feat(esptool): Upgrade to esptool v5 --- .github/scripts/package_esptool.sh | 129 ++++++++++++++++++++++ package/package_esp32_index.template.json | 70 ++++++------ platform.txt | 12 +- 3 files changed, 170 insertions(+), 41 deletions(-) create mode 100755 .github/scripts/package_esptool.sh diff --git a/.github/scripts/package_esptool.sh b/.github/scripts/package_esptool.sh new file mode 100755 index 00000000000..32b87b277e9 --- /dev/null +++ b/.github/scripts/package_esptool.sh @@ -0,0 +1,129 @@ +#!/bin/bash + +set -euo pipefail + +# Check version argument +if [[ $# -ne 3 ]]; then + echo "Usage: $0 " + echo "Example: $0 5.0.dev1 /tmp/esptool /tmp/esptool-5.0.dev1.json" + exit 1 +fi + +VERSION=$1 +BASE_FOLDER=$2 +JSON_PATH=$3 + +export COPYFILE_DISABLE=1 + +shopt -s nullglob # So for loop doesn't run if no matches + +# Function to update JSON for a given host +function update_json_for_host { + local host=$1 + local archive=$2 + + # Extract the old url from the JSON for this host, then replace only the filename + old_url=$(jq -r --arg host "$host" ' + .packages[].tools[] | select(.name == "esptool_py") | .systems[] | select(.host == $host) | .url // empty + ' "$tmp_json") + if [[ -n "$old_url" ]]; then + base_url="${old_url%/*}" + url="$base_url/$archive" + else + echo "No old url found for $host" + exit 1 + fi + + archiveFileName="$archive" + checksum="SHA-256:$(shasum -a 256 "$archive" | awk '{print $1}')" + size=$(stat -f%z "$archive") + + # Use jq to update the JSON + jq --arg host "$host" \ + --arg url "$url" \ + --arg archiveFileName "$archiveFileName" \ + --arg checksum "$checksum" \ + --arg size "$size" \ + ' + .packages[].tools[] + |= if .name == "esptool_py" then + .systems = ( + ((.systems // []) | map(select(.host != $host))) + [{ + host: $host, + url: $url, + archiveFileName: $archiveFileName, + checksum: $checksum, + size: $size + }] + ) + else + . + end + ' "$tmp_json" > "$tmp_json.new" && mv "$tmp_json.new" "$tmp_json" +} + +cd "$BASE_FOLDER" + +# Delete all archives before starting +rm -f esptool-*.tar.gz esptool-*.zip + +for dir in esptool-*; do + # Check if directory exists and is a directory + if [[ ! -d "$dir" ]]; then + continue + fi + + base="${dir#esptool-}" + + # Add 'linux-' prefix if base doesn't contain linux/macos/win64 + if [[ "$base" != *linux* && "$base" != *macos* && "$base" != *win64* ]]; then + base="linux-${base}" + fi + + if [[ "$dir" == esptool-win* ]]; then + # Windows zip archive + zipfile="esptool-v${VERSION}-${base}.zip" + echo "Creating $zipfile from $dir ..." + zip -r "$zipfile" "$dir" + else + # Non-Windows: set permissions and tar.gz archive + tarfile="esptool-v${VERSION}-${base}.tar.gz" + echo "Setting permissions and creating $tarfile from $dir ..." + chmod -R u=rwx,g=rx,o=rx "$dir" + tar -cvzf "$tarfile" "$dir" + fi +done + +# After the for loop, update the JSON for each archive +# Create a temporary JSON file to accumulate changes +tmp_json="${JSON_PATH}.tmp" +cp "$JSON_PATH" "$tmp_json" + +for archive in esptool-v"${VERSION}"-*.tar.gz esptool-v"${VERSION}"-*.zip; do + [ -f "$archive" ] || continue + + echo "Updating JSON for $archive" + + # Determine host from archive name + case "$archive" in + *linux-amd64*) host="x86_64-pc-linux-gnu" ;; + *linux-armv7*) host="arm-linux-gnueabihf" ;; + *linux-aarch64*) host="aarch64-linux-gnu" ;; + *macos-amd64*) host="x86_64-apple-darwin" ;; + *macos-arm64*) host="arm64-apple-darwin" ;; + *win64*) hosts=("x86_64-mingw32" "i686-mingw32") ;; + *) echo "Unknown host for $archive"; continue ;; + esac + + # For win64, loop over both hosts; otherwise, use a single host + if [[ "$archive" == *win64* ]]; then + for host in "${hosts[@]}"; do + update_json_for_host "$host" "$archive" + done + else + update_json_for_host "$host" "$archive" + fi +done + +# After all archives are processed, move the temporary JSON to the final file +mv "$tmp_json" "$JSON_PATH" diff --git a/package/package_esp32_index.template.json b/package/package_esp32_index.template.json index eecc7c10788..3ae2ff09ee6 100644 --- a/package/package_esp32_index.template.json +++ b/package/package_esp32_index.template.json @@ -81,7 +81,7 @@ { "packager": "esp32", "name": "esptool_py", - "version": "4.9.dev3" + "version": "5.0.dev1" }, { "packager": "esp32", @@ -469,56 +469,56 @@ }, { "name": "esptool_py", - "version": "4.9.dev3", + "version": "5.0.dev1", "systems": [ { - "host": "x86_64-pc-linux-gnu", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-linux-amd64.tar.gz", - "archiveFileName": "esptool-v4.9.dev3-linux-amd64.tar.gz", - "checksum": "SHA-256:4ecaf51836cbf4ea3c19840018bfef3b0b8cd8fc3c95f6e1e043ca5bbeab9bf0", - "size": "64958202" + "host": "aarch64-linux-gnu", + "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-linux-aarch64.tar.gz", + "archiveFileName": "esptool-v5.0.dev1-linux-aarch64.tar.gz", + "checksum": "SHA-256:bfafa7a7723ebbabfd8b6e3ca5ae00bfead0331de923754aeddb43b2c116a078", + "size": "58241736" }, { - "host": "arm-linux-gnueabihf", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-linux-armv7.tar.gz", - "archiveFileName": "esptool-v4.9.dev3-linux-armv7.tar.gz", - "checksum": "SHA-256:fff818573bce483ee793ac83c8211f6abf764aa3350f198228859f696a0a0b36", - "size": "31530030" + "host": "x86_64-pc-linux-gnu", + "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-linux-amd64.tar.gz", + "archiveFileName": "esptool-v5.0.dev1-linux-amd64.tar.gz", + "checksum": "SHA-256:acd0486e96586b99d053a1479acbbbfcae8667227c831cdc53a171f9ccfa27ee", + "size": "100740042" }, { - "host": "aarch64-linux-gnu", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-linux-aarch64.tar.gz", - "archiveFileName": "esptool-v4.9.dev3-linux-aarch64.tar.gz", - "checksum": "SHA-256:5b274bdff2f62e6a07c3c1dfa51b1128924621f661747eca3dbe0f77972f2f06", - "size": "33663882" + "host": "arm-linux-gnueabihf", + "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-linux-armv7.tar.gz", + "archiveFileName": "esptool-v5.0.dev1-linux-armv7.tar.gz", + "checksum": "SHA-256:ea77a38681506761bbb7b0b39c130811ed565667b67ebbdb4d6dcc6cb6e07368", + "size": "53451939" }, { "host": "x86_64-apple-darwin", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-macos-amd64.tar.gz", - "archiveFileName": "esptool-v4.9.dev3-macos-amd64.tar.gz", - "checksum": "SHA-256:c733c83b58fcf5f642fbb2fddb8ff24640c2c785126cba0821fb70c4a5ceea7a", - "size": "32767836" + "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-macos-amd64.tar.gz", + "archiveFileName": "esptool-v5.0.dev1-macos-amd64.tar.gz", + "checksum": "SHA-256:900a8e90731208bee96647e0e207a43612b9452c2120c4fdc0ff4c6be226257b", + "size": "59631998" }, { "host": "arm64-apple-darwin", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-macos-arm64.tar.gz", - "archiveFileName": "esptool-v4.9.dev3-macos-arm64.tar.gz", - "checksum": "SHA-256:83c195a15981e6a5e7a130db2ccfb21e2d8093912e5b003681f9a5abadd71af7", - "size": "30121441" + "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-macos-arm64.tar.gz", + "archiveFileName": "esptool-v5.0.dev1-macos-arm64.tar.gz", + "checksum": "SHA-256:3653f4de73cb4fc6a25351eaf663708e91c65ae3265d75bd54ca4315a4350bb4", + "size": "56349992" }, { - "host": "i686-mingw32", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-win64.zip", - "archiveFileName": "esptool-v4.9.dev3-win64.zip", - "checksum": "SHA-256:890051a4fdc684ff6f4af18d0bb27d274ca940ee0eef716a9455f8c64b25b215", - "size": "36072564" + "host": "x86_64-mingw32", + "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-win64.zip", + "archiveFileName": "esptool-v5.0.dev1-win64.zip", + "checksum": "SHA-256:1e8fd89645daf94f2d4406ec73c9004e617ea921079515f9fd749205eece4d6d", + "size": "59102658" }, { - "host": "x86_64-mingw32", - "url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-win64.zip", - "archiveFileName": "esptool-v4.9.dev3-win64.zip", - "checksum": "SHA-256:890051a4fdc684ff6f4af18d0bb27d274ca940ee0eef716a9455f8c64b25b215", - "size": "36072564" + "host": "i686-mingw32", + "url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-win64.zip", + "archiveFileName": "esptool-v5.0.dev1-win64.zip", + "checksum": "SHA-256:1e8fd89645daf94f2d4406ec73c9004e617ea921079515f9fd749205eece4d6d", + "size": "59102658" } ] }, diff --git a/platform.txt b/platform.txt index f41a8b4a764..8bfa69a79e0 100644 --- a/platform.txt +++ b/platform.txt @@ -120,7 +120,7 @@ recipe.hooks.prebuild.2.pattern.windows=cmd /c if not exist "{build.path}\partit recipe.hooks.prebuild.3.pattern.windows=cmd /c if not exist "{build.path}\partitions.csv" COPY "{runtime.platform.path}\tools\partitions\{build.partitions}.csv" "{build.path}\partitions.csv" # Check if custom bootloader exist: source > variant > build.boot -recipe.hooks.prebuild.4.pattern_args=--chip {build.mcu} elf2image --flash_mode {build.flash_mode} --flash_freq {build.img_freq} --flash_size {build.flash_size} -o +recipe.hooks.prebuild.4.pattern_args=--chip {build.mcu} elf2image --flash-mode {build.flash_mode} --flash-freq {build.img_freq} --flash-size {build.flash_size} -o recipe.hooks.prebuild.4.pattern=/usr/bin/env bash -c "[ -f "{build.source.path}"/bootloader.bin ] && cp -f "{build.source.path}"/bootloader.bin "{build.path}"/{build.project_name}.bootloader.bin || ( [ -f "{build.variant.path}"/{build.custom_bootloader}.bin ] && cp "{build.variant.path}"/{build.custom_bootloader}.bin "{build.path}"/{build.project_name}.bootloader.bin || "{tools.esptool_py.path}"/{tools.esptool_py.cmd} {recipe.hooks.prebuild.4.pattern_args} "{build.path}"/{build.project_name}.bootloader.bin "{compiler.sdk.path}"/bin/bootloader_{build.boot}_{build.boot_freq}.elf )" recipe.hooks.prebuild.4.pattern.windows=cmd /c IF EXIST "{build.source.path}\bootloader.bin" ( COPY /y "{build.source.path}\bootloader.bin" "{build.path}\{build.project_name}.bootloader.bin" ) ELSE ( IF EXIST "{build.variant.path}\{build.custom_bootloader}.bin" ( COPY "{build.variant.path}\{build.custom_bootloader}.bin" "{build.path}\{build.project_name}.bootloader.bin" ) ELSE ( "{tools.esptool_py.path}\{tools.esptool_py.cmd}" {recipe.hooks.prebuild.4.pattern_args} "{build.path}\{build.project_name}.bootloader.bin" "{compiler.sdk.path}\bin\bootloader_{build.boot}_{build.boot_freq}.elf" ) ) @@ -164,7 +164,7 @@ recipe.c.combine.pattern="{compiler.path}{compiler.c.elf.cmd}" {compiler.c.elf.f recipe.objcopy.partitions.bin.pattern={tools.gen_esp32part.cmd} -q "{build.path}/partitions.csv" "{build.path}/{build.project_name}.partitions.bin" ## Create bin -recipe.objcopy.bin.pattern_args=--chip {build.mcu} elf2image --flash_mode "{build.flash_mode}" --flash_freq "{build.img_freq}" --flash_size "{build.flash_size}" --elf-sha256-offset 0xb0 -o "{build.path}/{build.project_name}.bin" "{build.path}/{build.project_name}.elf" +recipe.objcopy.bin.pattern_args=--chip {build.mcu} elf2image --flash-mode "{build.flash_mode}" --flash-freq "{build.img_freq}" --flash-size "{build.flash_size}" --elf-sha256-offset 0xb0 -o "{build.path}/{build.project_name}.bin" "{build.path}/{build.project_name}.elf" recipe.objcopy.bin.pattern="{tools.esptool_py.path}/{tools.esptool_py.cmd}" {recipe.objcopy.bin.pattern_args} ## Create Insights Firmware Package @@ -177,7 +177,7 @@ recipe.hooks.objcopy.postobjcopy.2.pattern=/usr/bin/env bash -c "[ ! -d "{build. recipe.hooks.objcopy.postobjcopy.2.pattern.windows=cmd /c if exist "{build.path}\libraries\ESP_SR" if exist "{compiler.sdk.path}\esp_sr\srmodels.bin" COPY /y "{compiler.sdk.path}\esp_sr\srmodels.bin" "{build.path}\srmodels.bin" # Create merged binary -recipe.hooks.objcopy.postobjcopy.3.pattern_args=--chip {build.mcu} merge_bin -o "{build.path}/{build.project_name}.merged.bin" --fill-flash-size {build.flash_size} --flash_mode keep --flash_freq keep --flash_size keep {build.bootloader_addr} "{build.path}/{build.project_name}.bootloader.bin" 0x8000 "{build.path}/{build.project_name}.partitions.bin" 0xe000 "{runtime.platform.path}/tools/partitions/boot_app0.bin" 0x10000 "{build.path}/{build.project_name}.bin" +recipe.hooks.objcopy.postobjcopy.3.pattern_args=--chip {build.mcu} merge-bin -o "{build.path}/{build.project_name}.merged.bin" --pad-to-size {build.flash_size} --flash-mode keep --flash-freq keep --flash-size keep {build.bootloader_addr} "{build.path}/{build.project_name}.bootloader.bin" 0x8000 "{build.path}/{build.project_name}.partitions.bin" 0xe000 "{runtime.platform.path}/tools/partitions/boot_app0.bin" 0x10000 "{build.path}/{build.project_name}.bin" recipe.hooks.objcopy.postobjcopy.3.pattern="{tools.esptool_py.path}/{tools.esptool_py.cmd}" {recipe.hooks.objcopy.postobjcopy.3.pattern_args} ## Save bin @@ -294,14 +294,14 @@ debug.additional_config=debug_config.{build.mcu} tools.esptool_py.upload.protocol=serial tools.esptool_py.upload.params.verbose= tools.esptool_py.upload.params.quiet= -tools.esptool_py.upload.pattern_args=--chip {build.mcu} --port "{serial.port}" --baud {upload.speed} {upload.flags} --before default_reset --after hard_reset write_flash {upload.erase_cmd} -z --flash_mode keep --flash_freq keep --flash_size keep {build.bootloader_addr} "{build.path}/{build.project_name}.bootloader.bin" 0x8000 "{build.path}/{build.project_name}.partitions.bin" 0xe000 "{runtime.platform.path}/tools/partitions/boot_app0.bin" 0x10000 "{build.path}/{build.project_name}.bin" {upload.extra_flags} +tools.esptool_py.upload.pattern_args=--chip {build.mcu} --port "{serial.port}" --baud {upload.speed} {upload.flags} --before default-reset --after hard-reset write-flash {upload.erase_cmd} -z --flash-mode keep --flash-freq keep --flash-size keep {build.bootloader_addr} "{build.path}/{build.project_name}.bootloader.bin" 0x8000 "{build.path}/{build.project_name}.partitions.bin" 0xe000 "{runtime.platform.path}/tools/partitions/boot_app0.bin" 0x10000 "{build.path}/{build.project_name}.bin" {upload.extra_flags} tools.esptool_py.upload.pattern="{path}/{cmd}" {upload.pattern_args} ## Program Application ## ------------------- tools.esptool_py.program.params.verbose= tools.esptool_py.program.params.quiet= -tools.esptool_py.program.pattern_args=--chip {build.mcu} --port "{serial.port}" --baud {upload.speed} {upload.flags} --before default_reset --after hard_reset write_flash -z --flash_mode keep --flash_freq keep --flash_size keep 0x10000 "{build.path}/{build.project_name}.bin" +tools.esptool_py.program.pattern_args=--chip {build.mcu} --port "{serial.port}" --baud {upload.speed} {upload.flags} --before default-reset --after hard-reset write-flash -z --flash-mode keep --flash-freq keep --flash-size keep 0x10000 "{build.path}/{build.project_name}.bin" tools.esptool_py.program.pattern="{path}/{cmd}" {program.pattern_args} ## Erase Chip (before burning the bootloader) @@ -309,7 +309,7 @@ tools.esptool_py.program.pattern="{path}/{cmd}" {program.pattern_args} tools.esptool_py.erase.protocol=serial tools.esptool_py.erase.params.verbose= tools.esptool_py.erase.params.quiet= -tools.esptool_py.erase.pattern_args=--chip {build.mcu} --port "{serial.port}" --baud {upload.speed} {upload.flags} --before default_reset --after hard_reset erase_flash +tools.esptool_py.erase.pattern_args=--chip {build.mcu} --port "{serial.port}" --baud {upload.speed} {upload.flags} --before default-reset --after hard-reset erase-flash tools.esptool_py.erase.pattern="{path}/{cmd}" {erase.pattern_args} ## Burn Bootloader From 2e6db3c74a46096b3866e9bc0199cad4fe0ded17 Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Thu, 5 Jun 2025 14:45:13 +0300 Subject: [PATCH 2/4] fix(script): Update script for better handling of esptool --- .github/scripts/package_esptool.sh | 129 ---------------- .github/scripts/update_esptool.py | 236 +++++++++++++++++++++++++++++ 2 files changed, 236 insertions(+), 129 deletions(-) delete mode 100755 .github/scripts/package_esptool.sh create mode 100644 .github/scripts/update_esptool.py diff --git a/.github/scripts/package_esptool.sh b/.github/scripts/package_esptool.sh deleted file mode 100755 index 32b87b277e9..00000000000 --- a/.github/scripts/package_esptool.sh +++ /dev/null @@ -1,129 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Check version argument -if [[ $# -ne 3 ]]; then - echo "Usage: $0 " - echo "Example: $0 5.0.dev1 /tmp/esptool /tmp/esptool-5.0.dev1.json" - exit 1 -fi - -VERSION=$1 -BASE_FOLDER=$2 -JSON_PATH=$3 - -export COPYFILE_DISABLE=1 - -shopt -s nullglob # So for loop doesn't run if no matches - -# Function to update JSON for a given host -function update_json_for_host { - local host=$1 - local archive=$2 - - # Extract the old url from the JSON for this host, then replace only the filename - old_url=$(jq -r --arg host "$host" ' - .packages[].tools[] | select(.name == "esptool_py") | .systems[] | select(.host == $host) | .url // empty - ' "$tmp_json") - if [[ -n "$old_url" ]]; then - base_url="${old_url%/*}" - url="$base_url/$archive" - else - echo "No old url found for $host" - exit 1 - fi - - archiveFileName="$archive" - checksum="SHA-256:$(shasum -a 256 "$archive" | awk '{print $1}')" - size=$(stat -f%z "$archive") - - # Use jq to update the JSON - jq --arg host "$host" \ - --arg url "$url" \ - --arg archiveFileName "$archiveFileName" \ - --arg checksum "$checksum" \ - --arg size "$size" \ - ' - .packages[].tools[] - |= if .name == "esptool_py" then - .systems = ( - ((.systems // []) | map(select(.host != $host))) + [{ - host: $host, - url: $url, - archiveFileName: $archiveFileName, - checksum: $checksum, - size: $size - }] - ) - else - . - end - ' "$tmp_json" > "$tmp_json.new" && mv "$tmp_json.new" "$tmp_json" -} - -cd "$BASE_FOLDER" - -# Delete all archives before starting -rm -f esptool-*.tar.gz esptool-*.zip - -for dir in esptool-*; do - # Check if directory exists and is a directory - if [[ ! -d "$dir" ]]; then - continue - fi - - base="${dir#esptool-}" - - # Add 'linux-' prefix if base doesn't contain linux/macos/win64 - if [[ "$base" != *linux* && "$base" != *macos* && "$base" != *win64* ]]; then - base="linux-${base}" - fi - - if [[ "$dir" == esptool-win* ]]; then - # Windows zip archive - zipfile="esptool-v${VERSION}-${base}.zip" - echo "Creating $zipfile from $dir ..." - zip -r "$zipfile" "$dir" - else - # Non-Windows: set permissions and tar.gz archive - tarfile="esptool-v${VERSION}-${base}.tar.gz" - echo "Setting permissions and creating $tarfile from $dir ..." - chmod -R u=rwx,g=rx,o=rx "$dir" - tar -cvzf "$tarfile" "$dir" - fi -done - -# After the for loop, update the JSON for each archive -# Create a temporary JSON file to accumulate changes -tmp_json="${JSON_PATH}.tmp" -cp "$JSON_PATH" "$tmp_json" - -for archive in esptool-v"${VERSION}"-*.tar.gz esptool-v"${VERSION}"-*.zip; do - [ -f "$archive" ] || continue - - echo "Updating JSON for $archive" - - # Determine host from archive name - case "$archive" in - *linux-amd64*) host="x86_64-pc-linux-gnu" ;; - *linux-armv7*) host="arm-linux-gnueabihf" ;; - *linux-aarch64*) host="aarch64-linux-gnu" ;; - *macos-amd64*) host="x86_64-apple-darwin" ;; - *macos-arm64*) host="arm64-apple-darwin" ;; - *win64*) hosts=("x86_64-mingw32" "i686-mingw32") ;; - *) echo "Unknown host for $archive"; continue ;; - esac - - # For win64, loop over both hosts; otherwise, use a single host - if [[ "$archive" == *win64* ]]; then - for host in "${hosts[@]}"; do - update_json_for_host "$host" "$archive" - done - else - update_json_for_host "$host" "$archive" - fi -done - -# After all archives are processed, move the temporary JSON to the final file -mv "$tmp_json" "$JSON_PATH" diff --git a/.github/scripts/update_esptool.py b/.github/scripts/update_esptool.py new file mode 100644 index 00000000000..8e61925ae5f --- /dev/null +++ b/.github/scripts/update_esptool.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 + +# This script is used to re-package the esptool if needed and update the JSON file +# for the Arduino ESP32 platform. +# +# The script has only been tested on macOS. +# +# For regular esptool releases, the generated packages already contain the correct permissions, +# extensions and are uploaded to the GitHub release assets. In this case, the script will only +# update the JSON file with the information from the GitHub release. +# +# The script can be used in two modes: +# 1. Local build: The build artifacts must be already downloaded and extracted in the base_folder. +# This is useful for esptool versions that are not yet released and that are grabbed from the +# GitHub build artifacts. +# 2. Release build: The script will get the release information from GitHub and update the JSON file. +# This is useful for esptool versions that are already released and that are uploaded to the +# GitHub release assets. +# +# For local build, the artifacts must be already downloaded and extracted in the base_folder +# set with the -l option. +# For example, a base folder "esptool" should contain the following folders extracted directly +# from the GitHub build artifacts: +# esptool/esptool-linux-aarch64 +# esptool/esptool-linux-amd64 +# esptool/esptool-linux-armv7 +# esptool/esptool-macos-amd64 +# esptool/esptool-macos-arm64 +# esptool/esptool-windows-amd64 + +import argparse +import json +import os +import shutil +import stat +import tarfile +import zipfile +import hashlib +import requests +from pathlib import Path + +def compute_sha256(filepath): + sha256 = hashlib.sha256() + with open(filepath, "rb") as f: + for block in iter(lambda: f.read(4096), b""): + sha256.update(block) + return f"SHA-256:{sha256.hexdigest()}" + +def get_file_size(filepath): + return os.path.getsize(filepath) + +def update_json_for_host(tmp_json_path, version, host, url, archiveFileName, checksum, size): + with open(tmp_json_path) as f: + data = json.load(f) + + for pkg in data.get("packages", []): + for tool in pkg.get("tools", []): + if tool.get("name") == "esptool_py": + tool["version"] = version + + if url is None: + # If the URL is not set, we need to find the old URL and update it + for system in tool.get("systems", []): + if system.get("host") == host: + url = system.get("url").replace(system.get("archiveFileName"), archiveFileName) + break + else: + print(f"No old URL found for host {host}. Using empty URL.") + url = "" + + # Preserve existing systems order and update or append the new system + systems = tool.get("systems", []) + system_updated = False + for i, system in enumerate(systems): + if system.get("host") == host: + systems[i] = { + "host": host, + "url": url, + "archiveFileName": archiveFileName, + "checksum": checksum, + "size": str(size), + } + system_updated = True + break + + if not system_updated: + systems.append({ + "host": host, + "url": url, + "archiveFileName": archiveFileName, + "checksum": checksum, + "size": str(size), + }) + tool["systems"] = systems + + with open(tmp_json_path, "w") as f: + json.dump(data, f, indent=2, sort_keys=False, ensure_ascii=False) + f.write("\n") + +def update_tools_dependencies(tmp_json_path, version): + with open(tmp_json_path) as f: + data = json.load(f) + + for pkg in data.get("packages", []): + for platform in pkg.get("platforms", []): + for dep in platform.get("toolsDependencies", []): + if dep.get("name") == "esptool_py": + dep["version"] = version + + with open(tmp_json_path, "w") as f: + json.dump(data, f, indent=2, sort_keys=False, ensure_ascii=False) + f.write("\n") + +def create_archives(version, base_folder): + archive_files = [] + + for dirpath in Path(base_folder).glob("esptool-*"): + if not dirpath.is_dir(): + continue + + base = dirpath.name[len("esptool-"):] + + if "windows" in dirpath.name: + zipfile_name = f"esptool-v{version}-{base}.zip" + print(f"Creating {zipfile_name} from {dirpath} ...") + with zipfile.ZipFile(zipfile_name, "w", zipfile.ZIP_DEFLATED) as zipf: + for root, _, files in os.walk(dirpath): + for file in files: + full_path = os.path.join(root, file) + zipf.write(full_path, os.path.relpath(full_path, start=dirpath)) + archive_files.append(zipfile_name) + else: + tarfile_name = f"esptool-v{version}-{base}.tar.gz" + print(f"Creating {tarfile_name} from {dirpath} ...") + for root, dirs, files in os.walk(dirpath): + for name in dirs + files: + os.chmod(os.path.join(root, name), stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | + stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) + with tarfile.open(tarfile_name, "w:gz") as tar: + tar.add(dirpath, arcname=dirpath.name) + archive_files.append(tarfile_name) + + return archive_files + +def determine_hosts(archive_name): + if "linux-amd64" in archive_name: + return ["x86_64-pc-linux-gnu"] + elif "linux-armv7" in archive_name: + return ["arm-linux-gnueabihf"] + elif "linux-aarch64" in archive_name: + return ["aarch64-linux-gnu"] + elif "macos-amd64" in archive_name: + return ["x86_64-apple-darwin"] + elif "macos-arm64" in archive_name: + return ["arm64-apple-darwin"] + elif "windows-amd64" in archive_name: + return ["x86_64-mingw32", "i686-mingw32"] + else: + return [] + +def update_json_from_local_build(tmp_json_path, version, base_folder, archive_files): + for archive in archive_files: + print(f"Processing archive: {archive}") + hosts = determine_hosts(archive) + if not hosts: + print(f"Skipping unknown archive type: {archive}") + continue + + archive_path = Path(archive) + checksum = compute_sha256(archive_path) + size = get_file_size(archive_path) + + for host in hosts: + update_json_for_host(tmp_json_path, version, host, None, archive_path.name, checksum, size) + +def update_json_from_release(tmp_json_path, version, release_info): + assets = release_info.get("assets", []) + for asset in assets: + if (asset.get("name").endswith(".tar.gz") or asset.get("name").endswith(".zip")) and "esptool" in asset.get("name"): + asset_fname = asset.get("name") + print(f"Processing asset: {asset_fname}") + hosts = determine_hosts(asset_fname) + if not hosts: + print(f"Skipping unknown archive type: {asset_fname}") + continue + + asset_url = asset.get("url") + asset_checksum = asset.get("digest") + asset_size = asset.get("size") + if asset_checksum is None: + asset_checksum = "" + print(f"Asset {asset_fname} has no checksum. Please set the checksum in the JSON file.") + + for host in hosts: + update_json_for_host(tmp_json_path, version, host, asset_url, asset_fname, asset_checksum, asset_size) + +def get_release_info(version): + url = f"https://api.github.com/repos/espressif/esptool/releases/tags/v{version}" + response = requests.get(url) + response.raise_for_status() + return response.json() + +def main(): + parser = argparse.ArgumentParser(description="Repack esptool and update JSON metadata.") + parser.add_argument("version", help="Version of the esptool (e.g. 5.0.dev1)") + parser.add_argument("-l", "--local", dest="base_folder", help="Enable local build mode and set the base folder with unpacked artifacts") + args = parser.parse_args() + + script_dir = Path(__file__).resolve().parent + json_path = (script_dir / "../../package/package_esp32_index.template.json").resolve() + tmp_json_path = Path(str(json_path) + ".tmp") + shutil.copy(json_path, tmp_json_path) + + local_build = args.base_folder is not None + + if local_build: + os.chdir(args.base_folder) + os.environ['COPYFILE_DISABLE'] = 'true' # this disables including resource forks in tar files on macOS + # Clear any existing archive files + for file in Path(args.base_folder).glob("esptool-*.*"): + file.unlink() + archive_files = create_archives(args.version, args.base_folder) + update_json_from_local_build(tmp_json_path, args.version, args.base_folder, archive_files) + else: + release_info = get_release_info(args.version) + update_json_from_release(tmp_json_path, args.version, release_info) + + print(f"Updating esptool version fields to {args.version}") + update_tools_dependencies(tmp_json_path, args.version) + + shutil.move(tmp_json_path, json_path) + print(f"Done. JSON updated at {json_path}") + +if __name__ == "__main__": + main() From 667999e377a012608e1d47fd2b4c226f4b983f83 Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Thu, 5 Jun 2025 14:48:58 +0300 Subject: [PATCH 3/4] fix(script): Get proper download url --- .github/scripts/update_esptool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/update_esptool.py b/.github/scripts/update_esptool.py index 8e61925ae5f..95a0725dc97 100644 --- a/.github/scripts/update_esptool.py +++ b/.github/scripts/update_esptool.py @@ -185,7 +185,7 @@ def update_json_from_release(tmp_json_path, version, release_info): print(f"Skipping unknown archive type: {asset_fname}") continue - asset_url = asset.get("url") + asset_url = asset.get("browser_download_url") asset_checksum = asset.get("digest") asset_size = asset.get("size") if asset_checksum is None: From 2418fdd34296b69d21505a0f6d2fa4af3ef5dd4d Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Thu, 5 Jun 2025 14:50:29 +0300 Subject: [PATCH 4/4] fix(script): Apply copilot suggestions --- .github/scripts/update_esptool.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/scripts/update_esptool.py b/.github/scripts/update_esptool.py index 95a0725dc97..dd5de5526c3 100644 --- a/.github/scripts/update_esptool.py +++ b/.github/scripts/update_esptool.py @@ -185,15 +185,15 @@ def update_json_from_release(tmp_json_path, version, release_info): print(f"Skipping unknown archive type: {asset_fname}") continue - asset_url = asset.get("browser_download_url") - asset_checksum = asset.get("digest") - asset_size = asset.get("size") - if asset_checksum is None: - asset_checksum = "" - print(f"Asset {asset_fname} has no checksum. Please set the checksum in the JSON file.") - - for host in hosts: - update_json_for_host(tmp_json_path, version, host, asset_url, asset_fname, asset_checksum, asset_size) + asset_url = asset.get("browser_download_url") + asset_checksum = asset.get("digest") + asset_size = asset.get("size") + if asset_checksum is None: + asset_checksum = "" + print(f"Asset {asset_fname} has no checksum. Please set the checksum in the JSON file.") + + for host in hosts: + update_json_for_host(tmp_json_path, version, host, asset_url, asset_fname, asset_checksum, asset_size) def get_release_info(version): url = f"https://api.github.com/repos/espressif/esptool/releases/tags/v{version}"