Tools - mkbuildoptglobals refactoring & attempt to fix caching

Optional include for build-dir header to exclude it from core.a when no
build opts are created / used. Preventive reads before writes, too
Plus, missing change to makecorever.py doing read before write
Prefer to use --output-dir in scripts, tests/device *must* be
out-of-tree for caching to work properly (avoid changing core nested dirs)

Assume 'aggressive caching' is normal mode of operation, no need for special treatment
(can be checked by looking at .d files in the build dir)

Even CI uses arduino-cli ~/.cache/arduino/{sketch,core} for building
Allow sketches without globals.h to share core.a, rebuild otherwise
(same sketch cache is retained, core.a & .o's reused)

Drop manual logging in favour of 'import logging'. Arduino-CLI might still need
tweaks in debug mode, though. Output flushing does not always happen promptly.

Allow multiple entries for the same name, which are then merged.
Allow spaces between @ and the signature words.
Allow to use line right after signature.
Syntax warnings show relevant snippet from globals.h

Author: mcspr

.github/workflows/build-host.yml

@@ -1,5 +1,5 @@
 # Run host test suite under valgrind for runtime checking of code.
-# Also, a quick test that the mocking builds work at all
+# Also, a quick test that the mock builds are actually working
 
 name: Build on host OS
 

.github/workflows/build-ide.yml

@@ -9,8 +9,7 @@ permissions:
   contents: read
 
 jobs:
-
-  # Examples are built in parallel to avoid CI total job time limitation
+  # download toolchain and check whether gcc basic .c compilation works
   sanity-check:
     runs-on: ubuntu-latest
     defaults:
@@ -28,6 +27,23 @@ jobs:
       run: |
         bash ./tests/sanity_check.sh
 
+  # verify that any scripts that will or may be used by the builder are working
+  tooling-check:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: false
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.x'
+    - run: |
+        python ./tools/test_mkbuildoptglobals.py --quiet
+
+  # Examples are built in parallel to avoid CI total job time limitation
   build-linux:
     name: Linux - LwIP ${{ matrix.lwip }} (${{ matrix.chunk }})
     runs-on: ubuntu-latest

package/build_boards_manager_package.sh

@@ -117,7 +117,7 @@ $SED -E "s/name=([a-zA-Z0-9\ -]+).*/name=\1(${ver})/g"\
 #echo "#define ARDUINO_ESP8266_GIT_DESC `git describe --tags 2>/dev/null`" >>${outdir}/cores/esp8266/core_version.h
 #echo "#define ARDUINO_ESP8266_RELEASE_${ver_define}" >>${outdir}/cores/esp8266/core_version.h
 #echo "#define ARDUINO_ESP8266_RELEASE \"${ver_define}\"" >>${outdir}/cores/esp8266/core_version.h
-python3 ${srcdir}/tools/makecorever.py -b ${outdir} -i cores/esp8266 -p ${srcdir} -v ${plain_ver} -r
+python3 ${srcdir}/tools/makecorever.py --git-root ${srcdir} --version ${plain_ver} --release ${outdir}/cores/esp8266/core_version.h
 
 # Zip the package
 pushd package/versions/${visiblever}

platform.txt

@@ -66,17 +66,19 @@ build.spiffs_blocksize=
 build.iramfloat=-DFP_IN_IROM
 
 # Fully qualified file names for processing sketch global options
-globals.h.source.fqfn={build.source.path}/{build.project_name}.globals.h
-commonhfile.fqfn={build.core.path}/CommonHFile.h
-build.opt.fqfn={build.path}/core/build.opt
+globals.h.build.source.fqfn={build.source.path}/{build.project_name}.globals.h
+globals.h.build.fqfn={build.path}/{build.project_name}.globals.h
+common.h.fqfn={build.core.path}/CommonHFile.h
+build.opt.fqfn={build.path}/sketch/build.opt
 build.opt.flags="@{build.opt.fqfn}"
 mkbuildoptglobals.extra_flags=
 
 compiler.path={runtime.tools.xtensa-lx106-elf-gcc.path}/bin/
 compiler.sdk.path={runtime.platform.path}/tools/sdk
 
 compiler.libc.path={runtime.platform.path}/tools/sdk/libc/xtensa-lx106-elf
-compiler.cpreprocessor.flags=-D__ets__ -DICACHE_FLASH -U__STRICT_ANSI__ -D_GNU_SOURCE -DESP8266 {build.debug_optim} {build.opt.flags} "-I{compiler.sdk.path}/include" "-I{compiler.sdk.path}/{build.lwip_include}" "-I{compiler.libc.path}/include" "-I{build.path}/core"
+compiler.cpreprocessor.extra_flags=
+compiler.cpreprocessor.flags=-D__ets__ -DICACHE_FLASH -U__STRICT_ANSI__ -D_GNU_SOURCE -DESP8266 {build.debug_optim} {build.opt.flags} "-I{compiler.sdk.path}/include" "-I{compiler.sdk.path}/{build.lwip_include}" "-I{compiler.libc.path}/include" "-I{build.path}/core" {compiler.cpreprocessor.extra_flags}
 
 # support precompiled libraries in IDE v1.8.6+
 compiler.libraries.ldflags=
@@ -123,8 +125,8 @@ recipe.hooks.sketch.prebuild.pattern="{runtime.tools.python3.path}/python3" -I "
 # This is quite a working hack. This form of prebuild hook, while intuitive, is not explicitly documented.
 recipe.hooks.prebuild.1.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.makecorever}" --git-root "{runtime.platform.path}" --version "{version}" "{build.path}/core/core_version.h"
 
-# Handle processing sketch global options
-recipe.hooks.prebuild.2.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.mkbuildoptglobals}" "{runtime.ide.path}" {runtime.ide.version} "{build.path}" "{build.opt.fqfn}" "{globals.h.source.fqfn}" "{commonhfile.fqfn}" {mkbuildoptglobals.extra_flags}
+# Handle core & sketch global options
+recipe.hooks.prebuild.2.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.mkbuildoptglobals}" {mkbuildoptglobals.extra_flags} build --build-opt "{build.opt.fqfn}" --source-sketch-header "{globals.h.build.source.fqfn}" --build-sketch-header "{globals.h.build.fqfn}" --common-header "{common.h.fqfn}"
 
 
 ## Build the app.ld linker file

tests/common.sh

@@ -132,7 +132,7 @@ function build_sketches()
     build_cmd+=${cli_path}
     build_cmd+=" compile"\
 " --warnings=all"\
-" --build-path $build_dir"\
+" --output-path $build_dir"\
 " --fqbn $fqbn"\
 " --libraries $library_path"\
 " --output-dir $build_out"
@@ -311,7 +311,7 @@ function install_core()
     printf "%s\n" \
         "compiler.c.extra_flags=-Wall -Wextra $debug_flags" \
         "compiler.cpp.extra_flags=-Wall -Wextra $debug_flags" \
-        "mkbuildoptglobals.extra_flags=--ci --cache_core" \
+        "recipe.hooks.prebuild.1.pattern=\"{runtime.tools.python3.path}/python3\" -I \"{runtime.tools.makecorever}\" --git-root \"{runtime.platform.path}\" --version \"{version}\" \"{runtime.platform.path}/cores/esp8266/core_version.h\"" \
             > ${core_path}/platform.local.txt
     echo -e "\n----platform.local.txt----"
     cat platform.local.txt

tests/device/Makefile

@@ -2,7 +2,17 @@ SHELL := /bin/bash
 
 ESP8266_CORE_PATH ?= $(shell git rev-parse --show-toplevel)
 
+# arduino-cli core/ & sketch/ build and cache directories
+# by default, share build location with every other sketch
+CACHE_DIR ?= $(HOME)/.cache/arduino
+ifneq ("$(findstring $(ESP8266_CORE_PATH),$(CACHE_DIR))", "")
+$(warning "CACHE_DIR is located in ESP8266_CORE_PATH, core.a caching will be disabled")
+endif
+
+# binaries (compile --output-dir=...) & test output goes in here.
 BUILD_DIR ?= $(PWD)/build
+
+# where to look for BSTest scripts
 BS_DIR ?= $(PWD)/libraries/BSTest
 
 PYTHON ?= python3
@@ -67,7 +77,7 @@ list: showtestlist
 
 all: count tests test_report
 
-$(TEST_LIST): | virtualenv $(TEST_CONFIG) $(BUILD_DIR) $(HARDWARE_DIR)
+$(TEST_LIST): | virtualenv $(TEST_CONFIG) $(CACHE_DIR) $(BUILD_DIR) $(HARDWARE_DIR)
 
 .NOTPARALLEL: $(TEST_LIST)
 
@@ -78,18 +88,21 @@ showtestlist:
 	@printf '%s\n' $(TEST_LIST)
 	@echo "--------------------------------"
 
-$(TEST_LIST): LOCAL_BUILD_DIR=$(BUILD_DIR)/$(notdir $@)
-$(TEST_LIST): LOCAL_DATA_IMG=data.img
+$(TEST_LIST): LOCAL_BUILD_DIR=$(BUILD_DIR)/$(notdir $@)/
+$(TEST_LIST): LOCAL_TEST_RESULT_XML=$(LOCAL_BUILD_DIR)/$(TEST_RESULT_XML)
+$(TEST_LIST): LOCAL_DATA_DIR=$(LOCAL_BUILD_DIR)/data/
+$(TEST_LIST): LOCAL_DATA_IMG=$(LOCAL_BUILD_DIR)/data.img
 
 define build-arduino
-	rm -f $(LOCAL_BUILD_DIR)/build.options.json
-	$(BUILD_TOOL) compile \
-		$(BUILD_FLAGS) \
-		--libraries "$(PWD)/libraries" \
-		--warnings=all \
-		--build-path $(LOCAL_BUILD_DIR) \
-		--fqbn=$(FQBN) \
-		$@
+	export ARDUINO_BUILD_CACHE_PATH="$(CACHE_DIR)"; \
+		$(BUILD_TOOL) config dump; \
+		$(BUILD_TOOL) compile \
+			$(BUILD_FLAGS) \
+			--libraries "$(PWD)/libraries" \
+			--output-dir "$(LOCAL_BUILD_DIR)" \
+			--warnings=all \
+			--fqbn=$(FQBN) \
+			$@
 endef
 
 define build-mock
@@ -98,7 +111,7 @@ define build-mock
 	$(VENV_PYTHON) $(BS_DIR)/runner.py \
 		$(BS_FLAGS) \
 		--name $(basename $(notdir $@)) \
-		--output $(LOCAL_BUILD_DIR)/$(TEST_RESULT_XML) \
+		--output $(LOCAL_TEST_RESULT_XML) \
 		--env-file $(TEST_CONFIG) \
 		$(call mock_script,$@) \
 		executable "$(ESP8266_CORE_PATH)/tests/host/bin/$(@:%.ino=%)" || echo ""`
@@ -111,17 +124,17 @@ define upload-data
 		(cd $(dir $@) && ./make_data.py ) || echo "Filesystem creation skipped"
 	@test -d $(dir $@)/data/ && ( \
 		$(MKFS) \
-			--create $(dir $@)/data/ \
+			--create $(LOCAL_DATA_DIR) \
 			--size 0xFB000 \
 			--block 8192 \
 			--page 256 \
-			$(LOCAL_BUILD_DIR)/$(LOCAL_DATA_IMG) && \
+			$(LOCAL_DATA_IMG) && \
 		$(ESPTOOL) \
 			--chip esp8266 \
 			--port $(UPLOAD_PORT) \
 			--baud $(UPLOAD_BAUD) \
 			--after no_reset \
-			write_flash 0x300000 $(LOCAL_BUILD_DIR)/$(LOCAL_DATA_IMG) ) \
+			write_flash 0x300000 $(LOCAL_DATA_IMG) ) \
 		&& (echo "Uploaded filesystem") \
 		|| (echo "Filesystem upload skipped")
 endef
@@ -149,7 +162,7 @@ define run-test
 	$(VENV_PYTHON) $(BS_DIR)/runner.py \
 		$(BS_FLAGS) \
 		--name $(basename $(notdir $@)) \
-		--output $(LOCAL_BUILD_DIR)/$(TEST_RESULT_XML) \
+		--output $(LOCAL_TEST_RESULT_XML) \
 		--env-file $(TEST_CONFIG) \
 		$(call mock_script,$@) \
 		port $(UPLOAD_PORT) \
@@ -160,6 +173,7 @@ $(TEST_LIST):
 	@echo "--------------------------------"
 	@echo "Running test '$@' of $(words $(TEST_LIST)) tests"
 	mkdir -p $(LOCAL_BUILD_DIR)
+	mkdir -p $(CACHE_DIR)
 ifneq ("$(NO_BUILD)","1")
 	@echo Building $(notdir $@)
 ifeq ("$(MOCK)", "1")
@@ -187,13 +201,17 @@ $(TEST_REPORT_HTML): $(TEST_REPORT_XML) | virtualenv
 test_report: $(TEST_REPORT_HTML)
 	@echo "Test report generated in $(TEST_REPORT_HTML)"
 
+$(CACHE_DIR):
+	@mkdir -p $@
+
 $(BUILD_DIR):
-	@mkdir -p $(BUILD_DIR)
+	@mkdir -p $@
 
 virtualenv:
 	@make -C $(BS_DIR) PYTHON=$(PYTHON) virtualenv
 
 clean:
+	rm -rf $(CACHE_DIR)
 	rm -rf $(BUILD_DIR)
 	rm -rf $(BS_DIR)/virtualenv
 	rm -f $(TEST_REPORT_HTML) $(TEST_REPORT_XML)

tests/device/test_libc/test_libc.ino.globals.h

@@ -1,4 +1,3 @@
 /*@create-file:build.opt@
-
 -fno-builtin
 */

tests/sanity_check.sh

@@ -5,8 +5,9 @@ source "$root/tests/common.sh"
 
 pushd "$root"/tools
 python3 get.py -q
-
+python3 makecorever.py --git-root "$root" "$root/cores/esp8266/core_version.h"
 popd
+
 pushd "$cache_dir"
 
 gcc="$root/tools/xtensa-lx106-elf/bin/xtensa-lx106-elf-gcc"\

tools/makecorever.py

@@ -56,7 +56,6 @@ def check_git(*args: str, cwd: Optional[str]):
 
 
 def generate(
-    out: TextIO,
     *,
     git_root: pathlib.Path,
     hash_length: int = 8,
@@ -118,7 +117,7 @@ def git(*args):
 #define ARDUINO_ESP8266_DEV       1 // development version
 """
 
-    out.write(text)
+    return text
 
 
 if __name__ == "__main__":
@@ -158,20 +157,18 @@ def git(*args):
 
     args = parser.parse_args()
 
-    def select_output(s: str) -> TextIO:
-        if not s:
-            return sys.stdout
+    contents = generate(
+        git_root=args.git_root,
+        hash_length=args.hash_length,
+        release=args.release,
+        version=args.version,
+    )
 
-        out = pathlib.Path(s)
+    if args.output:
+        out = pathlib.Path(args.output)
         out.parent.mkdir(parents=True, exist_ok=True)
 
-        return out.open("w", encoding="utf-8")
-
-    with select_output(args.output) as out:
-        generate(
-            out,
-            git_root=args.git_root,
-            hash_length=args.hash_length,
-            release=args.release,
-            version=args.version,
-        )
+        if not out.exists() or contents != out.read_text(encoding="utf-8"):
+            out.write_text(contents, encoding="utf-8")
+    else:
+        print(contents, file=sys.stdout)