From 39d0dbabb6002cbb999750a28d59aa8c8158a4df Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 14 Oct 2025 09:41:14 +1030 Subject: [PATCH 1/2] configure: set SED to gsed (MacOS et al) or sed (GNU). We use some GNU sed features and it's easier to require gsed than fix them all (and keep noticing when they break). MacOS comes with gsed already! Signed-off-by: Rusty Russell --- configure | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/configure b/configure index 7375b3d4f912..10ab1762932b 100755 --- a/configure +++ b/configure @@ -82,6 +82,24 @@ default_python() done } +# We want GNU sed, which, among other things, recognizes '\t'. +# We don't want to get confused with some random program called gsed, +# so we actually test. +default_sed() +{ + SED_BINS="gsed sed" + for s in $SED_BINS; do + if [ "$(which $s)" != "" ] ; then + if [ "$(printf x | $s 's/x/\t/')" = " " ]; then + echo "$s" + return + fi + fi + done + echo "No valid sed found?" >&2 + exit 1 +} + # Takes PYTHON var default_pytest() { @@ -184,6 +202,7 @@ set_defaults() fi echo CSANFLAGS = $CSANFLAGS PYTHON=${PYTHON-$(default_python)} + SED=${SED-$(default_sed)} PYTEST=${PYTEST-$(default_pytest $PYTHON)} COPTFLAGS=${COPTFLAGS-$(default_coptflags "$DEBUGBUILD")} CONFIGURATOR_CC=${CONFIGURATOR_CC-$CC} @@ -212,6 +231,7 @@ usage() echo " To override compile line for configurator itself" usage_with_default "PYTEST" "$PYTEST" usage_with_default "VALGRIND" "$VALGRIND" + usage_with_default "SED" "$SED" echo "Options include:" usage_with_default "--prefix=" "$PREFIX" @@ -554,6 +574,7 @@ add_var SHA256SUM "$SHA256SUM" add_var FUZZING "$FUZZING" add_var RUST "$RUST" add_var PYTHON "$PYTHON" +add_var SED "$SED" # Hack to avoid sha256 name clash with libwally: will be fixed when that # becomes a standalone shared lib. From 9de0b677013c482b390209f38fe4193b7f77aeaa Mon Sep 17 00:00:00 2001 From: Sangbida Chaudhuri <101164840+sangbida@users.noreply.github.com> Date: Tue, 14 Oct 2025 10:36:43 +1030 Subject: [PATCH 2/2] makefile: use SED from configure in most places. Some simple cases are left alone, but anything called from make uses $SED. --- Makefile | 28 ++++++++++++++-------------- contrib/plugins/cowsay.sh | 13 ++++++++----- doc/Makefile | 10 +++++----- plugins/Makefile | 4 ++-- tools/check-manpage.sh | 4 ++-- tools/md2man.sh | 4 ++-- tools/mockup.sh | 2 +- 7 files changed, 34 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index 04f2f3a262e2..32f92e672f1a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ #! /usr/bin/make # Extract version from git, or if we're from a zipfile, use dirname -VERSION=$(shell git describe --tags --always --dirty=-modded --abbrev=7 2>/dev/null || pwd | sed -n 's|.*/c\{0,1\}lightning-v\{0,1\}\([0-9a-f.rc\-]*\)$$|v\1|gp') +VERSION=$(shell git describe --tags --always --dirty=-modded --abbrev=7 2>/dev/null || pwd | $(SED) -n 's|.*/c\{0,1\}lightning-v\{0,1\}\([0-9a-f.rc\-]*\)$$|v\1|gp') # Next release. CLN_NEXT_VERSION := v25.12 @@ -318,7 +318,7 @@ else # Git doesn't maintain timestamps, so we only regen if sources actually changed: # We place the SHA inside some generated files so we can tell if they need updating. # Usage: $(call SHA256STAMP_CHANGED) -SHA256STAMP_CHANGED = [ x"`sed -n 's/.*SHA256STAMP:\([a-f0-9]*\).*/\1/p' $@ 2>/dev/null`" != x"`cat $(sort $(filter-out FORCE,$^)) | $(SHA256SUM) | cut -c1-64`" ] +SHA256STAMP_CHANGED = [ x"`$(SED) -n 's/.*SHA256STAMP:\([a-f0-9]*\).*/\1/p' $@ 2>/dev/null`" != x"`cat $(sort $(filter-out FORCE,$^)) | $(SHA256SUM) | cut -c1-64`" ] # Usage: $(call SHA256STAMP,commentprefix,commentpostfix) SHA256STAMP = echo "$(1) SHA256STAMP:"`cat $(sort $(filter-out FORCE,$^)) | $(SHA256SUM) | cut -c1-64`"$(2)" >> $@ endif @@ -326,22 +326,22 @@ endif # generate-wire.py --page [header|impl] hdrfilename wirename < csv > file %_wiregen.h: %_wire.csv $(WIRE_GEN_DEPS) @if $(call SHA256STAMP_CHANGED); then \ - $(call VERBOSE,"wiregen $@",tools/generate-wire.py --page header $($@_args) $@ `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \ + $(call VERBOSE,"wiregen $@",tools/generate-wire.py --page header $($@_args) $@ `basename $< .csv | $(SED) 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \ fi %_wiregen.c: %_wire.csv $(WIRE_GEN_DEPS) @if $(call SHA256STAMP_CHANGED); then \ - $(call VERBOSE,"wiregen $@",tools/generate-wire.py --page impl $($@_args) ${@:.c=.h} `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \ + $(call VERBOSE,"wiregen $@",tools/generate-wire.py --page impl $($@_args) ${@:.c=.h} `basename $< .csv | $(SED) 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \ fi %_printgen.h: %_wire.csv $(WIRE_GEN_DEPS) @if $(call SHA256STAMP_CHANGED); then \ - $(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page header $($@_args) $@ `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \ + $(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page header $($@_args) $@ `basename $< .csv | $(SED) 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \ fi %_printgen.c: %_wire.csv $(WIRE_GEN_DEPS) @if $(call SHA256STAMP_CHANGED); then \ - $(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page impl $($@_args) ${@:.c=.h} `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \ + $(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page impl $($@_args) ${@:.c=.h} `basename $< .csv | $(SED) 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \ fi RUST_PROFILE ?= debug @@ -411,7 +411,7 @@ ALL_TEST_GEN += $(GRPC_GEN) $(GRPC_GEN) &: cln-grpc/proto/node.proto cln-grpc/proto/primitives.proto $(PYTHON) -m grpc_tools.protoc -I cln-grpc/proto cln-grpc/proto/node.proto --python_out=$(GRPC_PATH)/ --grpc_python_out=$(GRPC_PATH)/ --experimental_allow_proto3_optional $(PYTHON) -m grpc_tools.protoc -I cln-grpc/proto cln-grpc/proto/primitives.proto --python_out=$(GRPC_PATH)/ --experimental_allow_proto3_optional - find $(GRPC_DIR)/ -type f -name "*.py" -print0 | xargs -0 sed -i'.bak' -e 's/^import \(.*\)_pb2 as .*__pb2/from pyln.grpc import \1_pb2 as \1__pb2/g' + find $(GRPC_DIR)/ -type f -name "*.py" -print0 | xargs -0 $(SED) -i'.bak' -e 's/^import \(.*\)_pb2 as .*__pb2/from pyln.grpc import \1_pb2 as \1__pb2/g' find $(GRPC_DIR)/ -type f -name "*.py.bak" -print0 | xargs -0 rm -f # We make pretty much everything depend on these. @@ -444,7 +444,7 @@ mkdocs.yml: $(MANPAGES:=.md) @$(call VERBOSE, "genidx $@", \ find doc -maxdepth 1 -name '*\.[0-9]\.md' | \ cut -b 5- | LC_ALL=C sort | \ - sed 's/\(.*\)\.\(.*\).*\.md/- "\1": "\1.\2.md"/' | \ + $(SED) 's/\(.*\)\.\(.*\).*\.md/- "\1": "\1.\2.md"/' | \ $(PYTHON) devtools/blockreplace.py mkdocs.yml manpages --language=yml --indent " " \ ) @@ -559,7 +559,7 @@ check-python-flake8: @# E731 do not assign a lambda expression, use a def @# W503: line break before binary operator @# E741: ambiguous variable name - @uv run flake8 --ignore=E501,E731,E741,W503,F541,E275 --exclude $(shell echo ${PYTHON_GENERATED} | sed 's/ \+/,/g') ${PYSRC} + @uv run flake8 --ignore=E501,E731,E741,W503,F541,E275 --exclude $(shell echo ${PYTHON_GENERATED} | $(SED) 's/ \+/,/g') ${PYSRC} check-pytest-pyln-proto: PATH=$(PYLN_PATH) PYTHONPATH=$(MY_CHECK_PYTHONPATH) uv run $(PYTEST) contrib/pyln-proto/tests/ @@ -717,7 +717,7 @@ $(ALL_TEST_PROGRAMS:=.o): $(ALL_GEN_SOURCES) update-ccan: mv ccan ccan.old - DIR=$$(pwd)/ccan; cd ../ccan && ./tools/create-ccan-tree -a $$DIR `cd $$DIR.old/ccan && find * -name _info | sed s,/_info,, | $(SORT)` $(CCAN_NEW) + DIR=$$(pwd)/ccan; cd ../ccan && ./tools/create-ccan-tree -a $$DIR `cd $$DIR.old/ccan && find * -name _info | $(SED) s,/_info,, | $(SORT)` $(CCAN_NEW) mkdir -p ccan/tools/configurator cp ../ccan/tools/configurator/configurator.c ../ccan/doc/configurator.1 ccan/tools/configurator/ $(MAKE) ccan/config.h @@ -772,8 +772,8 @@ update-pyln-versions: $(PYLNS:%=update-pyln-version-%) update-pyln-version-%: @if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi @echo "Updating contrib/pyln-$* to $(NEW_VERSION)" - @sed -i.bak 's/^version = .*/version = "$(NEW_VERSION)"/' contrib/pyln-$*/pyproject.toml && rm contrib/pyln-$*/pyproject.toml.bak - @sed -i.bak 's/^__version__ = .*/__version__ = "$(NEW_VERSION)"/' contrib/pyln-$*/pyln/$*/__init__.py && rm contrib/pyln-$*/pyln/$*/__init__.py.bak + @$(SED) -i.bak 's/^version = .*/version = "$(NEW_VERSION)"/' contrib/pyln-$*/pyproject.toml && rm contrib/pyln-$*/pyproject.toml.bak + @$(SED) -i.bak 's/^__version__ = .*/__version__ = "$(NEW_VERSION)"/' contrib/pyln-$*/pyln/$*/__init__.py && rm contrib/pyln-$*/pyln/$*/__init__.py.bak pyln-release: $(PYLNS:%=pyln-release-%) @@ -805,7 +805,7 @@ update-lock: update-reckless-version: @if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi @echo "Updating tools/reckless to $(NEW_VERSION)" - @sed -i.bak "s/__VERSION__ = '.*'/__VERSION__ = '$(NEW_VERSION)'/" tools/reckless && rm tools/reckless.bak + @$(SED) -i.bak "s/__VERSION__ = '.*'/__VERSION__ = '$(NEW_VERSION)'/" tools/reckless && rm tools/reckless.bak update-dot-version: @if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi @@ -816,7 +816,7 @@ update-mocks: $(ALL_TEST_PROGRAMS:%=update-mocks/%.c) $(ALL_TEST_PROGRAMS:%=update-mocks/%.c): $(ALL_GEN_HEADERS) $(EXTERNAL_LIBS) libccan.a ccan/ccan/cdump/tools/cdump-enumstr config.vars update-mocks/%: % $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES) - @MAKE=$(MAKE) tools/update-mocks.sh "$*" $(SUPPRESS_OUTPUT) + @MAKE=$(MAKE) SED=$(SED) tools/update-mocks.sh "$*" $(SUPPRESS_OUTPUT) unittest/%: % bolt-precheck BOLTDIR=$(LOCAL_BOLTDIR) $(VG) $(VG_TEST_ARGS) $* > /dev/null diff --git a/contrib/plugins/cowsay.sh b/contrib/plugins/cowsay.sh index 1762aee0a152..8b3de14fe97f 100755 --- a/contrib/plugins/cowsay.sh +++ b/contrib/plugins/cowsay.sh @@ -1,5 +1,8 @@ #! /bin/sh +# Set sed using best effort, for MacOS. +if which gsed > /dev/null; then SED="gsed"; else SED="sed"; fi + maybe_cowsay() { cowsay || cat <","description":"Have a cow, man!"}]}}' # Eg. {"jsonrpc":"2.0","id":5,"method":"init","params":{"options":{},"configuration":{"lightning-dir":"/home/rusty/.lightning","rpc-file":"lightning-rpc","startup":false}}}\n\n read -r JSON read -r _ -id=$(echo "$JSON" | sed 's/.*"id" *: *\([^,]*\),.*/\1/') +id=$(echo "$JSON" | $SED 's/.*"id" *: *\([^,]*\),.*/\1/') echo '{"jsonrpc":"2.0","id":'"$id"',"result":{}}' # eg. { "jsonrpc" : "2.0", "method" : "cowsay", "id" : 6, "params" :[ "hello"] } while read -r JSON; do read -r _ - id=$(echo "$JSON" | sed 's/.*"id" *: *\([^,]*\),.*/\1/') - params=$(echo "$JSON" | sed 's/.*"params" *: *//' | tr -d '[{}]"') + id=$(echo "$JSON" | $SED 's/.*"id" *: *\([^,]*\),.*/\1/') + params=$(echo "$JSON" | $SED 's/.*"params" *: *//' | tr -d '[{}]"') echo '{"jsonrpc":"2.0","id":'"$id"',"result":{"format-hint":"simple","cowsay":"' # FIXME: lightning-cli does not unescape \\, so we replace with an L. - printf "%s" "$params" | maybe_cowsay | sed 's/\\/L/g' | sed ':a;N;$!ba;s/\n/\\n/g' | tr '\012' '"' + printf "%s" "$params" | maybe_cowsay | $SED 's/\\/L/g' | $SED ':a;N;$!ba;s/\n/\\n/g' | tr '\012' '"' echo '}}' done diff --git a/doc/Makefile b/doc/Makefile index b8498984e857..c4c2192abfa4 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -226,9 +226,9 @@ doc/.doc_version: version_gen.h @if cmp $@.new $@ >/dev/null 2>&1; then rm -f $@.new; else mv $@.new $@; $(ECHO) Documentation version updated to `cat doc/.doc_version`; fi $(PREFIXED_MANPAGES): doc/lightning-%: doc/%.md tools/md2man.sh doc/.doc_version - @VERSION=`cat doc/.doc_version` tools/md2man.sh $(LOWDOWN) $< + @VERSION=`cat doc/.doc_version` SED=$(SED) tools/md2man.sh $(LOWDOWN) $< $(NON_PREFIXED_MANPAGES): doc/%: doc/%.md tools/md2man.sh doc/.doc_version - @VERSION=`cat doc/.doc_version` tools/md2man.sh $(LOWDOWN) $< + @VERSION=`cat doc/.doc_version` SED=$(SED) tools/md2man.sh $(LOWDOWN) $< doc/protocol-%.svg: test/test_protocol test/test_protocol --svg < test/commits/$*.script > $@ @@ -263,8 +263,8 @@ check: check-manpages # This needs plugins, too. check-manpages: all-programs check-config-docs default-targets - @tools/check-manpage.sh cli/lightning-cli doc/lightning-cli.1.md - @tools/check-manpage.sh "lightningd/lightningd --lightning-dir=/tmp/" doc/lightningd-config.5.md + @SED=$(SED) tools/check-manpage.sh cli/lightning-cli doc/lightning-cli.1.md + @SED=$(SED) tools/check-manpage.sh "lightningd/lightningd --lightning-dir=/tmp/" doc/lightningd-config.5.md @awk '/^$$/ { do { getline } while ($$0 ~ /^( {4,}|\t)/) } /^\s*```/ { do { getline } while ($$0 !~ /^\s*```/) } /^([^`_\\]|`([^`\\]|\\.)*`|\b_|_\b|\\.)*\B_\B/ { print "" ; print "Unescaped underscore at " FILENAME ":" NR ":" ; print ; ret = 1 } ENDFILE { NR = 0 } END { exit ret }' doc/*.[0-9].md # Makes sure that fields mentioned in schema are in man page, and vice versa. @@ -281,7 +281,7 @@ doc/index.rst: $(MARKDOWNPAGES_WITH_EXT) $(NON_PREFIXED_MARKDOWNPAGES_WITH_EXT) for m in $(MARKDOWNPAGES_WITH_EXT) $(NON_PREFIXED_MARKDOWNPAGES_WITH_EXT); do \ base=$$(basename "$$m"); \ echo "$$base" | \ - sed -E 's/^(.*)\.([0-9]+)\.md$$/\1 <\1.\2.md>/; t; s/^(.*)\.md$$/\1 <\1.md>/'; \ + $(SED) -E 's/^(.*)\.([0-9]+)\.md$$/\1 <\1.\2.md>/; t; s/^(.*)\.md$$/\1 <\1.md>/'; \ done | \ LC_ALL=C sort | \ $(PYTHON) devtools/blockreplace.py doc/index.rst manpages --language=rst --indent " " \ diff --git a/plugins/Makefile b/plugins/Makefile index b1d5120b6f5c..96147d8b9041 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -277,9 +277,9 @@ plugins/sql-schema_gen.h: $(SQL_SCHEMA_PARTS) SEP=""; \ echo "\"{"; \ for l in $(SQL_LISTRPCS); do \ - echo "$$SEP\"$$l\":{\"properties\":$$(cat plugins/sql-schema_$${l}_gen.h)}" | sed s/\"/\\\"/g; \ + echo "$$SEP\"$$l\":{\"properties\":$$(cat plugins/sql-schema_$${l}_gen.h)}" | $(SED) s/\"/\\\"/g; \ SEP=","; \ - done | sed "s/\\\"/\\\\\"/g"; \ + done | $(SED) "s/\\\"/\\\\\"/g"; \ echo "}\"") | tr -d " \n" > $@ \ ) diff --git a/tools/check-manpage.sh b/tools/check-manpage.sh index e50be1e9c6ca..e8b245b760a9 100755 --- a/tools/check-manpage.sh +++ b/tools/check-manpage.sh @@ -10,7 +10,7 @@ get_cmd_opts() { # Trim out -- after first one: ensure width sufficient to give desc # on same line, and ignore single-letter prefix e.g. -X|--ex - COLUMNS=1000 $1 --help | sed -n 's/^\(-.|\)\?\(--[^ ]*\)\( \| \).*/\2/p' | while IFS=$'\n' read -r opt; do + COLUMNS=1000 $1 --help | $SED -n 's/^\(-.|\)\?\(--[^ ]*\)\( \| \).*/\2/p' | while IFS=$'\n' read -r opt; do case "$opt" in # We don't document dev options. --dev-*) @@ -46,7 +46,7 @@ if [ -z "$CMD_OPTNAMES" ]; then fi # Now, gather (long) opt names from man page, make sure they match. -MAN_OPTNAMES=$(grep -vi 'deprecated in' "$2" | sed -E -n 's,^\* \*\*(--)?([^*/]*)\*\*(/\*\*-.\*\*)?(=?).*,\2\4,p'| sort) +MAN_OPTNAMES=$(grep -vi 'deprecated in' "$2" | $SED -E -n 's,^\* \*\*(--)?([^*/]*)\*\*(/\*\*-.\*\*)?(=?).*,\2\4,p'| sort) if [ "$CMD_OPTNAMES" != "$MAN_OPTNAMES" ]; then echo "diff of command names vs manpage names": diff --git a/tools/md2man.sh b/tools/md2man.sh index 42bf60c07d80..7a81268e1e39 100755 --- a/tools/md2man.sh +++ b/tools/md2man.sh @@ -29,14 +29,14 @@ TITLELINE="$(head -n1 "$SOURCE")" # Replace lightning-cli with $ lightning-cli but do not replace it if it is preceded with ( # because it is used in the examples to run it in the shell, eg. $(lightning-cli listpeerchannels) # shellcheck disable=SC2016 # These are not variables, shellcheck! -SOURCE=$(tail -n +3 "$SOURCE" | sed -E ' +SOURCE=$(tail -n +3 "$SOURCE" | $SED -E ' :a;N;$!ba; s#(\(lightning-cli)#\x1#ig; s#lightning-cli#$ lightning-cli#g; s#\x1#(lightning-cli#g; ' | # Lowdown requires a blank line before every preformatted text block -sed ' +$SED ' /^$/{:0;N;/\n$/b0};s/^[[:blank:]]*```/\n\0/; /\n[[:blank:]]*```/{:1;n;/^[[:blank:]]*```/!b1} ') diff --git a/tools/mockup.sh b/tools/mockup.sh index e9808504ecdf..c98215c79292 100755 --- a/tools/mockup.sh +++ b/tools/mockup.sh @@ -73,5 +73,5 @@ for SYMBOL; do echo "/* Generated stub for $SYMBOL */" - tail -n "+${LINE}" < "$FILE" | head -n "$NUM" | sed 's/^extern *//' | sed 's/PRINTF_FMT([^)]*)//' | sed 's/NON_NULL_ARGS([^)]*)//' | sed 's/NO_NULL_ARGS//g' | sed 's/NORETURN//g' | sed 's/RETURNS_NONNULL//g' | sed 's/LAST_ARG_NULL//g' | sed 's/WARN_UNUSED_RESULT//g' | sed 's/,/ UNNEEDED,/g' | sed 's/\([a-z0-9A-Z*_]* [a-z0-9A-Z*_]*\));/\1 UNNEEDED);/' | sed "s/;\$/$STUB/" | sed 's/[[:space:]]*$//' + tail -n "+${LINE}" < "$FILE" | head -n "$NUM" | $SED 's/^extern *//' | $SED 's/PRINTF_FMT([^)]*)//' | $SED 's/NON_NULL_ARGS([^)]*)//' | $SED 's/NO_NULL_ARGS//g' | $SED 's/NORETURN//g' | $SED 's/RETURNS_NONNULL//g' | $SED 's/LAST_ARG_NULL//g' | $SED 's/WARN_UNUSED_RESULT//g' | $SED 's/,/ UNNEEDED,/g' | $SED 's/\([a-z0-9A-Z*_]* [a-z0-9A-Z*_]*\));/\1 UNNEEDED);/' | $SED "s/;\$/$STUB/" | $SED 's/[[:space:]]*$//' done