diff --git a/Makefile b/Makefile index 04f2f3a262e2..32f92e672f1a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ #! /usr/bin/make # Extract version from git, or if we're from a zipfile, use dirname -VERSION=$(shell git describe --tags --always --dirty=-modded --abbrev=7 2>/dev/null || pwd | sed -n 's|.*/c\{0,1\}lightning-v\{0,1\}\([0-9a-f.rc\-]*\)$$|v\1|gp') +VERSION=$(shell git describe --tags --always --dirty=-modded --abbrev=7 2>/dev/null || pwd | $(SED) -n 's|.*/c\{0,1\}lightning-v\{0,1\}\([0-9a-f.rc\-]*\)$$|v\1|gp') # Next release. CLN_NEXT_VERSION := v25.12 @@ -318,7 +318,7 @@ else # Git doesn't maintain timestamps, so we only regen if sources actually changed: # We place the SHA inside some generated files so we can tell if they need updating. # Usage: $(call SHA256STAMP_CHANGED) -SHA256STAMP_CHANGED = [ x"`sed -n 's/.*SHA256STAMP:\([a-f0-9]*\).*/\1/p' $@ 2>/dev/null`" != x"`cat $(sort $(filter-out FORCE,$^)) | $(SHA256SUM) | cut -c1-64`" ] +SHA256STAMP_CHANGED = [ x"`$(SED) -n 's/.*SHA256STAMP:\([a-f0-9]*\).*/\1/p' $@ 2>/dev/null`" != x"`cat $(sort $(filter-out FORCE,$^)) | $(SHA256SUM) | cut -c1-64`" ] # Usage: $(call SHA256STAMP,commentprefix,commentpostfix) SHA256STAMP = echo "$(1) SHA256STAMP:"`cat $(sort $(filter-out FORCE,$^)) | $(SHA256SUM) | cut -c1-64`"$(2)" >> $@ endif @@ -326,22 +326,22 @@ endif # generate-wire.py --page [header|impl] hdrfilename wirename < csv > file %_wiregen.h: %_wire.csv $(WIRE_GEN_DEPS) @if $(call SHA256STAMP_CHANGED); then \ - $(call VERBOSE,"wiregen $@",tools/generate-wire.py --page header $($@_args) $@ `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \ + $(call VERBOSE,"wiregen $@",tools/generate-wire.py --page header $($@_args) $@ `basename $< .csv | $(SED) 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \ fi %_wiregen.c: %_wire.csv $(WIRE_GEN_DEPS) @if $(call SHA256STAMP_CHANGED); then \ - $(call VERBOSE,"wiregen $@",tools/generate-wire.py --page impl $($@_args) ${@:.c=.h} `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \ + $(call VERBOSE,"wiregen $@",tools/generate-wire.py --page impl $($@_args) ${@:.c=.h} `basename $< .csv | $(SED) 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \ fi %_printgen.h: %_wire.csv $(WIRE_GEN_DEPS) @if $(call SHA256STAMP_CHANGED); then \ - $(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page header $($@_args) $@ `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \ + $(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page header $($@_args) $@ `basename $< .csv | $(SED) 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \ fi %_printgen.c: %_wire.csv $(WIRE_GEN_DEPS) @if $(call SHA256STAMP_CHANGED); then \ - $(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page impl $($@_args) ${@:.c=.h} `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \ + $(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page impl $($@_args) ${@:.c=.h} `basename $< .csv | $(SED) 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \ fi RUST_PROFILE ?= debug @@ -411,7 +411,7 @@ ALL_TEST_GEN += $(GRPC_GEN) $(GRPC_GEN) &: cln-grpc/proto/node.proto cln-grpc/proto/primitives.proto $(PYTHON) -m grpc_tools.protoc -I cln-grpc/proto cln-grpc/proto/node.proto --python_out=$(GRPC_PATH)/ --grpc_python_out=$(GRPC_PATH)/ --experimental_allow_proto3_optional $(PYTHON) -m grpc_tools.protoc -I cln-grpc/proto cln-grpc/proto/primitives.proto --python_out=$(GRPC_PATH)/ --experimental_allow_proto3_optional - find $(GRPC_DIR)/ -type f -name "*.py" -print0 | xargs -0 sed -i'.bak' -e 's/^import \(.*\)_pb2 as .*__pb2/from pyln.grpc import \1_pb2 as \1__pb2/g' + find $(GRPC_DIR)/ -type f -name "*.py" -print0 | xargs -0 $(SED) -i'.bak' -e 's/^import \(.*\)_pb2 as .*__pb2/from pyln.grpc import \1_pb2 as \1__pb2/g' find $(GRPC_DIR)/ -type f -name "*.py.bak" -print0 | xargs -0 rm -f # We make pretty much everything depend on these. @@ -444,7 +444,7 @@ mkdocs.yml: $(MANPAGES:=.md) @$(call VERBOSE, "genidx $@", \ find doc -maxdepth 1 -name '*\.[0-9]\.md' | \ cut -b 5- | LC_ALL=C sort | \ - sed 's/\(.*\)\.\(.*\).*\.md/- "\1": "\1.\2.md"/' | \ + $(SED) 's/\(.*\)\.\(.*\).*\.md/- "\1": "\1.\2.md"/' | \ $(PYTHON) devtools/blockreplace.py mkdocs.yml manpages --language=yml --indent " " \ ) @@ -559,7 +559,7 @@ check-python-flake8: @# E731 do not assign a lambda expression, use a def @# W503: line break before binary operator @# E741: ambiguous variable name - @uv run flake8 --ignore=E501,E731,E741,W503,F541,E275 --exclude $(shell echo ${PYTHON_GENERATED} | sed 's/ \+/,/g') ${PYSRC} + @uv run flake8 --ignore=E501,E731,E741,W503,F541,E275 --exclude $(shell echo ${PYTHON_GENERATED} | $(SED) 's/ \+/,/g') ${PYSRC} check-pytest-pyln-proto: PATH=$(PYLN_PATH) PYTHONPATH=$(MY_CHECK_PYTHONPATH) uv run $(PYTEST) contrib/pyln-proto/tests/ @@ -717,7 +717,7 @@ $(ALL_TEST_PROGRAMS:=.o): $(ALL_GEN_SOURCES) update-ccan: mv ccan ccan.old - DIR=$$(pwd)/ccan; cd ../ccan && ./tools/create-ccan-tree -a $$DIR `cd $$DIR.old/ccan && find * -name _info | sed s,/_info,, | $(SORT)` $(CCAN_NEW) + DIR=$$(pwd)/ccan; cd ../ccan && ./tools/create-ccan-tree -a $$DIR `cd $$DIR.old/ccan && find * -name _info | $(SED) s,/_info,, | $(SORT)` $(CCAN_NEW) mkdir -p ccan/tools/configurator cp ../ccan/tools/configurator/configurator.c ../ccan/doc/configurator.1 ccan/tools/configurator/ $(MAKE) ccan/config.h @@ -772,8 +772,8 @@ update-pyln-versions: $(PYLNS:%=update-pyln-version-%) update-pyln-version-%: @if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi @echo "Updating contrib/pyln-$* to $(NEW_VERSION)" - @sed -i.bak 's/^version = .*/version = "$(NEW_VERSION)"/' contrib/pyln-$*/pyproject.toml && rm contrib/pyln-$*/pyproject.toml.bak - @sed -i.bak 's/^__version__ = .*/__version__ = "$(NEW_VERSION)"/' contrib/pyln-$*/pyln/$*/__init__.py && rm contrib/pyln-$*/pyln/$*/__init__.py.bak + @$(SED) -i.bak 's/^version = .*/version = "$(NEW_VERSION)"/' contrib/pyln-$*/pyproject.toml && rm contrib/pyln-$*/pyproject.toml.bak + @$(SED) -i.bak 's/^__version__ = .*/__version__ = "$(NEW_VERSION)"/' contrib/pyln-$*/pyln/$*/__init__.py && rm contrib/pyln-$*/pyln/$*/__init__.py.bak pyln-release: $(PYLNS:%=pyln-release-%) @@ -805,7 +805,7 @@ update-lock: update-reckless-version: @if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi @echo "Updating tools/reckless to $(NEW_VERSION)" - @sed -i.bak "s/__VERSION__ = '.*'/__VERSION__ = '$(NEW_VERSION)'/" tools/reckless && rm tools/reckless.bak + @$(SED) -i.bak "s/__VERSION__ = '.*'/__VERSION__ = '$(NEW_VERSION)'/" tools/reckless && rm tools/reckless.bak update-dot-version: @if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi @@ -816,7 +816,7 @@ update-mocks: $(ALL_TEST_PROGRAMS:%=update-mocks/%.c) $(ALL_TEST_PROGRAMS:%=update-mocks/%.c): $(ALL_GEN_HEADERS) $(EXTERNAL_LIBS) libccan.a ccan/ccan/cdump/tools/cdump-enumstr config.vars update-mocks/%: % $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES) - @MAKE=$(MAKE) tools/update-mocks.sh "$*" $(SUPPRESS_OUTPUT) + @MAKE=$(MAKE) SED=$(SED) tools/update-mocks.sh "$*" $(SUPPRESS_OUTPUT) unittest/%: % bolt-precheck BOLTDIR=$(LOCAL_BOLTDIR) $(VG) $(VG_TEST_ARGS) $* > /dev/null diff --git a/configure b/configure index 7375b3d4f912..10ab1762932b 100755 --- a/configure +++ b/configure @@ -82,6 +82,24 @@ default_python() done } +# We want GNU sed, which, among other things, recognizes '\t'. +# We don't want to get confused with some random program called gsed, +# so we actually test. +default_sed() +{ + SED_BINS="gsed sed" + for s in $SED_BINS; do + if [ "$(which $s)" != "" ] ; then + if [ "$(printf x | $s 's/x/\t/')" = " " ]; then + echo "$s" + return + fi + fi + done + echo "No valid sed found?" >&2 + exit 1 +} + # Takes PYTHON var default_pytest() { @@ -184,6 +202,7 @@ set_defaults() fi echo CSANFLAGS = $CSANFLAGS PYTHON=${PYTHON-$(default_python)} + SED=${SED-$(default_sed)} PYTEST=${PYTEST-$(default_pytest $PYTHON)} COPTFLAGS=${COPTFLAGS-$(default_coptflags "$DEBUGBUILD")} CONFIGURATOR_CC=${CONFIGURATOR_CC-$CC} @@ -212,6 +231,7 @@ usage() echo " To override compile line for configurator itself" usage_with_default "PYTEST" "$PYTEST" usage_with_default "VALGRIND" "$VALGRIND" + usage_with_default "SED" "$SED" echo "Options include:" usage_with_default "--prefix=" "$PREFIX" @@ -554,6 +574,7 @@ add_var SHA256SUM "$SHA256SUM" add_var FUZZING "$FUZZING" add_var RUST "$RUST" add_var PYTHON "$PYTHON" +add_var SED "$SED" # Hack to avoid sha256 name clash with libwally: will be fixed when that # becomes a standalone shared lib. diff --git a/contrib/plugins/cowsay.sh b/contrib/plugins/cowsay.sh index 1762aee0a152..8b3de14fe97f 100755 --- a/contrib/plugins/cowsay.sh +++ b/contrib/plugins/cowsay.sh @@ -1,5 +1,8 @@ #! /bin/sh +# Set sed using best effort, for MacOS. +if which gsed > /dev/null; then SED="gsed"; else SED="sed"; fi + maybe_cowsay() { cowsay || cat <","description":"Have a cow, man!"}]}}' # Eg. {"jsonrpc":"2.0","id":5,"method":"init","params":{"options":{},"configuration":{"lightning-dir":"/home/rusty/.lightning","rpc-file":"lightning-rpc","startup":false}}}\n\n read -r JSON read -r _ -id=$(echo "$JSON" | sed 's/.*"id" *: *\([^,]*\),.*/\1/') +id=$(echo "$JSON" | $SED 's/.*"id" *: *\([^,]*\),.*/\1/') echo '{"jsonrpc":"2.0","id":'"$id"',"result":{}}' # eg. { "jsonrpc" : "2.0", "method" : "cowsay", "id" : 6, "params" :[ "hello"] } while read -r JSON; do read -r _ - id=$(echo "$JSON" | sed 's/.*"id" *: *\([^,]*\),.*/\1/') - params=$(echo "$JSON" | sed 's/.*"params" *: *//' | tr -d '[{}]"') + id=$(echo "$JSON" | $SED 's/.*"id" *: *\([^,]*\),.*/\1/') + params=$(echo "$JSON" | $SED 's/.*"params" *: *//' | tr -d '[{}]"') echo '{"jsonrpc":"2.0","id":'"$id"',"result":{"format-hint":"simple","cowsay":"' # FIXME: lightning-cli does not unescape \\, so we replace with an L. - printf "%s" "$params" | maybe_cowsay | sed 's/\\/L/g' | sed ':a;N;$!ba;s/\n/\\n/g' | tr '\012' '"' + printf "%s" "$params" | maybe_cowsay | $SED 's/\\/L/g' | $SED ':a;N;$!ba;s/\n/\\n/g' | tr '\012' '"' echo '}}' done diff --git a/doc/Makefile b/doc/Makefile index b8498984e857..c4c2192abfa4 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -226,9 +226,9 @@ doc/.doc_version: version_gen.h @if cmp $@.new $@ >/dev/null 2>&1; then rm -f $@.new; else mv $@.new $@; $(ECHO) Documentation version updated to `cat doc/.doc_version`; fi $(PREFIXED_MANPAGES): doc/lightning-%: doc/%.md tools/md2man.sh doc/.doc_version - @VERSION=`cat doc/.doc_version` tools/md2man.sh $(LOWDOWN) $< + @VERSION=`cat doc/.doc_version` SED=$(SED) tools/md2man.sh $(LOWDOWN) $< $(NON_PREFIXED_MANPAGES): doc/%: doc/%.md tools/md2man.sh doc/.doc_version - @VERSION=`cat doc/.doc_version` tools/md2man.sh $(LOWDOWN) $< + @VERSION=`cat doc/.doc_version` SED=$(SED) tools/md2man.sh $(LOWDOWN) $< doc/protocol-%.svg: test/test_protocol test/test_protocol --svg < test/commits/$*.script > $@ @@ -263,8 +263,8 @@ check: check-manpages # This needs plugins, too. check-manpages: all-programs check-config-docs default-targets - @tools/check-manpage.sh cli/lightning-cli doc/lightning-cli.1.md - @tools/check-manpage.sh "lightningd/lightningd --lightning-dir=/tmp/" doc/lightningd-config.5.md + @SED=$(SED) tools/check-manpage.sh cli/lightning-cli doc/lightning-cli.1.md + @SED=$(SED) tools/check-manpage.sh "lightningd/lightningd --lightning-dir=/tmp/" doc/lightningd-config.5.md @awk '/^$$/ { do { getline } while ($$0 ~ /^( {4,}|\t)/) } /^\s*```/ { do { getline } while ($$0 !~ /^\s*```/) } /^([^`_\\]|`([^`\\]|\\.)*`|\b_|_\b|\\.)*\B_\B/ { print "" ; print "Unescaped underscore at " FILENAME ":" NR ":" ; print ; ret = 1 } ENDFILE { NR = 0 } END { exit ret }' doc/*.[0-9].md # Makes sure that fields mentioned in schema are in man page, and vice versa. @@ -281,7 +281,7 @@ doc/index.rst: $(MARKDOWNPAGES_WITH_EXT) $(NON_PREFIXED_MARKDOWNPAGES_WITH_EXT) for m in $(MARKDOWNPAGES_WITH_EXT) $(NON_PREFIXED_MARKDOWNPAGES_WITH_EXT); do \ base=$$(basename "$$m"); \ echo "$$base" | \ - sed -E 's/^(.*)\.([0-9]+)\.md$$/\1 <\1.\2.md>/; t; s/^(.*)\.md$$/\1 <\1.md>/'; \ + $(SED) -E 's/^(.*)\.([0-9]+)\.md$$/\1 <\1.\2.md>/; t; s/^(.*)\.md$$/\1 <\1.md>/'; \ done | \ LC_ALL=C sort | \ $(PYTHON) devtools/blockreplace.py doc/index.rst manpages --language=rst --indent " " \ diff --git a/plugins/Makefile b/plugins/Makefile index b1d5120b6f5c..96147d8b9041 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -277,9 +277,9 @@ plugins/sql-schema_gen.h: $(SQL_SCHEMA_PARTS) SEP=""; \ echo "\"{"; \ for l in $(SQL_LISTRPCS); do \ - echo "$$SEP\"$$l\":{\"properties\":$$(cat plugins/sql-schema_$${l}_gen.h)}" | sed s/\"/\\\"/g; \ + echo "$$SEP\"$$l\":{\"properties\":$$(cat plugins/sql-schema_$${l}_gen.h)}" | $(SED) s/\"/\\\"/g; \ SEP=","; \ - done | sed "s/\\\"/\\\\\"/g"; \ + done | $(SED) "s/\\\"/\\\\\"/g"; \ echo "}\"") | tr -d " \n" > $@ \ ) diff --git a/tools/check-manpage.sh b/tools/check-manpage.sh index e50be1e9c6ca..e8b245b760a9 100755 --- a/tools/check-manpage.sh +++ b/tools/check-manpage.sh @@ -10,7 +10,7 @@ get_cmd_opts() { # Trim out -- after first one: ensure width sufficient to give desc # on same line, and ignore single-letter prefix e.g. -X|--ex - COLUMNS=1000 $1 --help | sed -n 's/^\(-.|\)\?\(--[^ ]*\)\( \| \).*/\2/p' | while IFS=$'\n' read -r opt; do + COLUMNS=1000 $1 --help | $SED -n 's/^\(-.|\)\?\(--[^ ]*\)\( \| \).*/\2/p' | while IFS=$'\n' read -r opt; do case "$opt" in # We don't document dev options. --dev-*) @@ -46,7 +46,7 @@ if [ -z "$CMD_OPTNAMES" ]; then fi # Now, gather (long) opt names from man page, make sure they match. -MAN_OPTNAMES=$(grep -vi 'deprecated in' "$2" | sed -E -n 's,^\* \*\*(--)?([^*/]*)\*\*(/\*\*-.\*\*)?(=?).*,\2\4,p'| sort) +MAN_OPTNAMES=$(grep -vi 'deprecated in' "$2" | $SED -E -n 's,^\* \*\*(--)?([^*/]*)\*\*(/\*\*-.\*\*)?(=?).*,\2\4,p'| sort) if [ "$CMD_OPTNAMES" != "$MAN_OPTNAMES" ]; then echo "diff of command names vs manpage names": diff --git a/tools/md2man.sh b/tools/md2man.sh index 42bf60c07d80..7a81268e1e39 100755 --- a/tools/md2man.sh +++ b/tools/md2man.sh @@ -29,14 +29,14 @@ TITLELINE="$(head -n1 "$SOURCE")" # Replace lightning-cli with $ lightning-cli but do not replace it if it is preceded with ( # because it is used in the examples to run it in the shell, eg. $(lightning-cli listpeerchannels) # shellcheck disable=SC2016 # These are not variables, shellcheck! -SOURCE=$(tail -n +3 "$SOURCE" | sed -E ' +SOURCE=$(tail -n +3 "$SOURCE" | $SED -E ' :a;N;$!ba; s#(\(lightning-cli)#\x1#ig; s#lightning-cli#$ lightning-cli#g; s#\x1#(lightning-cli#g; ' | # Lowdown requires a blank line before every preformatted text block -sed ' +$SED ' /^$/{:0;N;/\n$/b0};s/^[[:blank:]]*```/\n\0/; /\n[[:blank:]]*```/{:1;n;/^[[:blank:]]*```/!b1} ') diff --git a/tools/mockup.sh b/tools/mockup.sh index e9808504ecdf..c98215c79292 100755 --- a/tools/mockup.sh +++ b/tools/mockup.sh @@ -73,5 +73,5 @@ for SYMBOL; do echo "/* Generated stub for $SYMBOL */" - tail -n "+${LINE}" < "$FILE" | head -n "$NUM" | sed 's/^extern *//' | sed 's/PRINTF_FMT([^)]*)//' | sed 's/NON_NULL_ARGS([^)]*)//' | sed 's/NO_NULL_ARGS//g' | sed 's/NORETURN//g' | sed 's/RETURNS_NONNULL//g' | sed 's/LAST_ARG_NULL//g' | sed 's/WARN_UNUSED_RESULT//g' | sed 's/,/ UNNEEDED,/g' | sed 's/\([a-z0-9A-Z*_]* [a-z0-9A-Z*_]*\));/\1 UNNEEDED);/' | sed "s/;\$/$STUB/" | sed 's/[[:space:]]*$//' + tail -n "+${LINE}" < "$FILE" | head -n "$NUM" | $SED 's/^extern *//' | $SED 's/PRINTF_FMT([^)]*)//' | $SED 's/NON_NULL_ARGS([^)]*)//' | $SED 's/NO_NULL_ARGS//g' | $SED 's/NORETURN//g' | $SED 's/RETURNS_NONNULL//g' | $SED 's/LAST_ARG_NULL//g' | $SED 's/WARN_UNUSED_RESULT//g' | $SED 's/,/ UNNEEDED,/g' | $SED 's/\([a-z0-9A-Z*_]* [a-z0-9A-Z*_]*\));/\1 UNNEEDED);/' | $SED "s/;\$/$STUB/" | $SED 's/[[:space:]]*$//' done