Add pnut-awk #659

Workflow file for this run

	name: run-tests

	on:
	pull_request: # Should trigger on pull requests for all branches
	branches:
	- '**' # Matches all branches

	# Cancel previous runs if a new one is triggered
	concurrency:
	group: "${{ github.ref }}"
	cancel-in-progress: true

	jobs:
	build-without-warnings:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@v2

	- name: Install gcc and clang
	run: \|
	set -e
	sudo apt-get update
	sudo apt-get install -y build-essential clang-14

	- name: Build with warnings
	run: \|
	set -e

	export CFLAGS="-Wall -Werror"
	# Testing with gcc clang in case they have different warnings
	for comp in gcc clang-14; do
	make pnut-sh CC=$comp
	make pnut-awk CC=$comp
	make pnut-exe CC=$comp TARGET=Linux.i386 EXE_ONE_PASS=0
	make pnut-exe CC=$comp TARGET=Linux.i386 EXE_ONE_PASS=1
	make pnut-exe CC=$comp TARGET=Linux.i386 SAFE=1 NICE_UX=1
	make pnut-exe CC=$comp TARGET=Linux.x86_64 EXE_ONE_PASS=0
	make pnut-exe CC=$comp TARGET=Linux.x86_64 EXE_ONE_PASS=1
	make pnut-exe CC=$comp TARGET=Linux.x86_64 SAFE=1 NICE_UX=1
	make pnut-exe CC=$comp TARGET=Darwin.x86_64 EXE_ONE_PASS=0
	make pnut-exe CC=$comp TARGET=Darwin.x86_64 SAFE=1 NICE_UX=1
	done

	catch-bad-whitespace:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@v2

	- name: Check for trailing whitespace and tab characters
	run: \|
	set -e
	check_dir() { # $1 is the file extension, $2 is the directory and $3 the max depth to check
	result=$(find $2 -maxdepth ${3:-1000} -name "$1" -type f -exec grep -l -E " +$" {} \;)
	if [ -n "$result" ]; then
	echo "Trailing whitespace found in the following files:"
	echo "$result"
	exit 1
	fi

	result=$(find $2 -maxdepth ${3:-1000} -name "$1" -type f -exec egrep -l "$(printf '\t')" {} \;)
	if [ -n "$result" ]; then
	echo "Tab characters found in the following files:"
	echo "$result"
	exit 1
	fi
	}

	check_dir "*.c" . 1 # Check pnut source files
	check_dir "*.c" tests # and the tests directory recursively
	check_dir "*.c" examples 1 # and examples

	compile-in-safe-mode:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@v2

	- name: Install dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y coreutils time

	- name: Compile pnut-sh, pnut-exe and tests in safe mode
	run: \|
	set -e
	export SAFE=1

	make pnut-sh.sh
	make pnut-sh.sh MINIMAL=1

	make pnut-awk.awk
	make pnut-awk.awk MINIMAL=1

	for target in Linux.x86_64 Linux.i386; do
	for minimal in 0 1; do
	for one_pass in 0 1; do
	make pnut-exe TARGET=$target MINIMAL=$minimal EXE_ONE_PASS=$one_pass
	done
	done
	done

	./run-tests.sh sh --safe --compile-only
	./run-tests.sh x86_64_linux --safe --compile-only
	./run-tests.sh x86_64_linux --safe --one-pass-generator --compile-only
	./run-tests.sh i386_linux --safe --compile-only
	./run-tests.sh i386_linux --safe --one-pass-generator --compile-only

	tests-exe: # Run tests for pnut-exe on all supported platforms and architectures
	strategy:
	matrix:
	include:
	- target: i386_linux
	host: ubuntu-latest
	- target: x86_64_linux
	host: ubuntu-latest
	- target: x86_64_mac
	host: macos-latest
	runs-on: ${{ matrix.host }}
	steps:
	- name: Checkout code
	uses: actions/checkout@v2

	- name: Install utils
	run: \|
	if [ ${{ matrix.host }} = "macos-latest" ]; then
	brew install coreutils
	else
	sudo apt-get update
	sudo apt-get install -y coreutils time
	fi

	- name: Run ${{ matrix.target }} tests on ${{ matrix.host }}
	run: \|
	set -e
	./run-tests.sh ${{ matrix.target }}
	PNUT_OPTIONS="-D USE_STACK_FOR_GLOBALS" ./run-tests.sh ${{ matrix.target }}
	if [ ${{ matrix.host }} != "macos-latest" ]; then
	PNUT_OPTIONS="-D USE_BRK_SYSCALL" ./run-tests.sh ${{ matrix.target }}
	fi

	- name: Run ${{ matrix.target }} tests on ${{ matrix.host }} with one pass generator
	run: \|
	set -e
	if [ ${{ matrix.host }} = "macos-latest" ]; then
	echo "Skipping one pass generator on macOS"
	else
	./run-tests.sh ${{ matrix.target }} --one-pass-generator
	PNUT_OPTIONS="-D USE_STACK_FOR_GLOBALS" ./run-tests.sh ${{ matrix.target }} --one-pass-generator
	PNUT_OPTIONS="-D USE_BRK_SYSCALL" ./run-tests.sh ${{ matrix.target }} --one-pass-generator
	fi

	tests-shell: # Run tests for pnut-sh on all supported shells
	strategy:
	matrix:
	shell: ["bash", "dash", "ksh", "mksh", "yash", "zsh", "osh"]
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@v2

	- name: Install ${{ matrix.shell }} shell
	run: \|
	set -e
	sudo apt-get update
	if [[ "${{ matrix.shell }}" != "osh" ]]; then
	sudo apt-get install -y coreutils time ${{ matrix.shell }}
	else
	OSH_VERSION="0.33.0"
	sudo apt-get install -y coreutils time build-essential wget
	wget https://oils.pub/download/oils-for-unix-$OSH_VERSION.tar.gz
	tar -xzf oils-for-unix-$OSH_VERSION.tar.gz
	cd oils-for-unix-$OSH_VERSION
	./configure # completes very quickly
	_build/oils.sh # 30-60 seconds
	sudo ./install
	fi

	- name: Run tests with ${{ matrix.shell }}
	run: \|
	set -e
	./run-tests.sh sh --shell ${{ matrix.shell }}

	- name: Run tests with ${{ matrix.shell }} (fast)
	run: \|
	set -e
	./run-tests.sh sh --shell ${{ matrix.shell }} --fast

	bootstrap-pnut-sh:
	strategy:
	matrix:
	shell: ["bash", "dash", "ksh", "mksh", "yash", "zsh", "osh"]
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@v2

	- name: Install ${{ matrix.shell }} shell
	run: \|
	set -e
	sudo apt-get update
	if [[ "${{ matrix.shell }}" != "osh" ]]; then
	sudo apt-get install -y coreutils time ${{ matrix.shell }}
	else
	OSH_VERSION="0.33.0"
	sudo apt-get install -y coreutils time build-essential wget
	wget https://oils.pub/download/oils-for-unix-$OSH_VERSION.tar.gz
	tar -xzf oils-for-unix-$OSH_VERSION.tar.gz
	cd oils-for-unix-$OSH_VERSION
	./configure # completes very quickly
	_build/oils.sh # 30-60 seconds
	sudo ./install
	fi

	- name: Bootstrap pnut-sh.sh on ${{ matrix.shell }}
	run: \|
	set -e
	export BOOTSTRAP_SHELL=${{ matrix.shell }}
	make bootstrap-pnut-sh MINIMAL=1
	make bootstrap-pnut-sh MINIMAL=0

	bootstrap-pnut-awk:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@v2

	- name: Install AWK and dependencies
	run: \|
	set -e
	sudo apt-get update
	sudo apt-get install -y coreutils time gawk mawk

	- name: Bootstrap pnut-awk with AWK
	run: \|
	set -e
	for awk_impl in gawk mawk; do
	export BOOTSTRAP_AWK="$awk_impl --posix"
	# For mawk, only test with MINIMAL=0 because it fails to parse the
	# larger file (hitting some internal limit of mawk?).
	if [ $awk_impl = "mawk" ]; then
	make bootstrap-pnut-awk MINIMAL=1
	else
	make bootstrap-pnut-awk MINIMAL=1
	make bootstrap-pnut-awk MINIMAL=0
	fi
	done

	bootstrap-pnut-exe-on-shell:
	strategy:
	matrix:
	shell: ["bash", "dash", "ksh", "mksh", "yash", "zsh", "osh"]
	target: ["Linux.i386", "Linux.x86_64", "Darwin.x86_64"]
	include:
	- target: Linux.i386
	host: ubuntu-latest
	- target: Linux.x86_64
	host: ubuntu-latest
	- target: Darwin.x86_64
	host: macos-latest
	runs-on: ${{ matrix.host }}
	steps:
	- name: Checkout code
	uses: actions/checkout@v2

	- name: Install ${{ matrix.shell }} shell
	run: \|
	set -e
	SHELL_PACKAGE_NAME=""
	# osh must be built from source, so we handle it separately
	if [[ "${{ matrix.shell }}" != "osh" ]]; then
	SHELL_PACKAGE_NAME=${{ matrix.shell }}
	fi

	if [ ${{ matrix.host }} = "macos-latest" ]; then
	brew install coreutils $SHELL_PACKAGE_NAME
	else
	sudo apt-get update
	sudo apt-get install -y coreutils time build-essential $SHELL_PACKAGE_NAME
	fi

	if [[ "${{ matrix.shell }}" = "osh" ]]; then
	OSH_VERSION="0.33.0"
	wget https://oils.pub/download/oils-for-unix-$OSH_VERSION.tar.gz
	tar -xzf oils-for-unix-$OSH_VERSION.tar.gz
	cd oils-for-unix-$OSH_VERSION
	./configure # completes very quickly
	_build/oils.sh # 30-60 seconds
	sudo ./install
	fi

	- name: Bootstrap pnut-exe with ${{ matrix.target }} backend on ${{ matrix.shell }}
	run: \|
	set -e
	export BOOTSTRAP_SHELL=${{ matrix.shell }}
	export TARGET=${{ matrix.target }}

	do_bootstrap() {
	one_pass=$1
	minimal=$2
	if [ ${{ matrix.host }} = "macos-latest" ] && [ $one_pass -eq 1 ]; then
	# One pass generator is not supported on macOS
	return
	fi
	make bootstrap-pnut-exe EXE_ONE_PASS=$one_pass MINIMAL=$minimal
	make bootstrap-pnut-exe-from-pnut-shell EXE_ONE_PASS=$one_pass MINIMAL=$minimal
	make bootstrap-pnut-exe-from-shell EXE_ONE_PASS=$one_pass MINIMAL=$minimal
	}

	# For fast shells (ksh, dash, bash), test both the one-pass and minimal options
	case "${{ matrix.shell }}" in
	"ksh"\|"dash"\|"bash")
	for minimal in 0 1; do
	for one_pass in 0 1; do
	do_bootstrap $one_pass $minimal
	done
	done
	;;
	*)
	# For slower shells (mksh, yash, zsh, osh), only test the standard
	# bootstrap with the one-pass and minimal options, unless on macOS
	minimal=1
	if [ ${{ matrix.host }} = "macos-latest" ]; then
	one_pass=0
	else
	one_pass=1
	fi

	do_bootstrap $one_pass $minimal
	;;
	esac

	bootstrap-pnut-exe-on-awk:
	strategy:
	matrix:
	target: ["Linux.i386", "Linux.x86_64", "Darwin.x86_64"]
	include:
	- target: Linux.i386
	host: ubuntu-latest
	- target: Linux.x86_64
	host: ubuntu-latest
	- target: Darwin.x86_64
	host: macos-latest
	runs-on: ${{ matrix.host }}
	steps:
	- name: Checkout code
	uses: actions/checkout@v2

	- name: Install AWK and dependencies
	run: \|
	set -e
	if [ ${{ matrix.host }} = "macos-latest" ]; then
	brew install coreutils gawk mawk
	else
	sudo apt-get update
	sudo apt-get install -y coreutils time build-essential gawk mawk
	fi

	- name: Bootstrap pnut-exe with ${{ matrix.target }} backend on AWK
	run: \|
	set -e
	export BOOTSTRAP_AWK=awk
	export TARGET=${{ matrix.target }}

	do_bootstrap() {
	one_pass=$1
	minimal=$2
	if [ ${{ matrix.host }} = "macos-latest" ] && [ $one_pass -eq 1 ]; then
	# One pass generator is not supported on macOS
	return
	fi
	make bootstrap-pnut-exe EXE_ONE_PASS=$one_pass MINIMAL=$minimal
	make bootstrap-pnut-exe-from-pnut-awk EXE_ONE_PASS=$one_pass MINIMAL=$minimal
	make bootstrap-pnut-exe-from-awk EXE_ONE_PASS=$one_pass MINIMAL=$minimal
	}

	for awk_impl in gawk mawk; do
	export BOOTSTRAP_AWK="$awk_impl --posix"
	for minimal in 0 1; do
	if [ $awk_impl = "mawk" ] && [ $minimal -eq 0 ]; then
	# For mawk, only test with MINIMAL=0 because it fails to parse
	# the larger file (hitting some internal limit of mawk?).
	continue
	fi
	for one_pass in 0 1; do
	do_bootstrap $one_pass $minimal
	done
	done
	done

	bootstrap-pnut-sh-awk-with-pnut-exe:
	strategy:
	matrix:
	target: ["Linux.i386", "Linux.x86_64", "Darwin.x86_64"]
	include:
	- target: Linux.i386
	host: ubuntu-latest
	- target: Linux.x86_64
	host: ubuntu-latest
	- target: Darwin.x86_64
	host: macos-latest
	runs-on: ${{ matrix.host }}
	steps:
	- name: Checkout code
	uses: actions/checkout@v2

	- name: Install dependencies
	run: \|
	if [ ${{ matrix.host }} = "macos-latest" ]; then
	brew install coreutils
	else
	sudo apt-get update
	sudo apt-get install -y coreutils time
	fi

	- name: Bootstrap pnut-sh with pnut-exe on ${{ matrix.target }} backend
	run: \|
	set -e
	export TARGET=${{ matrix.target }}
	make bootstrap-pnut-sh-with-pnut-exe MINIMAL=0
	make bootstrap-pnut-sh-with-pnut-exe MINIMAL=1

	make bootstrap-pnut-awk-with-pnut-exe MINIMAL=0
	make bootstrap-pnut-awk-with-pnut-exe MINIMAL=1

	bootstrap-bash-2_05a:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@v2

	- name: Setup woody environment
	run: \|
	set -e
	sudo apt-get update
	sudo apt-get install -y debootstrap # debootstrap allows us to create a minimal Debian environment
	tar -czf ../pnut.tar.gz . # Make a copy of the pnut repo (excluding the woody directory)
	sudo debootstrap --arch=i386 woody woody http://archive.debian.org/debian
	sudo cp ../pnut.tar.gz woody/pnut.tar.gz # Copy the pnut repo into the woody environment
	# Check that the woody environment has the right bash version
	sudo chroot woody /bin/bash -c "bash --version" \| grep 2.05a \|\| { echo "Bash 2.05a is required for this job"; exit 1; }
	sudo chroot woody /bin/bash -c "mkdir pnut && tar -xvzf pnut.tar.gz -C pnut" # Extract the pnut repo into the woody environment
	sudo chroot woody /bin/bash -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential time timeout" # Install gcc and time utils

	- name: Run tests and bootstraps with bash 2.05a
	run: \|
	set -e
	# Unset of undeclared variables is not supported in bash 2.05a
	COMPATIBILITY_OPTIONS="-DRT_FREE_UNSETS_VARS_NOT"
	# The following snippet produces a different result in bash 2.05a
	# __us_buf256="Garfield hates Mondays\n"
	# __temp="${__us_buf256#????????????????}"
	# __us_buf16="${__us_buf256%"$__temp"}"
	# __us_buf256="$__temp"
	COMPATIBILITY_OPTIONS="$COMPATIBILITY_OPTIONS -DSH_OPTIMIZE_LONG_LINES_NOT"

	# Multiline script to run inside the woody chroot
	sudo chroot woody /bin/bash << EOF
	set -e

	cd pnut

	# Default make options for bash 2.05a
	export PNUT_BUILD_OPT="$COMPATIBILITY_OPTIONS"
	export TARGET=Linux.i386
	export BOOTSTRAP_SHELL=bash
	export EXE_ONE_PASS=1
	export CFLAGS=" " # Clear -std=c99 flag which is not supported by woody's gcc

	# Run tests with bash 2.05a
	PNUT_OPTIONS='$COMPATIBILITY_OPTIONS' ./run-tests.sh sh --shell bash

	# Run tests with bash 2.05a (fast)
	# Keeping the fast option on bash 2.05a because its very slow otherwise
	PNUT_OPTIONS='$COMPATIBILITY_OPTIONS' ./run-tests.sh sh --shell bash --fast

	# Bootstrap pnut-exe with pnut-exe with bash 2.05a (fastest step)
	make bootstrap-pnut-exe MINIMAL=1
	make bootstrap-pnut-exe MINIMAL=0

	# Bootstrap pnut-sh.sh with bash 2.05a
	make bootstrap-pnut-sh MINIMAL=1
	make bootstrap-pnut-sh

	# Bootstrap pnut-exe.sh with bash 2.05a
	make bootstrap-pnut-exe-from-pnut-shell MINIMAL=1
	make bootstrap-pnut-exe-from-pnut-shell MINIMAL=0

	# Bootstrap pnut-exe with bash 2.05a
	make bootstrap-pnut-exe-from-shell MINIMAL=1
	make bootstrap-pnut-exe-from-shell MINIMAL=0
	EOF

	pnut-variants-run:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@v2

	- name: Install ksh shell
	run: \|
	set -e
	sudo apt-get update
	sudo apt-get install -y coreutils time ksh dash bash

	- name: Run pnut reader, tokenizer and parser on different shells
	run: \|
	set -e
	variants="reader tokenizer parser"
	for variant in $variants; do
	bash ./run-pnut-variant.sh --${variant} --shell ksh
	bash ./run-pnut-variant.sh --${variant} --shell dash
	bash ./run-pnut-variant.sh --${variant} --shell bash
	done

	compile-with-M2-Planet:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@v2

	- name: Install gcc
	run: \|
	set -e
	sudo apt-get update
	sudo apt-get install -y build-essential

	- name: Install M2-Planet
	run: \|
	set -e
	git clone https://github.com/oriansj/M2-Planet.git M2-Planet --depth 1 --recursive
	# cee2359694740a7c1b5a85a16b0f96f5aa74e1ac is from Nov 5th 2025
	cd M2-Planet
	git checkout cee2359694740a7c1b5a85a16b0f96f5aa74e1ac
	# Build M2-Planet
	make
	# Sanity check
	./bin/M2-Planet --version

	- name: Install mescc-tools
	run: \|
	set -e
	git clone https://github.com/oriansj/mescc-tools.git mescc-tools --depth 1 --recursive
	cd mescc-tools
	# 9b1375115f9175d876c360dbbfd7e231dd9f2a2f is from Oct 9th 2025
	git checkout 9b1375115f9175d876c360dbbfd7e231dd9f2a2f
	# Build mescc-tools
	make
	# Sanity checks
	./bin/M1 --version
	./bin/hex2 --version
	./bin/blood-elf --version
	./bin/get_machine --version

	- name: Compile pnut-exe with M2-Planet
	run: \|
	set -e

	# Use the M2libc from mescc-tools and not the one from M2-Planet
	M2LIB="./M2-Planet/M2libc"

	# Compile to M1 using M2-Planet
	./M2-Planet/bin/M2-Planet --architecture x86 -D target_i386_linux -D NO_TERNARY_SUPPORT -D ONE_PASS_GENERATOR -D SMALL_HEAP -I $M2LIB --expand-includes --debug pnut.c > pnut.M1

	# Generate basic DWARF debug info
	./mescc-tools/bin/blood-elf -f pnut.M1 --little-endian --entry _start -o pnut-footer.M1

	# Compile to hex2 (for debug: -f pnut-footer.M1)
	./mescc-tools/bin/M1 -f $M2LIB/x86/x86_defs.M1 -f $M2LIB/x86/libc-full.M1 -f pnut.M1 -f pnut-footer.M1 --architecture x86 --little-endian -o pnut.hex2

	# Assemble
	./mescc-tools/bin/hex2 --architecture x86 --little-endian --file $M2LIB/x86/ELF-x86-debug.hex2 --file pnut.hex2 --base-address 0x8048000 --output pnut

	# Recompile pnut with itself
	./pnut pnut.c -Dtarget_i386_linux -D ONE_PASS_GENERATOR -o pnut-exe-by-pnut-exe

	# Recompile pnut-exe-by-pnut-exe with itself
	./pnut-exe-by-pnut-exe pnut.c -Dtarget_i386_linux -D ONE_PASS_GENERATOR -o pnut-exe-by-pnut-exe-by-pnut-exe

	# Fixed point?
	wc pnut-exe-by-pnut-exe pnut-exe-by-pnut-exe-by-pnut-exe
	sha256sum pnut-exe-by-pnut-exe pnut-exe-by-pnut-exe-by-pnut-exe
	diff pnut-exe-by-pnut-exe pnut-exe-by-pnut-exe-by-pnut-exe 2>&1 > /dev/null \
	\|\| { echo "Binaries differ after recompilation" ; exit 1; }

	success-message:
	runs-on: ubuntu-latest
	needs: [ build-without-warnings
	, catch-bad-whitespace
	, compile-in-safe-mode
	, tests-exe
	, tests-shell
	, bootstrap-pnut-sh
	, bootstrap-pnut-awk
	, bootstrap-pnut-exe-on-shell
	, bootstrap-pnut-exe-on-awk
	, bootstrap-pnut-sh-awk-with-pnut-exe
	, bootstrap-bash-2_05a
	, pnut-variants-run
	, compile-with-M2-Planet
	]

	steps:
	- name: Tests finished
	run: \|
	echo " ,-~~-.___."
	echo " / \| ' \\ Pnut tests completed...."
	echo "( ) 0"
	echo " \\_/-, ,----'"
	echo " ==== //"
	echo " / \\-'~; /~~~(O)"
	echo " / __/~\| / \|"
	echo "=( _____\| (_________\| "

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add pnut-awk #659

Workflow file

Add pnut-awk #659

Uh oh!

Workflow file for this run