Add OmniRouter: multimodal tools + model collections for agentic workflows #3758

Workflow file for this run

.github/workflows/cpp_server_build_test_release.yml at fe24184

	name: C++ Server Build, Test, and Release 🚀

	on:
	push:
	branches: ["main"]
	tags:
	- v*
	pull_request:
	merge_group:
	workflow_dispatch:
	inputs:
	enable_signing:
	description: 'Enable MSI signing with SignPath (for testing)'
	required: false
	default: false
	type: boolean

	permissions:
	contents: write
	actions: read # Required for SignPath to read workflow/job details

	env:
	LEMONADE_DISABLE_SYSTEMD_JOURNAL: "1"

	jobs:
	# ========================================================================
	# BUILD JOBS - Run on rai-160-sdk workers
	# ========================================================================

	build-lemonade-server-installer:
	name: Build Lemonade Server Installer
	runs-on: windows-latest
	outputs:
	unsigned-artifact-id: ${{ steps.upload-unsigned-msi.outputs.artifact-id }}
	steps:
	- uses: actions/checkout@v5
	with:
	clean: true
	fetch-depth: 0

	- name: Install CMake if not available
	shell: PowerShell
	run: \|
	# Check if CMake is already installed
	$cmakeInstalled = Get-Command cmake -ErrorAction SilentlyContinue

	if (-not $cmakeInstalled) {
	Write-Host "CMake not found, installing..." -ForegroundColor Yellow

	# Download CMake installer
	$cmakeVersion = "3.28.1"
	$cmakeUrl = "https://github.com/Kitware/CMake/releases/download/v$cmakeVersion/cmake-$cmakeVersion-windows-x86_64.msi"
	$cmakeInstaller = "cmake-installer.msi"

	Invoke-WebRequest -Uri $cmakeUrl -OutFile $cmakeInstaller -UseBasicParsing

	# Install CMake silently
	Start-Process msiexec.exe -ArgumentList "/i $cmakeInstaller /quiet /norestart" -Wait

	# Add CMake to PATH for this session AND future steps
	$cmakePath = "C:\Program Files\CMake\bin"
	$env:PATH = "$cmakePath;$env:PATH"

	# Persist to GITHUB_PATH for future steps
	echo $cmakePath >> $env:GITHUB_PATH

	# Verify installation
	cmake --version
	if ($LASTEXITCODE -ne 0) {
	Write-Host "ERROR: CMake installation failed!" -ForegroundColor Red
	exit 1
	}

	Write-Host "CMake installed successfully and added to PATH!" -ForegroundColor Green
	} else {
	Write-Host "CMake is already installed:" -ForegroundColor Green
	cmake --version
	}

	- name: Install WiX Toolset 5.0.2 (CLI)
	shell: PowerShell
	run: \|
	$ErrorActionPreference = "Stop"

	Write-Host "Downloading WiX Toolset 5.0.2 CLI..." -ForegroundColor Cyan
	$wixUri = "https://github.com/wixtoolset/wix/releases/download/v5.0.2/wix-cli-x64.msi"
	$msiPath = "$env:RUNNER_TEMP\wix-cli-x64.msi"

	Invoke-WebRequest -UseBasicParsing -Uri $wixUri -OutFile $msiPath

	Write-Host "Installing WiX Toolset 5.0.2 CLI..." -ForegroundColor Cyan
	$p = Start-Process "msiexec.exe" -ArgumentList @("/i", "`"$msiPath`"", "/qn", "/norestart") -PassThru -Wait
	if ($p.ExitCode -ne 0) {
	Write-Host "WiX installer exited with code $($p.ExitCode)" -ForegroundColor Red
	exit $p.ExitCode
	}

	- name: Verify WiX installation
	shell: PowerShell
	run: \|
	$ErrorActionPreference = "Stop"

	# WiX CLI MSI does not always add itself to PATH in non-interactive installs,
	# so we locate it explicitly and then update PATH for subsequent steps.
	$wixDirs = @(
	"C:\Program Files\WiX Toolset v5.0\bin",
	"C:\Program Files (x86)\WiX Toolset v5.0\bin"
	)

	$wixExe = $null
	foreach ($dir in $wixDirs) {
	if (Test-Path (Join-Path $dir "wix.exe")) {
	$wixExe = Join-Path $dir "wix.exe"
	break
	}
	}

	if (-not $wixExe) {
	Write-Host "ERROR: wix.exe not found after installation." -ForegroundColor Red
	Get-ChildItem -Recurse "C:\Program Files" -Filter wix.exe -ErrorAction SilentlyContinue \| Select-Object -First 20 \| Format-List FullName
	exit 1
	}

	$wixDir = Split-Path $wixExe -Parent

	# Persist wix.exe directory to PATH for all subsequent steps
	"$wixDir" \| Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append

	Write-Host "Using WiX from: $wixExe" -ForegroundColor Green
	& $wixExe --version

	- name: Cache FetchContent dependencies
	uses: actions/cache@v5
	with:
	path: build/_deps
	key: fetchcontent-windows-${{ hashFiles('CMakeLists.txt') }}
	restore-keys: \|
	fetchcontent-windows-

	- name: Run setup.ps1 to configure environment
	shell: PowerShell
	run: \|
	$ErrorActionPreference = "Stop"
	.\setup.ps1

	- name: Build installers
	shell: PowerShell
	run: \|
	$ErrorActionPreference = "Stop"

	Write-Host "Building C++ server, Tauri app, web app, and MSI installers..." -ForegroundColor Cyan

	# Single build command: wix_installers depends on all C++ targets,
	# tauri-app, and web-app via CMake dependency graph
	cmake --build --preset windows --target wix_installers
	if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

	# Verify outputs
	$failures = @()
	@(
	"build\Release\lemond.exe",
	"build\Release\lemonade-server.exe",
	"build\Release\LemonadeServer.exe",
	"build\app\lemonade-app.exe",
	"build\resources\web-app\index.html",
	"lemonade-server-minimal.msi",
	"lemonade.msi"
	) \| ForEach-Object {
	if (-not (Test-Path $_)) {
	Write-Host "ERROR: $_ not found!" -ForegroundColor Red
	$failures += $_
	}
	}

	if ($failures.Count -gt 0) { exit 1 }
	Write-Host "Build and packaging successful!" -ForegroundColor Green

	- name: Upload Lemonade Server Installers
	id: upload-unsigned-msi
	uses: actions/upload-artifact@v7
	with:
	name: Lemonade_Server_MSI
	path: \|
	lemonade-server-minimal.msi
	lemonade.msi
	retention-days: 7

	sign-msi-installers:
	name: Sign MSI Installers with SignPath
	runs-on: windows-latest
	needs: build-lemonade-server-installer
	# Sign on tag pushes (releases) or when manually enabled via workflow_dispatch
	if: startsWith(github.ref, 'refs/tags/v') \|\| inputs.enable_signing == true
	steps:
	- name: Sign MSI Installers with SignPath
	id: sign-msi
	uses: signpath/github-action-submit-signing-request@v2
	with:
	api-token: '${{ secrets.SIGNPATH_API_TOKEN }}'
	organization-id: '8103545b-7814-4edc-86d6-a91dc2a2291b'
	project-slug: 'lemonade'
	signing-policy-slug: 'release-signing'
	github-artifact-id: '${{ needs.build-lemonade-server-installer.outputs.unsigned-artifact-id }}'
	wait-for-completion: true
	wait-for-completion-timeout-in-seconds: 3600
	output-artifact-directory: 'signed-msi'
	parameters: \|
	version: "${{ startsWith(github.ref, 'refs/tags/') && github.ref_name \|\| 'test' }}"

	- name: Verify Signed MSI Files
	shell: PowerShell
	run: \|
	Write-Host "Verifying signed MSI files..." -ForegroundColor Cyan

	if (-not (Test-Path "signed-msi\lemonade-server-minimal.msi")) {
	Write-Host "ERROR: Signed lemonade-server-minimal.msi not found!" -ForegroundColor Red
	exit 1
	}

	if (-not (Test-Path "signed-msi\lemonade.msi")) {
	Write-Host "ERROR: Signed lemonade.msi not found!" -ForegroundColor Red
	exit 1
	}

	Write-Host "Signed MSI files verified!" -ForegroundColor Green
	Get-ChildItem -Path "signed-msi" -Recurse \| Format-Table Name, Length

	- name: Upload Signed MSI Installers
	uses: actions/upload-artifact@v7
	with:
	name: Lemonade_Server_MSI_Signed
	path: \|
	signed-msi/lemonade-server-minimal.msi
	signed-msi/lemonade.msi
	retention-days: 7

	build-lemonade-deb:
	name: Build Lemonade .deb Package
	runs-on: ubuntu-latest
	outputs:
	version: ${{ steps.get_version.outputs.version }}
	container:
	image: ghcr.io/lemonade-sdk/lemonade/build-environment:ubuntu24.04
	credentials:
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}
	steps:
	- uses: actions/checkout@v5
	with:
	clean: true
	fetch-depth: 0

	- name: Configure git safe directory
	run: git config --global --add safe.directory $(pwd)

	- name: Prepare Debian build
	id: get_version
	uses: ./.github/actions/prepare-debian-build
	with:
	release: '24.04'
	codename: 'noble'

	- name: Build Debian package
	id: build_deb
	uses: ./.github/actions/build-debian-package
	with:
	package-type: binary
	deb-version: ${{ steps.get_version.outputs.version }}

	- name: Upload .deb package
	uses: actions/upload-artifact@v7
	with:
	name: lemonade-deb
	path: ${{ steps.build_deb.outputs.output-dir }}/*.deb
	retention-days: 7

	build-lemonade-rpm:
	name: Build Lemonade .rpm Package
	runs-on: ubuntu-latest
	container:
	image: fedora:latest
	outputs:
	version: ${{ steps.get_version.outputs.version }}
	steps:
	- uses: actions/checkout@v5
	with:
	clean: true
	fetch-depth: 0

	- name: Install RPM packaging tools
	shell: bash
	run: \|
	set -e
	dnf install -y rpm-build

	- name: Get version from CMakeLists.txt
	id: get_version
	uses: ./.github/actions/get-version

	- name: Build Linux .rpm package
	shell: bash
	run: \|
	set -e

	echo "Running setup.sh to configure build environment..."
	bash setup.sh

	echo "Building lemond and lemonade-server for Fedora..."
	cmake --build --preset default

	RPM_FILE="lemonade-server-${LEMONADE_VERSION}.x86_64.rpm"

	cd build

	echo "Creating .rpm package with CPack..."
	cpack -G RPM -V

	if [ ! -f "$RPM_FILE" ]; then
	echo "ERROR: .rpm package not created!"
	echo "Contents of build directory:"
	ls -lR .
	exit 1
	fi

	echo "Package information:"
	rpm -qip "$RPM_FILE"

	- name: Upload .rpm package
	uses: actions/upload-artifact@v7
	with:
	name: lemonade-rpm
	path: build/lemonade-server-${{ env.LEMONADE_VERSION }}.x86_64.rpm
	retention-days: 7

	build-lemonade-macos-dmg:
	name: Build Lemonade macOS .dmg (with Tauri App)
	runs-on: macos-latest
	outputs:
	version: ${{ steps.get_version.outputs.version }}
	has_signing: ${{ steps.check_signing.outputs.has_signing }}
	steps:
	- uses: actions/checkout@v5
	with:
	clean: true
	fetch-depth: 0

	- name: Get version from CMakeLists.txt
	id: get_version
	uses: ./.github/actions/get-version

	- name: Check signing secrets
	id: check_signing
	shell: bash
	env:
	APP_CONNECT_KEY: ${{ secrets.MACOS_APP_CONNECT_KEY_GERAMY }}
	run: \|
	if [ -n "$APP_CONNECT_KEY" ]; then
	echo "has_signing=true" >> $GITHUB_OUTPUT
	echo "Signing secrets available - will build signed .pkg"
	else
	echo "has_signing=false" >> $GITHUB_OUTPUT
	echo "No signing secrets - will build and test locally"
	fi

	- name: Setup macOS Keychain
	if: steps.check_signing.outputs.has_signing == 'true'
	uses: ./.github/actions/setup-macos-keychain
	with:
	dev-signing-key: ${{ secrets.MACOS_DEV_SIGNING_IDENTITY_KEY }}
	inst-signing-key: ${{ secrets.MACOS_INST_SIGNING_IDENTITY_KEY }}
	certificate-password: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
	api-key: ${{ secrets.MACOS_APP_CONNECT_KEY_GERAMY }}
	api-key-id: '3WFZZ8F948'
	api-issuer-id: '2e545619-8206-4d14-9ba9-ef23eff841b2'

	- name: Build macOS .dmg
	uses: ./.github/actions/build-macos-dmg
	with:
	include-tauri: 'true'
	skip-packaging: ${{ steps.check_signing.outputs.has_signing != 'true' }}

	- name: Upload .pkg package
	if: steps.check_signing.outputs.has_signing == 'true'
	uses: actions/upload-artifact@v7
	with:
	name: lemonade-macos-pkg
	path: build/*.pkg
	retention-days: 7

	# ---- Unsigned local install + test path ----
	- name: Install binaries locally (unsigned path)
	if: steps.check_signing.outputs.has_signing != 'true'
	shell: bash
	run: \|
	set -e
	echo "Installing binaries locally (no signing)..."

	# Copy binaries to /usr/local/bin
	sudo cp build/lemond /usr/local/bin/
	sudo cp build/lemonade-server /usr/local/bin/
	sudo cp build/lemonade /usr/local/bin/
	sudo chmod 755 /usr/local/bin/lemond
	sudo chmod 755 /usr/local/bin/lemonade-server
	sudo chmod 755 /usr/local/bin/lemonade

	# Copy resources
	sudo mkdir -p "/Library/Application Support/Lemonade/resources"
	if [ -d "build/resources" ]; then
	sudo cp -R build/resources/* "/Library/Application Support/Lemonade/resources/" 2>/dev/null \|\| true
	fi

	# Run the postinst script for directory setup
	echo "Running post-install script..."
	sudo bash src/cpp/postinst-full-mac "" "/"

	# Verify installation
	echo "Verifying installation..."
	/usr/local/bin/lemonade-server --version
	/usr/local/bin/lemond --version
	/usr/local/bin/lemonade --version

	echo "Local installation complete!"

	- name: Verify server is running (unsigned path)
	if: steps.check_signing.outputs.has_signing != 'true'
	shell: bash
	run: \|
	echo "Checking server health..."
	for i in $(seq 1 15); do
	if curl -sf http://localhost:13305/live > /dev/null 2>&1; then
	echo "Server is running and healthy"
	exit 0
	fi
	sleep 2
	done

	# Launchd may not start reliably in CI. Fall back to manual start.
	echo "Server not reachable after 30s — starting manually..."
	/usr/local/bin/lemonade-server serve --no-tray > /tmp/lemonade-server.log 2>&1 &

	for i in $(seq 1 15); do
	if curl -sf http://localhost:13305/live > /dev/null 2>&1; then
	echo "Server is running and healthy (started manually)"
	exit 0
	fi
	echo "Waiting for server... ($i/15)"
	sleep 2
	done
	echo "ERROR: Server did not start within 60 seconds"
	cat /tmp/lemonade-server.log 2>/dev/null \|\| true
	exit 1

	- name: Setup Python and virtual environment (unsigned path)
	if: steps.check_signing.outputs.has_signing != 'true'
	uses: ./.github/actions/setup-venv
	with:
	venv-name: '.venv'
	python-version: '3.10'
	requirements-file: 'test/requirements.txt'

	- name: Run CLI tests (unsigned path)
	if: steps.check_signing.outputs.has_signing != 'true'
	shell: bash
	env:
	LEMONADE_CI_MODE: "True"
	PYTHONIOENCODING: utf-8
	GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
	HF_HOME: ${{ github.workspace }}/hf-cache
	GGML_METAL_NO_RESIDENCY: "1"
	run: \|
	set -e
	echo "Running CLI tests..."
	.venv/bin/python test/server_cli.py --server-binary /usr/local/bin/lemonade-server
	.venv/bin/python test/server_cli2.py --server-binary /usr/local/bin/lemonade-server
	echo "CLI tests PASSED!"

	- name: Run endpoint tests (unsigned path)
	if: steps.check_signing.outputs.has_signing != 'true'
	shell: bash
	env:
	LEMONADE_CI_MODE: "True"
	PYTHONIOENCODING: utf-8
	GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
	HF_HOME: ${{ github.workspace }}/hf-cache
	GGML_METAL_NO_RESIDENCY: "1"
	run: \|
	set -e
	echo "Running endpoint tests..."
	.venv/bin/python test/server_endpoints.py --server-binary /usr/local/bin/lemonade-server
	echo "Endpoint tests PASSED!"

	- name: Run Ollama API tests (unsigned path)
	if: steps.check_signing.outputs.has_signing != 'true'
	shell: bash
	env:
	LEMONADE_CI_MODE: "True"
	PYTHONIOENCODING: utf-8
	GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
	HF_HOME: ${{ github.workspace }}/hf-cache
	GGML_METAL_NO_RESIDENCY: "1"
	run: \|
	set -e
	echo "Running Ollama API tests..."
	.venv/bin/python test/test_ollama.py --server-binary /usr/local/bin/lemonade-server
	echo "Ollama API tests PASSED!"

	- name: Run streaming error tests (unsigned path)
	if: steps.check_signing.outputs.has_signing != 'true'
	shell: bash
	env:
	LEMONADE_CI_MODE: "True"
	PYTHONIOENCODING: utf-8
	GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
	HF_HOME: ${{ github.workspace }}/hf-cache
	GGML_METAL_NO_RESIDENCY: "1"
	run: \|
	set -e
	echo "Running streaming error termination tests..."
	.venv/bin/python test/server_streaming_errors.py --server-binary /usr/local/bin/lemonade-server
	echo "Streaming error tests PASSED!"

	- name: Run environment variable tests (unsigned path)
	if: steps.check_signing.outputs.has_signing != 'true'
	shell: bash
	env:
	LEMONADE_CI_MODE: "True"
	PYTHONIOENCODING: utf-8
	run: \|
	set -e
	echo "Running environment variable tests..."
	.venv/bin/python test/server_env_vars.py --lemond-binary /usr/local/bin/lemond
	echo "Environment variable tests PASSED!"

	- name: Cleanup keychain
	if: always() && steps.check_signing.outputs.has_signing == 'true'
	shell: bash
	run: \|
	if [ -n "$SIGNING_KEYCHAIN_PATH" ] && [ -f "$SIGNING_KEYCHAIN_PATH" ]; then
	echo "Cleaning up temporary keychain..."
	security delete-keychain "$SIGNING_KEYCHAIN_PATH" 2>/dev/null \|\| true
	fi

	build-lemonade-appimage:
	name: Build Lemonade AppImage
	runs-on: ubuntu-latest
	outputs:
	version: ${{ steps.get_version.outputs.version }}
	steps:
	- uses: actions/checkout@v5
	with:
	clean: true
	fetch-depth: 0

	- name: Get version from CMakeLists.txt
	id: get_version
	uses: ./.github/actions/get-version

	- name: Run setup.sh to configure environment
	shell: bash
	run: \|
	set -e
	echo "Running setup.sh to configure build environment..."
	bash setup.sh
	echo "Build environment configured successfully!"

	- name: Build AppImage
	shell: bash
	run: \|
	set -e

	echo "Building Lemonade AppImage..."
	cmake --build --preset default --target appimage

	# Verify AppImage was created
	APPIMAGE_FILE="build/app-appimage/lemonade-app-${LEMONADE_VERSION}-x86_64.AppImage"
	if [ ! -f "$APPIMAGE_FILE" ]; then
	echo "ERROR: AppImage not created!"
	echo "Contents of build/app-appimage directory:"
	ls -lh build/app-appimage/ \|\| echo "Directory does not exist"
	exit 1
	fi

	echo "AppImage created successfully!"
	ls -lh "$APPIMAGE_FILE"

	# Verify it's executable
	chmod +x "$APPIMAGE_FILE"
	"$APPIMAGE_FILE" --appimage-help \| head -5

	- name: Upload AppImage artifact
	uses: actions/upload-artifact@v7
	with:
	name: lemonade-appimage
	path: build/app-appimage/lemonade-app-${{ env.LEMONADE_VERSION }}-x86_64.AppImage
	retention-days: 7

	build-lemonade-embeddable-linux:
	name: Build Embeddable Lemonade (Linux)
	runs-on: ubuntu-latest
	outputs:
	version: ${{ steps.get_version.outputs.version }}
	steps:
	- uses: actions/checkout@v4
	with:
	clean: true
	fetch-depth: 0

	- name: Get version from CMakeLists.txt
	id: get_version
	uses: ./.github/actions/get-version

	- name: Install minimal build dependencies
	run: \|
	set -e
	sudo apt-get update
	sudo apt-get install -y cmake ninja-build g++ pkg-config libssl-dev libdrm-dev

	- name: Build embeddable archive
	shell: bash
	run: \|
	set -e

	# Do not use setup.sh — we intentionally avoid installing system
	# libraries so that FetchContent statically links all deps, making
	# the embeddable binary portable across Ubuntu versions.
	cmake --preset default -DBUILD_WEB_APP=OFF
	cmake --build --preset default --target embeddable

	ARCHIVE="build/lemonade-embeddable-${LEMONADE_VERSION}-ubuntu-x64.tar.gz"
	test -f "$ARCHIVE"
	echo "Archive contents:"
	tar tzf "$ARCHIVE"
	ls -lh "$ARCHIVE"

	- name: Upload embeddable archive
	uses: actions/upload-artifact@v4
	with:
	name: lemonade-embeddable-linux
	path: build/lemonade-embeddable-${{ env.LEMONADE_VERSION }}-ubuntu-x64.tar.gz
	retention-days: 7

	build-lemonade-embeddable-windows:
	name: Build Embeddable Lemonade (Windows)
	runs-on: windows-latest
	outputs:
	version: ${{ steps.get_version.outputs.version }}
	steps:
	- uses: actions/checkout@v4
	with:
	clean: true
	fetch-depth: 0

	- name: Install CMake if not available
	shell: PowerShell
	run: \|
	$cmakeInstalled = Get-Command cmake -ErrorAction SilentlyContinue

	if (-not $cmakeInstalled) {
	Write-Host "CMake not found, installing..." -ForegroundColor Yellow

	$cmakeVersion = "3.28.1"
	$cmakeUrl = "https://github.com/Kitware/CMake/releases/download/v$cmakeVersion/cmake-$cmakeVersion-windows-x86_64.msi"
	$cmakeInstaller = "cmake-installer.msi"

	Invoke-WebRequest -Uri $cmakeUrl -OutFile $cmakeInstaller -UseBasicParsing

	Start-Process msiexec.exe -ArgumentList "/i $cmakeInstaller /quiet /norestart" -Wait

	$cmakePath = "C:\Program Files\CMake\bin"
	$env:PATH = "$cmakePath;$env:PATH"
	echo $cmakePath >> $env:GITHUB_PATH

	cmake --version
	if ($LASTEXITCODE -ne 0) {
	Write-Host "ERROR: CMake installation failed!" -ForegroundColor Red
	exit 1
	}

	Write-Host "CMake installed successfully!" -ForegroundColor Green
	} else {
	Write-Host "CMake is already installed:" -ForegroundColor Green
	cmake --version
	}

	- name: Get version from CMakeLists.txt
	id: get_version
	uses: ./.github/actions/get-version

	- name: Cache FetchContent dependencies
	uses: actions/cache@v4
	with:
	path: build/_deps
	key: fetchcontent-windows-embeddable-${{ hashFiles('CMakeLists.txt') }}
	restore-keys: fetchcontent-windows-embeddable-

	- name: Build embeddable archive
	shell: PowerShell
	run: \|
	$ErrorActionPreference = "Stop"

	cmake --preset windows -DBUILD_WEB_APP=OFF
	if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

	cmake --build --preset windows --target embeddable
	if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

	$archive = "build\lemonade-embeddable-$($env:LEMONADE_VERSION)-windows-x64.zip"
	if (-not (Test-Path $archive)) { throw "Archive not found: $archive" }
	Write-Host "Archive created:" -ForegroundColor Green
	Get-ChildItem $archive

	- name: Upload embeddable archive
	uses: actions/upload-artifact@v4
	with:
	name: lemonade-embeddable-windows
	path: build/lemonade-embeddable-${{ env.LEMONADE_VERSION }}-windows-x64.zip
	retention-days: 7


	# ========================================================================
	# TEST JOBS - Inference tests on self-hosted runners
	# ========================================================================

	test-exe-inference:
	name: Test .exe - ${{ matrix.name }}
	runs-on: ${{ matrix.runner }}
	needs: build-lemonade-server-installer
	# Skip inference tests when signing is enabled (tag pushes or manual workflow_dispatch)
	if: ${{ !startsWith(github.ref, 'refs/tags/') && inputs.enable_signing != true }}
	strategy:
	fail-fast: false
	matrix:
	include:
	- name: llamacpp
	script: server_llm.py
	extra_args: "--wrapped-server llamacpp"
	backends: "vulkan rocm"
	runner: [rai300_400, Windows]
	- name: ryzenai
	script: server_llm.py
	extra_args: "--wrapped-server ryzenai"
	backends: "cpu hybrid npu"
	runner: [rai300_400, Windows]
	- name: flm
	script: server_llm.py
	extra_args: "--wrapped-server flm"
	backends: "npu"
	runner: [rai300_400, Windows]
	- name: whisper
	script: server_whisper.py
	extra_args: "--wrapped-server whispercpp"
	backends: "cpu npu"
	runner: [rai300_400, Windows]
	- name: flm-whisper
	script: server_whisper.py
	extra_args: "--wrapped-server flm"
	backends: "npu"
	runner: [rai300_400, Windows]
	- name: stable-diffusion
	script: server_sd.py
	extra_args: ""
	backends: "cpu"
	runner: [rai300_400, Windows]
	- name: text-to-speech
	script: server_tts.py
	extra_args: ""
	backends: ""
	runner: [rai300_400, Windows]
	- name: stable-diffusion (stx-halo)
	script: server_sd.py
	extra_args: ""
	backends: "rocm"
	runner: [stx-halo, Windows]
	env:
	LEMONADE_CI_MODE: "True"
	LEMONADE_CACHE_DIR: ".\\ci-cache"
	PYTHONIOENCODING: utf-8
	GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
	steps:
	- uses: actions/checkout@v5

	- name: Cleanup processes
	uses: ./.github/actions/cleanup-processes-windows

	- name: Set environment variables
	shell: PowerShell
	run: \|
	$cwd = (Get-Item .\).FullName
	echo "HF_HOME=$cwd\hf-cache" >> $Env:GITHUB_ENV
	echo "LEMONADE_INSTALL_PATH=$cwd\lemonade_server_install" >> $Env:GITHUB_ENV

	- name: Install and Verify Lemonade Server
	uses: ./.github/actions/install-lemonade-server-msi
	with:
	install-path: ${{ env.LEMONADE_INSTALL_PATH }}

	- name: Setup Python and virtual environment
	uses: ./.github/actions/setup-venv
	with:
	venv-name: '.venv'
	python-version: '3.10'
	requirements-file: 'test/requirements.txt'

	- name: Install FLM backend for FLM wrapped-server tests
	if: ${{ runner.os == 'Windows' && contains(matrix.extra_args, '--wrapped-server flm') }}
	shell: PowerShell
	run: \|
	$ErrorActionPreference = "Stop"
	$lemonadeExe = Join-Path $env:LEMONADE_INSTALL_PATH "bin\lemonade.exe"
	Write-Host "Installing FLM backend for CI inference tests..." -ForegroundColor Cyan
	& $lemonadeExe backends install flm:npu
	if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

	- name: Run tests
	shell: PowerShell
	env:
	HF_HOME: ${{ env.HF_HOME }}
	run: \|
	$ErrorActionPreference = "Stop"
	$venvPython = ".\.venv\Scripts\python.exe"
	$serverExe = Join-Path $env:LEMONADE_INSTALL_PATH "bin\lemonade-server.exe"
	$extraArgs = "${{ matrix.extra_args }}" -split " " \| Where-Object { $_ }
	$backends = "${{ matrix.backends }}" -split " " \| Where-Object { $_ }

	if ($backends.Count -eq 0) {
	Write-Host "Running test/${{ matrix.script }} ${{ matrix.extra_args }}" -ForegroundColor Cyan
	& $venvPython test/${{ matrix.script }} @extraArgs --server-binary $serverExe
	if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
	} else {
	foreach ($backend in $backends) {
	Write-Host "Running test/${{ matrix.script }} ${{ matrix.extra_args }} --backend $backend" -ForegroundColor Cyan
	& $venvPython test/${{ matrix.script }} @extraArgs --backend $backend --server-binary $serverExe
	if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
	}
	}

	- name: Capture and upload server logs
	if: always()
	uses: ./.github/actions/capture-server-logs
	with:
	artifact-name: server-logs-exe-${{ matrix.name }}

	- name: Cleanup
	if: always()
	uses: ./.github/actions/cleanup-processes-windows

	test-deb-inference:
	name: Test .deb - ${{ matrix.name }}
	runs-on: [rai300_400, Linux]
	needs: build-lemonade-deb
	# Skip inference tests when signing is enabled (tag pushes or manual workflow_dispatch)
	if: ${{ !startsWith(github.ref, 'refs/tags/') && inputs.enable_signing != true }}
	strategy:
	fail-fast: false
	matrix:
	include:
	- name: llamacpp
	script: server_llm.py
	extra_args: "--wrapped-server llamacpp"
	backends: "vulkan rocm"
	- name: stable-diffusion
	script: server_sd.py
	extra_args: ""
	backends: "cpu rocm"
	- name: whisper
	script: server_whisper.py
	extra_args: "--wrapped-server whispercpp"
	backends: "cpu vulkan"
	- name: flm
	script: server_llm.py
	extra_args: "--wrapped-server flm"
	backends: "npu"
	- name: text-to-speech
	script: server_tts.py
	extra_args: ""
	backends: ""
	env:
	LEMONADE_CI_MODE: "True"
	PYTHONIOENCODING: utf-8
	GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
	LEMONADE_VERSION: ${{ needs.build-lemonade-deb.outputs.version }}
	steps:
	- uses: actions/checkout@v5

	- name: Cleanup processes
	uses: ./.github/actions/cleanup-processes-linux

	- name: Set HF_HOME environment variable
	run: echo "HF_HOME=$PWD/hf-cache" >> $GITHUB_ENV

	- name: Install FLM backend for FLM wrapped-server tests
	if: ${{ contains(matrix.extra_args, '--wrapped-server flm') }}
	run: \|
	set -e
	FLM_VERSION=$(jq -r '.flm.npu' src/cpp/resources/backend_versions.json)
	FLM_VERSION_NUM=$(echo $FLM_VERSION \| sed 's/^v//')
	echo "Installing FLM ${FLM_VERSION} for CI inference tests..."
	curl -L -o /tmp/fastflowlm.deb "https://github.com/FastFlowLM/FastFlowLM/releases/download/${FLM_VERSION}/fastflowlm_${FLM_VERSION_NUM}_ubuntu24.04_amd64.deb"
	mkdir -p /tmp/flm-extract
	dpkg-deb -x /tmp/fastflowlm.deb /tmp/flm-extract
	# Add extracted FLM binary to PATH (Linux uses system PATH for FLM discovery)
	FLM_BIN=$(find /tmp/flm-extract -name flm -type f \| head -1)
	chmod +x "$FLM_BIN"
	echo "$(dirname "$FLM_BIN")" >> $GITHUB_PATH
	export PATH="$(dirname "$FLM_BIN"):$PATH"
	rm /tmp/fastflowlm.deb
	flm version

	- name: Install Lemonade (.deb)
	uses: ./.github/actions/install-lemonade-deb
	with:
	version: ${{ env.LEMONADE_VERSION }}

	- name: Setup Python and virtual environment
	uses: ./.github/actions/setup-venv
	with:
	venv-name: '.venv'
	python-version: '3.10'
	requirements-file: 'test/requirements.txt'

	- name: Run tests
	env:
	HF_HOME: ${{ env.HF_HOME }}
	run: \|
	set -e
	if [ -z "${{ matrix.backends }}" ]; then
	echo "Running test/${{ matrix.script }} ${{ matrix.extra_args }}"
	.venv/bin/python test/${{ matrix.script }} ${{ matrix.extra_args }} --server-binary lemonade-server
	else
	for backend in ${{ matrix.backends }}; do
	echo "Running test/${{ matrix.script }} ${{ matrix.extra_args }} --backend $backend"
	.venv/bin/python test/${{ matrix.script }} ${{ matrix.extra_args }} --backend $backend --server-binary lemonade-server
	done
	fi

	- name: Capture and upload server logs
	if: always()
	uses: ./.github/actions/capture-server-logs
	with:
	artifact-name: server-logs-deb-${{ matrix.name }}

	- name: Cleanup
	if: always()
	uses: ./.github/actions/cleanup-processes-linux

	test-rpm-package:
	name: Test .rpm - Fedora
	runs-on: ubuntu-latest
	needs: build-lemonade-rpm
	container:
	image: fedora:latest
	env:
	LEMONADE_VERSION: ${{ needs.build-lemonade-rpm.outputs.version }}
	steps:
	- name: Download Lemonade .rpm Package
	uses: actions/download-artifact@v7
	with:
	name: lemonade-rpm
	path: .

	- name: Install and verify Lemonade (.rpm)
	shell: bash
	run: \|
	set -e

	RPM_FILE="lemonade-server-${LEMONADE_VERSION}.x86_64.rpm"

	if [ ! -f "$RPM_FILE" ]; then
	echo "ERROR: .rpm file not found: $RPM_FILE"
	ls -la *.rpm 2>/dev/null \|\| echo "No .rpm files found in current directory"
	exit 1
	fi

	dnf install -y shadow-utils "$RPM_FILE"

	echo "Installed package information:"
	rpm -qi lemonade-server

	echo "Installed file list:"
	rpm -ql lemonade-server \| sort

	test -f /opt/bin/lemonade-server
	test -f /opt/bin/lemond

	/opt/bin/lemonade-server --version
	/opt/bin/lemond --version

	test-embeddable-linux:
	name: Test Embeddable (Linux)
	runs-on: ubuntu-latest
	needs: build-lemonade-embeddable-linux
	env:
	LEMONADE_VERSION: ${{ needs.build-lemonade-embeddable-linux.outputs.version }}
	steps:
	- name: Download embeddable archive
	uses: actions/download-artifact@v4
	with:
	name: lemonade-embeddable-linux
	path: .

	- name: Validate archive structure
	shell: bash
	run: \|
	set -e

	ARCHIVE="lemonade-embeddable-${LEMONADE_VERSION}-ubuntu-x64.tar.gz"
	test -f "$ARCHIVE"
	tar xzf "$ARCHIVE"

	DIR="lemonade-embeddable-${LEMONADE_VERSION}-ubuntu-x64"

	# Verify expected files exist
	test -f "$DIR/lemond"
	test -f "$DIR/lemonade"
	test -f "$DIR/LICENSE"
	test -f "$DIR/resources/server_models.json"
	test -f "$DIR/resources/backend_versions.json"
	test -f "$DIR/resources/defaults.json"
	echo "All expected files present"

	# Verify NO forbidden files
	FORBIDDEN=0
	if ls "$DIR"/lemonade-server* 2>/dev/null; then
	echo "ERROR: Found lemonade-server (legacy CLI) in archive"; FORBIDDEN=1
	fi
	if [ -d "$DIR/resources/web-app" ]; then
	echo "ERROR: Found web-app directory in archive"; FORBIDDEN=1
	fi
	if ls "$DIR"/lemonade-app* 2>/dev/null; then
	echo "ERROR: Found desktop app in archive"; FORBIDDEN=1
	fi
	if ls "$DIR"/lemonade-tray* 2>/dev/null; then
	echo "ERROR: Found tray app in archive"; FORBIDDEN=1
	fi
	if ls "$DIR"/LemonadeServer* 2>/dev/null; then
	echo "ERROR: Found LemonadeServer in archive"; FORBIDDEN=1
	fi
	if [ "$FORBIDDEN" -ne 0 ]; then exit 1; fi
	echo "No forbidden files found"

	- name: Test version commands
	shell: bash
	run: \|
	set -e
	DIR="lemonade-embeddable-${LEMONADE_VERSION}-ubuntu-x64"
	"$DIR/lemond" --version
	"$DIR/lemonade" --version

	- name: Test lemond startup and health check
	shell: bash
	run: \|
	set -e
	DIR="lemonade-embeddable-${LEMONADE_VERSION}-ubuntu-x64"
	cd "$DIR"

	./lemond ./ &
	LEMOND_PID=$!

	# Wait for server to become healthy
	for i in $(seq 1 15); do
	if curl -sf http://localhost:13305/api/v1/health > /dev/null 2>&1; then
	echo "Server is healthy!"
	break
	fi
	if [ $i -eq 15 ]; then
	echo "ERROR: Server failed to start within 30 seconds"
	kill $LEMOND_PID 2>/dev/null \|\| true
	exit 1
	fi
	sleep 2
	done

	# Verify health endpoint responds with valid JSON
	HEALTH=$(curl -sf http://localhost:13305/api/v1/health)
	echo "Health response: $HEALTH"

	# Clean shutdown
	kill $LEMOND_PID 2>/dev/null \|\| true
	wait $LEMOND_PID 2>/dev/null \|\| true
	echo "Embeddable Linux smoke test PASSED!"

	test-embeddable-windows:
	name: Test Embeddable (Windows)
	runs-on: windows-latest
	needs: build-lemonade-embeddable-windows
	env:
	LEMONADE_VERSION: ${{ needs.build-lemonade-embeddable-windows.outputs.version }}
	steps:
	- name: Download embeddable archive
	uses: actions/download-artifact@v4
	with:
	name: lemonade-embeddable-windows
	path: .

	- name: Validate archive structure
	shell: PowerShell
	run: \|
	$ErrorActionPreference = "Stop"

	$archiveDir = "lemonade-embeddable-$($env:LEMONADE_VERSION)-windows-x64"
	Expand-Archive -Path "$archiveDir.zip" -DestinationPath .

	# Verify expected files
	$expected = @(
	"$archiveDir\lemond.exe",
	"$archiveDir\lemonade.exe",
	"$archiveDir\LICENSE",
	"$archiveDir\resources\server_models.json",
	"$archiveDir\resources\backend_versions.json",
	"$archiveDir\resources\defaults.json"
	)
	$failures = @()
	foreach ($f in $expected) {
	if (-not (Test-Path $f)) {
	Write-Host "ERROR: $f not found!" -ForegroundColor Red
	$failures += $f
	}
	}
	if ($failures.Count -gt 0) { exit 1 }
	Write-Host "All expected files present" -ForegroundColor Green

	# Verify NO forbidden files
	$forbidden = @(
	"$archiveDir\lemonade-server.exe",
	"$archiveDir\LemonadeServer.exe",
	"$archiveDir\lemonade-app.exe",
	"$archiveDir\lemonade-tray.exe",
	"$archiveDir\resources\web-app"
	)
	foreach ($f in $forbidden) {
	if (Test-Path $f) {
	Write-Host "ERROR: Forbidden file found: $f" -ForegroundColor Red
	exit 1
	}
	}
	Write-Host "No forbidden files found" -ForegroundColor Green

	- name: Test version commands
	shell: PowerShell
	run: \|
	$ErrorActionPreference = "Stop"
	$archiveDir = "lemonade-embeddable-$($env:LEMONADE_VERSION)-windows-x64"
	& "$archiveDir\lemond.exe" --version
	& "$archiveDir\lemonade.exe" --version

	- name: Test lemond startup and health check
	shell: PowerShell
	run: \|
	$ErrorActionPreference = "Stop"
	$archiveDir = "lemonade-embeddable-$($env:LEMONADE_VERSION)-windows-x64"

	Push-Location $archiveDir
	$proc = Start-Process -FilePath ".\lemond.exe" -ArgumentList ".\" -PassThru -NoNewWindow

	# Wait for server to become healthy
	$healthy = $false
	for ($i = 0; $i -lt 15; $i++) {
	try {
	$response = Invoke-WebRequest -Uri "http://localhost:13305/api/v1/health" -UseBasicParsing -TimeoutSec 2
	if ($response.StatusCode -eq 200) {
	Write-Host "Server is healthy!" -ForegroundColor Green
	Write-Host "Health response: $($response.Content)"
	$healthy = $true
	break
	}
	} catch {}
	Start-Sleep -Seconds 2
	}

	if (-not $healthy) {
	Write-Host "ERROR: Server failed to start within 30 seconds" -ForegroundColor Red
	Stop-Process $proc -Force -ErrorAction SilentlyContinue
	Pop-Location
	exit 1
	}

	Stop-Process $proc -Force -ErrorAction SilentlyContinue
	Pop-Location
	Write-Host "Embeddable Windows smoke test PASSED!" -ForegroundColor Green

	test-dmg-inference:
	name: Test .dmg - llamacpp (metal)
	runs-on: macos-latest
	needs: build-lemonade-macos-dmg
	# Skip inference tests when signing is enabled (tag pushes or manual workflow_dispatch)
	# Also skip when no signing secrets (tests already ran inline in build job)
	if: ${{ needs.build-lemonade-macos-dmg.outputs.has_signing == 'true' && !startsWith(github.ref, 'refs/tags/') && inputs.enable_signing != true }}
	env:
	LEMONADE_CI_MODE: "True"
	PYTHONIOENCODING: utf-8
	GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
	LEMONADE_VERSION: ${{ needs.build-lemonade-macos-dmg.outputs.version }}
	GGML_METAL_NO_RESIDENCY: "1"
	steps:
	- uses: actions/checkout@v5

	- name: Set HF_HOME environment variable
	run: echo "HF_HOME=$PWD/hf-cache" >> $GITHUB_ENV

	- name: Install Lemonade Server (.pkg)
	uses: ./.github/actions/install-lemonade-server-dmg
	with:
	version: ${{ env.LEMONADE_VERSION }}

	- name: Setup Python and virtual environment
	uses: ./.github/actions/setup-venv
	with:
	venv-name: '.venv'
	python-version: '3.10'
	requirements-file: 'test/requirements.txt'

	- name: Test llamacpp (metal)
	env:
	HF_HOME: ${{ env.HF_HOME }}
	run: \|
	set -e
	.venv/bin/python test/server_llm.py --wrapped-server llamacpp --backend metal --server-binary /usr/local/bin/lemonade-server

	- name: Capture and upload server logs
	if: always()
	uses: ./.github/actions/capture-server-logs
	with:
	artifact-name: server-logs-dmg-llamacpp

	# ========================================================================
	# CLI AND ENDPOINTS TESTS - Run on GitHub-hosted runners (no GPU needed)
	# ========================================================================

	test-cli-endpoints-linux:
	name: Test ${{ matrix.test_type }} (ubuntu-latest)
	runs-on: ubuntu-latest
	needs: build-lemonade-deb
	container:
	image: ghcr.io/lemonade-sdk/lemonade/build-environment:ubuntu24.04
	credentials:
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}
	strategy:
	fail-fast: false
	matrix:
	test_type: [cli, endpoints, ollama, llamacpp-system, streaming-errors, env-vars]
	env:
	LEMONADE_CI_MODE: "True"
	PYTHONIOENCODING: utf-8
	GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
	LEMONADE_VERSION: ${{ needs.build-lemonade-deb.outputs.version }}
	steps:
	- uses: actions/checkout@v5

	- name: Configure git safe directory
	run: git config --global --add safe.directory $(pwd)

	- name: Install pip and test dependencies
	shell: bash
	run: \|
	set -e
	# Install pip if not already available
	if ! command -v pip3 &> /dev/null; then
	apt-get update
	apt-get install -y python3-pip python3-venv
	fi
	# Create venv and install test dependencies
	python3 -m venv .venv
	.venv/bin/pip install --upgrade pip
	.venv/bin/pip install -r test/requirements.txt

	- name: Install Lemonade (.deb)
	uses: ./.github/actions/install-lemonade-deb
	with:
	version: ${{ env.LEMONADE_VERSION }}

	- name: Set environment
	shell: bash
	run: \|
	echo "HF_HOME=$PWD/hf-cache" >> $GITHUB_ENV
	echo "VENV_PYTHON=.venv/bin/python" >> $GITHUB_ENV
	echo "SERVER_BINARY=lemonade-server" >> $GITHUB_ENV

	- name: Run tests
	shell: bash
	run: \|
	set -e
	VENV_PYTHON=.venv/bin/python
	SERVER_BINARY=lemonade-server

	if [ "${{ matrix.test_type }}" = "cli" ]; then
	echo "Running CLI tests..."
	$VENV_PYTHON test/server_cli.py --server-binary "$SERVER_BINARY"
	$VENV_PYTHON test/server_cli.py --server-binary "$SERVER_BINARY" --ephemeral
	$VENV_PYTHON test/server_cli.py --server-binary "$SERVER_BINARY" --listen-all
	$VENV_PYTHON test/server_cli.py --server-binary "$SERVER_BINARY" --api-key
	$VENV_PYTHON test/server_cli2.py --server-binary "$SERVER_BINARY"
	elif [ "${{ matrix.test_type }}" = "endpoints" ]; then
	echo "Running endpoint tests..."
	$VENV_PYTHON test/server_endpoints.py --server-binary "$SERVER_BINARY"
	$VENV_PYTHON test/server_endpoints.py --server-binary "$SERVER_BINARY" --server-per-test
	elif [ "${{ matrix.test_type }}" = "ollama" ]; then
	echo "Running Ollama API tests..."
	$VENV_PYTHON test/test_ollama.py --server-binary "$SERVER_BINARY"
	elif [ "${{ matrix.test_type }}" = "llamacpp-system" ]; then
	echo "Running LlamaCpp System Backend tests..."
	$VENV_PYTHON test/test_llamacpp_system_backend.py --server-binary "$SERVER_BINARY"
	elif [ "${{ matrix.test_type }}" = "streaming-errors" ]; then
	echo "Running streaming error termination tests..."
	$VENV_PYTHON test/server_streaming_errors.py --server-binary "$SERVER_BINARY"
	elif [ "${{ matrix.test_type }}" = "env-vars" ]; then
	echo "Running environment variable tests..."
	$VENV_PYTHON test/server_env_vars.py --lemond-binary ./deb-extract/usr/bin/lemond
	fi

	echo "${{ matrix.test_type }} tests PASSED!"

	test-cli-endpoints:
	name: Test ${{ matrix.test_type }} (${{ matrix.os }})
	runs-on: ${{ matrix.os }}
	needs:
	- build-lemonade-server-installer
	- build-lemonade-macos-dmg
	strategy:
	fail-fast: false
	matrix:
	os: [windows-latest, macos-latest]
	test_type: [cli, endpoints, ollama, llamacpp-system, streaming-errors]
	include:
	- os: macos-latest
	test_type: env-vars
	env:
	LEMONADE_CI_MODE: "True"
	PYTHONIOENCODING: utf-8
	GGML_METAL_NO_RESIDENCY: "1"
	GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
	LEMONADE_VERSION: ${{ needs.build-lemonade-macos-dmg.outputs.version }}
	steps:
	- uses: actions/checkout@v5

	# ---- Windows Setup ----
	- name: Setup (Windows)
	if: runner.os == 'Windows'
	shell: powershell
	run: \|
	$cwd = (Get-Item .).FullName
	echo "HF_HOME=$cwd\hf-cache" >> $Env:GITHUB_ENV
	echo "LEMONADE_INSTALL_PATH=$cwd\lemonade_server_install" >> $Env:GITHUB_ENV

	- name: Install Lemonade Server (Windows)
	if: runner.os == 'Windows'
	uses: ./.github/actions/install-lemonade-server-msi
	with:
	install-path: ${{ env.LEMONADE_INSTALL_PATH }}

	- name: Set paths (Windows)
	if: runner.os == 'Windows'
	shell: powershell
	run: \|
	echo "VENV_PYTHON=.venv/Scripts/python.exe" >> $Env:GITHUB_ENV
	echo "SERVER_BINARY=$Env:LEMONADE_INSTALL_PATH\bin\lemonade-server.exe" >> $Env:GITHUB_ENV

	# ---- macOS Setup (only when signed .pkg is available) ----
	- name: Download .pkg package
	if: runner.os == 'macOS' && needs.build-lemonade-macos-dmg.outputs.has_signing == 'true'
	uses: actions/download-artifact@v7
	with:
	name: lemonade-macos-pkg
	path: .

	- name: Install Lemonade Server (.pkg)
	id: install-pkg
	if: runner.os == 'macOS' && needs.build-lemonade-macos-dmg.outputs.has_signing == 'true'
	uses: ./.github/actions/install-lemonade-server-dmg
	with:
	version: ${{ env.LEMONADE_VERSION }}
	download-artifact: 'false'

	- name: Set environment (macOS)
	if: runner.os == 'macOS' && needs.build-lemonade-macos-dmg.outputs.has_signing == 'true'
	shell: bash
	run: \|
	echo "HF_HOME=$PWD/hf-cache" >> $GITHUB_ENV
	echo "VENV_PYTHON=.venv/bin/python" >> $GITHUB_ENV
	echo "SERVER_BINARY=${{ steps.install-pkg.outputs.bin-path }}/lemonade-server" >> $GITHUB_ENV

	# ---- Common Setup ----
	- name: Setup Python and virtual environment
	uses: ./.github/actions/setup-venv
	with:
	venv-name: '.venv'
	python-version: '3.10'
	requirements-file: 'test/requirements.txt'

	# ---- Run Tests ----
	- name: Run ${{ matrix.test_type }} tests
	if: ${{ !(runner.os == 'macOS' && needs.build-lemonade-macos-dmg.outputs.has_signing != 'true') }}
	shell: bash
	env:
	HF_HOME: ${{ env.HF_HOME }}
	run: \|
	set -e # Exit on error

	if [ "${{ matrix.test_type }}" = "cli" ]; then
	echo "Running CLI tests..."
	$VENV_PYTHON test/server_cli.py --server-binary "$SERVER_BINARY"
	$VENV_PYTHON test/server_cli2.py --server-binary "$SERVER_BINARY"
	elif [ "${{ matrix.test_type }}" = "endpoints" ]; then
	echo "Running endpoint tests..."
	$VENV_PYTHON test/server_endpoints.py --server-binary "$SERVER_BINARY"
	elif [ "${{ matrix.test_type }}" = "ollama" ]; then
	echo "Running Ollama API tests..."
	$VENV_PYTHON test/test_ollama.py --server-binary "$SERVER_BINARY"
	elif [ "${{ matrix.test_type }}" = "llamacpp-system" ]; then
	echo "Running LlamaCpp System Backend tests..."
	$VENV_PYTHON test/test_llamacpp_system_backend.py --server-binary "$SERVER_BINARY"
	elif [ "${{ matrix.test_type }}" = "streaming-errors" ]; then
	echo "Running streaming error termination tests..."
	$VENV_PYTHON test/server_streaming_errors.py --server-binary "$SERVER_BINARY"
	elif [ "${{ matrix.test_type }}" = "env-vars" ]; then
	echo "Running environment variable tests..."
	LEMOND_BINARY="$(dirname "$SERVER_BINARY")/lemond"
	$VENV_PYTHON test/server_env_vars.py --lemond-binary "$LEMOND_BINARY"
	fi

	echo "${{ matrix.test_type }} tests PASSED!"

	- name: Capture and upload server logs
	if: always()
	uses: ./.github/actions/capture-server-logs
	with:
	artifact-name: server-logs-${{ matrix.os }}-${{ matrix.test_type }}

	# ========================================================================
	# API KEY TESTS - Separate job with LEMONADE_API_KEY env var
	# ========================================================================

	test-cli-apikey-linux:
	name: Test API Key (ubuntu-latest)
	runs-on: ubuntu-latest
	needs: build-lemonade-deb
	container:
	image: ghcr.io/lemonade-sdk/lemonade/build-environment:ubuntu24.04
	credentials:
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}
	env:
	LEMONADE_CI_MODE: "True"
	LEMONADE_API_KEY: "test-api-key-12345"
	PYTHONIOENCODING: utf-8
	GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
	LEMONADE_VERSION: ${{ needs.build-lemonade-deb.outputs.version }}
	steps:
	- uses: actions/checkout@v5

	- name: Configure git safe directory
	run: git config --global --add safe.directory $(pwd)

	- name: Install pip and test dependencies
	shell: bash
	run: \|
	set -e
	if ! command -v pip3 &> /dev/null; then
	apt-get update
	apt-get install -y python3-pip python3-venv
	fi
	python3 -m venv .venv
	.venv/bin/pip install --upgrade pip
	.venv/bin/pip install -r test/requirements.txt

	- name: Install Lemonade (.deb)
	uses: ./.github/actions/install-lemonade-deb
	with:
	version: ${{ env.LEMONADE_VERSION }}

	- name: Verify API key enforcement
	shell: bash
	run: \|
	set -e
	PASS=0
	FAIL=0

	# 1. No API key → must get 401
	HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:13305/api/v1/health 2>/dev/null \|\| echo "000")
	if [ "$HTTP_CODE" = "401" ]; then
	echo "PASS: No API key → $HTTP_CODE"
	PASS=$((PASS+1))
	else
	echo "FAIL: No API key → $HTTP_CODE (expected 401)"
	FAIL=$((FAIL+1))
	fi

	# 2. Wrong API key → must get 401
	HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer wrong-key" http://127.0.0.1:13305/api/v1/health 2>/dev/null \|\| echo "000")
	if [ "$HTTP_CODE" = "401" ]; then
	echo "PASS: Wrong API key → $HTTP_CODE"
	PASS=$((PASS+1))
	else
	echo "FAIL: Wrong API key → $HTTP_CODE (expected 401)"
	FAIL=$((FAIL+1))
	fi

	# 3. Correct API key → must get 200
	HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $LEMONADE_API_KEY" http://127.0.0.1:13305/api/v1/health 2>/dev/null \|\| echo "000")
	if [ "$HTTP_CODE" = "200" ]; then
	echo "PASS: Correct API key → $HTTP_CODE"
	PASS=$((PASS+1))
	else
	echo "FAIL: Correct API key → $HTTP_CODE (expected 200)"
	FAIL=$((FAIL+1))
	fi

	echo ""
	echo "Results: $PASS passed, $FAIL failed"
	if [ "$FAIL" -gt 0 ]; then
	echo "ERROR: API key enforcement tests failed"
	exit 1
	fi

	- name: Run CLI tests with API key
	shell: bash
	run: \|
	set -e
	# CLI commands should work because LEMONADE_API_KEY is in the env
	# and the CLI client reads it automatically
	.venv/bin/python test/server_cli.py --server-binary lemonade-server

	- name: Capture and upload server logs
	if: always()
	uses: ./.github/actions/capture-server-logs
	with:
	artifact-name: server-logs-apikey-ubuntu-latest

	test-cli-apikey:
	name: Test API Key (windows-latest)
	runs-on: windows-latest
	needs:
	- build-lemonade-server-installer
	env:
	LEMONADE_CI_MODE: "True"
	LEMONADE_API_KEY: "test-api-key-12345"
	PYTHONIOENCODING: utf-8
	GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
	steps:
	- uses: actions/checkout@v5

	- name: Setup (Windows)
	shell: powershell
	run: \|
	$cwd = (Get-Item .).FullName
	echo "HF_HOME=$cwd\hf-cache" >> $Env:GITHUB_ENV
	echo "LEMONADE_INSTALL_PATH=$cwd\lemonade_server_install" >> $Env:GITHUB_ENV

	- name: Install Lemonade Server (Windows)
	uses: ./.github/actions/install-lemonade-server-msi
	with:
	install-path: ${{ env.LEMONADE_INSTALL_PATH }}

	- name: Set paths (Windows)
	shell: powershell
	run: \|
	echo "VENV_PYTHON=.venv/Scripts/python.exe" >> $Env:GITHUB_ENV
	echo "SERVER_BINARY=$Env:LEMONADE_INSTALL_PATH\bin\lemonade-server.exe" >> $Env:GITHUB_ENV

	- name: Setup Python and virtual environment
	uses: ./.github/actions/setup-venv
	with:
	venv-name: '.venv'
	python-version: '3.10'
	requirements-file: 'test/requirements.txt'

	- name: Verify API key enforcement
	shell: bash
	run: \|
	set -e
	PASS=0
	FAIL=0

	# 1. No API key → must get 401
	HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:13305/api/v1/health 2>/dev/null \|\| echo "000")
	if [ "$HTTP_CODE" = "401" ]; then
	echo "PASS: No API key → $HTTP_CODE"
	PASS=$((PASS+1))
	else
	echo "FAIL: No API key → $HTTP_CODE (expected 401)"
	FAIL=$((FAIL+1))
	fi

	# 2. Wrong API key → must get 401
	HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer wrong-key" http://localhost:13305/api/v1/health 2>/dev/null \|\| echo "000")
	if [ "$HTTP_CODE" = "401" ]; then
	echo "PASS: Wrong API key → $HTTP_CODE"
	PASS=$((PASS+1))
	else
	echo "FAIL: Wrong API key → $HTTP_CODE (expected 401)"
	FAIL=$((FAIL+1))
	fi

	# 3. Correct API key → must get 200
	HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $LEMONADE_API_KEY" http://localhost:13305/api/v1/health 2>/dev/null \|\| echo "000")
	if [ "$HTTP_CODE" = "200" ]; then
	echo "PASS: Correct API key → $HTTP_CODE"
	PASS=$((PASS+1))
	else
	echo "FAIL: Correct API key → $HTTP_CODE (expected 200)"
	FAIL=$((FAIL+1))
	fi

	echo ""
	echo "Results: $PASS passed, $FAIL failed"
	if [ "$FAIL" -gt 0 ]; then
	echo "ERROR: API key enforcement tests failed"
	exit 1
	fi

	- name: Run CLI tests with API key
	shell: bash
	env:
	HF_HOME: ${{ env.HF_HOME }}
	run: \|
	set -e
	$VENV_PYTHON test/server_cli.py --server-binary "$SERVER_BINARY"

	- name: Capture and upload server logs
	if: always()
	uses: ./.github/actions/capture-server-logs
	with:
	artifact-name: server-logs-apikey-windows-latest

	# ========================================================================
	# RELEASE JOB - Add artifacts to GitHub release
	# ========================================================================

	release:
	name: Create GitHub Release
	runs-on: ubuntu-latest
	needs:
	- sign-msi-installers
	- build-lemonade-rpm
	- build-lemonade-macos-dmg
	- build-lemonade-appimage
	- build-lemonade-embeddable-linux
	- build-lemonade-embeddable-windows
	- test-cli-endpoints
	- test-rpm-package
	- test-embeddable-linux
	- test-embeddable-windows
	if: startsWith(github.ref, 'refs/tags/v')
	env:
	LEMONADE_VERSION: ${{ needs.build-lemonade-rpm.outputs.version }}
	steps:
	- name: Checkout for release notes action
	uses: actions/checkout@v5
	with:
	sparse-checkout: .github

	- name: Download Signed Lemonade Server Installer (Windows)
	uses: actions/download-artifact@v7
	with:
	name: Lemonade_Server_MSI_Signed
	path: .

	- name: Download Lemonade .rpm Package
	uses: actions/download-artifact@v7
	with:
	name: lemonade-rpm
	path: .

	- name: Download Lemonade macOS .pkg Package
	uses: actions/download-artifact@v7
	with:
	name: lemonade-macos-pkg
	path: .

	- name: Download Lemonade AppImage (Linux)
	uses: actions/download-artifact@v7
	with:
	name: lemonade-appimage
	path: .

	- name: Download Lemonade Embeddable (Linux)
	uses: actions/download-artifact@v4
	with:
	name: lemonade-embeddable-linux
	path: .

	- name: Download Lemonade Embeddable (Windows)
	uses: actions/download-artifact@v4
	with:
	name: lemonade-embeddable-windows
	path: .

	- name: Verify release artifacts
	run: \|
	echo "Release artifacts:"
	ls -lh lemonade-server-minimal.msi
	ls -lh lemonade.msi
	ls -lh lemonade-server-${LEMONADE_VERSION}.x86_64.rpm
	ls -lh *.pkg
	ls -lh lemonade-app-${LEMONADE_VERSION}-x86_64.AppImage
	ls -lh lemonade-embeddable-${LEMONADE_VERSION}-ubuntu-x64.tar.gz
	ls -lh lemonade-embeddable-${LEMONADE_VERSION}-windows-x64.zip

	- name: Generate release notes
	id: release-notes
	uses: ./.github/actions/generate-release-notes
	with:
	version: ${{ env.LEMONADE_VERSION }}
	repo: ${{ github.repository }}
	github_token: ${{ secrets.GITHUB_TOKEN }}

	- name: Create Release
	uses: softprops/action-gh-release@v2
	with:
	name: ${{ github.ref_name }}
	body_path: ${{ steps.release-notes.outputs.release_notes_file }}
	files: \|
	lemonade-server-minimal.msi
	lemonade.msi
	lemonade-server-${{ env.LEMONADE_VERSION }}.x86_64.rpm
	*.pkg
	lemonade-app-${{ env.LEMONADE_VERSION }}-x86_64.AppImage
	lemonade-embeddable-${{ env.LEMONADE_VERSION }}-ubuntu-x64.tar.gz
	lemonade-embeddable-${{ env.LEMONADE_VERSION }}-windows-x64.zip
	fail_on_unmatched_files: true

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add OmniRouter: multimodal tools + model collections for agentic workflows #3758

Workflow file

Add OmniRouter: multimodal tools + model collections for agentic workflows #3758

Uh oh!

Workflow file for this run