Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 0 additions & 8 deletions .changeset/README.md

This file was deleted.

11 changes: 0 additions & 11 deletions .changeset/config.json

This file was deleted.

39 changes: 0 additions & 39 deletions .github/workflows/publish.yml

This file was deleted.

219 changes: 200 additions & 19 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -1,38 +1,219 @@
name: Release

on:
push:
branches: [main]
workflow_dispatch:
inputs:
bump:
description: "patch: bug fixes (0.1.0→0.1.1) | minor: new features (0.1.0→0.2.0) | major: breaking changes (0.1.0→1.0.0)"
type: choice
default: "patch"
options:
- patch
- minor
- major
release_engine:
description: "Release @mondaycom/sensei-engine"
type: boolean
default: true
release_sdk:
description: "Release @mondaycom/sensei-sdk"
type: boolean
default: true
release_cli:
description: "Release @mondaycom/sensei-cli"
type: boolean
default: true

concurrency: ${{ github.workflow }}-${{ github.ref }}
permissions:
contents: write
id-token: write

concurrency:
group: release
cancel-in-progress: false

jobs:
release:
name: Release
runs-on: ubuntu-latest

permissions:
contents: write
pull-requests: write

steps:
- uses: actions/checkout@v4
with:
token: ${{ secrets.GITHUB_TOKEN }}

- uses: actions/setup-node@v4
with:
node-version: 20
cache: npm
registry-url: "https://registry.npmjs.org"

- run: npm install
- run: npm run build

- name: Create Release Pull Request or Publish to npm
id: changesets
uses: changesets/action@v1
with:
title: "chore: version packages"
commit: "chore: version packages"
publish: npx changeset publish
- name: Validate selection
env:
RELEASE_ENGINE: ${{ inputs.release_engine }}
RELEASE_SDK: ${{ inputs.release_sdk }}
RELEASE_CLI: ${{ inputs.release_cli }}
run: |
if [[ "$RELEASE_ENGINE" != "true" && \
"$RELEASE_SDK" != "true" && \
"$RELEASE_CLI" != "true" ]]; then
echo "::error::No packages selected for release"
exit 1
fi
if [[ "$RELEASE_ENGINE" != "true" ]]; then
if [[ "$RELEASE_SDK" == "true" || "$RELEASE_CLI" == "true" ]]; then
echo "::error::Cannot release sdk/cli without engine. Engine must be included to ensure dependency versions are correct."
exit 1
fi
fi

- name: Bump versions
id: versions
env:
BUMP: ${{ inputs.bump }}
RELEASE_ENGINE: ${{ inputs.release_engine }}
RELEASE_SDK: ${{ inputs.release_sdk }}
RELEASE_CLI: ${{ inputs.release_cli }}
run: |
if [[ "$RELEASE_ENGINE" == "true" ]]; then
cd packages/engine
NEW_VERSION=$(npm version "$BUMP" --no-git-tag-version | sed 's/^v//')
echo "engine_version=${NEW_VERSION}" >> "$GITHUB_OUTPUT"
echo "Engine bumped to ${NEW_VERSION}"
cd ../..
fi

if [[ "$RELEASE_SDK" == "true" ]]; then
cd packages/sdk
NEW_VERSION=$(npm version "$BUMP" --no-git-tag-version | sed 's/^v//')
echo "sdk_version=${NEW_VERSION}" >> "$GITHUB_OUTPUT"
echo "SDK bumped to ${NEW_VERSION}"
cd ../..
fi

if [[ "$RELEASE_CLI" == "true" ]]; then
cd packages/cli
NEW_VERSION=$(npm version "$BUMP" --no-git-tag-version | sed 's/^v//')
echo "cli_version=${NEW_VERSION}" >> "$GITHUB_OUTPUT"
echo "CLI bumped to ${NEW_VERSION}"
cd ../..
fi

- name: Update lockfile
run: npm install

- name: Build
run: npm run build

- name: Test
run: npx vitest run

- name: Publish engine
if: inputs.release_engine == true
run: npm publish --workspace packages/engine --access public --provenance
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

- name: Publish sdk
if: inputs.release_sdk == true
run: npm publish --workspace packages/sdk --access public --provenance
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

- name: Publish cli
if: inputs.release_cli == true
run: npm publish --workspace packages/cli --access public --provenance
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

- name: Commit and tag
env:
ENGINE_VERSION: ${{ steps.versions.outputs.engine_version }}
SDK_VERSION: ${{ steps.versions.outputs.sdk_version }}
CLI_VERSION: ${{ steps.versions.outputs.cli_version }}
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"

git add packages/*/package.json package-lock.json

MSG="release [skip ci]:"
TAGS=()

if [[ -n "$ENGINE_VERSION" ]]; then
MSG="${MSG} engine@${ENGINE_VERSION}"
TAGS+=("@mondaycom/sensei-engine@${ENGINE_VERSION}")
fi
if [[ -n "$SDK_VERSION" ]]; then
MSG="${MSG} sdk@${SDK_VERSION}"
TAGS+=("@mondaycom/sensei-sdk@${SDK_VERSION}")
fi
if [[ -n "$CLI_VERSION" ]]; then
MSG="${MSG} cli@${CLI_VERSION}"
TAGS+=("@mondaycom/sensei-cli@${CLI_VERSION}")
fi

git commit -m "${MSG}"

for TAG in "${TAGS[@]}"; do
git tag "${TAG}"
done

git push origin HEAD:main --tags --atomic

- name: Create GitHub Releases
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ENGINE_VERSION: ${{ steps.versions.outputs.engine_version }}
SDK_VERSION: ${{ steps.versions.outputs.sdk_version }}
CLI_VERSION: ${{ steps.versions.outputs.cli_version }}
run: |
LAST_TAG=""

if [[ -n "$ENGINE_VERSION" ]]; then
LAST_TAG="@mondaycom/sensei-engine@${ENGINE_VERSION}"
gh release create "$LAST_TAG" \
--title "sensei-engine ${ENGINE_VERSION}" \
--generate-notes \
--latest=false
fi
if [[ -n "$SDK_VERSION" ]]; then
LAST_TAG="@mondaycom/sensei-sdk@${SDK_VERSION}"
gh release create "$LAST_TAG" \
--title "sensei-sdk ${SDK_VERSION}" \
--generate-notes \
--latest=false
fi
if [[ -n "$CLI_VERSION" ]]; then
LAST_TAG="@mondaycom/sensei-cli@${CLI_VERSION}"
gh release create "$LAST_TAG" \
--title "sensei-cli ${CLI_VERSION}" \
--generate-notes \
--latest=false
fi

# Mark the last release as "Latest"
if [[ -n "$LAST_TAG" ]]; then
gh release edit "$LAST_TAG" --latest
fi

- name: Summary
if: always()
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
ENGINE_VERSION: ${{ steps.versions.outputs.engine_version }}
SDK_VERSION: ${{ steps.versions.outputs.sdk_version }}
CLI_VERSION: ${{ steps.versions.outputs.cli_version }}
run: |
echo "## Release Summary" >> "$GITHUB_STEP_SUMMARY"
echo "" >> "$GITHUB_STEP_SUMMARY"
echo "| Package | Version |" >> "$GITHUB_STEP_SUMMARY"
echo "|---------|---------|" >> "$GITHUB_STEP_SUMMARY"
if [[ -n "$ENGINE_VERSION" ]]; then
echo "| @mondaycom/sensei-engine | ${ENGINE_VERSION} |" >> "$GITHUB_STEP_SUMMARY"
fi
if [[ -n "$SDK_VERSION" ]]; then
echo "| @mondaycom/sensei-sdk | ${SDK_VERSION} |" >> "$GITHUB_STEP_SUMMARY"
fi
if [[ -n "$CLI_VERSION" ]]; then
echo "| @mondaycom/sensei-cli | ${CLI_VERSION} |" >> "$GITHUB_STEP_SUMMARY"
fi
18 changes: 9 additions & 9 deletions ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,15 @@ Sensei is a standalone, open-source agent qualification engine. It runs test sui
## Tech Stack

- **Language:** TypeScript (Node.js)
- **Package manager:** npm (published as `@sensei/cli`, `@sensei/engine`, `@sensei/sdk`)
- **Package manager:** npm (published as `@mondaycom/sensei-cli`, `@mondaycom/sensei-engine`, `@mondaycom/sensei-sdk`)
- **Test definition:** YAML (declarative) + TypeScript SDK (programmatic)
- **LLM Judge:** OpenAI / any OpenAI-compatible API (Anthropic via proxy)
- **Output:** JSON reports, HTML reports, terminal output
- **CI/CD:** GitHub Actions integration

## Core Components

### 1. Engine (`@sensei/engine`)
### 1. Engine (`@mondaycom/sensei-engine`)

The core library. No CLI, no HTTP — pure evaluation logic.

Expand Down Expand Up @@ -254,7 +254,7 @@ Response: { output: "..." } or { output: { content: "..." } }

Features: Supports both string and object output formats, health check via `/input_schema`, retry with backoff.

### 6. CLI (`@sensei/cli`)
### 6. CLI (`@mondaycom/sensei-cli`)

```
sensei run [options]
Expand All @@ -273,7 +273,7 @@ sensei report Render a report from a previous JSON result
--input <path> Path to JSON result file
```

### 7. SDK (`@sensei/sdk`)
### 7. SDK (`@mondaycom/sensei-sdk`)

Programmatic suite building and utilities.

Expand Down Expand Up @@ -385,8 +385,8 @@ scenarios:
AgentTalent uses Sensei as a library dependency:

```typescript
import { SuiteLoader, Runner, Judge, Comparator, createAdapter } from '@sensei/engine';
import type { KPIResult } from '@sensei/engine';
import { SuiteLoader, Runner, Judge, Comparator, createAdapter } from '@mondaycom/sensei-engine';
import type { KPIResult } from '@mondaycom/sensei-engine';

async function evaluateCandidate(agentUrl: string, suiteFile: string) {
// Load the suite
Expand Down Expand Up @@ -444,7 +444,7 @@ async function evaluateCandidate(agentUrl: string, suiteFile: string) {
```
sensei/
├── packages/
│ ├── engine/ # @sensei/engine
│ ├── engine/ # @mondaycom/sensei-engine
│ │ ├── src/
│ │ │ ├── types.ts # Core types + LAYER_WEIGHTS + BADGE_THRESHOLDS
│ │ │ ├── schema.ts # Zod schemas (SuiteDefinitionSchema, etc.)
Expand All @@ -464,7 +464,7 @@ sensei/
│ │ ├── tests/ # 100+ engine tests
│ │ ├── package.json
│ │ └── tsconfig.json
│ ├── cli/ # @sensei/cli
│ ├── cli/ # @mondaycom/sensei-cli
│ │ ├── src/
│ │ │ ├── index.ts # Entry point (commander)
│ │ │ ├── loader.ts # Suite loader (YAML + JSON with Zod validation)
Expand All @@ -478,7 +478,7 @@ sensei/
│ │ │ └── report.ts # sensei report
│ │ ├── tests/ # CLI + E2E tests
│ │ └── package.json
│ └── sdk/ # @sensei/sdk
│ └── sdk/ # @mondaycom/sensei-sdk
│ ├── src/
│ │ ├── index.ts # Public API exports
│ │ ├── builder.ts # SuiteBuilder + scenario() + kpi() helpers
Expand Down
Loading
Loading