loreai/.github/workflows/eval.yml at main · BYK/loreai · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
name: Lore Eval Suite

on:
  # Live mode on weekly schedule
  schedule:
    - cron: '0 6 * * 1' # Monday 6am UTC

  # Manual trigger with mode selection
  workflow_dispatch:
    inputs:
      mode:
        description: 'Eval mode'
        type: choice
        options:
          - fixture
          - live
        default: fixture
      dimensions:
        description: 'Comma-separated dimensions (or "all")'
        default: 'all'
      baselines:
        description: 'Comma-separated baselines (or empty for defaults)'
        default: ''

jobs:
  eval-fixture:
    if: github.event_name == 'workflow_dispatch' && inputs.mode == 'fixture'
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v6
      - uses: oven-sh/setup-bun@v2
        with:
          bun-version: latest
      - uses: actions/setup-node@v6
        with:
          node-version: '24'
          registry-url: 'https://registry.npmjs.org'
      - run: bun install
      - run: bun packages/core/eval/run.ts --mode fixture
      - uses: actions/upload-artifact@v7
        if: always()
        with:
          name: eval-fixture-results
          path: packages/core/eval/results/
          retention-days: 30

  eval-live:
    if: >
      github.event_name == 'schedule' ||
      (github.event_name == 'workflow_dispatch' && inputs.mode == 'live')
    runs-on: ubuntu-latest
    timeout-minutes: 120
    permissions:
      models: read
    steps:
      - uses: actions/checkout@v6
      - uses: oven-sh/setup-bun@v2
        with:
          bun-version: latest
      - uses: actions/setup-node@v6
        with:
          node-version: '24'
          registry-url: 'https://registry.npmjs.org'
      - run: bun install

      - name: Run live eval
        run: >
          bun packages/core/eval/run.ts
          --mode live
          --dimensions ${{ inputs.dimensions || 'all' }}
          --concurrency 1
          ${{ inputs.baselines && format('--baselines {0}', inputs.baselines) || '' }}
        env:
          # Use ANTHROPIC_API_KEY if available, fall back to GH Models PAT
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          GITHUB_TOKEN: ${{ secrets.GH_MODELS_TOKEN || secrets.GITHUB_TOKEN }}

      - uses: actions/upload-artifact@v7
        if: always()
        with:
          name: eval-live-results
          path: packages/core/eval/results/
          retention-days: 90

      - name: Post summary
        if: always()
        run: |
          LATEST=$(ls -t packages/core/eval/results/*.jsonl 2>/dev/null | head -1)
          if [ -n "$LATEST" ]; then
            bun packages/core/eval/run.ts --summarize "$LATEST" >> $GITHUB_STEP_SUMMARY
          fi

  eval-auto-memory:
    if: >
      github.event_name == 'schedule' ||
      (github.event_name == 'workflow_dispatch' && inputs.mode == 'live')
    runs-on: ubuntu-latest
    timeout-minutes: 120
    continue-on-error: true
    permissions:
      models: read
    steps:
      - uses: actions/checkout@v6
      - uses: oven-sh/setup-bun@v2
        with:
          bun-version: latest
      - uses: actions/setup-node@v6
        with:
          node-version: '24'
          registry-url: 'https://registry.npmjs.org'
      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - run: pip install auto-memory
      - run: bun install

      - name: Run auto-memory baseline eval
        run: >
          bun packages/core/eval/run.ts
          --mode live
          --baselines auto-mem0,lore
          --dimensions recall,preferences,cross-project
          --concurrency 1
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          GITHUB_TOKEN: ${{ secrets.GH_MODELS_TOKEN || secrets.GITHUB_TOKEN }}

      - uses: actions/upload-artifact@v7
        if: always()
        with:
          name: eval-auto-memory-results
          path: packages/core/eval/results/
          retention-days: 90