feat: streaming + multiturn

.github/workflows/build_and_test.yml

@@ -37,7 +37,7 @@ jobs:
         run: uv run python3 -m pytest --runslow .
 
       - name: Check Python Types
-        run: uvx ty check
+        run: uv run ty check
 
       - name: Build Core
         run: uv build

.github/workflows/format_and_lint.yml

@@ -40,12 +40,12 @@ jobs:
 
       - name: Lint with ruff
         run: |
-          uvx ruff check
+          uv run ruff check
 
       - name: Format with ruff
         run: |
-          uvx ruff format --check .
+          uv run ruff format --check .
 
       - name: Typecheck with ty
         run: |
-          uvx ty check
+          uv run ty check

.gitignore

@@ -21,3 +21,5 @@ libs/server/build
 dist/
 
 .mcp.json
+
+test_output/

app/desktop/WinInnoSetup.iss

@@ -3,7 +3,7 @@
 
 #define MyAppPath "build\dist\Kiln"
 #define MyAppName "Kiln"
-#define MyAppVersion "0.24.0"
+#define MyAppVersion "0.25.0"
 #define MyAppPublisher "Chesterfield Laboratories Inc"
 #define MyAppURL "https://kiln.tech"
 #define MyAppExeName "Kiln.exe"

app/desktop/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "kiln-studio-desktop"
-version = "0.24.0"
+version = "0.25.0"
 description = "The Kiln Desktop App. Download from https://kiln.tech"
 requires-python = ">=3.10"
 dependencies = [

app/desktop/studio_server/test_copilot_api.py

@@ -13,11 +13,13 @@
 from app.desktop.studio_server.copilot_api import connect_copilot_api
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
+from kiln_server.custom_errors import connect_custom_errors
 
 
 @pytest.fixture
 def app():
     app = FastAPI()
+    connect_custom_errors(app)
     connect_copilot_api(app)
     return app
 
@@ -123,7 +125,7 @@ def test_clarify_spec_no_api_key(self, client, clarify_spec_input):
 
             response = client.post("/api/copilot/clarify_spec", json=clarify_spec_input)
             assert response.status_code == 401
-            assert "API key not configured" in response.json()["detail"]
+            assert "API key not configured" in response.json()["message"]
 
     def test_clarify_spec_success(self, client, clarify_spec_input, mock_api_key):
         mock_output = MagicMock(spec=ClarifySpecOutput)
@@ -194,7 +196,7 @@ def test_clarify_spec_no_response(self, client, clarify_spec_input, mock_api_key
         ):
             response = client.post("/api/copilot/clarify_spec", json=clarify_spec_input)
             assert response.status_code == 500
-            assert "Failed to analyze spec" in response.json()["detail"]
+            assert "Failed to analyze spec" in response.json()["message"]
 
     def test_clarify_spec_validation_error(
         self, client, clarify_spec_input, mock_api_key
@@ -210,7 +212,7 @@ def test_clarify_spec_validation_error(
         ):
             response = client.post("/api/copilot/clarify_spec", json=clarify_spec_input)
             assert response.status_code == 422
-            assert "Validation error from server" in response.json()["detail"]
+            assert "Validation error from server" in response.json()["message"]
 
 
 class TestRefineSpec:
@@ -223,7 +225,7 @@ def test_refine_spec_no_api_key(self, client, refine_spec_input):
 
             response = client.post("/api/copilot/refine_spec", json=refine_spec_input)
             assert response.status_code == 401
-            assert "API key not configured" in response.json()["detail"]
+            assert "API key not configured" in response.json()["message"]
 
     def test_refine_spec_success(self, client, refine_spec_input, mock_api_key):
         mock_output = MagicMock(spec=RefineSpecApiOutput)
@@ -259,7 +261,7 @@ def test_refine_spec_no_response(self, client, refine_spec_input, mock_api_key):
         ):
             response = client.post("/api/copilot/refine_spec", json=refine_spec_input)
             assert response.status_code == 500
-            assert "Failed to refine spec" in response.json()["detail"]
+            assert "Failed to refine spec" in response.json()["message"]
 
     def test_refine_spec_validation_error(
         self, client, refine_spec_input, mock_api_key
@@ -275,7 +277,7 @@ def test_refine_spec_validation_error(
         ):
             response = client.post("/api/copilot/refine_spec", json=refine_spec_input)
             assert response.status_code == 422
-            assert "Validation error from server" in response.json()["detail"]
+            assert "Validation error from server" in response.json()["message"]
 
 
 class TestGenerateBatch:
@@ -290,7 +292,7 @@ def test_generate_batch_no_api_key(self, client, generate_batch_input):
                 "/api/copilot/generate_batch", json=generate_batch_input
             )
             assert response.status_code == 401
-            assert "API key not configured" in response.json()["detail"]
+            assert "API key not configured" in response.json()["message"]
 
     def test_generate_batch_success(self, client, generate_batch_input, mock_api_key):
         mock_output = MagicMock(spec=GenerateBatchOutput)
@@ -328,7 +330,7 @@ def test_generate_batch_no_response(
                 "/api/copilot/generate_batch", json=generate_batch_input
             )
             assert response.status_code == 500
-            assert "Failed to generate synthetic data" in response.json()["detail"]
+            assert "Failed to generate synthetic data" in response.json()["message"]
 
     def test_generate_batch_validation_error(
         self, client, generate_batch_input, mock_api_key
@@ -346,4 +348,4 @@ def test_generate_batch_validation_error(
                 "/api/copilot/generate_batch", json=generate_batch_input
             )
             assert response.status_code == 422
-            assert "Validation error from server" in response.json()["detail"]
+            assert "Validation error from server" in response.json()["message"]

app/desktop/studio_server/test_data_gen_api.py

@@ -3,6 +3,7 @@
 import pytest
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
+from kiln_server.custom_errors import connect_custom_errors
 from kiln_ai.datamodel import (
     DataSource,
     DataSourceType,
@@ -31,6 +32,7 @@
 @pytest.fixture
 def app():
     app = FastAPI()
+    connect_custom_errors(app)
     connect_data_gen_api(app)
     return app
 

app/desktop/studio_server/test_eval_api.py

@@ -15,6 +15,7 @@
 from fastapi import FastAPI, HTTPException
 from fastapi.responses import StreamingResponse
 from fastapi.testclient import TestClient
+from kiln_server.custom_errors import connect_custom_errors
 from kiln_ai.adapters.ml_model_list import ModelProviderName
 from kiln_ai.datamodel import (
     DataSource,
@@ -55,6 +56,7 @@
 @pytest.fixture
 def app():
     app = FastAPI()
+    connect_custom_errors(app)
     connect_evals_api(app)
     return app
 
@@ -189,7 +191,7 @@ def test_get_eval_not_found(client, mock_task, mock_task_from_id):
     response = client.get("/api/projects/project1/tasks/task1/eval/non_existent")
 
     assert response.status_code == 404
-    assert response.json()["detail"] == "Eval not found. ID: non_existent"
+    assert response.json()["message"] == "Eval not found. ID: non_existent"
 
 
 @pytest.fixture
@@ -513,7 +515,7 @@ async def test_run_eval_config_no_run_configs_error(
 
         assert response.status_code == 400
         assert (
-            response.json()["detail"]
+            response.json()["message"]
             == "No run config ids provided. At least one run config id is required."
         )
 
@@ -785,7 +787,7 @@ def test_update_run_config_prompt_name_no_prompt(
         json={"prompt_name": "New Name"},
     )
     assert response.status_code == 400
-    assert "no frozen prompt" in response.json()["detail"].lower()
+    assert "no frozen prompt" in response.json()["message"].lower()
 
 
 @pytest.fixture
@@ -1354,7 +1356,7 @@ def test_delete_eval_not_found(client):
 
     # Verify the response
     assert response.status_code == 404
-    assert response.json()["detail"] == "Eval not found. ID: nonexistent_eval"
+    assert response.json()["message"] == "Eval not found. ID: nonexistent_eval"
 
 
 async def test_create_eval_then_delete_on_spec_failure(
@@ -1477,7 +1479,7 @@ def test_update_eval_train_set_filter_id_when_already_set(
     assert response.status_code == 400
     assert (
         "Train set filter is already set and cannot be changed"
-        in response.json()["detail"]
+        in response.json()["message"]
     )
 
 
@@ -1527,7 +1529,7 @@ def test_update_eval_not_found(client):
         )
 
     assert response.status_code == 404
-    assert "Eval not found" in response.json()["detail"]
+    assert "Eval not found" in response.json()["message"]
 
 
 def test_update_eval_empty_request(client, mock_task_from_id, mock_eval, mock_task):
@@ -1759,7 +1761,7 @@ async def test_get_eval_progress_not_found(client, mock_task_from_id, mock_task)
 
         # Verify the response
         assert response.status_code == 404
-        assert response.json()["detail"] == "Eval not found. ID: non_existent"
+        assert response.json()["message"] == "Eval not found. ID: non_existent"
         mock_eval_from_id.assert_called_once_with("project1", "task1", "non_existent")
 
 
@@ -1810,7 +1812,7 @@ async def test_set_current_eval_config_not_found(
 
     # Verify the response
     assert response.status_code == 400
-    assert response.json()["detail"] == "Eval config not found."
+    assert response.json()["message"] == "Eval config not found."
 
 
 @pytest.mark.parametrize(
@@ -1901,7 +1903,7 @@ async def test_create_task_run_config_invalid_temperature_values(
         },
     )
     assert response.status_code == 422
-    error_detail = response.json()["detail"]
+    error_detail = response.json()["message"]
     assert "temperature must be between 0 and 2" in str(error_detail)
 
     # Test temperature above 2
@@ -1919,7 +1921,7 @@ async def test_create_task_run_config_invalid_temperature_values(
         },
     )
     assert response.status_code == 422
-    error_detail = response.json()["detail"]
+    error_detail = response.json()["message"]
     assert "temperature must be between 0 and 2" in str(error_detail)
 
 
@@ -1945,7 +1947,7 @@ async def test_create_task_run_config_invalid_top_p_values(
         },
     )
     assert response.status_code == 422
-    error_detail = response.json()["detail"]
+    error_detail = response.json()["message"]
     assert "top_p must be between 0 and 1" in str(error_detail)
 
     # Test top_p above 1
@@ -1963,7 +1965,7 @@ async def test_create_task_run_config_invalid_top_p_values(
         },
     )
     assert response.status_code == 422
-    error_detail = response.json()["detail"]
+    error_detail = response.json()["message"]
     assert "top_p must be between 0 and 1" in str(error_detail)
 
 
@@ -2226,7 +2228,7 @@ def test_get_eval_configs_score_summary_no_filter_id(
 
         assert response.status_code == 400
         assert (
-            response.json()["detail"]
+            response.json()["message"]
             == "No eval configs filter id set, cannot get eval configs score summary."
         )
         mock_eval_from_id.assert_called_once_with("project1", "task1", "eval1")