fix: make ignore lists run time and not compile time

.github/workflows/test-query-compiler-template.yml

@@ -8,7 +8,7 @@ on:
       ignored_tests_list:
         type: string
         required: true
-      should_panic_tests_list:
+      should_fail_tests_list:
         type: string
         required: true
 
@@ -34,7 +34,7 @@ jobs:
       WASM_BUILD_PROFILE: "profiling" # Include debug info for proper backtraces
       WORKSPACE_ROOT: ${{ github.workspace }}
       IGNORED_TESTS: ${{ inputs.ignored_tests_list }}
-      SHOULD_PANIC_TESTS: ${{ inputs.should_panic_tests_list }}
+      SHOULD_FAIL_TESTS: ${{ inputs.should_fail_tests_list }}
 
     runs-on: ubuntu-latest
     steps:

.github/workflows/test-query-compiler.yml

@@ -33,4 +33,4 @@ jobs:
     with:
       setup_task: ${{ matrix.adapter.setup_task }}
       ignored_tests_list: ${{ matrix.ignored_tests_list }}
-      should_panic_tests_list: ${{ matrix.should_panic_tests_list }}
+      should_fail_tests_list: ${{ matrix.should_fail_tests_list }}

libs/driver-adapters/package.json

@@ -10,8 +10,12 @@
   "license": "Apache-2.0",
   "scripts": {
     "build": "pnpm build:prisma && pnpm build:executor",
+    "build:qe": "pnpm build:prisma && pnpm build:executor-qe",
+    "build:qc": "pnpm build:prisma && pnpm build:executor-qc",
     "build:prisma": "pnpm -r --parallel dev",
     "build:executor": "pnpm -r --filter executor build",
+    "build:executor-qe": "pnpm -r --filter executor build:qe",
+    "build:executor-qc": "pnpm -r --filter executor build:qc",
     "lint": "pnpm -r run lint",
     "clean": "git clean -dXf -e !libs/driver-adapters"
   },

query-engine/connector-test-kit-rs/query-test-macros/src/connector_test.rs

@@ -43,6 +43,7 @@ pub fn connector_test_impl(attr: TokenStream, input: TokenStream) -> TokenStream
 
     // The shell function retains the name of the original test definition.
     let test_fn_ident = test_function.sig.ident;
+    let test_fn_ident_string = test_fn_ident.to_string();
 
     // Rename original test function to run_<orig_name>.
     let runner_fn_ident = Ident::new(&format!("run_{test_fn_ident}"), Span::call_site());
@@ -82,6 +83,7 @@ pub fn connector_test_impl(attr: TokenStream, input: TokenStream) -> TokenStream
                 &[#(#db_extensions),*],
                 #referential_override,
                 #runner_fn_ident,
+                #test_fn_ident_string
             );
         }
 
@@ -90,4 +92,3 @@ pub fn connector_test_impl(attr: TokenStream, input: TokenStream) -> TokenStream
 
     test.into()
 }
-

query-engine/connector-test-kit-rs/query-test-macros/src/relation_link_test.rs

@@ -41,6 +41,7 @@ pub fn relation_link_test_impl(attr: TokenStream, input: TokenStream) -> TokenSt
 
     // The shell function retains the name of the original test definition.
     let test_fn_ident = test_function.sig.ident;
+    let test_fn_ident_string = test_fn_ident.to_string();
 
     // Rename original test function to run_<orig_name>.
     let runner_fn_ident = Ident::new(&format!("run_{test_fn_ident}"), Span::call_site());
@@ -66,7 +67,8 @@ pub fn relation_link_test_impl(attr: TokenStream, input: TokenStream) -> TokenSt
                 &[#exclude],
                 enumflags2::make_bitflags!(ConnectorCapability::{#(#required_capabilities)|*}),
                 (#suite_name, #test_name),
-                #runner_fn_ident
+                #runner_fn_ident,
+                #test_fn_ident_string
             )
         }
 

query-engine/connector-test-kit-rs/query-tests-setup/Cargo.toml

@@ -35,6 +35,7 @@ prisma-metrics.path = "../../../libs/metrics"
 quaint.workspace = true
 jsonrpc-core = "17"
 insta.workspace = true
+futures.workspace = true
 panic-utils.path = "../../../libs/panic-utils"
 
 # Only this version is vetted, upgrade only after going through the code,

query-engine/connector-test-kit-rs/query-tests-setup/src/ignore_lists.rs

@@ -0,0 +1,48 @@
+use std::{
+    fs::File,
+    io::{BufRead, BufReader},
+    path::PathBuf,
+    sync::OnceLock,
+};
+
+static IGNORED_TESTS: OnceLock<Vec<String>> = OnceLock::new();
+static SHOULD_FAIL_TESTS: OnceLock<Vec<String>> = OnceLock::new();
+
+pub fn is_ignored(test_name: &str) -> bool {
+    is_in_list(test_name, "IGNORED_TESTS", &IGNORED_TESTS)
+}
+
+pub fn is_expected_to_fail(test_name: &str) -> bool {
+    is_in_list(test_name, "SHOULD_FAIL_TESTS", &SHOULD_FAIL_TESTS)
+}
+
+fn is_in_list(test_name: &str, env_var: &'static str, cache: &OnceLock<Vec<String>>) -> bool {
+    let list_file = match std::env::var(env_var) {
+        Ok(file) => file,
+        Err(_) => return false,
+    };
+
+    let tests = cache.get_or_init(|| {
+        let workspace_root = std::env::var("WORKSPACE_ROOT").expect("WORKSPACE_ROOT env must be set");
+
+        let path = PathBuf::from(workspace_root).join(list_file);
+        let file = File::open(path).expect("could not open file");
+        let reader = BufReader::new(file);
+
+        reader
+            .lines()
+            .map(|line| line.expect("could not read line"))
+            .map(|line| {
+                let trimmed = line.trim();
+                if line != trimmed {
+                    trimmed.to_owned()
+                } else {
+                    line
+                }
+            })
+            .filter(|line| !line.is_empty() && !line.starts_with('#'))
+            .collect()
+    });
+
+    tests.iter().any(|t| t == test_name)
+}

query-engine/connector-test-kit-rs/query-tests-setup/src/lib.rs

@@ -2,6 +2,7 @@ mod config;
 mod connector_tag;
 mod datamodel_rendering;
 mod error;
+mod ignore_lists;
 mod logging;
 mod query_result;
 mod runner;
@@ -21,10 +22,12 @@ pub use schema_gen::*;
 pub use templating::*;
 
 use colored::Colorize;
+use futures::{future::Either, FutureExt};
 use once_cell::sync::Lazy;
 use prisma_metrics::{MetricRecorder, MetricRegistry, WithMetricsInstrumentation};
 use psl::datamodel_connector::ConnectorCapabilities;
 use std::future::Future;
+use std::panic::AssertUnwindSafe;
 use tokio::runtime::Builder;
 use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender};
 use tracing_futures::WithSubscriber;
@@ -100,6 +103,7 @@ pub fn run_relation_link_test<F>(
     required_capabilities: ConnectorCapabilities,
     (suite_name, test_name): (&str, &str),
     test_fn: F,
+    test_function_name: &'static str,
 ) where
     F: (for<'a> AsyncFn<'a, Runner, DatamodelWithParams, TestResult<()>>) + 'static,
 {
@@ -123,6 +127,8 @@ pub fn run_relation_link_test<F>(
         required_capabilities,
         (suite_name, test_name),
         &boxify(test_fn),
+        std::any::type_name::<F>(),
+        test_function_name,
     )
 }
 
@@ -137,7 +143,17 @@ fn run_relation_link_test_impl(
     required_capabilities: ConnectorCapabilities,
     (suite_name, test_name): (&str, &str),
     test_fn: &dyn for<'a> Fn(&'a Runner, &'a DatamodelWithParams) -> BoxFuture<'a, TestResult<()>>,
+    test_fn_full_name: &'static str,
+    original_test_function_name: &'static str,
 ) {
+    let full_test_name = build_full_test_name(test_fn_full_name, original_test_function_name);
+
+    if ignore_lists::is_ignored(&full_test_name) {
+        return;
+    }
+
+    let expected_to_fail = ignore_lists::is_expected_to_fail(&full_test_name);
+
     static RELATION_TEST_IDX: Lazy<Option<usize>> =
         Lazy::new(|| std::env::var("RELATION_TEST_IDX").ok().and_then(|s| s.parse().ok()));
 
@@ -171,8 +187,25 @@ fn run_relation_link_test_impl(
                         .await
                         .unwrap();
 
-
-                    test_fn(&runner, &dm).with_subscriber(test_tracing_subscriber(
+                    let test_future = if expected_to_fail {
+                        Either::Left(async {
+                            match AssertUnwindSafe(test_fn(&runner, &dm)).catch_unwind().await {
+                                Ok(Ok(_)) => panic!("expected this test to fail but it succeeded"),
+                                Ok(Err(err)) => {
+                                    eprintln!("test failed as expected: {err}");
+                                    Ok::<(), TestError>(())
+                                }
+                                Err(_) => {
+                                    eprintln!("test panicked as expected");
+                                    Ok(())
+                                }
+                            }
+                        })
+                    } else {
+                        Either::Right(test_fn(&runner, &dm))
+                    };
+
+                    test_future.with_subscriber(test_tracing_subscriber(
                         ENV_LOG_LEVEL.to_string(),
                         log_tx,
                     )).with_recorder(recorder)
@@ -217,6 +250,7 @@ pub fn run_connector_test<T>(
     db_extensions: &[&str],
     referential_override: Option<String>,
     test_fn: T,
+    test_function_name: &'static str,
 ) where
     T: ConnectorTestFn,
 {
@@ -238,6 +272,8 @@ pub fn run_connector_test<T>(
         db_extensions,
         referential_override,
         &boxify(test_fn),
+        std::any::type_name::<T>(),
+        test_function_name,
     )
 }
 
@@ -254,9 +290,17 @@ fn run_connector_test_impl(
     db_extensions: &[&str],
     referential_override: Option<String>,
     test_fn: &dyn Fn(Runner) -> BoxFuture<'static, TestResult<()>>,
+    test_fn_full_name: &'static str,
+    original_test_function_name: &'static str,
 ) {
     let (connector, version) = CONFIG.test_connector().unwrap();
 
+    let full_test_name = build_full_test_name(test_fn_full_name, original_test_function_name);
+
+    if ignore_lists::is_ignored(&full_test_name) {
+        return;
+    }
+
     if !should_run(&connector, &version, only, exclude, capabilities) {
         return;
     }
@@ -292,7 +336,25 @@ fn run_connector_test_impl(
         .unwrap();
         let schema_id = runner.schema_id();
 
-        if let Err(err) = test_fn(runner)
+        let test_future = if ignore_lists::is_expected_to_fail(&full_test_name) {
+            Either::Left(async {
+                match AssertUnwindSafe(test_fn(runner)).catch_unwind().await {
+                    Ok(Ok(_)) => panic!("expected this test to fail but it succeeded"),
+                    Ok(Err(err)) => {
+                        eprintln!("test failed as expected: {err}");
+                        Ok(())
+                    }
+                    Err(_) => {
+                        eprintln!("test panicked as expected");
+                        Ok(())
+                    }
+                }
+            })
+        } else {
+            Either::Right(test_fn(runner))
+        };
+
+        if let Err(err) = test_future
             .with_subscriber(test_tracing_subscriber(ENV_LOG_LEVEL.to_string(), log_tx))
             .with_recorder(recorder)
             .await
@@ -306,6 +368,13 @@ fn run_connector_test_impl(
     });
 }
 
+fn build_full_test_name(test_fn_full_name: &'static str, original_test_function_name: &'static str) -> String {
+    let mut parts = test_fn_full_name.split("::").skip(1).collect::<Vec<_>>();
+    let last = parts.len() - 1;
+    parts[last] = original_test_function_name;
+    parts.join("::")
+}
+
 pub type LogEmit = UnboundedSender<String>;
 pub struct TestLogCapture {
     rx: UnboundedReceiver<String>,