diff --git a/tests/testthat/test-test-files.R b/tests/testthat/test-test-files.R
index 42f51e076..e4a568390 100644
--- a/tests/testthat/test-test-files.R
+++ b/tests/testthat/test-test-files.R
@@ -46,6 +46,16 @@ test_that("can control if warnings errors", {
   expect_error(test_warning(stop_on_warning = FALSE), NA)
 })
 
+test_that("can control if warnings from code outside of `testthat()` generate errors", {
+  withr::local_envvar(TESTTHAT_PARALLEL = "FALSE")
+  test_warning2 <- function(...) {
+    test_dir(test_path("test-warning2"), reporter = check_reporter(), ...)
+  }
+
+  expect_error(test_warning2(stop_on_warning = TRUE), "Tests generated warnings")
+  expect_error(test_warning2(stop_on_warning = FALSE), NA)
+})
+
 # test_file ---------------------------------------------------------------
 
 test_that("can test single file", {
diff --git a/tests/testthat/test-warning2/test-warning2.R b/tests/testthat/test-warning2/test-warning2.R
new file mode 100644
index 000000000..50ee08f48
--- /dev/null
+++ b/tests/testthat/test-warning2/test-warning2.R
@@ -0,0 +1,6 @@
+test_that(
+  desc = "no error",
+  code = expect_no_error(
+    {1 + 1}
+  )
+)