Skip to content

Commit 3b14f82

Browse files
gaborcsardigaborcsardi
authored
Fix reporting for parallel tests (#1828)
Closes #1827.
1 parent f31b024 commit 3b14f82

11 files changed

+91
-30
lines changed

‎R/reporter-list.R

+9-2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ ListReporter <- R6::R6Class("ListReporter",
1414
current_start_time = NA,
1515
current_expectations = NULL,
1616
current_file = NULL,
17+
current_context = NULL,
18+
current_test = NULL,
1719
results = NULL,
1820

1921
initialize = function() {
@@ -23,8 +25,13 @@ ListReporter <- R6::R6Class("ListReporter",
2325
},
2426

2527
start_test = function(context, test) {
26-
self$current_expectations <- Stack$new()
27-
self$current_start_time <- proc.time()
28+
if (!identical(self$current_context, context) ||
29+
!identical(self$current_test, test)) {
30+
self$current_context <- context
31+
self$current_test <- test
32+
self$current_expectations <- Stack$new()
33+
self$current_start_time <- proc.time()
34+
}
2835
},
2936

3037
add_result = function(context, test, result) {

‎tests/testthat/test-parallel-crash.R

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ test_that("crash", {
99
do <- function() {
1010
err <- NULL
1111
tryCatch(
12-
testthat::test_local(".", reporter = "silent", stop_on_failure = FALSE),
12+
testthat::test_local(".", reporter = "summary", stop_on_failure = FALSE),
1313
error = function(e) err <<- e
1414
)
1515
err

‎tests/testthat/test-parallel-outside.R

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@
22
test_that("error outside of test_that()", {
33
withr::local_envvar(TESTTHAT_PARALLEL = "TRUE")
44
err <- tryCatch(
5-
suppressMessages(testthat::test_local(
5+
capture.output(suppressMessages(testthat::test_local(
66
test_path("test-parallel", "outside"),
7-
reporter = "silent"
8-
)),
7+
reporter = "summary"
8+
))),
99
error = function(e) e
1010
)
1111

‎tests/testthat/test-parallel-setup.R

+3-3
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@ test_that("error in parallel setup code", {
33
skip_on_covr()
44
withr::local_envvar(TESTTHAT_PARALLEL = "TRUE")
55
err <- tryCatch(
6-
suppressMessages(testthat::test_local(
6+
capture.output(suppressMessages(testthat::test_local(
77
test_path("test-parallel", "setup"),
8-
reporter = "silent"
9-
)),
8+
reporter = "summary"
9+
))),
1010
error = function(e) e
1111
)
1212
expect_s3_class(err, "testthat_process_error")

‎tests/testthat/test-parallel-startup.R

+3-3
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@ test_that("startup error", {
33
skip_on_covr()
44
withr::local_envvar(TESTTHAT_PARALLEL = "TRUE")
55
err <- tryCatch(
6-
suppressMessages(testthat::test_local(
6+
capture.output(suppressMessages(testthat::test_local(
77
test_path("test-parallel", "startup"),
8-
reporter = "silent"
9-
)),
8+
reporter = "summary"
9+
))),
1010
error = function(e) e
1111
)
1212
expect_s3_class(err, "testthat_process_error")

‎tests/testthat/test-parallel-teardown.R

+3-3
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@ test_that("teardown error", {
33
skip("teardown errors are ignored")
44
withr::local_envvar(TESTTHAT_PARALLEL = "TRUE")
55
err <- tryCatch(
6-
suppressMessages(testthat::test_local(
6+
capture.output(suppressMessages(testthat::test_local(
77
test_path("test-parallel", "teardown"),
8-
reporter = "silent"
9-
)),
8+
reporter = "summary"
9+
))),
1010
error = function(e) e
1111
)
1212
expect_s3_class(err, "testthat_process_error")

‎tests/testthat/test-parallel.R

+39-15
Original file line numberDiff line numberDiff line change
@@ -24,27 +24,45 @@ test_that("detect number of cpus to use", {
2424

2525
test_that("ok", {
2626
withr::local_envvar(c(TESTTHAT_PARALLEL = "TRUE"))
27-
suppressMessages(ret <- test_local(
27+
# we cannot run these with the silent reporter, because it is not
28+
# parallel compatible, and they'll not run in parallel
29+
capture.output(suppressMessages(ret <- test_local(
2830
test_path("test-parallel", "ok"),
29-
reporter = "silent",
31+
reporter = "summary",
3032
stop_on_failure = FALSE
31-
))
33+
)))
3234
tdf <- as.data.frame(ret)
3335
tdf <- tdf[order(tdf$file), ]
3436
expect_equal(tdf$failed, c(0,1,0))
3537
expect_equal(tdf$skipped, c(FALSE, FALSE, TRUE))
3638
})
3739

40+
test_that("fail", {
41+
withr::local_envvar(c(TESTTHAT_PARALLEL = "TRUE"))
42+
# we cannot run these with the silent reporter, because it is not
43+
# parallel compatible, and they'll not run in parallel
44+
capture.output(suppressMessages(ret <- test_local(
45+
test_path("test-parallel", "fail"),
46+
reporter = "summary",
47+
stop_on_failure = FALSE
48+
)))
49+
tdf <- as.data.frame(ret)
50+
tdf <- tdf[order(tdf$file), ]
51+
expect_equal(tdf$failed, c(1))
52+
})
53+
3854
test_that("snapshots", {
3955
withr::local_envvar(c(TESTTHAT_PARALLEL = "TRUE"))
4056
on.exit(unlink(tmp, recursive = TRUE), add = TRUE)
4157
dir.create(tmp <- tempfile("testthat-snap-"))
4258
file.copy(test_path("test-parallel", "snap"), tmp, recursive = TRUE)
43-
suppressMessages(ret <- test_local(
59+
# we cannot run these with the silent reporter, because it is not
60+
# parallel compatible, and they'll not run in parallel
61+
capture.output(suppressMessages(ret <- test_local(
4462
file.path(tmp, "snap"),
45-
reporter = "silent",
63+
reporter = "summary",
4664
stop_on_failure = FALSE
47-
))
65+
)))
4866
tdf <- as.data.frame(ret)
4967
tdf <- tdf[order(tdf$file), ]
5068
expect_equal(tdf$failed, c(0,0,1))
@@ -60,11 +78,13 @@ test_that("new snapshots are added", {
6078
dir.create(tmp <- tempfile("testthat-snap-"))
6179
file.copy(test_path("test-parallel", "snap"), tmp, recursive = TRUE)
6280
unlink(file.path(tmp, "snap", "tests", "testthat", "_snaps", "snap-2.md"))
63-
suppressMessages(ret <- test_local(
81+
# we cannot run these with the silent reporter, because it is not
82+
# parallel compatible, and they'll not run in parallel
83+
capture.output(suppressMessages(ret <- test_local(
6484
file.path(tmp, "snap"),
65-
reporter = "silent",
85+
reporter = "summary",
6686
stop_on_failure = FALSE
67-
))
87+
)))
6888
tdf <- as.data.frame(ret)
6989
tdf <- tdf[order(tdf$file), ]
7090
expect_equal(tdf$failed, c(0,0,1))
@@ -83,11 +103,13 @@ test_that("snapshots are removed if test file has no snapshots", {
83103
"test_that(\"2\", { expect_true(TRUE) })",
84104
file.path(tmp, "snap", "tests", "testthat", "test-snap-2.R")
85105
)
86-
suppressMessages(ret <- test_local(
106+
# we cannot run these with the silent reporter, because it is not
107+
# parallel compatible, and they'll not run in parallel
108+
capture.output(suppressMessages(ret <- test_local(
87109
file.path(tmp, "snap"),
88-
reporter = "silent",
110+
reporter = "summary",
89111
stop_on_failure = FALSE
90-
))
112+
)))
91113
tdf <- as.data.frame(ret)
92114
tdf <- tdf[order(tdf$file), ]
93115
expect_equal(tdf$failed, c(0,0,1))
@@ -104,11 +126,13 @@ test_that("snapshots are removed if test file is removed", {
104126
file.copy(test_path("test-parallel", "snap"), tmp, recursive = TRUE)
105127
unlink(file.path(tmp, "snap", "tests", "testthat", "test-snap-2.R"))
106128
withr::local_envvar(CI = NA_character_)
107-
suppressMessages(ret <- test_local(
129+
# we cannot run these with the silent reporter, because it is not
130+
# parallel compatible, and they'll not run in parallel
131+
capture.output(suppressMessages(ret <- test_local(
108132
file.path(tmp, "snap"),
109-
reporter = "silent",
133+
reporter = "summary",
110134
stop_on_failure = FALSE
111-
))
135+
)))
112136
tdf <- as.data.frame(ret)
113137
tdf <- tdf[order(tdf$file), ]
114138
expect_equal(tdf$failed, c(0,1))

‎tests/testthat/test-parallel/fail/DESCRIPTION

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
Package: ok
2+
Title: What the Package Does (One Line, Title Case)
3+
Version: 0.0.0.9000
4+
Authors@R:
5+
person(given = "First",
6+
family = "Last",
7+
role = c("aut", "cre"),
8+
email = "[email protected]",
9+
comment = c(ORCID = "YOUR-ORCID-ID"))
10+
Description: What the package does (one paragraph).
11+
License: `use_mit_license()`, `use_gpl3_license()` or friends to
12+
pick a license
13+
Encoding: UTF-8
14+
LazyData: true
15+
Roxygen: list(markdown = TRUE)
16+
RoxygenNote: 7.1.1
17+
Suggests:
18+
testthat
19+
Config/testthat/parallel: true
20+
Config/testthat/edition: 3

‎tests/testthat/test-parallel/fail/NAMESPACE

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Generated by roxygen2: do not edit by hand
2+

‎tests/testthat/test-parallel/fail/tests/testthat.R

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
library(testthat)
2+
library(ok)
3+
4+
test_check("ok")

‎tests/testthat/test-parallel/fail/tests/testthat/test-bad.R

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
2+
test_that("bad test", {
3+
expect_true(FALSE)
4+
})

0 commit comments

Comments
 (0)