Skip to content

Commit 32d8eb3

Browse files
hadleyhadley
authored
Capture tracebacks for wrapped expectations (#1330)
Fixes #1307
1 parent 81073d0 commit 32d8eb3

22 files changed

+143
-23
lines changed

‎NEWS.md

+4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
# testthat (development version)
22

3+
* Failing expectations now include a backtrace when they're not called directly
4+
from within `test_that()` but are instead wrapped in some helper function
5+
(#1307).
6+
37
* `CheckReporter` now only records warnings when not on CRAN. Otherwise
48
failed CRAN revdep checks tend to be cluttered up with warnings (#1300).
59
It automatically cleans up `testthat-problems.rds` left over from previous

‎R/expect-comparison.R

+2-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@ expect_compare <- function(operator = c("<", "<=", ">", ">="), act, exp) {
3636
}
3737
expect(
3838
if (!is.na(cmp)) cmp else FALSE,
39-
sprintf("%s is %s %s. Difference: %.3g", act$lab, msg, exp$lab, act$val - exp$val)
39+
sprintf("%s is %s %s. Difference: %.3g", act$lab, msg, exp$lab, act$val - exp$val),
40+
trace_env = caller_env()
4041
)
4142
invisible(act$val)
4243
}

‎R/expect-constant.R

+2-1
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,8 @@ expect_waldo_constant <- function(act, constant, info) {
7777
act$lab, deparse(constant),
7878
paste0(comp, collapse = "\n\n")
7979
),
80-
info = info
80+
info = info,
81+
trace_env = caller_env()
8182
)
8283

8384
invisible(act$val)

‎R/expect-equality.R

+2-1
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,8 @@ expect_waldo_equal <- function(type, act, exp, info, ...) {
122122
exp$lab, "`expected`",
123123
paste0(comp, collapse = "\n\n")
124124
),
125-
info = info
125+
info = info,
126+
trace_env = caller_env()
126127
)
127128

128129
invisible(act$val)

‎R/expect-inheritance.R

+6-6
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,7 @@ expect_s3_class <- function(object, class, exact = FALSE) {
7474
} else if (is.character(class)) {
7575
if (!isS3(act$val)) {
7676
fail(sprintf("%s is not an S3 object", act$lab))
77-
}
78-
if (exact) {
77+
} else if (exact) {
7978
expect(
8079
identical(class(act$val), class),
8180
sprintf("%s has class %s, not %s.", act$lab, act$class, exp_lab)
@@ -108,11 +107,12 @@ expect_s4_class <- function(object, class) {
108107
} else if (is.character(class)) {
109108
if (!isS4(act$val)) {
110109
fail(sprintf("%s is not an S4 object", act$lab))
110+
} else {
111+
expect(
112+
methods::is(act$val, class),
113+
sprintf("%s inherits from %s not %s.", act$lab, act_val_lab, exp_lab)
114+
)
111115
}
112-
expect(
113-
methods::is(act$val, class),
114-
sprintf("%s inherits from %s not %s.", act$lab, act_val_lab, exp_lab)
115-
)
116116
} else {
117117
abort("`class` must be a NA or a character vector")
118118
}

‎R/expect-known.R

+2-1
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,8 @@ compare_file <- function(path, lines, ..., update = TRUE, info = NULL) {
9999
"Results have changed from known value recorded in %s.\n\n%s",
100100
encodeString(path, quote = "'"), paste0(comp, collapse = "\n\n")
101101
),
102-
info = info
102+
info = info,
103+
trace_env = caller_env()
103104
)
104105
}
105106

‎R/expect-that.R

+2-3
Original file line numberDiff line numberDiff line change
@@ -50,11 +50,10 @@ expect_that <- function(object, condition, info = NULL, label = NULL) {
5050
#' test_that("this test fails", fail())
5151
#' test_that("this test succeeds", succeed())
5252
#' }
53-
fail <- function(message = "Failure has been forced", info = NULL) {
54-
expect(FALSE, message, info = info)
53+
fail <- function(message = "Failure has been forced", info = NULL, trace_env = caller_env()) {
54+
expect(FALSE, message, info = info, trace_env = trace_env)
5555
}
5656

57-
5857
#' @rdname fail
5958
#' @export
6059
succeed <- function(message = "Success has been forced", info = NULL) {

‎R/expect-vector.R

+3-2
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,13 @@
1919
expect_vector <- function(object, ptype = NULL, size = NULL) {
2020
act <- quasi_label(enquo(object), arg = "object")
2121

22+
message <- NULL
2223
tryCatch(
2324
vctrs::vec_assert(act$val, ptype = ptype, size = size, arg = act$lab),
2425
vctrs_error_assert = function(e) {
25-
expect(FALSE, e$message)
26+
message <<- e$message
2627
}
2728
)
2829

29-
expect(TRUE, "success")
30+
expect(is.null(message), message)
3031
}

‎R/expectation.R

+24-1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@
1111
#' supplied when you need to forward a srcref captured elsewhere.
1212
#' @param trace An optional backtrace created by [rlang::trace_back()].
1313
#' When supplied, the expectation is displayed with the backtrace.
14+
#' @param trace_env If `is.null(trace)`, this is used to automatically
15+
#' generate a traceback running from `test_code()`/`test_file()` to
16+
#' `trace_env`. You'll generally only need to set this if you're wrapping
17+
#' an expectation inside another function.
1418
#' @return An expectation object. Signals the expectation condition
1519
#' with a `continue_test` restart.
1620
#'
@@ -31,7 +35,11 @@
3135
#'
3236
#' @seealso [exp_signal()]
3337
#' @export
34-
expect <- function(ok, failure_message, info = NULL, srcref = NULL, trace = NULL) {
38+
expect <- function(ok, failure_message,
39+
info = NULL,
40+
srcref = NULL,
41+
trace = NULL,
42+
trace_env = caller_env()) {
3543
type <- if (ok) "success" else "failure"
3644

3745
# Preserve existing API which appear to be used in package test code
@@ -48,10 +56,25 @@ expect <- function(ok, failure_message, info = NULL, srcref = NULL, trace = NULL
4856
}
4957
}
5058

59+
if (!ok) {
60+
if (is.null(trace)) {
61+
trace <- trace_back(
62+
top = getOption("testthat_topenv"),
63+
bottom = trace_env
64+
)
65+
}
66+
67+
# Only show if there's at least one function apart from the expectation
68+
if (trace_length(trace) <= 1) {
69+
trace <- NULL
70+
}
71+
}
72+
5173
exp <- expectation(type, message, srcref = srcref, trace = trace)
5274
exp_signal(exp)
5375
}
5476

77+
5578
#' Construct an expectation object
5679
#'
5780
#' For advanced use only. If you are creating your own expectation, you should

‎R/snapshot.R

+2-1
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,8 @@ expect_snapshot_helper <- function(lab, val,
228228
lab,
229229
paste0(comp, collapse = "\n\n"),
230230
hint
231-
)
231+
),
232+
trace_env = caller_env()
232233
)
233234
}
234235

‎R/source.R

+2
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ source_file <- function(path, env = test_env(), chdir = TRUE,
3131
old_dir <- setwd(dirname(path))
3232
on.exit(setwd(old_dir), add = TRUE)
3333
}
34+
35+
withr::local_options(testthat_topenv = env)
3436
if (wrap) {
3537
invisible(test_code(NULL, exprs, env))
3638
} else {

‎R/test-that.R

+2
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,8 @@ test_code <- function(test, code, env = test_env(), reporter = get_reporter(), s
186186
old <- options(rlang_trace_top_env = test_env)[[1]]
187187
on.exit(options(rlang_trace_top_env = old), add = TRUE)
188188

189+
withr::local_options(testthat_topenv = test_env)
190+
189191
tryCatch(
190192
withCallingHandlers(
191193
{

‎man/expect.Rd

+13-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎man/fail.Rd

+10-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎tests/testthat/_snaps/reporter-check.md

+8
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@
2121

2222
`actual`: FALSE
2323
`expected`: TRUE
24+
Backtrace:
25+
x
26+
1. \-f() reporters/tests.R:17:2
27+
2. \-testthat::expect_true(FALSE) reporters/tests.R:16:7
2428
-- Error (tests.R:23:3): Error:1 -----------------------------------------------
2529
Error: stop
2630
-- Error (tests.R:31:3): errors get tracebacks ---------------------------------
@@ -68,6 +72,10 @@
6872

6973
`actual`: FALSE
7074
`expected`: TRUE
75+
Backtrace:
76+
x
77+
1. \-f() reporters/tests.R:17:2
78+
2. \-testthat::expect_true(FALSE) reporters/tests.R:16:7
7179
-- Error (tests.R:23:3): Error:1 -----------------------------------------------
7280
Error: stop
7381
-- Error (tests.R:31:3): errors get tracebacks ---------------------------------

‎tests/testthat/_snaps/reporter-junit.md

+4-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,10 @@
1616
<failure type="failure" message="FALSE is not TRUE (tests.R:17:3)">FALSE is not TRUE
1717

1818
`actual`: FALSE
19-
`expected`: TRUE </failure>
19+
`expected`: TRUE
20+
Backtrace:
21+
1. f()
22+
2. testthat::expect_true(FALSE)</failure>
2023
</testcase>
2124
</testsuite>
2225
<testsuite name="Errors" timestamp="1999:12:31 23:59:59" hostname="nodename" tests="2" skipped="0" failures="0" errors="2" time="0">

‎tests/testthat/_snaps/reporter-progress.md

+29-2
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,9 @@
163163
| | 0 6 1 | reporters/backtraces
164164
/ | 1 6 1 | reporters/backtraces
165165
- | 1 7 1 | reporters/backtraces
166-
x | 1 7 1 | reporters/backtraces
166+
\ | 1 8 1 | reporters/backtraces
167+
| | 1 9 1 | reporters/backtraces
168+
x | 1 9 1 | reporters/backtraces
167169
--------------------------------------------------------------------------------
168170
Error (backtraces.R:6:3): errors thrown at block level are entraced
169171
Error: foo
@@ -233,10 +235,32 @@
233235
24. f(x - 1) reporters/backtraces.R:56:4
234236
25. f(x - 1) reporters/backtraces.R:56:4
235237
26. f(x - 1) reporters/backtraces.R:56:4
238+
239+
Failure (backtraces.R:66:1): (code run outside of `test_that()`)
240+
FALSE is not TRUE
241+
242+
`actual`: FALSE
243+
`expected`: TRUE
244+
Backtrace:
245+
1. f() reporters/backtraces.R:66:0
246+
2. g() reporters/backtraces.R:62:5
247+
3. h() reporters/backtraces.R:63:5
248+
4. testthat::expect_true(FALSE) reporters/backtraces.R:64:5
249+
250+
Failure (backtraces.R:69:3): nested expectations get backtraces
251+
FALSE is not TRUE
252+
253+
`actual`: FALSE
254+
`expected`: TRUE
255+
Backtrace:
256+
1. f() reporters/backtraces.R:69:2
257+
2. g() reporters/backtraces.R:62:5
258+
3. h() reporters/backtraces.R:63:5
259+
4. testthat::expect_true(FALSE) reporters/backtraces.R:64:5
236260
--------------------------------------------------------------------------------
237261

238262
== Results =====================================================================
239-
[ FAIL 7 | WARN 1 | SKIP 0 | PASS 1 ]
263+
[ FAIL 9 | WARN 1 | SKIP 0 | PASS 1 ]
240264

241265
I believe in you!
242266

@@ -263,6 +287,9 @@
263287

264288
`actual`: FALSE
265289
`expected`: TRUE
290+
Backtrace:
291+
1. f()
292+
2. testthat::expect_true(FALSE)
266293

267294

268295
[ FAIL 2 | WARN 0 | SKIP 0 | PASS 1 ]

‎tests/testthat/_snaps/reporter-stop.md

+3
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@
1212

1313
`actual`: FALSE
1414
`expected`: TRUE
15+
Backtrace:
16+
1. f()
17+
2. testthat::expect_true(FALSE)
1518

1619
-- Error (tests.R:23:3): Error:1 -----------------------------------------------
1720
Error: stop

‎tests/testthat/_snaps/reporter-summary.md

+9
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@
2727

2828
`actual`: FALSE
2929
`expected`: TRUE
30+
Backtrace:
31+
1. f() reporters/tests.R:17:2
32+
2. testthat::expect_true(FALSE) reporters/tests.R:16:7
3033

3134
-- 3. Error (tests.R:23:3): Error:1 --------------------------------------------
3235
Error: stop
@@ -69,6 +72,9 @@
6972

7073
`actual`: FALSE
7174
`expected`: TRUE
75+
Backtrace:
76+
1. f() reporters/tests.R:17:2
77+
2. testthat::expect_true(FALSE) reporters/tests.R:16:7
7278

7379
-- 3. Error (tests.R:23:3): Error:1 --------------------------------------------
7480
Error: stop
@@ -111,6 +117,9 @@
111117

112118
`actual`: FALSE
113119
`expected`: TRUE
120+
Backtrace:
121+
1. f() reporters/tests.R:17:2
122+
2. testthat::expect_true(FALSE) reporters/tests.R:16:7
114123
... and 2 more
115124

116125

‎tests/testthat/_snaps/reporter-tap.md

+3
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@
1414
1515
`actual`: FALSE
1616
`expected`: TRUE
17+
Backtrace:
18+
1. f() reporters/tests.R:17:2
19+
2. testthat::expect_true(FALSE) reporters/tests.R:16:7
1720
# Context Errors
1821
not ok 4 Error:1
1922
Error: stop

‎tests/testthat/_snaps/reporter-teamcity.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818

1919
##teamcity[testSuiteStarted name='Failure:2a']
2020
##teamcity[testStarted name='expectation 1']
21-
##teamcity[testFailed name='expectation 1' message='FALSE is not TRUE' details='|n`actual`: FALSE|n`expected`: TRUE ']
21+
##teamcity[testFailed name='expectation 1' message='FALSE is not TRUE' details='|n`actual`: FALSE|n`expected`: TRUE |nBacktrace:|n 1. f()|n 2. testthat::expect_true(FALSE)']
2222
##teamcity[testFinished name='expectation 1']
2323
##teamcity[testSuiteFinished name='Failure:2a']
2424

‎tests/testthat/reporters/backtraces.R

+10
Original file line numberDiff line numberDiff line change
@@ -58,3 +58,13 @@ test_that("deep stacks are trimmed", {
5858
f(25)
5959
})
6060

61+
# Expectations ----------------------------------------------------------------
62+
f <- function() g()
63+
g <- function() h()
64+
h <- function() expect_true(FALSE)
65+
66+
f()
67+
68+
test_that("nested expectations get backtraces", {
69+
f()
70+
})

0 commit comments

Comments
 (0)