Skip to content

Commit 1b97f78

Browse files
igerberclaude
andcommitted
Fix NaN t-statistics across 7 locations for consistent undefined inference
Replace `else 0.0` with `else np.nan` when SE is non-finite or zero in t-stat calculations across sun_abraham.py, triple_diff.py, and diagnostics.py. Add CI guards returning (NaN, NaN) for 4 downstream confidence interval computations. Matches the CallawaySantAnna pattern established in PR #97. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent 3f8e9cb commit 1b97f78

7 files changed

Lines changed: 401 additions & 11 deletions

File tree

diff_diff/diagnostics.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -662,7 +662,7 @@ def permutation_test(
662662
ci_upper = np.percentile(valid_effects, (1 - alpha / 2) * 100)
663663

664664
# T-stat from original estimate
665-
t_stat = original_att / se if se > 0 else 0.0
665+
t_stat = original_att / se if np.isfinite(se) and se > 0 else np.nan
666666

667667
return PlaceboTestResults(
668668
test_type="permutation",
@@ -783,14 +783,14 @@ def leave_one_out_test(
783783
# Statistics of LOO distribution
784784
mean_effect = np.mean(valid_effects)
785785
se = np.std(valid_effects, ddof=1) if len(valid_effects) > 1 else 0.0
786-
t_stat = mean_effect / se if se > 0 else 0.0
786+
t_stat = mean_effect / se if np.isfinite(se) and se > 0 else np.nan
787787

788788
# Use t-distribution for p-value
789789
df = len(valid_effects) - 1 if len(valid_effects) > 1 else 1
790790
p_value = compute_p_value(t_stat, df=df)
791791

792792
# CI
793-
conf_int = compute_confidence_interval(mean_effect, se, alpha, df=df)
793+
conf_int = compute_confidence_interval(mean_effect, se, alpha, df=df) if np.isfinite(se) and se > 0 else (np.nan, np.nan)
794794

795795
return PlaceboTestResults(
796796
test_type="leave_one_out",

diff_diff/sun_abraham.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -600,9 +600,9 @@ def fit(
600600
coef_index_map,
601601
)
602602

603-
overall_t = overall_att / overall_se if overall_se > 0 else 0.0
603+
overall_t = overall_att / overall_se if np.isfinite(overall_se) and overall_se > 0 else np.nan
604604
overall_p = compute_p_value(overall_t)
605-
overall_ci = compute_confidence_interval(overall_att, overall_se, self.alpha)
605+
overall_ci = compute_confidence_interval(overall_att, overall_se, self.alpha) if np.isfinite(overall_se) and overall_se > 0 else (np.nan, np.nan)
606606

607607
# Run bootstrap if requested
608608
bootstrap_results = None
@@ -623,7 +623,7 @@ def fit(
623623

624624
# Update results with bootstrap inference
625625
overall_se = bootstrap_results.overall_att_se
626-
overall_t = overall_att / overall_se if overall_se > 0 else 0.0
626+
overall_t = overall_att / overall_se if np.isfinite(overall_se) and overall_se > 0 else np.nan
627627
overall_p = bootstrap_results.overall_att_p_value
628628
overall_ci = bootstrap_results.overall_att_ci
629629

@@ -640,7 +640,7 @@ def fit(
640640
eff_val = event_study_effects[e]["effect"]
641641
se_val = event_study_effects[e]["se"]
642642
event_study_effects[e]["t_stat"] = (
643-
eff_val / se_val if se_val > 0 else 0.0
643+
eff_val / se_val if np.isfinite(se_val) and se_val > 0 else np.nan
644644
)
645645

646646
# Convert cohort effects to storage format
@@ -878,9 +878,9 @@ def _compute_iw_effects(
878878
agg_var = float(weight_vec @ vcov_subset @ weight_vec)
879879
agg_se = np.sqrt(max(agg_var, 0))
880880

881-
t_stat = agg_effect / agg_se if agg_se > 0 else 0.0
881+
t_stat = agg_effect / agg_se if np.isfinite(agg_se) and agg_se > 0 else np.nan
882882
p_val = compute_p_value(t_stat)
883-
ci = compute_confidence_interval(agg_effect, agg_se, self.alpha)
883+
ci = compute_confidence_interval(agg_effect, agg_se, self.alpha) if np.isfinite(agg_se) and agg_se > 0 else (np.nan, np.nan)
884884

885885
event_study_effects[e] = {
886886
"effect": agg_effect,

diff_diff/triple_diff.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -598,14 +598,14 @@ def fit(
598598
)
599599

600600
# Compute inference
601-
t_stat = att / se if se > 0 else 0.0
601+
t_stat = att / se if np.isfinite(se) and se > 0 else np.nan
602602
df = n_obs - 8 # Approximate df (8 cell means)
603603
if covariates:
604604
df -= len(covariates)
605605
df = max(df, 1)
606606

607607
p_value = compute_p_value(t_stat, df=df)
608-
conf_int = compute_confidence_interval(att, se, self.alpha, df=df)
608+
conf_int = compute_confidence_interval(att, se, self.alpha, df=df) if np.isfinite(se) and se > 0 else (np.nan, np.nan)
609609

610610
# Get number of clusters if clustering
611611
n_clusters = None

docs/methodology/REGISTRY.md

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ This document provides the academic foundations and key implementation requireme
1616
- [TripleDifference](#tripledifference)
1717
- [TROP](#trop)
1818
4. [Diagnostics & Sensitivity](#diagnostics--sensitivity)
19+
- [PlaceboTests](#placebotests)
1920
- [BaconDecomposition](#bacondecomposition)
2021
- [HonestDiD](#honestdid)
2122
- [PreTrendsPower](#pretrendspower)
@@ -319,6 +320,11 @@ where weights ŵ_{g,e} = n_{g,e} / Σ_g n_{g,e} (sample share of cohort g at eve
319320
- Detection: Pivoted QR decomposition with tolerance `1e-07` (R's `qr()` default)
320321
- Handling: Warns and drops linearly dependent columns, sets NA for dropped coefficients (R-style, matches `lm()`)
321322
- Parameter: `rank_deficient_action` controls behavior: "warn" (default), "error", or "silent"
323+
- NaN inference for undefined statistics:
324+
- t_stat: Uses NaN (not 0.0) when SE is non-finite or zero
325+
- p_value and CI: Also NaN when t_stat is NaN
326+
- Applies to overall ATT, per-effect event study, and aggregated event study
327+
- **Note**: Defensive enhancement matching CallawaySantAnna behavior; R's `fixest::sunab()` may produce Inf/NaN without warning
322328

323329
**Reference implementation(s):**
324330
- R: `fixest::sunab()` (Laurent Bergé's implementation)
@@ -429,6 +435,10 @@ Doubly robust estimator:
429435
- Propensity scores near 0/1: trimmed at `pscore_trim` (default 0.01)
430436
- Empty cells: raises ValueError with diagnostic message
431437
- Collinear covariates: automatic detection and warning
438+
- NaN inference for undefined statistics:
439+
- t_stat: Uses NaN (not 0.0) when SE is non-finite or zero
440+
- p_value and CI: Also NaN when t_stat is NaN
441+
- **Note**: Defensive enhancement; reference implementation behavior not yet documented
432442

433443
**Reference implementation(s):**
434444
- Authors' replication code (forthcoming)
@@ -656,6 +666,18 @@ For joint method, LOOCV works as follows:
656666

657667
# Diagnostics & Sensitivity
658668

669+
## PlaceboTests
670+
671+
**Module:** `diff_diff/diagnostics.py`
672+
673+
*Edge cases:*
674+
- NaN inference for undefined statistics:
675+
- `permutation_test`: t_stat is NaN when permutation SE is zero (all permutations produce identical estimates)
676+
- `leave_one_out_test`: t_stat, p_value, CI are NaN when LOO SE is zero (all LOO effects identical)
677+
- **Note**: Defensive enhancement matching CallawaySantAnna NaN convention
678+
679+
---
680+
659681
## BaconDecomposition
660682

661683
**Primary source:** [Goodman-Bacon, A. (2021). Difference-in-differences with variation in treatment timing. *Journal of Econometrics*, 225(2), 254-277.](https://doi.org/10.1016/j.jeconom.2021.03.014)

tests/test_diagnostics.py

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -672,3 +672,128 @@ def test_returns_dict_structure(self, simple_panel_data):
672672
# Check that each result is either PlaceboTestResults or error dict
673673
for key, value in results.items():
674674
assert isinstance(value, (PlaceboTestResults, dict))
675+
676+
677+
class TestDiagnosticsTStatNaN:
678+
"""Tests for NaN t_stat when SE is invalid in diagnostic functions."""
679+
680+
def test_permutation_test_tstat_nan_when_se_zero(self):
681+
"""permutation_test t_stat is NaN when SE is zero (all permutations identical)."""
682+
np.random.seed(42)
683+
684+
# Create data where all units have deterministic outcomes
685+
# so permutation distribution has zero variance
686+
n_units = 20
687+
data = []
688+
for unit in range(n_units):
689+
is_treated = unit < n_units // 2
690+
for post in [0, 1]:
691+
y = 5.0
692+
if is_treated and post == 1:
693+
y += 2.0
694+
data.append({
695+
"unit": unit,
696+
"post": post,
697+
"outcome": y,
698+
"treated": int(is_treated),
699+
})
700+
701+
df = pd.DataFrame(data)
702+
703+
import warnings
704+
with warnings.catch_warnings(record=True):
705+
warnings.simplefilter("always")
706+
result = permutation_test(
707+
df,
708+
outcome="outcome",
709+
treatment="treated",
710+
time="post",
711+
unit="unit",
712+
n_permutations=20,
713+
seed=42,
714+
)
715+
716+
se = result.se
717+
t_stat = result.t_stat
718+
719+
if not np.isfinite(se) or se == 0:
720+
assert np.isnan(t_stat), (
721+
f"permutation t_stat should be NaN when SE={se}, got {t_stat}"
722+
)
723+
else:
724+
expected = result.original_effect / se
725+
assert np.isclose(t_stat, expected), (
726+
f"permutation t_stat should be effect/SE, "
727+
f"expected {expected}, got {t_stat}"
728+
)
729+
730+
def test_leave_one_out_tstat_nan_when_se_zero(self):
731+
"""leave_one_out_test t_stat and CI are NaN when SE is zero."""
732+
np.random.seed(42)
733+
734+
# Create data where leaving out any unit gives identical results
735+
# (deterministic outcomes, no noise)
736+
n_units = 20
737+
data = []
738+
for unit in range(n_units):
739+
is_treated = unit < n_units // 2
740+
for post in [0, 1]:
741+
y = 5.0
742+
if is_treated and post == 1:
743+
y += 2.0
744+
data.append({
745+
"unit": unit,
746+
"post": post,
747+
"outcome": y,
748+
"treated": int(is_treated),
749+
})
750+
751+
df = pd.DataFrame(data)
752+
753+
import warnings
754+
with warnings.catch_warnings(record=True):
755+
warnings.simplefilter("always")
756+
result = leave_one_out_test(
757+
df,
758+
outcome="outcome",
759+
treatment="treated",
760+
time="post",
761+
unit="unit",
762+
)
763+
764+
se = result.se
765+
t_stat = result.t_stat
766+
767+
if not np.isfinite(se) or se == 0:
768+
assert np.isnan(t_stat), (
769+
f"LOO t_stat should be NaN when SE={se}, got {t_stat}"
770+
)
771+
ci = result.conf_int
772+
assert np.isnan(ci[0]) and np.isnan(ci[1]), (
773+
f"LOO conf_int should be (NaN, NaN) when SE={se}, got {ci}"
774+
)
775+
776+
def test_permutation_tstat_consistency(self, simple_panel_data):
777+
"""permutation_test t_stat = effect/SE when SE is valid."""
778+
result = permutation_test(
779+
simple_panel_data,
780+
outcome="outcome",
781+
treatment="treated",
782+
time="post",
783+
unit="unit",
784+
n_permutations=50,
785+
seed=42,
786+
)
787+
788+
se = result.se
789+
t_stat = result.t_stat
790+
791+
if not np.isfinite(se) or se == 0:
792+
assert np.isnan(t_stat), (
793+
f"t_stat should be NaN when SE={se}, got {t_stat}"
794+
)
795+
else:
796+
expected = result.original_effect / se
797+
assert np.isclose(t_stat, expected), (
798+
f"t_stat should be effect/SE, expected {expected}, got {t_stat}"
799+
)

0 commit comments

Comments
 (0)