Skip to content

Commit b606f64

Browse files
igerberclaude
andcommitted
Fix CI review: survey-weighted jackknife guards, benchmark timing, docstring
P1: Add effective-support guards checking composed weights (omega * w_control) and treated survey weights, plus per-iteration zero-sum guards in LOO loops. P2: Restore R benchmark total_seconds to placebo-only (matches se field). P3: Update fit() docstring for non-bootstrap survey rejection, add 4 survey jackknife tests, update REGISTRY.md edge cases. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 56c71fa commit b606f64

5 files changed

Lines changed: 125 additions & 4 deletions

File tree

benchmarks/R/benchmark_synthdid.R

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ se_jk_matrix <- vcov(tau_hat, method = "jackknife")
8787
se_jackknife <- as.numeric(sqrt(se_jk_matrix[1, 1]))
8888
se_jk_time <- as.numeric(difftime(Sys.time(), se_jk_start, units = "secs"))
8989

90-
total_time <- estimation_time + se_time + se_jk_time
90+
total_time <- estimation_time + se_time # placebo only, matches `se` field
9191

9292
# Compute noise level and regularization (to match Python's auto-computed values)
9393
N0 <- setup$N0

diff_diff/synthetic_did.py

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -223,8 +223,9 @@ def fit( # type: ignore[override]
223223
survey_design : SurveyDesign, optional
224224
Survey design specification. Only pweight weight_type is supported.
225225
Strata/PSU/FPC are supported via Rao-Wu rescaled bootstrap when
226-
variance_method='bootstrap'. Placebo variance does not support
227-
strata/PSU/FPC; use variance_method='bootstrap' for full designs.
226+
variance_method='bootstrap'. Non-bootstrap variance methods
227+
(placebo, jackknife) do not support strata/PSU/FPC; use
228+
variance_method='bootstrap' for full designs.
228229
229230
Returns
230231
-------
@@ -1210,6 +1211,29 @@ def _jackknife_se(
12101211
)
12111212
return np.nan, np.array([])
12121213

1214+
# --- Effective-support guards for survey-weighted path ---
1215+
if w_control is not None:
1216+
effective_control = unit_weights * w_control
1217+
if np.sum(effective_control > 0) <= 1:
1218+
warnings.warn(
1219+
"Jackknife variance requires more than 1 control unit with "
1220+
"positive effective weight (omega * survey_weight). "
1221+
"Consider variance_method='placebo'.",
1222+
UserWarning,
1223+
stacklevel=3,
1224+
)
1225+
return np.nan, np.array([])
1226+
1227+
if w_treated is not None and np.sum(w_treated > 0) <= 1:
1228+
warnings.warn(
1229+
"Jackknife variance requires more than 1 treated unit with "
1230+
"positive survey weight. "
1231+
"Consider variance_method='placebo'.",
1232+
UserWarning,
1233+
stacklevel=3,
1234+
)
1235+
return np.nan, np.array([])
1236+
12131237
jackknife_estimates = np.empty(n)
12141238

12151239
# --- Precompute treated means (constant across control-LOO) ---
@@ -1238,6 +1262,10 @@ def _jackknife_se(
12381262
# Compose with survey weights if present
12391263
if w_control is not None:
12401264
omega_jk = omega_jk * w_control[mask]
1265+
if omega_jk.sum() == 0:
1266+
jackknife_estimates[j] = np.nan
1267+
mask[j] = True
1268+
continue
12411269
omega_jk = omega_jk / omega_jk.sum()
12421270

12431271
jackknife_estimates[j] = compute_sdid_estimator(
@@ -1259,6 +1287,10 @@ def _jackknife_se(
12591287
# Recompute treated means from remaining units
12601288
if w_treated is not None:
12611289
w_t_jk = w_treated[mask]
1290+
if w_t_jk.sum() == 0:
1291+
jackknife_estimates[n_control + k] = np.nan
1292+
mask[k] = True
1293+
continue
12621294
t_pre_mean = np.average(
12631295
Y_pre_treated[:, mask], axis=1, weights=w_t_jk
12641296
)

docs/methodology/REGISTRY.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1510,6 +1510,7 @@ Convergence criterion: stop when objective decrease < min_decrease² (default mi
15101510
- **Jackknife with single treated unit**: Returns NaN SE. Cannot leave-one-out with N_tr=1; R returns NA for the same condition.
15111511
- **Jackknife with single nonzero-weight control**: Returns NaN SE. Leaving out the only effective control is not meaningful.
15121512
- **Jackknife with non-finite LOO estimate**: Returns NaN SE. Unlike bootstrap/placebo, jackknife is deterministic and cannot skip failed iterations; NaN propagates through `var()` (matches R behavior).
1513+
- **Jackknife with survey weights**: Guards on effective positive support (omega * w_control > 0 and w_treated > 0) after composition, not raw FW counts. Returns NaN SE if fewer than 2 effective controls or 2 positive-weight treated units. Per-iteration zero-sum guards return NaN for individual LOO iterations when remaining composed weights sum to zero.
15131514
- **Note:** Survey support: weights, strata, PSU, and FPC are all supported. Full-design surveys use Rao-Wu rescaled bootstrap (Phase 6); non-bootstrap variance methods (`variance_method="placebo"` or `"jackknife"`) require weights-only (strata/PSU/FPC require bootstrap). Both sides weighted per WLS regression interpretation: treated-side means are survey-weighted (Frank-Wolfe target and ATT formula); control-side synthetic weights are composed with survey weights post-optimization (ω_eff = ω * w_co, renormalized). Frank-Wolfe optimization itself is unweighted — survey importance enters after trajectory-matching. Covariate residualization uses WLS with survey weights. Placebo, jackknife, and bootstrap SE preserve survey weights on both sides.
15141515

15151516
**Reference implementation(s):**

tests/test_methodology_sdid.py

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -611,6 +611,80 @@ def test_jackknife_n_bootstrap_none_in_results(self):
611611
)
612612
assert results.n_bootstrap is None
613613

614+
def test_jackknife_with_pweights(self):
615+
"""Jackknife should produce finite SE with survey pweights."""
616+
from diff_diff.survey import SurveyDesign
617+
618+
df = _make_panel(n_control=15, n_treated=3, seed=42)
619+
# Add unit-constant survey weights
620+
unit_weights = {u: 1.0 + u * 0.1 for u in df["unit"].unique()}
621+
df["weight"] = df["unit"].map(unit_weights)
622+
623+
sdid = SyntheticDiD(variance_method="jackknife", seed=42)
624+
results = sdid.fit(
625+
df, outcome="outcome", treatment="treated",
626+
unit="unit", time="period",
627+
post_periods=list(range(5, 8)),
628+
survey_design=SurveyDesign(weights="weight"),
629+
)
630+
assert results.se > 0
631+
assert np.isfinite(results.se)
632+
assert results.variance_method == "jackknife"
633+
634+
def test_jackknife_zero_effective_control_nan(self):
635+
"""Zero-weight controls after composition -> NaN SE."""
636+
from diff_diff.survey import SurveyDesign
637+
638+
# 3 controls, 2 treated. Set all but 1 control survey weight to 0
639+
# so effective support <= 1.
640+
df = _make_panel(n_control=3, n_treated=2, seed=42)
641+
weights = {}
642+
control_units = sorted(df.loc[df["treated"] == 0, "unit"].unique())
643+
treated_units = sorted(df.loc[df["treated"] == 1, "unit"].unique())
644+
# Only first control gets positive weight
645+
for i, u in enumerate(control_units):
646+
weights[u] = 1.0 if i == 0 else 0.0
647+
for u in treated_units:
648+
weights[u] = 1.0
649+
df["weight"] = df["unit"].map(weights)
650+
651+
with warnings.catch_warnings():
652+
warnings.simplefilter("ignore")
653+
sdid = SyntheticDiD(variance_method="jackknife", seed=42)
654+
results = sdid.fit(
655+
df, outcome="outcome", treatment="treated",
656+
unit="unit", time="period",
657+
post_periods=list(range(5, 7)),
658+
survey_design=SurveyDesign(weights="weight"),
659+
)
660+
assert np.isnan(results.se)
661+
662+
def test_jackknife_zero_treated_weight_nan(self):
663+
"""Single positive-weight treated unit with survey -> NaN SE."""
664+
from diff_diff.survey import SurveyDesign
665+
666+
df = _make_panel(n_control=10, n_treated=2, seed=42)
667+
weights = {}
668+
treated_units = sorted(df.loc[df["treated"] == 1, "unit"].unique())
669+
control_units = sorted(df.loc[df["treated"] == 0, "unit"].unique())
670+
for u in control_units:
671+
weights[u] = 1.0
672+
# Only first treated unit gets positive weight
673+
weights[treated_units[0]] = 1.0
674+
weights[treated_units[1]] = 0.0
675+
df["weight"] = df["unit"].map(weights)
676+
677+
with warnings.catch_warnings():
678+
warnings.simplefilter("ignore")
679+
sdid = SyntheticDiD(variance_method="jackknife", seed=42)
680+
results = sdid.fit(
681+
df, outcome="outcome", treatment="treated",
682+
unit="unit", time="period",
683+
post_periods=list(range(5, 7)),
684+
survey_design=SurveyDesign(weights="weight"),
685+
)
686+
assert np.isnan(results.se)
687+
614688

615689
# =============================================================================
616690
# Jackknife SE - R Golden Value Parity

tests/test_survey_phase5.py

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,21 @@ def test_full_design_bootstrap_smoke(self, sdid_survey_data, survey_design_full)
198198
def test_full_design_placebo_raises(self, sdid_survey_data, survey_design_full):
199199
"""Placebo variance with full design raises NotImplementedError."""
200200
est = SyntheticDiD(variance_method="placebo", n_bootstrap=50, seed=42)
201-
with pytest.raises(NotImplementedError, match="placebo.*does not support strata/PSU/FPC"):
201+
with pytest.raises(NotImplementedError, match="does not support strata/PSU/FPC"):
202+
est.fit(
203+
sdid_survey_data,
204+
outcome="outcome",
205+
treatment="treated",
206+
unit="unit",
207+
time="time",
208+
post_periods=[6, 7, 8, 9],
209+
survey_design=survey_design_full,
210+
)
211+
212+
def test_full_design_jackknife_raises(self, sdid_survey_data, survey_design_full):
213+
"""Jackknife variance with full design raises NotImplementedError."""
214+
est = SyntheticDiD(variance_method="jackknife", seed=42)
215+
with pytest.raises(NotImplementedError, match="does not support strata/PSU/FPC"):
202216
est.fit(
203217
sdid_survey_data,
204218
outcome="outcome",

0 commit comments

Comments
 (0)