Filter NaN effects in Hausman pretest, add clustered aggregation tests

Address rerun review findings:
- Filter common (g,t) cells with non-finite effect estimates before
  building Hausman delta/covariance (prevents NaN poisoning from
  no-valid-pairs cells)
- Make _nan_result return recommendation="inconclusive" for consistency
  (reject=False + "pt_post" was misleading)
- Add clustered SE tests for aggregate=event_study and aggregate=all
  (exercises cluster path through aggregation methods)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: igerber

diff_diff/efficient_did.py

@@ -1077,17 +1077,21 @@ def _compute_survey_eif_se(self, eif_vals: np.ndarray) -> float:
         vcov = compute_survey_vcov(X_ones, eif_vals, self._unit_resolved_survey)
         return float(np.sqrt(np.abs(vcov[0, 0])))
 
-    def _eif_se(self, eif_vals: np.ndarray, n_units: int) -> float:
+    def _eif_se(
+        self,
+        eif_vals: np.ndarray,
+        n_units: int,
+        cluster_indices: Optional[np.ndarray] = None,
+        n_clusters: Optional[int] = None,
+    ) -> float:
         """Compute SE from aggregated EIF scores.
 
         Dispatches to survey TSL when ``_unit_resolved_survey`` is set
         (during fit), otherwise uses cluster-robust or standard formula.
         """
         if self._unit_resolved_survey is not None:
             return self._compute_survey_eif_se(eif_vals)
-        return _compute_se_from_eif(
-            eif_vals, n_units, self._cluster_indices, self._n_clusters
-        )
+        return _compute_se_from_eif(eif_vals, n_units, cluster_indices, n_clusters)
 
     # -- Aggregation helpers --------------------------------------------------
 
@@ -1196,7 +1200,7 @@ def _aggregate_overall(
 
         # SE = sqrt(mean(EIF^2) / n) — standard IF-based SE
         # (dispatches to survey TSL or cluster-robust when active)
-        se = self._eif_se(agg_eif_total, n_units)
+        se = self._eif_se(agg_eif_total, n_units, cluster_indices, n_clusters)
 
         return overall_att, se
 
@@ -1290,7 +1294,7 @@ def _aggregate_event_study(
                 )
                 agg_eif = agg_eif + wif
 
-            agg_se = self._eif_se(agg_eif, n_units)
+            agg_se = self._eif_se(agg_eif, n_units, cluster_indices, n_clusters)
 
             t_stat, p_val, ci = safe_inference(
                 agg_eff, agg_se, alpha=self.alpha, df=self._survey_df
@@ -1354,7 +1358,7 @@ def _aggregate_by_group(
             agg_eif = np.zeros(n_units)
             for k, gt in enumerate(g_gts):
                 agg_eif += w[k] * eif_by_gt[gt]
-            agg_se = self._eif_se(agg_eif, n_units)
+            agg_se = self._eif_se(agg_eif, n_units, cluster_indices, n_clusters)
 
             t_stat, p_val, ci = safe_inference(
                 agg_eff, agg_se, alpha=self.alpha, df=self._survey_df
@@ -1455,7 +1459,7 @@ def hausman_pretest(
             set(result_all.group_time_effects.keys()) & set(result_post.group_time_effects.keys())
         )
 
-        def _nan_result(recommendation: str = "pt_post") -> HausmanPretestResult:
+        def _nan_result() -> HausmanPretestResult:
             return HausmanPretestResult(
                 statistic=np.nan,
                 p_value=np.nan,
@@ -1464,13 +1468,23 @@ def _nan_result(recommendation: str = "pt_post") -> HausmanPretestResult:
                 alpha=alpha,
                 att_all=result_all.overall_att,
                 att_post=result_post.overall_att,
-                recommendation=recommendation,
+                recommendation="inconclusive",
                 gt_details=None,
             )
 
         if not common_gts:
             return _nan_result()
 
+        # Filter out (g,t) cells with non-finite effect estimates
+        common_gts = [
+            gt
+            for gt in common_gts
+            if np.isfinite(result_all.group_time_effects[gt]["effect"])
+            and np.isfinite(result_post.group_time_effects[gt]["effect"])
+        ]
+        if not common_gts:
+            return _nan_result()
+
         k = len(common_gts)
 
         # Build EIF matrices for common (g,t) pairs: (n_units, k)
@@ -1572,7 +1586,7 @@ def _eif_cov(eif_mat: np.ndarray) -> np.ndarray:
         # Effective rank = number of positive eigenvalues
         effective_rank = int(np.sum(eigvals > tol))
         if effective_rank == 0:
-            return _nan_result("pt_all")
+            return _nan_result()
 
         # Compute H = delta' @ pinv(V) @ delta
         V_pinv = np.linalg.pinv(V, rcond=tol / max_eigval if max_eigval > 0 else 1e-10)

tests/test_efficient_did.py

@@ -588,7 +588,7 @@ def test_hausman_homogeneous_trends_fail_to_reject(self):
         assert np.isfinite(pretest.p_value)
         assert pretest.df > 0
         # With homogeneous trends, should generally fail to reject
-        assert pretest.recommendation in ("pt_all", "pt_post")
+        assert pretest.recommendation in ("pt_all", "pt_post", "inconclusive")
 
     def test_hausman_differential_trends_detects(self):
         """DGP with cohort-specific trends → test detects or warns."""
@@ -636,7 +636,7 @@ def test_hausman_differential_trends_detects(self):
         # Both are acceptable outcomes for a DGP that violates PT-All
         if np.isfinite(pretest.statistic):
             assert pretest.statistic >= 0
-        assert pretest.recommendation in ("pt_all", "pt_post")
+        assert pretest.recommendation in ("pt_all", "pt_post", "inconclusive")
 
     def test_hausman_gt_details(self):
         """gt_details should have expected columns."""
@@ -650,7 +650,7 @@ def test_hausman_recommendation_field(self):
         """recommendation should be pt_all or pt_post."""
         df = _make_staggered_panel(n_per_group=80, n_control=100)
         pretest = EfficientDiD.hausman_pretest(df, "y", "unit", "time", "first_treat")
-        assert pretest.recommendation in ("pt_all", "pt_post")
+        assert pretest.recommendation in ("pt_all", "pt_post", "inconclusive")
         if pretest.reject:
             assert pretest.recommendation == "pt_post"
         else:
@@ -700,7 +700,7 @@ def test_hausman_clustered(self):
         pretest = EfficientDiD.hausman_pretest(
             df, "y", "unit", "time", "first_treat", cluster="cluster_id"
         )
-        assert pretest.recommendation in ("pt_all", "pt_post")
+        assert pretest.recommendation in ("pt_all", "pt_post", "inconclusive")
         assert pretest.df >= 0
 
     def test_hausman_last_cohort(self):
@@ -719,7 +719,7 @@ def test_hausman_last_cohort(self):
             "first_treat",
             control_group="last_cohort",
         )
-        assert pretest.recommendation in ("pt_all", "pt_post")
+        assert pretest.recommendation in ("pt_all", "pt_post", "inconclusive")
         assert np.isfinite(pretest.att_all)
         assert np.isfinite(pretest.att_post)
 
@@ -803,6 +803,27 @@ def test_clustered_se_at_least_as_large(self):
         assert result_clustered.overall_se > 0
         assert result_unclustered.overall_se > 0
 
+    def test_clustered_aggregate_event_study(self):
+        """Clustered SE with aggregate='event_study' should produce finite results."""
+        df = self._make_clustered_panel(n_clusters=60, units_per_cluster=3)
+        result = EfficientDiD(cluster="cluster_id").fit(
+            df, "y", "unit", "time", "first_treat", aggregate="event_study"
+        )
+        assert result.event_study_effects is not None
+        for e, d in result.event_study_effects.items():
+            assert np.isfinite(d["se"])
+
+    def test_clustered_aggregate_all(self):
+        """Clustered SE with aggregate='all' should produce finite results."""
+        df = self._make_clustered_panel(n_clusters=60, units_per_cluster=3)
+        result = EfficientDiD(cluster="cluster_id").fit(
+            df, "y", "unit", "time", "first_treat", aggregate="all"
+        )
+        assert result.event_study_effects is not None
+        assert result.group_effects is not None
+        for g, d in result.group_effects.items():
+            assert np.isfinite(d["se"])
+
     def test_cluster_bootstrap(self, ci_params):
         """Cluster bootstrap should produce finite inference."""
         n_boot = ci_params.bootstrap(99)