diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py
index 38be038efcaa5..ffb2145ace390 100644
--- a/pandas/core/arrays/datetimes.py
+++ b/pandas/core/arrays/datetimes.py
@@ -817,7 +817,29 @@ def _add_offset(self, offset: BaseOffset) -> Self:
             result = type(self)._from_sequence(res_values, dtype=self.dtype)
 
         else:
-            result = type(self)._simple_new(res_values, dtype=res_values.dtype)
+            units = [
+                "ns",
+                "us",
+                "ms",
+                "s",
+            ]
+            res_unit = self.unit
+            # Only try to adjust unit if both units are recognized
+            try:
+                if hasattr(offset, "offset"):
+                    offset_td = Timedelta(offset.offset)
+                    offset_unit = offset_td.unit
+                    if self.unit in units and offset_unit in units:
+                        idx_self = units.index(self.unit)
+                        idx_offset = units.index(offset_unit)
+                        res_unit = units[min(idx_self, idx_offset)]
+            except Exception:
+                res_unit = self.unit
+            dtype = tz_to_dtype(self.tz, unit=res_unit)
+            if res_values.dtype != f"datetime64[{res_unit}]":
+                res_values = res_values.astype(f"datetime64[{res_unit}]")
+            result = type(self)._simple_new(res_values, dtype=dtype)
+
             if offset.normalize:
                 result = result.normalize()
                 result._freq = None
diff --git a/pandas/tests/arrays/test_datetimes.py b/pandas/tests/arrays/test_datetimes.py
index 199e3572732a0..505b49f7b7fb4 100644
--- a/pandas/tests/arrays/test_datetimes.py
+++ b/pandas/tests/arrays/test_datetimes.py
@@ -844,3 +844,20 @@ def test_factorize_sort_without_freq():
     tda = dta - dta[0]
     with pytest.raises(NotImplementedError, match=msg):
         tda.factorize(sort=True)
+
+
+def test_dt64_non_nano_offset_no_rounding():
+    # GH#56586
+    dti = pd.date_range("2016-01-01", periods=3, unit="s")
+    offset = pd.offsets.CustomBusinessDay(offset=pd.Timedelta("1ms"))
+    result = dti + offset
+
+    assert result.dtype == np.dtype("datetime64[ms]")
+    expected = pd.DatetimeIndex(
+        [
+            pd.Timestamp("2016-01-02 00:00:00.001"),
+            pd.Timestamp("2016-01-03 00:00:00.001"),
+            pd.Timestamp("2016-01-04 00:00:00.001"),
+        ]
+    )
+    tm.assert_index_equal(result, expected)