VER: Release 0.25.0

See release notes.

Author: nmacholl

CHANGELOG.md

@@ -1,8 +1,20 @@
 # Changelog
 
+## 0.25.0 - 2024-01-09
+
+#### Breaking changes
+- Removed deprecated `pretty_px` parameter for `DBNStore.to_df`; `price_type` can be used instead
+
+#### Bug fixes
+- Fixed an issue where the `Live` client would not raise an exception when reading an incompatible DBN version
+- Fixed an issue where sending lots of subscriptions could cause a `BufferError`
+- Fixed an issue where `Historical.batch.download` was slow
+- Fixed an issue where `Historical.timeseries.get_range` was slow
+- Fixed an issue where reading a DBN file with non-empty metadata symbol mappings and mixed `SType` would cause an error when mapping symbols (credit: Jakob Lövhall)
+
 ## 0.24.1 - 2023-12-15
 
-##### Enhancements
+#### Enhancements
 - Added new publisher value for OPRA MIAX Sapphire
 
 #### Bug fixes

databento/common/dbnstore.py

@@ -4,7 +4,6 @@
 import decimal
 import itertools
 import logging
-import warnings
 from collections.abc import Generator
 from collections.abc import Iterator
 from io import BytesIO
@@ -826,7 +825,6 @@ def to_csv(
     @overload
     def to_df(
         self,
-        pretty_px: bool | None = ...,
         price_type: Literal["fixed", "float", "decimal"] = ...,
         pretty_ts: bool = ...,
         map_symbols: bool = ...,
@@ -838,7 +836,6 @@ def to_df(
     @overload
     def to_df(
         self,
-        pretty_px: bool | None = ...,
         price_type: Literal["fixed", "float", "decimal"] = ...,
         pretty_ts: bool = ...,
         map_symbols: bool = ...,
@@ -849,7 +846,6 @@ def to_df(
 
     def to_df(
         self,
-        pretty_px: bool | None = None,
         price_type: Literal["fixed", "float", "decimal"] = "float",
         pretty_ts: bool = True,
         map_symbols: bool = True,
@@ -861,11 +857,6 @@ def to_df(
 
         Parameters
         ----------
-        pretty_px : bool, default True
-            This parameter is deprecated and will be removed in a future release.
-            If all price columns should be converted from `int` to `float` at
-            the correct scale (using the fixed-precision scalar 1e-9). Null
-            prices are replaced with NaN.
         price_type : str, default "float"
             The price type to use for price fields.
             If "fixed", prices will have a type of `int` in fixed decimal format; each unit representing 1e-9 or 0.000000001.
@@ -899,20 +890,6 @@ def to_df(
             If the schema for the array cannot be determined.
 
         """
-        if pretty_px is True:
-            warnings.warn(
-                'The argument `pretty_px` is deprecated and will be removed in a future release; `price_type="float"` can be used instead.',
-                DeprecationWarning,
-                stacklevel=2,
-            )
-        elif pretty_px is False:
-            price_type = "fixed"
-            warnings.warn(
-                'The argument `pretty_px` is deprecated and will be removed in a future release; `price_type="fixed"` can be used instead.',
-                DeprecationWarning,
-                stacklevel=2,
-            )
-
         schema = validate_maybe_enum(schema, Schema, "schema")
         if schema is None:
             if self.schema is None:

databento/common/symbology.py

@@ -242,8 +242,8 @@ def insert_metadata(self, metadata: Metadata) -> None:
             # Nothing to do
             return
 
-        stype_in = SType(metadata.stype_in)
-        stype_out = SType(metadata.stype_out)
+        stype_in = SType(metadata.stype_in) if metadata.stype_in is not None else None
+        stype_out = SType(metadata.stype_out) if metadata.stype_out is not None else None
 
         for symbol_in, entries in metadata.mappings.items():
             for entry in entries:

databento/historical/api/batch.py

@@ -370,7 +370,7 @@ def _download_file(
 
             logger.debug("Starting download of file %s", output_path.name)
             with open(output_path, mode=mode) as f:
-                for chunk in response.iter_content():
+                for chunk in response.iter_content(chunk_size=None):
                     f.write(chunk)
             logger.debug("Download of %s completed", output_path.name)
 

databento/historical/http.py

@@ -23,7 +23,6 @@
 from databento.common.system import USER_AGENT
 
 
-_32KIB = 1024 * 32  # 32_768
 WARNING_HEADER_FIELD: str = "X-Warning"
 
 
@@ -133,7 +132,7 @@ def _stream(
             else:
                 writer = open(path, "x+b")
 
-            for chunk in response.iter_content(chunk_size=_32KIB):
+            for chunk in response.iter_content(chunk_size=None):
                 writer.write(chunk)
 
             if path is None:

databento/live/protocol.py

@@ -283,6 +283,7 @@ def subscribe(
         stype_in_valid = validate_enum(stype_in, SType, "stype_in")
         symbols_list = optional_symbols_list_to_list(symbols, stype_in_valid)
 
+        subscription_bytes: list[bytes] = []
         for batch in chunk(symbols_list, SYMBOL_LIST_BATCH_SIZE):
             batch_str = ",".join(batch)
             message = SubscriptionRequest(
@@ -291,8 +292,9 @@ def subscribe(
                 symbols=batch_str,
                 start=optional_datetime_to_unix_nanoseconds(start),
             )
+            subscription_bytes.append(bytes(message))
 
-            self.transport.write(bytes(message))
+        self.transport.writelines(subscription_bytes)
 
     def start(
         self,
@@ -312,8 +314,6 @@ def _process_dbn(self, data: bytes) -> None:
         try:
             self._dbn_decoder.write(bytes(data))
             records = self._dbn_decoder.decode()
-        except ValueError:
-            pass  # expected for partial records
         except Exception:
             logger.exception("error decoding DBN record")
             self.__transport.close()

databento/live/session.py

@@ -366,12 +366,15 @@ def subscribe(
                     loop=self._loop,
                 )
 
-            self._protocol.subscribe(
+        asyncio.run_coroutine_threadsafe(
+            self._subscribe_task(
                 schema=schema,
                 symbols=symbols,
                 stype_in=stype_in,
                 start=start,
-            )
+            ),
+            loop=self._loop,
+        ).result()
 
     def resume_reading(self) -> None:
         """
@@ -477,7 +480,7 @@ async def _connect_task(
         )
 
         try:
-            await asyncio.wait_for(
+            session_id = await asyncio.wait_for(
                 protocol.authenticated,
                 timeout=AUTH_TIMEOUT_SECONDS,
             )
@@ -488,9 +491,29 @@ async def _connect_task(
             ) from None
         except ValueError as exc:
             raise BentoError(f"User authentication failed: {exc!s}") from None
+        else:
+            logger.info("assigned session id %s", session_id)
 
         logger.info(
             "authentication with remote gateway completed",
         )
 
         return transport, protocol
+
+    async def _subscribe_task(
+        self,
+        schema: Schema | str,
+        symbols: Iterable[str] | Iterable[Number] | str | Number = ALL_SYMBOLS,
+        stype_in: SType | str = SType.RAW_SYMBOL,
+        start: str | int | None = None,
+    ) -> None:
+        with self._lock:
+            if self._protocol is None:
+                return
+
+            self._protocol.subscribe(
+                schema=schema,
+                symbols=symbols,
+                stype_in=stype_in,
+                start=start,
+            )

databento/version.py

@@ -1 +1 @@
-__version__ = "0.24.1"
+__version__ = "0.25.0"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "databento"
-version = "0.24.1"
+version = "0.25.0"
 description = "Official Python client library for Databento"
 authors = [
     "Databento <[email protected]>",

tests/test_common_symbology.py

@@ -166,7 +166,7 @@ def create_metadata(
     dataset: str = "UNIT.TEST",
     start: int = UNDEF_TIMESTAMP,
     end: int = UNDEF_TIMESTAMP,
-    stype_in: SType = SType.RAW_SYMBOL,
+    stype_in: SType | None = SType.RAW_SYMBOL,
     stype_out: SType = SType.INSTRUMENT_ID,
     schema: Schema = Schema.TRADES,
     limit: int | None = None,
@@ -198,10 +198,18 @@ def test_instrument_map(
     assert instrument_map._data == {}
 
 
+@pytest.mark.parametrize(
+    "stype_in",
+    [
+        SType.RAW_SYMBOL,
+        None,
+    ],
+)
 def test_instrument_map_insert_metadata(
     instrument_map: InstrumentMap,
     start_date: pd.Timestamp,
     end_date: pd.Timestamp,
+    stype_in: SType | None,
 ) -> None:
     """
     Test the insertion of DBN Metadata.
@@ -224,6 +232,7 @@ def test_instrument_map_insert_metadata(
     ]
 
     metadata = create_metadata(
+        stype_in=stype_in,
         mappings=mappings,
     )