Add S3_ENDPOINT to customize S3 endpoint URL (#432)

Author: Edward Park

CHANGELOG.md

@@ -5,18 +5,21 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.17.3] - 2024-02-29
+
+### Added
+- Added the environment variable `S3_ENDPOINT` to accomodate for nonstandard S3 Endpoint URLs when asking for presigned URLs
+
 ## [0.17.2](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.17.2) - 2024-02-28
 
 ### Modified
 - In `Dataset.create_slice`, the `reference_ids` parameter is now optional. If left unspecified, it will create an empty slice
 
-
 ## [0.17.1](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.17.1) - 2024-02-22
 
 ### Added
 - Environment variable `NUCLEUS_SKIP_SSL_VERIFY` to skip SSL verification on requests
 
-
 ## [0.17.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.17.0) - 2024-02-06
 
 ### Added
@@ -26,11 +29,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixes
 - Fix test `test_models.test_remove_invalid_tag_from_model`
 
-## [0.17.1] - 2024-02-22
-
-### Added
-- Environment variable `NUCLEUS_SKIP_SSL_VERIFY` to skip SSL verification on requests
-
 ## [0.16.18](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.16.18) - 2024-02-06
 
 ### Added

nucleus/utils.py

@@ -3,6 +3,7 @@
 import io
 import json
 import os
+import urllib.request
 import uuid
 from collections import defaultdict
 from typing import IO, TYPE_CHECKING, Dict, List, Sequence, Tuple, Type, Union
@@ -381,9 +382,14 @@ def serialize_and_write_to_presigned_url(
 ):
     """This helper function can be used to serialize a list of API objects to NDJSON."""
     request_id = uuid.uuid4().hex
+    route = f"dataset/{dataset_id}/signedUrl/{request_id}"
+    if os.environ.get("S3_ENDPOINT") is not None:
+        route += "?s3Endpoint=" + urllib.request.pathname2url(
+            os.environ["S3_ENDPOINT"]
+        )
     response = client.make_request(
         payload={},
-        route=f"dataset/{dataset_id}/signedUrl/{request_id}",
+        route=route,
         requests_command=requests.get,
     )
 

pyproject.toml

@@ -25,7 +25,7 @@ ignore = ["E501", "E741", "E731", "F401"]  # Easy ignore for getting it running
 
 [tool.poetry]
 name = "scale-nucleus"
-version = "0.17.2"
+version = "0.17.3"
 description = "The official Python client library for Nucleus, the Data Platform for AI"
 license =  "MIT"
 authors = ["Scale AI Nucleus Team <[email protected]>"]