codegen: add Python urllib3 client

.github/workflows/python.yml

@@ -0,0 +1,99 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Python
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - ready_for_review
+      - reopened
+    paths:
+      - spec/**
+      - python/**
+      - .github/workflows/python.yml
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+env:
+  # This env var is used by Swatinem/rust-cache@v2 for the cache
+  # key, so we set it to make sure it is always consistent.
+  CARGO_TERM_COLOR: always
+  # Disable full debug symbol generation to speed up CI build and keep memory down
+  # "1" means line tables only, which is useful for panic tracebacks.
+  RUSTFLAGS: "-C debuginfo=1"
+  RUST_BACKTRACE: "1"
+  # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html
+  # CI builds are faster with incremental disabled.
+  CARGO_INCREMENTAL: "0"
+  CARGO_BUILD_JOBS: "1"
+
+jobs:
+  linux-build:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        python-version: [ 3.11 ] # Ray does not support 3.12 yet.
+    steps:
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      # pin the toolchain version to avoid surprises
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: stable
+      - uses: rui314/setup-mold@v1
+      - name: Install cargo-llvm-cov
+        uses: taiki-e/install-action@cargo-llvm-cov
+      - name: Checkout lance repo
+        uses: actions/checkout@v4
+        with:
+          repository: lancedb/lance
+          path: lance
+      - name: Checkout lance-catalog repo
+        uses: actions/checkout@v4
+        with:
+          path: lance-catalog
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: lance/python
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+      - name: Set up Python virtual environment
+        run: |
+          python -m venv venv
+          source venv/bin/activate
+          pip install maturin
+          pip install poetry
+      - name: Install Lance Python SDK
+        working-directory: lance/python
+        run: |
+          source ../../venv/bin/activate
+          maturin develop
+      - name: Test lance_catalog_urllib3_client with Python ${{ matrix.python-version }}
+        working-directory: lance-catalog/python/lance_catalog_urllib3_client
+        run: |
+          source ../../../venv/bin/activate
+          poetry install
+          poetry run pytest

.github/workflows/spec.yml

@@ -37,7 +37,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.13'
+          python-version: 3.11 # Ray does not support 3.12 yet.
           cache: 'pip'
       - name: Set up Java
         uses: actions/setup-java@v4
@@ -56,6 +56,9 @@ jobs:
       - name: Generate Rust
         working-directory: rust
         run: make gen
+      - name: Generate Python
+        working-directory: python
+        run: make gen
       - name: Check no difference in codegen
         run: |
           output=$(git diff)

README.md

@@ -13,6 +13,7 @@ and invoke operations in Lance Catalog to read, write and manage Lance tables in
 |--------------------------------------------------------------------------------|-----------------------------------------------------|
 | [spec](./spec)                                                                 | Lance Catalog OpenAPI specification                 |
 | [rust/lance-catalog-reqwest-client](./rust/lance-catalog-reqwest-client)       | Generated Rust reqwest client for Lance Catalog     |
+| [python/lance_catalog_urllib3_client](./python/lance_catalog_urllib3_client)   | Generated Python urllib3 client for Lance Catalog   |
 | [java/lance-catalog-apache-client](./java/lance-catalog-apache-client)         | Generated Java Apache HTTP client for Lance Catalog |
 | [java/lance-catalog-springboot-server](./java/lance-catalog-springboot-server) | Generated Java SpringBoot server for Lance          |
 
@@ -50,6 +51,34 @@ make gen-rust-reqwest-client
 make build
 ```
 
+### Develop Python
+
+We use [poetry](https://python-poetry.org/) for managing dependency of Python modules.
+If you are already in a virtual environment for [OpenAPI Generator](#install-openapi-generator),
+you can just run the following to install poetry in the same environment:
+
+```bash
+pip install -r python/requirements.txt
+```
+
+To run codegen and build actions:
+
+```bash
+cd python
+
+# clean all existing generated modules
+make clean
+
+# clean, and then generate all clients and servers
+make gen 
+
+# clean and generate the python urllib3 client
+make gen-python-urllib3-client
+
+# clean, generate and build all clients and servers
+make build
+```
+
 ### Develop Java
 
 ```bash

python/Makefile

@@ -0,0 +1,33 @@
+
+VERSION = 0.0.1
+
+clean-python-urllib3-client:
+	rm -rf lance-catalog-urllib3-client/**
+
+gen-python-urllib3-client: clean-python-urllib3-client
+	openapi-generator-cli generate \
+	  -i ../spec/catalog.yaml \
+	  -g python \
+	  -o lance_catalog_urllib3_client \
+	  --additional-properties=packageName=lance_catalog_urllib3_client,packageVersion=$(VERSION),library=urllib3
+  	# TODO: using .openapi-generator-ignore was not working, manually delete unused files for now
+	rm -rf lance_catalog_urllib3_client/.github
+	rm -rf lance_catalog_urllib3_client/.gitignore
+	rm -rf lance_catalog_urllib3_client/.gitlab-ci.yml
+	rm -rf lance_catalog_urllib3_client/.travis.yml
+	rm -rf lance_catalog_urllib3_client/git_push.sh
+	rm -rf lance_catalog_urllib3_client/requirements.txt
+	rm -rf lance_catalog_urllib3_client/setup.cfg
+	rm -rf lance_catalog_urllib3_client/setup.py
+	rm -rf lance_catalog_urllib3_client/test-requirements.txt
+	rm -rf lance_catalog_urllib3_client/tox.ini
+
+build-python-urllib3-client: gen-python-urllib3-client
+	cd lance_catalog_urllib3_client; \
+		poetry run pytest
+
+clean: clean-python-urllib3-client
+
+gen: gen-python-urllib3-client
+
+build: build-python-urllib3-client

python/lance_catalog_urllib3_client/.openapi-generator-ignore

@@ -0,0 +1,23 @@
+# OpenAPI Generator Ignore
+# Generated by openapi-generator https://github.com/openapitools/openapi-generator
+
+# Use this file to prevent files from being overwritten by the generator.
+# The patterns follow closely to .gitignore or .dockerignore.
+
+# As an example, the C# client generator defines ApiClient.cs.
+# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line:
+#ApiClient.cs
+
+# You can match any string of characters against a directory, file or extension with a single asterisk (*):
+#foo/*/qux
+# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux
+
+# You can recursively match patterns against a directory, file or extension with a double asterisk (**):
+#foo/**/qux
+# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux
+
+# You can also negate patterns with an exclamation (!).
+# For example, you can ignore all files in a docs folder with the file extension .md:
+#docs/*.md
+# Then explicitly reverse the ignore rule for a single file:
+#!docs/README.md

python/lance_catalog_urllib3_client/.openapi-generator/FILES

@@ -0,0 +1,34 @@
+.github/workflows/python.yml
+.gitignore
+.gitlab-ci.yml
+.travis.yml
+README.md
+docs/CreateNamespaceRequest.md
+docs/CreateNamespaceResponse.md
+docs/ErrorModel.md
+docs/GetNamespaceResponse.md
+docs/ListNamespacesResponse.md
+docs/NamespaceApi.md
+git_push.sh
+lance_catalog_urllib3_client/__init__.py
+lance_catalog_urllib3_client/api/__init__.py
+lance_catalog_urllib3_client/api/namespace_api.py
+lance_catalog_urllib3_client/api_client.py
+lance_catalog_urllib3_client/api_response.py
+lance_catalog_urllib3_client/configuration.py
+lance_catalog_urllib3_client/exceptions.py
+lance_catalog_urllib3_client/models/__init__.py
+lance_catalog_urllib3_client/models/create_namespace_request.py
+lance_catalog_urllib3_client/models/create_namespace_response.py
+lance_catalog_urllib3_client/models/error_model.py
+lance_catalog_urllib3_client/models/get_namespace_response.py
+lance_catalog_urllib3_client/models/list_namespaces_response.py
+lance_catalog_urllib3_client/py.typed
+lance_catalog_urllib3_client/rest.py
+pyproject.toml
+requirements.txt
+setup.cfg
+setup.py
+test-requirements.txt
+test/__init__.py
+tox.ini

python/lance_catalog_urllib3_client/.openapi-generator/VERSION

@@ -0,0 +1 @@
+7.12.0

python/lance_catalog_urllib3_client/README.md

@@ -0,0 +1,120 @@
+# lance-catalog-urllib3-client
+**Lance Catalog** is an OpenAPI specification on top of the storage-based Lance format.
+It provides an integration point for catalog service like Apache Hive MetaStore (HMS), Apache Gravitino, etc.
+to store and use Lance tables. To integrate, the catalog service implements a **Lance Catalog Adapter**,
+which is a REST server that converts the Lance catalog requests to native requests against the catalog service.
+Different tools can integrate with Lance Catalog using the generated OpenAPI clients in various languages,
+and invoke operations in Lance Catalog to read, write and manage Lance tables in the integrated catalog services.
+
+
+This Python package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project:
+
+- API version: 0.0.1
+- Package version: 0.0.1
+- Generator version: 7.12.0
+- Build package: org.openapitools.codegen.languages.PythonClientCodegen
+
+## Requirements.
+
+Python 3.8+
+
+## Installation & Usage
+### pip install
+
+If the python package is hosted on a repository, you can install directly using:
+
+```sh
+pip install git+https://github.com/GIT_USER_ID/GIT_REPO_ID.git
+```
+(you may need to run `pip` with root permission: `sudo pip install git+https://github.com/GIT_USER_ID/GIT_REPO_ID.git`)
+
+Then import the package:
+```python
+import lance_catalog_urllib3_client
+```
+
+### Setuptools
+
+Install via [Setuptools](http://pypi.python.org/pypi/setuptools).
+
+```sh
+python setup.py install --user
+```
+(or `sudo python setup.py install` to install the package for all users)
+
+Then import the package:
+```python
+import lance_catalog_urllib3_client
+```
+
+### Tests
+
+Execute `pytest` to run the tests.
+
+## Getting Started
+
+Please follow the [installation procedure](#installation--usage) and then run the following:
+
+```python
+
+import lance_catalog_urllib3_client
+from lance_catalog_urllib3_client.rest import ApiException
+from pprint import pprint
+
+# Defining the host is optional and defaults to http://localhost:2333
+# See configuration.py for a list of all supported configuration parameters.
+configuration = lance_catalog_urllib3_client.Configuration(
+    host = "http://localhost:2333"
+)
+
+
+
+# Enter a context with an instance of the API client
+with lance_catalog_urllib3_client.ApiClient(configuration) as api_client:
+    # Create an instance of the API class
+    api_instance = lance_catalog_urllib3_client.NamespaceApi(api_client)
+    create_namespace_request = lance_catalog_urllib3_client.CreateNamespaceRequest() # CreateNamespaceRequest | 
+
+    try:
+        # Create a new namespace. A catalog can manage one or more namespaces. A namespace is used to manage one or more tables. There are three modes when trying to create a namespace:   * CREATE: Create the namespace if it does not exist. If a namespace of the same name already exists, the operation fails with 400.   * EXIST_OK: Create the namespace if it does not exist. If a namespace of the same name already exists, the operation succeeds and the existing namespace is kept.   * OVERWRITE: Create the namespace if it does not exist. If a namespace of the same name already exists, the existing namespace is dropped and a new namespace with this name with no table is created. 
+        api_response = api_instance.create_namespace(create_namespace_request)
+        print("The response of NamespaceApi->create_namespace:\n")
+        pprint(api_response)
+    except ApiException as e:
+        print("Exception when calling NamespaceApi->create_namespace: %s\n" % e)
+
+```
+
+## Documentation for API Endpoints
+
+All URIs are relative to *http://localhost:2333*
+
+Class | Method | HTTP request | Description
+------------ | ------------- | ------------- | -------------
+*NamespaceApi* | [**create_namespace**](docs/NamespaceApi.md#create_namespace) | **POST** /v1/namespaces | Create a new namespace. A catalog can manage one or more namespaces. A namespace is used to manage one or more tables. There are three modes when trying to create a namespace:   * CREATE: Create the namespace if it does not exist. If a namespace of the same name already exists, the operation fails with 400.   * EXIST_OK: Create the namespace if it does not exist. If a namespace of the same name already exists, the operation succeeds and the existing namespace is kept.   * OVERWRITE: Create the namespace if it does not exist. If a namespace of the same name already exists, the existing namespace is dropped and a new namespace with this name with no table is created. 
+*NamespaceApi* | [**drop_namespace**](docs/NamespaceApi.md#drop_namespace) | **DELETE** /v1/namespaces/{ns} | Drop a namespace from the catalog. Namespace must be empty.
+*NamespaceApi* | [**get_namespace**](docs/NamespaceApi.md#get_namespace) | **GET** /v1/namespaces/{ns} | Get information about a namespace
+*NamespaceApi* | [**list_namespaces**](docs/NamespaceApi.md#list_namespaces) | **GET** /v1/namespaces | List all namespaces in the catalog. 
+*NamespaceApi* | [**namespace_exists**](docs/NamespaceApi.md#namespace_exists) | **HEAD** /v1/namespaces/{ns} | Check if a namespace exists
+
+
+## Documentation For Models
+
+ - [CreateNamespaceRequest](docs/CreateNamespaceRequest.md)
+ - [CreateNamespaceResponse](docs/CreateNamespaceResponse.md)
+ - [ErrorModel](docs/ErrorModel.md)
+ - [GetNamespaceResponse](docs/GetNamespaceResponse.md)
+ - [ListNamespacesResponse](docs/ListNamespacesResponse.md)
+
+
+<a id="documentation-for-authorization"></a>
+## Documentation For Authorization
+
+Endpoints do not require authorization.
+
+
+## Author
+
+
+
+