From 722d4d84a8c7007d5f305def71d08a8156aadbc5 Mon Sep 17 00:00:00 2001 From: Vladislav Polyakov <polrk@ydb.tech> Date: Sun, 20 Apr 2025 18:43:16 +0300 Subject: [PATCH 01/12] Add devcontainer configuration for YDB Python SDK - Create Dockerfile for building the SDK environment - Add docker-compose configuration for YDB services - Include YDB configuration file for storage and channel profiles - Set up initialization and configuration scripts for YDB CLI - Configure Prometheus for monitoring YDB services --- .devcontainer/Dockerfile | 8 +++++ .devcontainer/commands/initialize.sh | 5 +++ .devcontainer/commands/postCreate.sh | 22 ++++++++++++ .devcontainer/commands/postStart.sh | 8 +++++ .devcontainer/compose.yml | 51 ++++++++++++++++++++++++++ .devcontainer/devcontainer.json | 54 ++++++++++++++++++++++++++++ 6 files changed, 148 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100755 .devcontainer/commands/initialize.sh create mode 100755 .devcontainer/commands/postCreate.sh create mode 100755 .devcontainer/commands/postStart.sh create mode 100644 .devcontainer/compose.yml create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000..723c1843 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,8 @@ +FROM mcr.microsoft.com/devcontainers/python:3.12-bookworm + +# [Optional] Uncomment if you want to install more tools +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends <your-pkg> + +# [Optional] Uncomment if you want to install ydb cli +RUN curl -fsSL https://raw.githubusercontent.com/ydb-platform/ydb/refs/heads/main/ydb/apps/ydb/install/install.sh | bash diff --git a/.devcontainer/commands/initialize.sh b/.devcontainer/commands/initialize.sh new file mode 100755 index 00000000..9be0294d --- /dev/null +++ b/.devcontainer/commands/initialize.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e + +git config --local user.email "$(git config user.email)" +git config --local user.name "$(git config user.name)" diff --git a/.devcontainer/commands/postCreate.sh b/.devcontainer/commands/postCreate.sh new file mode 100755 index 00000000..f05b6be9 --- /dev/null +++ b/.devcontainer/commands/postCreate.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e + + +# Set up YDB profile if ydb cli exists +if which ydb > /dev/null 2>&1; then + ENDPOINT=$(echo ${YDB_CONNECTION_STRING_SECURE:-$YDB_CONNECTION_STRING} | awk -F/ '{print $1 "//" $3}') + DATABASE=$(echo ${YDB_CONNECTION_STRING_SECURE:-$YDB_CONNECTION_STRING} | cut -d/ -f4-) + CA_FILE_OPTION="" + + if [ -n "$YDB_SSL_ROOT_CERTIFICATES_FILE" ]; then + ENDPOINT="${ENDPOINT/grpc:/grpcs:}" + CA_FILE_OPTION="--ca-file ${YDB_SSL_ROOT_CERTIFICATES_FILE}" + fi + + ydb config profile replace local \ + --endpoint "$ENDPOINT" \ + --database "/$DATABASE" \ + $CA_FILE_OPTION + + ydb config profile activate local +fi diff --git a/.devcontainer/commands/postStart.sh b/.devcontainer/commands/postStart.sh new file mode 100755 index 00000000..4ffe6fe3 --- /dev/null +++ b/.devcontainer/commands/postStart.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + + +# Install dependencies +pip install -r requirements.txt +# Install test dependencies +# pip install -r test-requirements.txt diff --git a/.devcontainer/compose.yml b/.devcontainer/compose.yml new file mode 100644 index 00000000..56039007 --- /dev/null +++ b/.devcontainer/compose.yml @@ -0,0 +1,51 @@ +volumes: + ydb-data: + # driver: local + # driver_opts: + # type: tmpfs + # device: tmpfs + # o: size=80g + ydb-certs: + +services: + sdk: + build: + context: . + dockerfile: Dockerfile + hostname: sdk + + volumes: + - ydb-certs:/ydb_certs + - ../:/workspaces/ydb-python-sdk:cached + + environment: + - YDB_IMAGE_VERSION=24.3 + - YDB_CREDENTIALS_USER=root + - YDB_CREDENTIALS_PASSWORD=1234 + - YDB_CONNECTION_STRING=grpc://ydb:2136/local + - YDB_CONNECTION_STRING_SECURE=grpcs://ydb:2135/local + - YDB_SSL_ROOT_CERTIFICATES_FILE=/ydb_certs/ca.pem + + # Overrides default command so things don't shut down after the process ends. + command: sleep infinity + + ydb: + image: ghcr.io/ydb-platform/local-ydb:24.3 + restart: unless-stopped + hostname: ydb + platform: linux/amd64 + + ports: + - 2135:2135 + - 2136:2136 + - 8765:8765 + + volumes: + - ydb-data:/ydb_data + - ydb-certs:/ydb_certs + + environment: + - YDB_USE_IN_MEMORY_PDISKS=true + - GRPC_TLS_PORT=2135 + - GRPC_PORT=2136 + - MON_PORT=8765 diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..836e0f3c --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,54 @@ +{ + "name": "Python & YDB", + "service": "sdk", + "dockerComposeFile": "compose.yml", + "workspaceFolder": "/workspaces/ydb-python-sdk", + // Allows the container to use ptrace, which is useful for debugging. + "capAdd": [ + "SYS_PTRACE" + ], + // Disables seccomp, which can be necessary for some debugging tools to function correctly. + "securityOpt": [ + "seccomp=unconfined" + ], + // Features to add to the dev container. More info: https://containers.dev/features. + "features": { + "ghcr.io/devcontainers/features/git": {}, + "ghcr.io/devcontainers/features/common-utils": {}, + "ghcr.io/devcontainers/features/github-cli:1": {} + }, + // Use 'forwardPorts' to make a list of ports inside the container available locally. + "forwardPorts": [ + 2135, + 2136, + 8765, + 9090, + 9464 + ], + // Use 'initializeCommand' to run commands before the container is created. + "initializeCommand": "chmod +x .devcontainer/commands/initialize.sh && .devcontainer/commands/initialize.sh", + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "chmod +x .devcontainer/commands/postCreate.sh && .devcontainer/commands/postCreate.sh", + // Use 'postStartCommand' to run commands after the container is started. + "postStartCommand": "chmod +x .devcontainer/commands/postStart.sh && .devcontainer/commands/postStart.sh", + // Configure tool-specific properties. + "customizations": { + "vscode": { + "extensions": [ + "ms-python.autopep8", + "ms-python.debugpy", + "ms-python.flake8", + "ms-python.isort", + "ms-python.pylint", + "ms-python.python", + "ms-python.vscode-pylance", + "ms-python.vscode-python-envs" + ] + } + }, + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + "remoteUser": "root", + "mounts": [ + "source=${localEnv:HOME}/.ssh,target=/root/.ssh,type=bind,readonly" + ] +} From c7b7bfaf71ab49418a6950530510f252ac8195b1 Mon Sep 17 00:00:00 2001 From: Vladislav Polyakov <polrk@ydb.tech> Date: Tue, 22 Apr 2025 12:52:58 +0300 Subject: [PATCH 02/12] Update devcontainer mount to use specific SSH signing key Signed-off-by: Vladislav Polyakov <polrk@ydb.tech> --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 836e0f3c..21e43676 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -49,6 +49,6 @@ // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. "remoteUser": "root", "mounts": [ - "source=${localEnv:HOME}/.ssh,target=/root/.ssh,type=bind,readonly" + "source=${localEnv:HOME}/.ssh/id_ed25519_signing,target=/root/.ssh/id_ed25519_signing,type=bind,readonly" ] } From 98d2d96622247c0753f1eae5034846b05d16d91d Mon Sep 17 00:00:00 2001 From: Vladislav Polyakov <polrk@ydb.tech> Date: Tue, 22 Apr 2025 14:50:13 +0300 Subject: [PATCH 03/12] Add sign-off configuration and GPG settings in initialization scripts Signed-off-by: Vladislav Polyakov <polrk@ydb.tech> --- .devcontainer/commands/initialize.sh | 3 ++- .devcontainer/commands/postCreate.sh | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.devcontainer/commands/initialize.sh b/.devcontainer/commands/initialize.sh index 9be0294d..926031b6 100755 --- a/.devcontainer/commands/initialize.sh +++ b/.devcontainer/commands/initialize.sh @@ -1,5 +1,6 @@ #!/bin/bash set -e -git config --local user.email "$(git config user.email)" +git config --local format.signoff true git config --local user.name "$(git config user.name)" +git config --local user.email "$(git config user.email)" diff --git a/.devcontainer/commands/postCreate.sh b/.devcontainer/commands/postCreate.sh index f05b6be9..40078bf5 100755 --- a/.devcontainer/commands/postCreate.sh +++ b/.devcontainer/commands/postCreate.sh @@ -1,6 +1,11 @@ #!/bin/bash set -e +if git config --get commit.gpgsign | grep -q true; then + git config --global gpg.format ssh + git config --global gpg.ssh.defaultKeyCommand 'ssh-add -L' + git config --global gpg.ssh.allowedSigners '~/.ssh/allowed_signers' +fi # Set up YDB profile if ydb cli exists if which ydb > /dev/null 2>&1; then From 646f5c5b45bab7be253e03723d223c57e16c0147 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk <vgvoleg@gmail.com> Date: Wed, 23 Apr 2025 18:53:01 +0300 Subject: [PATCH 04/12] Update postStart.sh --- .devcontainer/commands/postStart.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.devcontainer/commands/postStart.sh b/.devcontainer/commands/postStart.sh index 4ffe6fe3..bf38ac35 100755 --- a/.devcontainer/commands/postStart.sh +++ b/.devcontainer/commands/postStart.sh @@ -4,5 +4,8 @@ set -e # Install dependencies pip install -r requirements.txt -# Install test dependencies -# pip install -r test-requirements.txt +# Install ydb package +pip install -e . +# Install tox for CI +pip install tox + From b29f9bf43b0c9584c98e1aaaf455091d190d114a Mon Sep 17 00:00:00 2001 From: Vladislav Polyakov <polRk@ydb.tech> Date: Fri, 25 Apr 2025 08:14:01 +0000 Subject: [PATCH 05/12] chore: simplify tox compose setup and support devcontainers Signed-off-by: Vladislav Polyakov <polRk@ydb.tech> --- compose.yml | 22 ++++++++++++++++++++++ docker-compose-tls.yml | 14 -------------- docker-compose.yml | 11 ----------- tox.ini | 15 ++++++++------- 4 files changed, 30 insertions(+), 32 deletions(-) create mode 100644 compose.yml delete mode 100644 docker-compose-tls.yml delete mode 100644 docker-compose.yml diff --git a/compose.yml b/compose.yml new file mode 100644 index 00000000..2834961c --- /dev/null +++ b/compose.yml @@ -0,0 +1,22 @@ +networks: + default: + external: true + name: ydb-python-sdk_default + +services: + py-sdk-ydb: + image: ydbplatform/local-ydb:trunk + restart: always + hostname: py-sdk-ydb + ports: + - 2135 + - 2136 + - 8765 + volumes: + - ./ydb_certs:/ydb_certs + environment: + - YDB_USE_IN_MEMORY_PDISKS=true + - YDB_ENABLE_COLUMN_TABLES=true + - GRPC_TLS_PORT=2135 + - GRPC_PORT=2136 + - MON_PORT=8765 diff --git a/docker-compose-tls.yml b/docker-compose-tls.yml deleted file mode 100644 index f0a4b328..00000000 --- a/docker-compose-tls.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: "3.9" -services: - ydb: - image: ydbplatform/local-ydb:trunk - restart: always - ports: - - 2136:2136 - - 2135:2135 - hostname: localhost - volumes: - - ./ydb_certs:/ydb_certs - environment: - - YDB_USE_IN_MEMORY_PDISKS=true - - YDB_ENABLE_COLUMN_TABLES=true diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 1a466fab..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: "3.3" -services: - ydb: - image: ydbplatform/local-ydb:trunk - restart: always - ports: - - 2136:2136 - hostname: localhost - environment: - - YDB_USE_IN_MEMORY_PDISKS=true - - YDB_ENABLE_COLUMN_TABLES=true diff --git a/tox.ini b/tox.ini index f91e7d8a..a458b180 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,7 @@ usedevelop = True install_command = pip install {opts} {packages} setenv = PYTHONPATH = {env:PYTHONPATH}{:}{toxinidir} + REMOTE_CONTAINERS = {env:REMOTE_CONTAINERS} deps = -r{toxinidir}/test-requirements.txt @@ -32,14 +33,14 @@ deps = [testenv:py] commands = - pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs} + pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=compose.yml {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<6.0.0 [testenv:py-proto4] commands = - pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs} + pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=compose.yml {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<5.0.0 @@ -48,35 +49,35 @@ deps = commands = pytest -v -m "not tls" \ --cov-report html:cov_html --cov=ydb \ - --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs} + --docker-compose-remove-volumes --docker-compose=compose.yml {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<5.0.0 [testenv:py-proto3] commands = - pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs} + pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=compose.yml {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<4.0.0 [testenv:py-tls] commands = - pytest -v -m tls --docker-compose-remove-volumes --docker-compose=docker-compose-tls.yml {posargs} + pytest -v -m tls --docker-compose-remove-volumes --docker-compose=compose.yml {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<6.0.0 [testenv:py-tls-proto4] commands = - pytest -v -m tls --docker-compose-remove-volumes --docker-compose=docker-compose-tls.yml {posargs} + pytest -v -m tls --docker-compose-remove-volumes --docker-compose=compose.yml {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<5.0.0 [testenv:py-tls-proto3] commands = - pytest -v -m tls --docker-compose-remove-volumes --docker-compose=docker-compose-tls.yml {posargs} + pytest -v -m tls --docker-compose-remove-volumes --docker-compose=compose.yml {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<4.0.0 From 51f0ac23faa93cd93bd9c90ac6ae93940ef81e9c Mon Sep 17 00:00:00 2001 From: Vladislav Polyakov <polRk@ydb.tech> Date: Fri, 25 Apr 2025 08:14:34 +0000 Subject: [PATCH 06/12] chore(devcontainer): fix port forwarding Signed-off-by: Vladislav Polyakov <polRk@ydb.tech> --- .devcontainer/compose.yml | 17 ++++++++++------- .devcontainer/devcontainer.json | 11 +++++------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.devcontainer/compose.yml b/.devcontainer/compose.yml index 56039007..9e6c5fae 100644 --- a/.devcontainer/compose.yml +++ b/.devcontainer/compose.yml @@ -1,3 +1,5 @@ +name: ydb-python-sdk + volumes: ydb-data: # driver: local @@ -19,9 +21,10 @@ services: - ../:/workspaces/ydb-python-sdk:cached environment: - - YDB_IMAGE_VERSION=24.3 - - YDB_CREDENTIALS_USER=root - - YDB_CREDENTIALS_PASSWORD=1234 + - YDB_VERSION=25.1 + - YDB_STATIC_CREDENTIALS_USER=root + - YDB_STATIC_CREDENTIALS_PASSWORD=1234 + - YDB_STATIC_CREDENTIALS_ENDPOINT=grpc://ydb:2136 - YDB_CONNECTION_STRING=grpc://ydb:2136/local - YDB_CONNECTION_STRING_SECURE=grpcs://ydb:2135/local - YDB_SSL_ROOT_CERTIFICATES_FILE=/ydb_certs/ca.pem @@ -30,15 +33,15 @@ services: command: sleep infinity ydb: - image: ghcr.io/ydb-platform/local-ydb:24.3 + image: ydbplatform/local-ydb:25.1 restart: unless-stopped hostname: ydb platform: linux/amd64 ports: - - 2135:2135 - - 2136:2136 - - 8765:8765 + - 2135 + - 2136 + - 8765 volumes: - ydb-data:/ydb_data diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 21e43676..5af5f41f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -15,15 +15,14 @@ "features": { "ghcr.io/devcontainers/features/git": {}, "ghcr.io/devcontainers/features/common-utils": {}, - "ghcr.io/devcontainers/features/github-cli:1": {} + "ghcr.io/devcontainers/features/github-cli:1": {}, + "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {} }, // Use 'forwardPorts' to make a list of ports inside the container available locally. "forwardPorts": [ - 2135, - 2136, - 8765, - 9090, - 9464 + "ydb:2135", + "ydb:2136", + "ydb:8765" ], // Use 'initializeCommand' to run commands before the container is created. "initializeCommand": "chmod +x .devcontainer/commands/initialize.sh && .devcontainer/commands/initialize.sh", From aacd910d798cf1352062238c2adf89f79d593927 Mon Sep 17 00:00:00 2001 From: Vladislav Polyakov <polRk@ydb.tech> Date: Fri, 25 Apr 2025 08:14:48 +0000 Subject: [PATCH 07/12] chore(devcontainer): use python 3.9 Signed-off-by: Vladislav Polyakov <polRk@ydb.tech> --- .devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 723c1843..7ac807b1 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/devcontainers/python:3.12-bookworm +FROM mcr.microsoft.com/devcontainers/python:3.9-bookworm # [Optional] Uncomment if you want to install more tools # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ From 117606366488f4ebcefb83b4c0449822e3f4685f Mon Sep 17 00:00:00 2001 From: Vladislav Polyakov <polRk@ydb.tech> Date: Fri, 25 Apr 2025 08:15:28 +0000 Subject: [PATCH 08/12] test: support run inside devcontainer Signed-off-by: Vladislav Polyakov <polRk@ydb.tech> --- tests/conftest.py | 21 ++++++++++++++++----- tests/ssl/test_ssl.py | 5 +++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index a8177f46..f8360a2a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,7 @@ @pytest.fixture(scope="module") def docker_compose_file(pytestconfig): - return os.path.join(str(pytestconfig.rootdir), "docker-compose.yml") + return os.path.join(str(pytestconfig.rootdir), "compose.yml") def wait_container_ready(driver): @@ -32,9 +32,14 @@ def wait_container_ready(driver): @pytest.fixture(scope="module") def endpoint(pytestconfig, module_scoped_container_getter): - with ydb.Driver(endpoint="localhost:2136", database="/local") as driver: + e = "grpc://localhost:2136" + if os.environ.get('REMOTE_CONTAINERS') is not None: + e = "grpc://py-sdk-ydb:2136" + + with ydb.Driver(endpoint=e, database="/local") as driver: wait_container_ready(driver) - yield "localhost:2136" + + yield e @pytest.fixture(scope="session") @@ -47,13 +52,19 @@ def secure_endpoint(pytestconfig, session_scoped_container_getter): assert os.path.exists(ca_path) os.environ["YDB_SSL_ROOT_CERTIFICATES_FILE"] = ca_path + + e = "grpcs://localhost:2135" + if os.environ.get('REMOTE_CONTAINERS') is not None: + e = "grpcs://py-sdk-ydb:2135" + with ydb.Driver( - endpoint="grpcs://localhost:2135", + endpoint=e, database="/local", root_certificates=ydb.load_ydb_root_certificate(), ) as driver: wait_container_ready(driver) - yield "localhost:2135" + + yield e @pytest.fixture(scope="module") diff --git a/tests/ssl/test_ssl.py b/tests/ssl/test_ssl.py index 231f082a..204f36f8 100644 --- a/tests/ssl/test_ssl.py +++ b/tests/ssl/test_ssl.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import os import ydb import pytest @@ -6,8 +7,8 @@ @pytest.mark.tls def test_connect_secure(secure_endpoint, database): with ydb.Driver( - endpoint="grpcs://localhost:2135", - database="/local", + endpoint=secure_endpoint, + database=database, root_certificates=ydb.load_ydb_root_certificate(), ) as driver: driver.wait(timeout=10) From 28e89871fef92a78e42c8ee478f76040cc3cbfe6 Mon Sep 17 00:00:00 2001 From: Vladislav Polyakov <polRk@ydb.tech> Date: Fri, 25 Apr 2025 11:29:07 +0300 Subject: [PATCH 09/12] test: support running in both enviro Signed-off-by: Vladislav Polyakov <polRk@ydb.tech> --- .devcontainer/compose.yml | 1 + compose.remote.yml | 22 ++++++++++++++++++++++ compose.yml | 7 +------ tests/conftest.py | 6 +++++- tox.ini | 16 +++++++++------- 5 files changed, 38 insertions(+), 14 deletions(-) create mode 100644 compose.remote.yml diff --git a/.devcontainer/compose.yml b/.devcontainer/compose.yml index 9e6c5fae..54a6795e 100644 --- a/.devcontainer/compose.yml +++ b/.devcontainer/compose.yml @@ -28,6 +28,7 @@ services: - YDB_CONNECTION_STRING=grpc://ydb:2136/local - YDB_CONNECTION_STRING_SECURE=grpcs://ydb:2135/local - YDB_SSL_ROOT_CERTIFICATES_FILE=/ydb_certs/ca.pem + - TEST_COMPOSE_FILE=compose.remote.yml # Overrides default command so things don't shut down after the process ends. command: sleep infinity diff --git a/compose.remote.yml b/compose.remote.yml new file mode 100644 index 00000000..505d5706 --- /dev/null +++ b/compose.remote.yml @@ -0,0 +1,22 @@ +networks: + default: + name: ydb-python-sdk_default + external: true + +services: + py-sdk-ydb: + image: ydbplatform/local-ydb:trunk + restart: always + hostname: py-sdk-ydb + ports: + - 2135 + - 2136 + - 8765 + volumes: + - ./ydb_certs:/ydb_certs + environment: + - YDB_USE_IN_MEMORY_PDISKS=true + - YDB_ENABLE_COLUMN_TABLES=true + - GRPC_TLS_PORT=2135 + - GRPC_PORT=2136 + - MON_PORT=8765 diff --git a/compose.yml b/compose.yml index 2834961c..ba1fe12e 100644 --- a/compose.yml +++ b/compose.yml @@ -1,13 +1,8 @@ -networks: - default: - external: true - name: ydb-python-sdk_default - services: py-sdk-ydb: image: ydbplatform/local-ydb:trunk restart: always - hostname: py-sdk-ydb + hostname: localhost ports: - 2135 - 2136 diff --git a/tests/conftest.py b/tests/conftest.py index f8360a2a..f88d7149 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,11 @@ @pytest.fixture(scope="module") def docker_compose_file(pytestconfig): - return os.path.join(str(pytestconfig.rootdir), "compose.yml") + f = "compose.yml" + if os.environ.get('REMOTE_CONTAINERS') is not None: + f = "compose.remote.yml" + + return os.path.join(str(pytestconfig.rootdir), f) def wait_container_ready(driver): diff --git a/tox.ini b/tox.ini index a458b180..6fbe024f 100644 --- a/tox.ini +++ b/tox.ini @@ -10,9 +10,11 @@ install_command = pip install {opts} {packages} setenv = PYTHONPATH = {env:PYTHONPATH}{:}{toxinidir} REMOTE_CONTAINERS = {env:REMOTE_CONTAINERS} + TEST_COMPOSE_FILE = {env:TEST_COMPOSE_FILE:compose.yml} deps = -r{toxinidir}/test-requirements.txt + [testenv:dev-proto5] commands = deps = @@ -33,14 +35,14 @@ deps = [testenv:py] commands = - pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=compose.yml {posargs} + pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose={env:TEST_COMPOSE_FILE} {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<6.0.0 [testenv:py-proto4] commands = - pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=compose.yml {posargs} + pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose={env:TEST_COMPOSE_FILE} {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<5.0.0 @@ -49,35 +51,35 @@ deps = commands = pytest -v -m "not tls" \ --cov-report html:cov_html --cov=ydb \ - --docker-compose-remove-volumes --docker-compose=compose.yml {posargs} + --docker-compose-remove-volumes --docker-compose={env:TEST_COMPOSE_FILE} {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<5.0.0 [testenv:py-proto3] commands = - pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=compose.yml {posargs} + pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose={env:TEST_COMPOSE_FILE} {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<4.0.0 [testenv:py-tls] commands = - pytest -v -m tls --docker-compose-remove-volumes --docker-compose=compose.yml {posargs} + pytest -v -m tls --docker-compose-remove-volumes --docker-compose={env:TEST_COMPOSE_FILE} {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<6.0.0 [testenv:py-tls-proto4] commands = - pytest -v -m tls --docker-compose-remove-volumes --docker-compose=compose.yml {posargs} + pytest -v -m tls --docker-compose-remove-volumes --docker-compose={env:TEST_COMPOSE_FILE} {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<5.0.0 [testenv:py-tls-proto3] commands = - pytest -v -m tls --docker-compose-remove-volumes --docker-compose=compose.yml {posargs} + pytest -v -m tls --docker-compose-remove-volumes --docker-compose={env:TEST_COMPOSE_FILE} {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<4.0.0 From e49d5e35852bee5619fd8b238f634d7a6f84a9e7 Mon Sep 17 00:00:00 2001 From: Vladislav Polyakov <polRk@ydb.tech> Date: Tue, 17 Jun 2025 17:33:50 +0300 Subject: [PATCH 10/12] docs: update CONTRIBUTING.md with detailed dev container and Codespaces setup instructions Signed-off-by: Vladislav Polyakov <polRk@ydb.tech> --- CONTRIBUTING.md | 110 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 109 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6c56cb00..78248840 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,6 +4,114 @@ YDB is a free and open project and we appreciate to receive contributions from our community. +## Development Environment Setup + +### Using Dev Containers (Recommended) + +This repository includes a complete development environment using Docker containers that provides everything you need to start contributing immediately. The devcontainer setup includes: + +- **Python 3.9** development environment with all necessary dependencies +- **YDB server** running locally in a container +- **Pre-configured tools**: Git, GitHub CLI, YDB CLI, and essential Python packages +- **VS Code extensions**: Python development tools, linting, formatting, and debugging support + +#### Prerequisites + +- [Docker](https://www.docker.com/get-started) installed and running +- [VS Code](https://code.visualstudio.com/) with the [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) + +#### Quick Start with Dev Containers + +1. Clone the repository: + ```bash + git clone https://github.com/ydb-platform/ydb-python-sdk.git + cd ydb-python-sdk + ``` + +2. Open in VS Code: + ```bash + code . + ``` + +3. When prompted, click "Reopen in Container" or use the Command Palette (`Ctrl+Shift+P` / `Cmd+Shift+P`) and select "Dev Containers: Reopen in Container" + +4. Wait for the container to build and start (first time may take a few minutes) + +5. The development environment is ready! You can now run tests, debug code, and develop new features. + +### Using GitHub Codespaces + +GitHub Codespaces provides a cloud-based development environment that works directly in your browser or VS Code. It's perfect for quick contributions without setting up a local environment. + +#### Quick Start with Codespaces + +1. Navigate to the [repository on GitHub](https://github.com/ydb-platform/ydb-python-sdk) +2. Click the green "Code" button +3. Select the "Codespaces" tab +4. Click "Create codespace on main" (or your desired branch) +5. Wait for the environment to initialize (usually 2-3 minutes) +6. Start coding directly in the browser or connect with your local VS Code + +#### What's Included in the Development Environment + +When you use either dev containers or Codespaces, the following environment is automatically set up: + +**Container Services:** +- **SDK Container (`sdk`)**: Your main development environment running Python 3.9 on Debian Bookworm +- **YDB Container (`ydb`)**: Local YDB server (version 25.1) for testing and development + +**Development Tools:** +- **YDB CLI**: Pre-installed and configured to connect to the local YDB instance +- **Python Environment**: All project dependencies installed via `pip install -e .` +- **Git Configuration**: Automatic setup for signed commits (if configured) +- **VS Code Extensions**: Python development stack including linting, formatting, and debugging + +**Network Configuration:** +- **Port 2135**: YDB gRPC with TLS +- **Port 2136**: YDB gRPC without TLS +- **Port 8765**: YDB Monitoring interface +- These ports are automatically forwarded and accessible from your local machine + +**Environment Variables:** +The following environment variables are pre-configured for immediate use: +- `YDB_CONNECTION_STRING=grpc://ydb:2136/local` - Standard connection +- `YDB_CONNECTION_STRING_SECURE=grpcs://ydb:2135/local` - Secure connection +- `YDB_SSL_ROOT_CERTIFICATES_FILE=/ydb_certs/ca.pem` - SSL certificates +- `YDB_STATIC_CREDENTIALS_USER=root` and `YDB_STATIC_CREDENTIALS_PASSWORD=1234` - Test credentials + +**Automatic Setup Process:** +1. **Initialize**: Git configuration for signed commits and user settings +2. **Post-Create**: YDB CLI profile setup and GPG configuration for SSH signing +3. **Post-Start**: Installation of Python dependencies, SDK package, and testing tools (tox) + +#### Running Tests in the Development Environment + +Once your environment is ready, you can run the test suite: + +```bash +# Run all tests +tox + +# Run specific test categories +python -m pytest tests/ + +# Run with specific Python version +tox -e py39 +``` + +#### Connecting to the Local YDB Instance + +The YDB CLI is pre-configured to connect to the local instance: + +```bash +# Run a simple query +echo "SELECT 1;" | ydb + +# Access the web interface +# Open http://localhost:8765 in your browser (when using local dev containers) +# In codespaces you can access it via the provided URL in the terminal output. +``` + ## Contributing code changes If you would like to contribute a new feature or a bug fix, please discuss your idea first on the GitHub issue. @@ -19,4 +127,4 @@ if any changes are needed, we would love to work with you to get your pull reque ## Other questions -If you have any questions, please mail us at info@ydb.tech. \ No newline at end of file +If you have any questions, please mail us at info@ydb.tech. From 16e1876f187ef2dc01e0d5eb8829e0a64ec88407 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk <vgvoleg@gmail.com> Date: Tue, 17 Jun 2025 19:00:45 +0300 Subject: [PATCH 11/12] small fixes --- test-requirements.txt | 4 ++-- tests/conftest.py | 6 +++--- tests/ssl/test_ssl.py | 1 - 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 012697ec..7fff8494 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,7 +25,7 @@ pycparser==2.20 PyNaCl==1.4.0 pyparsing==2.4.7 pyrsistent==0.18.0 -pytest<8.0.0 +pytest<9.0.0 pytest-asyncio==0.21.0 pytest-docker-compose==3.2.1 python-dotenv==0.18.0 @@ -46,6 +46,6 @@ sqlalchemy==1.4.26 pylint-protobuf cython freezegun==1.2.2 -pytest-cov +# pytest-cov yandexcloud -e . diff --git a/tests/conftest.py b/tests/conftest.py index f88d7149..33b25f95 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,7 @@ @pytest.fixture(scope="module") def docker_compose_file(pytestconfig): f = "compose.yml" - if os.environ.get('REMOTE_CONTAINERS') is not None: + if os.environ.get("REMOTE_CONTAINERS") is not None: f = "compose.remote.yml" return os.path.join(str(pytestconfig.rootdir), f) @@ -37,7 +37,7 @@ def wait_container_ready(driver): @pytest.fixture(scope="module") def endpoint(pytestconfig, module_scoped_container_getter): e = "grpc://localhost:2136" - if os.environ.get('REMOTE_CONTAINERS') is not None: + if os.environ.get("REMOTE_CONTAINERS") is not None: e = "grpc://py-sdk-ydb:2136" with ydb.Driver(endpoint=e, database="/local") as driver: @@ -58,7 +58,7 @@ def secure_endpoint(pytestconfig, session_scoped_container_getter): os.environ["YDB_SSL_ROOT_CERTIFICATES_FILE"] = ca_path e = "grpcs://localhost:2135" - if os.environ.get('REMOTE_CONTAINERS') is not None: + if os.environ.get("REMOTE_CONTAINERS") is not None: e = "grpcs://py-sdk-ydb:2135" with ydb.Driver( diff --git a/tests/ssl/test_ssl.py b/tests/ssl/test_ssl.py index 204f36f8..29cf3bbf 100644 --- a/tests/ssl/test_ssl.py +++ b/tests/ssl/test_ssl.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import os import ydb import pytest From c62ce3c2a412f327170eebfb4291b55f2df1b707 Mon Sep 17 00:00:00 2001 From: Vladislav Polyakov <polRk@ydb.tech> Date: Wed, 18 Jun 2025 15:27:07 +0300 Subject: [PATCH 12/12] docs: update CONTRIBUTING.md to reference BUILD.md for development environment setup Signed-off-by: Vladislav Polyakov <polRk@ydb.tech> --- BUILD.md | 13 +++++- CONTRIBUTING.md | 108 +----------------------------------------------- 2 files changed, 12 insertions(+), 109 deletions(-) diff --git a/BUILD.md b/BUILD.md index 864815f9..283a0215 100644 --- a/BUILD.md +++ b/BUILD.md @@ -2,7 +2,7 @@ This document has detailed instructions on how to build ydb-python-sdk from source and run style and unit tests. -### Pre-requisites +## Pre-requisites - Install [Docker](https://docs.docker.com/engine/install/). - Install [Python](https://docs.python.org/3.8/) @@ -52,4 +52,13 @@ Use the command below for regenerate protobuf code. ```sh make protobuf -``` \ No newline at end of file +``` + +## Using Dev Containers + +This repository includes Dev Containers configuration for a quick development environment setup. For general information about Dev Containers, see the [official guide](https://code.visualstudio.com/docs/devcontainers/containers). + +The dev container automatically sets up: +- A local YDB database instance for testing +- YDB CLI with a pre-configured profile named `local` that connects to the database +- All Python dependencies and development tools diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 78248840..d25f739e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,113 +4,7 @@ YDB is a free and open project and we appreciate to receive contributions from our community. -## Development Environment Setup - -### Using Dev Containers (Recommended) - -This repository includes a complete development environment using Docker containers that provides everything you need to start contributing immediately. The devcontainer setup includes: - -- **Python 3.9** development environment with all necessary dependencies -- **YDB server** running locally in a container -- **Pre-configured tools**: Git, GitHub CLI, YDB CLI, and essential Python packages -- **VS Code extensions**: Python development tools, linting, formatting, and debugging support - -#### Prerequisites - -- [Docker](https://www.docker.com/get-started) installed and running -- [VS Code](https://code.visualstudio.com/) with the [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) - -#### Quick Start with Dev Containers - -1. Clone the repository: - ```bash - git clone https://github.com/ydb-platform/ydb-python-sdk.git - cd ydb-python-sdk - ``` - -2. Open in VS Code: - ```bash - code . - ``` - -3. When prompted, click "Reopen in Container" or use the Command Palette (`Ctrl+Shift+P` / `Cmd+Shift+P`) and select "Dev Containers: Reopen in Container" - -4. Wait for the container to build and start (first time may take a few minutes) - -5. The development environment is ready! You can now run tests, debug code, and develop new features. - -### Using GitHub Codespaces - -GitHub Codespaces provides a cloud-based development environment that works directly in your browser or VS Code. It's perfect for quick contributions without setting up a local environment. - -#### Quick Start with Codespaces - -1. Navigate to the [repository on GitHub](https://github.com/ydb-platform/ydb-python-sdk) -2. Click the green "Code" button -3. Select the "Codespaces" tab -4. Click "Create codespace on main" (or your desired branch) -5. Wait for the environment to initialize (usually 2-3 minutes) -6. Start coding directly in the browser or connect with your local VS Code - -#### What's Included in the Development Environment - -When you use either dev containers or Codespaces, the following environment is automatically set up: - -**Container Services:** -- **SDK Container (`sdk`)**: Your main development environment running Python 3.9 on Debian Bookworm -- **YDB Container (`ydb`)**: Local YDB server (version 25.1) for testing and development - -**Development Tools:** -- **YDB CLI**: Pre-installed and configured to connect to the local YDB instance -- **Python Environment**: All project dependencies installed via `pip install -e .` -- **Git Configuration**: Automatic setup for signed commits (if configured) -- **VS Code Extensions**: Python development stack including linting, formatting, and debugging - -**Network Configuration:** -- **Port 2135**: YDB gRPC with TLS -- **Port 2136**: YDB gRPC without TLS -- **Port 8765**: YDB Monitoring interface -- These ports are automatically forwarded and accessible from your local machine - -**Environment Variables:** -The following environment variables are pre-configured for immediate use: -- `YDB_CONNECTION_STRING=grpc://ydb:2136/local` - Standard connection -- `YDB_CONNECTION_STRING_SECURE=grpcs://ydb:2135/local` - Secure connection -- `YDB_SSL_ROOT_CERTIFICATES_FILE=/ydb_certs/ca.pem` - SSL certificates -- `YDB_STATIC_CREDENTIALS_USER=root` and `YDB_STATIC_CREDENTIALS_PASSWORD=1234` - Test credentials - -**Automatic Setup Process:** -1. **Initialize**: Git configuration for signed commits and user settings -2. **Post-Create**: YDB CLI profile setup and GPG configuration for SSH signing -3. **Post-Start**: Installation of Python dependencies, SDK package, and testing tools (tox) - -#### Running Tests in the Development Environment - -Once your environment is ready, you can run the test suite: - -```bash -# Run all tests -tox - -# Run specific test categories -python -m pytest tests/ - -# Run with specific Python version -tox -e py39 -``` - -#### Connecting to the Local YDB Instance - -The YDB CLI is pre-configured to connect to the local instance: - -```bash -# Run a simple query -echo "SELECT 1;" | ydb - -# Access the web interface -# Open http://localhost:8765 in your browser (when using local dev containers) -# In codespaces you can access it via the provided URL in the terminal output. -``` +For information about setting up your development environment, please see [BUILD.md](BUILD.md). ## Contributing code changes