Collect variables in .env, extend README.md, extend build process.

Author: hgkegft

.env

@@ -0,0 +1,4 @@
+PLANKQ_API_KEY="plqk_4Rk6rDnjDp6D38cRQXgGigFymWpCt3R7jw27kUZAcP"
+PLANKQ_TOKEN_URL="https://gateway.platform.planqk.de/token"
+PLANQK_DATA_POOL_URL="https://platform.planqk.de/qc-catalog/data-pools"
+PLANQK_OPEN_API_SPEC_URL="https://platform.planqk.de/qc-catalog/docs"

README.md

@@ -2,4 +2,18 @@
 ````shell
 .\install.sh
 pip install .
-````
+````
+In *install.sh* the python client library is generated from the openapi-spec at https://platform.planqk.de/qc-catalog/docs.
+Addintionally there are some problems with generating the correct types which are also handled by replacing the lines with the script.
+
+# Test
+````shell
+pytest -s -m MARK .
+````
+
+Values for **MARK** are defined in the pyproject.toml
+- **interactive**: only run tests with user interaction
+- **slow_service**: only run tests wich are slow because of the service creation
+- **auto**: only run tests that run fully automatic
+
+By setting **log_cli_level** to **DEBUG** you get a finer logging output.

install.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Read .env
+source .env
+
 # Install openapi-generator-cli globally
 npm install -g @openapitools/openapi-generator-cli
 
@@ -10,7 +13,7 @@ mkdir generator-output && cd generator-output || exit
 openapi-generator-cli version-manager set 5.4.0
 
 # Generate Python client using the specified OpenAPI URL
-openapi-generator-cli generate -g python -i https://platform.planqk.de/qc-catalog/docs
+openapi-generator-cli generate -g python -i $PLANQK_OPEN_API_SPEC_URL
 
 # Copy the generated client to the parent directory
 cp -R openapi_client ../src

pyproject.toml

@@ -19,7 +19,8 @@ dependencies = [
     "requests>=2.31.0",
     "python-dateutil>=2.8.2",
     "urllib3>=2.2.0",
-    "names-generator>=0.1.0"
+    "names-generator>=0.1.0",
+    "python-dotenv>=1.0.1"
 ]
 dynamic = ["version"]
 
@@ -32,9 +33,9 @@ version_file = "src/pyplanqk/version.py"
 
 [tool.pytest.ini_options]
 markers = [
-    "previously_wrong: only run previously wrong tests",
     "interactive: only run tests with user interaction",
     "slow_service: only run tests wich are slow because of the service creation",
+    "auto: only run tests that run fully automatic",
 ]
 log_cli = true
-log_cli_level = "INFO"
+log_cli_level = "DEBUG"

src/pyplanqk/helpers.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import time
 from typing import Dict
 
@@ -106,3 +107,12 @@ def wait_for_service_job_to_be_finished(
             return False
         else:
             logger.debug(f"{status_timer + 1}|{timeout} Wait for job...")
+
+
+def get_path_delimiter() -> str:
+    path_delimiter: str = ""
+    if os.name == "nt":
+        path_delimiter: str = "\\"
+    elif os.name == "posix":
+        path_delimiter: str = "/"
+    return path_delimiter

src/pyplanqk/high_level_actions.py

@@ -1,12 +1,17 @@
+from dotenv import load_dotenv
+
 from pyplanqk.low_level_actions import *
 
 logger = logging.getLogger(__name__)
 
+load_dotenv(".env")
+PLANKQ_TOKEN_URL = os.getenv("PLANKQ_TOKEN_URL")
+
 
 class PyPlanQK:
     def __init__(self, api_key):
         self.api_key = {"apiKey": api_key}
-        self.token_url = "https://gateway.platform.planqk.de/token"
+        self.token_url = PLANKQ_TOKEN_URL
 
     def create_service(self, config: Dict[str, Any]) -> Dict[str, Any]:
         service_name = None
@@ -50,6 +55,7 @@ def execute_service(
 
         try:
             if data_ref is not None:
+                logger.debug(f"triggering serice job with data pool: {data_ref}.")
                 job = trigger_service_job(
                     service_name=service_name,
                     api_key=self.api_key,
@@ -58,6 +64,7 @@ def execute_service(
                     params=params,
                 )
             else:
+                logger.debug(f"triggering serice job with data upload: {data}.")
                 job = trigger_service_job(
                     service_name=service_name,
                     api_key=self.api_key,
@@ -84,18 +91,21 @@ def create_data_pool(self, data_pool_name: Optional[str], file) -> Dict[str, Any
             if data_pool is not None:
                 logger.info(f"Data pool: {data_pool_name} already created.")
                 return data_pool
+            logger.debug(f"data pool: {data_pool_name} not found. Creating...")
+            logger.debug(f"data pool: {data_pool_name} not found. Creating...")
 
             create_data_pool(data_pool_name, self.api_key["apiKey"])
-
+            logger.debug(f"data pool: {data_pool_name} created. Adding data...")
             add_data_to_data_pool(data_pool_name, file, self.api_key["apiKey"])
-
+            logger.debug(f"data added to data pool")
             file_infos = get_data_pool_file_information(
                 data_pool_name, self.api_key["apiKey"]
             )
-            file_name = file.name.split("/")[-1]
+            file_name = file.name.split(get_path_delimiter())[-1]
             file_info = file_infos[file_name]
             return file_info
         except Exception as e:
             logger.error(f"Creation of data pool: {data_pool_name} failed.")
+            logger.error(f"file: {file.name} could not be added to data pool.")
             logger.error(e)
             raise e

src/pyplanqk/low_level_actions.py

@@ -1,5 +1,4 @@
 import json
-import os
 from typing import Any, List, Optional
 
 from openapi_client.apis import ServicePlatformApplicationsApi
@@ -718,8 +717,10 @@ def get_data_pool(data_pool_name: str, api_key: str) -> Optional[Dict[str, str]]
         found_data_pool = None
         for data_pool in data_pools:
             if data_pool_name == data_pool["name"]:
+                logger.debug(f"Get Pool: Found it!")
                 found_data_pool = data_pool
-
+                return found_data_pool
+        logger.debug(f"Get Pool: Didn't found it!")
         return found_data_pool
     except Exception as e:
         logger.error("Get data pool failed.")
@@ -781,9 +782,14 @@ def get_data_pool_file_information(data_pool_name: str, api_key: str) -> Dict[st
 
 def add_data_to_data_pool(data_pool_name: str, file, api_key: str) -> bool:
     logger.debug("Add data to data pool.")
-
     try:
-        data_pool = get_data_pool(data_pool_name, api_key)
+        for count in range(10):
+            logger.debug(f"Get pool try: {count+1}")
+            data_pool = get_data_pool(data_pool_name, api_key)
+            if data_pool is not None:
+                break
+            time.sleep(1)
+
         assert data_pool is not None
         data_pool_id = data_pool["id"]
 

src/pyplanqk/version.py

@@ -12,5 +12,5 @@
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.0.post42+g60c51ed.d20240215'
-__version_tuple__ = version_tuple = (0, 0, 'g60c51ed.d20240215')
+__version__ = version = '0.0.post44+g500054a.d20240216'
+__version_tuple__ = version_tuple = (0, 0, 'g500054a.d20240216')

tests/conftest.py

@@ -1,13 +1,19 @@
-from typing import Dict, Tuple
+from typing import Tuple
 
 import pytest
+from dotenv import load_dotenv
 from names_generator import generate_name
 from util import *
 
+load_dotenv(".env")
+PLANKQ_API_KEY = os.getenv("PLANKQ_API_KEY")
+PLANKQ_TOKEN_URL = os.getenv("PLANKQ_API_KEY")
+PLANQK_DATA_POOL_URL = os.getenv("PLANQK_DATA_POOL_URL")
+
 
 @pytest.fixture(scope="function")
 def api_key() -> Dict[str, str]:
-    api_key = "plqk_4Rk6rDnjDp6D38cRQXgGigFymWpCt3R7jw27kUZAcP"
+    api_key = PLANKQ_API_KEY
     api_key = {"apiKey": api_key}
     return api_key
 
@@ -24,7 +30,7 @@ def step() -> int:
 
 @pytest.fixture(scope="function")
 def token_url() -> str:
-    return "https://gateway.platform.planqk.de/token"
+    return PLANKQ_TOKEN_URL
 
 
 @pytest.fixture(scope="function")
@@ -79,7 +85,7 @@ def config() -> Dict[str, Any]:
 @pytest.fixture(scope="function")
 def data_pool(api_key: Dict[str, str]) -> Dict[str, Any]:
     data_pool_name = f"data_pool_{generate_name()}"
-    url = "https://platform.planqk.de/qc-catalog/data-pools"
+    url = PLANQK_DATA_POOL_URL
 
     headers = {"Content-Type": "application/json", "X-Auth-Token": api_key["apiKey"]}
 
@@ -92,10 +98,10 @@ def data_pool(api_key: Dict[str, str]) -> Dict[str, Any]:
 
 @pytest.fixture(scope="function")
 def data_pool_with_data(
-    api_key: Dict[str, str], train_data: Dict[str, list], train_params: Dict[str, list]
+        api_key: Dict[str, str], train_data: Dict[str, list], train_params: Dict[str, list]
 ) -> Dict[str, Any]:
     data_pool_name = f"data_pool_{generate_name()}"
-    url = "https://platform.planqk.de/qc-catalog/data-pools"
+    url = PLANQK_DATA_POOL_URL
 
     headers = {"Content-Type": "application/json", "X-Auth-Token": api_key["apiKey"]}
 
@@ -106,7 +112,7 @@ def data_pool_with_data(
 
     # save_data(train_data, train_params)
 
-    file = open("tests/data/data.json", "rb")
+    file = open(f"{get_test_data_path()}data.json", "rb")
     result = add_data_to_data_pool(data_pool_name, file, api_key["apiKey"])
     assert result
 
@@ -117,7 +123,7 @@ def data_pool_with_data(
 def access_token() -> str:
     consumer_key = "wC1Dkq6ZPW7CRBUSB1oL5cURA_ga"
     consumer_secret = "o5oiJrJrNLbIuJPGk46vJyHfDj4a"
-    token_url = "https://gateway.platform.planqk.de/token"
+    token_url = PLANKQ_TOKEN_URL
     access_token = get_access_token(consumer_key, consumer_secret, token_url)
     assert access_token is not None
     assert len(access_token) == 1037
@@ -126,7 +132,7 @@ def access_token() -> str:
 
 @pytest.fixture(scope="function")
 def service_info(
-    config: Dict[str, Any], api_key: Dict[str, str]
+        config: Dict[str, Any], api_key: Dict[str, str]
 ) -> Tuple[Dict[str, Any], Dict[str, Any]]:
     service = create_managed_service(config, api_key)
     service_id = service["id"]
@@ -139,7 +145,7 @@ def service_info(
 
 @pytest.fixture(scope="function")
 def internally_published_service(
-    service_info: Tuple[Dict[str, Any], Dict[str, Any]], api_key: Dict[str, str]
+        service_info: Tuple[Dict[str, Any], Dict[str, Any]], api_key: Dict[str, str]
 ) -> Dict[str, Any]:
     simple_service, config = service_info
 
@@ -172,7 +178,7 @@ def application_with_auth(api_key: Dict[str, str]) -> Tuple[Dict[str, Any], str,
 
 @pytest.fixture(scope="function")
 def full_application(
-    config: Dict[str, Any], api_key: Dict[str, str]
+        config: Dict[str, Any], api_key: Dict[str, str]
 ) -> Tuple[Dict[str, Any], Dict[str, Any], str, str]:
     application_name = f"application_{generate_name()}"
     application = create_application(application_name, api_key)
@@ -199,7 +205,7 @@ def full_application(
 
 @pytest.fixture(scope="function")
 def service_job(
-    data: Dict[str, Any], params: Dict[str, Any], api_key: Dict[str, str]
+        data: Dict[str, Any], params: Dict[str, Any], api_key: Dict[str, str]
 ) -> JobDto:
     logger.debug("Trigger service job")
     configuration = Configuration(api_key=api_key)

tests/data/data.json

@@ -2,7 +2,13 @@
 import logging
 from typing import Any, Dict
 
-from conftest import cleanup_services_and_applications, get_data, get_params
+import pytest
+from conftest import (
+    cleanup_services_and_applications,
+    get_data,
+    get_params,
+    get_test_data_path,
+)
 from names_generator import generate_name
 
 import pyplanqk
@@ -11,6 +17,7 @@
 logger = logging.getLogger(__name__)
 
 
+@pytest.mark.auto
 def test_service_job_from_data_upload(config: Dict[str, Any], api_key: Dict[str, str]):
     print()
     logger.debug("test_service_job_from_data_upload")
@@ -23,15 +30,20 @@ def test_service_job_from_data_upload(config: Dict[str, Any], api_key: Dict[str,
         train_params = get_params(maxiter=100, reps=3)
 
         plnqk = pyplanqk.PyPlanQK(api_key["apiKey"])
+
+        logger.debug(f"start create_service now\n")
         service = plnqk.create_service(config)
+        logger.debug(f"create_service done\n")
         assert service is not None
         services.append(service)
 
         service_name = service["name"]
+        logger.debug(f"start execute_service {service_name} now\n")
         result = plnqk.execute_service(
             service_name, data=train_data, params=train_params
         )
         assert result is not None
+        logger.debug(f"execute_service {service_name} done\n")
 
         model = result["model"]
 
@@ -54,20 +66,22 @@ def test_service_job_from_data_upload(config: Dict[str, Any], api_key: Dict[str,
         assert False
 
 
+@pytest.mark.auto
 def test_service_job_from_data_pool(config: Dict[str, Any], api_key: Dict[str, str]):
     print()
     logger.debug("test_service_job_from_data_pool")
 
+    data_path = get_test_data_path() + "data.json"
     applications = []
     services = []
 
     try:
         train_data = get_data(num_samples_train=800, num_samples_test=200)
-        f_data = open("/tmp/data.json", "w")
+        f_data = open(data_path, "w")
+
         json.dump(train_data, f_data)
         f_data.close()
-
-        f_data = open("/tmp/data.json", "rb")
+        f_data = open(data_path, "rb")
 
         train_params = get_params(maxiter=100, reps=3)