From 34127c1c35cfeab81c80ba8e2b1379d2bcf91572 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 23 Sep 2025 10:36:45 -0500 Subject: [PATCH 01/33] support custom TopCP images --- .../TopCP_code_generator/query_translate.py | 11 +- .../tests/test_custom_image_tag.py | 91 +++++++++++ helm/servicex/templates/app/configmap.yaml | 7 + helm/servicex/values.yaml | 15 +- servicex_app/servicex_app/code_gen_adapter.py | 2 +- .../resources/transformation/submit.py | 84 ++++++++++- .../test_custom_image_validation.py | 141 ++++++++++++++++++ 7 files changed, 345 insertions(+), 6 deletions(-) create mode 100644 code_generator_TopCPToolkit/tests/test_custom_image_tag.py create mode 100644 servicex_app/servicex_app_test/test_custom_image_validation.py diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py index ac7ff9ac3..773a5ea69 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py @@ -36,6 +36,12 @@ "ifTrue": ["--no-filter"], "ifFalse": None, }, + "image_tag": { + "properType": str, + "properTypeString": "string", + "default": "2.17.0-25.2.45", + "optional": True, + }, } @@ -52,9 +58,12 @@ def generate_files_from_query(query, query_file_path): "customConfig", ] - # ensure all keys are specified + # ensure all required keys are specified for key in options: if key not in jquery: + # Skip optional parameters + if options[key].get("optional", False): + continue raise ValueError( key + " must be specified. May be type None or ", options[key]["properTypeString"], diff --git a/code_generator_TopCPToolkit/tests/test_custom_image_tag.py b/code_generator_TopCPToolkit/tests/test_custom_image_tag.py new file mode 100644 index 000000000..18c2cf60e --- /dev/null +++ b/code_generator_TopCPToolkit/tests/test_custom_image_tag.py @@ -0,0 +1,91 @@ +"""Test custom image tag functionality for TopCP code generator.""" + +import json +import os +import tempfile +import unittest + +from servicex.TopCP_code_generator.query_translate import generate_files_from_query + + +class TestCustomImageTag(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + import shutil + shutil.rmtree(self.temp_dir) + + def test_custom_image_tag_in_query_validation(self): + """Test that custom image tag is accepted in query validation.""" + query = { + "reco": "reco_config_content", + "parton": None, + "particle": None, + "max_events": -1, + "no_systematics": False, + "no_filter": False, + "image_tag": "v2.20.0_v0.2" + } + + # This should not raise an exception + generate_files_from_query(json.dumps(query), self.temp_dir) + + def test_image_tag_not_required(self): + """Test that image_tag is optional and doesn't break existing queries.""" + query = { + "reco": "reco_config_content", + "parton": None, + "particle": None, + "max_events": -1, + "no_systematics": False, + "no_filter": False + } + + # This should not raise an exception + generate_files_from_query(json.dumps(query), self.temp_dir) + + def test_image_tag_none_allowed(self): + """Test that None image tag is accepted.""" + query = { + "reco": "reco_config_content", + "parton": None, + "particle": None, + "max_events": -1, + "no_systematics": False, + "no_filter": False, + "image_tag": None + } + + # This should not raise an exception + generate_files_from_query(json.dumps(query), self.temp_dir) + + def test_generated_files_created_with_image_tag(self): + """Test that all expected files are generated when image_tag is present.""" + query = { + "reco": "reco_config_content", + "parton": None, + "particle": None, + "max_events": -1, + "no_systematics": False, + "no_filter": False, + "image_tag": "v2.20.0_v0.2" + } + + generate_files_from_query(json.dumps(query), self.temp_dir) + + # Check that reco.yaml is created + reco_file = os.path.join(self.temp_dir, "reco.yaml") + self.assertTrue(os.path.exists(reco_file)) + + with open(reco_file, 'r') as f: + content = f.read() + self.assertEqual(content, "reco_config_content") + + # Check that generated_transformer.py is created + transformer_file = os.path.join(self.temp_dir, "generated_transformer.py") + self.assertTrue(os.path.exists(transformer_file)) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/helm/servicex/templates/app/configmap.yaml b/helm/servicex/templates/app/configmap.yaml index e543e2608..67067878a 100644 --- a/helm/servicex/templates/app/configmap.yaml +++ b/helm/servicex/templates/app/configmap.yaml @@ -178,6 +178,13 @@ data: CODE_GEN_IMAGES = { {{ join "," $code_gen_images }} } {{- end }} + {{- if .Values.codeGen.topcp.enabled }} + # TopCP custom image configuration + TOPCP_ALLOWED_REPOSITORIES = {{ .Values.codeGen.topcp.allowedRepositories | toJson }} + TOPCP_IMAGE_TAG_PATTERN = '{{ .Values.codeGen.topcp.imageTagPattern }}' + TOPCP_DEFAULT_BASE_IMAGE = '{{ .Values.codeGen.topcp.defaultBaseImage }}' + {{- end }} + {{- $didFinders := list }} {{- if .Values.didFinder.CERNOpenData.enabled }} {{- $didFinders = append $didFinders "cernopendata" }} diff --git a/helm/servicex/values.yaml b/helm/servicex/values.yaml index 40de0b959..02d1707fd 100644 --- a/helm/servicex/values.yaml +++ b/helm/servicex/values.yaml @@ -108,6 +108,12 @@ codeGen: tag: develop defaultScienceContainerImage: sslhep/servicex_science_image_topcp defaultScienceContainerTag: 2.17.0-25.2.45 + # Configuration for custom image tag validation + allowedRepositories: + - "sslhep/servicex_science_image_topcp" + - "registry.gitlab.com/topcp-project/toolkit" + imageTagPattern: "^v?\\d+\\.\\d+\\.\\d+[-_]v?\\d+\\.\\d+$" + defaultBaseImage: "sslhep/servicex_science_image_topcp" didFinder: CERNOpenData: @@ -146,7 +152,8 @@ logging: minio: volumePermissions: image: - repository: bitnami/bitnami-shell-archived +# repository: bitnami/bitnami-shell-archived + repository: bitnami/os-shell auth: rootPassword: leftfoot1 rootUser: miniouser @@ -168,7 +175,8 @@ postgres: postgresql: volumePermissions: image: - repository: bitnami/bitnami-shell-archived +# repository: bitnami/bitnami-shell-archived + repository: bitnami/os-shell global: postgresql: auth: @@ -189,7 +197,8 @@ rabbitmq: volumePermissions: enabled: true image: - repository: bitnami/bitnami-shell-archived +# repository: bitnami/bitnami-shell-archived + repository: bitnami/os-shell extraConfiguration: |- consumer_timeout = 3600000 diff --git a/servicex_app/servicex_app/code_gen_adapter.py b/servicex_app/servicex_app/code_gen_adapter.py index ea88516c5..b4361acd2 100644 --- a/servicex_app/servicex_app/code_gen_adapter.py +++ b/servicex_app/servicex_app/code_gen_adapter.py @@ -53,7 +53,7 @@ def generate_code_for_selection( :param namespace: Namespace in which to place resulting ConfigMap. :param user_codegen_name: Name provided by user for selecting the codegen URL from config dictionary - :returns a tuple of (config map name, default transformer image) + :returns a tuple of (config map name, default transformer image, language, command) """ from io import BytesIO from zipfile import ZipFile diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 51e57668e..3aec30274 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -26,6 +26,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import uuid +import re from datetime import datetime, timezone from typing import Optional, List @@ -39,7 +40,63 @@ from werkzeug.exceptions import BadRequest +def validate_custom_image_tag(image_tag: str, config: dict, codegen_name: str) -> tuple[str, str]: + """ + Validate custom image tag for TopCP transformations. + + :param image_tag: The custom image tag provided by user + :param config: Flask application configuration + :param codegen_name: The code generator name (e.g., 'topcp') + :returns: tuple of (validated_image_name, validated_tag) + :raises: BadRequest if validation fails + """ + if not image_tag or not image_tag.strip(): + return None, None + + # Only validate for TopCP requests + if codegen_name != 'topcp': + return None, None + + # Get validation configuration (with defaults) + allowed_repos = config.get('TOPCP_ALLOWED_REPOSITORIES', [ + 'sslhep/servicex_science_image_topcp', + 'registry.gitlab.com/topcp-project/toolkit' + ]) + tag_pattern = config.get('TOPCP_IMAGE_TAG_PATTERN', r'^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$') + default_base_image = config.get('TOPCP_DEFAULT_BASE_IMAGE', 'sslhep/servicex_science_image_topcp') + + # Validate tag format + if not re.match(tag_pattern, image_tag): + raise BadRequest(f"Invalid TopCP image tag format: {image_tag}. Expected format: v2.20.0_v0.2") + + # For now, use the default base image with the custom tag + # In the future, this could be extended to allow custom repositories + validated_image = default_base_image + validated_tag = image_tag + + return validated_image, validated_tag + + class SubmitTransformationRequest(ServiceXResource): + + def _extract_custom_image_tag(self, selection: str, codegen_name: str) -> Optional[str]: + """ + Extract custom image tag from TopCP selection query. + + :param selection: The query selection string + :param codegen_name: The code generator name + :returns: Custom image tag if present and valid, None otherwise + """ + if codegen_name != 'topcp': + return None + + try: + import json + query = json.loads(selection) + return query.get('image_tag') + except (json.JSONDecodeError, AttributeError): + # Invalid JSON or non-dict selection + return None @classmethod def make_api( cls, @@ -212,6 +269,9 @@ def post(self): files=0, ) + # Extract custom image tag from TopCP queries before code generation + custom_image_tag = self._extract_custom_image_tag(request_rec.selection, user_codegen_name) + # The first thing to do is make sure the requested selection is correct, # and can generate the requested code ( @@ -223,7 +283,29 @@ def post(self): request_rec, namespace, user_codegen_name ) - request_rec.image = codegen_transformer_image + # Handle custom image tag for TopCP + if custom_image_tag: + try: + validated_image, validated_tag = validate_custom_image_tag( + custom_image_tag, config, user_codegen_name + ) + if validated_image and validated_tag: + # Override the default science container image + request_rec.image = f"{validated_image}:{validated_tag}" + current_app.logger.info( + f"Using custom TopCP image: {request_rec.image}", + extra={"requestId": request_id} + ) + else: + request_rec.image = codegen_transformer_image + except BadRequest as e: + current_app.logger.error( + f"Invalid custom image tag: {str(e)}", + extra={"requestId": request_id} + ) + return {"message": str(e)}, 400 + else: + request_rec.image = codegen_transformer_image # Check to make sure the transformer docker image actually exists (if enabled) if config["TRANSFORMER_VALIDATE_DOCKER_IMAGE"]: diff --git a/servicex_app/servicex_app_test/test_custom_image_validation.py b/servicex_app/servicex_app_test/test_custom_image_validation.py new file mode 100644 index 000000000..43a40d193 --- /dev/null +++ b/servicex_app/servicex_app_test/test_custom_image_validation.py @@ -0,0 +1,141 @@ +"""Test custom image tag validation for TopCP transformations.""" + +import json +import unittest +from werkzeug.exceptions import BadRequest + +from servicex_app.resources.transformation.submit import validate_custom_image_tag, SubmitTransformationRequest + + +class TestCustomImageValidation(unittest.TestCase): + + def setUp(self): + self.config = { + 'TOPCP_ALLOWED_REPOSITORIES': [ + 'sslhep/servicex_science_image_topcp', + 'registry.gitlab.com/topcp-project/toolkit' + ], + 'TOPCP_IMAGE_TAG_PATTERN': r'^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$', + 'TOPCP_DEFAULT_BASE_IMAGE': 'sslhep/servicex_science_image_topcp' + } + + def test_valid_image_tag_format(self): + """Test that valid image tag formats are accepted.""" + test_cases = [ + "v2.20.0_v0.2", + "2.20.0_v0.2", + "v1.15.0-v1.0", + "3.0.0_v2.5" + ] + + for tag in test_cases: + with self.subTest(tag=tag): + image, result_tag = validate_custom_image_tag(tag, self.config, 'topcp') + self.assertEqual(image, 'sslhep/servicex_science_image_topcp') + self.assertEqual(result_tag, tag) + + def test_invalid_image_tag_format(self): + """Test that invalid image tag formats are rejected.""" + test_cases = [ + "latest", + "2.20.0", + "v2.20.0", + "invalid-tag", + "v2.20.0_v0.2.1", # Too many version components + "2.20_v0.2", # Missing patch version + ] + + for tag in test_cases: + with self.subTest(tag=tag): + with self.assertRaises(BadRequest): + validate_custom_image_tag(tag, self.config, 'topcp') + + def test_non_topcp_codegen_returns_none(self): + """Test that non-TopCP code generators return None.""" + image, tag = validate_custom_image_tag("v2.20.0_v0.2", self.config, 'uproot') + self.assertIsNone(image) + self.assertIsNone(tag) + + def test_empty_image_tag_returns_none(self): + """Test that empty image tag returns None.""" + test_cases = [None, "", " "] + + for tag in test_cases: + with self.subTest(tag=tag): + image, result_tag = validate_custom_image_tag(tag, self.config, 'topcp') + self.assertIsNone(image) + self.assertIsNone(result_tag) + + def test_missing_config_uses_defaults(self): + """Test that missing configuration uses default values.""" + minimal_config = {} + + image, tag = validate_custom_image_tag("v2.20.0_v0.2", minimal_config, 'topcp') + self.assertEqual(image, 'sslhep/servicex_science_image_topcp') + self.assertEqual(tag, "v2.20.0_v0.2") + + +class TestImageTagExtraction(unittest.TestCase): + + def setUp(self): + self.submit_request = SubmitTransformationRequest() + + def test_extract_image_tag_from_topcp_query(self): + """Test extraction of image tag from TopCP query.""" + query = { + "reco": "reco_config", + "parton": None, + "particle": None, + "max_events": -1, + "no_systematics": False, + "no_filter": False, + "image_tag": "v2.20.0_v0.2" + } + + result = self.submit_request._extract_custom_image_tag(json.dumps(query), 'topcp') + self.assertEqual(result, "v2.20.0_v0.2") + + def test_extract_image_tag_missing_returns_none(self): + """Test that missing image tag returns None.""" + query = { + "reco": "reco_config", + "parton": None, + "particle": None, + "max_events": -1, + "no_systematics": False, + "no_filter": False + } + + result = self.submit_request._extract_custom_image_tag(json.dumps(query), 'topcp') + self.assertIsNone(result) + + def test_extract_image_tag_non_topcp_returns_none(self): + """Test that non-TopCP codegen returns None.""" + query = { + "selection": "some_selection", + "image_tag": "v2.20.0_v0.2" + } + + result = self.submit_request._extract_custom_image_tag(json.dumps(query), 'uproot') + self.assertIsNone(result) + + def test_extract_image_tag_invalid_json_returns_none(self): + """Test that invalid JSON returns None.""" + invalid_json = "{ invalid json }" + + result = self.submit_request._extract_custom_image_tag(invalid_json, 'topcp') + self.assertIsNone(result) + + def test_extract_image_tag_none_value(self): + """Test that None image tag value is returned as None.""" + query = { + "reco": "reco_config", + "image_tag": None + } + + result = self.submit_request._extract_custom_image_tag(json.dumps(query), 'topcp') + self.assertIsNone(result) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From ba7101415838b6c80e8b350d251e0503ce643103 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 23 Sep 2025 15:37:26 +0000 Subject: [PATCH 02/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../tests/test_custom_image_tag.py | 15 ++-- .../resources/transformation/submit.py | 47 ++++++++---- .../test_custom_image_validation.py | 74 +++++++++---------- 3 files changed, 77 insertions(+), 59 deletions(-) diff --git a/code_generator_TopCPToolkit/tests/test_custom_image_tag.py b/code_generator_TopCPToolkit/tests/test_custom_image_tag.py index 18c2cf60e..19ffd5344 100644 --- a/code_generator_TopCPToolkit/tests/test_custom_image_tag.py +++ b/code_generator_TopCPToolkit/tests/test_custom_image_tag.py @@ -14,6 +14,7 @@ def setUp(self): def tearDown(self): import shutil + shutil.rmtree(self.temp_dir) def test_custom_image_tag_in_query_validation(self): @@ -25,7 +26,7 @@ def test_custom_image_tag_in_query_validation(self): "max_events": -1, "no_systematics": False, "no_filter": False, - "image_tag": "v2.20.0_v0.2" + "image_tag": "v2.20.0_v0.2", } # This should not raise an exception @@ -39,7 +40,7 @@ def test_image_tag_not_required(self): "particle": None, "max_events": -1, "no_systematics": False, - "no_filter": False + "no_filter": False, } # This should not raise an exception @@ -54,7 +55,7 @@ def test_image_tag_none_allowed(self): "max_events": -1, "no_systematics": False, "no_filter": False, - "image_tag": None + "image_tag": None, } # This should not raise an exception @@ -69,7 +70,7 @@ def test_generated_files_created_with_image_tag(self): "max_events": -1, "no_systematics": False, "no_filter": False, - "image_tag": "v2.20.0_v0.2" + "image_tag": "v2.20.0_v0.2", } generate_files_from_query(json.dumps(query), self.temp_dir) @@ -78,7 +79,7 @@ def test_generated_files_created_with_image_tag(self): reco_file = os.path.join(self.temp_dir, "reco.yaml") self.assertTrue(os.path.exists(reco_file)) - with open(reco_file, 'r') as f: + with open(reco_file, "r") as f: content = f.read() self.assertEqual(content, "reco_config_content") @@ -87,5 +88,5 @@ def test_generated_files_created_with_image_tag(self): self.assertTrue(os.path.exists(transformer_file)) -if __name__ == '__main__': - unittest.main() \ No newline at end of file +if __name__ == "__main__": + unittest.main() diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 3aec30274..4fe0e789d 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -40,7 +40,9 @@ from werkzeug.exceptions import BadRequest -def validate_custom_image_tag(image_tag: str, config: dict, codegen_name: str) -> tuple[str, str]: +def validate_custom_image_tag( + image_tag: str, config: dict, codegen_name: str +) -> tuple[str, str]: """ Validate custom image tag for TopCP transformations. @@ -54,20 +56,29 @@ def validate_custom_image_tag(image_tag: str, config: dict, codegen_name: str) - return None, None # Only validate for TopCP requests - if codegen_name != 'topcp': + if codegen_name != "topcp": return None, None # Get validation configuration (with defaults) - allowed_repos = config.get('TOPCP_ALLOWED_REPOSITORIES', [ - 'sslhep/servicex_science_image_topcp', - 'registry.gitlab.com/topcp-project/toolkit' - ]) - tag_pattern = config.get('TOPCP_IMAGE_TAG_PATTERN', r'^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$') - default_base_image = config.get('TOPCP_DEFAULT_BASE_IMAGE', 'sslhep/servicex_science_image_topcp') + allowed_repos = config.get( + "TOPCP_ALLOWED_REPOSITORIES", + [ + "sslhep/servicex_science_image_topcp", + "registry.gitlab.com/topcp-project/toolkit", + ], + ) + tag_pattern = config.get( + "TOPCP_IMAGE_TAG_PATTERN", r"^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$" + ) + default_base_image = config.get( + "TOPCP_DEFAULT_BASE_IMAGE", "sslhep/servicex_science_image_topcp" + ) # Validate tag format if not re.match(tag_pattern, image_tag): - raise BadRequest(f"Invalid TopCP image tag format: {image_tag}. Expected format: v2.20.0_v0.2") + raise BadRequest( + f"Invalid TopCP image tag format: {image_tag}. Expected format: v2.20.0_v0.2" + ) # For now, use the default base image with the custom tag # In the future, this could be extended to allow custom repositories @@ -79,7 +90,9 @@ def validate_custom_image_tag(image_tag: str, config: dict, codegen_name: str) - class SubmitTransformationRequest(ServiceXResource): - def _extract_custom_image_tag(self, selection: str, codegen_name: str) -> Optional[str]: + def _extract_custom_image_tag( + self, selection: str, codegen_name: str + ) -> Optional[str]: """ Extract custom image tag from TopCP selection query. @@ -87,16 +100,18 @@ def _extract_custom_image_tag(self, selection: str, codegen_name: str) -> Option :param codegen_name: The code generator name :returns: Custom image tag if present and valid, None otherwise """ - if codegen_name != 'topcp': + if codegen_name != "topcp": return None try: import json + query = json.loads(selection) - return query.get('image_tag') + return query.get("image_tag") except (json.JSONDecodeError, AttributeError): # Invalid JSON or non-dict selection return None + @classmethod def make_api( cls, @@ -270,7 +285,9 @@ def post(self): ) # Extract custom image tag from TopCP queries before code generation - custom_image_tag = self._extract_custom_image_tag(request_rec.selection, user_codegen_name) + custom_image_tag = self._extract_custom_image_tag( + request_rec.selection, user_codegen_name + ) # The first thing to do is make sure the requested selection is correct, # and can generate the requested code @@ -294,14 +311,14 @@ def post(self): request_rec.image = f"{validated_image}:{validated_tag}" current_app.logger.info( f"Using custom TopCP image: {request_rec.image}", - extra={"requestId": request_id} + extra={"requestId": request_id}, ) else: request_rec.image = codegen_transformer_image except BadRequest as e: current_app.logger.error( f"Invalid custom image tag: {str(e)}", - extra={"requestId": request_id} + extra={"requestId": request_id}, ) return {"message": str(e)}, 400 else: diff --git a/servicex_app/servicex_app_test/test_custom_image_validation.py b/servicex_app/servicex_app_test/test_custom_image_validation.py index 43a40d193..c0198f46a 100644 --- a/servicex_app/servicex_app_test/test_custom_image_validation.py +++ b/servicex_app/servicex_app_test/test_custom_image_validation.py @@ -4,34 +4,32 @@ import unittest from werkzeug.exceptions import BadRequest -from servicex_app.resources.transformation.submit import validate_custom_image_tag, SubmitTransformationRequest +from servicex_app.resources.transformation.submit import ( + validate_custom_image_tag, + SubmitTransformationRequest, +) class TestCustomImageValidation(unittest.TestCase): def setUp(self): self.config = { - 'TOPCP_ALLOWED_REPOSITORIES': [ - 'sslhep/servicex_science_image_topcp', - 'registry.gitlab.com/topcp-project/toolkit' + "TOPCP_ALLOWED_REPOSITORIES": [ + "sslhep/servicex_science_image_topcp", + "registry.gitlab.com/topcp-project/toolkit", ], - 'TOPCP_IMAGE_TAG_PATTERN': r'^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$', - 'TOPCP_DEFAULT_BASE_IMAGE': 'sslhep/servicex_science_image_topcp' + "TOPCP_IMAGE_TAG_PATTERN": r"^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$", + "TOPCP_DEFAULT_BASE_IMAGE": "sslhep/servicex_science_image_topcp", } def test_valid_image_tag_format(self): """Test that valid image tag formats are accepted.""" - test_cases = [ - "v2.20.0_v0.2", - "2.20.0_v0.2", - "v1.15.0-v1.0", - "3.0.0_v2.5" - ] + test_cases = ["v2.20.0_v0.2", "2.20.0_v0.2", "v1.15.0-v1.0", "3.0.0_v2.5"] for tag in test_cases: with self.subTest(tag=tag): - image, result_tag = validate_custom_image_tag(tag, self.config, 'topcp') - self.assertEqual(image, 'sslhep/servicex_science_image_topcp') + image, result_tag = validate_custom_image_tag(tag, self.config, "topcp") + self.assertEqual(image, "sslhep/servicex_science_image_topcp") self.assertEqual(result_tag, tag) def test_invalid_image_tag_format(self): @@ -42,17 +40,17 @@ def test_invalid_image_tag_format(self): "v2.20.0", "invalid-tag", "v2.20.0_v0.2.1", # Too many version components - "2.20_v0.2", # Missing patch version + "2.20_v0.2", # Missing patch version ] for tag in test_cases: with self.subTest(tag=tag): with self.assertRaises(BadRequest): - validate_custom_image_tag(tag, self.config, 'topcp') + validate_custom_image_tag(tag, self.config, "topcp") def test_non_topcp_codegen_returns_none(self): """Test that non-TopCP code generators return None.""" - image, tag = validate_custom_image_tag("v2.20.0_v0.2", self.config, 'uproot') + image, tag = validate_custom_image_tag("v2.20.0_v0.2", self.config, "uproot") self.assertIsNone(image) self.assertIsNone(tag) @@ -62,7 +60,7 @@ def test_empty_image_tag_returns_none(self): for tag in test_cases: with self.subTest(tag=tag): - image, result_tag = validate_custom_image_tag(tag, self.config, 'topcp') + image, result_tag = validate_custom_image_tag(tag, self.config, "topcp") self.assertIsNone(image) self.assertIsNone(result_tag) @@ -70,8 +68,8 @@ def test_missing_config_uses_defaults(self): """Test that missing configuration uses default values.""" minimal_config = {} - image, tag = validate_custom_image_tag("v2.20.0_v0.2", minimal_config, 'topcp') - self.assertEqual(image, 'sslhep/servicex_science_image_topcp') + image, tag = validate_custom_image_tag("v2.20.0_v0.2", minimal_config, "topcp") + self.assertEqual(image, "sslhep/servicex_science_image_topcp") self.assertEqual(tag, "v2.20.0_v0.2") @@ -89,10 +87,12 @@ def test_extract_image_tag_from_topcp_query(self): "max_events": -1, "no_systematics": False, "no_filter": False, - "image_tag": "v2.20.0_v0.2" + "image_tag": "v2.20.0_v0.2", } - result = self.submit_request._extract_custom_image_tag(json.dumps(query), 'topcp') + result = self.submit_request._extract_custom_image_tag( + json.dumps(query), "topcp" + ) self.assertEqual(result, "v2.20.0_v0.2") def test_extract_image_tag_missing_returns_none(self): @@ -103,39 +103,39 @@ def test_extract_image_tag_missing_returns_none(self): "particle": None, "max_events": -1, "no_systematics": False, - "no_filter": False + "no_filter": False, } - result = self.submit_request._extract_custom_image_tag(json.dumps(query), 'topcp') + result = self.submit_request._extract_custom_image_tag( + json.dumps(query), "topcp" + ) self.assertIsNone(result) def test_extract_image_tag_non_topcp_returns_none(self): """Test that non-TopCP codegen returns None.""" - query = { - "selection": "some_selection", - "image_tag": "v2.20.0_v0.2" - } + query = {"selection": "some_selection", "image_tag": "v2.20.0_v0.2"} - result = self.submit_request._extract_custom_image_tag(json.dumps(query), 'uproot') + result = self.submit_request._extract_custom_image_tag( + json.dumps(query), "uproot" + ) self.assertIsNone(result) def test_extract_image_tag_invalid_json_returns_none(self): """Test that invalid JSON returns None.""" invalid_json = "{ invalid json }" - result = self.submit_request._extract_custom_image_tag(invalid_json, 'topcp') + result = self.submit_request._extract_custom_image_tag(invalid_json, "topcp") self.assertIsNone(result) def test_extract_image_tag_none_value(self): """Test that None image tag value is returned as None.""" - query = { - "reco": "reco_config", - "image_tag": None - } + query = {"reco": "reco_config", "image_tag": None} - result = self.submit_request._extract_custom_image_tag(json.dumps(query), 'topcp') + result = self.submit_request._extract_custom_image_tag( + json.dumps(query), "topcp" + ) self.assertIsNone(result) -if __name__ == '__main__': - unittest.main() \ No newline at end of file +if __name__ == "__main__": + unittest.main() From 86d536d545a03f2ef3e6fc4605a25cb2983f6c57 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 23 Sep 2025 10:52:48 -0500 Subject: [PATCH 03/33] resolve flake8 issues --- .../tests/test_custom_image_tag.py | 2 +- .../resources/transformation/submit.py | 17 +++++++++-------- .../test_custom_image_validation.py | 6 ++++-- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/code_generator_TopCPToolkit/tests/test_custom_image_tag.py b/code_generator_TopCPToolkit/tests/test_custom_image_tag.py index 18c2cf60e..635228a85 100644 --- a/code_generator_TopCPToolkit/tests/test_custom_image_tag.py +++ b/code_generator_TopCPToolkit/tests/test_custom_image_tag.py @@ -88,4 +88,4 @@ def test_generated_files_created_with_image_tag(self): if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 3aec30274..2e16772b3 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -58,16 +58,15 @@ def validate_custom_image_tag(image_tag: str, config: dict, codegen_name: str) - return None, None # Get validation configuration (with defaults) - allowed_repos = config.get('TOPCP_ALLOWED_REPOSITORIES', [ - 'sslhep/servicex_science_image_topcp', - 'registry.gitlab.com/topcp-project/toolkit' - ]) - tag_pattern = config.get('TOPCP_IMAGE_TAG_PATTERN', r'^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$') - default_base_image = config.get('TOPCP_DEFAULT_BASE_IMAGE', 'sslhep/servicex_science_image_topcp') + tag_pattern = config.get('TOPCP_IMAGE_TAG_PATTERN', + r'^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$') + default_base_image = config.get('TOPCP_DEFAULT_BASE_IMAGE', + 'sslhep/servicex_science_image_topcp') # Validate tag format if not re.match(tag_pattern, image_tag): - raise BadRequest(f"Invalid TopCP image tag format: {image_tag}. Expected format: v2.20.0_v0.2") + raise BadRequest(f"Invalid TopCP image tag format: {image_tag}. " + f"Expected format: v2.20.0_v0.2") # For now, use the default base image with the custom tag # In the future, this could be extended to allow custom repositories @@ -97,6 +96,7 @@ def _extract_custom_image_tag(self, selection: str, codegen_name: str) -> Option except (json.JSONDecodeError, AttributeError): # Invalid JSON or non-dict selection return None + @classmethod def make_api( cls, @@ -270,7 +270,8 @@ def post(self): ) # Extract custom image tag from TopCP queries before code generation - custom_image_tag = self._extract_custom_image_tag(request_rec.selection, user_codegen_name) + custom_image_tag = self._extract_custom_image_tag( + request_rec.selection, user_codegen_name) # The first thing to do is make sure the requested selection is correct, # and can generate the requested code diff --git a/servicex_app/servicex_app_test/test_custom_image_validation.py b/servicex_app/servicex_app_test/test_custom_image_validation.py index 43a40d193..ce02bf41a 100644 --- a/servicex_app/servicex_app_test/test_custom_image_validation.py +++ b/servicex_app/servicex_app_test/test_custom_image_validation.py @@ -4,7 +4,9 @@ import unittest from werkzeug.exceptions import BadRequest -from servicex_app.resources.transformation.submit import validate_custom_image_tag, SubmitTransformationRequest +from servicex_app.resources.transformation.submit import ( + validate_custom_image_tag, SubmitTransformationRequest +) class TestCustomImageValidation(unittest.TestCase): @@ -138,4 +140,4 @@ def test_extract_image_tag_none_value(self): if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From b518d4086aca9f0f84623e8c0ece934c67bdace7 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 23 Sep 2025 10:55:27 -0500 Subject: [PATCH 04/33] Revert "resolve flake8 issues" This reverts commit 86d536d545a03f2ef3e6fc4605a25cb2983f6c57. --- .../tests/test_custom_image_tag.py | 2 +- .../resources/transformation/submit.py | 17 ++++++++--------- .../test_custom_image_validation.py | 6 ++---- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/code_generator_TopCPToolkit/tests/test_custom_image_tag.py b/code_generator_TopCPToolkit/tests/test_custom_image_tag.py index 635228a85..18c2cf60e 100644 --- a/code_generator_TopCPToolkit/tests/test_custom_image_tag.py +++ b/code_generator_TopCPToolkit/tests/test_custom_image_tag.py @@ -88,4 +88,4 @@ def test_generated_files_created_with_image_tag(self): if __name__ == '__main__': - unittest.main() + unittest.main() \ No newline at end of file diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 2e16772b3..3aec30274 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -58,15 +58,16 @@ def validate_custom_image_tag(image_tag: str, config: dict, codegen_name: str) - return None, None # Get validation configuration (with defaults) - tag_pattern = config.get('TOPCP_IMAGE_TAG_PATTERN', - r'^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$') - default_base_image = config.get('TOPCP_DEFAULT_BASE_IMAGE', - 'sslhep/servicex_science_image_topcp') + allowed_repos = config.get('TOPCP_ALLOWED_REPOSITORIES', [ + 'sslhep/servicex_science_image_topcp', + 'registry.gitlab.com/topcp-project/toolkit' + ]) + tag_pattern = config.get('TOPCP_IMAGE_TAG_PATTERN', r'^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$') + default_base_image = config.get('TOPCP_DEFAULT_BASE_IMAGE', 'sslhep/servicex_science_image_topcp') # Validate tag format if not re.match(tag_pattern, image_tag): - raise BadRequest(f"Invalid TopCP image tag format: {image_tag}. " - f"Expected format: v2.20.0_v0.2") + raise BadRequest(f"Invalid TopCP image tag format: {image_tag}. Expected format: v2.20.0_v0.2") # For now, use the default base image with the custom tag # In the future, this could be extended to allow custom repositories @@ -96,7 +97,6 @@ def _extract_custom_image_tag(self, selection: str, codegen_name: str) -> Option except (json.JSONDecodeError, AttributeError): # Invalid JSON or non-dict selection return None - @classmethod def make_api( cls, @@ -270,8 +270,7 @@ def post(self): ) # Extract custom image tag from TopCP queries before code generation - custom_image_tag = self._extract_custom_image_tag( - request_rec.selection, user_codegen_name) + custom_image_tag = self._extract_custom_image_tag(request_rec.selection, user_codegen_name) # The first thing to do is make sure the requested selection is correct, # and can generate the requested code diff --git a/servicex_app/servicex_app_test/test_custom_image_validation.py b/servicex_app/servicex_app_test/test_custom_image_validation.py index ce02bf41a..43a40d193 100644 --- a/servicex_app/servicex_app_test/test_custom_image_validation.py +++ b/servicex_app/servicex_app_test/test_custom_image_validation.py @@ -4,9 +4,7 @@ import unittest from werkzeug.exceptions import BadRequest -from servicex_app.resources.transformation.submit import ( - validate_custom_image_tag, SubmitTransformationRequest -) +from servicex_app.resources.transformation.submit import validate_custom_image_tag, SubmitTransformationRequest class TestCustomImageValidation(unittest.TestCase): @@ -140,4 +138,4 @@ def test_extract_image_tag_none_value(self): if __name__ == '__main__': - unittest.main() + unittest.main() \ No newline at end of file From 8286d955e87989412e7be2004b07095c30f120db Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 23 Sep 2025 11:05:12 -0500 Subject: [PATCH 05/33] fix broken test --- .../servicex/TopCP_code_generator/query_translate.py | 2 +- .../servicex_app/resources/transformation/submit.py | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py index 773a5ea69..0b5518e82 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py @@ -87,7 +87,7 @@ def generate_files_from_query(query, query_file_path): ) # check for reco.yaml, parton.yaml and particle.yaml files - if isinstance(jquery[key], str): + if isinstance(jquery[key], str) and "fileName" in options[key]: with open( os.path.join(query_file_path, options[key]["fileName"]), "w" ) as file: diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 4fe0e789d..9d9469c5d 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -60,13 +60,6 @@ def validate_custom_image_tag( return None, None # Get validation configuration (with defaults) - allowed_repos = config.get( - "TOPCP_ALLOWED_REPOSITORIES", - [ - "sslhep/servicex_science_image_topcp", - "registry.gitlab.com/topcp-project/toolkit", - ], - ) tag_pattern = config.get( "TOPCP_IMAGE_TAG_PATTERN", r"^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$" ) From 2e706b7643a5e79df206f6489c6aff63070dbc44 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 23 Sep 2025 11:10:05 -0500 Subject: [PATCH 06/33] add back allowed_repos logic --- .../resources/transformation/submit.py | 16 ++++++++++-- .../test_custom_image_validation.py | 26 +++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 9d9469c5d..07a7dce3b 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -60,6 +60,13 @@ def validate_custom_image_tag( return None, None # Get validation configuration (with defaults) + allowed_repos = config.get( + "TOPCP_ALLOWED_REPOSITORIES", + [ + "sslhep/servicex_science_image_topcp", + "registry.gitlab.com/topcp-project/toolkit", + ], + ) tag_pattern = config.get( "TOPCP_IMAGE_TAG_PATTERN", r"^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$" ) @@ -73,8 +80,13 @@ def validate_custom_image_tag( f"Invalid TopCP image tag format: {image_tag}. Expected format: v2.20.0_v0.2" ) - # For now, use the default base image with the custom tag - # In the future, this could be extended to allow custom repositories + # Validate that the default base image is in the allowed repositories + if default_base_image not in allowed_repos: + raise BadRequest( + f"Default base image {default_base_image} is not in allowed repositories: " + f"{allowed_repos}" + ) + validated_image = default_base_image validated_tag = image_tag diff --git a/servicex_app/servicex_app_test/test_custom_image_validation.py b/servicex_app/servicex_app_test/test_custom_image_validation.py index c0198f46a..9bfc9f4ec 100644 --- a/servicex_app/servicex_app_test/test_custom_image_validation.py +++ b/servicex_app/servicex_app_test/test_custom_image_validation.py @@ -72,6 +72,32 @@ def test_missing_config_uses_defaults(self): self.assertEqual(image, "sslhep/servicex_science_image_topcp") self.assertEqual(tag, "v2.20.0_v0.2") + def test_repository_validation_success(self): + """Test that repository validation passes when default image is in allowed repos.""" + config = { + "TOPCP_ALLOWED_REPOSITORIES": [ + "sslhep/servicex_science_image_topcp", + "registry.gitlab.com/topcp-project/toolkit", + ], + "TOPCP_DEFAULT_BASE_IMAGE": "sslhep/servicex_science_image_topcp", + } + + image, tag = validate_custom_image_tag("v2.20.0_v0.2", config, "topcp") + self.assertEqual(image, "sslhep/servicex_science_image_topcp") + self.assertEqual(tag, "v2.20.0_v0.2") + + def test_repository_validation_failure(self): + """Test that repository validation fails when default image is not in allowed repos.""" + config = { + "TOPCP_ALLOWED_REPOSITORIES": [ + "registry.gitlab.com/topcp-project/toolkit", + ], + "TOPCP_DEFAULT_BASE_IMAGE": "sslhep/servicex_science_image_topcp", + } + + with self.assertRaises(BadRequest): + validate_custom_image_tag("v2.20.0_v0.2", config, "topcp") + class TestImageTagExtraction(unittest.TestCase): From fe976f3e14ffb4492b05283645b8999efbf38de5 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 23 Sep 2025 11:17:11 -0500 Subject: [PATCH 07/33] remove accidental values changes --- helm/servicex/values.yaml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/helm/servicex/values.yaml b/helm/servicex/values.yaml index 02d1707fd..4c1ad0a58 100644 --- a/helm/servicex/values.yaml +++ b/helm/servicex/values.yaml @@ -152,8 +152,7 @@ logging: minio: volumePermissions: image: -# repository: bitnami/bitnami-shell-archived - repository: bitnami/os-shell + repository: bitnami/bitnami-shell-archived auth: rootPassword: leftfoot1 rootUser: miniouser @@ -175,8 +174,7 @@ postgres: postgresql: volumePermissions: image: -# repository: bitnami/bitnami-shell-archived - repository: bitnami/os-shell + repository: bitnami/bitnami-shell-archived global: postgresql: auth: @@ -197,8 +195,7 @@ rabbitmq: volumePermissions: enabled: true image: -# repository: bitnami/bitnami-shell-archived - repository: bitnami/os-shell + repository: bitnami/bitnami-shell-archived extraConfiguration: |- consumer_timeout = 3600000 From e12925647fd016a4f8be50c847a0672ba4566281 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Wed, 8 Oct 2025 09:33:47 -0500 Subject: [PATCH 08/33] moving towards MVP (app server is failing due to bad mount) --- Procfile | 3 +- code_generator_TopCPToolkit/Dockerfile | 7 +- .../TopCP_code_generator/query_translate.py | 18 ++- .../request_translator.py | 20 ++- .../{ => servicex}/boot.sh | 2 +- .../templates/transform_single_file.py | 13 +- helm/servicex/templates/app/configmap.yaml | 1 + .../templates/codegen/deployment.yaml | 9 ++ helm/servicex/values.yaml | 22 ++-- servicex_app/pyproject.toml | 2 +- servicex_app/servicex_app/code_gen_adapter.py | 10 +- .../servicex_app/docker_repo_adapter.py | 82 ++++++++++++ .../resources/transformation/submit.py | 120 +++--------------- 13 files changed, 179 insertions(+), 130 deletions(-) rename code_generator_TopCPToolkit/{ => servicex}/boot.sh (54%) diff --git a/Procfile b/Procfile index cfe64a9c2..8ddfaea4b 100644 --- a/Procfile +++ b/Procfile @@ -1,4 +1,5 @@ -minikube-mount: minikube mount $LOCAL_DIR:/mnt/servicex +minikube-mount-app: minikube mount $CHART_DIR:/mnt/servicex +minikube-mount-topcp: minikube mount $LOCAL_DIR/code_generator_TopCPToolkit/servicex:/mnt/topcp helm-install: sleep 5; cd $CHART_DIR && helm install -f $VALUES_FILE servicex . && tail -f /dev/null port-forward-app: sleep 30 && cd $LOCAL_DIR && bash local/port-forward.sh app port-forward-minio: sleep 20 && cd $LOCAL_DIR && bash local/port-forward.sh minio diff --git a/code_generator_TopCPToolkit/Dockerfile b/code_generator_TopCPToolkit/Dockerfile index ec8adf0ce..98136d379 100644 --- a/code_generator_TopCPToolkit/Dockerfile +++ b/code_generator_TopCPToolkit/Dockerfile @@ -17,9 +17,6 @@ RUN poetry config virtualenvs.create false && \ RUN pip install gunicorn -COPY boot.sh ./ -RUN chmod 755 boot.sh - COPY transformer_capabilities.json ./ RUN chmod 644 transformer_capabilities.json @@ -29,9 +26,11 @@ RUN chmod 777 -R servicex COPY app.conf . RUN chmod 755 app.conf +RUN chmod 755 /home/servicex/servicex/boot.sh + USER servicex ENV CODEGEN_CONFIG_FILE="/home/servicex/app.conf" EXPOSE 5000 -ENTRYPOINT ["./boot.sh"] +ENTRYPOINT ["/home/servicex/servicex/boot.sh"] diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py index 0b5518e82..1644e6fad 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py @@ -36,10 +36,10 @@ "ifTrue": ["--no-filter"], "ifFalse": None, }, - "image_tag": { + "docker_image": { "properType": str, "properTypeString": "string", - "default": "2.17.0-25.2.45", + "default": "sslhep/servicex_science_image_topcp:2.17.0-25.2.45", "optional": True, }, } @@ -48,6 +48,15 @@ def generate_files_from_query(query, query_file_path): jquery = json.loads(query) + # transformer_image = jquery.get("docker_image", "sslhep/servicex_science_image_topcp:2.17.0-25.2.45") + # metadata = { + # "transformer_image": transformer_image + # } + # with open( + # os.path.join(query_file_path, "image_metadata.json"), "w" + # ) as metadata_file: + # json.dump(metadata, metadata_file) + runTopCommand = [ "runTop_el.py", "-i", @@ -70,7 +79,10 @@ def generate_files_from_query(query, query_file_path): ) for key in jquery: - # ensure only aviable options are allowed + if key == "docker_image": + continue + + # ensure only available options are allowed if key not in options: raise KeyError( key + " is not implemented. Available keys: " + str(options.keys()) diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py index 76ebcf0c8..d1ff541cd 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py @@ -26,6 +26,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os +import json import shutil from . import query_translate from servicex_codegen.code_generator import ( @@ -62,11 +63,26 @@ def generate_code(self, query, cache_path: str): capabilities_path = os.environ.get( "CAPABILITIES_PATH", "/home/servicex/transformer_capabilities.json" ) + + # Generate query files first to create any metadata + query_translate.generate_files_from_query(query, query_file_path) + + # Copy capabilities file shutil.copyfile( capabilities_path, os.path.join(query_file_path, "transformer_capabilities.json"), ) - query_translate.generate_files_from_query(query, query_file_path) - return GeneratedFileResult(_hash, query_file_path) + # Check if custom transformer image was specified + # image_metadata_path = os.path.join(query_file_path, "image_metadata.json") + # transformer_image = "sslhep/servicex_science_image_topcp:2.17.0-25.2.45" + # if os.path.exists(image_metadata_path): + # with open(image_metadata_path, "r") as f: + # metadata = json.load(f) + # transformer_image = metadata.get("transformer_image", transformer_image) + # + # generated_file = GeneratedFileResult(_hash, query_file_path) + # generated_file.image = transformer_image + # + # return generated_file diff --git a/code_generator_TopCPToolkit/boot.sh b/code_generator_TopCPToolkit/servicex/boot.sh similarity index 54% rename from code_generator_TopCPToolkit/boot.sh rename to code_generator_TopCPToolkit/servicex/boot.sh index 599ab4610..350504fd7 100644 --- a/code_generator_TopCPToolkit/boot.sh +++ b/code_generator_TopCPToolkit/servicex/boot.sh @@ -4,7 +4,7 @@ action=${1:-web_service} if [ "$action" = "web_service" ] ; then mkdir instance - exec gunicorn -b :5000 --workers=2 --threads=1 --access-logfile - --error-logfile - "servicex.TopCP_code_generator:create_app()" + exec gunicorn -b :5000 --reload --workers=2 --threads=1 --access-logfile - --error-logfile - "servicex.TopCP_code_generator:create_app()" else echo "Unknown action '$action'" fi diff --git a/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py b/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py index 264b1c51f..92722f3aa 100644 --- a/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py +++ b/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py @@ -12,24 +12,27 @@ def transform_single_file(file_path: str, output_path: Path, output_format: str) # create input.txt file for event loop and insert file_path as only line with open("input.txt", "w") as f: f.write(file_path) - # move reco.yaml, parton.yaml and particle.yaml if they exit to CONFIG_LOC loacation + + # Get CONFIG_LOC with a sensible default (current working directory) + config_loc = os.environ.get("CONFIG_LOC", os.getcwd()) + + # move reco.yaml, parton.yaml and particle.yaml if they exist to CONFIG_LOC location if os.path.exists("/generated/reco.yaml"): shutil.copyfile( "/generated/reco.yaml", - os.path.join(os.environ.get("CONFIG_LOC"), "reco.yaml"), + os.path.join(config_loc, "reco.yaml"), ) if os.path.exists("/generated/parton.yaml"): shutil.copyfile( "/generated/parton.yaml", - os.path.join(os.environ.get("CONFIG_LOC"), "parton.yaml"), + os.path.join(config_loc, "parton.yaml"), ) if os.path.exists("/generated/particle.yaml"): shutil.copyfile( "/generated/particle.yaml", - os.path.join(os.environ.get("CONFIG_LOC"), "particle.yaml"), + os.path.join(config_loc, "particle.yaml"), ) - generated_transformer.runTop_el() subprocess.run(["mv", "output.root", output_path]) diff --git a/helm/servicex/templates/app/configmap.yaml b/helm/servicex/templates/app/configmap.yaml index 67067878a..7f0be9c96 100644 --- a/helm/servicex/templates/app/configmap.yaml +++ b/helm/servicex/templates/app/configmap.yaml @@ -183,6 +183,7 @@ data: TOPCP_ALLOWED_REPOSITORIES = {{ .Values.codeGen.topcp.allowedRepositories | toJson }} TOPCP_IMAGE_TAG_PATTERN = '{{ .Values.codeGen.topcp.imageTagPattern }}' TOPCP_DEFAULT_BASE_IMAGE = '{{ .Values.codeGen.topcp.defaultBaseImage }}' + TOPCP_DEFAULT_BASE_TAG = '{{ .Values.codeGen.topcp.defaultBaseTag }}' {{- end }} {{- $didFinders := list }} diff --git a/helm/servicex/templates/codegen/deployment.yaml b/helm/servicex/templates/codegen/deployment.yaml index 97d225ae9..02460ca29 100644 --- a/helm/servicex/templates/codegen/deployment.yaml +++ b/helm/servicex/templates/codegen/deployment.yaml @@ -35,5 +35,14 @@ spec: imagePullPolicy: {{ .pullPolicy }} ports: - containerPort: 5000 + volumeMounts: + - name: host-topcp-volume + mountPath: /home/servicex/servicex + + volumes: + - name: host-topcp-volume + hostPath: + path: /mnt/topcp + type: DirectoryOrCreate {{- end }} {{- end }} diff --git a/helm/servicex/values.yaml b/helm/servicex/values.yaml index 4c1ad0a58..556882012 100644 --- a/helm/servicex/values.yaml +++ b/helm/servicex/values.yaml @@ -104,15 +104,14 @@ codeGen: topcp: enabled: true image: sslhep/servicex_code_gen_topcp - pullPolicy: Always + pullPolicy: IfNotPresent tag: develop defaultScienceContainerImage: sslhep/servicex_science_image_topcp defaultScienceContainerTag: 2.17.0-25.2.45 - # Configuration for custom image tag validation allowedRepositories: - - "sslhep/servicex_science_image_topcp" - - "registry.gitlab.com/topcp-project/toolkit" - imageTagPattern: "^v?\\d+\\.\\d+\\.\\d+[-_]v?\\d+\\.\\d+$" + - "sslhep" + - "mattshirley" + imageTagPattern: ".*" defaultBaseImage: "sslhep/servicex_science_image_topcp" didFinder: @@ -152,7 +151,8 @@ logging: minio: volumePermissions: image: - repository: bitnami/bitnami-shell-archived +# repository: bitnami/bitnami-shell-archived + repository: bitnami/os-shell auth: rootPassword: leftfoot1 rootUser: miniouser @@ -174,7 +174,8 @@ postgres: postgresql: volumePermissions: image: - repository: bitnami/bitnami-shell-archived +# repository: bitnami/bitnami-shell-archived + repository: bitnami/os-shell global: postgresql: auth: @@ -195,7 +196,8 @@ rabbitmq: volumePermissions: enabled: true image: - repository: bitnami/bitnami-shell-archived +# repository: bitnami/bitnami-shell-archived + repository: bitnami/os-shell extraConfiguration: |- consumer_timeout = 3600000 @@ -218,8 +220,8 @@ transformer: sidecarImage: sslhep/servicex_sidecar_transformer sidecarTag: develop - sidecarPullPolicy: Always - scienceContainerPullPolicy: Always + sidecarPullPolicy: IfNotPresent + scienceContainerPullPolicy: IfNotPresent language: python exec: # replace me diff --git a/servicex_app/pyproject.toml b/servicex_app/pyproject.toml index 7dab5eaa3..7f70bdc26 100644 --- a/servicex_app/pyproject.toml +++ b/servicex_app/pyproject.toml @@ -34,7 +34,7 @@ blinker = "^1.5" pre-commit = "^2.20.0" minio = "^7.1.12" flask-migrate = "^3.1.0" -psycopg2 = "^2.9.5" +psycopg2-binary = "^2.9.5" python-logstash = "^0.4.8" humanize = "^4.4.0" gunicorn = "^23.0.0" diff --git a/servicex_app/servicex_app/code_gen_adapter.py b/servicex_app/servicex_app/code_gen_adapter.py index b4361acd2..7917a0eb7 100644 --- a/servicex_app/servicex_app/code_gen_adapter.py +++ b/servicex_app/servicex_app/code_gen_adapter.py @@ -25,6 +25,8 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from typing import Optional + import requests from requests_toolbelt.multipart import decoder @@ -43,7 +45,11 @@ def post_request(self, post_url, post_obj): return result def generate_code_for_selection( - self, request_record: TransformRequest, namespace: str, user_codegen_name: str + self, + request_record: TransformRequest, + namespace: str, + user_codegen_name: str, + custom_image: Optional[str] = None, ) -> tuple[str, str, str, str]: """ Generates the C++ code for a request's selection string. @@ -68,6 +74,8 @@ def generate_code_for_selection( if not post_url: raise ValueError(f"{user_codegen_name}, code generator unavailable for use") + print("generate_code_for_selection") + print(request_record.selection) result = self.post_request( post_url + "/servicex/generated-code", post_obj={ diff --git a/servicex_app/servicex_app/docker_repo_adapter.py b/servicex_app/servicex_app/docker_repo_adapter.py index 01976ed40..46e8773cd 100644 --- a/servicex_app/servicex_app/docker_repo_adapter.py +++ b/servicex_app/servicex_app/docker_repo_adapter.py @@ -64,3 +64,85 @@ def check_image_exists(self, tagged_image: str) -> bool: f"last updated {r.json()['last_updated']}" ) return True + + @servicex_retry() + def get_image_manifest(self, repo: str, image: str, tag: str) -> requests.Response: + """Get Docker image manifest from registry API v2.""" + # Try Docker Hub v2 API first for manifest + query = f"https://registry-1.docker.io/v2/{repo}/{image}/manifests/{tag}" + headers = {"Accept": "application/vnd.docker.distribution.manifest.v2+json"} + try: + r = requests.get(query, headers=headers, timeout=REQUEST_TIMEOUT) + if r.status_code == 200: + return r + except Exception: + pass + + # Fall back to Docker Hub v1 API for basic info + query = f"{self.registry_endpoint}/v2/repositories/{repo}/{image}/tags/{tag}" + r = requests.get(query, timeout=REQUEST_TIMEOUT) + return r + + def get_image_info(self, tagged_image: str) -> dict: + """ + Get detailed information about a Docker image including layers and configuration. + + :param tagged_image: Full Docker image name, e.g. "sslhep/servicex_app:latest" + :return: Dictionary containing image metadata, layers, config, etc. + """ + search_result = re.search("(.+)/(.+):(.+)", tagged_image) + if not search_result or len(search_result.groups()) != 3: + current_app.logger.warning(f"Invalid image format: {tagged_image}") + return None + + (repo, image, tag) = search_result.groups() + + try: + # Get manifest/detailed info + r = self.get_image_manifest(repo, image, tag) + + if r.status_code != 200: + current_app.logger.warning( + f"Failed to get image info for {tagged_image}: {r.status_code}" + ) + return None + + manifest_data = r.json() + + # Extract relevant information depending on API version + image_info = { + "digest": manifest_data.get("digest"), + "layers": [], + "config": {}, + "history": [], + } + + # Handle Docker Registry v2 manifest format + if "layers" in manifest_data: + image_info["layers"] = [ + layer.get("digest", "") for layer in manifest_data["layers"] + ] + + # Handle config blob reference + if "config" in manifest_data: + config_digest = manifest_data["config"].get("digest") + if config_digest: + # For full implementation, we'd fetch the config blob here + # For now, store the reference + image_info["config"] = {"digest": config_digest} + + # Handle Docker Hub v1 API response format + if "last_updated" in manifest_data: + image_info["last_updated"] = manifest_data["last_updated"] + + # Add some basic metadata + image_info["tag_info"] = manifest_data + + current_app.logger.debug(f"Retrieved image info for {tagged_image}") + return image_info + + except Exception as e: + current_app.logger.warning( + f"Error retrieving image info for {tagged_image}: {str(e)}" + ) + return None diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 07a7dce3b..5d977fc98 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -27,6 +27,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import uuid import re +import json from datetime import datetime, timezone from typing import Optional, List @@ -39,84 +40,7 @@ from servicex_app.resources.servicex_resource import ServiceXResource from werkzeug.exceptions import BadRequest - -def validate_custom_image_tag( - image_tag: str, config: dict, codegen_name: str -) -> tuple[str, str]: - """ - Validate custom image tag for TopCP transformations. - - :param image_tag: The custom image tag provided by user - :param config: Flask application configuration - :param codegen_name: The code generator name (e.g., 'topcp') - :returns: tuple of (validated_image_name, validated_tag) - :raises: BadRequest if validation fails - """ - if not image_tag or not image_tag.strip(): - return None, None - - # Only validate for TopCP requests - if codegen_name != "topcp": - return None, None - - # Get validation configuration (with defaults) - allowed_repos = config.get( - "TOPCP_ALLOWED_REPOSITORIES", - [ - "sslhep/servicex_science_image_topcp", - "registry.gitlab.com/topcp-project/toolkit", - ], - ) - tag_pattern = config.get( - "TOPCP_IMAGE_TAG_PATTERN", r"^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$" - ) - default_base_image = config.get( - "TOPCP_DEFAULT_BASE_IMAGE", "sslhep/servicex_science_image_topcp" - ) - - # Validate tag format - if not re.match(tag_pattern, image_tag): - raise BadRequest( - f"Invalid TopCP image tag format: {image_tag}. Expected format: v2.20.0_v0.2" - ) - - # Validate that the default base image is in the allowed repositories - if default_base_image not in allowed_repos: - raise BadRequest( - f"Default base image {default_base_image} is not in allowed repositories: " - f"{allowed_repos}" - ) - - validated_image = default_base_image - validated_tag = image_tag - - return validated_image, validated_tag - - class SubmitTransformationRequest(ServiceXResource): - - def _extract_custom_image_tag( - self, selection: str, codegen_name: str - ) -> Optional[str]: - """ - Extract custom image tag from TopCP selection query. - - :param selection: The query selection string - :param codegen_name: The code generator name - :returns: Custom image tag if present and valid, None otherwise - """ - if codegen_name != "topcp": - return None - - try: - import json - - query = json.loads(selection) - return query.get("image_tag") - except (json.JSONDecodeError, AttributeError): - # Invalid JSON or non-dict selection - return None - @classmethod def make_api( cls, @@ -243,6 +167,9 @@ def post(self): file_list = args.get("file-list") user_codegen_name = args.get("codegen") + print("CODEGEN!!!") + print(config["CODE_GEN_IMAGES"]) + print(user_codegen_name) code_gen_image_name = config["CODE_GEN_IMAGES"].get(user_codegen_name, None) namespace = config["TRANSFORMER_NAMESPACE"] @@ -271,6 +198,9 @@ def post(self): # TODO: need to check to make sure bucket was created # WHat happens if object-store and object_store is None? + print("SELECTION!!!") + print(args["selection"]) + request_rec = TransformRequest( request_id=str(request_id), title=args.get("title"), @@ -289,10 +219,12 @@ def post(self): files=0, ) - # Extract custom image tag from TopCP queries before code generation - custom_image_tag = self._extract_custom_image_tag( - request_rec.selection, user_codegen_name - ) + + selection = json.loads(args["selection"]) + custom_docker_image = None + if "docker_image" in selection: + custom_docker_image = selection["docker_image"] + del selection["docker_image"] # The first thing to do is make sure the requested selection is correct, # and can generate the requested code @@ -305,27 +237,11 @@ def post(self): request_rec, namespace, user_codegen_name ) - # Handle custom image tag for TopCP - if custom_image_tag: - try: - validated_image, validated_tag = validate_custom_image_tag( - custom_image_tag, config, user_codegen_name - ) - if validated_image and validated_tag: - # Override the default science container image - request_rec.image = f"{validated_image}:{validated_tag}" - current_app.logger.info( - f"Using custom TopCP image: {request_rec.image}", - extra={"requestId": request_id}, - ) - else: - request_rec.image = codegen_transformer_image - except BadRequest as e: - current_app.logger.error( - f"Invalid custom image tag: {str(e)}", - extra={"requestId": request_id}, - ) - return {"message": str(e)}, 400 + print("TEST!!") + print(codegen_transformer_image) + + if custom_docker_image: + request_rec.image = custom_docker_image else: request_rec.image = codegen_transformer_image From 03864761e6a6182c3fd4ea89edda8be0355eb15b Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 21 Oct 2025 13:13:52 -0500 Subject: [PATCH 09/33] update directory structure --- Procfile | 4 ++-- code_generator_TopCPToolkit/Dockerfile | 4 ++-- code_generator_TopCPToolkit/{servicex => }/boot.sh | 0 helm/servicex/templates/codegen/deployment.yaml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename code_generator_TopCPToolkit/{servicex => }/boot.sh (100%) mode change 100644 => 100755 diff --git a/Procfile b/Procfile index 8ddfaea4b..8b0636d50 100644 --- a/Procfile +++ b/Procfile @@ -1,5 +1,5 @@ -minikube-mount-app: minikube mount $CHART_DIR:/mnt/servicex -minikube-mount-topcp: minikube mount $LOCAL_DIR/code_generator_TopCPToolkit/servicex:/mnt/topcp +minikube-mount-app: minikube mount $LOCAL_DIR:/mnt/servicex +minikube-mount-topcp: minikube mount $LOCAL_DIR/code_generator_TopCPToolkit:/mnt/topcp helm-install: sleep 5; cd $CHART_DIR && helm install -f $VALUES_FILE servicex . && tail -f /dev/null port-forward-app: sleep 30 && cd $LOCAL_DIR && bash local/port-forward.sh app port-forward-minio: sleep 20 && cd $LOCAL_DIR && bash local/port-forward.sh minio diff --git a/code_generator_TopCPToolkit/Dockerfile b/code_generator_TopCPToolkit/Dockerfile index 98136d379..79a3a34ec 100644 --- a/code_generator_TopCPToolkit/Dockerfile +++ b/code_generator_TopCPToolkit/Dockerfile @@ -26,11 +26,11 @@ RUN chmod 777 -R servicex COPY app.conf . RUN chmod 755 app.conf -RUN chmod 755 /home/servicex/servicex/boot.sh +RUN chmod 755 /home/servicex/boot.sh USER servicex ENV CODEGEN_CONFIG_FILE="/home/servicex/app.conf" EXPOSE 5000 -ENTRYPOINT ["/home/servicex/servicex/boot.sh"] +ENTRYPOINT ["/home/servicex/boot.sh"] \ No newline at end of file diff --git a/code_generator_TopCPToolkit/servicex/boot.sh b/code_generator_TopCPToolkit/boot.sh old mode 100644 new mode 100755 similarity index 100% rename from code_generator_TopCPToolkit/servicex/boot.sh rename to code_generator_TopCPToolkit/boot.sh diff --git a/helm/servicex/templates/codegen/deployment.yaml b/helm/servicex/templates/codegen/deployment.yaml index 02460ca29..8f029b61f 100644 --- a/helm/servicex/templates/codegen/deployment.yaml +++ b/helm/servicex/templates/codegen/deployment.yaml @@ -37,7 +37,7 @@ spec: - containerPort: 5000 volumeMounts: - name: host-topcp-volume - mountPath: /home/servicex/servicex + mountPath: /home/servicex volumes: - name: host-topcp-volume From d0776b043d20da1edc0f6b21b726cb3c99305b1d Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 21 Oct 2025 16:13:18 -0500 Subject: [PATCH 10/33] tighten flake8 --- .../TopCP_code_generator/query_translate.py | 9 --------- .../TopCP_code_generator/request_translator.py | 13 ------------- .../servicex/templates/transform_single_file.py | 1 - .../servicex_app/resources/transformation/submit.py | 5 +---- 4 files changed, 1 insertion(+), 27 deletions(-) diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py index 1644e6fad..76e87e96d 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py @@ -48,15 +48,6 @@ def generate_files_from_query(query, query_file_path): jquery = json.loads(query) - # transformer_image = jquery.get("docker_image", "sslhep/servicex_science_image_topcp:2.17.0-25.2.45") - # metadata = { - # "transformer_image": transformer_image - # } - # with open( - # os.path.join(query_file_path, "image_metadata.json"), "w" - # ) as metadata_file: - # json.dump(metadata, metadata_file) - runTopCommand = [ "runTop_el.py", "-i", diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py index d1ff541cd..8e8a3839a 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py @@ -26,7 +26,6 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os -import json import shutil from . import query_translate from servicex_codegen.code_generator import ( @@ -74,15 +73,3 @@ def generate_code(self, query, cache_path: str): ) return GeneratedFileResult(_hash, query_file_path) - # Check if custom transformer image was specified - # image_metadata_path = os.path.join(query_file_path, "image_metadata.json") - # transformer_image = "sslhep/servicex_science_image_topcp:2.17.0-25.2.45" - # if os.path.exists(image_metadata_path): - # with open(image_metadata_path, "r") as f: - # metadata = json.load(f) - # transformer_image = metadata.get("transformer_image", transformer_image) - # - # generated_file = GeneratedFileResult(_hash, query_file_path) - # generated_file.image = transformer_image - # - # return generated_file diff --git a/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py b/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py index 92722f3aa..d42db1585 100644 --- a/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py +++ b/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py @@ -3,7 +3,6 @@ from pathlib import Path import subprocess import shutil -import generated_transformer instance = os.environ.get("INSTANCE_NAME", "Unknown") diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 5d977fc98..6f1bea80b 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -26,7 +26,6 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import uuid -import re import json from datetime import datetime, timezone from typing import Optional, List @@ -40,6 +39,7 @@ from servicex_app.resources.servicex_resource import ServiceXResource from werkzeug.exceptions import BadRequest + class SubmitTransformationRequest(ServiceXResource): @classmethod def make_api( @@ -100,7 +100,6 @@ def _initialize_dataset_manager( request_id: str, config: dict, ) -> DatasetManager: - # did xor file_list if bool(did) == bool(file_list): raise BadRequest("Must provide did or file-list but not both") @@ -152,7 +151,6 @@ def post(self): uuid.uuid4() ) # make sure we have a request id for all messages try: - try: args = self.parser.parse_args() except BadRequest as bad_request: @@ -219,7 +217,6 @@ def post(self): files=0, ) - selection = json.loads(args["selection"]) custom_docker_image = None if "docker_image" in selection: From f1b7b43dffda323a3c0e8b95f888568b0d9b2d39 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 21 Oct 2025 16:34:53 -0500 Subject: [PATCH 11/33] remove comments and unused env vars --- helm/servicex/templates/app/configmap.yaml | 3 --- servicex_app/servicex_app/code_gen_adapter.py | 2 -- .../servicex_app/resources/transformation/submit.py | 9 --------- 3 files changed, 14 deletions(-) diff --git a/helm/servicex/templates/app/configmap.yaml b/helm/servicex/templates/app/configmap.yaml index e7615fd91..e35ec9645 100644 --- a/helm/servicex/templates/app/configmap.yaml +++ b/helm/servicex/templates/app/configmap.yaml @@ -186,9 +186,6 @@ data: {{- if .Values.codeGen.topcp.enabled }} # TopCP custom image configuration TOPCP_ALLOWED_REPOSITORIES = {{ .Values.codeGen.topcp.allowedRepositories | toJson }} - TOPCP_IMAGE_TAG_PATTERN = '{{ .Values.codeGen.topcp.imageTagPattern }}' - TOPCP_DEFAULT_BASE_IMAGE = '{{ .Values.codeGen.topcp.defaultBaseImage }}' - TOPCP_DEFAULT_BASE_TAG = '{{ .Values.codeGen.topcp.defaultBaseTag }}' {{- end }} {{- $didFinders := list }} diff --git a/servicex_app/servicex_app/code_gen_adapter.py b/servicex_app/servicex_app/code_gen_adapter.py index 7917a0eb7..b6ede3034 100644 --- a/servicex_app/servicex_app/code_gen_adapter.py +++ b/servicex_app/servicex_app/code_gen_adapter.py @@ -74,8 +74,6 @@ def generate_code_for_selection( if not post_url: raise ValueError(f"{user_codegen_name}, code generator unavailable for use") - print("generate_code_for_selection") - print(request_record.selection) result = self.post_request( post_url + "/servicex/generated-code", post_obj={ diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 6f1bea80b..82eaaeb9f 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -165,9 +165,6 @@ def post(self): file_list = args.get("file-list") user_codegen_name = args.get("codegen") - print("CODEGEN!!!") - print(config["CODE_GEN_IMAGES"]) - print(user_codegen_name) code_gen_image_name = config["CODE_GEN_IMAGES"].get(user_codegen_name, None) namespace = config["TRANSFORMER_NAMESPACE"] @@ -196,9 +193,6 @@ def post(self): # TODO: need to check to make sure bucket was created # WHat happens if object-store and object_store is None? - print("SELECTION!!!") - print(args["selection"]) - request_rec = TransformRequest( request_id=str(request_id), title=args.get("title"), @@ -234,9 +228,6 @@ def post(self): request_rec, namespace, user_codegen_name ) - print("TEST!!") - print(codegen_transformer_image) - if custom_docker_image: request_rec.image = custom_docker_image else: From 8a4fbb7fed83cd63b507fac72319a80710d37f04 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Oct 2025 21:37:35 +0000 Subject: [PATCH 12/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- code_generator_TopCPToolkit/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code_generator_TopCPToolkit/Dockerfile b/code_generator_TopCPToolkit/Dockerfile index 79a3a34ec..6da3bd25b 100644 --- a/code_generator_TopCPToolkit/Dockerfile +++ b/code_generator_TopCPToolkit/Dockerfile @@ -33,4 +33,4 @@ USER servicex ENV CODEGEN_CONFIG_FILE="/home/servicex/app.conf" EXPOSE 5000 -ENTRYPOINT ["/home/servicex/boot.sh"] \ No newline at end of file +ENTRYPOINT ["/home/servicex/boot.sh"] From 0fd050d9277f7991a3f45013bf379285b7e0c8e1 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 23 Oct 2025 14:12:34 -0500 Subject: [PATCH 13/33] rewrite custom_docker_image logic --- .../servicex_app/resources/transformation/submit.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 82eaaeb9f..471638b91 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -211,11 +211,14 @@ def post(self): files=0, ) - selection = json.loads(args["selection"]) + print(f"selection: {args.get('selection')}") custom_docker_image = None - if "docker_image" in selection: - custom_docker_image = selection["docker_image"] - del selection["docker_image"] + try: + selection = json.loads(args["selection"]) + if "docker_image" in selection: + custom_docker_image = selection["docker_image"] + except json.decoder.JSONDecodeError: + pass # The first thing to do is make sure the requested selection is correct, # and can generate the requested code From d7bb6cd0a49180b7a5f976aea4e0a47982e2386c Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 28 Oct 2025 15:30:27 -0500 Subject: [PATCH 14/33] add back runTop_el --- .../TopCP_code_generator/query_translate.py | 4 - .../templates/transform_single_file.py | 2 + .../templates/codegen/deployment.yaml | 4 + servicex_app/pyproject.toml | 2 +- servicex_app/servicex_app/code_gen_adapter.py | 2 +- .../servicex_app/docker_repo_adapter.py | 82 ------------------- 6 files changed, 8 insertions(+), 88 deletions(-) diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py index 76e87e96d..4923b0d6a 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py @@ -39,7 +39,6 @@ "docker_image": { "properType": str, "properTypeString": "string", - "default": "sslhep/servicex_science_image_topcp:2.17.0-25.2.45", "optional": True, }, } @@ -70,9 +69,6 @@ def generate_files_from_query(query, query_file_path): ) for key in jquery: - if key == "docker_image": - continue - # ensure only available options are allowed if key not in options: raise KeyError( diff --git a/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py b/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py index d42db1585..7c2c047b1 100644 --- a/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py +++ b/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py @@ -3,6 +3,7 @@ from pathlib import Path import subprocess import shutil +import generated_transformer instance = os.environ.get("INSTANCE_NAME", "Unknown") @@ -32,6 +33,7 @@ def transform_single_file(file_path: str, output_path: Path, output_format: str) os.path.join(config_loc, "particle.yaml"), ) + generated_transformer.runTop_el() subprocess.run(["mv", "output.root", output_path]) diff --git a/helm/servicex/templates/codegen/deployment.yaml b/helm/servicex/templates/codegen/deployment.yaml index 8f029b61f..e4b02ffe9 100644 --- a/helm/servicex/templates/codegen/deployment.yaml +++ b/helm/servicex/templates/codegen/deployment.yaml @@ -35,14 +35,18 @@ spec: imagePullPolicy: {{ .pullPolicy }} ports: - containerPort: 5000 + {{- if eq $codeGenName "topcp" }} volumeMounts: - name: host-topcp-volume mountPath: /home/servicex + {{- end }} + {{- if eq $codeGenName "topcp" }} volumes: - name: host-topcp-volume hostPath: path: /mnt/topcp type: DirectoryOrCreate + {{- end }} {{- end }} {{- end }} diff --git a/servicex_app/pyproject.toml b/servicex_app/pyproject.toml index 1b9b01fa1..5964b38b2 100644 --- a/servicex_app/pyproject.toml +++ b/servicex_app/pyproject.toml @@ -34,7 +34,7 @@ blinker = "^1.5" pre-commit = "^2.20.0" minio = "^7.1.12" flask-migrate = "^3.1.0" -psycopg2-binary = "^2.9.5" +psycopg2 = "^2.9.5" python-logstash = "^0.4.8" humanize = "^4.4.0" gunicorn = "^23.0.0" diff --git a/servicex_app/servicex_app/code_gen_adapter.py b/servicex_app/servicex_app/code_gen_adapter.py index b6ede3034..c088f3c06 100644 --- a/servicex_app/servicex_app/code_gen_adapter.py +++ b/servicex_app/servicex_app/code_gen_adapter.py @@ -59,7 +59,7 @@ def generate_code_for_selection( :param namespace: Namespace in which to place resulting ConfigMap. :param user_codegen_name: Name provided by user for selecting the codegen URL from config dictionary - :returns a tuple of (config map name, default transformer image, language, command) + :returns a tuple of (config map name, default transformer image) """ from io import BytesIO from zipfile import ZipFile diff --git a/servicex_app/servicex_app/docker_repo_adapter.py b/servicex_app/servicex_app/docker_repo_adapter.py index 46e8773cd..01976ed40 100644 --- a/servicex_app/servicex_app/docker_repo_adapter.py +++ b/servicex_app/servicex_app/docker_repo_adapter.py @@ -64,85 +64,3 @@ def check_image_exists(self, tagged_image: str) -> bool: f"last updated {r.json()['last_updated']}" ) return True - - @servicex_retry() - def get_image_manifest(self, repo: str, image: str, tag: str) -> requests.Response: - """Get Docker image manifest from registry API v2.""" - # Try Docker Hub v2 API first for manifest - query = f"https://registry-1.docker.io/v2/{repo}/{image}/manifests/{tag}" - headers = {"Accept": "application/vnd.docker.distribution.manifest.v2+json"} - try: - r = requests.get(query, headers=headers, timeout=REQUEST_TIMEOUT) - if r.status_code == 200: - return r - except Exception: - pass - - # Fall back to Docker Hub v1 API for basic info - query = f"{self.registry_endpoint}/v2/repositories/{repo}/{image}/tags/{tag}" - r = requests.get(query, timeout=REQUEST_TIMEOUT) - return r - - def get_image_info(self, tagged_image: str) -> dict: - """ - Get detailed information about a Docker image including layers and configuration. - - :param tagged_image: Full Docker image name, e.g. "sslhep/servicex_app:latest" - :return: Dictionary containing image metadata, layers, config, etc. - """ - search_result = re.search("(.+)/(.+):(.+)", tagged_image) - if not search_result or len(search_result.groups()) != 3: - current_app.logger.warning(f"Invalid image format: {tagged_image}") - return None - - (repo, image, tag) = search_result.groups() - - try: - # Get manifest/detailed info - r = self.get_image_manifest(repo, image, tag) - - if r.status_code != 200: - current_app.logger.warning( - f"Failed to get image info for {tagged_image}: {r.status_code}" - ) - return None - - manifest_data = r.json() - - # Extract relevant information depending on API version - image_info = { - "digest": manifest_data.get("digest"), - "layers": [], - "config": {}, - "history": [], - } - - # Handle Docker Registry v2 manifest format - if "layers" in manifest_data: - image_info["layers"] = [ - layer.get("digest", "") for layer in manifest_data["layers"] - ] - - # Handle config blob reference - if "config" in manifest_data: - config_digest = manifest_data["config"].get("digest") - if config_digest: - # For full implementation, we'd fetch the config blob here - # For now, store the reference - image_info["config"] = {"digest": config_digest} - - # Handle Docker Hub v1 API response format - if "last_updated" in manifest_data: - image_info["last_updated"] = manifest_data["last_updated"] - - # Add some basic metadata - image_info["tag_info"] = manifest_data - - current_app.logger.debug(f"Retrieved image info for {tagged_image}") - return image_info - - except Exception as e: - current_app.logger.warning( - f"Error retrieving image info for {tagged_image}: {str(e)}" - ) - return None From e5024d01c56addc15cd6e8ac460c6b9cea4854a1 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 28 Oct 2025 15:33:02 -0500 Subject: [PATCH 15/33] remove old test files --- .../tests/test_custom_image_tag.py | 92 ---------- .../test_custom_image_validation.py | 167 ------------------ 2 files changed, 259 deletions(-) delete mode 100644 code_generator_TopCPToolkit/tests/test_custom_image_tag.py delete mode 100644 servicex_app/servicex_app_test/test_custom_image_validation.py diff --git a/code_generator_TopCPToolkit/tests/test_custom_image_tag.py b/code_generator_TopCPToolkit/tests/test_custom_image_tag.py deleted file mode 100644 index 19ffd5344..000000000 --- a/code_generator_TopCPToolkit/tests/test_custom_image_tag.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Test custom image tag functionality for TopCP code generator.""" - -import json -import os -import tempfile -import unittest - -from servicex.TopCP_code_generator.query_translate import generate_files_from_query - - -class TestCustomImageTag(unittest.TestCase): - def setUp(self): - self.temp_dir = tempfile.mkdtemp() - - def tearDown(self): - import shutil - - shutil.rmtree(self.temp_dir) - - def test_custom_image_tag_in_query_validation(self): - """Test that custom image tag is accepted in query validation.""" - query = { - "reco": "reco_config_content", - "parton": None, - "particle": None, - "max_events": -1, - "no_systematics": False, - "no_filter": False, - "image_tag": "v2.20.0_v0.2", - } - - # This should not raise an exception - generate_files_from_query(json.dumps(query), self.temp_dir) - - def test_image_tag_not_required(self): - """Test that image_tag is optional and doesn't break existing queries.""" - query = { - "reco": "reco_config_content", - "parton": None, - "particle": None, - "max_events": -1, - "no_systematics": False, - "no_filter": False, - } - - # This should not raise an exception - generate_files_from_query(json.dumps(query), self.temp_dir) - - def test_image_tag_none_allowed(self): - """Test that None image tag is accepted.""" - query = { - "reco": "reco_config_content", - "parton": None, - "particle": None, - "max_events": -1, - "no_systematics": False, - "no_filter": False, - "image_tag": None, - } - - # This should not raise an exception - generate_files_from_query(json.dumps(query), self.temp_dir) - - def test_generated_files_created_with_image_tag(self): - """Test that all expected files are generated when image_tag is present.""" - query = { - "reco": "reco_config_content", - "parton": None, - "particle": None, - "max_events": -1, - "no_systematics": False, - "no_filter": False, - "image_tag": "v2.20.0_v0.2", - } - - generate_files_from_query(json.dumps(query), self.temp_dir) - - # Check that reco.yaml is created - reco_file = os.path.join(self.temp_dir, "reco.yaml") - self.assertTrue(os.path.exists(reco_file)) - - with open(reco_file, "r") as f: - content = f.read() - self.assertEqual(content, "reco_config_content") - - # Check that generated_transformer.py is created - transformer_file = os.path.join(self.temp_dir, "generated_transformer.py") - self.assertTrue(os.path.exists(transformer_file)) - - -if __name__ == "__main__": - unittest.main() diff --git a/servicex_app/servicex_app_test/test_custom_image_validation.py b/servicex_app/servicex_app_test/test_custom_image_validation.py deleted file mode 100644 index 9bfc9f4ec..000000000 --- a/servicex_app/servicex_app_test/test_custom_image_validation.py +++ /dev/null @@ -1,167 +0,0 @@ -"""Test custom image tag validation for TopCP transformations.""" - -import json -import unittest -from werkzeug.exceptions import BadRequest - -from servicex_app.resources.transformation.submit import ( - validate_custom_image_tag, - SubmitTransformationRequest, -) - - -class TestCustomImageValidation(unittest.TestCase): - - def setUp(self): - self.config = { - "TOPCP_ALLOWED_REPOSITORIES": [ - "sslhep/servicex_science_image_topcp", - "registry.gitlab.com/topcp-project/toolkit", - ], - "TOPCP_IMAGE_TAG_PATTERN": r"^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$", - "TOPCP_DEFAULT_BASE_IMAGE": "sslhep/servicex_science_image_topcp", - } - - def test_valid_image_tag_format(self): - """Test that valid image tag formats are accepted.""" - test_cases = ["v2.20.0_v0.2", "2.20.0_v0.2", "v1.15.0-v1.0", "3.0.0_v2.5"] - - for tag in test_cases: - with self.subTest(tag=tag): - image, result_tag = validate_custom_image_tag(tag, self.config, "topcp") - self.assertEqual(image, "sslhep/servicex_science_image_topcp") - self.assertEqual(result_tag, tag) - - def test_invalid_image_tag_format(self): - """Test that invalid image tag formats are rejected.""" - test_cases = [ - "latest", - "2.20.0", - "v2.20.0", - "invalid-tag", - "v2.20.0_v0.2.1", # Too many version components - "2.20_v0.2", # Missing patch version - ] - - for tag in test_cases: - with self.subTest(tag=tag): - with self.assertRaises(BadRequest): - validate_custom_image_tag(tag, self.config, "topcp") - - def test_non_topcp_codegen_returns_none(self): - """Test that non-TopCP code generators return None.""" - image, tag = validate_custom_image_tag("v2.20.0_v0.2", self.config, "uproot") - self.assertIsNone(image) - self.assertIsNone(tag) - - def test_empty_image_tag_returns_none(self): - """Test that empty image tag returns None.""" - test_cases = [None, "", " "] - - for tag in test_cases: - with self.subTest(tag=tag): - image, result_tag = validate_custom_image_tag(tag, self.config, "topcp") - self.assertIsNone(image) - self.assertIsNone(result_tag) - - def test_missing_config_uses_defaults(self): - """Test that missing configuration uses default values.""" - minimal_config = {} - - image, tag = validate_custom_image_tag("v2.20.0_v0.2", minimal_config, "topcp") - self.assertEqual(image, "sslhep/servicex_science_image_topcp") - self.assertEqual(tag, "v2.20.0_v0.2") - - def test_repository_validation_success(self): - """Test that repository validation passes when default image is in allowed repos.""" - config = { - "TOPCP_ALLOWED_REPOSITORIES": [ - "sslhep/servicex_science_image_topcp", - "registry.gitlab.com/topcp-project/toolkit", - ], - "TOPCP_DEFAULT_BASE_IMAGE": "sslhep/servicex_science_image_topcp", - } - - image, tag = validate_custom_image_tag("v2.20.0_v0.2", config, "topcp") - self.assertEqual(image, "sslhep/servicex_science_image_topcp") - self.assertEqual(tag, "v2.20.0_v0.2") - - def test_repository_validation_failure(self): - """Test that repository validation fails when default image is not in allowed repos.""" - config = { - "TOPCP_ALLOWED_REPOSITORIES": [ - "registry.gitlab.com/topcp-project/toolkit", - ], - "TOPCP_DEFAULT_BASE_IMAGE": "sslhep/servicex_science_image_topcp", - } - - with self.assertRaises(BadRequest): - validate_custom_image_tag("v2.20.0_v0.2", config, "topcp") - - -class TestImageTagExtraction(unittest.TestCase): - - def setUp(self): - self.submit_request = SubmitTransformationRequest() - - def test_extract_image_tag_from_topcp_query(self): - """Test extraction of image tag from TopCP query.""" - query = { - "reco": "reco_config", - "parton": None, - "particle": None, - "max_events": -1, - "no_systematics": False, - "no_filter": False, - "image_tag": "v2.20.0_v0.2", - } - - result = self.submit_request._extract_custom_image_tag( - json.dumps(query), "topcp" - ) - self.assertEqual(result, "v2.20.0_v0.2") - - def test_extract_image_tag_missing_returns_none(self): - """Test that missing image tag returns None.""" - query = { - "reco": "reco_config", - "parton": None, - "particle": None, - "max_events": -1, - "no_systematics": False, - "no_filter": False, - } - - result = self.submit_request._extract_custom_image_tag( - json.dumps(query), "topcp" - ) - self.assertIsNone(result) - - def test_extract_image_tag_non_topcp_returns_none(self): - """Test that non-TopCP codegen returns None.""" - query = {"selection": "some_selection", "image_tag": "v2.20.0_v0.2"} - - result = self.submit_request._extract_custom_image_tag( - json.dumps(query), "uproot" - ) - self.assertIsNone(result) - - def test_extract_image_tag_invalid_json_returns_none(self): - """Test that invalid JSON returns None.""" - invalid_json = "{ invalid json }" - - result = self.submit_request._extract_custom_image_tag(invalid_json, "topcp") - self.assertIsNone(result) - - def test_extract_image_tag_none_value(self): - """Test that None image tag value is returned as None.""" - query = {"reco": "reco_config", "image_tag": None} - - result = self.submit_request._extract_custom_image_tag( - json.dumps(query), "topcp" - ) - self.assertIsNone(result) - - -if __name__ == "__main__": - unittest.main() From 0125f414d6ec7586e94f9ff41105fb4325a54bc5 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Wed, 29 Oct 2025 13:50:07 -0500 Subject: [PATCH 16/33] fix local volume mount --- helm/servicex/templates/codegen/deployment.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/helm/servicex/templates/codegen/deployment.yaml b/helm/servicex/templates/codegen/deployment.yaml index e4b02ffe9..b14e66ed3 100644 --- a/helm/servicex/templates/codegen/deployment.yaml +++ b/helm/servicex/templates/codegen/deployment.yaml @@ -36,17 +36,25 @@ spec: ports: - containerPort: 5000 {{- if eq $codeGenName "topcp" }} + {{- if eq $.Values.app.environment "dev" }} + {{- if $.Values.app.reload }} volumeMounts: - name: host-topcp-volume mountPath: /home/servicex {{- end }} + {{- end }} + {{- end }} {{- if eq $codeGenName "topcp" }} + {{- if eq $.Values.app.environment "dev" }} + {{- if $.Values.app.reload }} volumes: - name: host-topcp-volume hostPath: path: /mnt/topcp type: DirectoryOrCreate {{- end }} + {{- end }} + {{- end }} {{- end }} {{- end }} From b49b9ee32f39847bbc825087f5357963a515b22f Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Wed, 29 Oct 2025 14:15:21 -0500 Subject: [PATCH 17/33] change reload to mountLocal --- helm/servicex/templates/codegen/deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/servicex/templates/codegen/deployment.yaml b/helm/servicex/templates/codegen/deployment.yaml index b14e66ed3..b5204764e 100644 --- a/helm/servicex/templates/codegen/deployment.yaml +++ b/helm/servicex/templates/codegen/deployment.yaml @@ -37,7 +37,7 @@ spec: - containerPort: 5000 {{- if eq $codeGenName "topcp" }} {{- if eq $.Values.app.environment "dev" }} - {{- if $.Values.app.reload }} + {{- if $.Values.app.mountLocal }} volumeMounts: - name: host-topcp-volume mountPath: /home/servicex @@ -47,7 +47,7 @@ spec: {{- if eq $codeGenName "topcp" }} {{- if eq $.Values.app.environment "dev" }} - {{- if $.Values.app.reload }} + {{- if $.Values.app.mountLocal }} volumes: - name: host-topcp-volume hostPath: From 668633645997fe1cee4cccf9f7e9b6c836fb503f Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 30 Oct 2025 20:50:35 -0500 Subject: [PATCH 18/33] update helm charts and other config stuff --- code_generator_TopCPToolkit/boot.sh | 14 ++++- .../request_translator.py | 2 - .../templates/transform_single_file.py | 4 +- helm/servicex/templates/app/configmap.yaml | 2 +- .../templates/codegen/deployment.yaml | 5 ++ helm/servicex/values.yaml | 4 +- .../resources/transformation/submit.py | 55 ++++++++++++++++--- 7 files changed, 68 insertions(+), 18 deletions(-) diff --git a/code_generator_TopCPToolkit/boot.sh b/code_generator_TopCPToolkit/boot.sh index 350504fd7..76cd19549 100755 --- a/code_generator_TopCPToolkit/boot.sh +++ b/code_generator_TopCPToolkit/boot.sh @@ -1,10 +1,22 @@ #!/bin/bash +# Initialize reload flag +RELOAD="" + +# Parse command line arguments +for arg in "$@" +do + if [ "$arg" = "--reload" ]; then + RELOAD="--reload" + break + fi +done + # Running the web server? action=${1:-web_service} if [ "$action" = "web_service" ] ; then mkdir instance - exec gunicorn -b :5000 --reload --workers=2 --threads=1 --access-logfile - --error-logfile - "servicex.TopCP_code_generator:create_app()" + exec gunicorn -b :5000 $RELOAD --workers=2 --threads=1 --access-logfile - --error-logfile - "servicex.TopCP_code_generator:create_app()" else echo "Unknown action '$action'" fi diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py index 8e8a3839a..3a7ef9021 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py @@ -63,10 +63,8 @@ def generate_code(self, query, cache_path: str): "CAPABILITIES_PATH", "/home/servicex/transformer_capabilities.json" ) - # Generate query files first to create any metadata query_translate.generate_files_from_query(query, query_file_path) - # Copy capabilities file shutil.copyfile( capabilities_path, os.path.join(query_file_path, "transformer_capabilities.json"), diff --git a/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py b/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py index 7c2c047b1..7efe01a6d 100644 --- a/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py +++ b/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py @@ -13,10 +13,8 @@ def transform_single_file(file_path: str, output_path: Path, output_format: str) with open("input.txt", "w") as f: f.write(file_path) - # Get CONFIG_LOC with a sensible default (current working directory) - config_loc = os.environ.get("CONFIG_LOC", os.getcwd()) - # move reco.yaml, parton.yaml and particle.yaml if they exist to CONFIG_LOC location + config_loc = os.environ.get("CONFIG_LOC", os.getcwd()) if os.path.exists("/generated/reco.yaml"): shutil.copyfile( "/generated/reco.yaml", diff --git a/helm/servicex/templates/app/configmap.yaml b/helm/servicex/templates/app/configmap.yaml index e35ec9645..b95b0c681 100644 --- a/helm/servicex/templates/app/configmap.yaml +++ b/helm/servicex/templates/app/configmap.yaml @@ -185,7 +185,7 @@ data: {{- if .Values.codeGen.topcp.enabled }} # TopCP custom image configuration - TOPCP_ALLOWED_REPOSITORIES = {{ .Values.codeGen.topcp.allowedRepositories | toJson }} + TOPCP_ALLOWED_IMAGES = {{ .Values.codeGen.topcp.allowedImages | toJson }} {{- end }} {{- $didFinders := list }} diff --git a/helm/servicex/templates/codegen/deployment.yaml b/helm/servicex/templates/codegen/deployment.yaml index b5204764e..6330ec560 100644 --- a/helm/servicex/templates/codegen/deployment.yaml +++ b/helm/servicex/templates/codegen/deployment.yaml @@ -19,6 +19,11 @@ spec: containers: - name: {{ $.Release.Name }}-code-gen-{{ $codeGenName }} image: {{ .image }}:{{ .tag }} + {{- if eq $.Values.app.environment "dev" }} + {{- if $.Values.app.reload }} + command: [ "./boot.sh", "web_service", "--reload" ] + {{- end }} + {{- end }} env: - name: INSTANCE_NAME value: {{ $.Release.Name }} diff --git a/helm/servicex/values.yaml b/helm/servicex/values.yaml index cda785e1c..ff460f2b0 100644 --- a/helm/servicex/values.yaml +++ b/helm/servicex/values.yaml @@ -108,8 +108,8 @@ codeGen: tag: develop defaultScienceContainerImage: sslhep/servicex_science_image_topcp defaultScienceContainerTag: 2.17.0-25.2.45 - allowedRepositories: - - "sslhep" + allowedImages: + - "sslhep/servicex_science_image_topcp:" defaultBaseImage: "sslhep/servicex_science_image_topcp" didFinder: diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 471638b91..b4404600e 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -27,6 +27,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import uuid import json +import os from datetime import datetime, timezone from typing import Optional, List @@ -40,6 +41,36 @@ from werkzeug.exceptions import BadRequest +def validate_custom_docker_image(image_name: str) -> bool: + allowed_images_json = os.environ.get("TOPCP_ALLOWED_IMAGES") + + if not allowed_images_json: + raise BadRequest( + "Custom Docker images are not allowed." + ) + + try: + allowed_prefixes = json.loads(allowed_images_json) + + if not isinstance(allowed_prefixes, list): + raise BadRequest( + "TopCP allowed images are improperly configured." + ) + + for prefix in allowed_prefixes: + if image_name.startswith(prefix): + return True + + raise BadRequest( + f"Custom Docker image '{image_name}' not allowed." + ) + + except json.JSONDecodeError as e: + raise BadRequest( + "TopCP allowed images are improperly configured." + ) + + class SubmitTransformationRequest(ServiceXResource): @classmethod def make_api( @@ -211,15 +242,6 @@ def post(self): files=0, ) - print(f"selection: {args.get('selection')}") - custom_docker_image = None - try: - selection = json.loads(args["selection"]) - if "docker_image" in selection: - custom_docker_image = selection["docker_image"] - except json.decoder.JSONDecodeError: - pass - # The first thing to do is make sure the requested selection is correct, # and can generate the requested code ( @@ -231,6 +253,21 @@ def post(self): request_rec, namespace, user_codegen_name ) + custom_docker_image = None + if user_codegen_name == "topcp": + try: + selection = json.loads(args["selection"]) + if "docker_image" in selection: + custom_docker_image = selection["docker_image"] + try: + validate_custom_docker_image(custom_docker_image) + except BadRequest as e: + current_app.logger.error( + str(e), extra={"requestId": request_id} + ) + except json.decoder.JSONDecodeError: + pass + if custom_docker_image: request_rec.image = custom_docker_image else: From 0f4f3f44140f3665f771c92f4f60042582de86cf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 31 Oct 2025 01:50:51 +0000 Subject: [PATCH 19/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../resources/transformation/submit.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index b4404600e..38411b165 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -45,30 +45,22 @@ def validate_custom_docker_image(image_name: str) -> bool: allowed_images_json = os.environ.get("TOPCP_ALLOWED_IMAGES") if not allowed_images_json: - raise BadRequest( - "Custom Docker images are not allowed." - ) + raise BadRequest("Custom Docker images are not allowed.") try: allowed_prefixes = json.loads(allowed_images_json) if not isinstance(allowed_prefixes, list): - raise BadRequest( - "TopCP allowed images are improperly configured." - ) + raise BadRequest("TopCP allowed images are improperly configured.") for prefix in allowed_prefixes: if image_name.startswith(prefix): return True - raise BadRequest( - f"Custom Docker image '{image_name}' not allowed." - ) + raise BadRequest(f"Custom Docker image '{image_name}' not allowed.") except json.JSONDecodeError as e: - raise BadRequest( - "TopCP allowed images are improperly configured." - ) + raise BadRequest("TopCP allowed images are improperly configured.") class SubmitTransformationRequest(ServiceXResource): From 8aeff9eb38ed20db850efe3f90fc5443630918b4 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 30 Oct 2025 20:57:03 -0500 Subject: [PATCH 20/33] resolve flake8 --- servicex_app/servicex_app/resources/transformation/submit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index b4404600e..2b01e9500 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -65,7 +65,7 @@ def validate_custom_docker_image(image_name: str) -> bool: f"Custom Docker image '{image_name}' not allowed." ) - except json.JSONDecodeError as e: + except json.JSONDecodeError: raise BadRequest( "TopCP allowed images are improperly configured." ) From 981504b9964e8acc5c1ea7de4d3fe26108fdc6c2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 31 Oct 2025 01:58:40 +0000 Subject: [PATCH 21/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- servicex_app/servicex_app/resources/transformation/submit.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 6e6512f3d..ef1a50485 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -60,9 +60,7 @@ def validate_custom_docker_image(image_name: str) -> bool: raise BadRequest(f"Custom Docker image '{image_name}' not allowed.") except json.JSONDecodeError: - raise BadRequest( - "TopCP allowed images are improperly configured." - ) + raise BadRequest("TopCP allowed images are improperly configured.") class SubmitTransformationRequest(ServiceXResource): From 28523ed9bf1db86217f0223f4292bd6db63ac172 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 30 Oct 2025 21:16:03 -0500 Subject: [PATCH 22/33] move boot.sh --- code_generator_TopCPToolkit/Dockerfile | 4 ++-- code_generator_TopCPToolkit/{ => servicex}/boot.sh | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename code_generator_TopCPToolkit/{ => servicex}/boot.sh (100%) diff --git a/code_generator_TopCPToolkit/Dockerfile b/code_generator_TopCPToolkit/Dockerfile index 6da3bd25b..98136d379 100644 --- a/code_generator_TopCPToolkit/Dockerfile +++ b/code_generator_TopCPToolkit/Dockerfile @@ -26,11 +26,11 @@ RUN chmod 777 -R servicex COPY app.conf . RUN chmod 755 app.conf -RUN chmod 755 /home/servicex/boot.sh +RUN chmod 755 /home/servicex/servicex/boot.sh USER servicex ENV CODEGEN_CONFIG_FILE="/home/servicex/app.conf" EXPOSE 5000 -ENTRYPOINT ["/home/servicex/boot.sh"] +ENTRYPOINT ["/home/servicex/servicex/boot.sh"] diff --git a/code_generator_TopCPToolkit/boot.sh b/code_generator_TopCPToolkit/servicex/boot.sh similarity index 100% rename from code_generator_TopCPToolkit/boot.sh rename to code_generator_TopCPToolkit/servicex/boot.sh From ce26f8f31b521b580758bd17043eb00d62cf02b2 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 30 Oct 2025 21:33:05 -0500 Subject: [PATCH 23/33] update boot.sh command --- helm/servicex/templates/codegen/deployment.yaml | 4 +++- helm/servicex/values.yaml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/helm/servicex/templates/codegen/deployment.yaml b/helm/servicex/templates/codegen/deployment.yaml index 6330ec560..b6b4d97e1 100644 --- a/helm/servicex/templates/codegen/deployment.yaml +++ b/helm/servicex/templates/codegen/deployment.yaml @@ -20,8 +20,10 @@ spec: - name: {{ $.Release.Name }}-code-gen-{{ $codeGenName }} image: {{ .image }}:{{ .tag }} {{- if eq $.Values.app.environment "dev" }} + {{- if eq $codeGenName "topcp" }} {{- if $.Values.app.reload }} - command: [ "./boot.sh", "web_service", "--reload" ] + command: [ "./servicex/boot.sh", "web_service", "--reload" ] + {{- end }} {{- end }} {{- end }} env: diff --git a/helm/servicex/values.yaml b/helm/servicex/values.yaml index ff460f2b0..4333cdf3e 100644 --- a/helm/servicex/values.yaml +++ b/helm/servicex/values.yaml @@ -104,7 +104,7 @@ codeGen: topcp: enabled: true image: sslhep/servicex_code_gen_topcp - pullPolicy: Always + pullPolicy: IfNotPresent tag: develop defaultScienceContainerImage: sslhep/servicex_science_image_topcp defaultScienceContainerTag: 2.17.0-25.2.45 From 6c2059f86d1b63190b34ae98a453e9a7cff18eb9 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Fri, 31 Oct 2025 16:19:01 -0500 Subject: [PATCH 24/33] update test coverage --- .../resources/transformation/submit.py | 13 +- .../resources/transformation/test_submit.py | 203 +++++++++++++++++- 2 files changed, 207 insertions(+), 9 deletions(-) diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index ef1a50485..7c0f3ab35 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -38,7 +38,7 @@ from servicex_app.did_parser import DIDParser from servicex_app.models import TransformRequest, db, TransformStatus from servicex_app.resources.servicex_resource import ServiceXResource -from werkzeug.exceptions import BadRequest +from werkzeug.exceptions import BadRequest, HTTPException def validate_custom_docker_image(image_name: str) -> bool: @@ -251,14 +251,9 @@ def post(self): selection = json.loads(args["selection"]) if "docker_image" in selection: custom_docker_image = selection["docker_image"] - try: - validate_custom_docker_image(custom_docker_image) - except BadRequest as e: - current_app.logger.error( - str(e), extra={"requestId": request_id} - ) + validate_custom_docker_image(custom_docker_image) except json.decoder.JSONDecodeError: - pass + raise BadRequest("Malformed JSON submitted") if custom_docker_image: request_rec.image = custom_docker_image @@ -310,6 +305,8 @@ def post(self): "Transformation request submitted!", extra={"requestId": request_id} ) return {"request_id": str(request_id)} + except HTTPException: + raise except Exception as eek: current_app.logger.exception( "Got exception while submitting transformation request", diff --git a/servicex_app/servicex_app_test/resources/transformation/test_submit.py b/servicex_app/servicex_app_test/resources/transformation/test_submit.py index 4eba86c1b..e13095ffe 100644 --- a/servicex_app/servicex_app_test/resources/transformation/test_submit.py +++ b/servicex_app/servicex_app_test/resources/transformation/test_submit.py @@ -25,17 +25,22 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import json +import os from datetime import datetime, timezone -from unittest.mock import ANY +from unittest.mock import ANY, patch +import pytest from celery import Celery from pytest import fixture +from werkzeug.exceptions import BadRequest from servicex_app import LookupResultProcessor from servicex_app.code_gen_adapter import CodeGenAdapter from servicex_app.dataset_manager import DatasetManager from servicex_app.models import Dataset from servicex_app.models import TransformRequest, DatasetStatus, TransformStatus +from servicex_app.resources.transformation.submit import validate_custom_docker_image from servicex_app.transformer_manager import TransformerManager from servicex_app_test.resource_test_base import ResourceTestBase @@ -588,3 +593,199 @@ def test_submit_transformation_with_title( saved_obj = TransformRequest.lookup(request_id) assert saved_obj assert saved_obj.title == title + + +class TestValidateCustomDockerImage: + """Tests for the validate_custom_docker_image function""" + + def test_validate_with_matching_prefix(self): + """Test validation succeeds when image matches an allowed prefix""" + with patch.dict( + os.environ, + {"TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]'}, + ): + result = validate_custom_docker_image( + "sslhep/servicex_science_image_topcp:2.17.0" + ) + assert result is True + + def test_validate_with_multiple_prefixes(self): + """Test validation with multiple allowed prefixes""" + with patch.dict( + os.environ, + { + "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]' + }, + ): + assert ( + validate_custom_docker_image( + "sslhep/servicex_science_image_topcp:latest" + ) + is True + ) + assert ( + validate_custom_docker_image("docker.io/ssl-hep/custom:v1") is True + ) + + def test_validate_with_no_matching_prefix(self): + """Test validation fails when image doesn't match any allowed prefix""" + with patch.dict( + os.environ, + {"TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]'}, + ): + with pytest.raises(BadRequest, match="not allowed"): + validate_custom_docker_image("unauthorized/image:latest") + + def test_validate_with_no_env_variable(self): + """Test validation fails when TOPCP_ALLOWED_IMAGES is not set""" + with patch.dict(os.environ, {}, clear=True): + with pytest.raises(BadRequest, match="Custom Docker images are not allowed"): + validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") + + def test_validate_with_invalid_json(self): + """Test validation fails with invalid JSON in env variable""" + with patch.dict(os.environ, {"TOPCP_ALLOWED_IMAGES": "not-valid-json"}): + with pytest.raises(BadRequest, match="improperly configured"): + validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") + + def test_validate_with_non_list_json(self): + """Test validation fails when JSON is not a list""" + with patch.dict(os.environ, {"TOPCP_ALLOWED_IMAGES": '{"key": "value"}'}): + with pytest.raises(BadRequest, match="improperly configured"): + validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") + + def test_validate_with_empty_list(self): + """Test validation fails when allowed list is empty""" + with patch.dict(os.environ, {"TOPCP_ALLOWED_IMAGES": "[]"}): + with pytest.raises(BadRequest, match="not allowed"): + validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") + + +class TestSubmitTransformationRequestCustomImage(ResourceTestBase): + """Tests for custom Docker image feature in transformation requests""" + + @staticmethod + def _generate_transformation_request(**kwargs): + request = { + "did": "123-45-678", + "selection": "test-string", + "result-destination": "object-store", + "result-format": "root-file", + "workers": 10, + "codegen": "topcp", + } + request.update(kwargs) + return request + + @fixture + def mock_dataset_manager_from_did(self, mocker): + dm = mocker.Mock() + dm.dataset = Dataset( + name="rucio://123-45-678", + did_finder="rucio", + lookup_status=DatasetStatus.looking, + last_used=datetime.now(tz=timezone.utc), + last_updated=datetime.fromtimestamp(0), + ) + dm.name = "rucio://123-45-678" + dm.id = 42 + mock_from_did = mocker.patch.object(DatasetManager, "from_did", return_value=dm) + return mock_from_did + + @fixture + def mock_codegen(self, mocker): + mock_code_gen = mocker.MagicMock(CodeGenAdapter) + mock_code_gen.generate_code_for_selection.return_value = ( + "my-cm", + "sslhep/servicex_science_image_topcp:2.17.0", + "bash", + "echo", + ) + return mock_code_gen + + def test_submit_topcp_with_custom_docker_image( + self, mock_dataset_manager_from_did, mock_codegen, mock_app_version + ): + """Test submitting a TopCP transformation with a valid custom docker image""" + extra_config = { + "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} + } + with patch.dict( + os.environ, + {"TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]'}, + ): + client = self._test_client(code_gen_service=mock_codegen, extra_config=extra_config) + with client.application.app_context(): + selection_dict = {"docker_image": "sslhep/servicex_science_image_topcp:custom"} + request = self._generate_transformation_request( + selection=json.dumps(selection_dict) + ) + + response = client.post( + "/servicex/transformation", json=request, headers=self.fake_header() + ) + assert response.status_code == 200 + request_id = response.json["request_id"] + + saved_obj = TransformRequest.lookup(request_id) + assert saved_obj + assert saved_obj.image == "sslhep/servicex_science_image_topcp:custom" + + def test_submit_topcp_with_invalid_custom_docker_image( + self, mock_dataset_manager_from_did, mock_codegen + ): + """Test submitting a TopCP transformation with an invalid custom docker image returns 400""" + extra_config = { + "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} + } + with patch.dict( + os.environ, + {"TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]'}, + ): + client = self._test_client(code_gen_service=mock_codegen, extra_config=extra_config) + with client.application.app_context(): + selection_dict = {"docker_image": "unauthorized/image:latest"} + request = self._generate_transformation_request( + selection=json.dumps(selection_dict) + ) + + response = client.post( + "/servicex/transformation", json=request, headers=self.fake_header() + ) + assert response.status_code == 400 + + def test_submit_topcp_with_non_json_selection( + self, mock_dataset_manager_from_did, mock_codegen, mock_app_version + ): + """Test submitting a TopCP transformation with non-JSON selection string""" + extra_config = { + "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} + } + client = self._test_client(code_gen_service=mock_codegen, extra_config=extra_config) + with client.application.app_context(): + request = self._generate_transformation_request(selection="not-json-string") + + response = client.post( + "/servicex/transformation", json=request, headers=self.fake_header() + ) + assert response.status_code == 400 + + def test_submit_custom_topcp_without_env_var( + self, mock_dataset_manager_from_did, mock_codegen + ): + """Test submitting TopCP with custom image when TOPCP_ALLOWED_IMAGES is not set""" + extra_config = { + "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} + } + with patch.dict(os.environ, {}, clear=True): + client = self._test_client(code_gen_service=mock_codegen, extra_config=extra_config) + with client.application.app_context(): + selection_dict = {"docker_image": "sslhep/servicex_science_image_topcp:custom"} + request = self._generate_transformation_request( + selection=json.dumps(selection_dict) + ) + + response = client.post( + "/servicex/transformation", json=request, headers=self.fake_header() + ) + assert response.status_code == 400 From 3dfd2ad8e5dca333025fb304dc678f74f4017e35 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 31 Oct 2025 21:19:24 +0000 Subject: [PATCH 25/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../resources/transformation/test_submit.py | 48 +++++++++++++------ 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/servicex_app/servicex_app_test/resources/transformation/test_submit.py b/servicex_app/servicex_app_test/resources/transformation/test_submit.py index e13095ffe..c07ff818b 100644 --- a/servicex_app/servicex_app_test/resources/transformation/test_submit.py +++ b/servicex_app/servicex_app_test/resources/transformation/test_submit.py @@ -623,9 +623,7 @@ def test_validate_with_multiple_prefixes(self): ) is True ) - assert ( - validate_custom_docker_image("docker.io/ssl-hep/custom:v1") is True - ) + assert validate_custom_docker_image("docker.io/ssl-hep/custom:v1") is True def test_validate_with_no_matching_prefix(self): """Test validation fails when image doesn't match any allowed prefix""" @@ -639,26 +637,36 @@ def test_validate_with_no_matching_prefix(self): def test_validate_with_no_env_variable(self): """Test validation fails when TOPCP_ALLOWED_IMAGES is not set""" with patch.dict(os.environ, {}, clear=True): - with pytest.raises(BadRequest, match="Custom Docker images are not allowed"): - validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") + with pytest.raises( + BadRequest, match="Custom Docker images are not allowed" + ): + validate_custom_docker_image( + "sslhep/servicex_science_image_topcp:2.17.0" + ) def test_validate_with_invalid_json(self): """Test validation fails with invalid JSON in env variable""" with patch.dict(os.environ, {"TOPCP_ALLOWED_IMAGES": "not-valid-json"}): with pytest.raises(BadRequest, match="improperly configured"): - validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") + validate_custom_docker_image( + "sslhep/servicex_science_image_topcp:2.17.0" + ) def test_validate_with_non_list_json(self): """Test validation fails when JSON is not a list""" with patch.dict(os.environ, {"TOPCP_ALLOWED_IMAGES": '{"key": "value"}'}): with pytest.raises(BadRequest, match="improperly configured"): - validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") + validate_custom_docker_image( + "sslhep/servicex_science_image_topcp:2.17.0" + ) def test_validate_with_empty_list(self): """Test validation fails when allowed list is empty""" with patch.dict(os.environ, {"TOPCP_ALLOWED_IMAGES": "[]"}): with pytest.raises(BadRequest, match="not allowed"): - validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") + validate_custom_docker_image( + "sslhep/servicex_science_image_topcp:2.17.0" + ) class TestSubmitTransformationRequestCustomImage(ResourceTestBase): @@ -714,9 +722,13 @@ def test_submit_topcp_with_custom_docker_image( os.environ, {"TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]'}, ): - client = self._test_client(code_gen_service=mock_codegen, extra_config=extra_config) + client = self._test_client( + code_gen_service=mock_codegen, extra_config=extra_config + ) with client.application.app_context(): - selection_dict = {"docker_image": "sslhep/servicex_science_image_topcp:custom"} + selection_dict = { + "docker_image": "sslhep/servicex_science_image_topcp:custom" + } request = self._generate_transformation_request( selection=json.dumps(selection_dict) ) @@ -742,7 +754,9 @@ def test_submit_topcp_with_invalid_custom_docker_image( os.environ, {"TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]'}, ): - client = self._test_client(code_gen_service=mock_codegen, extra_config=extra_config) + client = self._test_client( + code_gen_service=mock_codegen, extra_config=extra_config + ) with client.application.app_context(): selection_dict = {"docker_image": "unauthorized/image:latest"} request = self._generate_transformation_request( @@ -761,7 +775,9 @@ def test_submit_topcp_with_non_json_selection( extra_config = { "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} } - client = self._test_client(code_gen_service=mock_codegen, extra_config=extra_config) + client = self._test_client( + code_gen_service=mock_codegen, extra_config=extra_config + ) with client.application.app_context(): request = self._generate_transformation_request(selection="not-json-string") @@ -778,9 +794,13 @@ def test_submit_custom_topcp_without_env_var( "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} } with patch.dict(os.environ, {}, clear=True): - client = self._test_client(code_gen_service=mock_codegen, extra_config=extra_config) + client = self._test_client( + code_gen_service=mock_codegen, extra_config=extra_config + ) with client.application.app_context(): - selection_dict = {"docker_image": "sslhep/servicex_science_image_topcp:custom"} + selection_dict = { + "docker_image": "sslhep/servicex_science_image_topcp:custom" + } request = self._generate_transformation_request( selection=json.dumps(selection_dict) ) From 5aa97d6421723a2382c036b3c5d9b1b85932d7b5 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Fri, 31 Oct 2025 16:25:58 -0500 Subject: [PATCH 26/33] resolve flake8 --- .../resources/transformation/test_submit.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/servicex_app/servicex_app_test/resources/transformation/test_submit.py b/servicex_app/servicex_app_test/resources/transformation/test_submit.py index e13095ffe..f76d308f0 100644 --- a/servicex_app/servicex_app_test/resources/transformation/test_submit.py +++ b/servicex_app/servicex_app_test/resources/transformation/test_submit.py @@ -614,7 +614,8 @@ def test_validate_with_multiple_prefixes(self): with patch.dict( os.environ, { - "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]' + "TOPCP_ALLOWED_IMAGES": + '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]' }, ): assert ( @@ -734,7 +735,7 @@ def test_submit_topcp_with_custom_docker_image( def test_submit_topcp_with_invalid_custom_docker_image( self, mock_dataset_manager_from_did, mock_codegen ): - """Test submitting a TopCP transformation with an invalid custom docker image returns 400""" + """Submitting a TopCP transformation with an invalid custom docker image fails""" extra_config = { "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} } From 8cce09658c79fec35c5a4a004add086121337cba Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 31 Oct 2025 21:27:16 +0000 Subject: [PATCH 27/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../servicex_app_test/resources/transformation/test_submit.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/servicex_app/servicex_app_test/resources/transformation/test_submit.py b/servicex_app/servicex_app_test/resources/transformation/test_submit.py index f66866126..2559d8711 100644 --- a/servicex_app/servicex_app_test/resources/transformation/test_submit.py +++ b/servicex_app/servicex_app_test/resources/transformation/test_submit.py @@ -614,8 +614,7 @@ def test_validate_with_multiple_prefixes(self): with patch.dict( os.environ, { - "TOPCP_ALLOWED_IMAGES": - '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]' + "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]' }, ): assert ( From 01c453af046d5aaa50d2f5b4aba4f81066719749 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Fri, 31 Oct 2025 16:32:38 -0500 Subject: [PATCH 28/33] resolve flake8 again --- .../resources/transformation/test_submit.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/servicex_app/servicex_app_test/resources/transformation/test_submit.py b/servicex_app/servicex_app_test/resources/transformation/test_submit.py index 2559d8711..b480d7cc3 100644 --- a/servicex_app/servicex_app_test/resources/transformation/test_submit.py +++ b/servicex_app/servicex_app_test/resources/transformation/test_submit.py @@ -46,7 +46,6 @@ class TestSubmitTransformationRequest(ResourceTestBase): - @staticmethod def _generate_transformation_request(**kwargs): request = { @@ -437,7 +436,6 @@ def test_submit_transformation_request_no_docker_check( code_gen_service=mock_codegen, ) with client.application.app_context(): - request = self._generate_transformation_request() response = client.post( "/servicex/transformation", json=request, headers=self.fake_header() @@ -565,7 +563,6 @@ def test_submit_transformation_auth_enabled( extra_config={"ENABLE_AUTH": True}, code_gen_service=mock_codegen ) with client.application.app_context(): - response = client.post( "/servicex/transformation", json=self._generate_transformation_request(), @@ -614,7 +611,8 @@ def test_validate_with_multiple_prefixes(self): with patch.dict( os.environ, { - "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]' + "TOPCP_ALLOWED_IMAGES": """["sslhep/servicex_science_image_topcp:", + "docker.io/ssl-hep/"]""" }, ): assert ( From 91343d260d7bb0ef140471678a505b7aba98e4ab Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 4 Nov 2025 17:57:03 -0600 Subject: [PATCH 29/33] remove unneeded defaultBaseImage from values.yaml --- helm/servicex/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/helm/servicex/values.yaml b/helm/servicex/values.yaml index 4333cdf3e..80d99dce7 100644 --- a/helm/servicex/values.yaml +++ b/helm/servicex/values.yaml @@ -110,7 +110,6 @@ codeGen: defaultScienceContainerTag: 2.17.0-25.2.45 allowedImages: - "sslhep/servicex_science_image_topcp:" - defaultBaseImage: "sslhep/servicex_science_image_topcp" didFinder: CERNOpenData: From a172125fd116cca3f29cb6a2ec57a4a216c40245 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Mon, 10 Nov 2025 16:57:21 -0600 Subject: [PATCH 30/33] properly source app config values --- helm/servicex/templates/app/configmap.yaml | 2 +- .../servicex_app/resources/transformation/submit.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/helm/servicex/templates/app/configmap.yaml b/helm/servicex/templates/app/configmap.yaml index b95b0c681..59add3396 100644 --- a/helm/servicex/templates/app/configmap.yaml +++ b/helm/servicex/templates/app/configmap.yaml @@ -185,7 +185,7 @@ data: {{- if .Values.codeGen.topcp.enabled }} # TopCP custom image configuration - TOPCP_ALLOWED_IMAGES = {{ .Values.codeGen.topcp.allowedImages | toJson }} + TOPCP_ALLOWED_IMAGES = {{ .Values.codeGen.topcp.allowedImages | toJson | quote }} {{- end }} {{- $didFinders := list }} diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 7c0f3ab35..6cbfc7a2a 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -42,17 +42,15 @@ def validate_custom_docker_image(image_name: str) -> bool: - allowed_images_json = os.environ.get("TOPCP_ALLOWED_IMAGES") + allowed_images_json = current_app.config.get("TOPCP_ALLOWED_IMAGES") if not allowed_images_json: raise BadRequest("Custom Docker images are not allowed.") try: allowed_prefixes = json.loads(allowed_images_json) - if not isinstance(allowed_prefixes, list): raise BadRequest("TopCP allowed images are improperly configured.") - for prefix in allowed_prefixes: if image_name.startswith(prefix): return True @@ -252,6 +250,8 @@ def post(self): if "docker_image" in selection: custom_docker_image = selection["docker_image"] validate_custom_docker_image(custom_docker_image) + print("test!!") + print(custom_docker_image) except json.decoder.JSONDecodeError: raise BadRequest("Malformed JSON submitted") From 78ddeac71bb8c04af0d4c62dd03fdab6ec1d68cf Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Mon, 10 Nov 2025 17:00:16 -0600 Subject: [PATCH 31/33] resolve flake8 --- servicex_app/servicex_app/resources/transformation/submit.py | 1 - 1 file changed, 1 deletion(-) diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 6cbfc7a2a..f2b445518 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -27,7 +27,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import uuid import json -import os from datetime import datetime, timezone from typing import Optional, List From 6bbd73b477667485b4edb96002df5c6f8a076147 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Mon, 10 Nov 2025 17:30:32 -0600 Subject: [PATCH 32/33] update tests to use app.config --- .../resources/transformation/submit.py | 2 - .../resources/transformation/test_submit.py | 147 +++++++++--------- 2 files changed, 73 insertions(+), 76 deletions(-) diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index f2b445518..6209d8750 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -249,8 +249,6 @@ def post(self): if "docker_image" in selection: custom_docker_image = selection["docker_image"] validate_custom_docker_image(custom_docker_image) - print("test!!") - print(custom_docker_image) except json.decoder.JSONDecodeError: raise BadRequest("Malformed JSON submitted") diff --git a/servicex_app/servicex_app_test/resources/transformation/test_submit.py b/servicex_app/servicex_app_test/resources/transformation/test_submit.py index b480d7cc3..31addbf00 100644 --- a/servicex_app/servicex_app_test/resources/transformation/test_submit.py +++ b/servicex_app/servicex_app_test/resources/transformation/test_submit.py @@ -26,9 +26,8 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json -import os from datetime import datetime, timezone -from unittest.mock import ANY, patch +from unittest.mock import ANY import pytest from celery import Celery @@ -592,15 +591,16 @@ def test_submit_transformation_with_title( assert saved_obj.title == title -class TestValidateCustomDockerImage: +class TestValidateCustomDockerImage(ResourceTestBase): """Tests for the validate_custom_docker_image function""" def test_validate_with_matching_prefix(self): """Test validation succeeds when image matches an allowed prefix""" - with patch.dict( - os.environ, - {"TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]'}, - ): + extra_config = { + "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]' + } + client = self._test_client(extra_config=extra_config) + with client.application.app_context(): result = validate_custom_docker_image( "sslhep/servicex_science_image_topcp:2.17.0" ) @@ -608,13 +608,11 @@ def test_validate_with_matching_prefix(self): def test_validate_with_multiple_prefixes(self): """Test validation with multiple allowed prefixes""" - with patch.dict( - os.environ, - { - "TOPCP_ALLOWED_IMAGES": """["sslhep/servicex_science_image_topcp:", - "docker.io/ssl-hep/"]""" - }, - ): + extra_config = { + "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]' + } + client = self._test_client(extra_config=extra_config) + with client.application.app_context(): assert ( validate_custom_docker_image( "sslhep/servicex_science_image_topcp:latest" @@ -625,16 +623,18 @@ def test_validate_with_multiple_prefixes(self): def test_validate_with_no_matching_prefix(self): """Test validation fails when image doesn't match any allowed prefix""" - with patch.dict( - os.environ, - {"TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]'}, - ): + extra_config = { + "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]' + } + client = self._test_client(extra_config=extra_config) + with client.application.app_context(): with pytest.raises(BadRequest, match="not allowed"): validate_custom_docker_image("unauthorized/image:latest") def test_validate_with_no_env_variable(self): """Test validation fails when TOPCP_ALLOWED_IMAGES is not set""" - with patch.dict(os.environ, {}, clear=True): + client = self._test_client() + with client.application.app_context(): with pytest.raises( BadRequest, match="Custom Docker images are not allowed" ): @@ -644,7 +644,9 @@ def test_validate_with_no_env_variable(self): def test_validate_with_invalid_json(self): """Test validation fails with invalid JSON in env variable""" - with patch.dict(os.environ, {"TOPCP_ALLOWED_IMAGES": "not-valid-json"}): + extra_config = {"TOPCP_ALLOWED_IMAGES": "not-valid-json"} + client = self._test_client(extra_config=extra_config) + with client.application.app_context(): with pytest.raises(BadRequest, match="improperly configured"): validate_custom_docker_image( "sslhep/servicex_science_image_topcp:2.17.0" @@ -652,7 +654,9 @@ def test_validate_with_invalid_json(self): def test_validate_with_non_list_json(self): """Test validation fails when JSON is not a list""" - with patch.dict(os.environ, {"TOPCP_ALLOWED_IMAGES": '{"key": "value"}'}): + extra_config = {"TOPCP_ALLOWED_IMAGES": '{"key": "value"}'} + client = self._test_client(extra_config=extra_config) + with client.application.app_context(): with pytest.raises(BadRequest, match="improperly configured"): validate_custom_docker_image( "sslhep/servicex_science_image_topcp:2.17.0" @@ -660,7 +664,9 @@ def test_validate_with_non_list_json(self): def test_validate_with_empty_list(self): """Test validation fails when allowed list is empty""" - with patch.dict(os.environ, {"TOPCP_ALLOWED_IMAGES": "[]"}): + extra_config = {"TOPCP_ALLOWED_IMAGES": "[]"} + client = self._test_client(extra_config=extra_config) + with client.application.app_context(): with pytest.raises(BadRequest, match="not allowed"): validate_custom_docker_image( "sslhep/servicex_science_image_topcp:2.17.0" @@ -714,57 +720,51 @@ def test_submit_topcp_with_custom_docker_image( ): """Test submitting a TopCP transformation with a valid custom docker image""" extra_config = { - "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} + "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"}, + "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]', } - with patch.dict( - os.environ, - {"TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]'}, - ): - client = self._test_client( - code_gen_service=mock_codegen, extra_config=extra_config + client = self._test_client( + code_gen_service=mock_codegen, extra_config=extra_config + ) + with client.application.app_context(): + selection_dict = { + "docker_image": "sslhep/servicex_science_image_topcp:custom" + } + request = self._generate_transformation_request( + selection=json.dumps(selection_dict) ) - with client.application.app_context(): - selection_dict = { - "docker_image": "sslhep/servicex_science_image_topcp:custom" - } - request = self._generate_transformation_request( - selection=json.dumps(selection_dict) - ) - response = client.post( - "/servicex/transformation", json=request, headers=self.fake_header() - ) - assert response.status_code == 200 - request_id = response.json["request_id"] + response = client.post( + "/servicex/transformation", json=request, headers=self.fake_header() + ) + assert response.status_code == 200 + request_id = response.json["request_id"] - saved_obj = TransformRequest.lookup(request_id) - assert saved_obj - assert saved_obj.image == "sslhep/servicex_science_image_topcp:custom" + saved_obj = TransformRequest.lookup(request_id) + assert saved_obj + assert saved_obj.image == "sslhep/servicex_science_image_topcp:custom" def test_submit_topcp_with_invalid_custom_docker_image( self, mock_dataset_manager_from_did, mock_codegen ): """Submitting a TopCP transformation with an invalid custom docker image fails""" extra_config = { - "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} + "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"}, + "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]', } - with patch.dict( - os.environ, - {"TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]'}, - ): - client = self._test_client( - code_gen_service=mock_codegen, extra_config=extra_config + client = self._test_client( + code_gen_service=mock_codegen, extra_config=extra_config + ) + with client.application.app_context(): + selection_dict = {"docker_image": "unauthorized/image:latest"} + request = self._generate_transformation_request( + selection=json.dumps(selection_dict) ) - with client.application.app_context(): - selection_dict = {"docker_image": "unauthorized/image:latest"} - request = self._generate_transformation_request( - selection=json.dumps(selection_dict) - ) - response = client.post( - "/servicex/transformation", json=request, headers=self.fake_header() - ) - assert response.status_code == 400 + response = client.post( + "/servicex/transformation", json=request, headers=self.fake_header() + ) + assert response.status_code == 400 def test_submit_topcp_with_non_json_selection( self, mock_dataset_manager_from_did, mock_codegen, mock_app_version @@ -791,19 +791,18 @@ def test_submit_custom_topcp_without_env_var( extra_config = { "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} } - with patch.dict(os.environ, {}, clear=True): - client = self._test_client( - code_gen_service=mock_codegen, extra_config=extra_config + client = self._test_client( + code_gen_service=mock_codegen, extra_config=extra_config + ) + with client.application.app_context(): + selection_dict = { + "docker_image": "sslhep/servicex_science_image_topcp:custom" + } + request = self._generate_transformation_request( + selection=json.dumps(selection_dict) ) - with client.application.app_context(): - selection_dict = { - "docker_image": "sslhep/servicex_science_image_topcp:custom" - } - request = self._generate_transformation_request( - selection=json.dumps(selection_dict) - ) - response = client.post( - "/servicex/transformation", json=request, headers=self.fake_header() - ) - assert response.status_code == 400 + response = client.post( + "/servicex/transformation", json=request, headers=self.fake_header() + ) + assert response.status_code == 400 From 1f20afe347d363010c965efa35361f433dd224b3 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Mon, 10 Nov 2025 17:32:45 -0600 Subject: [PATCH 33/33] resolve flake8 --- .../servicex_app_test/resources/transformation/test_submit.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/servicex_app/servicex_app_test/resources/transformation/test_submit.py b/servicex_app/servicex_app_test/resources/transformation/test_submit.py index 31addbf00..393eaaa1c 100644 --- a/servicex_app/servicex_app_test/resources/transformation/test_submit.py +++ b/servicex_app/servicex_app_test/resources/transformation/test_submit.py @@ -609,7 +609,9 @@ def test_validate_with_matching_prefix(self): def test_validate_with_multiple_prefixes(self): """Test validation with multiple allowed prefixes""" extra_config = { - "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]' + "TOPCP_ALLOWED_IMAGES": ( + '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]' + ) } client = self._test_client(extra_config=extra_config) with client.application.app_context():