diff --git a/.github/workflows/deploy-test-studio-crc.yml b/.github/workflows/deploy-test-studio-crc.yml index 2af1989..9239776 100644 --- a/.github/workflows/deploy-test-studio-crc.yml +++ b/.github/workflows/deploy-test-studio-crc.yml @@ -177,6 +177,7 @@ jobs: export NON_INTERACTIVE=true export DEPLOYMENT_ENV="crc" export OC_PROJECT="geostudio-test" + export STORAGE_MODE="cloud-object-storage" # Get cluster URL from CRC export CLUSTER_URL=$(oc get IngressController default -n openshift-ingress-operator -o jsonpath='{.status.domain}') diff --git a/.github/workflows/deploy-test-studio-kind.yml b/.github/workflows/deploy-test-studio-kind.yml index 0d28281..ee316f3 100644 --- a/.github/workflows/deploy-test-studio-kind.yml +++ b/.github/workflows/deploy-test-studio-kind.yml @@ -165,6 +165,7 @@ jobs: # Set non-interactive mode and required variables export NON_INTERACTIVE=true export cluster_node_name="$WORKER_NODE" + export STORAGE_MODE="cloud-object-storage" export in_cluster_storage_class_type="Default" export NON_COS_STORAGE_CLASS="standard" export OC_PROJECT=default diff --git a/.secrets.baseline b/.secrets.baseline index 1b56160..94e2a13 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1,9 +1,9 @@ { "exclude": { - "files": "^.secrets.baseline$", + "files": null, "lines": null }, - "generated_at": "2025-11-18T08:32:46Z", + "generated_at": "2026-03-23T12:10:53Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -77,174 +77,172 @@ } ], "results": { - "Local-lima-vm.md": [ + ".github/workflows/deploy-test-studio-crc.yml": [ { - "hashed_secret": "4cbb11fd97e365dc0c1eab9aba3a60ac28b9f10a", + "hashed_secret": "5dce7cf340967ee46d0c7837259edf87dc783e0f", "is_secret": false, "is_verified": false, - "line_number": 149, + "line_number": 691, "type": "Secret Keyword", "verified_result": null - }, + } + ], + ".github/workflows/deploy-test-studio-kind.yml": [ { - "hashed_secret": "9d3fccd580bb161ad5fe3e0e1ac5c5b8825f4913", + "hashed_secret": "5dce7cf340967ee46d0c7837259edf87dc783e0f", "is_secret": false, "is_verified": false, - "line_number": 240, + "line_number": 400, "type": "Secret Keyword", "verified_result": null - }, + } + ], + "README.md": [ { - "hashed_secret": "7cd974289f88ebb8cbd99a7b1abc8daee0c28389", + "hashed_secret": "4d9bfd0e8de7017b695cb5991006816c106a3205", "is_secret": false, "is_verified": false, - "line_number": 298, + "line_number": 296, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "4d9bfd0e8de7017b695cb5991006816c106a3205", + "hashed_secret": "236a00044010f54e7d9d32fabe715750c4fc55ba", "is_secret": false, "is_verified": false, - "line_number": 345, + "line_number": 298, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "4f4944a7117fd2e95169da2b40af33b68a65a161", + "hashed_secret": "7cd974289f88ebb8cbd99a7b1abc8daee0c28389", "is_secret": false, "is_verified": false, - "line_number": 413, + "line_number": 301, "type": "Secret Keyword", "verified_result": null - } - ], - "Local-minikube.md": [ + }, { "hashed_secret": "4cbb11fd97e365dc0c1eab9aba3a60ac28b9f10a", "is_secret": false, "is_verified": false, - "line_number": 128, + "line_number": 303, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "9d3fccd580bb161ad5fe3e0e1ac5c5b8825f4913", + "hashed_secret": "d9e9019d9eb455a3d72a3bc252c26927bb148a10", "is_secret": false, "is_verified": false, - "line_number": 216, + "line_number": 325, "type": "Secret Keyword", "verified_result": null - }, + } + ], + "deploy_studio_k8s.sh": [ { - "hashed_secret": "7cd974289f88ebb8cbd99a7b1abc8daee0c28389", + "hashed_secret": "973d12df7be0124e5459faaf32f1f154761c04c9", "is_secret": false, "is_verified": false, - "line_number": 274, + "line_number": 14, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "4d9bfd0e8de7017b695cb5991006816c106a3205", + "hashed_secret": "90e243a59a66e7e7019b9b8f90128b484d8271e6", "is_secret": false, "is_verified": false, - "line_number": 321, + "line_number": 17, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "4f4944a7117fd2e95169da2b40af33b68a65a161", + "hashed_secret": "6ae774b1f2afa1c184255ec69cc57182e95e666b", "is_secret": false, "is_verified": false, - "line_number": 364, + "line_number": 174, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "bb7989c7e3b85c7488b9c82c051f5a48ce7a35b1", + "hashed_secret": "f1f48dd750683ff3ac5aef31c438f54f7e5d4a9b", "is_secret": false, "is_verified": false, - "line_number": 606, + "line_number": 217, "type": "Secret Keyword", "verified_result": null - } - ], - "Local_dev_deployment.md": [ + }, { - "hashed_secret": "27d999fff6529e289bd8f0fe933fd83957f03a72", + "hashed_secret": "0e5a73ebd25b4a5cf6e4a7808c54098efa4ebe4c", "is_secret": false, "is_verified": false, - "line_number": 54, + "line_number": 218, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "d033e22ae348aeb5660fc2140aec35850c4da997", + "hashed_secret": "88f4d1e9f4d0c712ee4bd57dc98f9e6ba2e4f5b6", "is_secret": false, "is_verified": false, - "line_number": 70, + "line_number": 241, "type": "Secret Keyword", "verified_result": null - } - ], - "README.md": [ + }, { - "hashed_secret": "4d9bfd0e8de7017b695cb5991006816c106a3205", + "hashed_secret": "8846702e5cfa953399ddf8365418a62b770da625", "is_secret": false, "is_verified": false, - "line_number": 181, + "line_number": 248, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "d9e9019d9eb455a3d72a3bc252c26927bb148a10", + "hashed_secret": "ef62f7910a4fe3ae4cac61b2de5f1e99975826f2", "is_secret": false, "is_verified": false, - "line_number": 205, + "line_number": 284, "type": "Secret Keyword", "verified_result": null - } - ], - "Studio_Deployment_instructions.md": [ + }, { - "hashed_secret": "9d3fccd580bb161ad5fe3e0e1ac5c5b8825f4913", + "hashed_secret": "66512948965f5d8379a7d0898b553bc4884c98a8", "is_secret": false, "is_verified": false, - "line_number": 234, + "line_number": 288, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "4f4944a7117fd2e95169da2b40af33b68a65a161", + "hashed_secret": "b6217e9c997a979a4fcd2df9a22090eb2942dad4", "is_secret": false, "is_verified": false, - "line_number": 376, + "line_number": 378, "type": "Secret Keyword", "verified_result": null } ], - "deploy_studio.sh": [ + "deploy_studio_lima.sh": [ { - "hashed_secret": "2ace62c1befa19e3ea37dd52be9f6d508c5163e6", + "hashed_secret": "973d12df7be0124e5459faaf32f1f154761c04c9", "is_secret": false, "is_verified": false, - "line_number": 164, + "line_number": 14, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "6ae774b1f2afa1c184255ec69cc57182e95e666b", + "hashed_secret": "90e243a59a66e7e7019b9b8f90128b484d8271e6", "is_secret": false, "is_verified": false, - "line_number": 233, + "line_number": 17, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "b4ef6764d15387bf38148918388d736c33efb499", + "hashed_secret": "6ae774b1f2afa1c184255ec69cc57182e95e666b", "is_secret": false, "is_verified": false, - "line_number": 260, + "line_number": 124, "type": "Secret Keyword", "verified_result": null }, @@ -252,7 +250,7 @@ "hashed_secret": "f1f48dd750683ff3ac5aef31c438f54f7e5d4a9b", "is_secret": false, "is_verified": false, - "line_number": 314, + "line_number": 166, "type": "Secret Keyword", "verified_result": null }, @@ -260,23 +258,15 @@ "hashed_secret": "0e5a73ebd25b4a5cf6e4a7808c54098efa4ebe4c", "is_secret": false, "is_verified": false, - "line_number": 315, - "type": "Secret Keyword", - "verified_result": null - }, - { - "hashed_secret": "c4f3d1d86375f5851344d132126f24a933aa2844", - "is_secret": false, - "is_verified": false, - "line_number": 331, + "line_number": 167, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "90e243a59a66e7e7019b9b8f90128b484d8271e6", + "hashed_secret": "88f4d1e9f4d0c712ee4bd57dc98f9e6ba2e4f5b6", "is_secret": false, "is_verified": false, - "line_number": 374, + "line_number": 190, "type": "Secret Keyword", "verified_result": null }, @@ -284,7 +274,7 @@ "hashed_secret": "8846702e5cfa953399ddf8365418a62b770da625", "is_secret": false, "is_verified": false, - "line_number": 377, + "line_number": 197, "type": "Secret Keyword", "verified_result": null }, @@ -292,7 +282,7 @@ "hashed_secret": "ef62f7910a4fe3ae4cac61b2de5f1e99975826f2", "is_secret": false, "is_verified": false, - "line_number": 408, + "line_number": 232, "type": "Secret Keyword", "verified_result": null }, @@ -300,7 +290,7 @@ "hashed_secret": "66512948965f5d8379a7d0898b553bc4884c98a8", "is_secret": false, "is_verified": false, - "line_number": 412, + "line_number": 236, "type": "Secret Keyword", "verified_result": null }, @@ -308,25 +298,25 @@ "hashed_secret": "b6217e9c997a979a4fcd2df9a22090eb2942dad4", "is_secret": false, "is_verified": false, - "line_number": 494, + "line_number": 319, "type": "Secret Keyword", "verified_result": null } ], - "deploy_studio_local.sh": [ + "deploy_studio_ocp.sh": [ { "hashed_secret": "6ae774b1f2afa1c184255ec69cc57182e95e666b", "is_secret": false, "is_verified": false, - "line_number": 82, + "line_number": 501, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "f1f48dd750683ff3ac5aef31c438f54f7e5d4a9b", + "hashed_secret": "b4ef6764d15387bf38148918388d736c33efb499", "is_secret": false, "is_verified": false, - "line_number": 123, + "line_number": 533, "type": "Secret Keyword", "verified_result": null }, @@ -334,7 +324,31 @@ "hashed_secret": "0e5a73ebd25b4a5cf6e4a7808c54098efa4ebe4c", "is_secret": false, "is_verified": false, - "line_number": 124, + "line_number": 590, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "c4f3d1d86375f5851344d132126f24a933aa2844", + "is_secret": false, + "is_verified": false, + "line_number": 610, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "88f4d1e9f4d0c712ee4bd57dc98f9e6ba2e4f5b6", + "is_secret": false, + "is_verified": false, + "line_number": 660, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "90e243a59a66e7e7019b9b8f90128b484d8271e6", + "is_secret": false, + "is_verified": false, + "line_number": 665, "type": "Secret Keyword", "verified_result": null }, @@ -342,7 +356,31 @@ "hashed_secret": "8846702e5cfa953399ddf8365418a62b770da625", "is_secret": false, "is_verified": false, - "line_number": 151, + "line_number": 669, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "ef7e6794ca9c6a06b54b66f279237fb8daaaeea8", + "is_secret": false, + "is_verified": false, + "line_number": 694, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "e21322cb43df32d78df21fcfbc130c96a12ac986", + "is_secret": false, + "is_verified": false, + "line_number": 700, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "2ace62c1befa19e3ea37dd52be9f6d508c5163e6", + "is_secret": false, + "is_verified": false, + "line_number": 711, "type": "Secret Keyword", "verified_result": null }, @@ -350,7 +388,7 @@ "hashed_secret": "ef62f7910a4fe3ae4cac61b2de5f1e99975826f2", "is_secret": false, "is_verified": false, - "line_number": 185, + "line_number": 758, "type": "Secret Keyword", "verified_result": null }, @@ -358,7 +396,7 @@ "hashed_secret": "66512948965f5d8379a7d0898b553bc4884c98a8", "is_secret": false, "is_verified": false, - "line_number": 189, + "line_number": 762, "type": "Secret Keyword", "verified_result": null }, @@ -366,7 +404,7 @@ "hashed_secret": "b6217e9c997a979a4fcd2df9a22090eb2942dad4", "is_secret": false, "is_verified": false, - "line_number": 279, + "line_number": 877, "type": "Secret Keyword", "verified_result": null } @@ -376,25 +414,17 @@ "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", "is_secret": false, "is_verified": false, - "line_number": 44, + "line_number": 41, "type": "Basic Auth Credentials", "verified_result": null } ], - "deployment-scripts/minio.sh": [ - { - "hashed_secret": "2ace62c1befa19e3ea37dd52be9f6d508c5163e6", - "is_secret": false, - "is_verified": false, - "line_number": 165, - "type": "Secret Keyword", - "verified_result": null - }, + "deployment-scripts/geoserver-deployment.yaml": [ { - "hashed_secret": "f37fd28d63c206f7ddee188f9b1eabfd5749abee", + "hashed_secret": "255a762c36cdae2fe950e67e8b8a4efb260e7540", "is_secret": false, "is_verified": false, - "line_number": 167, + "line_number": 197, "type": "Secret Keyword", "verified_result": null } @@ -404,282 +434,998 @@ "hashed_secret": "d033e22ae348aeb5660fc2140aec35850c4da997", "is_secret": false, "is_verified": false, - "line_number": 9, + "line_number": 13, "type": "Secret Keyword", "verified_result": null - } - ], - "deployment-values-details.md": [ + }, { - "hashed_secret": "93ac8946882128457cd9e283b30ca851945e6690", + "hashed_secret": "44caf6b000ff51b6c5966cfca4f9c013aa8dee9d", "is_secret": false, "is_verified": false, - "line_number": 10, + "line_number": 31, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "1af17e73721dbe0c40011b82ed4bb1a7dbe3ce29", + "hashed_secret": "f1f48dd750683ff3ac5aef31c438f54f7e5d4a9b", "is_secret": false, "is_verified": false, - "line_number": 12, + "line_number": 36, "type": "Secret Keyword", "verified_result": null - }, + } + ], + "deployment-scripts/template/populate-buckets-minio-pvc.yaml": [ { - "hashed_secret": "8b142a91cfb6e617618ad437cedf74a6745f8926", + "hashed_secret": "d30d31c0d3b2a36c927a1824bb0b5c4901d1bf3b", "is_secret": false, "is_verified": false, - "line_number": 26, + "line_number": 29, "type": "Secret Keyword", "verified_result": null - }, + } + ], + "docs/deployment/crc_deployment.md": [ { - "hashed_secret": "4c082b6d02a995fba0cc92f3dbe83b697be1f966", + "hashed_secret": "7cd974289f88ebb8cbd99a7b1abc8daee0c28389", "is_secret": false, "is_verified": false, - "line_number": 27, + "line_number": 193, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "4af07521e94438fcc37c966f7630940db3524a2e", + "hashed_secret": "4cbb11fd97e365dc0c1eab9aba3a60ac28b9f10a", "is_secret": false, "is_verified": false, - "line_number": 56, - "type": "Basic Auth Credentials", + "line_number": 198, + "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "97c368c0c24e4533d11e295b978d4d6bc2a8101c", + "hashed_secret": "d56d985300d4b52eb6e189be006f44f8d23c5ec9", "is_secret": false, "is_verified": false, - "line_number": 60, + "line_number": 203, "type": "Secret Keyword", "verified_result": null } ], - "geospatial-studio-pipelines/templates/secrets.yaml": [ + "docs/deployment/detailed_deployment_cluster.md": [ { - "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3", + "hashed_secret": "9d3fccd580bb161ad5fe3e0e1ac5c5b8825f4913", "is_secret": false, "is_verified": false, - "line_number": 15, - "type": "Basic Auth Credentials", + "line_number": 243, + "type": "Secret Keyword", "verified_result": null - } - ], - "geospatial-studio-pipelines/values.yaml": [ + }, { - "hashed_secret": "66cf0e57ff0c4392b01bfadcd28759c621938a02", + "hashed_secret": "0e5a73ebd25b4a5cf6e4a7808c54098efa4ebe4c", "is_secret": false, "is_verified": false, - "line_number": 27, + "line_number": 320, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "64955456d46405ff355b3abc0cf74a4ff9bdf283", + "hashed_secret": "7cd974289f88ebb8cbd99a7b1abc8daee0c28389", "is_secret": false, "is_verified": false, - "line_number": 31, + "line_number": 338, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "90c3cc8d4eae70f6a1e7c2a9c88a57d5d9623deb", + "hashed_secret": "4d9bfd0e8de7017b695cb5991006816c106a3205", "is_secret": false, "is_verified": false, - "line_number": 41, + "line_number": 385, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "8e9e0a058630388dcbb497903ec1a5b03e3781f9", + "hashed_secret": "90e243a59a66e7e7019b9b8f90128b484d8271e6", "is_secret": false, "is_verified": false, - "line_number": 46, + "line_number": 423, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "055abb8b464ae0ab2bb7e9d2fa150c641fee8334", + "hashed_secret": "4f4944a7117fd2e95169da2b40af33b68a65a161", "is_secret": false, "is_verified": false, - "line_number": 51, + "line_number": 490, "type": "Secret Keyword", "verified_result": null } ], - "geospatial-studio/charts/gfm-studio-gateway/values.yaml": [ + "docs/deployment/detailed_deployment_k8s.md": [ { - "hashed_secret": "4f4944a7117fd2e95169da2b40af33b68a65a161", + "hashed_secret": "6ae774b1f2afa1c184255ec69cc57182e95e666b", "is_secret": false, "is_verified": false, - "line_number": 273, + "line_number": 230, "type": "Secret Keyword", "verified_result": null - } - ], - "geospatial-studio/values.yaml": [ + }, { - "hashed_secret": "66cf0e57ff0c4392b01bfadcd28759c621938a02", + "hashed_secret": "0e5a73ebd25b4a5cf6e4a7808c54098efa4ebe4c", "is_secret": false, "is_verified": false, - "line_number": 27, + "line_number": 339, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "8b142a91cfb6e617618ad437cedf74a6745f8926", + "hashed_secret": "88f4d1e9f4d0c712ee4bd57dc98f9e6ba2e4f5b6", "is_secret": false, "is_verified": false, - "line_number": 53, + "line_number": 378, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "4c082b6d02a995fba0cc92f3dbe83b697be1f966", + "hashed_secret": "90e243a59a66e7e7019b9b8f90128b484d8271e6", "is_secret": false, "is_verified": false, - "line_number": 54, + "line_number": 391, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "4af07521e94438fcc37c966f7630940db3524a2e", + "hashed_secret": "8846702e5cfa953399ddf8365418a62b770da625", "is_secret": false, "is_verified": false, - "line_number": 99, - "type": "Basic Auth Credentials", + "line_number": 399, + "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "97c368c0c24e4533d11e295b978d4d6bc2a8101c", + "hashed_secret": "ef62f7910a4fe3ae4cac61b2de5f1e99975826f2", "is_secret": false, "is_verified": false, - "line_number": 103, + "line_number": 462, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "87cc2bd0c04fc6d36f506792f73956d670217fba", + "hashed_secret": "66512948965f5d8379a7d0898b553bc4884c98a8", "is_secret": false, "is_verified": false, - "line_number": 124, + "line_number": 466, "type": "Secret Keyword", "verified_result": null - } - ], - "populate-studio/payloads/templates/template-clay_v1_seg.json": [ + }, { - "hashed_secret": "04f97eb8bcc0c1e1dfc1498251b7988d83042023", + "hashed_secret": "4d9bfd0e8de7017b695cb5991006816c106a3205", "is_secret": false, "is_verified": false, - "line_number": 373, - "type": "Base64 High Entropy String", + "line_number": 634, + "type": "Secret Keyword", "verified_result": null - } - ], - "populate-studio/payloads/templates/template-reg.json": [ + }, { - "hashed_secret": "13b8281bb3f1779373722c3bf6ab0a591dc1b56b", + "hashed_secret": "236a00044010f54e7d9d32fabe715750c4fc55ba", "is_secret": false, "is_verified": false, - "line_number": 357, - "type": "Base64 High Entropy String", + "line_number": 636, + "type": "Secret Keyword", "verified_result": null - } - ], - "populate-studio/payloads/templates/template-seg.json": [ + }, { - "hashed_secret": "74fffb127df2531fd89a23729be6acd95ae1eadf", + "hashed_secret": "7cd974289f88ebb8cbd99a7b1abc8daee0c28389", "is_secret": false, "is_verified": false, - "line_number": 367, - "type": "Base64 High Entropy String", + "line_number": 637, + "type": "Secret Keyword", "verified_result": null - } - ], - "populate-studio/payloads/templates/template-terramind_seg.json": [ + }, { - "hashed_secret": "28fb62850e226755529d69f9cc3dfa9fc538332c", + "hashed_secret": "4cbb11fd97e365dc0c1eab9aba3a60ac28b9f10a", "is_secret": false, "is_verified": false, - "line_number": 366, - "type": "Base64 High Entropy String", + "line_number": 639, + "type": "Secret Keyword", "verified_result": null } ], - "populate-studio/payloads/templates/template-timm_convnext_seg.json": [ + "docs/deployment/detailed_deployment_local.md": [ { - "hashed_secret": "0ea380996f981df678a1e0935c9bff4e980952d1", + "hashed_secret": "9d3fccd580bb161ad5fe3e0e1ac5c5b8825f4913", "is_secret": false, "is_verified": false, - "line_number": 366, - "type": "Base64 High Entropy String", + "line_number": 367, + "type": "Secret Keyword", "verified_result": null - } - ], - "populate-studio/payloads/templates/template-timm_resnet_seg.json": [ + }, { - "hashed_secret": "9cca9058e26a37baa4739971ccdd7af65ebfaa6f", + "hashed_secret": "90e243a59a66e7e7019b9b8f90128b484d8271e6", "is_secret": false, "is_verified": false, - "line_number": 366, - "type": "Base64 High Entropy String", + "line_number": 542, + "type": "Secret Keyword", "verified_result": null - } - ], - "run_deploy_steps.sh": [ + }, { - "hashed_secret": "f1f48dd750683ff3ac5aef31c438f54f7e5d4a9b", + "hashed_secret": "4f4944a7117fd2e95169da2b40af33b68a65a161", "is_secret": false, "is_verified": false, - "line_number": 120, + "line_number": 572, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "0e5a73ebd25b4a5cf6e4a7808c54098efa4ebe4c", + "hashed_secret": "4d9bfd0e8de7017b695cb5991006816c106a3205", "is_secret": false, "is_verified": false, - "line_number": 121, + "line_number": 734, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "ef62f7910a4fe3ae4cac61b2de5f1e99975826f2", + "hashed_secret": "236a00044010f54e7d9d32fabe715750c4fc55ba", "is_secret": false, "is_verified": false, - "line_number": 138, + "line_number": 736, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "66512948965f5d8379a7d0898b553bc4884c98a8", + "hashed_secret": "7cd974289f88ebb8cbd99a7b1abc8daee0c28389", "is_secret": false, "is_verified": false, - "line_number": 140, + "line_number": 738, "type": "Secret Keyword", "verified_result": null }, { - "hashed_secret": "b6217e9c997a979a4fcd2df9a22090eb2942dad4", + "hashed_secret": "4cbb11fd97e365dc0c1eab9aba3a60ac28b9f10a", "is_secret": false, "is_verified": false, - "line_number": 208, + "line_number": 740, "type": "Secret Keyword", "verified_result": null } ], - "tests/api-data/03-ft-templates.json": [ + "docs/getting-started/first-steps.md": [ { - "hashed_secret": "74fffb127df2531fd89a23729be6acd95ae1eadf", + "hashed_secret": "1c58bd92003bbaa0538e249fff6ee19a270dec5f", "is_secret": false, "is_verified": false, - "line_number": 5, - "type": "Base64 High Entropy String", + "line_number": 47, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "90e243a59a66e7e7019b9b8f90128b484d8271e6", + "is_secret": false, + "is_verified": false, + "line_number": 55, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "bc565f6e909ec7d3c18e2ff5d9eeb2300ff20b7f", + "is_secret": false, + "is_verified": false, + "line_number": 63, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "4d9bfd0e8de7017b695cb5991006816c106a3205", + "is_secret": false, + "is_verified": false, + "line_number": 77, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "d9e9019d9eb455a3d72a3bc252c26927bb148a10", + "is_secret": false, + "is_verified": false, + "line_number": 104, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "df74c0d60053f76c7f3d013152414f3026c71957", + "is_secret": false, + "is_verified": false, + "line_number": 112, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "a8253456364f1bfc7da7ae4a1db5b45d106317a5", + "is_secret": false, + "is_verified": false, + "line_number": 125, + "type": "Secret Keyword", + "verified_result": null + } + ], + "docs/getting-started/getting-started-SDK.md": [ + { + "hashed_secret": "98dad5e4b406f17d94cc970f1332d3b32d263927", + "is_secret": false, + "is_verified": false, + "line_number": 39, + "type": "Secret Keyword", + "verified_result": null + } + ], + "docs/payloads/templates/template-clay_v1_seg.json": [ + { + "hashed_secret": "04f97eb8bcc0c1e1dfc1498251b7988d83042023", + "is_secret": false, + "is_verified": false, + "line_number": 380, + "type": "Base64 High Entropy String", + "verified_result": null + } + ], + "docs/payloads/templates/template-reg.json": [ + { + "hashed_secret": "13b8281bb3f1779373722c3bf6ab0a591dc1b56b", + "is_secret": false, + "is_verified": false, + "line_number": 357, + "type": "Base64 High Entropy String", + "verified_result": null + } + ], + "docs/payloads/templates/template-seg.json": [ + { + "hashed_secret": "74fffb127df2531fd89a23729be6acd95ae1eadf", + "is_secret": false, + "is_verified": false, + "line_number": 374, + "type": "Base64 High Entropy String", + "verified_result": null + } + ], + "docs/payloads/templates/template-terramind_seg.json": [ + { + "hashed_secret": "28fb62850e226755529d69f9cc3dfa9fc538332c", + "is_secret": false, + "is_verified": false, + "line_number": 373, + "type": "Base64 High Entropy String", + "verified_result": null + } + ], + "docs/payloads/templates/template-timm_convnext_seg.json": [ + { + "hashed_secret": "0ea380996f981df678a1e0935c9bff4e980952d1", + "is_secret": false, + "is_verified": false, + "line_number": 373, + "type": "Base64 High Entropy String", + "verified_result": null + } + ], + "docs/payloads/templates/template-timm_resnet_seg.json": [ + { + "hashed_secret": "9cca9058e26a37baa4739971ccdd7af65ebfaa6f", + "is_secret": false, + "is_verified": false, + "line_number": 373, + "type": "Base64 High Entropy String", + "verified_result": null + } + ], + "docs/reference/sample-payloads.md": [ + { + "hashed_secret": "ece33d80e9fdd21f657d54103482b47367fc9f4e", + "is_secret": false, + "is_verified": false, + "line_number": 571, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "ddd7047c3e79fe19f557c73d384fe55cc232c749", + "is_secret": false, + "is_verified": false, + "line_number": 578, + "type": "Secret Keyword", + "verified_result": null + } + ], + "docs/user-guides/fine-tuning/fine-tuning-service.md": [ + { + "hashed_secret": "4bb195e6267cf132b3468fa88b108a7d0d45128f", + "is_secret": false, + "is_verified": false, + "line_number": 272, + "type": "Secret Keyword", + "verified_result": null + } + ], + "docs/user-guides/inference/inference-service.md": [ + { + "hashed_secret": "bbc4e9d52252171a3a306be55086c65b126189e8", + "is_secret": false, + "is_verified": false, + "line_number": 19, + "type": "Secret Keyword", + "verified_result": null + } + ], + "geospatial-studio/charts/geospatial-studio-pipelines/templates/pvc.yaml": [ + { + "hashed_secret": "1687da9ce4ce99db215ea0f72e13cbde74038fa7", + "is_secret": false, + "is_verified": false, + "line_number": 78, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "d2e2ab0f407e4ee3cf2ab87d61c31b25a74085e5", + "is_secret": false, + "is_verified": false, + "line_number": 112, + "type": "Secret Keyword", + "verified_result": null + } + ], + "geospatial-studio/charts/geospatial-studio-pipelines/templates/secrets.yaml": [ + { + "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3", + "is_secret": false, + "is_verified": false, + "line_number": 12, + "type": "Basic Auth Credentials", + "verified_result": null + } + ], + "geospatial-studio/charts/geospatial-studio-pipelines/values.yaml": [ + { + "hashed_secret": "66cf0e57ff0c4392b01bfadcd28759c621938a02", + "is_secret": false, + "is_verified": false, + "line_number": 23, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "64955456d46405ff355b3abc0cf74a4ff9bdf283", + "is_secret": false, + "is_verified": false, + "line_number": 39, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "90c3cc8d4eae70f6a1e7c2a9c88a57d5d9623deb", + "is_secret": false, + "is_verified": false, + "line_number": 49, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "8e9e0a058630388dcbb497903ec1a5b03e3781f9", + "is_secret": false, + "is_verified": false, + "line_number": 54, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "055abb8b464ae0ab2bb7e9d2fa150c641fee8334", + "is_secret": false, + "is_verified": false, + "line_number": 59, + "type": "Secret Keyword", + "verified_result": null + } + ], + "geospatial-studio/charts/gfm-mlflow/templates/pvc.yaml": [ + { + "hashed_secret": "a462d873bb54fe73a7454a394e7a82421e2391bb", + "is_secret": false, + "is_verified": false, + "line_number": 58, + "type": "Secret Keyword", + "verified_result": null + } + ], + "geospatial-studio/charts/gfm-studio-gateway/templates/pvc.yaml": [ + { + "hashed_secret": "f17581e821fb790afd02935d5cb7dda40b6d2e30", + "is_secret": false, + "is_verified": false, + "line_number": 111, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "b6a8ab7d5d1e79a572d3811a70fdf31da8a319e7", + "is_secret": false, + "is_verified": false, + "line_number": 144, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "611bfa6fb1e19c85e64aaddb9e09c0ec54cf3a49", + "is_secret": false, + "is_verified": false, + "line_number": 177, + "type": "Secret Keyword", + "verified_result": null + } + ], + "geospatial-studio/charts/gfm-studio-gateway/values.yaml": [ + { + "hashed_secret": "4f4944a7117fd2e95169da2b40af33b68a65a161", + "is_secret": false, + "is_verified": false, + "line_number": 337, + "type": "Secret Keyword", + "verified_result": null + } + ], + "geospatial-studio/values.yaml": [ + { + "hashed_secret": "8b142a91cfb6e617618ad437cedf74a6745f8926", + "is_secret": false, + "is_verified": false, + "line_number": 54, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "4c082b6d02a995fba0cc92f3dbe83b697be1f966", + "is_secret": false, + "is_verified": false, + "line_number": 55, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "a9649a4c6e02455e4b810918720ebdf98bdd010e", + "is_secret": false, + "is_verified": false, + "line_number": 58, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "4af07521e94438fcc37c966f7630940db3524a2e", + "is_secret": false, + "is_verified": false, + "line_number": 104, + "type": "Basic Auth Credentials", + "verified_result": null + }, + { + "hashed_secret": "97c368c0c24e4533d11e295b978d4d6bc2a8101c", + "is_secret": false, + "is_verified": false, + "line_number": 108, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "87cc2bd0c04fc6d36f506792f73956d670217fba", + "is_secret": false, + "is_verified": false, + "line_number": 129, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "66cf0e57ff0c4392b01bfadcd28759c621938a02", + "is_secret": false, + "is_verified": false, + "line_number": 309, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "64955456d46405ff355b3abc0cf74a4ff9bdf283", + "is_secret": false, + "is_verified": false, + "line_number": 312, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "90c3cc8d4eae70f6a1e7c2a9c88a57d5d9623deb", + "is_secret": false, + "is_verified": false, + "line_number": 321, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "8e9e0a058630388dcbb497903ec1a5b03e3781f9", + "is_secret": false, + "is_verified": false, + "line_number": 325, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "055abb8b464ae0ab2bb7e9d2fa150c641fee8334", + "is_secret": false, + "is_verified": false, + "line_number": 329, + "type": "Secret Keyword", + "verified_result": null + } + ], + "populate-studio/getting-started-notebook.ipynb": [ + { + "hashed_secret": "5243abcc9135d638bc73e8adcb52cb56e56a0dac", + "is_secret": false, + "is_verified": false, + "line_number": 135, + "type": "Secret Keyword", + "verified_result": null + } + ], + "populate-studio/payloads/templates/template-clay_v1_seg.json": [ + { + "hashed_secret": "04f97eb8bcc0c1e1dfc1498251b7988d83042023", + "is_secret": false, + "is_verified": false, + "line_number": 380, + "type": "Base64 High Entropy String", + "verified_result": null + } + ], + "populate-studio/payloads/templates/template-reg.json": [ + { + "hashed_secret": "13b8281bb3f1779373722c3bf6ab0a591dc1b56b", + "is_secret": false, + "is_verified": false, + "line_number": 357, + "type": "Base64 High Entropy String", + "verified_result": null + } + ], + "populate-studio/payloads/templates/template-seg.json": [ + { + "hashed_secret": "74fffb127df2531fd89a23729be6acd95ae1eadf", + "is_secret": false, + "is_verified": false, + "line_number": 374, + "type": "Base64 High Entropy String", + "verified_result": null + } + ], + "populate-studio/payloads/templates/template-terramind_seg.json": [ + { + "hashed_secret": "955a175cdec0a9cecd700bcd733f58c948d0b5dd", + "is_secret": false, + "is_verified": false, + "line_number": 373, + "type": "Base64 High Entropy String", + "verified_result": null + } + ], + "populate-studio/payloads/templates/template-timm_convnext_seg.json": [ + { + "hashed_secret": "0ea380996f981df678a1e0935c9bff4e980952d1", + "is_secret": false, + "is_verified": false, + "line_number": 373, + "type": "Base64 High Entropy String", + "verified_result": null + } + ], + "populate-studio/payloads/templates/template-timm_resnet_seg.json": [ + { + "hashed_secret": "9cca9058e26a37baa4739971ccdd7af65ebfaa6f", + "is_secret": false, + "is_verified": false, + "line_number": 373, + "type": "Base64 High Entropy String", + "verified_result": null + } + ], + "tests/api-data/03-ft-templates.json": [ + { + "hashed_secret": "74fffb127df2531fd89a23729be6acd95ae1eadf", + "is_secret": false, + "is_verified": false, + "line_number": 5, + "type": "Base64 High Entropy String", + "verified_result": null + } + ], + "workshop/docs/introduction/key-concepts.md": [ + { + "hashed_secret": "11fa7c37d697f30e6aee828b4426a10f83ab2380", + "is_secret": false, + "is_verified": false, + "line_number": 391, + "type": "Secret Keyword", + "verified_result": null + } + ], + "workshop/docs/introduction/what-is-geospatial-studio.md": [ + { + "hashed_secret": "07689bd0b0346f445a287f64d2a889a4e7323bbe", + "is_secret": false, + "is_verified": false, + "line_number": 89, + "type": "Secret Keyword", + "verified_result": null + } + ], + "workshop/docs/notebooks/README.md": [ + { + "hashed_secret": "19da1f592a715038d9b178bf19c4d35214ab6dc2", + "is_secret": false, + "is_verified": false, + "line_number": 113, + "type": "Base64 High Entropy String", + "verified_result": null + }, + { + "hashed_secret": "f17121cb648a03059f7c6eaf6b181aa989f18704", + "is_secret": false, + "is_verified": false, + "line_number": 113, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "294e91ac28a946e8ff176b79c04ce2ca2b6dc36e", + "is_secret": false, + "is_verified": false, + "line_number": 119, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "95327cc98a8227f4518b53055c55c43db4258178", + "is_secret": false, + "is_verified": false, + "line_number": 119, + "type": "Base64 High Entropy String", + "verified_result": null + } + ], + "workshop/docs/notebooks/lab1-getting-started.ipynb": [ + { + "hashed_secret": "2ace62c1befa19e3ea37dd52be9f6d508c5163e6", + "is_secret": false, + "is_verified": false, + "line_number": 214, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "9941421dba5e7ef4e0ddd04e833896ce776a3923", + "is_secret": false, + "is_verified": false, + "line_number": 227, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "359c324f20253debdfc480c2e059015286394e9b", + "is_secret": false, + "is_verified": false, + "line_number": 415, + "type": "Secret Keyword", + "verified_result": null + } + ], + "workshop/docs/prework/cluster-deployment.md": [ + { + "hashed_secret": "bc2f74c22f98f7b6ffbc2f67453dbfa99bce9a32", + "is_secret": false, + "is_verified": false, + "line_number": 271, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "5d639eb72ee0f6161372b8030ed37f3c3cf4ce40", + "is_secret": false, + "is_verified": false, + "line_number": 432, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "a6778f1880744bd1a342a8e3789135412d8f9da2", + "is_secret": false, + "is_verified": false, + "line_number": 469, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "94c6d635704972aa5c960ddfe83f8d45ab8fbb40", + "is_secret": false, + "is_verified": false, + "line_number": 513, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "0028bd6f7855daf2a4985f9317f1d1109b8cae5f", + "is_secret": false, + "is_verified": false, + "line_number": 539, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "c303df00cd0a72b21c62900b758b06fc541664ce", + "is_secret": false, + "is_verified": false, + "line_number": 558, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "1c58bd92003bbaa0538e249fff6ee19a270dec5f", + "is_secret": false, + "is_verified": false, + "line_number": 631, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "f0a5d38db493f6e89d7ab252ed36258bcddefafb", + "is_secret": false, + "is_verified": false, + "line_number": 643, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "b04b60c63997e49e6d5d0842df240a3754470b7d", + "is_secret": false, + "is_verified": false, + "line_number": 913, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "5a9474bc6ecf315e8315a172b47ef5e1439f81ca", + "is_secret": false, + "is_verified": false, + "line_number": 1059, + "type": "Secret Keyword", + "verified_result": null + } + ], + "workshop/docs/prework/local-deployment.md": [ + { + "hashed_secret": "4d9bfd0e8de7017b695cb5991006816c106a3205", + "is_secret": false, + "is_verified": false, + "line_number": 224, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "236a00044010f54e7d9d32fabe715750c4fc55ba", + "is_secret": false, + "is_verified": false, + "line_number": 226, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "7cd974289f88ebb8cbd99a7b1abc8daee0c28389", + "is_secret": false, + "is_verified": false, + "line_number": 228, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "4cbb11fd97e365dc0c1eab9aba3a60ac28b9f10a", + "is_secret": false, + "is_verified": false, + "line_number": 229, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "f0a5d38db493f6e89d7ab252ed36258bcddefafb", + "is_secret": false, + "is_verified": false, + "line_number": 254, + "type": "Secret Keyword", + "verified_result": null + } + ], + "workshop/docs/prework/verification.md": [ + { + "hashed_secret": "4d9bfd0e8de7017b695cb5991006816c106a3205", + "is_secret": false, + "is_verified": false, + "line_number": 57, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "f0a5d38db493f6e89d7ab252ed36258bcddefafb", + "is_secret": false, + "is_verified": false, + "line_number": 100, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "05672043f03564fec5403ccfc5a42d2dfc4b5300", + "is_secret": false, + "is_verified": false, + "line_number": 108, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "236a00044010f54e7d9d32fabe715750c4fc55ba", + "is_secret": false, + "is_verified": false, + "line_number": 125, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "4cbb11fd97e365dc0c1eab9aba3a60ac28b9f10a", + "is_secret": false, + "is_verified": false, + "line_number": 139, + "type": "Secret Keyword", + "verified_result": null + } + ], + "workshop/docs/resources/troubleshooting.md": [ + { + "hashed_secret": "a8253456364f1bfc7da7ae4a1db5b45d106317a5", + "is_secret": false, + "is_verified": false, + "line_number": 247, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "6d9c68c603e465077bdd49c62347fe54717f83a3", + "is_secret": false, + "is_verified": false, + "line_number": 469, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "90e243a59a66e7e7019b9b8f90128b484d8271e6", + "is_secret": false, + "is_verified": false, + "line_number": 1040, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "6ae774b1f2afa1c184255ec69cc57182e95e666b", + "is_secret": false, + "is_verified": false, + "line_number": 1105, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "bc2f74c22f98f7b6ffbc2f67453dbfa99bce9a32", + "is_secret": false, + "is_verified": false, + "line_number": 1140, + "type": "Secret Keyword", "verified_result": null } ] diff --git a/deploy_studio_k8s.sh b/deploy_studio_k8s.sh index dd40c5c..0cfcb3a 100755 --- a/deploy_studio_k8s.sh +++ b/deploy_studio_k8s.sh @@ -46,35 +46,71 @@ export CLUSTER_NODE_NAME=$cluster_node_name kubectl label nodes ${CLUSTER_NODE_NAME} topology.kubernetes.io/region=us-east-1 topology.kubernetes.io/zone=us-east-1a echo "***********************************************************************************" -echo "-------------------------- Configure storage classes -----------------------------" -echo "-----------------------------------------------------------------------------------" -echo "----------- Verify the available in-cluster storage classes in your cluster -------" +echo "---------------------- Configure Storage Mode -----------------------------------" echo "-----------------------------------------------------------------------------------" echo "***********************************************************************************" -echo "************************ You will enter the following ***************************" -echo "------------------------ NON_COS_STORAGE_CLASS -----------------------------------" +echo "Select the storage mode for your deployment:" +echo " - cloud-object-storage: Use Cloud Object Storage (production) [DEFAULT]" +echo " - cluster-block-storage: Use in-cluster dynamic provisioning" +echo " - local-hostpath: Use local host directories (development/testing)" echo "***********************************************************************************" -in_cluster_storage_class_options="Default User-Supplied" -typeset in_cluster_storage_class_type -get_menu_selection \ -"Select a storage class for your cluster. You can use the default 'standard' class or provide a custom one." \ -in_cluster_storage_class_type \ -"$in_cluster_storage_class_options" +storage_mode_options="cloud-object-storage cluster-block-storage local-hostpath" +typeset storage_mode -if [[ "$in_cluster_storage_class_type" == "Default" ]]; then - export NON_COS_STORAGE_CLASS="standard" +get_menu_selection \ +"Select storage mode for your deployment:" \ +storage_mode \ +"$storage_mode_options" + +export STORAGE_MODE=$storage_mode +echo "STORAGE_MODE selected: **$STORAGE_MODE**" + +# Update env.sh with storage mode +sed -i -e "s/export STORAGE_MODE=.*/export STORAGE_MODE=${STORAGE_MODE}/g" workspace/${DEPLOYMENT_ENV}/env/env.sh + +if [[ "$STORAGE_MODE" == "cloud-object-storage" ]] || [[ "$STORAGE_MODE" == "cluster-block-storage" ]]; then + echo "***********************************************************************************" + echo "-------------------------- Configure storage classes -----------------------------" + echo "-----------------------------------------------------------------------------------" + + if [[ "$STORAGE_MODE" == "cloud-object-storage" ]]; then + export COS_STORAGE_CLASS="cos-s3-csi-s3fs-sc" + echo "COS_STORAGE_CLASS selected: **$COS_STORAGE_CLASS**" + fi + + # Set NON_COS_STORAGE_CLASS for both COS and cluster-block-storage + echo "----------- Verify the available in-cluster storage classes in your cluster -------" + echo "***********************************************************************************" + echo "************************ You will enter the following ***************************" + echo "------------------------ NON_COS_STORAGE_CLASS -----------------------------------" + echo "***********************************************************************************" + in_cluster_storage_class_options="Default User-Supplied" + typeset in_cluster_storage_class_type + + get_menu_selection \ + "Select a storage class for your cluster. You can use the default 'standard' class or provide a custom one." \ + in_cluster_storage_class_type \ + "$in_cluster_storage_class_options" + + if [[ "$in_cluster_storage_class_type" == "Default" ]]; then + export NON_COS_STORAGE_CLASS="standard" + else + typeset user_non_cos_storage_class + get_user_input "Enter NON_COS_STORAGE_CLASS: " user_non_cos_storage_class + echo "NON_COS_STORAGE_CLASS accepted: **$user_non_cos_storage_class**" + export NON_COS_STORAGE_CLASS=$user_non_cos_storage_class + fi + + # Update env.sh + sed -i -e "s/export COS_STORAGE_CLASS=.*/export COS_STORAGE_CLASS=${COS_STORAGE_CLASS:-cos-s3-csi-s3fs-sc}/g" workspace/${DEPLOYMENT_ENV}/env/env.sh + sed -i -e "s/export NON_COS_STORAGE_CLASS=.*/export NON_COS_STORAGE_CLASS=${NON_COS_STORAGE_CLASS:-standard}/g" workspace/${DEPLOYMENT_ENV}/env/env.sh else - typeset user_non_cos_storage_class - get_user_input "Enter NON_COS_STORAGE_CLASS: " user_non_cos_storage_class - echo "NON_COS_STORAGE_CLASS accepted: **$user_non_cos_storage_class**" - export NON_COS_STORAGE_CLASS=$user_non_cos_storage_class + echo "Using local-hostpath storage mode - no storage class configuration needed" + sed -i -e "s/export COS_STORAGE_CLASS=.*/export COS_STORAGE_CLASS=manual/g" workspace/${DEPLOYMENT_ENV}/env/env.sh + sed -i -e "s/export NON_COS_STORAGE_CLASS=.*/export NON_COS_STORAGE_CLASS=manual/g" workspace/${DEPLOYMENT_ENV}/env/env.sh fi -## Setup storage class for minio and default in cluster storage class -sed -i -e "s/export COS_STORAGE_CLASS=.*/export COS_STORAGE_CLASS=cos-s3-csi-s3fs-sc/g" workspace/${DEPLOYMENT_ENV}/env/env.sh -sed -i -e "s/export NON_COS_STORAGE_CLASS=.*/export NON_COS_STORAGE_CLASS=${NON_COS_STORAGE_CLASS}/g" workspace/${DEPLOYMENT_ENV}/env/env.sh - echo "----------------------------------------------------------------------" echo "---------------------- Deploying Minio -----------------------------" echo "----------------------------------------------------------------------" @@ -143,7 +179,7 @@ kubectl_wait_with_retry $KUBECTL_WAIT_RETRY_ATTEMPTS $KUBECTL_WAIT_RETRY_DELAY - # export POSTGRES_PASSWORD=$(kubectl get secret --namespace ${OC_PROJECT} postgresql -o jsonpath="{.data.postgres-password}" | base64 -d) -kubectl port-forward --namespace ${OC_PROJECT} svc/postgresql 54320:5432 >> studio-pf.log 2>&1 & +kubectl port-forward --namespace ${OC_PROJECT} svc/postgresql 54322:5432 >> studio-pf.log 2>&1 & sleep 5 # Update .env with the Postgres details for local connection @@ -151,6 +187,7 @@ sed -i -e "s/pg_username=.*/pg_username=postgres/g" workspace/${DEPLOYMENT_ENV}/ sed -i -e "s/pg_password=.*/pg_password=${POSTGRES_PASSWORD}/g" workspace/${DEPLOYMENT_ENV}/env/.env sed -i -e "s/pg_uri=.*/pg_uri=127.0.0.1/g" workspace/${DEPLOYMENT_ENV}/env/.env sed -i -e "s/pg_port=.*/pg_port=5432/g" workspace/${DEPLOYMENT_ENV}/env/.env +sed -i -e "s/pg_forwarded_port=.*/pg_forwarded_port=54322/g" workspace/${DEPLOYMENT_ENV}/env/.env sed -i -e "s/pg_original_db_name=.*/pg_original_db_name='postgres'/g" workspace/${DEPLOYMENT_ENV}/env/.env python deployment-scripts/create_studio_dbs.py --env-path workspace/${DEPLOYMENT_ENV}/env/.env diff --git a/deploy_studio_lima.sh b/deploy_studio_lima.sh index 0834da3..caf865f 100755 --- a/deploy_studio_lima.sh +++ b/deploy_studio_lima.sh @@ -87,6 +87,18 @@ sed -i -e "s/secret_access_key=.*/secret_access_key=minioadmin/g" workspace/${DE sed -i -e "s|endpoint=.*|endpoint=https://localhost:9000|g" workspace/${DEPLOYMENT_ENV}/env/.env sed -i -e "s/region=.*/region=us-east-1/g" workspace/${DEPLOYMENT_ENV}/env/.env +echo "***********************************************************************************" +echo "---------------------- Configure Storage Mode -----------------------------------" +echo "***********************************************************************************" +echo "Lima deployment uses cloud-object-storage mode with MinIO (default)" +echo "***********************************************************************************" + +export STORAGE_MODE="cloud-object-storage" +echo "STORAGE_MODE set to: **$STORAGE_MODE**" + +# Update env.sh with storage mode +sed -i -e "s/export STORAGE_MODE=.*/export STORAGE_MODE=${STORAGE_MODE}/g" workspace/${DEPLOYMENT_ENV}/env/env.sh + ## Setup storage class for minio and default in cluster storage class sed -i -e "s/export COS_STORAGE_CLASS=.*/export COS_STORAGE_CLASS=cos-s3-csi-s3fs-sc/g" workspace/${DEPLOYMENT_ENV}/env/env.sh sed -i -e "s/export NON_COS_STORAGE_CLASS=.*/export NON_COS_STORAGE_CLASS=local-path/g" workspace/${DEPLOYMENT_ENV}/env/env.sh diff --git a/deploy_studio_ocp.sh b/deploy_studio_ocp.sh index 6f32851..f9087e4 100755 --- a/deploy_studio_ocp.sh +++ b/deploy_studio_ocp.sh @@ -260,45 +260,88 @@ EOF fi echo "***********************************************************************************" - echo "----------------------- Configure s3 storage classes -----------------------------" + echo "---------------------- Configure Storage Mode -----------------------------------" echo "-----------------------------------------------------------------------------------" - echo "---------------- Verify the available storage classes in your cluster -------------" - echo "-----------------------------------------------------------------------------------" - echo "---------- You should already have setup the cloud object storage drivers ---------" - echo "-- See: https://cloud.ibm.com/docs/openshift?topic=openshift-storage_cos_install --" echo "***********************************************************************************" - echo "************************ You will enter the following ***************************" - echo "-------------------------- COS_STORAGE_CLASS -------------------------------------" - echo "------------------------ NON_COS_STORAGE_CLASS ---------------------------------" + echo "Select the storage mode for your deployment:" + echo " - cloud-object-storage: Use Cloud Object Storage (production) [DEFAULT]" + echo " - cluster-block-storage: Use in-cluster dynamic provisioning" + echo " - local-hostpath: Use local host directories (development/testing)" echo "***********************************************************************************" - while true; do - printf "%s " "Press enter to continue" - read ans + storage_mode_options="cloud-object-storage cluster-block-storage local-hostpath" + typeset storage_mode - typeset user_cos_storage_class - get_user_input "Enter COS_STORAGE_CLASS: " user_cos_storage_class - echo "COS_STORAGE_CLASS accepted: **$user_cos_storage_class**" - export COS_STORAGE_CLASS=$user_cos_storage_class + get_menu_selection \ + "Select storage mode for your deployment:" \ + storage_mode \ + "$storage_mode_options" - typeset user_non_cos_storage_class - get_user_input "Enter NON_COS_STORAGE_CLASS: " user_non_cos_storage_class - echo "NON_COS_STORAGE_CLASS accepted: **$user_non_cos_storage_class**" - export NON_COS_STORAGE_CLASS=$user_non_cos_storage_class + export STORAGE_MODE=$storage_mode + echo "STORAGE_MODE selected: **$STORAGE_MODE**" - sed -i -e "s/export COS_STORAGE_CLASS=.*/export COS_STORAGE_CLASS=${COS_STORAGE_CLASS}/g" workspace/${DEPLOYMENT_ENV}/env/env.sh - sed -i -e "s/export NON_COS_STORAGE_CLASS=.*/export NON_COS_STORAGE_CLASS=${NON_COS_STORAGE_CLASS}/g" workspace/${DEPLOYMENT_ENV}/env/env.sh + # Update env.sh with storage mode + sed -i -e "s/export STORAGE_MODE=.*/export STORAGE_MODE=${STORAGE_MODE}/g" workspace/${DEPLOYMENT_ENV}/env/env.sh - python deployment-scripts/validate-env-files.py \ - --env-file workspace/${DEPLOYMENT_ENV}/env/.env \ - --env-variables "" \ - --env-sh-file workspace/${DEPLOYMENT_ENV}/env/env.sh \ - --env-sh-variables "COS_STORAGE_CLASS,NON_COS_STORAGE_CLASS" + if [[ "$STORAGE_MODE" == "cloud-object-storage" ]] || [[ "$STORAGE_MODE" == "cluster-block-storage" ]]; then + echo "***********************************************************************************" + echo "----------------------- Configure s3 storage classes -----------------------------" + echo "-----------------------------------------------------------------------------------" + echo "---------------- Verify the available storage classes in your cluster -------------" + echo "-----------------------------------------------------------------------------------" - if [ $? -eq 0 ]; then - break + if [[ "$STORAGE_MODE" == "cloud-object-storage" ]]; then + echo "---------- You should already have setup the cloud object storage drivers ---------" + echo "-- See: https://cloud.ibm.com/docs/openshift?topic=openshift-storage_cos_install --" + fi + + echo "***********************************************************************************" + echo "************************ You will enter the following ***************************" + + if [[ "$STORAGE_MODE" == "cloud-object-storage" ]]; then + echo "-------------------------- COS_STORAGE_CLASS -------------------------------------" fi - done + + if [[ "$STORAGE_MODE" == "cluster-block-storage" ]]; then + echo "------------------------ NON_COS_STORAGE_CLASS -----------------------------------" + fi + + echo "***********************************************************************************" + + while true; do + printf "%s " "Press enter to continue" + read ans + + if [[ "$STORAGE_MODE" == "cloud-object-storage" ]]; then + typeset user_cos_storage_class + get_user_input "Enter COS_STORAGE_CLASS (cos-s3-csi-s3fs-sc or ibmc-s3fs-cos): " user_cos_storage_class + echo "COS_STORAGE_CLASS accepted: **$user_cos_storage_class**" + export COS_STORAGE_CLASS=$user_cos_storage_class + fi + + typeset user_non_cos_storage_class + get_user_input "Enter NON_COS_STORAGE_CLASS: " user_non_cos_storage_class + echo "NON_COS_STORAGE_CLASS accepted: **$user_non_cos_storage_class**" + export NON_COS_STORAGE_CLASS=$user_non_cos_storage_class + + sed -i -e "s/export COS_STORAGE_CLASS=.*/export COS_STORAGE_CLASS=${COS_STORAGE_CLASS:-ibmc-s3fs-cos}/g" workspace/${DEPLOYMENT_ENV}/env/env.sh + sed -i -e "s/export NON_COS_STORAGE_CLASS=.*/export NON_COS_STORAGE_CLASS=${NON_COS_STORAGE_CLASS:-standard}/g" workspace/${DEPLOYMENT_ENV}/env/env.sh + + python deployment-scripts/validate-env-files.py \ + --env-file workspace/${DEPLOYMENT_ENV}/env/.env \ + --env-variables "" \ + --env-sh-file workspace/${DEPLOYMENT_ENV}/env/env.sh \ + --env-sh-variables "COS_STORAGE_CLASS,NON_COS_STORAGE_CLASS" + + if [ $? -eq 0 ]; then + break + fi + done + else + echo "Using local-hostpath storage mode - no storage class configuration needed" + sed -i -e "s/export COS_STORAGE_CLASS=.*/export COS_STORAGE_CLASS=manual/g" workspace/${DEPLOYMENT_ENV}/env/env.sh + sed -i -e "s/export NON_COS_STORAGE_CLASS=.*/export NON_COS_STORAGE_CLASS=manual/g" workspace/${DEPLOYMENT_ENV}/env/env.sh + fi cloud_object_storage_type_options="Cluster-deployment Cloud-managed-instance" typeset cloud_object_storage_type @@ -457,11 +500,15 @@ EOF export POSTGRES_PASSWORD=devPostgresql123 - ./deployment-scripts/install-postgres.sh UPDATE_STORAGE DISABLE_PV DO_NOT_SET_SCC + if [[ "$DEPLOYMENT_ENV" == "crc" ]]; then + ./deployment-scripts/install-postgres.sh UPDATE_STORAGE DISABLE_PV + else + ./deployment-scripts/install-postgres.sh UPDATE_STORAGE DISABLE_PV DO_NOT_SET_SCC + fi kubectl_wait_with_retry $KUBECTL_WAIT_RETRY_ATTEMPTS $KUBECTL_WAIT_RETRY_DELAY --for=condition=ready pod/postgresql-0 -n ${OC_PROJECT} --timeout=300s - kubectl port-forward --namespace ${OC_PROJECT} svc/postgresql 54320:5432 & + kubectl port-forward --namespace ${OC_PROJECT} svc/postgresql 54321:5432 & sleep 5 # Update .env with the Postgres details for local connection @@ -469,10 +516,11 @@ EOF sed -i -e "s/pg_password=.*/pg_password=${POSTGRES_PASSWORD}/g" workspace/${DEPLOYMENT_ENV}/env/.env sed -i -e "s/pg_uri=.*/pg_uri=127.0.0.1/g" workspace/${DEPLOYMENT_ENV}/env/.env sed -i -e "s/pg_port=.*/pg_port=5432/g" workspace/${DEPLOYMENT_ENV}/env/.env + sed -i -e "s/pg_forwarded_port=.*/pg_forwarded_port=54321/g" workspace/${DEPLOYMENT_ENV}/env/.env sed -i -e "s/pg_original_db_name=.*/pg_original_db_name='postgres'/g" workspace/${DEPLOYMENT_ENV}/env/.env python deployment-scripts/create_studio_dbs.py --env-path workspace/${DEPLOYMENT_ENV}/env/.env - + # After creating dbs; reset pg uri and port sed -i -e "s/pg_uri=.*/pg_uri=postgresql.${OC_PROJECT}.svc.cluster.local/g" workspace/${DEPLOYMENT_ENV}/env/.env else echo "**********************************************************************" diff --git a/deployment-scripts/template/env.template.sh b/deployment-scripts/template/env.template.sh index b7a27ee..7c6d822 100644 --- a/deployment-scripts/template/env.template.sh +++ b/deployment-scripts/template/env.template.sh @@ -19,6 +19,7 @@ export PG_VERSION=18.2.0 export CLUSTER_URL= # Storage +export STORAGE_MODE=cloud-object-storage export SHARE_PIPELINE_PVC=false export COS_STORAGE_CLASS= export NON_COS_STORAGE_CLASS= diff --git a/deployment-scripts/values-file-generate.sh b/deployment-scripts/values-file-generate.sh index 1a0a8ed..43a2d8f 100755 --- a/deployment-scripts/values-file-generate.sh +++ b/deployment-scripts/values-file-generate.sh @@ -28,6 +28,7 @@ do -e "s|OBSERVABILITY_OTLP_TRACES_ENDPOINT|${OBSERVABILITY_OTLP_TRACES_ENDPOINT}|" \ -e "s|BUCKET_FINE_TUNING_MODELS|${BUCKET_FINE_TUNING_MODELS}|" \ -e "s|BUCKET_INFERENCE_AUXDATA|${BUCKET_INFERENCE_AUXDATA}|" \ + -e "s|STORAGE_MODE|${STORAGE_MODE}|" \ -e "s|NON_COS_STORAGE_CLASS|${NON_COS_STORAGE_CLASS}|" \ -e "s|BUCKET_TEMP_UPLOAD|${BUCKET_TEMP_UPLOAD}|" \ -e "s|BUCKET_INFERENCE|${BUCKET_INFERENCE}|" \ diff --git a/geospatial-studio/charts/geospatial-studio-pipelines/templates/pvc.yaml b/geospatial-studio/charts/geospatial-studio-pipelines/templates/pvc.yaml index b51afd5..97f75cc 100644 --- a/geospatial-studio/charts/geospatial-studio-pipelines/templates/pvc.yaml +++ b/geospatial-studio/charts/geospatial-studio-pipelines/templates/pvc.yaml @@ -2,7 +2,10 @@ # SPDX-License-Identifier: Apache-2.0 -{{- if ne .Values.environment "local" }} +{{- if eq .Values.global.storage.storageMode "cloud-object-storage" }} +# handles both ibmc-s3fs-cos and ibmc-s3fs-cos-perf +{{- if contains "ibmc-s3fs-cos" .Values.objectStorage.cos_storage_class }} +--- kind: PersistentVolumeClaim apiVersion: v1 metadata: @@ -21,10 +24,11 @@ spec: - ReadWriteMany resources: requests: - storage: 200Gi + storage: 50Gi storageClassName: {{ .Values.objectStorage.cos_storage_class }} # kubectl get sc for other COS Storage Class Options. --- {{- range $v := .Values.processors }}{{ if hasKey $v "extra_volumes"}}{{- range $e := $v.extra_volumes }}{{ if hasKey $e "bucket_name" }}{{ if $e.create }} +--- kind: PersistentVolumeClaim apiVersion: v1 metadata: @@ -43,7 +47,174 @@ spec: - ReadWriteMany resources: requests: - storage: 200Gi + storage: 50Gi storageClassName: {{ $.Values.objectStorage.cos_storage_class }} # kubectl get sc for other COS Storage Class Options. --- {{ end }}{{- end }}{{ end }}{{ end }}{{- end }} +{{- else if eq .Values.objectStorage.cos_storage_class "cos-s3-csi-s3fs-sc" }} +--- +apiVersion: v1 +kind: Secret +type: cos-s3-csi-driver +metadata: + name: pipelines-inference-cos-s3fs-secret + namespace: {{ .Values.namespace }} +stringData: + uid: "10001" + bucketName: {{ .Values.objectStorage.buckets.inference }} + cosEndpoint: {{ .Values.objectStorage.endpoint }} + accessKey: {{ .Values.objectStorage.access_key }} + secretKey: {{ .Values.objectStorage.secret_key }} + mountOptions: | + dbglevel=info + ssl_verify_hostname=0 + no_check_certificate +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.inference_volume.pvc_name }} + namespace: {{ .Values.namespace }} + annotations: + cos.csi.driver/secret: "pipelines-inference-cos-s3fs-secret" + cos.csi.driver/secret-namespace: {{ .Values.namespace }} +spec: + storageClassName: {{ .Values.objectStorage.cos_storage_class }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +{{- range $v := .Values.processors }}{{ if hasKey $v "extra_volumes"}}{{- range $e := $v.extra_volumes }}{{ if hasKey $e "bucket_name" }}{{ if $e.create }} +--- +apiVersion: v1 +kind: Secret +type: cos-s3-csi-driver +metadata: + name: {{ $e.claimName }}-cos-s3fs-secret + namespace: {{ $.Values.namespace }} +stringData: + uid: "10001" + bucketName: {{ $e.bucket_name }} + cosEndpoint: {{ $.Values.objectStorage.endpoint }} + accessKey: {{ $.Values.objectStorage.access_key }} + secretKey: {{ $.Values.objectStorage.secret_key }} + mountOptions: | + dbglevel=info + ssl_verify_hostname=0 + no_check_certificate +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ $e.claimName }} + namespace: {{ $.Values.namespace }} + annotations: + cos.csi.driver/secret: "{{ $e.claimName }}-cos-s3fs-secret" + cos.csi.driver/secret-namespace: {{ $.Values.namespace }} +spec: + storageClassName: {{ $.Values.objectStorage.cos_storage_class }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +--- {{ end }}{{- end }}{{ end }}{{ end }}{{- end }} +{{- end }} + +{{- else if eq .Values.global.storage.storageMode "cluster-block-storage" }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.inference_volume.pvc_name }} # Enter the name of the PVC. + namespace: {{ .Values.global.namespace }} +spec: + storageClassName: {{ .Values.non_cos_storage_class }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +{{- range $v := .Values.processors }}{{ if hasKey $v "extra_volumes"}}{{- range $e := $v.extra_volumes }}{{ if hasKey $e "bucket_name" }}{{ if $e.create }} +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ $e.claimName }} # Enter the name of the PVC. + namespace: {{ $.Values.namespace }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: {{ $.Values.non_cos_storage_class }} # kubectl get sc for other COS Storage Class Options. +--- {{ end }}{{- end }}{{ end }}{{ end }}{{- end }} + +{{- else if eq .Values.global.storage.storageMode "local-hostpath" }} +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pipelines-inference-pv-volume + labels: + type: local +spec: + storageClassName: manual + capacity: + storage: 20Gi + accessModes: + - ReadWriteMany + claimRef: + name: {{ .Values.inference_volume.pvc_name }} + namespace: {{ .Values.namespace }} + hostPath: + path: "/data/pipelines-inference-pvc" +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.inference_volume.pvc_name }} + namespace: {{ .Values.namespace }} +spec: + storageClassName: manual + accessModes: + - ReadWriteMany + resources: + requests: + storage: 20Gi +{{- range $v := .Values.processors }}{{ if hasKey $v "extra_volumes"}}{{- range $e := $v.extra_volumes }}{{ if hasKey $e "bucket_name" }}{{ if $e.create }} +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: {{ $e.claimName }}-pv-volume + labels: + type: local +spec: + storageClassName: manual + capacity: + storage: 10Gi + accessModes: + - ReadWriteMany + claimRef: + name: {{ $e.claimName }} + namespace: {{ $.Values.namespace }} + hostPath: + path: "/data/{{ $e.claimName }}" +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ $e.claimName }} + namespace: {{ $.Values.namespace }} +spec: + storageClassName: manual + accessModes: + - ReadWriteMany + resources: + requests: + storage: 10Gi +--- {{ end }}{{- end }}{{ end }}{{ end }}{{- end }} + {{- end }} diff --git a/geospatial-studio/charts/geospatial-studio-pipelines/values.yaml b/geospatial-studio/charts/geospatial-studio-pipelines/values.yaml index e286f22..01b05fa 100644 --- a/geospatial-studio/charts/geospatial-studio-pipelines/values.yaml +++ b/geospatial-studio/charts/geospatial-studio-pipelines/values.yaml @@ -97,6 +97,7 @@ objectStorage: inference_auxdata: BUCKET_INFERENCE_AUXDATA generic_python_processor: BUCKET_GENERIC_PYTHON_PROCESSOR cos_storage_class: COS_STORAGE_CLASS +non_cos_storage_class: NON_COS_STORAGE_CLASS serviceAccount: # Specifies serviceaccount to be used @@ -262,4 +263,4 @@ processors: claimName: generic-python-processor-pvc bucket_name: BUCKET_GENERIC_PYTHON_PROCESSOR mountPath: /generic_data - create: false + create: true diff --git a/geospatial-studio/charts/gfm-mlflow/templates/pvc.yaml b/geospatial-studio/charts/gfm-mlflow/templates/pvc.yaml index be3c460..7564612 100644 --- a/geospatial-studio/charts/gfm-mlflow/templates/pvc.yaml +++ b/geospatial-studio/charts/gfm-mlflow/templates/pvc.yaml @@ -3,7 +3,9 @@ {{- if .Values.global.storage.pvc.enabled }} -{{- if and (hasKey .Values.global "environment") (ne .Values.global.environment "local") }} +{{- if eq .Values.global.storage.storageMode "cloud-object-storage" }} +{{- if contains "ibmc-s3fs-cos" .Values.global.objectStorage.cos_storage_class }} +--- apiVersion: v1 kind: PersistentVolumeClaim metadata: @@ -28,8 +30,7 @@ spec: requests: storage: {{ .Values.persistence.gfmMlflowCOSPVC.capacity }} storageClassName: {{ .Values.global.objectStorage.cos_storage_class }} # kubectl get sc for other COS Storage Class Options. -{{- else if contains "cos-s3-csi-s3fs-sc" .Values.global.objectStorage.cos_storage_class }} - +{{- else if eq .Values.global.objectStorage.cos_storage_class "cos-s3-csi-s3fs-sc" }} --- apiVersion: v1 kind: Secret @@ -63,36 +64,24 @@ spec: resources: requests: storage: 10Gi +{{- end }} + +{{- else if eq .Values.global.storage.storageMode "cluster-block-storage" }} --- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.persistence.gfmMlflowCOSPVC.claimName }} # Enter the name of the PVC. + namespace: {{ .Values.global.namespace }} +spec: + storageClassName: {{ .Values.global.non_cos_storage_class }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi -{{- else }} -# kind: PersistentVolumeClaim -# apiVersion: v1 -# metadata: -# name: {{ .Values.persistence.gfmMlflowCOSPVC.claimName }} # Enter the name of the PVC. -# namespace: {{ .Values.global.namespace }} -# spec: -# accessModes: -# - ReadWriteOnce -# resources: -# requests: -# storage: 20Gi -# storageClassName: {{ .Values.global.non_cos_storage_class }} # kubectl get sc for other COS Storage Class Options. -# apiVersion: datashim.io/v1alpha1 -# kind: Dataset -# metadata: -# # name: gfm-mlflow-cos-pvc -# name: {{ .Values.persistence.gfmMlflowCOSPVC.claimName }} # Enter the name of the PVC. -# namespace: {{ .Values.global.namespace }} -# spec: -# local: -# type: "COS" -# accessKeyID: {{ .Values.global.objectStorage.access_key }} -# secretAccessKey: {{ .Values.global.objectStorage.secret_key }} -# endpoint: {{ .Values.global.objectStorage.endpoint }} -# bucket: {{ .Values.global.objectStorage.buckets.mlflow }} -# readonly: "false" -# region: {{ .Values.global.objectStorage.region }} +{{- else if eq .Values.global.storage.storageMode "local-hostpath" }} --- apiVersion: v1 kind: PersistentVolume @@ -103,7 +92,7 @@ metadata: spec: storageClassName: manual capacity: - storage: 60Gi + storage: 10Gi accessModes: - ReadWriteMany claimRef: @@ -123,10 +112,7 @@ spec: - ReadWriteMany resources: requests: - storage: 60Gi + storage: 10Gi --- - - - {{- end }} {{- end }} \ No newline at end of file diff --git a/geospatial-studio/charts/gfm-mlflow/values.yaml b/geospatial-studio/charts/gfm-mlflow/values.yaml index 51855bc..50694dd 100644 --- a/geospatial-studio/charts/gfm-mlflow/values.yaml +++ b/geospatial-studio/charts/gfm-mlflow/values.yaml @@ -79,7 +79,7 @@ resources: # resources, such as Minikube. If you do want to specify resources, uncomment the following # lines, adjust them as necessary, and remove the curly braces after 'resources:'. limits: - cpu: 2 + cpu: 1 memory: 8G requests: cpu: 1 @@ -107,7 +107,7 @@ persistence: gfmMlflowCOSPVC: create: false claimName: gfm-mlflow-cos-pvc - capacity: 200Gi + capacity: 50Gi # customStorageClass: ibmc-s3fs-cos-perf # objectStorage: # endpoint: http://s3.us-south.cloud-object-storage.appdomain.cloud diff --git a/geospatial-studio/charts/gfm-studio-gateway/templates/pvc.yaml b/geospatial-studio/charts/gfm-studio-gateway/templates/pvc.yaml index 1269566..41a7c57 100644 --- a/geospatial-studio/charts/gfm-studio-gateway/templates/pvc.yaml +++ b/geospatial-studio/charts/gfm-studio-gateway/templates/pvc.yaml @@ -3,7 +3,8 @@ {{- if .Values.global.storage.pvc.enabled }} -{{- if and (hasKey .Values.global "environment") (ne .Values.global.environment "local") }} +{{- if eq .Values.global.storage.storageMode "cloud-object-storage" }} +{{- if contains "ibmc-s3fs-cos" .Values.global.objectStorage.cos_storage_class }} --- kind: PersistentVolumeClaim apiVersion: v1 @@ -27,8 +28,7 @@ spec: - ReadWriteMany resources: requests: - storage: 200Gi - # storageClassName: {{ .Values.global.non_cos_storage_class }} # kubectl get sc for other COS Storage Class Options. + storage: 50Gi storageClassName: {{ .Values.global.objectStorage.cos_storage_class }} # kubectl get sc for other COS Storage Class Options. --- @@ -54,8 +54,7 @@ spec: - ReadWriteMany resources: requests: - storage: 200Gi - # storageClassName: {{ .Values.global.non_cos_storage_class }} # kubectl get sc for other COS Storage Class Options. + storage: 50Gi storageClassName: {{ .Values.global.objectStorage.cos_storage_class }} # kubectl get sc for other COS Storage Class Options. --- @@ -81,72 +80,10 @@ spec: - ReadWriteMany resources: requests: - storage: 200Gi - # storageClassName: {{ .Values.global.non_cos_storage_class }} # kubectl get sc for other COS Storage Class Options. + storage: 50Gi storageClassName: {{ .Values.global.objectStorage.cos_storage_class }} # kubectl get sc for other COS Storage Class Options. ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: {{ .Values.mountLocations.generic_python_processor_pvc_name }} # Enter the name of the PVC. - namespace: {{ .Values.global.namespace }} - annotations: - ibm.io/region: {{ .Values.global.objectStorage.region }} - ibm.io/auto-create-bucket: {{ .Values.global.objectStorage.create_bucket | quote }} - ibm.io/auto-delete-bucket: {{ .Values.global.objectStorage.delete_bucket | quote }} - ibm.io/bucket: {{ .Values.global.objectStorage.buckets.generic_python_processor }} # Enter the name (not the ID) of the standard class, regional bucket created in a previous step - # ibm.io/object-path: "pvc2" # Uncomment line and enter valid subdirectory if desired - ibm.io/secret-name: {{ .Values.global.appNames.gateway }}-cos-secret # Enter the name of the OpenShift Secret created in a previous step - ibm.io/quota-limit: "false" # Disable or enable a quota limit for your PVC. To use this annotation you must specify the -set quotaLimit=true option during installation. - ibm.io/endpoint: {{ .Values.global.objectStorage.endpoint }} - ibm.io/tls-cipher-suite: "default" - ibm.io/curl-debug: "true" - ibm.io/add-mount-param: "ssl_verify_hostname=0,no_check_certificate,dbglevel=info" -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 200Gi - # storageClassName: {{ .Values.global.non_cos_storage_class }} # kubectl get sc for other COS Storage Class Options. - storageClassName: {{ .Values.global.objectStorage.cos_storage_class }} # kubectl get sc for other COS Storage Class Options. - -{{- else if contains "cos-s3-csi-s3fs-sc" .Values.global.objectStorage.cos_storage_class }} - ---- -apiVersion: v1 -kind: Secret -type: cos-s3-csi-driver -metadata: - name: gateway-inference-cos-s3fs-secret - namespace: {{ .Values.global.namespace }} -stringData: - uid: "10001" # Provide uid to run as non root user. This must match runAsUser in SecurityContext of pod spec. - bucketName: {{ .Values.global.objectStorage.buckets.inference }} - cosEndpoint: {{ .Values.global.objectStorage.endpoint }} - accessKey: {{ .Values.global.objectStorage.access_key }} - secretKey: {{ .Values.global.objectStorage.secret_key }} - mountOptions: | - dbglevel=info - ssl_verify_hostname=0 - no_check_certificate ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: inference-shared-pvc # Enter the name of the PVC. - namespace: {{ .Values.global.namespace }} - annotations: - cos.csi.driver/secret: "gateway-inference-cos-s3fs-secret" - cos.csi.driver/secret-namespace: {{ .Values.global.namespace }} -spec: - storageClassName: {{ .Values.global.objectStorage.cos_storage_class }} - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi +{{- else if eq .Values.global.objectStorage.cos_storage_class "cos-s3-csi-s3fs-sc" }} --- apiVersion: v1 kind: Secret @@ -246,34 +183,17 @@ spec: resources: requests: storage: 20Gi ---- -apiVersion: v1 -kind: Secret -type: cos-s3-csi-driver -metadata: - name: gateway-auxdata-cos-s3fs-secret - namespace: {{ .Values.global.namespace }} -stringData: - uid: "10001" # Provide uid to run as non root user. This must match runAsUser in SecurityContext of pod spec. - bucketName: {{ .Values.global.objectStorage.buckets.inference_auxdata }} - cosEndpoint: {{ .Values.global.objectStorage.endpoint }} - accessKey: {{ .Values.global.objectStorage.access_key }} - secretKey: {{ .Values.global.objectStorage.secret_key }} - mountOptions: | - dbglevel=info - ssl_verify_hostname=0 - no_check_certificate +{{- end }} + +{{- else if eq .Values.global.storage.storageMode "cluster-block-storage" }} --- apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: {{ .Values.mountLocations.inference_auxdata_pvc_name }} # Enter the name of the PVC. + name: {{ .Values.mountLocations.files_pvc_name }} # Enter the name of the PVC. namespace: {{ .Values.global.namespace }} - annotations: - cos.csi.driver/secret: "gateway-auxdata-cos-s3fs-secret" - cos.csi.driver/secret-namespace: {{ .Values.global.namespace }} spec: - storageClassName: {{ .Values.global.objectStorage.cos_storage_class }} + storageClassName: {{ .Values.global.non_cos_storage_class }} accessModes: - ReadWriteOnce resources: @@ -281,108 +201,31 @@ spec: storage: 10Gi --- apiVersion: v1 -kind: Secret -type: cos-s3-csi-driver -metadata: - name: generic-python-processor-cos-s3fs-secret - namespace: {{ .Values.global.namespace }} -stringData: - uid: "10001" # Provide uid to run as non root user. This must match runAsUser in SecurityContext of pod spec. - bucketName: {{ .Values.global.objectStorage.buckets.generic_python_processor }} - cosEndpoint: {{ .Values.global.objectStorage.endpoint }} - accessKey: {{ .Values.global.objectStorage.access_key }} - secretKey: {{ .Values.global.objectStorage.secret_key }} - mountOptions: | - dbglevel=info - ssl_verify_hostname=0 - no_check_certificate ---- -apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: {{ .Values.mountLocations.generic_python_processor_pvc_name }} # Enter the name of the PVC. + name: {{ .Values.mountLocations.models_pvc_name }} # Enter the name of the PVC. namespace: {{ .Values.global.namespace }} - annotations: - cos.csi.driver/secret: "generic-python-processor-cos-s3fs-secret" - cos.csi.driver/secret-namespace: {{ .Values.global.namespace }} spec: - storageClassName: {{ .Values.global.objectStorage.cos_storage_class }} + storageClassName: {{ .Values.global.non_cos_storage_class }} accessModes: - ReadWriteOnce resources: requests: storage: 10Gi --- -{{- else }} ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: studio-pv-volume - labels: - type: local -spec: - storageClassName: manual - capacity: - storage: 15Gi - accessModes: - - ReadWriteOnce - hostPath: - path: "/data/studio-inference-pvc" ---- apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: inference-shared-pvc + name: {{ .Values.mountLocations.data_pvc_name }} # Enter the name of the PVC. + namespace: {{ .Values.global.namespace }} spec: - storageClassName: manual + storageClassName: {{ .Values.global.non_cos_storage_class }} accessModes: - ReadWriteOnce resources: requests: - storage: 10Gi -# apiVersion: datashim.io/v1alpha1 -# kind: Dataset -# metadata: -# name: inference-shared-pvc -# namespace: {{ .Values.global.namespace }} -# spec: -# local: -# type: "COS" -# accessKeyID: {{ .Values.global.objectStorage.access_key }} -# secretAccessKey: {{ .Values.global.objectStorage.secret_key }} -# endpoint: {{ .Values.global.objectStorage.endpoint }} -# bucket: {{ .Values.global.objectStorage.buckets.inference }} -# readonly: "false" -# region: {{ .Values.global.objectStorage.region }} ---- -# kind: PersistentVolumeClaim -# apiVersion: v1 -# metadata: -# name: {{ .Values.mountLocations.files_pvc_name }} # Enter the name of the PVC. -# namespace: {{ .Values.global.namespace }} -# spec: -# accessModes: -# - ReadWriteOnce -# resources: -# requests: -# storage: 20Gi -# storageClassName: {{ .Values.global.non_cos_storage_class }} # kubectl get sc for other COS Storage Class Options. -# apiVersion: datashim.io/v1alpha1 -# kind: Dataset -# metadata: -# # name: gfm-ft-files-pvc -# name: {{ .Values.mountLocations.files_pvc_name }} # Enter the name of the PVC. -# namespace: {{ .Values.global.namespace }} -# spec: -# local: -# type: "COS" -# accessKeyID: {{ .Values.global.objectStorage.access_key }} -# secretAccessKey: {{ .Values.global.objectStorage.secret_key }} -# endpoint: {{ .Values.global.objectStorage.endpoint }} -# bucket: {{ .Values.global.objectStorage.buckets.fineTuning }} -# readonly: "false" -# region: {{ .Values.global.objectStorage.region }} + storage: 20Gi +{{- else if eq .Values.global.storage.storageMode "local-hostpath" }} --- apiVersion: v1 kind: PersistentVolume @@ -393,7 +236,7 @@ metadata: spec: storageClassName: manual capacity: - storage: 60Gi + storage: 40Gi accessModes: - ReadWriteMany claimRef: @@ -413,36 +256,7 @@ spec: - ReadWriteMany resources: requests: - storage: 60Gi ---- - -# kind: PersistentVolumeClaim -# apiVersion: v1 -# metadata: -# name: {{ .Values.mountLocations.models_pvc_name }} # Enter the name of the PVC. -# namespace: {{ .Values.global.namespace }} -# spec: -# accessModes: -# - ReadWriteOnce -# resources: -# requests: -# storage: 20Gi -# storageClassName: {{ .Values.global.non_cos_storage_class }} # kubectl get sc for other COS Storage Class Options. -# apiVersion: datashim.io/v1alpha1 -# kind: Dataset -# metadata: -# # name: gfm-ft-models-pvc -# name: {{ .Values.mountLocations.models_pvc_name }} # Enter the name of the PVC. -# namespace: {{ .Values.global.namespace }} -# spec: -# local: -# type: "COS" -# accessKeyID: {{ .Values.global.objectStorage.access_key }} -# secretAccessKey: {{ .Values.global.objectStorage.secret_key }} -# endpoint: {{ .Values.global.objectStorage.endpoint }} -# bucket: {{ .Values.global.objectStorage.buckets.fineTuningModels }} -# readonly: "false" -# region: {{ .Values.global.objectStorage.region }} + storage: 40Gi --- apiVersion: v1 kind: PersistentVolume @@ -453,7 +267,7 @@ metadata: spec: storageClassName: manual capacity: - storage: 60Gi + storage: 40Gi accessModes: - ReadWriteMany claimRef: @@ -473,38 +287,7 @@ spec: - ReadWriteMany resources: requests: - storage: 60Gi ---- - - - -# kind: PersistentVolumeClaim -# apiVersion: v1 -# metadata: -# name: {{ .Values.mountLocations.data_pvc_name }} # Enter the name of the PVC. -# namespace: {{ .Values.global.namespace }} -# spec: -# accessModes: -# - ReadWriteOnce -# resources: -# requests: -# storage: 20Gi -# storageClassName: {{ .Values.global.non_cos_storage_class }} # kubectl get sc for other COS Storage Class Options. -# apiVersion: datashim.io/v1alpha1 -# kind: Dataset -# metadata: -# # name: gfm-ft-data-pvc -# name: {{ .Values.mountLocations.data_pvc_name }} # Enter the name of the PVC. -# namespace: {{ .Values.global.namespace }} -# spec: -# local: -# type: "COS" -# accessKeyID: {{ .Values.global.objectStorage.access_key }} -# secretAccessKey: {{ .Values.global.objectStorage.secret_key }} -# endpoint: {{ .Values.global.objectStorage.endpoint }} -# bucket: {{ .Values.global.objectStorage.buckets.datasetFactory }} -# readonly: "false" -# region: {{ .Values.global.objectStorage.region }} + storage: 40Gi --- apiVersion: v1 kind: PersistentVolume @@ -515,7 +298,7 @@ metadata: spec: storageClassName: manual capacity: - storage: 60Gi + storage: 40Gi accessModes: - ReadWriteMany claimRef: @@ -535,89 +318,7 @@ spec: - ReadWriteMany resources: requests: - storage: 60Gi ---- - - - ---- -# apiVersion: datashim.io/v1alpha1 -# kind: Dataset -# metadata: -# name: {{ .Values.mountLocations.inference_auxdata_pvc_name }} -# namespace: {{ .Values.global.namespace }} -# spec: -# local: -# type: "COS" -# accessKeyID: {{ .Values.global.objectStorage.access_key }} -# secretAccessKey: {{ .Values.global.objectStorage.secret_key }} -# endpoint: {{ .Values.global.objectStorage.endpoint }} -# bucket: {{ .Values.global.objectStorage.buckets.inference_auxdata }} -# readonly: "false" -# region: {{ .Values.global.objectStorage.region }} + storage: 40Gi --- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: inference-auxdata-pv-volume - labels: - type: local -spec: - storageClassName: manual - capacity: - storage: 60Gi - accessModes: - - ReadWriteMany - claimRef: - name: {{ .Values.mountLocations.inference_auxdata_pvc_name }} - namespace: {{ .Values.global.namespace }} - hostPath: - path: "/data/inference-auxdata-pvc" ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ .Values.mountLocations.inference_auxdata_pvc_name }} - namespace: {{ .Values.global.namespace }} -spec: - storageClassName: manual - accessModes: - - ReadWriteMany - resources: - requests: - storage: 60Gi ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: generic-python-processor-pv-volume - labels: - type: local -spec: - storageClassName: manual - capacity: - storage: 60Gi - accessModes: - - ReadWriteMany - claimRef: - name: {{ .Values.mountLocations.generic_python_processor_pvc_name }} - namespace: {{ .Values.global.namespace }} - hostPath: - path: "/generic_data" ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ .Values.mountLocations.generic_python_processor_pvc_name }} - namespace: {{ .Values.global.namespace }} -spec: - storageClassName: manual - accessModes: - - ReadWriteMany - resources: - requests: - storage: 60Gi - - {{- end }} {{- end }} diff --git a/geospatial-studio/values.yaml b/geospatial-studio/values.yaml index 55be9a8..795e4db 100644 --- a/geospatial-studio/values.yaml +++ b/geospatial-studio/values.yaml @@ -60,6 +60,7 @@ global: tlsKeyB64: tlsKeyB64 storage: + storageMode: STORAGE_MODE pvc: enabled: STORAGE_PVC_ENABLED filesystem: @@ -514,4 +515,4 @@ geospatial-studio-pipelines: claimName: generic-python-processor-pvc bucket_name: BUCKET_GENERIC_PYTHON_PROCESSOR mountPath: /generic_data - create: false + create: true