finalized the cos-to-sql sample code and readme instructions

Author: reggeenr

cos-to-sql/.dockerignore

@@ -0,0 +1,5 @@
+.dockerignore
+.gitignore
+build
+Dockerfile
+node_modules

cos-to-sql/Dockerfile

@@ -1,21 +1,21 @@
-FROM registry.access.redhat.com/ubi9/nodejs-22:latest AS build-env
-WORKDIR /app
+# Download dependencies in builder stage
+FROM registry.access.redhat.com/ubi9/nodejs-22:latest AS builder
 
-# Copy job dependency manifests to the container image.
-COPY package.json ./
+COPY --chown=${CNB_USER_ID}:${CNB_GROUP_ID} package.json package-lock.json /app/
+WORKDIR /app
+RUN npm ci --omit=dev
 
-# Install production dependencies.
-RUN npm install --production
 
 # Use a small distroless image for as runtime image
-FROM gcr.io/distroless/nodejs22-debian12
-
-WORKDIR /app
+FROM gcr.io/distroless/nodejs22
 
-# Copy local code to the container image.
-COPY ./src ./
+COPY --chown=1001:0 --from=builder /app/node_modules /app/node_modules
+COPY --chown=1001:0 app.mjs /app
+COPY --chown=1001:0 utils/ /app/utils
+COPY --chown=1001:0 public/ /app/public
 
-# Copy the dependencies and other project related files
-COPY --from=build-env /app ./
+USER 1001:0
+WORKDIR /app
 EXPOSE 8080
+
 CMD ["app.mjs"]

cos-to-sql/Dockerfile.job

@@ -1,4 +1,4 @@
-# IBM Cloud Code Engine - Integrate Cloud Object Storage and PostgreSQL through a job and an event subscription
+# IBM Cloud Code Engine - Integrate Cloud Object Storage and PostgreSQL through a app and an event subscription
 
 This sample demonstrates how to read CSV files hosted on a IBM Cloud Object Storage and save their contents line by line into relational PostgreSQL database.
 
@@ -7,123 +7,52 @@ This sample demonstrates how to read CSV files hosted on a IBM Cloud Object Stor
 Make sure the following [IBM Cloud CLI](https://cloud.ibm.com/docs/cli/reference/ibmcloud?topic=cloud-cli-getting-started) and the following list of plugins are installed
 - `ibmcloud plugin install code-engine`
 - `ibmcloud plugin install cloud-object-storage`
+- `ibmcloud plugin install secrets-manager`
 
 Install `jq`. On MacOS, you can use following [brew formulae](https://formulae.brew.sh/formula/jq) to do a `brew install jq`.
-## CLI Setup
 
-Login to IBM Cloud via the CLI
-```
-ibmcloud login 
-```
+## Setting up all IBM Cloud Service instances
 
-Target the `ca-tor` region:
+* Login to IBM Cloud via the CLI and target the `ca-tor` region:
 ```
 export REGION=ca-tor
-ibmcloud -r $REGION
-```
-
-Create the project:
-```
-ibmcloud code-engine project create -n ce-objectstorage-to-sql
-```
-
-Store the project guid:
-```
-export CE_ID=$(ibmcloud ce project current -o json | jq -r .guid)
-```
-
-Create the job:
-```
-ibmcloud code-engine job create \
-    --name csv-to-sql \
-    --source ./ \
-    --retrylimit 0 \
-    --cpu 0.25 \
-    --memory 0.5G \
-    --wait
+export RESOURCE_GROUP=Default
+ibmcloud login -r ${REGION} -g $RESOURCE_GROUP
 ```
 
-Create the app:
-```
-ibmcloud code-engine app create \
-    --name csv-to-sql-app \
-    --source ./ \
-    --cpu 0.25 \
-    --memory 0.5G \
-    --env COS_REGION=eu-es \
-    --env COS_TRUSTED_PROFILE_NAME=code-engine-cos-access \
-    --env SM_TRUSTED_PROFILE_NAME=code-engine-sm-access \
-    --env SM_SERVICE_URL=https://4e61488a-b76f-4d44-ba0e-ed2489d8a57a.private.eu-es.secrets-manager.appdomain.cloud \
-    --env SM_PG_SECRET_ID=04abb32c-bbe3-a069-e4c4-8899853ef053
+* Create the Code Engine project
 ```
+export CE_INSTANCE_NAME=cos-to-sql--ce
+ibmcloud code-engine project create -n ${CE_INSTANCE_NAME}
 
-Create the COS instance:
-```
-ibmcloud resource service-instance-create csv-to-sql-cos cloud-object-storage standard global
+export CE_INSTANCE_GUID=$(ibmcloud ce project current -o json | jq -r .guid)
+export CE_INSTANCE_ID=$(ibmcloud resource service-instance ${CE_INSTANCE_NAME} --output json | jq -r '.[0] | .id')
 ```
 
-Store the COS CRN:
-```
-export COS_ID=$(ibmcloud resource service-instance csv-to-sql-cos --output json | jq -r '.[0] | .id')
+* Create the COS instance
 ```
+export COS_INSTANCE_NAME=cos-to-sql--cos
+ibmcloud resource service-instance-create ${COS_INSTANCE_NAME} cloud-object-storage standard global
 
-Create an authorization policy to allow the Code Engine project receive events from COS:
-```
-ibmcloud iam authorization-policy-create codeengine cloud-object-storage \
-    "Notifications Manager" \
-    --source-service-instance-id $CE_ID \
-    --target-service-instance-id $COS_ID
+export COS_INSTANCE_ID=$(ibmcloud resource service-instance ${COS_INSTANCE_NAME} --output json | jq -r '.[0] | .id')
 ```
 
-Create a COS bucket:
+* Create a COS bucket
 ```
-ibmcloud cos config crn --crn $COS_ID --force
+ibmcloud cos config crn --crn ${COS_INSTANCE_ID} --force
 ibmcloud cos config auth --method IAM
-ibmcloud cos config region --region $REGION
-ibmcloud cos config endpoint-url --url s3.$REGION.cloud-object-storage.appdomain.cloud
-export BUCKET=$CE_ID-csv-to-sql
+ibmcloud cos config region --region ${REGION}
+ibmcloud cos config endpoint-url --url s3.${REGION}.cloud-object-storage.appdomain.cloud
+export COS_BUCKET_NAME=${CE_INSTANCE_GUID}-csv-to-sql
 ibmcloud cos bucket-create \
     --class smart \
-    --bucket $BUCKET
-```
-
-Creating a trusted profile that grants a Code Engine Job access to your COS bucket
-```
-REGION=eu-es
-RESOURCE_GROUP=Default
-COS_INSTANCE_NAME=csv-to-sql-cos
-COS_BUCKET=$CE_ID-csv-to-sql
-CE_PROJECT_NAME=ce-objectstorage-to-sql
-JOB_NAME=csv-to-sql
-TRUSTED_PROFILE_NAME=code-engine-cos-access
-
-CE_PROJECT_CRN=$(ibmcloud resource service-instance ${CE_PROJECT_NAME} --location ${REGION} -g ${RESOURCE_GROUP} --crn 2>/dev/null | grep ':codeengine:')
-COS_INSTANCE_ID=$(ibmcloud resource service-instance ${COS_INSTANCE_NAME} --crn 2>/dev/null | grep ':cloud-object-storage:')
-
-ibmcloud iam trusted-profile-create ${TRUSTED_PROFILE_NAME}
-ibmcloud iam trusted-profile-link-create ${TRUSTED_PROFILE_NAME} --name ce-job-${JOB_NAME} --cr-type CE --link-crn ${CE_PROJECT_CRN} --link-component-type job --link-component-name ${JOB_NAME}
-ibmcloud iam trusted-profile-policy-create ${TRUSTED_PROFILE_NAME} --roles "Content Reader" --service-name cloud-object-storage --service-instance ${COS_INSTANCE_ID} --resource-type bucket --resource ${COS_BUCKET}
-```
-
-Create the subscription for all COS events:
-```
-ibmcloud ce sub cos create \
-    --name coswatch \
-    --bucket $BUCKET \
-    --destination csv-to-sql \
-    --destination-type job
-
-ibmcloud ce sub cos create \
-    --name coswatch \
-    --bucket $BUCKET \
-    --destination csv-to-sql-app \
-    --destination-type app \
-    --path /cos-to-sql
+    --bucket $COS_BUCKET_NAME
 ```
 
-Create a PostgreSQL service instance:
+* Create the PostgreSQL instance
 ```
-ibmcloud resource service-instance-create csv-to-sql-postgresql databases-for-postgresql standard $REGION --service-endpoints private -p \
+export DB_INSTANCE_NAME=cos-to-sql--pg
+ibmcloud resource service-instance-create $DB_INSTANCE_NAME databases-for-postgresql standard ${REGION} --service-endpoints private -p \
  '{
     "disk_encryption_instance_crn": "none",
     "disk_encryption_key_crn": "none",
@@ -135,90 +64,113 @@ ibmcloud resource service-instance-create csv-to-sql-postgresql databases-for-po
     "service-endpoints": "private",
     "version": "16"
 }'
-```
 
-Create a trusted profile that grants this Code Engine job access to the COS instance 
+export DB_INSTANCE_ID=$(ibmcloud resource service-instance $DB_INSTANCE_NAME --location ${REGION} --output json | jq -r '.[0] | .id')
 ```
-REGION=eu-es
-RESOURCE_GROUP=Default
-COS_INSTANCE_NAME=my-first-cos
-COS_BUCKET=my-first-bucket-31292
-CE_PROJECT_NAME=trusted-profiles-test
-JOB_NAME=list-cos-files
-COS_TRUSTED_PROFILE_NAME=code-engine-cos-access
-SM_TRUSTED_PROFILE_NAME=code-engine-sm-access
-SM_SERVICE_URL=https://4e61488a-b76f-4d44-ba0e-ed2489d8a57a.private.eu-es.secrets-manager.appdomain.cloud
-SM_PG_SECRET_ID=
 
-CE_PROJECT_CRN=$(ibmcloud resource service-instance ${CE_PROJECT_NAME} --location ${REGION} -g ${RESOURCE_GROUP} --crn 2>/dev/null | grep ':codeengine:')
-COS_INSTANCE_ID=$(ibmcloud resource service-instance ${COS_INSTANCE_NAME} --crn 2>/dev/null | grep ':cloud-object-storage:')
+* Create the Secrets Manager instance
+```
+export SM_INSTANCE_NAME=cos-to-sql--sm
+ibmcloud resource service-instance-create $SM_INSTANCE_NAME secrets-manager 7713c3a8-3be8-4a9a-81bb-ee822fcaac3d ${REGION} -p \
+'{
+    "allowed_network": "public-and-private"
+}'
 
-ibmcloud iam trusted-profile-create ${TRUSTED_PROFILE_NAME}
-ibmcloud iam trusted-profile-link-create ${TRUSTED_PROFILE_NAME} --name ce-job-${JOB_NAME} --cr-type CE --link-crn ${CE_PROJECT_CRN} --link-component-type job --link-component-name ${JOB_NAME}
-ibmcloud iam trusted-profile-policy-create ${TRUSTED_PROFILE_NAME} --roles "Content Reader" --service-name cloud-object-storage --service-instance ${COS_INSTANCE_ID} --resource-type bucket --resource ${COS_BUCKET}
+export SM_INSTANCE_ID=$(ibmcloud resource service-instance $SM_INSTANCE_NAME --location ${REGION} --output json | jq -r '.[0] | .id')
+export SM_INSTANCE_GUID=$(ibmcloud resource service-instance $SM_INSTANCE_NAME --location ${REGION} --output json | jq -r '.[0] | .guid')
+export SECRETS_MANAGER_URL=https://$SM_INSTANCE_GUID.${REGION}.secrets-manager.appdomain.cloud
 ```
 
-Update the job by adding a binding to the PostgreSQL instance:
+* Create a S2S policy "Key Manager" between SM and the DB
 ```
-ibmcloud code-engine job update \
-    --name csv-to-sql \
-    --trusted-profiles-enabled true \
-    --env COS_REGION=${REGION} \
-    --env COS_TRUSTED_PROFILE_NAME=${COS_TRUSTED_PROFILE_NAME} \
-    --env SM_TRUSTED_PROFILE_NAME=${SM_TRUSTED_PROFILE_NAME}\
-    --env SM_SERVICE_URL=${SM_SERVICE_URL} \
-    --env SM_PG_SECRET_ID=${SM_PG_SECRET_ID}
+ibmcloud iam authorization-policy-create secrets-manager databases-for-postgresql \
+    "Key Manager" \
+    --source-service-instance-id $SM_INSTANCE_ID \
+    --target-service-instance-id $DB_INSTANCE_ID
 ```
 
-Create a Secrets Manager instance
-```
-ibmcloud resource service-instance-create credential-store secrets-manager 7713c3a8-3be8-4a9a-81bb-ee822fcaac3d eu-es -p \
-'{
-    "allowed_network": "private-only"
-}'
+* Create the service credential to access the PostgreSQL instance
 ```
+SM_SECRET_FOR_PG_NAME=pg-access-credentials
+ibmcloud secrets-manager secret-create --secret-type="service_credentials" --secret-name="$SM_SECRET_FOR_PG_NAME" --secret-source-service="{\"instance\": {\"crn\": \"$DB_INSTANCE_ID\"},\"parameters\": {},\"role\": {\"crn\": \"crn:v1:bluemix:public:iam::::serviceRole:Writer\"}}"
 
-// Create a Trusted Profile for Secrets Manager
+export SM_SECRET_FOR_PG_ID=$(ibmcloud sm secret-by-name --name $SM_SECRET_FOR_PG_NAME --secret-type service_credentials --secret-group-name default --output JSON|jq -r '.id')
+```
 
-// Create a S2S policy "Key Manager" between SM and the DB
 
-// Create a service credential for PG with automatic key rotation
+* Create the Code Engine app:
+```
+export CE_APP_NAME=csv-to-sql
+export TRUSTED_PROFILE_FOR_COS_NAME=cos-to-sql--ce-to-cos-access
+export TRUSTED_PROFILE_FOR_SM_NAME=cos-to-sql--ce-to-sm-access
 
+ibmcloud code-engine app create \
+    --name ${CE_APP_NAME} \
+    --source ./ \
+    --cpu 0.25 \
+    --memory 0.5G \
+    --trusted-profiles-enabled="true" \
+    --probe-ready type=http \
+    --probe-ready path=/readiness \
+    --probe-ready interval=30 \
+    --env COS_REGION=${REGION} \
+    --env COS_TRUSTED_PROFILE_NAME=${TRUSTED_PROFILE_FOR_COS_NAME} \
+    --env SM_TRUSTED_PROFILE_NAME=${TRUSTED_PROFILE_FOR_SM_NAME} \
+    --env SM_SERVICE_URL=${SECRETS_MANAGER_URL} \
+    --env SM_PG_SECRET_ID=${SM_SECRET_FOR_PG_ID}
+```
 
+## Trusted Profile setup
 
-// ibmcloud secrets-manager secret-by-name --secret-type service_credentials --name pg-credentials --secret-group-name  --service-url https://4e61488a-b76f-4d44-ba0e-ed2489d8a57a.private.eu-es.secrets-manager.appdomain.cloud
+* Create a trusted profile that grants a Code Engine app access to your COS bucket
+```
+ibmcloud iam trusted-profile-create ${TRUSTED_PROFILE_FOR_COS_NAME}
+ibmcloud iam trusted-profile-link-create ${TRUSTED_PROFILE_FOR_COS_NAME} --name ce-app-${CE_APP_NAME} --cr-type CE --link-crn ${CE_INSTANCE_ID} --link-component-type application --link-component-name ${CE_APP_NAME}
+ibmcloud iam trusted-profile-policy-create ${TRUSTED_PROFILE_FOR_COS_NAME} --roles "Content Reader" --service-name cloud-object-storage --service-instance ${COS_INSTANCE_ID} --resource-type bucket --resource ${COS_BUCKET_NAME}
+```
 
-// https://github.com/IBM/secrets-manager-node-sdk
 
-Upload a CSV file to COS, to initate an event that leads to a job execution:
+* Create the trusted profile to access Secrets Manager
 ```
-ibmcloud cos object-put \
-    --bucket $BUCKET \
-    --key users.csv \
-    --body ./samples/users.csv \
-    --content-type text/csv
+ibmcloud iam trusted-profile-create ${TRUSTED_PROFILE_FOR_SM_NAME}
+ibmcloud iam trusted-profile-link-create ${TRUSTED_PROFILE_FOR_SM_NAME} --name ce-app-${CE_APP_NAME} --cr-type CE --link-crn ${CE_INSTANCE_ID} --link-component-type application --link-component-name ${CE_APP_NAME}
+ibmcloud iam trusted-profile-policy-create ${TRUSTED_PROFILE_FOR_SM_NAME} --roles "SecretsReader" --service-name secrets-manager --service-instance ${SM_INSTANCE_ID}
 ```
 
-List all jobs to determine the one, that processes the COS bucket update:
+## Setting up eventing
+
+* Create an authorization policy to allow the Code Engine project receive events from COS:
 ```
-ibmcloud code-engine jobrun list \
-    --job csv-to-sql \
-    --sort-by age
+ibmcloud iam authorization-policy-create codeengine cloud-object-storage \
+    "Notifications Manager" \
+    --source-service-instance-id ${CE_INSTANCE_ID} \
+    --target-service-instance-id ${COS_INSTANCE_ID}
 ```
 
-Inspect the job execution by opening the logs:
+* Create the subscription for all COS events:
 ```
-ibmcloud code-engine jobrun logs \
-    --name <jobrun-name>
+ibmcloud ce sub cos create \
+    --name "coswatch-${CE_APP_NAME}" \
+    --bucket ${COS_BUCKET_NAME} \
+    --destination ${CE_APP_NAME} \
+    --destination-type app \
+    --path /cos-to-sql
 ```
 
-Or do the two commands in one, using this one-liner:
+## Verify the solution
+
+* Upload a CSV file to COS, to initate an event that leads to a job execution:
 ```
-jobrunname=$(ibmcloud ce jr list -j csv-to-sql -s age -o json | jq -r '.items[0] | .metadata.name') && ibmcloud ce jr logs -n $jobrunname -f
+ibmcloud cos object-put \
+    --bucket ${COS_BUCKET_NAME} \
+    --key users.csv \
+    --body ./samples/users.csv \
+    --content-type text/csv
 ```
 
-
+* Inspect the app execution by opening the logs:
+```
+ibmcloud code-engine app logs \
+    --name ${CE_APP_NAME} \
+    --follow
 ```
-kubectl patch jobdefinitions csv-to-sql --type='json' -p='[{"op": "add", "path": "/spec/template/mountComputeResourceToken", "value":true}]'
-kubectl patch ksvc csv-to-sql-app --type='json' -p='[{"op": "add", "path": "/spec/template/metadata/annotations/codeengine.cloud.ibm.com~1mount-compute-resource-token", "value":"true"}]'
-```