feat: update to support new ray-job-definition.json info

rather than depending on the job.json generated only when the ray job was submitted by us

WARNING: tests/plugin-codeflare/dashboard/inputs/2 still needs to be updated to have the new job definition file assets

Author: starpit

plugins/plugin-client-default/notebooks/dashboard.md

@@ -24,7 +24,7 @@ layout:
     execute: now
     outputOnly: true
     ---
-    description application "$LOGDIR/job.json"
+    description application "$LOGDIR"
     ```
 
 === "Workers"
@@ -34,7 +34,7 @@ layout:
     execute: now
     outputOnly: true
     ---
-    description workers "$LOGDIR/job.json"
+    description workers "$LOGDIR"
     ```
 
 ---

plugins/plugin-codeflare/src/controller/description.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import { join } from "path"
 import { Arguments, Registrar } from "@kui-shell/core"
 import { expand } from "../lib/util"
 
@@ -42,20 +43,27 @@ export type SummaryResponse = {
   source: string
 }
 
+/** Given the location of the staging directory, return the location of the ray job definition */
+function jobDefinition(filepath: string) {
+  return expand(join(filepath.replace(/'/g, ""), "ray-job-definition.json"))
+}
+
 async function app(args: Arguments) {
   const filepath = args.argvNoOptions[2]
   if (!filepath) {
     throw new Error("Usage: description application <filepath>")
   }
 
-  const jobInfo = JSON.parse(await args.REPL.qexec<string>(`vfs fslice ${expand(filepath)} 0`))
-  const { RAY_IMAGE } = jobInfo.runtimeEnv.env_vars
+  const jobInfo = JSON.parse(await args.REPL.qexec<string>(`vfs fslice ${jobDefinition(filepath)} 0`))
+  const { RAY_IMAGE } = jobInfo.runtime_env.env_vars
+
+  const status = jobInfo.status.toLowerCase()
 
   const summaryData = [
     { label: "Application Class", value: "Unknown" }, // TODO...
     { label: "Application Name", value: "Unknown" }, // TODO...
     { label: "Base Image", value: RAY_IMAGE },
-    { label: "Run Status", value: process.env.FOLLOW ? "Running" : "Done" }, // TODO...
+    { label: "Run Status", value: status ? status[0].toUpperCase() + status.slice(1) : "Unknown" },
   ]
 
   const React = await import("react")
@@ -72,8 +80,8 @@ async function workers(args: Arguments) {
     throw new Error("Usage: description workers <filepath>")
   }
 
-  const jobInfo = JSON.parse(await args.REPL.qexec<string>(`vfs fslice ${expand(filepath)} 0`))
-  const { KUBE_CONTEXT, KUBE_NS, WORKER_MEMORY, MIN_WORKERS, MAX_WORKERS } = jobInfo.runtimeEnv.env_vars
+  const jobInfo = JSON.parse(await args.REPL.qexec<string>(`vfs fslice ${jobDefinition(filepath)} 0`))
+  const { KUBE_CONTEXT, KUBE_NS, WORKER_MEMORY, MIN_WORKERS, MAX_WORKERS } = jobInfo.runtime_env.env_vars
 
   const summaryData = [
     { label: "Cluster Context", value: KUBE_CONTEXT.replace(/^[^/]+\//, "") },

tests/plugin-codeflare/dashboard/inputs/1/README.md

@@ -3,7 +3,7 @@
   "madwizard/apriori/arch": "x64",
   "madwizard/apriori/platform": "darwin",
   "madwizard/apriori/mac-installer": "Homebrew",
-  "madwizard/apriori/in-terminal": "Text",
+  "madwizard/apriori/in-terminal": "HTML",
   "Training####Fine Tuning": "Fine Tuning",
   "GLUE": "GLUE",
   "AWS####IBM": "AWS",
@@ -21,5 +21,12 @@
   "Number of CPUs####Number of GPUs": "{\"Number of CPUs\":4,\"Number of GPUs\":3}",
   "expand(echo ${A-error} ; echo ${B-4} ; echo ${C-5})": "3",
   "XXXXXX.11111####222222": "11111",
-  "YYYYYY.11111####222222": "222222"
+  "YYYYYY.11111####222222": "222222",
+  "My Cluster is Running Locally####My Cluster is Runing on Kubernetes": "My Cluster is Runing on Kubernetes",
+  "expand([ -n \"$RAY_ADDRESS\" ] && ray job list --address $RAY_ADDRESS | tail +2 | awk '{print $1}' | sed \"s/[:{' ]//g\", Ray Runs)": "07a2647f-3656-4e3e-836c-95a2fa841af6",
+  "expand([ -n \"$RAY_ADDRESS\" ] && curl $RAY_ADDRESS/api/jobs/ | jq -r 'keys | .[]', Ray Runs)": "d5a0d68f-a675-49ca-bff7-4ae762e6b146",
+  "My Cluster is Running Locally####My Cluster is Running on Kubernetes": "My Cluster is Running on Kubernetes",
+  "expand([ -n \"$RAY_ADDRESS\" ] && curl $RAY_ADDRESS/api/jobs/ | jq -r 'to_entries | sort_by(.value.start_time) | reverse | .[] | \"\\(.key) \\(.value.status) \\(.value.entrypoint)\"' | sed -E 's/python3 ([^[:space:]])+ //g' | awk '{a=$1;b=$2; $1=\"\";$2=\"\";print \"\\033;1m\" a, \"\\033[0;33m\" b \"\\033[0;2m\" $0 \"\\033[0m\"}', Ray Runs)": "\u001b;1ma88d4632-ab5c-4350-a770-d39a955c42c8 \u001b[0;33mRUNNING\u001b[0;2m  -v --datapath /tmp/ --modelpath /tmp/ --logpath /tmp/ --tblogpath s3://browsey/codeflare/a88d4632-ab5c-4350-a770-d39a955c42c8/tensorboard/ --num_workers 1\u001b[0m",
+  "expand([ -n \"$RAY_ADDRESS\" ] && curl $RAY_ADDRESS/api/jobs/ | jq -r 'to_entries | sort_by(.value.start_time) | reverse | .[] | \"\\(.key) \\(.value.status) \\(.value.entrypoint)\"' | sed -E 's/python3 ([^[:space:]])+ //g' | awk '{a=$1;b=$2; $1=\"\";$2=\"\";print a, \"\\033[33m\" b \"\\033[0;2m\" $0 \"\\033[0m\"}', Ray Runs)": "a88d4632-ab5c-4350-a770-d39a955c42c8 \u001b[33mRUNNING\u001b[0;2m  -v --datapath /tmp/ --modelpath /tmp/ --logpath /tmp/ --tblogpath s3://browsey/codeflare/a88d4632-ab5c-4350-a770-d39a955c42c8/tensorboard/ --num_workers 1\u001b[0m",
+  "expand([ -n \"$RAY_ADDRESS\" ] && curl $RAY_ADDRESS/api/jobs/ | jq -r 'to_entries | sort_by(.value.start_time) | reverse | .[] | \"\\(.key) \\(.value.status) \\(.value.start_time / 1000 | strflocaltime(\"%Y-%m-%dT%H:%M:%S\")) \\(.value.entrypoint)\"' | sed -E 's/python3 ([^[:space:]])+ //g' | awk '{a=$1;b=$2;c=$3; $1=\"\";$2=\"\";$3=\"\"; print a, \"\\033[0;36m\" c, \"\\033[0;1;33m\" b \"\\033[0;2m\" $0 \"\\033[0m\"}', Ray Runs)": "505b98b6-a258-4afd-bdc6-ddf84d3f2862 \u001b[0;36m2022-07-07T13:27:54 \u001b[0;1;33mRUNNING\u001b[0;2m   -v --datapath /tmp/ --modelpath /tmp/ --logpath /tmp/ --tblogpath s3://browsey/codeflare/505b98b6-a258-4afd-bdc6-ddf84d3f2862/tensorboard/ --num_workers 1\u001b[0m"
 }

tests/plugin-codeflare/dashboard/inputs/1/choices.json

@@ -1,13 +0,0 @@
-LAST SEEN   TYPE     REASON      OBJECT                              MESSAGE
-0s          Normal   Scheduled   pod/mycluster-ray-head-type-krlr4   Successfully assigned nvidia-gpu-operator/mycluster-ray-head-type-krlr4 to ip-10-0-128-169.ec2.internal
-0s          Normal   AddedInterface   pod/mycluster-ray-head-type-krlr4   Add eth0 [10.128.44.144/23] from openshift-sdn
-0s          Normal   Pulling          pod/mycluster-ray-head-type-krlr4   Pulling image "rayproject/ray-ml:1.13.0-py37-gpu"
-0s          Normal   Pulled           pod/mycluster-ray-head-type-krlr4   Successfully pulled image "rayproject/ray-ml:1.13.0-py37-gpu" in 6m48.700535275s
-0s          Normal   Created          pod/mycluster-ray-head-type-krlr4   Created container ray-node
-0s          Normal   Started          pod/mycluster-ray-head-type-krlr4   Started container ray-node
-0s          Normal   Scheduled        pod/mycluster-ray-worker-type-6r7hp   Successfully assigned nvidia-gpu-operator/mycluster-ray-worker-type-6r7hp to ip-10-0-133-106.ec2.internal
-0s          Normal   AddedInterface   pod/mycluster-ray-worker-type-6r7hp   Add eth0 [10.131.42.42/23] from openshift-sdn
-0s          Normal   Pulling          pod/mycluster-ray-worker-type-6r7hp   Pulling image "rayproject/ray-ml:1.13.0-py37-gpu"
-0s          Normal   Pulled           pod/mycluster-ray-worker-type-6r7hp   Successfully pulled image "rayproject/ray-ml:1.13.0-py37-gpu" in 6m14.380152399s
-0s          Normal   Created          pod/mycluster-ray-worker-type-6r7hp   Created container ray-node
-0s          Normal   Started          pod/mycluster-ray-worker-type-6r7hp   Started container ray-node

tests/plugin-codeflare/dashboard/inputs/1/events/kubernetes.txt

@@ -1 +1 @@
-87ed37bc-a837-4f20-aad0-0a12754452f4
+bf3d5456-fba9-42b1-9b55-119366b409ee