cocalc-api: jupyter tests, add endpoints to list/stop kernels

Author: haraldschilly

src/packages/conat/project/api/system.ts

@@ -34,6 +34,10 @@ export const system = {
 
   // jupyter stateless API
   jupyterExecute: true,
+
+  // jupyter kernel management
+  listJupyterKernels: true,
+  stopJupyterKernel: true,
 };
 
 export interface System {
@@ -74,4 +78,9 @@ export interface System {
   }) => Promise<void>;
 
   jupyterExecute: (opts: ProjectJupyterApiOptions) => Promise<object[]>;
+
+  listJupyterKernels: () => Promise<
+    { pid: number; connectionFile: string; kernel_name?: string }[]
+  >;
+  stopJupyterKernel: (opts: { pid: number }) => Promise<{ success: boolean }>;
 }

src/packages/jupyter/kernel/launch-kernel.ts

@@ -168,6 +168,11 @@ async function launchKernelSpec(
   } else {
     running_kernel = spawn(argv[0], argv.slice(1), full_spawn_options);
   }
+
+  // Store kernel info for tracking
+  running_kernel.connectionFile = connectionFile;
+  running_kernel.kernel_spec = kernel_spec;
+
   spawned.push(running_kernel);
 
   running_kernel.on("error", (code, signal) => {
@@ -221,6 +226,48 @@ async function ensureDirectoryExists(path: string) {
 
 // Clean up after any children created here
 const spawned: any[] = [];
+
+export interface RunningKernel {
+  pid: number;
+  connectionFile: string;
+  kernel_name?: string;
+}
+
+export function listRunningKernels(): RunningKernel[] {
+  return spawned
+    .filter((child) => child.pid)
+    .map((child) => ({
+      pid: child.pid,
+      connectionFile: child.connectionFile || "unknown",
+      kernel_name: child.kernel_spec?.name,
+    }));
+}
+
+export function stopKernel(pid: number): boolean {
+  const index = spawned.findIndex((child) => child.pid === pid);
+  if (index === -1) {
+    return false;
+  }
+
+  const child = spawned[index];
+  try {
+    // Try to kill the process group first (negative PID)
+    process.kill(-child.pid, "SIGKILL");
+  } catch (err) {
+    // If that fails, try killing the process directly
+    try {
+      child.kill("SIGKILL");
+    } catch (err2) {
+      logger.debug(`stopKernel: failed to kill ${child.pid}: ${err2}`);
+      return false;
+    }
+  }
+
+  // Remove from spawned array
+  spawned.splice(index, 1);
+  return true;
+}
+
 export function closeAll() {
   for (const child of spawned) {
     if (child.pid) {

src/packages/next/pages/api/conat/project.ts

@@ -40,7 +40,7 @@ export default async function handle(req, res) {
       throw Error("must specify project_id or use project-specific api key");
     }
     if (project_id0) {
-      // auth via project_id
+      // auth via project-specific API key
       if (project_id0 != project_id) {
         throw Error("project specific api key must match requested project");
       }

src/packages/project/conat/api/system.ts

@@ -50,7 +50,7 @@ export async function renameFile({ src, dest }: { src: string; dest: string }) {
 import { get_configuration } from "@cocalc/project/configuration";
 export { get_configuration as configuration };
 
-import { canonical_paths } from "../../browser-websocket/canonical-path";
+import { canonical_paths } from "@cocalc/project/browser-websocket/canonical-path";
 export { canonical_paths as canonicalPaths };
 
 import ensureContainingDirectoryExists from "@cocalc/backend/misc/ensure-containing-directory-exists";
@@ -107,3 +107,17 @@ export async function signal({
 
 import jupyterExecute from "@cocalc/jupyter/stateless-api/execute";
 export { jupyterExecute };
+
+import {
+  listRunningKernels,
+  stopKernel,
+} from "@cocalc/jupyter/kernel/launch-kernel";
+
+export async function listJupyterKernels() {
+  return listRunningKernels();
+}
+
+export async function stopJupyterKernel({ pid }: { pid: number }) {
+  const success = stopKernel(pid);
+  return { success };
+}

src/packages/server/projects/control/single-user.ts

@@ -153,20 +153,33 @@ class Project extends BaseProject {
       this.stateChanging = { state: "stopping" };
       await this.saveStateToDatabase(this.stateChanging);
       const pid = await getProjectPID(this.HOME);
-      const killProject = () => {
+
+      // First attempt: graceful shutdown with SIGTERM
+      // This allows the process to clean up child processes (e.g., Jupyter kernels)
+      let usedSigterm = false;
+      const killProject = (signal: NodeJS.Signals = "SIGTERM") => {
         try {
-          logger.debug(`stop: sending kill -${pid}`);
-          kill(-pid, "SIGKILL");
+          logger.debug(`stop: sending kill -${pid} with ${signal}`);
+          kill(-pid, signal);
         } catch (err) {
           // expected exception if no pid
           logger.debug(`stop: kill err ${err}`);
         }
       };
-      killProject();
+
+      // Try SIGTERM first for graceful shutdown
+      killProject("SIGTERM");
+      usedSigterm = true;
+
       await this.wait({
         until: async () => {
           if (await isProjectRunning(this.HOME)) {
-            killProject();
+            // After 5 seconds, escalate to SIGKILL
+            if (usedSigterm) {
+              logger.debug("stop: escalating to SIGKILL");
+              killProject("SIGKILL");
+              usedSigterm = false;
+            }
             return false;
           } else {
             return true;

src/python/cocalc-api/docs/index.md

@@ -10,13 +10,13 @@ Obtain a [CoCalc account API Key](https://doc.cocalc.com/apikeys.html) by going
 
 Using an account level API key, the cocalc_api Python library enabled you to do all of the following very easily from a Python script:
 
-- [search](api/system/) for other cocalc users by name or email address, and get the name associated to an account_id
-- list [your projects](api/projects), add and remove collaborators, copy files between projects and start, stop and create projects.
-- use the [Jupyter API](api/jupyter) to evaluate code using a kernel, either in an anonymous sandbox or in one of your projects.
-- read or write any data you have access to in the CoCalc [PostgreSQL database](api/database), as defined by [this schema](https://github.com/sagemathinc/cocalc/tree/master/src/packages/util/db-schema).
-- instantly send and receive [messages](api/messages) with any other cocalc users
-- create and manage users in an [organization](api/organizations), including automatically generating authentication links, so you're users do not have explicitly create a CoCalc account. You can see when they are active and send them messages.
+- [search](api/system.md) for other cocalc users by name or email address, and get the name associated to an account_id
+- list [your projects](api/projects.md), add and remove collaborators, copy files between projects and create, start, stop and delete projects.
+- use the [Jupyter API](api/jupyter.md) to list available kernels and evaluate code using a kernel, either in an anonymous sandbox or in one of your projects.
+- read or write any data you have access to in the CoCalc [PostgreSQL database](api/database.md), as defined by [this schema](https://github.com/sagemathinc/cocalc/tree/master/src/packages/util/db-schema).
+- instantly send and receive [messages](api/messages.md) with any other cocalc users
+- create and manage users in an [organization](api/organizations.md), including automatically generating authentication links, so you're users do not have explicitly create a CoCalc account. You can see when they are active and send them messages.
 
 Currently a project specific API key can be used to:
 
-- Run [shell commands](api/project) and Jupyter code in a specific project.
+- Run [shell commands](api/project.md) and Jupyter code in a specific project, and manage running Jupyter kernels (list and stop).

src/python/cocalc-api/src/cocalc_api/hub.py

@@ -289,7 +289,7 @@ def __init__(self, parent: "Hub"):
         self._parent = parent
 
     @api_method("jupyter.kernels")
-    def kernels(self, project_id: Optional[str] = None) -> dict[str, Any]:
+    def kernels(self, project_id: Optional[str] = None) -> list[dict[str, Any]]:
         """
         Get specifications of available Jupyter kernels.
 
@@ -298,7 +298,18 @@ def kernels(self, project_id: Optional[str] = None) -> dict[str, Any]:
                 If not given, a global anonymous project may be used.
 
         Returns:
-            dict[str, Any]: JSON response containing kernel specs.
+            list[dict[str, Any]]: List of kernel specification objects. Each kernel object
+                contains information like 'name', 'display_name', 'language', etc.
+
+        Examples:
+            Get available kernels for a project:
+
+            >>> import cocalc_api; hub = cocalc_api.Hub(api_key="sk-...")
+            >>> kernels = hub.jupyter.kernels(project_id='6e75dbf1-0342-4249-9dce-6b21648656e9')
+            >>> # Extract kernel names
+            >>> kernel_names = [k['name'] for k in kernels]
+            >>> 'python3' in kernel_names
+            True
         """
         ...
 

src/python/cocalc-api/src/cocalc_api/project.py

@@ -116,26 +116,84 @@ def jupyter_execute(
         kernel: str,
         history: Optional[list[str]] = None,
         path: Optional[str] = None,
-    ) -> dict[str, Any]:  # type: ignore[empty-body]
+    ) -> list[dict[str, Any]]:  # type: ignore[empty-body]
         """
         Execute code using a Jupyter kernel.
 
         Args:
             input (str): Code to execute.
-            kernel (str): Name of kernel to use. Get options using jupyter.kernels().
+            kernel (str): Name of kernel to use. Get options using hub.jupyter.kernels().
             history (Optional[list[str]]): Array of previous inputs (they get evaluated every time, but without output being captured).
             path (Optional[str]): File path context for execution.
 
         Returns:
-            dict[str, Any]: JSON response containing execution results.
+            list[dict[str, Any]]: List of output items. Each output item contains
+                execution results with 'data' field containing output by MIME type
+                (e.g., 'text/plain' for text output) or 'name'/'text' fields for
+                stream output (stdout/stderr).
 
         Examples:
             Execute a simple sum using a Jupyter kernel:
 
-            >>> import cocalc_api; project = cocalc_api.Project(api_key="sk-...")
-            >>> project.jupyter.execute(history=['a=100;print(a)'],
-                         input='sum(range(a+1))',
-                         kernel='python3')
-            {'output': [{'data': {'text/plain': '5050'}}], ...}
+            >>> import cocalc_api; project = cocalc_api.Project(api_key="sk-...", project_id='...')
+            >>> result = project.system.jupyter_execute(input='sum(range(100))', kernel='python3')
+            >>> result
+            [{'data': {'text/plain': '4950'}}]
+
+            Execute with history context:
+
+            >>> result = project.system.jupyter_execute(
+            ...     history=['a = 100'],
+            ...     input='sum(range(a + 1))',
+            ...     kernel='python3')
+            >>> result
+            [{'data': {'text/plain': '5050'}}]
+
+            Print statements produce stream output:
+
+            >>> result = project.system.jupyter_execute(input='print("Hello")', kernel='python3')
+            >>> result
+            [{'name': 'stdout', 'text': 'Hello\\n'}]
+        """
+        ...
+
+    @api_method("system.listJupyterKernels")
+    def list_jupyter_kernels(self) -> list[dict[str, Any]]:  # type: ignore[empty-body]
+        """
+        List all running Jupyter kernels in the project.
+
+        Returns:
+            list[dict[str, Any]]: List of running kernels. Each kernel has:
+                - pid (int): Process ID of the kernel
+                - connectionFile (str): Path to the kernel connection file
+                - kernel_name (str, optional): Name of the kernel (e.g., 'python3')
+
+        Examples:
+            List all running kernels:
+
+            >>> import cocalc_api; project = cocalc_api.Project(api_key="sk-...", project_id='...')
+            >>> kernels = project.system.list_jupyter_kernels()
+            >>> kernels
+            [{'pid': 12345, 'connectionFile': '/run/user/1000/jupyter/kernel-abc123.json', 'kernel_name': 'python3'}]
+        """
+        ...
+
+    @api_method("system.stopJupyterKernel")
+    def stop_jupyter_kernel(self, pid: int) -> dict[str, bool]:  # type: ignore[empty-body]
+        """
+        Stop a specific Jupyter kernel by process ID.
+
+        Args:
+            pid (int): Process ID of the kernel to stop
+
+        Returns:
+            dict[str, bool]: Dictionary with 'success' key indicating if the kernel was stopped
+
+        Examples:
+            Stop a kernel by PID:
+
+            >>> import cocalc_api; project = cocalc_api.Project(api_key="sk-...", project_id='...')
+            >>> project.system.stop_jupyter_kernel(pid=12345)
+            {'success': True}
         """
         ...