diff --git a/AGENTS.md b/AGENTS.md index 257f6b4334..d1b23afc8c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -374,6 +374,13 @@ git commit -s solution. - Documentation updates (`docs/`) are required for user-visible changes. +**Never push a branch to `origin`.** The `origin` remote is the shared +upstream repository (`openpmix/prrte`); pushing topic branches there is not +the project workflow. Always push your branch to your personal fork remote +instead, and open the pull request from the fork against the upstream +`master` (or the appropriate release branch). If you are unsure which remote +is the fork, run `git remote -v` and ask rather than guessing. + ### Testing PRRTE does not have a standalone unit test suite. Integration-level diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index ad1a2a3aab..0000000000 --- a/CLAUDE.md +++ /dev/null @@ -1,441 +0,0 @@ -# AGENTS.md — Orientation for AI Coding Agents and Human Contributors - -This document is an orientation guide for AI coding agents and their human operators working in the PRRTE code base. -This file is an *orientation map*, not the -full rulebook: the authoritative, human-maintained documentation lives -under [`docs/developers/`](docs/developers/) and -[`docs/contributing.rst`](docs/contributing.rst) (rendered at -). When this file and those docs disagree, -**the docs win** — and please fix this file. - -AI-assisted contributions are welcome. But PRRTE runs on the largest -supercomputers in the world and across a huge range of operating -systems and hardware. It is also used by numerous research groups -looking at developments such as elastic environments. We want careful, -portable code — not plausible-looking code that solves one problem in -one environment or use-case at the expense of others. Hold yourself to -the same bar as a thoughtful human contributor. - ---- - -## What is PRRTE? - -PRRTE (PMIx Reference RunTime Environment) is an open-source, production -runtime system for launching and managing parallel jobs in HPC environments. -Unlike a library, PRRTE exposes no public API for callers — it is a -collection of executable tools that users invoke directly. PRRTE acts as a -reference implementation of the PMIx-defined runtime services, and it relies -heavily on the PMIx project for its internal infrastructure: MCA -(Modular Component Architecture), utility routines, data structures, and -threading primitives. Many PMIx symbols and headers PRRTE uses are internal -PMIx interfaces, **not** the PMIx public library API. - -Source repository: https://github.com/openpmix/prrte -Documentation: https://docs.prrte.org/ - ---- - -## Terminology - -Understanding PRRTE's vocabulary is essential before reading or modifying -code. - -| Term | Meaning | -|------|---------| -| **DVM** | Distributed Virtual Machine — a persistent set of PRRTE daemons spread across an allocation, ready to launch jobs on demand. | -| **HNP** | Head Node Process — the `prte` process that acts as the DVM controller. Also called the DVM master. | -| **prted** | PRRTE daemon — one instance runs on each node in the DVM and manages local processes. | -| **job** | A set of processes launched together under a single `prun` invocation. Each job has a unique namespace. | -| **namespace** | A string that uniquely identifies a job within a DVM session (inherited from PMIx). | -| **rank** | A process's integer identifier within its namespace (global rank), within its node (node rank), or within the local set of processes on a node (local rank). | -| **app context** | A description of one executable to run: path, argv, environment, and process count. A single `prun` may specify multiple app contexts. | -| **session** | The lifetime of a DVM — from `prte` startup through `pterm` shutdown. | -| **schizo** | The "personality" framework. Schizo components translate between PRRTE's internal model and the command-line conventions of specific launchers (e.g., Open MPI's `mpirun`, SLURM's `srun`). | -| **MCA** | Modular Component Architecture — the plugin system, inherited from PMIx, that PRRTE uses for every swappable subsystem. | -| **framework** | An MCA abstraction layer that defines the interface for one functional area (e.g., `plm`, `rmaps`). | -| **component** | A plugin that implements a framework's interface for a specific environment or algorithm (e.g., `plm/slurm`). | -| **module** | The active instance of a component, returned after it wins selection. | - ---- - -## User-Facing Tools - -PRRTE provides no linkable library. All user interaction is through -executables. - -| Tool | Source | Purpose | -|------|--------|---------| -| `prte` | `src/tools/prte/` | Start a DVM (the HNP process). Allocates and connects daemons across an allocation. | -| `prun` | `src/tools/prun/` | Submit and manage a job within a running DVM. The everyday launch command. | -| `pterm` | `src/tools/pterm/` | Terminate a running DVM cleanly. | -| `prted` | `src/tools/prted/` | The per-node daemon, normally launched by the HNP — not invoked directly by users. | -| `prte_info` | `src/tools/prte_info/` | Query build configuration, list available MCA components, and display version information. | -| `pcc` | `src/tools/pcc/` | Compiler wrapper for applications that directly embed PMIx (not PRRTE — PRRTE has no linkable library). | - ---- - -## Source Layout - -``` -src/ - tools/ # Executable entry points (prte, prun, pterm, prted, prte_info, pcc) - mca/ # All MCA frameworks and their components - runtime/ # Global state, init/finalize, job/node/proc data structures - rml/ # Runtime Messaging Layer (point-to-point communication between daemons) - util/ # Internal utilities (hostfile parsing, name formatting, attributes, ...) - include/ # Internal header files (types.h, constants.h, ...) - pmix/ # Thin shim connecting PRRTE to its PMIx dependency - event/ # Libevent integration - hwloc/ # hwloc integration (topology, binding) -``` - -The three central data structures — `prte_job_t`, `prte_node_t`, and -`prte_proc_t` — are defined in `src/runtime/prte_globals.h` and carry -all runtime state for a running job. Code throughout the tree reaches -for these; understand them before touching launch, mapping, or error -handling paths. - ---- - -## MCA Frameworks - -Every swappable PRRTE subsystem is an MCA framework. The framework -directory contains a `base/` subdirectory (selection, default behaviors, -utility stubs) and one or more component subdirectories. - -| Framework | Location | Responsibility | -|-----------|----------|----------------| -| `ess` | `src/mca/ess/` | Environment-Specific Services — init/finalize for each process role (HNP, daemon, tool). | -| `plm` | `src/mca/plm/` | Process Launch Manager — spawns prted daemons across the system. Components: `ssh`, `slurm`, `lsf`, `pals`. | -| `ras` | `src/mca/ras/` | Resource Allocation Subsystem — discovers nodes and slots. Components: `slurm`, `pbs`, `lsf`, `flux`, `gridengine`, `hosts`. | -| `rmaps` | `src/mca/rmaps/` | Resource Mapping — assigns processes to nodes/slots. Components: `round_robin`, `ppr`, `rank_file`, `seq`. | -| `odls` | `src/mca/odls/` | PRRTE Daemon Local Launch Subsystem — the per-node daemon (prted) forks/execs application processes. | -| `iof` | `src/mca/iof/` | I/O Forwarding — routes stdout/stderr/stdin between daemons and the HNP. | -| `grpcomm` | `src/mca/grpcomm/` | Group Communication — collective operations among daemons (broadcast, barrier). | -| `errmgr` | `src/mca/errmgr/` | Error Manager — handles process faults, abnormal exits, and propagation of errors. | -| `state` | `src/mca/state/` | State Machine — drives the DVM and job lifecycle through defined states/transitions. | -| `schizo` | `src/mca/schizo/` | Personality layer — parses CLI options and environment for specific launcher personalities (prte, ompi). | -| `filem` | `src/mca/filem/` | File Management — pre-positions files across nodes before launch. | -| `prtereachable` | `src/mca/prtereachable/` | Reachability — determines which network interfaces can reach each node. | -| `prtebacktrace` | `src/mca/prtebacktrace/` | Backtrace support for crash diagnostics. | -| `prtedl` | `src/mca/prtedl/` | Dynamic linker abstraction (dlopen/dlclose). | -| `prteinstalldirs` | `src/mca/prteinstalldirs/` | Installation directory queries. | - -### MCA component structure - -Each component directory must contain: -- `.h` — component struct (`prte___component_t`) -- `.c` — component registration, open, query, close -- `_module.c` (or similar) — module implementation -- `Makefile.am` - -The component's `query` function returns a priority; the highest-priority -component that successfully opens wins selection. - ---- - -## PRRTE's Relationship with PMIx - -PRRTE depends on PMIx (minimum version `6.1.0`) and uses PMIx internals -extensively. This is not the same as calling the PMIx public library API. - -**What PRRTE takes from PMIx:** - -- **MCA infrastructure** (`src/mca/base/`, `src/mca/mca.h`) — the entire - plugin/component system is PMIx's code, not PRRTE's. -- **Data structures and classes** — `pmix_list_t`, `pmix_pointer_array_t`, - `pmix_hash_table_t`, `pmix_ring_buffer_t`, `pmix_value_array_t`. -- **Threading primitives** — `pmix_mutex_t`, `pmix_condition_t`, - `pmix_threads_*`. -- **Utility functions** — `pmix_argv_*`, `pmix_environ_*`, - `pmix_output_*`, `pmix_cmd_line_*`. -- **Event loop** — PMIx's libevent integration and progress threads. -- **Data packing/unpacking** — PMIx's buffer and bfrops subsystem. - -Include paths for these symbols come from PMIx's installed headers (found -via `pkg-config` or `--with-pmix=`). The shim at `src/pmix/pmix-internal.h` -and `src/pmix/pmix.c` is PRRTE's integration point — consult it before -reaching for PMIx symbols elsewhere. - -### Check capability flags - -PRRTE uses the `AC_PREPROC_IFELSE` idiom in its configure script to test for PMIx capability flags at build time. PRRTE's `config/prte_setup_pmix.m4` defines a helper macro `PRTE_CHECK_PMIX_CAP` that reduces the check to: - -```m4 -PRTE_CHECK_PMIX_CAP([MY_NEW_FEATURE], - [action if supported], - [action if not supported]) -``` - -The macro succeeds when `PMIX_CAP_MY_NEW_FEATURE` is defined in the installed `pmix_version.h`; it fails (and triggers the third argument) when the definition is absent, meaning the running PMIx predates the feature. - -Code requiring a specific PMIx feature that is covered by a capability flag must be protected by an appropriate `#if FOO` clause. - ---- - -## Coding Rules - -These rules apply to **all** PRRTE C source files without exception. - -### Mandatory header order - -`prte_config.h` **must be the first `#include`** in every `.c` file. -Nothing comes before it — not system headers, not other PRRTE headers. - -```c -#include "prte_config.h" - -#include /* system headers follow */ -... -#include "src/util/name_fns.h" /* then PRRTE/PMIx headers */ -``` - -### Symbol prefixes - -| Scope | Prefix | -|-------|--------| -| Exported (tools, public data) | `PRTE_` (macros) / `prte_` (functions, types) | -| Internal only | `prte_` with no `PRTE_EXPORT` annotation | -| MCA component | `prte___` | - -Do not use `PMIX_` or `pmix_` prefixes for new PRRTE symbols. - -### New files need the standard copyright/license header. - -Copy the multi-institution BSD header block — including the `Copyright (c) 2026 Nanook Consulting All rights reserved. -Copy the multi-institution BSD header block — including the `$COPYRIGHT$` and -`$HEADER$` tokens — from a neighboring file. If you substantially -change an existing file, add your copyright line to its block. - -### `#define` logical macros to `0` or `1`; never `#undef` them. - -Test with `#if FOO`, not `#ifdef FOO`, so a misspelling is a compiler -error, not a silent false. - -### Constant-on-left comparisons - -Always put the constant or NULL on the left side of an equality test: - -```c -if (NULL == ptr) /* correct */ -if (ptr == NULL) /* wrong — easy to mistype as assignment */ - -if (PRTE_SUCCESS == rc) /* correct */ -``` - -### Always brace blocks - -Use `{ }` around every conditional or loop body, even single-line ones. - -### Indentation - -4 spaces, never tab characters. - - -### Stay compiler-warning-free - -PRRTE strives to build with zero compiler warnings. Do not introduce -code that adds new warnings. `--enable-devel-check` is implied when -building from a git repo with `--enable-debug`; it instructs the compiler -to treat all warnings as errors and enables maximum warning coverage -(e.g., unused variables). CI runs with this setting, so your code must -be warning-free before submitting. - -### No hand-editing of generated files - -Do not modify files produced by autotools (`configure`, `Makefile.in`, etc.), pre-rendered documentation, or third-party vendored code. Edit the source code instead. - - -### Error handling - -Functions that can fail return `int`. Check every return value. Use -`PRTE_ERROR_LOG(rc)` to record unexpected errors. Use -`PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_*)` to trigger state -transitions rather than handling errors inline in launch paths. - -### Thread model - -PRRTE is event-driven and single-threaded on the progress thread. Use -the PRRTE event loop (`prte_event_base`) for deferred work. Do not block on -the progress thread. - -### Thread-shifting with caddies - -A **caddy** is a short-lived heap object whose sole job is to carry a request's parameters across states within the progress thread. Every caddy struct must contain at minimum: - -| Field | Type | Purpose | -|-------|------|---------| -| ev | `pmix_event_t` | Required by Libevent to queue the caddy; **must be named `ev`** | -| lock | `pmix_lock_t` | Thread synchronization (blocking operations wait on this; handlers wake it) | -| cbdata | `void *` | Opaque pointer passed through to the callback | -| callback pointer(s) | function pointer(s) | Cache the caller-supplied callback function(s) | - -The pattern: - -1. Allocate a caddy with `PMIX_NEW(caddy_type_t)`. -2. Assign the caddy's fields to point at the caller's parameters — **do not copy the data**. -3. Call `PRTE_PMIX_THREADSHIFT(cd, evbase, handler_fn)` to post the caddy to the progress thread's event queue. -4. The progress thread fires `handler_fn(cd)`, which performs the actual work. - -Never read or write shared library state outside of the progress thread; do it only inside the handler that runs on the progress thread. -Do not allocate a caddy on the stack — it must outlive the function that creates it. - - -### Memory management - -Use `PMIX_NEW` / `PMIX_RELEASE` (PMIx's reference-counted object system) -for all PRRTE objects that embed `pmix_object_t`. Raw allocations use -`malloc`/`free` directly — avoid `PMIX_MALLOC`/`PMIX_FREE` for plain -buffers unless interfacing with PMIx routines that require it. - -### C standard - -PRRTE targets C11. Do not add `-Wno-*` flags to suppress warnings — -fix the underlying issue. - ---- - -## Build System - -PRRTE uses GNU Autotools. The generated `configure` script is **not** in -the git repository; it is produced by running: - -```sh -./autogen.pl -``` - -This must be re-run whenever `configure.ac`, any `Makefile.am`, or any -`*.m4` file under `config/` is modified. After `autogen.pl`: - -```sh -./configure [options] -make -j$(nproc) -make install -``` - -Common configure options: - -| Option | Purpose | -|--------|---------| -| `--with-pmix=` | Path to PMIx installation (required if not in default search path) | -| `--with-hwloc=` | Path to hwloc installation | -| `--with-libevent=` | Path to libevent installation | -| `--with-slurm` | Enable SLURM support | -| `--with-lsf=` | Enable LSF support | -| `--with-pbs` | Enable PBS/Torque support | -| `--with-flux=` | Enable Flux support | -| `--enable-debug` | Build with debug symbols and extra assertions | -| `--enable-devel-check` | Enable strict compiler warnings (treat warnings as errors); on by default when `--enable-debug` is used in a git repo build | - -Version requirements: PMIx ≥ 6.1.0, hwloc ≥ 2.1.0, libevent ≥ 2.0.21. - ---- - -## State Machines - -PRRTE drives both the DVM lifecycle and individual job lifecycles through -explicit state machines defined in `src/mca/state/`. Process and job -states are declared in `src/mca/state/state_types.h`. When debugging -launch or termination problems, enable framework verbosity to trace -state transitions: - -```sh -prte --prtemca state_base_verbose 5 ... # trace job state transitions -prte --prtemca plm_base_verbose 5 ... # trace daemon launch -prte --prtemca rmaps_base_verbose 5 ... # trace process mapping -``` - -Key job states in order: `PRTE_JOB_STATE_INIT` → -`PRTE_JOB_STATE_ALLOCATE` → `PRTE_JOB_STATE_MAP` → -`PRTE_JOB_STATE_LAUNCH_DAEMONS` → `PRTE_JOB_STATE_RUNNING` → -`PRTE_JOB_STATE_TERMINATED`. - ---- - -## Contributing - -### Commit messages - -Write prose commit messages, not bullet lists. The subject line should -complete the sentence "If applied, this commit will …". The body must -explain **why** the change is needed, not just what it does. Keep the -subject line under 72 characters. - -All commits require a `Signed-off-by:` line (DCO): - -``` -git commit -s -``` - -### Pull requests - -- Open PRs against the `master` branch (development trunk). -- One logical change per PR. Split unrelated fixes. -- Describe the problem being solved in the PR description, not just the - solution. -- Documentation updates (`docs/`) are required for user-visible changes. - -### Testing - -PRRTE does not have a standalone unit test suite. Integration-level -testing is done by running actual parallel jobs through the DVM. When -modifying launch, mapping, or I/O forwarding paths, test with: - -```sh -prte --daemonize # start DVM -prun -n 4 hostname # basic launch smoke test -pterm # shut down DVM -``` - -For resource manager integration (SLURM, PBS, LSF), test within an actual -allocation on the relevant system. - -### Testing the mapper without launching - -Most `rmaps` work — mapping policies, ranking, and binding — can be verified -**without launching any daemons or processes**. `prterun` will compute and -print the complete job map (process-to-node placement, ranks, and CPU -bindings) and then stop, when given the `donotlaunch` runtime option together -with `--display map`: - -```sh -prterun --rtos donotlaunch --display map \ - --prtemca hwloc_use_topo_file test/topologies/test-topo.xml \ - -H node0:4,node1:4,node2:4 \ - --map-by node --rank-by node --bind-to core -n 8 hostname -``` - -This places the 8 procs round-robin across the three simulated nodes (3/3/2), -ranks them by node, and binds each to a core — and prints the result without -launching anything. - -What each piece does: - -- **`--rtos donotlaunch`** — run the mapper/ranker/binder, display the result, - and exit. Nothing is forked or exec'd; no DVM is started. (`--do-not-launch` - is a deprecated alias.) -- **`--display map`** — print the resulting map. Use `--display allocation` or - `--display map-devel` for more detail. (`--display-map` is a deprecated - alias.) -- **`--prtemca hwloc_use_topo_file `** — bind against a *simulated* node - topology loaded from an hwloc XML file instead of the local machine's - topology. Without it, binding is computed against the head node's own - topology (and `donotlaunch` warns that it could not probe the compute - nodes). A ready-made multi-core topology lives at - `test/topologies/test-topo.xml`; generate others with `lstopo file.xml`. -- **`-H node0:N,node1:M,...`** — declare the simulated nodes and their slot - counts. The counts (`N`, `M`, …) only need to be **at least** the number of - procs you want placed on each node; they bound oversubscription, not the - topology. All simulated nodes share the one topology from the XML file. - -The printed map shows, per node, each process's `App:` index, `Process rank:`, -and `Bound:` object — exactly the values produced by `src/mca/rmaps` and the -`src/hwloc` binding code — so different `--map-by` / `--rank-by` / `--bind-to` -combinations (and the by-node vs. by-slot rank distinctions that are only -visible across multiple nodes) can be compared directly. This is the fastest -way to confirm a mapping/ranking/binding change before any integration test. - -### Reporting bugs - -File issues at https://github.com/openpmix/prrte/issues. Include the -output of `prte_info --all` and a minimal reproduction case. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000000..47dc3e3d86 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/config/prte_setup_pmix.m4 b/config/prte_setup_pmix.m4 index 45da3ee618..9cbda53905 100644 --- a/config/prte_setup_pmix.m4 +++ b/config/prte_setup_pmix.m4 @@ -249,6 +249,27 @@ AC_DEFUN([PRTE_CHECK_PMIX],[ [$PRTE_PMIX_LTO_CAPABILITY], [Whether or not PMIx has the LTO capability flag set]) + dnl The elastic-DVM completion contract directs two PMIx events at the + dnl process that requested a DVM size change: PMIX_DVM_IS_READY on success + dnl and PMIX_ERR_DVM_MOD on failure. These are plain #define'd status + dnl codes, not behavioral capability flags, so probe for the symbols + dnl themselves rather than a PMIX_CAP_* flag. A PMIx that defines neither + dnl leaves the completion notification compiled out (see + dnl docs/plans/elastic_dvm/). + AC_MSG_CHECKING([for PMIx DVM modification event codes]) + AC_PREPROC_IFELSE( + [AC_LANG_PROGRAM([[#include +#if !defined(PMIX_DVM_IS_READY) || !defined(PMIX_ERR_DVM_MOD) +#error DVM modification event codes not present +#endif +]], [[]])], + [AC_MSG_RESULT([yes]) + AC_DEFINE([PRTE_HAVE_DVM_MOD_EVENTS], [1], + [Whether PMIx defines the DVM modification event codes])], + [AC_MSG_RESULT([no]) + AC_DEFINE([PRTE_HAVE_DVM_MOD_EVENTS], [0], + [Whether PMIx defines the DVM modification event codes])]) + # restore the global flags CPPFLAGS=$prte_external_pmix_save_CPPFLAGS LDFLAGS=$prte_external_pmix_save_LDFLAGS diff --git a/contrib/dockerswarm/.gitignore b/contrib/dockerswarm/.gitignore new file mode 100644 index 0000000000..558bf8e3a8 --- /dev/null +++ b/contrib/dockerswarm/.gitignore @@ -0,0 +1,2 @@ +# build.sh staging area (clean git-archive context + PMIx clone) +_work/ diff --git a/contrib/dockerswarm/Dockerfile b/contrib/dockerswarm/Dockerfile new file mode 100644 index 0000000000..5b14e5454b --- /dev/null +++ b/contrib/dockerswarm/Dockerfile @@ -0,0 +1,73 @@ +# Image for the PRRTE elastic-DVM test "swarm" (see README.md). +# +# Builds PMIx (cloned from git) and PRRTE (supplied in the build context by +# build.sh as a clean git-archive of the local committed tree) into /usr/local, +# plus the `elastic` test client and passwordless SSH between the container +# "nodes". Build this with ./build.sh, not a bare `docker build` -- build.sh +# prepares the PRRTE source the COPY below expects. + +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential gcc g++ make \ + autoconf automake libtool m4 flex \ + perl python3 git \ + libevent-dev libhwloc-dev zlib1g-dev \ + openssh-server openssh-client \ + iproute2 iputils-ping procps less vim ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# ---- build PMIx ---- +# Cloned with its submodules so autogen.pl runs normally. PMIx master is the +# default because the DVM size-change event codes (PMIX_DVM_IS_READY / +# PMIX_ERR_DVM_MOD) the elastic client registers for may not be in a release +# yet; override with --build-arg if you need a specific PMIx. +# Do NOT pass --with-libevent=/usr: it forces /usr/lib and misses the multiarch +# dir (/usr/lib/); let configure find the system libevent-dev/hwloc-dev. +ARG PMIX_REPO=https://github.com/openpmix/openpmix.git +ARG PMIX_REF=master +RUN git clone --recursive --depth=1 -b "$PMIX_REF" "$PMIX_REPO" /src/pmix \ + && cd /src/pmix \ + && ./autogen.pl \ + && ./configure --prefix=/usr/local \ + && make -j"$(nproc)" \ + && make install \ + && ldconfig + +# ---- build PRRTE (this repo's committed tree, from the build context) ---- +# build.sh git-archives the local tree into ./prrte with config/oac populated +# and .gitmodules removed, so autogen.pl skips its git-only submodule check and +# runs from the VERSION file. The archived tree has no .git, so configure does +# NOT imply --enable-devel-check (warnings-as-errors); --enable-debug still +# gives us debug symbols and assertions. +COPY prrte /src/prrte +RUN cd /src/prrte \ + && ./autogen.pl \ + && ./configure --prefix=/usr/local --with-pmix=/usr/local --enable-debug \ + && make -j"$(nproc)" \ + && make install \ + && ldconfig + +# ---- elastic test client ---- +COPY elastic.c /src/elastic.c +RUN gcc -O0 -g -o /usr/local/bin/elastic /src/elastic.c \ + -I/usr/local/include -L/usr/local/lib -lpmix \ + && ldconfig + +# ---- passwordless SSH between the container "nodes" ---- +RUN mkdir -p /var/run/sshd /root/.ssh \ + && ssh-keygen -t ed25519 -N "" -f /root/.ssh/id_ed25519 \ + && cp /root/.ssh/id_ed25519.pub /root/.ssh/authorized_keys \ + && printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n LogLevel ERROR\n' \ + > /root/.ssh/config \ + && chmod 600 /root/.ssh/* \ + && sed -i 's/#\?PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config \ + && printf '\nAcceptEnv *\n' >> /etc/ssh/sshd_config + +# make the install discoverable for any login shell too +RUN echo 'export PATH=/usr/local/bin:$PATH' > /etc/profile.d/prte.sh + +EXPOSE 22 +CMD ["/usr/sbin/sshd", "-D", "-e"] diff --git a/contrib/dockerswarm/README.md b/contrib/dockerswarm/README.md new file mode 100644 index 0000000000..f28200c56c --- /dev/null +++ b/contrib/dockerswarm/README.md @@ -0,0 +1,183 @@ +# PRRTE elastic-DVM test "swarm" + +A small, self-contained Docker harness for exercising PRRTE's **elastic DVM** +(grow/shrink) behavior — without a real PMIx scheduler — across several +container "nodes". It is the quickest way to bring up a multi-node DVM, drive a +grow and a shrink, and watch the two-phase completion events. + +It is **not** a Docker Swarm in the orchestration sense — just four plain +`ubuntu:24.04` containers on one bridge network, each acting as a DVM node. +"Swarm" is only the nickname. + +> Orientation for an AI agent or new contributor: this directory contains +> everything needed. Read this file top to bottom, then run the Quick start. +> The one thing that will silently waste your time is forgetting +> `--prtemca prte_elastic_mode 1` when starting the DVM — see §3. + +--- + +## 1. What's here + +| File | Purpose | +|------|---------| +| `build.sh` | Builds the `prte-elastic:latest` image from **this repo's committed tree** plus a cloned PMIx. Start here. | +| `Dockerfile` | Image recipe: builds PMIx + PRRTE + the `elastic` client into `/usr/local`, sets up passwordless SSH. Driven by `build.sh`. | +| `docker-compose.yml` | Defines the four nodes `prte-node1`..`prte-node4` on bridge network `dvm`. | +| `elastic.c` | The test client (`/usr/local/bin/elastic` in the image): issues a PMIx allocation request and waits for the phase-two completion event. | + +## 2. Prerequisites + +- Docker (with `docker compose`) and `git`. +- **Network access during the build** — the Dockerfile clones PMIx and installs + apt packages. +- Builds and runs as `aarch64` or `x86_64`; the image is arch-neutral. + +PMIx is cloned from **master** by default, because the DVM size-change event +codes the client registers for (`PMIX_DVM_IS_READY` / `PMIX_ERR_DVM_MOD`) may +not be in a tagged PMIx release yet. If you build against an older PMIx that +lacks them, PRRTE compiles those completion events out (and `elastic.c` will not +compile) — so stick with master unless you know your PMIx has the codes. + +## 3. Quick start + +```sh +# from this directory (contrib/dockerswarm/) +./build.sh # build prte-elastic:latest (PMIx master + this repo's HEAD) +docker compose up -d # start prte-node1 .. prte-node4 + +# convenience: run everything on the head node as root, elastic mode ON +RUN='docker exec -e PRTE_ALLOW_RUN_AS_ROOT=1 -e PRTE_ALLOW_RUN_AS_ROOT_CONFIRM=1 prte-node1 sh -c' + +# 1. start the DVM on node1 -- prte_elastic_mode is REQUIRED (see below) +$RUN 'cd /root && nohup prte --daemonize --prtemca prte_elastic_mode 1 \ + >/tmp/prte.out 2>&1 & sleep 6' + +# 2. baseline sanity (only node1 in the DVM so far) +$RUN 'prun --np 1 hostname' + +# 3. grow onto node2 + node3, then shrink node3 back out +$RUN 'elastic grow node2:2,node3:2' +$RUN 'elastic shrink node3' + +# 4. tear the test DVM down (leaves the containers running) +$RUN 'pterm' +``` + +Both `grow` and `shrink` should print +`PHASE 2 (completion): received event PMIX_DVM_IS_READY` followed by `SUCCESS`. + +> **The flag that bites you.** PRRTE gates *all* of the grow/shrink launch-fence +> and completion-event machinery behind the `prte_elastic_mode` MCA parameter +> (default off). If you start the DVM **without** `--prtemca prte_elastic_mode 1`, +> a grow returns phase-1 SUCCESS and even launches the daemons, but it **never +> completes** — no `PMIX_DVM_IS_READY`, the client times out after 60s, and the +> nodes never wire into the DVM. Always start with the flag. + +## 4. The `elastic` client + +``` +elastic grow # PMIX_ALLOC_NEW, naming nodes to ADD +elastic shrink # PMIX_ALLOC_RELEASE, naming nodes to REMOVE +``` + +(`extend` / `release` are accepted aliases.) It registers for both completion +codes, prints the phase-1 allocation response, then waits up to 60s for the +phase-2 event. + +A grow uses **`PMIX_ALLOC_NEW`** carrying `PMIX_ALLOC_NODE_LIST` (a new +reservation naming the nodes to add) — **not** `PMIX_ALLOC_EXTEND`, which only +enlarges an existing reservation. With no scheduler attached, PRRTE's `ras/hosts` +component satisfies these requests locally using the real node names, and +`plm/ssh` SSHes to the named nodes to launch their daemons (passwordless SSH is +baked into the image). + +## 5. What "success" looks like + +**Grow** (`elastic grow node2:2,node3:2`): phase-1 `PMIX_SUCCESS` with a +`pmix.alloc.id`, then phase-2 `PMIX_DVM_IS_READY`, and `prted` now running on +node2 and node3. + +> The grown nodes join the **reservation** created by the request, so a plain +> `prun -n 3 --map-by node hostname` still lands only on node1 — its default job +> allocation is node1's base pool, not the reservation. That is the elastic +> pooling model, not a failure. Confirm a grow by `prted` presence on the target +> nodes and the `PMIX_DVM_IS_READY` event, not by plain-prun placement: +> +> ```sh +> for n in 2 3; do docker exec prte-node$n pgrep -ax prted; done +> ``` + +**Shrink** (`elastic shrink node3`): phase-1 `PMIX_SUCCESS`, then phase-2 +`PMIX_DVM_IS_READY`, plus a **"PRRTE has lost communication with a remote +daemon"** notice naming the shrunk node. **That notice is expected** — shrink +completion is driven by the targeted daemon's actual *death* (the comm-failure +path), not by an acknowledgement. The HNP must survive it. Afterward: HNP alive, +node3's `prted` gone, node2's `prted` still alive, `prun -n 1 hostname` still +works. + +## 6. Cleanup hygiene (prevents "multiple possible servers") + +Between DVM runs, clean stale state on **every** node: + +```sh +for n in 1 2 3 4; do + docker exec prte-node$n sh -c 'pkill -9 -x prted; pkill -9 -x prte; + rm -rf /tmp/prte.* /tmp/prun.session.*; true' +done +``` + +A detached `prted --daemonize` **survives an HNP kill** — if you only kill +node1's `prte`, orphan `prted`s linger on the other nodes and the next DVM trips +over stale rendezvous files reporting *"multiple possible servers"*. The `pkill` +loop across all four nodes is mandatory, not optional. `pterm` is the clean way +to bring a healthy DVM down; still run the loop afterward to be safe. + +## 7. Rebuilding after a code change + +`build.sh` always rebuilds from your **committed** tree (it uses `git archive`). +After committing a change: + +```sh +./build.sh && docker compose up -d --force-recreate +``` + +For a fast edit/test loop on **uncommitted** PRRTE changes without a full image +rebuild, each running container already has the build tree at `/src/prrte`. Copy +the changed files in and run an incremental build **on all four nodes** (every +node runs a `prted` that loads the libraries, so an ABI/header change must be +consistent across the DVM): + +```sh +ROOT=$(git rev-parse --show-toplevel) +FILES="src/mca/ras/ras.h src/mca/ras/base/base.h \ + src/mca/ras/base/ras_base_allocate.c src/mca/errmgr/dvm/errmgr_dvm.c" +for n in 1 2 3 4; do + for f in $FILES; do docker cp "$ROOT/$f" "prte-node$n:/src/prrte/$f"; done + docker exec prte-node$n sh -c \ + 'cd /src/prrte && make -j"$(nproc)" && make install && ldconfig' +done +``` + +PMIx is untouched by PRRTE-only edits, so you never rebuild PMIx for a PRRTE +change. + +## 8. Known issue + +Re-growing a node **immediately after** shrinking it can fail its TCP +connect-back (`prted` exits 255) — the just-killed daemon/session may not have +fully torn down — and the HNP does not always survive that cleanly. Tracked as +[openpmix/prrte#2491](https://github.com/openpmix/prrte/issues/2491). Start a +fresh DVM between grow/shrink cycles to avoid it. A single grow→shrink cycle on +a fresh DVM is the reliable smoke test. + +## 9. Topology reference + +| Container | hostname | role | +|-----------|----------|------| +| `prte-node1` | node1 | head node (HNP) — start `prte` here, run all tools here | +| `prte-node2` | node2 | DVM node (grow target) | +| `prte-node3` | node3 | DVM node (grow/shrink target) | +| `prte-node4` | node4 | spare DVM node | + +Network: bridge `dvm`. To add more nodes, copy a service block in +`docker-compose.yml`. diff --git a/contrib/dockerswarm/build.sh b/contrib/dockerswarm/build.sh new file mode 100755 index 0000000000..77290916d2 --- /dev/null +++ b/contrib/dockerswarm/build.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# +# Copyright (c) 2026 Nanook Consulting All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# +# Build the prte-elastic:latest image used by the DVM test swarm (README.md). +# +# It exports this repo's committed PRRTE tree (default: HEAD) into a clean build +# context via `git archive` -- so the image contains exactly your committed code, +# with no host build artifacts and no architecture mismatch -- then builds the +# Dockerfile, which clones PMIx and compiles both into /usr/local. +# +# Test a different committed state with PRRTE_REF, or a specific PMIx with +# PMIX_REF / PMIX_REPO. Examples: +# ./build.sh +# PRRTE_REF=topic/lnch ./build.sh +# PMIX_REF=v6.1.0 ./build.sh +# +# Requires: docker, git, and network access during the build (for PMIx + apt). + +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +root="$(git -C "$here" rev-parse --show-toplevel)" + +IMAGE="${IMAGE:-prte-elastic:latest}" +PRRTE_REF="${PRRTE_REF:-HEAD}" +PMIX_REPO="${PMIX_REPO:-https://github.com/openpmix/openpmix.git}" +PMIX_REF="${PMIX_REF:-master}" + +ctx="$here/_work/context" +rm -rf "$ctx" +mkdir -p "$ctx" + +echo ">>> exporting PRRTE source ($PRRTE_REF) from $root" +git -C "$root" archive --prefix=prrte/ "$PRRTE_REF" | tar -x -C "$ctx" + +# git archive omits submodule contents, so add config/oac (needed by autogen's +# m4) separately. Make sure it is checked out on the host first. +echo ">>> adding config/oac submodule contents" +git -C "$root" submodule update --init -- config/oac >/dev/null 2>&1 || true +git -C "$root/config/oac" archive --prefix=prrte/config/oac/ HEAD | tar -x -C "$ctx" + +# Drop .gitmodules so the in-container autogen.pl skips its submodule check, +# which would otherwise run `git submodule status` and fail (no .git here). +rm -f "$ctx/prrte/.gitmodules" + +cp "$here/Dockerfile" "$here/elastic.c" "$ctx/" + +echo ">>> docker build $IMAGE (PMIx $PMIX_REF)" +docker build -t "$IMAGE" \ + --build-arg PMIX_REPO="$PMIX_REPO" \ + --build-arg PMIX_REF="$PMIX_REF" \ + "$ctx" + +rm -rf "$here/_work" +echo ">>> done: built $IMAGE" +echo ">>> next: docker compose up -d (then see README.md)" diff --git a/contrib/dockerswarm/docker-compose.yml b/contrib/dockerswarm/docker-compose.yml new file mode 100644 index 0000000000..094b80dbd6 --- /dev/null +++ b/contrib/dockerswarm/docker-compose.yml @@ -0,0 +1,33 @@ +# A 4-"node" PRRTE DVM on a private docker network for elastic grow/shrink +# testing. node1 is the head node (where you start `prte` and run the tests); +# node2..node4 are spare nodes the DVM can grow onto and shrink back out of. +# Each container just runs sshd; we `docker exec` into node1 to drive the DVM. +services: + node1: + image: prte-elastic:latest + hostname: node1 + container_name: prte-node1 + networks: [dvm] + tty: true + node2: + image: prte-elastic:latest + hostname: node2 + container_name: prte-node2 + networks: [dvm] + tty: true + node3: + image: prte-elastic:latest + hostname: node3 + container_name: prte-node3 + networks: [dvm] + tty: true + node4: + image: prte-elastic:latest + hostname: node4 + container_name: prte-node4 + networks: [dvm] + tty: true + +networks: + dvm: + driver: bridge diff --git a/contrib/dockerswarm/elastic.c b/contrib/dockerswarm/elastic.c new file mode 100644 index 0000000000..ba6498b593 --- /dev/null +++ b/contrib/dockerswarm/elastic.c @@ -0,0 +1,198 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2026 Nanook Consulting All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * Elastic-DVM test client. + * + * Connects to a running PRRTE DVM as a PMIx tool, registers handlers for the + * two DVM size-change completion events, and issues a grow or shrink request + * naming a node list. It then waits for the directed PMIX_DVM_IS_READY + * (success) or PMIX_ERR_DVM_MOD (failure) event so the two-phase completion + * contract can be observed end to end. + * + * elastic grow # grow the DVM onto these nodes + * elastic shrink # shrink the DVM by these nodes + * + * (extend/release are accepted as aliases for grow/shrink.) + * + * Build: gcc -o elastic elastic.c -lpmix + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +/* tiny lock so the main thread can block on async callbacks */ +typedef struct { + pthread_mutex_t mtx; + pthread_cond_t cond; + volatile int active; + pmix_status_t status; +} lock_t; + +static void lock_init(lock_t *l) { + pthread_mutex_init(&l->mtx, NULL); + pthread_cond_init(&l->cond, NULL); + l->active = 1; + l->status = PMIX_SUCCESS; +} +static void lock_wait(lock_t *l) { + pthread_mutex_lock(&l->mtx); + while (l->active) { + pthread_cond_wait(&l->cond, &l->mtx); + } + pthread_mutex_unlock(&l->mtx); +} +static void lock_wake(lock_t *l, pmix_status_t st) { + pthread_mutex_lock(&l->mtx); + l->status = st; + l->active = 0; + pthread_cond_signal(&l->cond); + pthread_mutex_unlock(&l->mtx); +} + +static lock_t complete_lock; /* fired by the DVM_IS_READY / ERR_DVM_MOD handler */ +static pmix_proc_t myproc; + +/* handler registration callback */ +static void reg_cb(pmix_status_t status, size_t evref, void *cbdata) { + lock_t *l = (lock_t *) cbdata; + (void) evref; + lock_wake(l, status); +} + +/* allocation-request response (phase one: acceptance) */ +static void alloc_cb(pmix_status_t status, pmix_info_t *info, size_t ninfo, + void *cbdata, pmix_release_cbfunc_t release_fn, void *release_cbdata) { + lock_t *l = (lock_t *) cbdata; + size_t n; + fprintf(stderr, ">>> PHASE 1 (acceptance): allocation request returned %s\n", + PMIx_Error_string(status)); + for (n = 0; n < ninfo; n++) { + if (PMIX_STRING == info[n].value.type) { + fprintf(stderr, " info[%zu] %s = %s\n", n, info[n].key, + info[n].value.data.string); + } else { + fprintf(stderr, " info[%zu] key=%s\n", n, info[n].key); + } + } + if (NULL != release_fn) { + release_fn(release_cbdata); + } + lock_wake(l, status); +} + +/* phase two: the directed completion event */ +static void completion_evh(size_t evhdlr_id, pmix_status_t status, + const pmix_proc_t *source, pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, void *cbdata) { + size_t n; + (void) evhdlr_id; (void) source; (void) results; (void) nresults; + + fprintf(stderr, "\n>>> PHASE 2 (completion): received event %s (%d)\n", + PMIx_Error_string(status), status); + for (n = 0; n < ninfo; n++) { + fprintf(stderr, " payload[%zu] key=%s\n", n, info[n].key); + } + if (PMIX_DVM_IS_READY == status) { + fprintf(stderr, ">>> SUCCESS: the DVM now reflects the requested size\n"); + } else if (PMIX_ERR_DVM_MOD == status) { + fprintf(stderr, ">>> FAILURE: the DVM modification did not happen\n"); + } + if (NULL != cbfunc) { + cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata); + } + lock_wake(&complete_lock, status); +} + +int main(int argc, char **argv) { + pmix_status_t rc; + pmix_info_t *info; + pmix_alloc_directive_t directive; + pmix_status_t codes[2]; + lock_t reglock; + lock_t alloclock; + const char *op, *nodelist; + + if (argc < 3) { + fprintf(stderr, "usage: %s \n", argv[0]); + return 1; + } + op = argv[1]; + nodelist = argv[2]; + /* A grow is a NEW reservation that names the nodes to add: PRRTE adds them + * to the pool and extends the DVM. PMIX_ALLOC_EXTEND only adds to an + * already-existing reservation, so it is not the right trigger here. */ + if (0 == strcmp(op, "grow") || 0 == strcmp(op, "extend")) { + directive = PMIX_ALLOC_NEW; + } else if (0 == strcmp(op, "shrink") || 0 == strcmp(op, "release")) { + directive = PMIX_ALLOC_RELEASE; + } else { + fprintf(stderr, "unknown op '%s' (want grow|shrink)\n", op); + return 1; + } + + if (PMIX_SUCCESS != (rc = PMIx_tool_init(&myproc, NULL, 0))) { + fprintf(stderr, "PMIx_tool_init failed: %s\n", PMIx_Error_string(rc)); + return 1; + } + fprintf(stderr, "tool %s:%d connected to the DVM\n", myproc.nspace, myproc.rank); + + /* register for BOTH completion codes in one handler */ + lock_init(&complete_lock); + lock_init(®lock); + codes[0] = PMIX_DVM_IS_READY; + codes[1] = PMIX_ERR_DVM_MOD; + PMIx_Register_event_handler(codes, 2, NULL, 0, completion_evh, reg_cb, ®lock); + lock_wait(®lock); + fprintf(stderr, "registered for PMIX_DVM_IS_READY / PMIX_ERR_DVM_MOD\n"); + + /* issue the size-change request naming the node list */ + lock_init(&alloclock); + PMIX_INFO_CREATE(info, 2); + PMIX_INFO_LOAD(&info[0], PMIX_ALLOC_NODE_LIST, nodelist, PMIX_STRING); + PMIX_INFO_LOAD(&info[1], PMIX_ALLOC_REQ_ID, "elastic-test", PMIX_STRING); + fprintf(stderr, "requesting %s of nodes [%s] ...\n", op, nodelist); + rc = PMIx_Allocation_request_nb(directive, info, 2, alloc_cb, &alloclock); + if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) { + fprintf(stderr, "PMIx_Allocation_request_nb failed immediately: %s\n", + PMIx_Error_string(rc)); + goto done; + } + lock_wait(&alloclock); + PMIX_INFO_FREE(info, 2); + + fprintf(stderr, "waiting for phase-two completion event (60s) ...\n"); + /* crude bounded wait so the tool can't hang forever in a test */ + { + struct timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + ts.tv_sec += 60; + pthread_mutex_lock(&complete_lock.mtx); + while (complete_lock.active) { + if (ETIMEDOUT == pthread_cond_timedwait(&complete_lock.cond, + &complete_lock.mtx, &ts)) { + fprintf(stderr, ">>> TIMEOUT: no completion event within 60s\n"); + break; + } + } + pthread_mutex_unlock(&complete_lock.mtx); + } + +done: + PMIx_tool_finalize(); + return 0; +} diff --git a/docs/how-things-work/index.rst b/docs/how-things-work/index.rst index 6c4cafa011..ac1a0b95af 100644 --- a/docs/how-things-work/index.rst +++ b/docs/how-things-work/index.rst @@ -12,3 +12,4 @@ find information on that subject here. session_dirs.rst per-app-mapping.rst schedulers/index.rst + state_machine.rst diff --git a/docs/how-things-work/state_machine.rst b/docs/how-things-work/state_machine.rst new file mode 100644 index 0000000000..53fb793197 --- /dev/null +++ b/docs/how-things-work/state_machine.rst @@ -0,0 +1,748 @@ +.. _state-machine-label: + +Job Launch State Machine +======================== + +PRRTE drives the full lifecycle of a job — from daemon launch through +application launch and termination — through an explicit, event-driven state +machine. Every transition is represented as an event posted to the PRRTE +progress thread; the callback for each state runs single-threaded, performs +its work, and posts the next event when done. Nothing blocks the calling +thread and there are no race conditions between state handlers. + +There are two cooperating state machines: one for **jobs** (tracking the +lifecycle of an entire job or the DVM itself) and one for **processes** +(tracking each individual application process). + +Architecture +------------ + +The state machine is implemented in ``src/mca/state/``. The **DVM module** +(``src/mca/state/dvm/state_dvm.c``) is used when ``prte`` runs as a +persistent Distributed Virtual Machine; it owns the authoritative ordered +table of states and callbacks. The **prted module** +(``src/mca/state/prted/state_prted.c``) runs inside each daemon and handles +only the small set of states relevant to a daemon's local work. + +The state machine is a linked list (``prte_job_states``) of +``(state, callback)`` pairs. The macro ``PRTE_ACTIVATE_JOB_STATE(jdata, +state)`` packages the job object and the target state into a *caddy* and +posts it to the event loop. The matching callback is looked up and invoked +asynchronously. + +Job State Definitions +--------------------- + +All job-state constants are defined in +``src/mca/plm/plm_types.h`` (lines 116–194). The states relevant to daemon +launch, in numeric order, are: + +.. list-table:: + :widths: 35 10 55 + :header-rows: 1 + + * - Name + - Value + - Meaning + * - ``PRTE_JOB_STATE_INIT`` + - 1 + - Job record created; ready to receive a job ID. + * - ``PRTE_JOB_STATE_INIT_COMPLETE`` + - 2 + - Job ID assigned; initial setup done. + * - ``PRTE_JOB_STATE_ALLOCATE`` + - 3 + - Ready to request resources from the scheduler/RAS. + * - ``PRTE_JOB_STATE_ALLOCATION_COMPLETE`` + - 4 + - Resource allocation finished. + * - ``PRTE_JOB_STATE_LAUNCH_DAEMONS`` + - 8 + - Ready to spawn ``prted`` processes. *Not* in the DVM default table; + registered by each PLM component at startup. + * - ``PRTE_JOB_STATE_DAEMONS_LAUNCHED`` + - 9 + - The PLM has initiated daemon spawning; waiting for daemons to call home. + * - ``PRTE_JOB_STATE_DAEMONS_REPORTED`` + - 10 + - All expected daemons have connected and sent their contact information. + * - ``PRTE_JOB_STATE_VM_READY`` + - 11 + - The DVM is fully operational; node map and wireup info have been + broadcast to all daemons. + * - ``PRTE_JOB_STATE_MAP`` + - 5 + - Ready to map processes to nodes. + * - ``PRTE_JOB_STATE_MAP_COMPLETE`` + - 6 + - Process mapping finished. + * - ``PRTE_JOB_STATE_SYSTEM_PREP`` + - 7 + - Final sanity checks and environment setup before launch. + * - ``PRTE_JOB_STATE_LAUNCH_APPS`` + - 12 + - Ready to send launch directives to daemons. + * - ``PRTE_JOB_STATE_SEND_LAUNCH_MSG`` + - 13 + - Launch message being assembled and sent. + * - ``PRTE_JOB_STATE_STARTED`` + - 20 + - At least one application process has been forked. + * - ``PRTE_JOB_STATE_LOCAL_LAUNCH_COMPLETE`` + - 18 + - All local processes on a daemon have attempted to launch. + * - ``PRTE_JOB_STATE_READY_FOR_DEBUG`` + - 19 + - All local processes report ready for a debugger attach. + * - ``PRTE_JOB_STATE_RUNNING`` + - 14 + - All processes across all daemons have been forked. + * - ``PRTE_JOB_STATE_REGISTERED`` + - 16 + - All processes have registered with the PMIx server (called + ``PMIx_Init``). + +Termination states (values ≥ 30) and error states (values ≥ 51) are +described at the bottom of this page. + +The Daemon Launch Sequence +-------------------------- + +The DVM module registers the following ordered table at startup +(``src/mca/state/dvm/state_dvm.c``, ``launch_states[]`` / +``launch_callbacks[]``): + +.. code-block:: text + + State Callback + ───────────────────────────────────────────────────────────────────── + PRTE_JOB_STATE_INIT prte_plm_base_setup_job + PRTE_JOB_STATE_INIT_COMPLETE init_complete (dvm-local) + PRTE_JOB_STATE_ALLOCATE prte_ras_base_allocate + PRTE_JOB_STATE_ALLOCATION_COMPLETE prte_plm_base_allocation_complete + PRTE_JOB_STATE_DAEMONS_LAUNCHED prte_plm_base_daemons_launched + PRTE_JOB_STATE_DAEMONS_REPORTED prte_plm_base_daemons_reported + PRTE_JOB_STATE_VM_READY vm_ready (dvm-local) + PRTE_JOB_STATE_MAP prte_rmaps_base_map_job + PRTE_JOB_STATE_MAP_COMPLETE prte_plm_base_mapping_complete + PRTE_JOB_STATE_SYSTEM_PREP prte_plm_base_complete_setup + PRTE_JOB_STATE_LAUNCH_APPS prte_plm_base_launch_apps + PRTE_JOB_STATE_SEND_LAUNCH_MSG prte_plm_base_send_launch_msg + PRTE_JOB_STATE_STARTED job_started (dvm-local) + PRTE_JOB_STATE_LOCAL_LAUNCH_COMPLETE prte_state_base_local_launch_complete + PRTE_JOB_STATE_READY_FOR_DEBUG ready_for_debug (dvm-local) + PRTE_JOB_STATE_RUNNING prte_plm_base_post_launch + PRTE_JOB_STATE_REGISTERED prte_plm_base_registered + PRTE_JOB_STATE_TERMINATED check_complete (dvm-local) + PRTE_JOB_STATE_NOTIFY_COMPLETED dvm_notify (dvm-local) + PRTE_JOB_STATE_NOTIFIED cleanup_job (dvm-local) + PRTE_JOB_STATE_ALL_JOBS_COMPLETE prte_quit + + (plus DAEMONS_TERMINATED → prte_quit and FORCED_EXIT → force_quit, + registered separately) + +Note that ``PRTE_JOB_STATE_LAUNCH_DAEMONS`` is **not** in this table. +Each Process Launch Manager (PLM) component—ssh, slurm, pals, lsf—inserts +its own ``launch_daemons`` callback for that state during its own ``init``. + +Step-by-step walk-through +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**1. INIT → prte_plm_base_setup_job** + +The job record is validated and initial app-context setup is performed. +On success the callback posts ``INIT_COMPLETE``. + +**2. INIT_COMPLETE → init_complete** + +The DVM-local ``init_complete`` immediately posts ``ALLOCATE`` so that a +potential DVM expansion can go through the allocation step. + +**3. ALLOCATE → prte_ras_base_allocate** + +The Resource Allocation Subsystem (RAS) queries the scheduler or hostfile +for available nodes and records them in the node pool. On completion it +posts ``ALLOCATION_COMPLETE``. + +**4. ALLOCATION_COMPLETE → prte_plm_base_allocation_complete** + +Decision point (``src/mca/plm/base/plm_base_launch_support.c``:186): + +* If ``PRTE_JOB_DO_NOT_LAUNCH`` is set (e.g., ``--map-by :display``), skip + daemon spawning entirely and jump straight to ``DAEMONS_REPORTED``. +* Otherwise, post ``LAUNCH_DAEMONS``. + +**5. LAUNCH_DAEMONS → ** + +This state is handled by the active PLM component, not by the DVM module. +The ssh PLM's handler (``src/mca/plm/ssh/plm_ssh_module.c``:1077) is +representative: + +a. Calls ``prte_plm_base_setup_virtual_machine()`` to compute which nodes + need new daemons (nodes already hosting a daemon from a prior job are + reused). +b. If no new daemons are needed (``map->num_new_daemons == 0``), fast-paths + to ``DAEMONS_REPORTED``. +c. Otherwise, builds the ``prted`` command line and spawns one daemon per + node via ssh (or pdsh, or the equivalent for slurm/pals/lsf). +d. Registers ``prte_plm_base_daemon_callback`` on + ``PRTE_RML_TAG_DAEMON_REPORT`` to hear from daemons as they start. +e. Posts ``DAEMONS_LAUNCHED`` to indicate spawning has been initiated. + +**6. DAEMONS_LAUNCHED → prte_plm_base_daemons_launched** + +This callback is intentionally a no-op +(``src/mca/plm/base/plm_base_launch_support.c``:218). The state machine +parks here and waits for daemons to call home asynchronously. + +**7. Daemons call home (asynchronous)** + +As each ``prted`` process starts up it: + +a. Initializes via its ESS (Environment-Specific Services) component. +b. Connects to the HNP (Head Node Process) via the RML. +c. Sends a report containing its process name, RML contact URI, node name, + and hwloc topology to the HNP on ``PRTE_RML_TAG_DAEMON_REPORT``. + +The HNP receives these reports in ``prte_plm_base_daemon_callback`` +(``src/mca/plm/base/plm_base_launch_support.c``:1237). For each arriving +daemon it: + +* Records the daemon's contact URI (stored via ``PMIx_Store_internal`` as + ``PMIX_PROC_URI``). +* Records the node name and hwloc topology. +* Marks the node ``PRTE_NODE_STATE_UP``. +* Increments ``jdatorted->num_reported``. +* Calls ``progress_daemons()`` (line 1173), which fires + ``DAEMONS_REPORTED`` once ``num_reported == num_procs``. + +**8. DAEMONS_REPORTED → prte_plm_base_daemons_reported** + +(``src/mca/plm/base/plm_base_launch_support.c``:118) + +* If using an unmanaged allocation (e.g., a hostfile), sets the default + slot count on each node according to ``--set-slots`` (cores, sockets, + hwthreads, or a literal number). +* Totals up ``jdata->total_slots_alloc``. +* Posts ``VM_READY``. + +At this point every daemon is up and the HNP knows how to reach each of +them. + +**9. VM_READY → vm_ready** + +(``src/mca/state/dvm/state_dvm.c``:261) + +If new daemons were actually launched (``PRTE_JOB_LAUNCHED_DAEMONS`` is +set) and more than one daemon is running: + +* Serializes the node map via ``prte_util_nidmap_create()`` into a buffer. +* Looks up each daemon's ``PMIX_PROC_URI`` and packs it into the same + buffer. +* Broadcasts the combined nidmap + wireup buffer to all daemons via + ``prte_grpcomm.xcast(PRTE_RML_TAG_WIREUP, &buf)``. + +After the broadcast: + +* Sets ``prte_dvm_ready = true``. +* If running as a persistent DVM (``prte`` without an immediate job), + prints ``"DVM ready\n"`` to stdout or writes a ``'K'`` byte on the + parent pipe so the caller knows the DVM is accepting work. +* Dispatches any jobs that arrived and were cached while the DVM was + starting (``prte_cache``). + +**The DVM is now fully operational.** For a standalone ``prterun`` +invocation the state machine continues immediately into the app-launch +phase below. + +Application Launch (after the DVM is ready) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Once the DVM is ready, each new application job that arrives at the HNP +(via ``PRTE_PLM_LAUNCH_JOB_CMD``) goes through a fast-path re-entry into +the state machine (``plm_base_receive.c``:470). If ``prte_dvm_ready`` is +not yet true (initial DVM startup still in progress), the job is stashed in +``prte_cache`` and flushed when ``vm_ready`` fires for the daemon job. +Otherwise the job enters the state machine immediately via +``prte_plm.spawn(jdata)``. + +A DVM can run many application jobs concurrently. Each follows the same +state machine independently. + +**10. MAP → prte_rmaps_base_map_job** + +(``src/mca/rmaps/base/``) + +The RMAPS framework assigns each application process to a specific node and +slot. The mapping policy (``--map-by slot``, ``--map-by node``, +``--map-by core``, ``--map-by ppr:N:L``, etc.) determines how processes +are distributed. + +Key actions: + +* Iterates over the node pool for the job's session. +* For each app context, calls the selected RMAPS component + (e.g., ``rmaps_round_robin``, ``rmaps_ppr``, ``rmaps_rank_file``). +* Each component calls ``prte_rmaps_base_claim_slot()`` to assign a process + to a node; this creates a ``prte_proc_t`` entry and links it to the node. +* Sets ``jdata->num_procs``. +* If ``--rank-by`` or ``--bind-to`` were specified, records those policies + in the map for use during launch. + +On completion, fires ``MAP_COMPLETE``. + +**11. MAP_COMPLETE → prte_plm_base_mapping_complete** + +(``plm_base_launch_support.c``:276) + +Posts ``SYSTEM_PREP``. + +**12. SYSTEM_PREP → prte_plm_base_complete_setup** + +(``plm_base_launch_support.c``) + +Performs pre-launch sanity checks and environment preparation: + +* Validates that there are enough slots for the requested process count. +* Constructs the environment for each app context (inheriting the HNP + environment, applying ``-x VAR``, ``--env-merge``, and PMIx-standard + keys). +* Calls ``prte_filem.preposition_files()`` to stage any required input + files to the compute nodes. The ``files_ready`` callback fires on + completion; on success it activates ``MAP`` — **wait, this is actually + activated from** ``vm_ready`` **for the app-job path; see below**. + +.. note:: + ``SYSTEM_PREP``'s callback ``prte_plm_base_complete_setup`` does the + environment/slot validation and then fires ``LAUNCH_APPS``. File + staging happens earlier, inside ``vm_ready``, before MAP is activated. + The call chain is: ``vm_ready`` → ``preposition_files`` → + ``files_ready`` → ``MAP`` → ... → ``SYSTEM_PREP`` → ``LAUNCH_APPS``. + +**13. LAUNCH_APPS → prte_plm_base_launch_apps** + +(``plm_base_launch_support.c``) + +Prepares the per-daemon launch data and posts ``SEND_LAUNCH_MSG``. + +**14. SEND_LAUNCH_MSG → prte_plm_base_send_launch_msg** + +(``plm_base_launch_support.c``) + +Builds and sends an ODLS (On-node Daemon Launch Subsystem) launch message +to each daemon that has local processes for this job. The message contains: + +* The job's namespace and process list. +* Per-process slot list (cpuset, binding directives). +* Application argv and environment. +* IOF (I/O Forwarding) channel setup — which file descriptors to forward + for each process. +* Any PMIx server info that the processes will need at init time. + +Each daemon receives the message via ``PRTE_RML_TAG_LAUNCH_APPS`` and +passes it to its ODLS component. The ODLS ``launch_local_procs()`` entry +point iterates over the local process list and ``fork``/``exec``'s each +one. After the exec, the child process calls ``PMIx_Init`` which connects +it to the daemon's embedded PMIx server. + +**15. STARTED → job_started** + +Fires once the first process has been forked on any daemon (triggered by +``PRTE_PLM_LOCAL_LAUNCH_COMP_CMD`` receipt at the HNP—see step 16). +Notifies the originating tool via a PMIx ``PMIX_EVENT_JOB_START`` event. + +**16. LOCAL_LAUNCH_COMPLETE** + +Each daemon sends ``PRTE_PLM_LOCAL_LAUNCH_COMP_CMD`` back to the HNP when +all of its local processes have attempted to start, carrying each process's +PID and state. The HNP handler (``plm_base_receive.c``:715) accumulates +``jdata->num_launched``; when the first process is counted it posts +``STARTED``; when all processes are counted it posts ``RUNNING``. + +**17. READY_FOR_DEBUG → ready_for_debug** + +Optional. If the job was submitted with ``--stop-on-exec``, +``--stop-in-init``, or ``--stop-in-app``, each daemon waits until all its +local processes signal readiness and then sends +``PRTE_PLM_READY_FOR_DEBUG_CMD`` to the HNP. When the HNP has heard from +all daemons it fires a ``PMIX_READY_FOR_DEBUG`` PMIx event to the +originating tool. + +**18. RUNNING → prte_plm_base_post_launch** + +All processes across the entire job are running. Post-launch cleanup: +timeout timers, progress callbacks, and similar housekeeping. + +**19. REGISTERED → prte_plm_base_registered** + +All application processes have called ``PMIx_Init`` and registered with +their local PMIx server. Each daemon accumulates its local count and +sends ``PRTE_PLM_REGISTERED_CMD`` to the HNP when all of its local +processes have registered. The HNP handler +(``plm_base_receive.c``:675) increments ``jdata->num_reported``; when the +count reaches ``jdata->num_procs`` it fires this state. + +Process State Machine +--------------------- + +The process state machine tracks individual application processes. It +runs on both the HNP (via the DVM module) and each daemon (via the prted +module), with the same set of states and a single callback +``prte_state_base_track_procs`` / ``track_procs``. + +.. list-table:: + :widths: 40 10 50 + :header-rows: 1 + + * - Name + - Value + - Meaning + * - ``PRTE_PROC_STATE_INIT`` + - 1 + - Process entry created by RMAPS. + * - ``PRTE_PROC_STATE_RUNNING`` + - 4 + - Daemon has forked the process. + * - ``PRTE_PROC_STATE_REGISTERED`` + - 5 + - Process called ``PMIx_Init``. + * - ``PRTE_PROC_STATE_IOF_COMPLETE`` + - 6 + - All I/O forwarding pipes have closed. + * - ``PRTE_PROC_STATE_WAITPID_FIRED`` + - 7 + - ``waitpid`` detected the process has exited. + * - ``PRTE_PROC_STATE_READY_FOR_DEBUG`` + - 9 + - Process is stopped and awaiting a debugger. + * - ``PRTE_PROC_STATE_TERMINATED`` + - 20 + - Process is fully cleaned up. + +A process is considered still running if its state is less than +``PRTE_PROC_STATE_UNTERMINATED`` (15). States ≥ +``PRTE_PROC_STATE_ERROR`` (50) indicate abnormal exit. + +On the daemon side (``src/mca/state/prted/state_prted.c``:314, +``track_procs``): + +* ``RUNNING``: increments ``jdata->num_launched``; when all local procs + are running, fires ``PRTE_JOB_STATE_LOCAL_LAUNCH_COMPLETE`` which + sends ``PRTE_PLM_LOCAL_LAUNCH_COMP_CMD`` to the HNP. +* ``REGISTERED``: increments ``jdata->num_reported``; when all local procs + have registered, sends ``PRTE_PLM_REGISTERED_CMD`` to the HNP. +* ``IOF_COMPLETE`` / ``WAITPID_FIRED``: when both flags are set for a + process, marks it ``TERMINATED`` and triggers job-completion accounting. + +Termination and Error States +---------------------------- + +**Boundary markers** (job states): + +* ``PRTE_JOB_STATE_UNTERMINATED`` (30): any state below this means the job + is still running. +* ``PRTE_JOB_STATE_ERROR`` (50): any state at or above this is an error. + +**Normal termination sequence**: + +``TERMINATED`` → ``NOTIFY_COMPLETED`` → ``NOTIFIED`` → ``ALL_JOBS_COMPLETE`` +→ ``prte_quit`` + +**Selected error states**: + +.. list-table:: + :widths: 50 10 + :header-rows: 1 + + * - Name + - Value + * - ``PRTE_JOB_STATE_KILLED_BY_CMD`` + - 51 + * - ``PRTE_JOB_STATE_ABORTED`` + - 52 + * - ``PRTE_JOB_STATE_FAILED_TO_START`` + - 53 + * - ``PRTE_JOB_STATE_NEVER_LAUNCHED`` + - 60 + * - ``PRTE_JOB_STATE_ALLOC_FAILED`` + - 68 + * - ``PRTE_JOB_STATE_MAP_FAILED`` + - 69 + * - ``PRTE_JOB_STATE_CANNOT_LAUNCH`` + - 70 + * - ``PRTE_JOB_STATE_FORCED_EXIT`` + - 64 + +All error states ultimately route to ``force_quit`` or ``prte_quit`` which +calls ``prte_plm.terminate_orteds()`` before exiting. + +Key Source Files +---------------- + +.. list-table:: + :widths: 45 55 + :header-rows: 1 + + * - File + - Role + * - ``src/mca/plm/plm_types.h`` + - All state constant definitions. + * - ``src/mca/state/dvm/state_dvm.c`` + - DVM job and proc state tables; ``vm_ready``, ``init_complete``, + ``check_complete``, ``dvm_notify``, ``cleanup_job``. + * - ``src/mca/state/prted/state_prted.c`` + - Per-daemon job and proc state tables; ``track_procs``, + ``track_jobs``. + * - ``src/mca/state/base/state_base_fns.c`` + - ``prte_state_base_activate_job_state`` — the core dispatch function. + * - ``src/mca/plm/base/plm_base_launch_support.c`` + - Most PLM base callbacks: ``prte_plm_base_setup_job``, + ``prte_plm_base_allocation_complete``, + ``prte_plm_base_daemons_launched``, + ``prte_plm_base_daemons_reported``, ``progress_daemons``, + ``prte_plm_base_daemon_callback``. + * - ``src/mca/plm/base/plm_base_receive.c`` + - HNP message handler: processes ``PRTE_PLM_LOCAL_LAUNCH_COMP_CMD`` + and ``PRTE_PLM_REGISTERED_CMD`` from daemons. + * - ``src/mca/plm/ssh/plm_ssh_module.c`` + - SSH PLM ``launch_daemons`` callback (line 1077). + * - ``src/mca/plm/slurm/plm_slurm_module.c`` + - SLURM PLM ``launch_daemons`` callback. + * - ``src/mca/plm/pals/plm_pals_module.c`` + - PALS PLM ``launch_daemons`` callback. + * - ``src/mca/plm/lsf/plm_lsf_module.c`` + - LSF PLM ``launch_daemons`` callback. + * - ``src/mca/ras/base/ras_base_allocate.c`` + - ``prte_ras_base_add_hosts()`` (thin async wrapper, line 771); + ``prte_ras_base_complete_request()`` (grow/shrink completion, line + 586); ``prte_ras_base_modify()`` (routes requests to RAS modules, + line 529). + * - ``src/mca/ras/hosts/ras_hosts.c`` + - ``ras/hosts`` module ``modify()`` entry point: parses hostfiles and + host lists and inserts nodes into the pool (line 340). + * - ``src/mca/ras/slurm/ras_slurm_modify_extend.c`` + - Slurm ``modify()`` entry for ``PMIX_ALLOC_EXTEND``; fires + ``LAUNCH_DAEMONS`` directly on the daemon job (line 752) instead of + routing through ``prte_ras_base_complete_request()`` — see the + launch-fence warning under *DVM Extension and the Daemon-Launch + Race*. + * - ``src/prted/prted_comm.c`` + - ``PRTE_DAEMON_SHRINK_CMD`` handler (line 469): checks daemon rank + list and exits cleanly if listed. + +Debugging +--------- + +Verbose output for each subsystem is controlled at runtime: + +.. code-block:: sh + + # Job state machine transitions + prte --prtemca state_base_verbose 5 ... + + # PLM (daemon launch, message receive) + prte --prtemca plm_base_verbose 5 ... + + # Process mapping + prte --prtemca rmaps_base_verbose 5 ... + + # Resource allocation + prte --prtemca ras_base_verbose 5 ... + +At verbosity level 5 the state machine also prints its full table at +startup via ``prte_state_base_print_job_state_machine()``. + +DVM Extension and the Daemon-Launch Race +----------------------------------------- + +Background +~~~~~~~~~~ + +A persistent DVM can have its node pool expanded at runtime in two ways: + +1. **App-triggered** (``src/mca/ras/base/ras_base_allocate.c``:771): + A job submitted with ``--add-host`` or ``--add-hostfile`` causes the RAS + base ``add_hosts()`` function — now a thin asynchronous wrapper — to + collect the directives into a ``prte_pmix_server_req_t`` with + ``req->key = "hosts"`` and ``req->allocdir = PMIX_ALLOC_EXTEND``. It + sets ``prte_dvm_ready = false`` to block concurrent job dispatch, then + posts the request to the event loop for ``prte_ras_base_modify()`` to + handle. ``prte_ras_base_modify()`` routes the request to the ``ras/hosts`` + module, whose ``modify()`` entry point + (``src/mca/ras/hosts/ras_hosts.c``:340) parses the hostfiles and host + lists and inserts new nodes into ``prte_node_pool``. On success the + common completion function ``prte_ras_base_complete_request()`` (line 586) + marks ``PRTE_JOB_EXTEND_DVM`` on the **daemon job** and fires + ``PRTE_JOB_STATE_LAUNCH_DAEMONS`` on the daemon job. Any application + jobs that arrive while ``prte_dvm_ready`` is false are stashed in + ``prte_cache`` and flushed when ``vm_ready()`` fires. + +2. **Scheduler push** (``src/mca/ras/slurm/ras_slurm_modify_extend.c``:752): + When Slurm grants additional nodes (e.g., in response to a + ``PMIx_Allocate`` call from an application), the Slurm RAS component + adds the nodes to the pool and fires ``PRTE_JOB_STATE_LAUNCH_DAEMONS`` + **directly on the daemon job**, setting ``PRTE_JOB_EXTEND_DVM`` on the + daemon job — bypassing ``prte_ras_base_complete_request()`` and leaving + ``prte_dvm_ready`` unchanged. + +In both cases ``setup_virtual_machine()`` is called (from within the PLM's +``launch_daemons`` callback) and detects the extension via the +``PRTE_JOB_EXTEND_DVM`` attribute on the daemon job. If new daemons are +needed it sets ``PRTE_JOB_LAUNCHED_DAEMONS`` on the daemon job and returns +with ``map->num_new_daemons > 0``. The PLM then spawns ``prted`` processes +on the new nodes and the state machine parks at ``DAEMONS_LAUNCHED`` until +they call home. + +.. warning:: + A RAS component that handles a modification request (grow or shrink) + must route its result through ``prte_ras_base_complete_request()`` + rather than activating ``PRTE_JOB_STATE_LAUNCH_DAEMONS`` directly on the + daemon job. ``prte_ras_base_complete_request()`` is the single point + that performs the bookkeeping the launch fence depends on: it sets + ``PRTE_JOB_EXTEND_DVM`` and resets ``prte_nidmap_communicated`` on the + grow path, and on the shrink path it records the + ``prte_shrink_campaign_t`` and raises ``prte_dvm_launch_fence`` *before* + any daemon is asked to leave. A component that fires + ``PRTE_JOB_STATE_LAUNCH_DAEMONS`` itself — as the Slurm scheduler-push + path historically does — skips this common handling and can leave the + fence out of step with the campaign it is supposed to gate, reopening + the daemon-launch race described below. New RAS modules, and any + reworking of the existing ones, should hand their results to + ``prte_ras_base_complete_request()`` and let it activate the state. + +DVM Shrink +~~~~~~~~~~ + +A DVM can also be **shrunk** at runtime by releasing nodes back to the +scheduler. The path runs through the same ``prte_ras_base_complete_request()`` +function, but with ``req->allocdir == PMIX_ALLOC_RELEASE``: + +1. The ``PMIX_ALLOC_RELEASE`` branch extracts the node list from + ``PMIX_ALLOC_NODE_LIST``, looks up each node's daemon rank in + ``prte_node_pool``, and packs the ranks into a + ``PRTE_DAEMON_SHRINK_CMD`` message. +2. The message is broadcast to all daemons via + ``prte_grpcomm.xcast(PRTE_RML_TAG_DAEMON)``. +3. Each daemon that receives ``PRTE_DAEMON_SHRINK_CMD`` + (``src/prted/prted_comm.c``:469) checks whether its own rank appears in + the unpacked list. If listed, it: + + a. Sets ``prte_abnormal_term_ordered = true``. + b. Fires a ``PMIX_EVENT_JOB_END`` PMIx event to notify any attached tools. + c. Activates ``PRTE_JOB_STATE_DAEMONS_TERMINATED`` and exits cleanly. + + The HNP needs no acknowledgement from the daemon: it learns that the + daemon is gone through the normal daemon-loss (comm-failure) path, which + is also the only event that guarantees the daemon's routes and node state + have actually been torn down (see below). + +Unlisted daemons silently discard the command and continue running. + +In addition, each RAS module may implement a ``release_allocation`` entry +point (added in ``src/mca/ras/ras.h``). The base function +``prte_ras_base_release_allocation()`` cycles active modules in priority +order (filtering by ``session->alloc_module`` when set) and is called +automatically from the ``prte_session_t`` destructor so that allocations are +released when their session object is destructed. + +Shrink Synchronisation Requirement +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``PRTE_DAEMON_SHRINK_CMD`` xcast is fire-and-forget: targeted daemons +exit on their own schedule, and the HNP must determine when all of them have +actually terminated. This creates two race windows that must be closed. + +**Race 1 — new job mapping onto a shrinking node** + +A job that reaches the ``VM_READY → MAP`` boundary while a shrink is in +progress may have its processes mapped onto a node whose daemon has already +received ``PRTE_DAEMON_SHRINK_CMD``. By the time the launch message is +sent the daemon may already have exited. + +**Race 2 — in-flight job at** ``LAUNCH_APPS`` + +A job that was fully mapped *before* a shrink started and then reaches +``LAUNCH_APPS`` (where launch data is packed and sent to each daemon) may +send to a daemon that dies in the window between MAP and the actual send. + +Closing both races requires: + +1. **Completion on actual daemon death** — the HNP records the targeted + daemon ranks in a ``prte_shrink_campaign_t`` and waits for each one to + leave the DVM. Departure is detected through the existing daemon-loss + (comm-failure) path in the ``errmgr/dvm`` component, which matches the + dead daemon's rank against the campaign's target list, drives the fence + counter down, and releases the fence once every target is gone. The + HNP does not rely on any acknowledgement from the daemon: the reason a + targeted daemon dies is irrelevant, and the comm-failure event is the + only signal that also guarantees the daemon's routes, ``num_daemons`` + count, and node state have been cleaned up. Each target slot is stamped + ``PMIX_RANK_INVALID`` once counted so a repeated comm event cannot + decrement the campaign twice. + +2. **Second hold point at** ``LAUNCH_APPS`` — ``prte_plm_base_launch_apps()`` + checks a dedicated ``prte_shrink_ntargets`` counter (nonzero only when a + shrink is in progress) and if nonzero parks the job in a second held-job + array (``prte_prelaunch_held_jobs``) rather than packing or sending any + launch data. This hold uses ``prte_shrink_ntargets`` rather than the + general ``prte_dvm_launch_fence`` so that a concurrent DVM grow does not + unnecessarily stall jobs that have already been mapped to existing nodes. + +3. **Remap on release** — when ``prte_dvm_launch_fence`` returns to zero, + jobs in ``prte_prelaunch_held_jobs`` that were mapped to any of the now-dead + daemon nodes are reset to ``MAP`` state so they are remapped to the + surviving nodes; jobs whose entire mapping lies on surviving nodes are + re-activated at ``LAUNCH_APPS`` without remapping. + +The full implementation plan is in :ref:`dvm-shrink-campaign-label`; the +shared fence mechanism it builds on is in :ref:`elastic-dvm-plan-label`. + +The Race Condition +~~~~~~~~~~~~~~~~~~ + +The app-triggered path partially mitigates the race by setting +``prte_dvm_ready = false`` in ``add_hosts()`` before the asynchronous +request is posted: any job that arrives after that point is stashed in +``prte_cache`` and is not dispatched until ``vm_ready()`` restores +``prte_dvm_ready = true``. + +The scheduler-push path does **not** clear ``prte_dvm_ready``. Because +``prte_dvm_ready`` otherwise remains ``true`` throughout DVM operation (it +is only cleared at shutdown), any application job that arrives while a +scheduler-initiated daemon launch is in flight is dispatched immediately: + +.. code-block:: text + + Thread of events (time →) + + Slurm grants new nodes + ras_slurm_modify_extend fires LAUNCH_DAEMONS on daemon job + PLM starts spawning prted on new nodes ← daemon launch in progress + App job B arrives, prte_dvm_ready==true, B is dispatched + B: INIT → ALLOCATE → VM_READY + B: MAP ← assigns procs to new nodes ← daemons NOT UP YET + B: SEND_LAUNCH_MSG → daemons fail to receive it + +The same race exists when multiple apps are running concurrently inside the +DVM and one of them triggers an allocation expansion: the other apps' +independent state machine progressions can interleave with the daemon launch +events. + +Required Change: Gate at the VM_READY → MAP Boundary +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To eliminate the race, all application jobs must be held at the +``VM_READY → MAP`` boundary whenever any daemon launch campaign is in +progress, regardless of which path (app-triggered or scheduler push) +initiated it. Jobs that are already past ``MAP`` (i.e., already launching +or running) are unaffected — their daemons are already up. + +The mechanism is a **global launch fence** — a counter +(``prte_dvm_launch_fence``) that tracks the number of in-progress daemon +launch campaigns. An app job that reaches the ``VM_READY → MAP`` transition +checks the fence; if it is nonzero the job parks itself in a held-job array +(``prte_held_jobs``) and is released when the fence reaches zero. + +The step-by-step implementation plan is in +:ref:`elastic-dvm-plan-label`, with the grow- and shrink-specific details +in :ref:`dvm-grow-campaign-label` and :ref:`dvm-shrink-campaign-label`. diff --git a/docs/plans/elastic_dvm/elastic_dvm_plan.rst b/docs/plans/elastic_dvm/elastic_dvm_plan.rst new file mode 100644 index 0000000000..10d2a264b3 --- /dev/null +++ b/docs/plans/elastic_dvm/elastic_dvm_plan.rst @@ -0,0 +1,467 @@ +.. _elastic-dvm-plan-label: + +Elastic DVM Implementation Plan +=============================== + +This document describes the implementation of the **launch fence** — the +shared mechanism that serialises application-job dispatch against in-progress +DVM grow and shrink campaigns, closing the race between a DVM size change and +concurrently-running application jobs. For background on the race itself see +:ref:`state-machine-label`, section *DVM Extension and the Daemon-Launch Race*. + +The externally observable contract this implementation delivers — the +job-admission and placement guarantees, and the two-phase completion +notification — is specified in :ref:`elastic-dvm-spec-label`, which is +authoritative for observable behavior. Where this plan and that specification +disagree about observable behavior, the specification wins and this plan must +be corrected. + +This plan is the **parent** of two campaign-specific plans: + +* :ref:`dvm-grow-campaign-label` — the grow (daemon-launch) path's per-campaign + fence accounting, failure rollback, and success/failure completion events. +* :ref:`dvm-shrink-campaign-label` — the shrink (node-removal) path's campaign + tracking, the second (``LAUNCH_APPS``) hold point, completion detection, the + RM-side resource release that runs at completion (a release may span multiple + allocations, so every active RAS module is offered the completed campaign), + and completion events. + +It covers the **shared** infrastructure both paths build on: the fence counter, +the held-job arrays, the ``VM_READY → MAP`` hold point, the fence-release +helper, and the completion-event emission common to both. + +The mechanism is a **global launch fence** — a counter +(``prte_dvm_launch_fence``) that tracks the number of in-progress daemon launch +campaigns. An app job that reaches the ``VM_READY → MAP`` transition checks the +fence; if it is nonzero the job parks itself in a held-job array +(``prte_held_jobs``) and is released when the fence reaches zero. + +The state machine is single-threaded on the progress thread, so no locking is +required anywhere in this plan. + +**Gated on elastic mode.** Every piece of this machinery is active only when +the DVM is in elastic mode (the pre-existing ``prte_elastic_mode`` MCA +parameter, off by default). The gate is applied at the two points that *raise* +the fence — grow-campaign creation in ``setup_virtual_machine()`` and +shrink-campaign creation in the ``PMIX_ALLOC_RELEASE`` handler — so that outside +elastic mode the fence is never raised, the campaign lists stay empty, and every +downstream check (the ``VM_READY`` and ``LAUNCH_APPS`` holds, the errmgr +campaign matching, the drains) is naturally inert. The consumer sites also +carry an explicit ``prte_elastic_mode`` guard so the non-elastic path is +provably identical to the pre-feature behavior — in particular, +``prte_plm_base_grow_target_failed()`` returns ``false`` immediately, so a +daemon loss on a fixed-size DVM is handled by the ordinary errmgr abort path +exactly as before. + +.. note:: + The app-triggered expansion path (``--add-host`` / ``--add-hostfile``) + already sets ``prte_dvm_ready = false`` in ``add_hosts()`` before posting + the asynchronous RAS modify request, which causes newly-arriving jobs to be + stashed in ``prte_cache`` rather than dispatched immediately. The launch + fence is still required for the scheduler-push path (e.g., Slurm firing + ``LAUNCH_DAEMONS`` directly) where ``prte_dvm_ready`` is never cleared, and + to ensure full correctness when both paths can interleave. + +Step 1 — New state constant +--------------------------- + +In ``src/mca/plm/plm_types.h``, add: + +.. code-block:: c + + /* value 17 is currently unused in the running-state band */ + #define PRTE_JOB_STATE_WAITING_FOR_DAEMONS 17 + +Add a corresponding string to ``src/util/error_strings.c``. + +This state is used purely as a marker so that debugging tools and verbose +output show clearly why a job is parked; no callback is registered for it. + +Step 2 — New global fence and held-job arrays +--------------------------------------------- + +In ``src/runtime/prte_globals.c`` and ``src/runtime/prte_globals.h``, add: + +.. code-block:: c + + /* counts in-progress daemon launch campaigns */ + int prte_dvm_launch_fence = 0; + + /* jobs parked at the VM_READY → MAP boundary */ + pmix_pointer_array_t *prte_held_jobs; + + /* jobs parked at the LAUNCH_APPS boundary during a shrink */ + pmix_pointer_array_t *prte_prelaunch_held_jobs; + +Initialize both arrays in ``src/runtime/prte_init.c`` alongside the existing +``prte_cache`` initialization: + +.. code-block:: c + + prte_held_jobs = PMIX_NEW(pmix_pointer_array_t); + pmix_pointer_array_init(prte_held_jobs, 1, INT_MAX, 1); + + prte_prelaunch_held_jobs = PMIX_NEW(pmix_pointer_array_t); + pmix_pointer_array_init(prte_prelaunch_held_jobs, 1, INT_MAX, 1); + +Destruct both in ``src/runtime/prte_finalize.c``. + +The grow and shrink campaign lists (``prte_grow_campaigns`` and +``prte_shrink_campaigns``) that drive the fence are declared, constructed, and +destructed alongside these globals; their types and lifecycles are specified in +the two child plans. + +Step 3 — Park jobs at the VM_READY → MAP boundary +-------------------------------------------------- + +This is the first of the two hold points and is shared by both paths: it stops +*any* in-progress campaign (grow or shrink) from letting a freshly-arriving job +map onto a node whose daemon is not ready. + +In ``vm_ready()``, the code at line 360 is reached only by app jobs (the +daemon-job branch returns at line 357). This is immediately before +``prte_filem.preposition_files()`` which leads to ``files_ready → MAP``. +Add the hold check here: + +.. code-block:: c + + /* position any required files */ + if (0 < prte_dvm_launch_fence) { + /* daemon launch in progress — park this job */ + caddy->jdata->state = PRTE_JOB_STATE_WAITING_FOR_DAEMONS; + PMIX_RETAIN(caddy->jdata); + pmix_pointer_array_add(prte_held_jobs, caddy->jdata); + PMIX_RELEASE(caddy); + return; + } + if (PRTE_SUCCESS != + prte_filem.preposition_files(caddy->jdata, files_ready, caddy->jdata)) { + PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_FILES_POSN_FAILED); + } + PMIX_RELEASE(caddy); + +The second hold point — at ``LAUNCH_APPS``, guarded by the shrink campaign list +rather than the fence counter — is shrink-specific and is described in +:ref:`dvm-shrink-campaign-label`. + +Step 4 — Held-job release helpers +--------------------------------- + +There are two distinct ways a held job leaves its parked state, and they are +**not** symmetric, so they are handled by two separate helpers rather than a +single ``bool success`` flag: + +* **Global success.** When the global fence reaches zero — every grow and + shrink campaign has completed *successfully* — both classes of held job are + admitted. This is ``prte_plm_base_fence_release()``. + +* **Grow failure.** When a grow campaign fails (see + :ref:`dvm-grow-campaign-label`, *Failure drain and rollback*), the spec + requires the whole **pre-map** held-job set to be aborted — the first-failure + semantics of a non-elastic launch. But a grow failure must **not** touch the + **pre-launch** held jobs: those are parked solely on account of an in-progress + shrink (the ``LAUNCH_APPS`` hold is gated on the shrink list, not the fence), + so they do not wait on the grow, and the spec's conformance guarantee #4 + states that a daemon failure may affect only the jobs waiting on the campaign + it belongs to. This asymmetric abort is ``prte_plm_base_abort_premap_held()``. + +Folding both paths into one ``fence_release(bool success)`` was the original +shape of this plan, but it could not honor the spec when a grow failed while a +shrink was still in progress. A single global ``success`` flag cannot express +"abort the grow's waiters but leave the shrink's waiters parked," and gating the +failure abort on the fence reaching zero would let a *later* shrink-success +release **admit** a pre-map job whose grow dependency had already failed (the +last campaign to drain — the shrink — would call the release with +``success == true``). Splitting the two release paths closes that gap: the +grow-failure abort fires immediately on the pre-map array regardless of the +fence value, and the success release is reached only when no campaign has +failed. + +Both helpers are declared in ``src/mca/plm/base/plm_private.h`` (the header the +errmgr, ras, and state callers already include for the existing +``prte_plm_base_*`` launch helpers) and defined in +``plm_base_launch_support.c``: + +.. code-block:: c + + /* SUCCESS release — invoked only when the global fence reaches zero, i.e. + * every grow and shrink campaign has completed successfully. Admits both + * classes of held job and defensively sweeps any residual campaigns. */ + void prte_plm_base_fence_release(void) + { + int _hi; + prte_job_t *_held; + + /* --- pre-map held jobs (parked at VM_READY) --- */ + for (_hi = 0; _hi < prte_held_jobs->size; _hi++) { + _held = (prte_job_t *) + pmix_pointer_array_get_item(prte_held_jobs, _hi); + if (NULL == _held) { + continue; + } + pmix_pointer_array_set_item(prte_held_jobs, _hi, NULL); + PRTE_ACTIVATE_JOB_STATE(_held, PRTE_JOB_STATE_VM_READY); + PMIX_RELEASE(_held); + } + + /* --- pre-launch held jobs (parked at LAUNCH_APPS) --- */ + for (_hi = 0; _hi < prte_prelaunch_held_jobs->size; _hi++) { + _held = (prte_job_t *) + pmix_pointer_array_get_item(prte_prelaunch_held_jobs, _hi); + if (NULL == _held) { + continue; + } + pmix_pointer_array_set_item(prte_prelaunch_held_jobs, _hi, NULL); + if (prte_plm_base_job_needs_remap(_held)) { + prte_plm_base_reset_proc_map(_held); + PRTE_ACTIVATE_JOB_STATE(_held, PRTE_JOB_STATE_MAP); + } else { + PRTE_ACTIVATE_JOB_STATE(_held, PRTE_JOB_STATE_LAUNCH_APPS); + } + PMIX_RELEASE(_held); + } + + /* Campaigns are removed individually as their last target drains, so + * both lists should be empty here. Sweep each defensively anyway — so + * a future change that can leave a residual campaign behind cannot wedge + * the fence — and sweep *both* kinds for symmetry, not just shrink. */ + { + prte_shrink_campaign_t *_sc, *_sn; + PMIX_LIST_FOREACH_SAFE(_sc, _sn, + &prte_shrink_campaigns, prte_shrink_campaign_t) { + pmix_list_remove_item(&prte_shrink_campaigns, &_sc->super); + PMIX_RELEASE(_sc); + } + } + { + prte_grow_campaign_t *_gc, *_gn; + PMIX_LIST_FOREACH_SAFE(_gc, _gn, + &prte_grow_campaigns, prte_grow_campaign_t) { + pmix_list_remove_item(&prte_grow_campaigns, &_gc->super); + PMIX_RELEASE(_gc); + } + } + } + + /* GROW-FAILURE abort — fails every job parked at the VM_READY -> MAP + * boundary to NEVER_LAUNCHED. Called from the grow failure drain only, and + * independent of the fence value, so a grow failure aborts its pre-map + * waiters even while a concurrent shrink keeps the fence nonzero. It + * deliberately leaves prte_prelaunch_held_jobs untouched: those jobs wait + * only on a shrink, never on the grow (conformance #4). */ + void prte_plm_base_abort_premap_held(void) + { + int _hi; + prte_job_t *_held; + + for (_hi = 0; _hi < prte_held_jobs->size; _hi++) { + _held = (prte_job_t *) + pmix_pointer_array_get_item(prte_held_jobs, _hi); + if (NULL == _held) { + continue; + } + pmix_pointer_array_set_item(prte_held_jobs, _hi, NULL); + PRTE_ACTIVATE_JOB_STATE(_held, PRTE_JOB_STATE_NEVER_LAUNCHED); + PMIX_RELEASE(_held); + } + } + +The pre-launch branch of ``fence_release()`` calls two shrink-specific +helpers — ``prte_plm_base_job_needs_remap()`` (does any held proc sit on a +departing daemon?) and ``prte_plm_base_reset_proc_map()`` (un-claim the previous +mapping so the job can be remapped onto survivors) — specified in +:ref:`dvm-shrink-campaign-label`. Because shrink completion treats a clean exit +and a crash identically (a targeted daemon's departure is always a success for +its campaign), neither held-job array is ever failed on the shrink path; the +only failure disposition of a held job is the grow-failure abort above. + +``prte_plm_base_fence_release()`` acts when the **global** fence reaches zero, +which requires *all* grow and shrink campaigns to have completed. The +per-campaign **completion event** (Step 5) is distinct: it fires for an +individual request's campaign when that campaign drains, independent of whether +other campaigns are still in flight. + +Step 5 — Completion-event emission (shared helper) +-------------------------------------------------- + +The spec's two-phase contract (see :ref:`elastic-dvm-spec-label`, +*Asynchronous size-change completion*) requires that, when an accepted DVM +operation finishes, the runtime deliver a directed event to the process that +requested the size change: ``PMIX_DVM_IS_READY`` on success or +``PMIX_ERR_DVM_MOD`` (carrying the underlying cause) on failure. + +Both campaign objects therefore record the requester so the event can be +directed once the campaign drains: + +.. code-block:: c + + pmix_proc_t requester; /* who requested the size change */ + char *alloc_id; /* PMIX_ALLOC_ID of the affected allocation */ + char *req_id; /* requester's PMIX_ALLOC_REQ_ID, or NULL */ + bool have_requester; /* false for a scheduler push */ + +These fields are populated where the campaign is created, from the allocation +request that drove the operation: + +* **Shrink** — directly in the ``PMIX_ALLOC_RELEASE`` handler, from the request + object: ``requester`` is the request's ``tproc``, and ``alloc_id`` / ``req_id`` + are read from its ``PMIX_ALLOC_ID`` / ``PMIX_ALLOC_REQ_ID`` info keys. +* **Grow** — in ``setup_virtual_machine()``, *indirectly through the session*. + The RAS reservation machinery already records the driving request on the + session and back-points every reserved node at it + (``add_nodes_to_session()`` sets ``node->session``; the session carries + ``requestor``, ``alloc_refid``, and ``user_refid``). The grow campaign reads + those from the first new daemon's ``node->session``, so the originating + request need not be threaded explicitly into the launch path. + +A size change initiated with no PMIx requester (a scheduler push, or the initial +DVM bring-up, where the session is the default one or its ``requestor`` rank is +``PMIX_RANK_INVALID``) leaves ``have_requester`` false and emits no event. + +A single shared helper, declared in ``src/mca/plm/base/plm_private.h`` and +defined in ``plm_base_launch_support.c``, performs the emission. Its prototype +takes a +``bool success`` rather than an event code, so that **the two new status codes +are named only inside the helper body** — call sites pass a bool and a cause and +never reference ``PMIX_DVM_IS_READY`` / ``PMIX_ERR_DVM_MOD`` themselves: + +.. code-block:: c + + /* success => emit PMIX_DVM_IS_READY; otherwise emit PMIX_ERR_DVM_MOD + * carrying `cause` (the underlying failure pmix_status_t) */ + void prte_plm_base_dvm_mod_notify(const pmix_proc_t *requester, + const char *alloc_id, + const char *req_id, + bool success, + pmix_status_t cause); + +It packs ``PMIX_ALLOC_ID`` (always), ``PMIX_ALLOC_REQ_ID`` (when ``req_id`` is +non-NULL), and — on failure — the underlying ``cause`` status (carried under +``PMIX_JOB_TERM_STATUS``, the standard ``pmix_status_t``-typed info key; the PMIx +contract for ``PMIX_ERR_DVM_MOD`` asks only for "any available information +describing the cause"), then delivers the event **only** to ``requester`` as a +directed, custom-range notification — the same ``PMIX_RANGE_CUSTOM`` mechanism +used for ``PMIX_ALLOC_TIMEOUT_WARNING``. + +The grow and shrink plans call this helper at their respective drain points: the +grow path inside ``prte_plm_base_grow_drain()`` (reached on success from +``vm_ready`` after the WIREUP xcast, and on failure from +``prte_plm_base_grow_target_failed()`` and the ``check_job_complete`` safety +net); the shrink path when a campaign's last target departs (success, in the +errmgr) and on the xcast-failure cleanup at campaign creation (failure). + +``PMIX_DVM_IS_READY`` and ``PMIX_ERR_DVM_MOD`` are plain ``#define``\ d +``pmix_status_t`` values (PMIx status codes are preprocessor macros, not enum +constants), so their availability is decided entirely by whether the installed +PMIx headers define the symbols — **no PMIx capability flag is involved** (the +``PRTE_CHECK_PMIX_CAP`` machinery is for ``PMIX_CAP_*`` behavioral flags and +does not apply here). + +To keep the project's ``#if FOO`` discipline — so a mistyped guard is a compile +error rather than a silently-false ``#ifdef`` — a probe in +``config/prte_setup_pmix.m4`` defines ``PRTE_HAVE_DVM_MOD_EVENTS`` to ``0`` or +``1`` from the presence of the two symbols: + +.. code-block:: none + + AC_MSG_CHECKING([for PMIx DVM modification event codes]) + AC_PREPROC_IFELSE( + [AC_LANG_PROGRAM([[#include + #if !defined(PMIX_DVM_IS_READY) || !defined(PMIX_ERR_DVM_MOD) + #error DVM modification event codes not present + #endif + ]], [[]])], + [AC_MSG_RESULT([yes]) + AC_DEFINE([PRTE_HAVE_DVM_MOD_EVENTS], [1], + [PMIx defines the DVM modification event codes])], + [AC_MSG_RESULT([no]) + AC_DEFINE([PRTE_HAVE_DVM_MOD_EVENTS], [0], + [PMIx defines the DVM modification event codes])]) + +The macro is defined to ``0`` or ``1`` on both branches (never ``#undef``\ ed), +so it can be tested with ``#if PRTE_HAVE_DVM_MOD_EVENTS``. Because the helper's +``bool``-based prototype keeps the two codes out of every call site, **only the +helper body needs the guard**: when ``PRTE_HAVE_DVM_MOD_EVENTS`` is ``0`` the +body compiles to a no-op (the ``bool``/``pmix_status_t`` prototype still +compiles), the call sites are unchanged, and no completion event is delivered — +exactly as the spec's backward-compatibility clause requires. Because this +touches ``*.m4``, ``./autogen.pl`` must be re-run before configuring. + +Summary of Files Changed (Shared Fence Infrastructure) +------------------------------------------------------- + +.. list-table:: + :widths: 50 50 + :header-rows: 1 + + * - File + - Change + * - ``src/mca/plm/plm_types.h`` + - Add ``PRTE_JOB_STATE_WAITING_FOR_DAEMONS = 17``. + * - ``src/util/error_strings.c`` + - Add string for ``PRTE_JOB_STATE_WAITING_FOR_DAEMONS``. + * - ``src/runtime/prte_globals.h`` + - Declare ``prte_dvm_launch_fence``, ``prte_held_jobs``, and + ``prte_prelaunch_held_jobs``. + * - ``src/runtime/prte_globals.c`` + - Define and initialize ``prte_dvm_launch_fence = 0``. + * - ``src/runtime/prte_init.c`` + - Allocate and init ``prte_held_jobs`` and ``prte_prelaunch_held_jobs``. + * - ``src/runtime/prte_finalize.c`` + - Destruct ``prte_held_jobs`` and ``prte_prelaunch_held_jobs``. + * - ``src/mca/plm/base/plm_base_launch_support.c`` + - Define ``prte_plm_base_fence_release()`` and + ``prte_plm_base_abort_premap_held()`` (Step 4) and + ``prte_plm_base_dvm_mod_notify()`` (Step 5). + * - ``src/mca/plm/base/plm_private.h`` + - Declare ``prte_plm_base_fence_release()``, + ``prte_plm_base_abort_premap_held()``, and + ``prte_plm_base_dvm_mod_notify()`` (and, for the grow path, + ``prte_plm_base_grow_drain()`` / ``prte_plm_base_grow_target_failed()`` — + see :ref:`dvm-grow-campaign-label`). This is the header the errmgr, ras, + and state callers already include. + * - ``src/mca/state/dvm/state_dvm.c`` + - In ``vm_ready``: add the ``VM_READY → MAP`` hold-check before + ``preposition_files`` (Step 3). + * - ``config/prte_setup_pmix.m4`` + - Add the ``AC_PREPROC_IFELSE`` probe that defines + ``PRTE_HAVE_DVM_MOD_EVENTS`` (``0``/``1``) from the presence of + ``PMIX_DVM_IS_READY`` / ``PMIX_ERR_DVM_MOD`` (Step 5). Re-run + ``autogen.pl`` afterward. + +For the grow path's file changes see the "Touched files" table in +:ref:`dvm-grow-campaign-label`; for the shrink path's, the "Touched files" +table in :ref:`dvm-shrink-campaign-label`. + +Design Invariants +----------------- + +**Shared fence** + +* The fence is a single ``int`` accessed only on the progress thread, so all + increments, decrements, and the zero test are race-free without locking. +* A job is parked iff the fence is nonzero at the ``VM_READY → MAP`` boundary + (Step 3); the ``LAUNCH_APPS`` hold (shrink plan) is gated on the shrink + campaign list, not the fence, so a concurrent grow does not stall an + already-mapped job on surviving nodes. +* ``prte_plm_base_fence_release()`` is the **success-only** release; it is + called only when the fence reaches zero, which requires *all* grow and shrink + campaigns to have completed successfully. The campaign lists are therefore + empty (or nearly so — ``fence_release`` does a defensive sweep of **both** the + grow and shrink lists for the degenerate case where some future change leaves + a partially-setup campaign behind). +* A grow failure is the only path that fails a held job. It calls + ``prte_plm_base_abort_premap_held()``, which aborts the pre-map held jobs + (``prte_held_jobs`` → ``NEVER_LAUNCHED``) immediately and independently of the + fence, and never touches ``prte_prelaunch_held_jobs``: those jobs wait only on + a shrink, so per conformance #4 a grow failure must leave them parked until the + shrink completes. +* The per-campaign completion event (Step 5) is independent of the global fence: + it fires when an individual request's campaign drains, even if other campaigns + keep the fence nonzero. + +**Grow fence** — see the "Why this is correct" and "Design" sections of +:ref:`dvm-grow-campaign-label`. + +**Shrink fence** — see the "Design Invariants" section of +:ref:`dvm-shrink-campaign-label`. diff --git a/docs/plans/elastic_dvm/elastic_dvm_spec.rst b/docs/plans/elastic_dvm/elastic_dvm_spec.rst new file mode 100644 index 0000000000..05fca0a7b5 --- /dev/null +++ b/docs/plans/elastic_dvm/elastic_dvm_spec.rst @@ -0,0 +1,556 @@ +.. _elastic-dvm-spec-label: + +Elastic DVM: Specification +========================== + +Purpose +------- + +This document specifies the externally observable behavior of the DVM +while it changes size — what an application, tool, or scheduler may rely +on when the DVM **grows** (a new-daemon launch campaign) or **shrinks** (a +node-removal campaign). It covers two distinct audiences and contracts: + +* **Job admission** — what happens to an application job that is submitted + *while* a grow or shrink is in progress (the bulk of this document). +* **Size-change completion** — how the process that *requested* the size + change learns, asynchronously, whether the DVM operation eventually + succeeded or failed (see `Asynchronous size-change completion`_). + +It defines *what* the runtime guarantees, not *how* it achieves it. The +companion design plans — :ref:`elastic-dvm-plan-label` (the shared fence +mechanism and completion-event helper), :ref:`dvm-grow-campaign-label` +(the grow path's per-campaign accounting), and +:ref:`dvm-shrink-campaign-label` (the shrink path's campaign tracking) — +describe the internal data structures, code paths, and implementation +order. Where this specification and those plans disagree about observable +behavior, **this specification is authoritative** and the plan must be +corrected. + +The whole of this behavior is gated on the DVM being in **elastic mode**, +selected by the pre-existing ``prte_elastic_mode`` MCA parameter (off by +default). When it is not set the DVM is fixed-size: none of the +job-admission deferral, parking, or completion-event machinery is active, +and the runtime behaves exactly as it did before this feature — a daemon +loss, for instance, is handled by the ordinary error path rather than +absorbed as a campaign event. Everything below describes the behavior +**when elastic mode is enabled**. + +Within elastic mode, the job-admission guarantees are stated purely in +terms of job lifecycle outcomes and introduce **no** new command-line +options, environment variables, or PMIx attributes: the grow and shrink +triggers that already exist (``--add-host`` / ``--add-hostfile``, a +scheduler-driven daemon launch, and a ``PMIX_ALLOC_RELEASE`` that removes +nodes) simply acquire correct concurrency semantics. The completion +contract does introduce two new PMIx event (status) codes — +``PMIX_DVM_IS_READY`` and ``PMIX_ERR_DVM_MOD`` — used to notify the +requester when the asynchronous DVM operation finishes; these are the only +new caller-visible interface this feature adds, and both are optional (see +`Backward compatibility and transparency`_). + +Scope +----- + +In scope +~~~~~~~~ + +* The admission guarantee for an application job that reaches the + map-eligible boundary while a grow or shrink is in progress. +* The placement guarantee for a job launched across a size change — onto + which set of nodes (pre- or post-change) it is permitted to map. +* The disposition of a job that was already mapped onto a node that is + about to depart in a shrink. +* The outcome of a daemon failure that occurs during a grow or a shrink, + the distinction between a failure that belongs to the in-progress + campaign and an unrelated one, and the rollback of a failed grow to the + DVM's pre-grow membership. +* The behavior when grow and shrink campaigns, or multiple campaigns of + the same kind, overlap in time. +* The two-phase model by which a dynamic allocation request that drives a + size change is answered: a synchronous response when the request is + *accepted*, and a later asynchronous event when the DVM operation + *completes* or *fails*. +* The two new PMIx event codes — ``PMIX_DVM_IS_READY`` and + ``PMIX_ERR_DVM_MOD`` — that carry the asynchronous completion result to + the requester, and the allocation-identifying payload each delivers. +* The ``PRTE_JOB_STATE_WAITING_FOR_DAEMONS`` job state reported for a + parked job in verbose and debugging output. + +Out of scope / non-goals +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* The *policy* that decides when the DVM grows or shrinks. That decision + is driven entirely by the existing triggers (a tool or application + expansion request, a scheduler action, or an allocation release); this + specification governs only how concurrent job submissions behave around + such a change, never whether the change happens. +* Node selection — which physical nodes are added or removed — is the + responsibility of the resource manager and the existing allocation and + mapping machinery, and is unchanged. +* The wire encoding of internal messages, the names and types of internal + counters and lists, and the precise timing of internal state + transitions are implementation details and are not specified here. +* This document does not redefine the meaning of any DVM size-change + trigger; it specifies only the admission and placement guarantees that + now hold while one is in flight. + +Definitions +----------- + +Grow campaign + A single in-progress launch of one or more new daemons that extends + the DVM onto additional nodes. A grow campaign is *complete* when + every new daemon has reported in and the head node has distributed the + wireup (nidmap) information to the DVM. + +Shrink campaign + A single in-progress removal of one or more nodes from the DVM, + initiated by an allocation release. A shrink campaign targets a fixed + set of daemon ranks and is *complete* when every targeted daemon has + actually departed the DVM. + +Size change + A grow campaign or a shrink campaign. More than one may be in + progress at the same time. + +Map-eligible boundary + The point in a job's lifecycle at which it is ready to be assigned to + nodes (the ``VM_READY → MAP`` transition). A job that has not yet + crossed this boundary has no node assignments. + +Launch boundary + The point at which a job's mapping is complete and the runtime is + ready to send launch data to the daemons that will host its processes + (the ``LAUNCH_APPS`` transition). A job at this boundary already has + node assignments. + +Parked job + A job that has been held at the map-eligible boundary or the launch + boundary because a size change is in progress. A parked job is + reported in the ``PRTE_JOB_STATE_WAITING_FOR_DAEMONS`` state. Parking + is invisible to the submitting tool beyond a delay in launch; the job + is neither failed nor restarted. + +Surviving node + A node that remains in the DVM after a shrink campaign completes. + +Departing node + A node whose daemon is a target of an in-progress shrink campaign. + +Size-change requester + The process whose request initiated a grow or shrink — for a dynamic + allocation this is the process that issued the ``PMIX_ALLOC_NEW`` / + ``PMIX_ALLOC_EXTEND`` (grow) or ``PMIX_ALLOC_RELEASE`` (shrink) request. + A size change initiated without a PMIx requester (for example a + scheduler pushing daemons directly) has no requester. + +Request acceptance + The point at which the runtime has finished *processing* a size-change + request — validated it and initiated the corresponding DVM operation — + and returns its synchronous response. Acceptance is **phase one**; it + does not assert that the operation has finished. + +Operation completion + The point at which the initiated DVM operation actually finishes — the + grow's new daemons are launched and wired, or the shrink's targeted + daemons have departed and the routing tree is repaired — or + definitively fails. Completion is **phase two** and is reported + asynchronously by event (see `Asynchronous size-change completion`_). + +Admission contract +------------------- + +The central guarantee is one of **non-destructive deferral**: + + A job submitted while a size change is in progress is never failed + merely because the DVM is changing size. It is held until the change + completes and then admitted onto the post-change set of nodes — except + that a job whose admission depends on a grow that *fails* is aborted + rather than launched onto an incomplete DVM. + +Two distinct placement hazards are closed by this contract, and a +conforming implementation must close both: + +#. **A job must never be mapped onto a node whose daemon is not ready.** + During a grow this means a node whose daemon has started but has not + yet received its wireup information; during a shrink it means a node + whose daemon is a departure target. + +#. **A job must never have launch data sent to a daemon that is + departing.** A job that completed mapping *before* a shrink began, and + was placed on a node now targeted for removal, must not transmit its + launch message to that daemon. + +Behavior during a grow +~~~~~~~~~~~~~~~~~~~~~~~~ + +While a grow campaign is in progress: + +* A job that reaches the map-eligible boundary is parked. It is admitted + only once **every** in-progress grow campaign has completed — that is, + only after the new daemons are not merely running but fully wired into + the DVM. This guarantees hazard 1: the job cannot be mapped onto a + node whose daemon is up but not yet wired. + +* A job that had already crossed the map-eligible boundary before the + grow began is **unaffected**: it continues to launch on the nodes it + was assigned, and a grow never stalls it. + +* When the grow completes successfully, every job parked at the + map-eligible boundary is admitted and proceeds to mapping, now able to + consider the newly added nodes. + +* If a daemon belonging to the grow campaign **fails** before the + campaign completes, the grow is treated as failed as a whole and is + **rolled back**: every daemon that the same campaign had already + started is terminated, and the nodes the campaign was adding leave the + DVM, so the DVM is restored to exactly the membership it had before that + campaign began. A failed grow never leaves the DVM half-extended with a + partial, un-wired set of new daemons. Every job parked on account of + the grow is then aborted (it never launches) rather than being admitted + onto an incomplete DVM. This matches the first-failure semantics of a + non-elastic launch. The rollback is scoped to the failed campaign: an + unrelated grow campaign running concurrently keeps its own daemons and + completes normally, and pre-existing daemons and nodes are untouched. + +* A daemon failure that does **not** belong to any in-progress grow + campaign (for example, a pre-existing daemon dying for an unrelated + reason) does not release parked jobs early and does not abort them; the + grow proceeds unaffected. + +Behavior during a shrink +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +While a shrink campaign is in progress: + +* A job that reaches the map-eligible boundary is parked, exactly as for + a grow, and is admitted only once every in-progress size change has + completed. This closes hazard 1 for the shrink case: a newly arriving + job cannot be mapped onto a node that is in the act of leaving. + +* A job that had already completed mapping and reaches the launch + boundary is parked **if and only if a shrink is in progress**, until + every targeted daemon has departed. This closes hazard 2. A grow in + progress does *not* park a job at the launch boundary, because a grow + removes no node and so cannot invalidate an existing mapping. + +* When the shrink completes, each parked job is admitted according to + whether the shrink invalidated its placement: + + - A job parked at the launch boundary whose processes were **not** + assigned to any departing node proceeds directly to launch on its + existing mapping. + - A job parked at the launch boundary that had one or more processes + assigned to a departing node is **remapped** onto the surviving + nodes and then launched. Its placement after the change reflects the + smaller DVM; the job is not failed. + - A job parked at the map-eligible boundary is admitted to mapping and + naturally considers only the surviving nodes. + +* Completion of a shrink is driven by the **actual departure** of each + targeted daemon, not by any advance announcement of intent to leave. A + daemon that exits cleanly in response to the shrink and a daemon that + crashes during the shrink are indistinguishable to the contract: in + both cases the node is leaving, and any job mapped onto it is remapped + onto survivors as described above. No job is admitted until the + departing daemons are genuinely gone and the DVM's view of its + membership has been updated to match. + +Concurrency +----------- + +* Any number of grow campaigns, any number of shrink campaigns, or a + mixture, may be in progress simultaneously. Parked jobs are admitted + only when **all** in-progress campaigns have completed; no campaign can + release another campaign's held jobs, and no campaign's completion is + consumed by an unrelated daemon event. + +* A grow campaign in progress concurrently with a shrink does not stall a + job at the launch boundary that has already been mapped onto surviving + nodes; only an in-progress shrink holds jobs there. + +* Each campaign is accounted for independently, so an overlapping or + partially-overlapping pair of campaigns cannot leave a job parked + indefinitely once the last campaign it is waiting on completes. + +Asynchronous size-change completion +----------------------------------- + +A dynamic allocation request that grows or shrinks the DVM is answered in +**two phases**, and the runtime separates the point at which the +*allocation is complete* from the point at which the *runtime is ready*. + +Two-phase model +~~~~~~~~~~~~~~~ + +#. **Acceptance (synchronous).** When the runtime has finished + *processing* the request — validated it, decided the resulting + session/reservation, and initiated the corresponding grow or shrink — + it returns the allocation response (status plus ``PMIX_ALLOC_ID``, as + specified in the companion allocation contract + ``node-reservation-spec.rst``). This response confirms only that the + request was **accepted** and the DVM operation has **begun**. It does + *not* assert that a grow's new daemons are up and wired, or that a + shrink's targeted daemons have departed. + +#. **Completion (asynchronous, by event).** When the DVM operation later + finishes — or fails — the runtime delivers a directed PMIx event to the + **size-change requester**. This is the signal that the *runtime is + ready* (or that the change will not happen), as distinct from the + acceptance in phase one. + +The two phases are decoupled because the DVM operation is inherently +asynchronous and unbounded in time (daemon launch and wireup, or daemon +termination and tree repair). Blocking the allocation response until the +operation finished would stall the requester and entangle the request's +validity with unrelated launch/teardown failures. Decoupling lets the +requester learn promptly that its request was accepted and then act only +when the runtime actually reflects the new size. + +Resource release at shrink completion +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The nodes a single shrink removes need not all come from the same +underlying allocation. A ``PMIX_ALLOC_RELEASE`` may name resources that +the runtime originally obtained from more than one source — for example +nodes acquired through different resource managers, or a mixture of +scheduler-provided and statically-configured nodes — so the set of +departing nodes can span several allocations, each managed by a different +resource component. + +Accordingly, when a shrink completes — every targeted daemon has departed +— the runtime offers the completed operation to **each** active resource +component in turn *before* it emits the completion event, giving each the +opportunity to release the share of the departing resources that belongs +to it back to its resource manager. What a component does with that +opportunity is up to the component: it may return the nodes to a +scheduler, defer, or do nothing if it has no stake in the operation. The +runtime guarantees only the *ordering* — that the release cycle is offered +to every component, and runs to completion, before the completion event is +delivered. It does not guarantee that any particular resource was in fact +handed back, since that is the component's decision, not the runtime's. + +Success event — ``PMIX_DVM_IS_READY`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When the DVM operation completes successfully — for a grow, the new +daemons are launched and wired into the DVM; for a shrink, every targeted +daemon has departed, the routing tree is repaired, and each resource +component has been given the opportunity to release the freed resources +back to its resource manager (see `Resource release at shrink +completion`_) — the runtime delivers a ``PMIX_DVM_IS_READY`` event to the +requester. After this event +the DVM reflects the requested size: a grow's new nodes are available to +spawn onto, a shrink's removed nodes are gone. The event payload carries: + +* ``PMIX_ALLOC_ID`` (``char*``) — the allocation whose operation + completed; always present. +* ``PMIX_ALLOC_REQ_ID`` (``char*``) — the requester's own request id, + included whenever one was supplied on the original request, so the + recipient can match the event by either identifier. + +Failure event — ``PMIX_ERR_DVM_MOD`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If the accepted DVM modification fails — a grow cannot launch or wire its +new daemons (and is rolled back per `Failure semantics`_), a shrink cannot +be carried out, or any other condition leaves the requested change +unrealized — the runtime delivers a ``PMIX_ERR_DVM_MOD`` event to the +requester. The event states that **no (further) DVM modification will be +made** for this request and that the DVM has been returned to a stable +state (for a failed grow, its pre-grow membership). The payload carries: + +* ``PMIX_ALLOC_ID`` (``char*``) — always present. +* ``PMIX_ALLOC_REQ_ID`` (``char*``) — when one was supplied. +* The **underlying cause** — the specific ``pmix_status_t`` that prevented + the modification (for example a daemon-launch failure versus a resource + error), conveyed in the event's info array so the requester can + distinguish what went wrong rather than only that *something* did. + +Both events are **directed to the requesting process only** — they are not +broadcast — mirroring the delivery of ``PMIX_ALLOC_TIMEOUT_WARNING`` +specified in ``node-reservation-spec.rst``. + +Delivery guarantees +~~~~~~~~~~~~~~~~~~~ + +* **Exactly one terminal event per operation.** Each accepted request + that initiates a DVM grow or shrink yields exactly one of + ``PMIX_DVM_IS_READY`` or ``PMIX_ERR_DVM_MOD`` to its requester. +* **No event for a phase-one rejection.** A request rejected during + *processing* (the error cases in the companion + ``node-reservation-spec.rst``, e.g. a malformed or unauthorized request) + fails synchronously in the allocation response and produces **no** + completion event; the phase-two events report only the outcome of an + *accepted* request's DVM operation. +* **No event when nothing changes.** A request that is accepted but + initiates no actual DVM size change (for example an extend that adds no + new daemons) is fully complete at acceptance and emits no asynchronous + event. +* **No requester, no directed event.** A size change with no PMIx + requester (a scheduler-driven push) updates the runtime's own state but + has no specific process to direct a completion event to; none is sent. + +Proposed PMIx status codes +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This contract requires two PMIx event codes that PRRTE cannot define on +its own (they belong to the PMIx standard and headers): + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Code + - Meaning + * - ``PMIX_DVM_IS_READY`` + - Non-error event: an accepted DVM size change has completed and the + runtime now reflects the new size. Carries ``PMIX_ALLOC_ID`` and, + when supplied, ``PMIX_ALLOC_REQ_ID``. + * - ``PMIX_ERR_DVM_MOD`` + - Error event: an accepted DVM size change failed and will not be + made; the DVM has returned to a stable state. Carries + ``PMIX_ALLOC_ID``, the underlying failure ``pmix_status_t``, and — + when supplied — ``PMIX_ALLOC_REQ_ID``. + +Because PMIx status codes are plain preprocessor ``#define``\ s, their use is +guarded at build time by a simple check for the codes' presence in the +installed PMIx headers — no PMIx *capability* flag is required. A PRRTE built +against a PMIx that defines neither code simply omits the completion +notification (see `Backward compatibility and transparency`_). + +Failure semantics +----------------- + +.. list-table:: + :header-rows: 1 + :widths: 50 50 + + * - Event + - Observable outcome + * - A grow-target daemon fails before its grow completes + - The grow fails as a whole and is rolled back: every daemon the same + campaign had already started is terminated and its nodes leave the + DVM, restoring the pre-grow membership; all jobs parked on account + of the grow are aborted and never launch. + * - A daemon unrelated to any in-progress grow fails during a grow + - The grow is unaffected; parked jobs are neither released early nor + aborted. + * - A shrink-target daemon exits cleanly + - Counts as that target's departure; when the last target departs the + shrink completes and parked jobs are admitted (remapped if they + were on a departing node). + * - A shrink-target daemon crashes during the shrink + - Handled identically to a clean exit: the node is leaving regardless, + and jobs mapped onto it are remapped onto survivors. + * - The same departing daemon emits more than one failure event + - Counted once; a repeated event for an already-departed target has no + further effect on admission. + * - The controlling daemon job is torn down with grow campaigns still + pending + - The pending grows are drained as failures so that no job is left + parked across the teardown. + +In every case a parked job has no partial effect: until it is admitted it +has launched no processes, and a job aborted because its grow failed +leaves nothing running. + +Observability +------------- + +This feature introduces two externally visible artifacts. + +The first is a **job state**. A parked job is reported as +``PRTE_JOB_STATE_WAITING_FOR_DAEMONS`` in verbose and debugging output (for +example under ``--prtemca state_base_verbose``), so an operator can see +precisely why a job has not yet been mapped or launched. The state is a +passive marker: it triggers no callback and changes no other behavior. +Once the size change the job is waiting on completes, the job advances out +of this state on its own. + +The second is the pair of **completion events** described under +`Asynchronous size-change completion`_: ``PMIX_DVM_IS_READY`` on success +and ``PMIX_ERR_DVM_MOD`` on failure, each directed to the requester of the +size change and carrying the allocation identifiers (and, on failure, the +underlying cause). Unlike the job state, these are an active interface a +requester registers an event handler for; they are the requester's only +signal that the asynchronous DVM operation has finished. + +Backward compatibility and transparency +---------------------------------------- + +The **job-admission** contract is transparent to every caller: + +* It is inert unless the DVM is in elastic mode (``prte_elastic_mode``, off + by default). A DVM started without that parameter is fixed-size and runs + exactly as it did before this feature — the launch fence is never raised, + no job is ever parked, and daemon losses follow the ordinary error path. +* No new command-line option, environment variable, or PMIx attribute is + defined or required for it (``prte_elastic_mode`` already existed). A + tool, application, or scheduler issues the same requests it always has. +* On a DVM that never grows or shrinks, no job is ever parked and behavior + is identical to a non-elastic DVM. +* The only difference a submitting caller can observe when a size change + *is* in progress is a launch delay for an affected job and, in debugging + output, the ``PRTE_JOB_STATE_WAITING_FOR_DAEMONS`` state — never a + spurious failure and never a launch onto a node that is not ready or is + leaving. + +The **completion** contract adds the two new PMIx event codes, which are +optional and degrade cleanly: + +* ``PMIX_DVM_IS_READY`` and ``PMIX_ERR_DVM_MOD`` are delivered only to a + requester that registers a handler for them; a requester that ignores + them is unaffected beyond losing the completion signal. +* When the underlying PMIx defines neither code, a PRRTE built against it + (the call sites are guarded by a preprocessor check for the two + ``#define``\ d status codes) omits the asynchronous completion + notification entirely. The allocation response + (phase one) is unchanged, so a request is still accepted and the DVM + still grows or shrinks; the requester simply receives no event-based + signal that the operation finished or failed, exactly as before this + feature existed. This is a functional gap, not merely a cosmetic one: + without the event a requester cannot reliably know when the runtime is + ready and must fall back to whatever coarse means it used previously. + +Conformance summary +------------------- + +A conforming implementation guarantees that: + +#. A job submitted while a size change is in progress is held, not failed, + solely on account of the change — and is then admitted onto the + post-change node set, with the sole exception of a job whose grow + dependency fails, which is aborted. +#. A job is never mapped onto a node whose daemon is not yet wired into + the DVM (grow) or is a departure target (shrink). +#. A job is never sent launch data destined for a departing daemon; a job + already mapped onto a departing node is remapped onto surviving nodes + before it launches. +#. A daemon failure affects only the jobs waiting on the campaign that + failure belongs to; an unrelated daemon loss neither releases nor + aborts parked jobs. +#. A grow that fails is rolled back to the DVM's pre-grow membership: the + campaign's already-started daemons are terminated and its nodes leave + the DVM, so a failed grow never leaves the DVM half-extended. The + rollback is scoped to the failed campaign and leaves concurrent + campaigns and pre-existing daemons untouched. +#. Shrink completion is driven by the actual departure of every targeted + daemon — clean exit and crash being indistinguishable — and is + idempotent against a daemon that reports its departure more than once. +#. Concurrent campaigns of either kind never deadlock a parked job: it is + admitted once, and only once, every campaign it is waiting on has + completed. +#. A dynamic allocation that drives a size change is answered in two + phases: a synchronous response on acceptance (which does not assert the + operation has finished), and exactly one asynchronous terminal event — + ``PMIX_DVM_IS_READY`` on success or ``PMIX_ERR_DVM_MOD`` (carrying the + underlying cause) on failure — directed to the requester when the DVM + operation completes. A phase-one rejection, a request that changes + nothing, and a requester-less scheduler push each produce no such event. +#. The job-admission contract adds no caller-visible interface; the + completion contract adds only the two optional PMIx event codes above, + and when the underlying PMIx lacks them the runtime omits the + completion notification while leaving every other guarantee intact. + The job state ``PRTE_JOB_STATE_WAITING_FOR_DAEMONS`` remains observable + for a parked job in verbose and debugging output. diff --git a/docs/plans/elastic_dvm/grow_campaign.rst b/docs/plans/elastic_dvm/grow_campaign.rst new file mode 100644 index 0000000000..942577da16 --- /dev/null +++ b/docs/plans/elastic_dvm/grow_campaign.rst @@ -0,0 +1,307 @@ +.. _dvm-grow-campaign-label: + +DVM Grow-Campaign Fence Tracking +================================ + +This document describes the implementation that makes the DVM **grow** +(daemon-launch) path account for the launch fence on a per-daemon, rank-tracked +basis, mirroring the design already used by the DVM **shrink** path +(:ref:`dvm-shrink-campaign-label`). For the shared fence mechanism itself and +the race it closes, see the parent plan :ref:`elastic-dvm-plan-label` and +:ref:`state-machine-label`, section *DVM Extension and the Daemon-Launch Race*. + +The state machine is single-threaded on the progress thread, so no locking is +required anywhere in this plan. + +The observable job-admission and placement guarantees that the grow path +upholds are specified in :ref:`elastic-dvm-spec-label`, which is +authoritative for observable behavior; this document describes the +implementation that delivers them. + +Motivation +---------- + +The launch fence (``prte_dvm_launch_fence``) holds application jobs at the +``VM_READY → MAP`` boundary while a daemon-launch campaign is in progress, so +that no job is mapped onto a node whose daemon is not yet up and wired. The +shrink path tracks the specific daemon ranks it is removing in a +``prte_shrink_campaign_t`` and resolves the fence one rank at a time as each +targeted daemon actually departs. + +The grow path, by contrast, originally encoded "a grow is in progress" as a +single boolean — ``PRTE_JOB_LAUNCHED_DAEMONS`` — set on the one daemon job, +together with a ``prte_dvm_launch_fence++`` performed once per campaign in +``prte_plm_base_setup_virtual_machine()``. The single decrement happened in +``vm_ready`` on success, or in the ``errmgr/dvm`` comm-failure handler if a +daemon died first. Because that boolean carries no identity, two defects +followed: + +#. **An unrelated daemon death consumed the campaign's token.** The + comm-failure handler decremented the fence and cleared the boolean whenever + *any* daemon died while a grow was in progress — there is only one daemon + job, and it carried the token. A pre-existing daemon dying mid-grow would + therefore release the held jobs early (reopening the very race the fence + exists to close) and clear the token, after which ``vm_ready`` skipped the + WIREUP xcast (it is gated on the same attribute), so the genuinely new + daemons could come up without ever receiving the nidmap/wireup buffer. + +#. **Concurrent campaigns could wedge the fence.** Two overlapping grows would + raise the fence to two but share the single boolean token, which can only be + cleared once. A daemon failure would clear it, leaving the fence stuck + above zero and the held jobs parked indefinitely. + +Both defects trace to the same root cause: the grow path tracked *that* a grow +was happening, not *which* daemons it was launching. + +Design +------ + +Track each grow campaign explicitly, recording the ranks being launched, and +hold the whole campaign's fence contribution until a single safe drain point. + +New campaign object +~~~~~~~~~~~~~~~~~~~~ + +In ``src/runtime/prte_globals.h`` / ``prte_globals.c``: + +.. code-block:: c + + typedef struct { + pmix_list_item_t super; + pmix_rank_t *targets; /* daemon ranks being launched */ + int ntargets; /* == this campaign's fence contribution */ + /* requester recorded for the spec's phase-two completion event */ + pmix_proc_t requester; /* who requested the grow */ + char *alloc_id; /* PMIX_ALLOC_ID of the allocation */ + char *req_id; /* PMIX_ALLOC_REQ_ID, or NULL */ + bool have_requester; /* false for a scheduler-driven push */ + } prte_grow_campaign_t; + PMIX_CLASS_DECLARATION(prte_grow_campaign_t); + + PRTE_EXPORT extern pmix_list_t prte_grow_campaigns; + +The campaign's destructor frees ``targets``, ``alloc_id``, and ``req_id``. + +The list is constructed in ``prte_init.c`` and destructed in +``prte_finalize.c`` alongside ``prte_shrink_campaigns``. A separate list (as +opposed to unifying with the shrink list) is used deliberately: the +``LAUNCH_APPS`` hold and the remap-on-release logic key off the *shrink* list's +non-emptiness, and a grow must **not** stall jobs that are already mapped onto +existing nodes. Keeping the lists separate leaves the working shrink path +untouched. + +Fence is campaign-granular, not per-rank +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Unlike shrink, the grow fence contribution is **held in full until the +campaign is drained as a unit**. This is the key correctness point: the +fence must not reach zero (for a successful grow) until *after* the WIREUP +xcast in ``vm_ready``, otherwise an application job arriving in the window +between "last daemon reported" and "wireup sent" would see a zero fence and +map onto daemons that are up but not yet wired. A naive per-rank decrement at +daemon-report time would reopen exactly that window. Holding the contribution +until ``vm_ready`` drains it preserves the original ordering guarantee. + +The per-rank ``targets`` array serves two purposes: to decide whether a +*failure* event belongs to this grow, and — when one does — to enumerate the +daemons that must be torn down to roll the DVM back to its pre-grow membership +(see `Rollback on failure`_). + +Lifecycle +~~~~~~~~~ + +#. **Create** — in ``prte_plm_base_setup_virtual_machine()``, when + ``map->num_new_daemons > 0``: build a ``prte_grow_campaign_t`` recording the + ``num_new_daemons`` consecutive vpids starting at ``map->daemon_vpid_start``, + record the requester / ``PMIX_ALLOC_ID`` / ``PMIX_ALLOC_REQ_ID`` for the + phase-two completion event, append it to ``prte_grow_campaigns``, and add + ``num_new_daemons`` to the fence. The requester is taken from the first new + daemon's ``node->session`` — the RAS reservation machinery back-points each + reserved node at the session that records the driving request (``requestor``, + ``alloc_refid``, ``user_refid``). When the grow was not driven by an + allocation request (the initial DVM bring-up, or a scheduler push — the + default session, or a session whose ``requestor`` rank is + ``PMIX_RANK_INVALID``), ``have_requester`` stays false and no event is + emitted. ``PRTE_JOB_LAUNCHED_DAEMONS`` is still set on the daemon job for its + unrelated uses (the WIREUP gate in ``vm_ready`` and the odls path); it is no + longer consulted for fence accounting. + +#. **Success drain** — ``vm_ready`` fires only once every expected daemon has + reported (``num_reported == num_procs``), which means any in-progress grow + campaigns have fully succeeded. After performing the WIREUP xcast, it calls + ``prte_plm_base_grow_drain(true)``, which removes every grow campaign, + subtracts each ``ntargets`` from the fence, emits a ``PMIX_DVM_IS_READY`` + completion event to each drained campaign's requester (via + ``prte_plm_base_dvm_mod_notify()`` — see :ref:`elastic-dvm-plan-label`, + Step 5), and — if the fence has reached zero — admits the held jobs by + calling ``prte_plm_base_fence_release()``. + +#. **Failure drain and rollback** — in the ``errmgr/dvm`` comm-failure / + ``FAILED_TO_START`` handler, the dead daemon's rank is passed to + ``prte_plm_base_grow_target_failed()``, which returns ``true`` iff the rank + belonged to an in-progress grow campaign. An unrelated daemon loss matches + nothing, returns ``false``, and is left to the errmgr's normal handling + (fixing defect 1). When the rank *is* a grow target, the function handles + the loss completely — it removes **that** campaign from the list, drops its + ``ntargets`` from the fence, rolls it back out of the DVM (see + `Rollback on failure`_), emits a ``PMIX_ERR_DVM_MOD`` completion event to its + requester, and aborts the pre-map held jobs via + ``prte_plm_base_abort_premap_held()`` (see :ref:`elastic-dvm-plan-label`, + Step 4) — and the errmgr, seeing the ``true`` return, ``goto``\ s its cleanup + so the general daemon-loss path (which would otherwise abort the whole DVM) + is skipped. The failure is **campaign-scoped**: only the matched campaign is + torn down, so a concurrent grow keeps its daemons and completes normally. + Mirroring the original single-token behavior, any grow failure fails the + whole **pre-map** held-job set — immediately, regardless of the fence value, + so a concurrent shrink cannot later admit a job whose grow dependency has + failed. It deliberately does **not** disturb the pre-launch + (``LAUNCH_APPS``) held jobs: those wait only on a shrink, not on the grow, so + per the spec's conformance guarantee #4 a grow failure must leave them + parked. + +#. **Safety net** — ``check_job_complete``'s "received NULL job" branch calls + ``prte_plm_base_grow_drain(false)`` to drain any still-pending grow campaigns + as failures, so pre-map held jobs are never parked across a daemon-job + teardown. (No rollback is needed there — the whole DVM is force-exiting.) + +The success drain (``grow_drain(true)`` from ``vm_ready``) still removes every +grow campaign in one pass and zeroes the fence's entire grow contribution, +independent of how many concurrent campaigns exist (fixing defect 2); the +*failure* path, by contrast, is per-campaign so an unrelated concurrent grow is +not dragged down with the failed one. + +Rollback on failure +~~~~~~~~~~~~~~~~~~~~ + +The spec (:ref:`elastic-dvm-spec-label`) requires that a failed grow leave the +DVM in its pre-grow state rather than half-extended. Failing the held jobs is +therefore necessary but not sufficient: the campaign's already-started daemons +and the nodes it was adding must also be removed. ``grow_target_failed()`` +performs this teardown (in the static helper ``grow_rollback()``) for the +matched campaign before notifying the requester and aborting the held jobs. + +The campaign's ``targets`` array enumerates every daemon rank the grow +launched. One of them is the rank whose loss triggered the failure; the +remainder may be in any state from "not yet reported" through "reported and +wired". Routing for the triggering rank is repaired here with +``prte_rml_route_lost()`` (the errmgr's own ``route_lost`` call is on the path +that the ``true`` return skips). Each *other* target is handled according to +whether a daemon actually came up: + +* **A target that started** (``PRTE_PROC_FLAG_ALIVE`` — it reported in) is + terminated using the same ``PRTE_DAEMON_SHRINK_CMD`` xcast the DVM shrink path + uses. It self-exits, and its departure is then reconciled on the normal + daemon-loss path (``route_lost`` succeeds, ``num_daemons`` is decremented) as + for any shrink — and because the campaign is already gone, that later event + returns ``false`` and is handled without a second rollback. + +* **A target that never started** (the ``FAILED_TO_START`` case — e.g. the + remote ``exec`` failed) has no daemon to signal, so no comm-failure event will + arrive for it; its launch-time ``num_daemons`` bump is reverted directly in + ``grow_rollback()``. + +In every case the node's daemon backpointer is cleared (``node->daemon = NULL``, +releasing the retain taken at assignment, and detaching any reservation +session), which removes the node from the mapper's usable set — the new nodes +carry no application procs, since the jobs that would have used them were held +at the fence and never launched, so clearing ``node->daemon`` is sufficient to +keep any later job off them. + +The rollback is strictly campaign-scoped: it touches only the ranks in the +failed campaign's ``targets`` array. A concurrently-running grow campaign +keeps its own daemons and completes normally, and no pre-existing daemon or +node is disturbed — the same identity-based discrimination that keeps an +unrelated daemon death from consuming the fence (defect 1) also keeps it out of +the rollback set. + +.. note:: + + Two edges remain, both within the rarely-exercised daemon-launch-failure + path and neither yet validated against a real multi-node allocation: a target + that is **slow to start** (neither ``ALIVE`` nor yet failed when the rollback + runs) is treated as never-started, so a later report-in or failure for it is + not specially handled; and node objects are detached via ``node->daemon`` + rather than physically removed from ``prte_node_pool`` (matching how the + shrink path leaves the pool), so ``num_nodes`` is not decremented. + +Why this is correct +------------------- + +* **Unrelated daemon death during a grow.** ``grow_target_failed()`` scans the + campaign target arrays; a non-target rank matches nothing, so the fence is + not touched, the held jobs are not released early, and the WIREUP xcast is + not skipped. + +* **Concurrent campaigns.** Each campaign is an independent object with its own + contribution. On success ``grow_drain()`` removes them all and the fence + reaches zero only when no grow contribution remains; on failure only the + matched campaign is removed. Either way there is no single token to exhaust. + +* **Wireup ordering.** The fence stays at its full value throughout the grow + and is dropped only when ``vm_ready`` drains it after the WIREUP xcast (on + success) or when a target dies (on failure). Jobs held at ``VM_READY → MAP`` + are thus admitted only once the new daemons are wired up. + +* **Partial failure.** A grow in which any target dies is failed as a whole: + the dying daemon triggers ``grow_target_failed()``, which rolls the matched + campaign back out of the DVM — terminating its started daemons via the shrink + command and detaching its nodes — and aborts the pre-map held jobs to + ``NEVER_LAUNCHED`` (the pre-launch held jobs, which wait only on a shrink, are + left untouched). This matches the original first-failure semantics for the + held jobs and, per the spec, leaves the DVM at its pre-grow membership rather + than half-extended; the errmgr skips its DVM-wide abort because the loss was + reported as handled. + +Touched files +------------- + +.. list-table:: + :widths: 45 55 + :header-rows: 1 + + * - File + - Change + * - ``src/runtime/prte_globals.{h,c}`` + - Add ``prte_grow_campaign_t`` (including the requester fields), + ``prte_grow_campaigns`` list, and class (destructor frees ``targets``, + ``alloc_id``, ``req_id``). + * - ``src/runtime/prte_init.c`` / ``prte_finalize.c`` + - Construct / destruct ``prte_grow_campaigns``. + * - ``src/mca/plm/base/plm_base_launch_support.c`` + - Create the campaign in ``setup_virtual_machine`` (recording the + requester from the first new daemon's ``node->session`` — its + ``requestor`` / ``alloc_refid`` / ``user_refid``); add + ``prte_plm_base_grow_drain()``, the static ``grow_rollback()``, and + ``prte_plm_base_grow_target_failed()``. ``grow_drain()`` (success drain / + teardown safety net) emits the per-campaign completion event via the + shared ``prte_plm_base_dvm_mod_notify()`` helper and, on success, admits + the held jobs via ``prte_plm_base_fence_release()`` when the fence reaches + zero. ``grow_target_failed()`` returns ``bool`` and, for the matched + campaign, removes it, drops its fence contribution, calls + ``grow_rollback()`` (terminate started daemons via the shrink command, + revert never-started daemon counts, detach nodes), emits + ``PMIX_ERR_DVM_MOD``, and aborts the pre-map held jobs. + * - ``src/mca/plm/base/plm_private.h`` + - Declare ``prte_plm_base_grow_drain()`` and the now-``bool``-returning + ``prte_plm_base_grow_target_failed()`` (alongside the shared + ``fence_release`` / ``abort_premap_held`` / ``dvm_mod_notify`` helpers, so + all the ``prte_plm_base_*`` launch-fence helpers live in the one header + the errmgr already includes). + * - ``src/mca/errmgr/dvm/errmgr_dvm.c`` + - In the daemon comm-failure block, ``goto cleanup`` when + ``prte_plm_base_grow_target_failed()`` returns ``true``: the grow rollback + has fully absorbed the loss, so the general daemon-loss handling (which + would otherwise abort the whole DVM) must be skipped. + * - ``src/mca/state/dvm/state_dvm.c`` + - Drain on success in ``vm_ready`` after WIREUP; drop the per-error fence + manipulation (the DVM is force-exiting); convert the + ``check_job_complete`` safety net to a drain. + +Follow-up +--------- + +* **Campaign-object unification.** The grow and shrink campaign objects are + structurally similar and could be unified into a single + ``prte_launch_campaign_t`` with a ``kind`` discriminator in a future cleanup. + That was intentionally deferred here to avoid disturbing the ``LAUNCH_APPS`` + hold and remap-on-release logic, which must remain shrink-only. diff --git a/docs/plans/elastic_dvm/index.rst b/docs/plans/elastic_dvm/index.rst new file mode 100644 index 0000000000..be84d2401b --- /dev/null +++ b/docs/plans/elastic_dvm/index.rst @@ -0,0 +1,12 @@ +Elastic DVM Plans +================= + +Implementation plans for dynamic DVM grow/shrink support. + +.. toctree:: + :maxdepth: 2 + + elastic_dvm_spec.rst + elastic_dvm_plan.rst + grow_campaign.rst + shrink_campaign.rst diff --git a/docs/plans/elastic_dvm/shrink_campaign.rst b/docs/plans/elastic_dvm/shrink_campaign.rst new file mode 100644 index 0000000000..a8c0b3c84b --- /dev/null +++ b/docs/plans/elastic_dvm/shrink_campaign.rst @@ -0,0 +1,561 @@ +.. _dvm-shrink-campaign-label: + +DVM Shrink-Campaign Fence Tracking +================================== + +This document describes the implementation of the DVM **shrink** (node-removal) +path: how a ``PMIX_ALLOC_RELEASE`` that removes daemons is tracked against the +launch fence, the second hold point that protects in-flight jobs, how campaign +completion is detected, and the completion event delivered to the requester. +For the shared fence mechanism it builds on — the counter, the held-job arrays, +the ``VM_READY → MAP`` hold point, the fence-release helper, and the +completion-event helper — see the parent plan :ref:`elastic-dvm-plan-label`. +The externally observable contract is specified in :ref:`elastic-dvm-spec-label`, +which is authoritative for observable behavior. + +The state machine is single-threaded on the progress thread, so no locking is +required anywhere in this plan. + +Background +---------- + +The ``PRTE_DAEMON_SHRINK_CMD`` xcast is fire-and-forget: daemons exit +asynchronously and the HNP has no built-in notification when all targeted +daemons have terminated. Two race windows must be closed: + +**Race 1 — new job maps onto a shrinking node.** A job that checks the +``VM_READY → MAP`` fence while a shrink is in progress may pass the fence +(if it was raised after the check), get mapped to a node whose daemon is +dying, and then send a launch message to a daemon that has already exited. + +**Race 2 — in-flight job at LAUNCH_APPS.** A job that completed MAP before +the shrink started and then enters ``prte_plm_base_launch_apps()`` may pack +and send launch data to a daemon that dies between MAP and the send. The +existing VM_READY fence does not protect this window because the job already +passed the fence before the shrink was initiated. + +Race 1 is covered by the shared fence (the fence is incremented for shrink — see +Step 1). Race 2 requires a second hold point guarded by checking the shrink +campaign list (nonempty only during shrink) so that a concurrent grow does not +unnecessarily hold jobs that have already been mapped to surviving nodes. + +Multiple concurrent shrink campaigns are supported: each campaign tracks its +own count of still-living targets and is removed from the list when all of +them have departed the DVM. + +Design Decision — Complete on Death, Not on Acknowledgement +----------------------------------------------------------- + +An earlier revision of this plan had each targeted daemon send an explicit +``PRTE_PLM_SHRINK_ACK_CMD`` to the HNP just before it exited, and the HNP +decremented the campaign on *receipt of the ACK*. The errmgr comm-failure +path existed only as a *fallback* for a daemon that crashed before it could +send its ACK. That design was abandoned for the following reasons. + +**The ACK is the wrong signal.** An ACK announces a daemon's *intent* to +leave; it is sent while the daemon is still alive and still a participant in +the DVM. But the state the fence protects against — a job being mapped onto, +or having launch data sent to, a departing daemon — is only safe once the +daemon's routes, its ``num_daemons`` count, and its node state have actually +been torn down. That teardown happens on the comm-failure path +(``errmgr_dvm.c``), *not* when the ACK is sent. Releasing held jobs on ACK +receipt could therefore unpark them into a DVM that still believed the +departing daemon was present. + +**The reason for departure carries no information.** The HNP only needs to +know that a target is gone, not *why*. A clean shrink exit and a crash have +identical consequences for the campaign: the node is being removed either +way, and the application processes beneath the daemon are killed (or die) +when it terminates. Distinguishing the two cases buys nothing, so the +"clean ACK vs. crash fallback" split was pure complexity. + +**Two decrement paths caused double-counting.** With both the ACK handler +and the errmgr fallback live, each target could be counted twice — once when +its ACK arrived (daemon still alive) and again when its subsequent death was +detected — because nothing marked a target as already counted and the +campaign was only removed once ``pending`` hit zero. Worked through for a +two-target campaign: + +.. code-block:: text + + camp: ntargets=2, pending=2 + daemon A acks -> pending=1, fence-=1 (A still alive) + daemon A dies -> errmgr matches A (still in targets, camp still listed) + -> pending=0 -> camp removed+released, fence-=1 + daemon B -> never counted; campaign already "complete" + +The campaign completed and the fence released while daemon B was still +present, re-opening exactly the race the fence was meant to close. + +**Resolution.** The ACK was removed entirely (the daemon-side send, the +``PRTE_PLM_SHRINK_ACK_CMD`` constant, and the HNP-side handler). Campaign +completion is driven solely by actual daemon departure on the comm-failure +path, which is both the authoritative event and the point at which the +relevant cleanup has occurred. To make the single decrement idempotent +against a daemon that emits more than one failure event, each matched target +slot is stamped ``PMIX_RANK_INVALID`` once counted (Step 5). + +Step 1 — Shrink campaign type, list, and fence increment +--------------------------------------------------------- + +**1a — Campaign type** + +Add the following type to ``src/runtime/prte_globals.h`` and define the +class instance in ``src/runtime/prte_globals.c``: + +.. code-block:: c + + /* one entry per in-progress shrink campaign */ + typedef struct { + pmix_list_item_t super; + pmix_rank_t *targets; /* daemon ranks being terminated */ + int ntargets; /* initial count */ + int pending; /* targets not yet known to have departed */ + /* requester recorded for the spec's phase-two completion event */ + pmix_proc_t requester; /* who issued the PMIX_ALLOC_RELEASE */ + char *alloc_id; /* PMIX_ALLOC_ID of the allocation */ + char *req_id; /* PMIX_ALLOC_REQ_ID, or NULL */ + bool have_requester; /* false for a scheduler-driven release */ + } prte_shrink_campaign_t; + PMIX_CLASS_DECLARATION(prte_shrink_campaign_t); + +In ``src/runtime/prte_globals.c``: + +.. code-block:: c + + static void campaign_destruct(prte_shrink_campaign_t *p) + { + free(p->targets); + free(p->alloc_id); + free(p->req_id); + } + PMIX_CLASS_INSTANCE(prte_shrink_campaign_t, pmix_list_item_t, + NULL, campaign_destruct); + +**1b — Global list** + +In ``src/runtime/prte_globals.h`` declare, and in ``src/runtime/prte_globals.c`` +define: + +.. code-block:: c + + pmix_list_t prte_shrink_campaigns; + +Initialize in ``src/runtime/prte_init.c``: + +.. code-block:: c + + PMIX_CONSTRUCT(&prte_shrink_campaigns, pmix_list_t); + +Destruct in ``src/runtime/prte_finalize.c``: + +.. code-block:: c + + PMIX_LIST_DESTRUCT(&prte_shrink_campaigns); + +**1c — Populate campaign and increment fence** + +In ``src/mca/ras/base/ras_base_allocate.c``, the ``PMIX_ALLOC_RELEASE`` +branch of ``prte_ras_base_complete_request()`` builds the daemon rank array +(``ranks``, count ``m``) and then calls ``free(ranks)`` before the xcast that +carries ``PRTE_DAEMON_SHRINK_CMD`` to the daemons. Insert the campaign setup +**before** ``free(ranks)``, recording the requester directly from the request +object (``req``) so the completion event can be directed at it. Guard the whole +setup on ``0 < m``: a release that removes no daemons creates no campaign, +exactly as the grow path creates none when ``map->num_new_daemons == 0``. The +file must ``#include "src/mca/plm/base/plm_private.h"`` to see +``prte_plm_base_dvm_mod_notify()``. + +.. code-block:: c + + /* record the campaign — must be before free(ranks). Skip entirely when the + * release removes no daemons (m == 0): an empty campaign would never drain + * (no target ever departs on the comm-failure path), so it would leave + * prte_shrink_campaigns non-empty forever — wedging every later job at the + * LAUNCH_APPS hold — and would emit no completion event. Mirrors the grow + * path's `map->num_new_daemons > 0` guard and the spec's "no event when + * nothing changes" clause. */ + if (0 < m) { + prte_shrink_campaign_t *_camp = PMIX_NEW(prte_shrink_campaign_t); + _camp->targets = (pmix_rank_t *) malloc(m * sizeof(pmix_rank_t)); + memcpy(_camp->targets, ranks, m * sizeof(pmix_rank_t)); + _camp->ntargets = m; + _camp->pending = m; + /* this path always has a requesting process (req->tproc); a + * scheduler-driven release that has no requester does not pass through + * here. Capture the requester and the allocation ids from the request. */ + PMIX_XFER_PROCID(&_camp->requester, &req->tproc); + for (n = 0; n < req->ninfo; n++) { + if (PMIx_Check_key(req->info[n].key, PMIX_ALLOC_ID)) { + _camp->alloc_id = strdup(req->info[n].value.data.string); + } else if (PMIx_Check_key(req->info[n].key, PMIX_ALLOC_REQ_ID)) { + _camp->req_id = strdup(req->info[n].value.data.string); + } + } + _camp->have_requester = true; + pmix_list_append(&prte_shrink_campaigns, &_camp->super); + prte_dvm_launch_fence += m; + } + free(ranks); + + /* existing xcast */ + if (PRTE_SUCCESS != (rc = prte_grpcomm.xcast(PRTE_RML_TAG_DAEMON, &msg))) { + PRTE_ERROR_LOG(rc); + /* clean up the campaign we just added (only if one was created), and + * tell the requester the DVM modification failed (spec phase-two + * failure event). rc is a PRTE code, so convert it to the + * pmix_status_t the event carries. */ + if (0 < m) { + prte_shrink_campaign_t *_camp = + (prte_shrink_campaign_t *) pmix_list_remove_last(&prte_shrink_campaigns); + prte_dvm_launch_fence -= _camp->pending; + if (_camp->have_requester) { + prte_plm_base_dvm_mod_notify(&_camp->requester, _camp->alloc_id, + _camp->req_id, false, + prte_pmix_convert_rc(rc)); + } + PMIX_RELEASE(_camp); + } + } + +Because the campaign is appended before the xcast, any ``VM_READY`` event +that fires on the progress thread after this point will see a nonzero fence +and park the job. + +Step 2 — Daemon exit (no acknowledgement) +------------------------------------------ + +A daemon that decides to exit in response to ``PRTE_DAEMON_SHRINK_CMD`` +does **not** send any acknowledgement to the HNP. After firing its +``PMIX_EVENT_JOB_END`` notification it simply activates +``PRTE_JOB_STATE_DAEMONS_TERMINATED`` and exits. + +The HNP tracks campaign completion through the daemon's *actual departure*, +not through a message announcing its intent to leave. An acknowledgement +sent before the daemon dies would be premature: the HNP cares only that the +daemon is gone — the reason is irrelevant, and the application processes +under it are killed when it terminates regardless. More importantly, the +acknowledgement would arrive *before* the daemon's routes, ``num_daemons`` +count, and node state have been torn down, so acting on it could release +held jobs into a DVM that still believes the departing daemon is present. +The comm-failure event (Step 5) is the only signal that coincides with that +cleanup, so it is the sole completion trigger. + +Step 3 — Second hold point at LAUNCH_APPS +------------------------------------------ + +In ``src/mca/plm/base/plm_base_launch_support.c``, +``prte_plm_base_launch_apps()`` (line 817), add a check after the job-state +guard but before packing any data: + +.. code-block:: c + + /* if a shrink is in progress, hold this job until all targeted + * daemons have departed the DVM, to prevent sending launch data to + * a dying daemon */ + if (!pmix_list_is_empty(&prte_shrink_campaigns)) { + jdata->state = PRTE_JOB_STATE_WAITING_FOR_DAEMONS; + PMIX_RETAIN(jdata); + pmix_pointer_array_add(prte_prelaunch_held_jobs, jdata); + PMIX_RELEASE(caddy); + return; + } + +Using ``!pmix_list_is_empty(...)`` rather than a counter means the check +automatically handles concurrent campaigns: the list is nonempty as long as +any shrink is in progress. Keying on the **shrink** list specifically (not the +shared fence counter) ensures a concurrent grow does not stall a job that has +already been mapped onto surviving nodes. + +Step 4 — Remap helpers +---------------------- + +The pre-launch branch of the shared ``prte_plm_base_fence_release()`` (parent +plan, Step 4) calls two shrink-specific helpers, both defined in +``plm_base_launch_support.c`` and declared in ``src/mca/plm/base/plm_private.h``. + +**``prte_plm_base_job_needs_remap(jdata)``** iterates over ``jdata->procs`` +and returns ``true`` if any proc's assigned node has a daemon rank appearing +in any active campaign: + +.. code-block:: c + + bool prte_plm_base_job_needs_remap(prte_job_t *jdata) + { + prte_shrink_campaign_t *camp; + prte_proc_t *proc; + int p, t; + + PMIX_LIST_FOREACH(camp, &prte_shrink_campaigns, prte_shrink_campaign_t) { + for (p = 0; p < jdata->procs->size; p++) { + proc = (prte_proc_t *) + pmix_pointer_array_get_item(jdata->procs, p); + if (NULL == proc || NULL == proc->node || + NULL == proc->node->daemon) continue; + for (t = 0; t < camp->ntargets; t++) { + if (camp->targets[t] == proc->node->daemon->name.rank) { + return true; + } + } + } + } + return false; + } + +**``prte_plm_base_reset_proc_map(jdata)``** un-claims all slot assignments +made during the previous MAP pass so that the job can be remapped cleanly. +Mirror the mapper's ``prte_rmaps_base_claim_slot()`` accounting, which does +``node->num_procs++`` and ``++node->slots_inuse`` for each non-tool proc: + +.. code-block:: c + + void prte_plm_base_reset_proc_map(prte_job_t *jdata) + { + int p, np; + prte_proc_t *proc; + prte_node_t *node; + prte_app_context_t *app; + + for (p = 0; p < jdata->procs->size; p++) { + proc = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, p); + if (NULL == proc) continue; + node = proc->node; + if (NULL != node) { + /* remove from node's proc list */ + for (np = 0; np < node->procs->size; np++) { + if (pmix_pointer_array_get_item(node->procs, np) == proc) { + pmix_pointer_array_set_item(node->procs, np, NULL); + node->num_procs--; + /* mirror claim_slot: tool procs do not count + * against slots_inuse */ + app = (prte_app_context_t *) + pmix_pointer_array_get_item(jdata->apps, + proc->app_idx); + if (NULL == app || + !PRTE_FLAG_TEST(app, PRTE_APP_FLAG_TOOL)) { + node->slots_inuse--; + } + break; + } + } + } + pmix_pointer_array_set_item(jdata->procs, p, NULL); + PMIX_RELEASE(proc); + } + jdata->num_procs = 0; + jdata->num_launched = 0; + } + +After remapping, the job re-enters ``prte_rmaps_base_map_job()`` which +re-creates proc objects on the surviving nodes using the original +``app->num_procs`` counts. + +Step 5 — Detect target departure in the errmgr and notify completion +-------------------------------------------------------------------- + +Campaign completion is driven entirely by the daemon-loss path: when a +targeted daemon leaves the DVM, the HNP's comm-failure handler matches its +rank against the active campaigns and drives the fence down. This is the +same event whether the daemon exited cleanly in response to the shrink +command or crashed, so a single code path covers both — there is no separate +"acknowledgement" message and no fallback to reconcile. + +In ``src/mca/errmgr/dvm/errmgr_dvm.c``, inside the ``PMIX_CHECK_NSPACE`` +daemon-proc block of ``proc_errors()`` (line 252), within the +``PRTE_PROC_STATE_COMM_FAILED`` / heartbeat-failed handler, add after the +"mark daemon as gone" logic: + +.. code-block:: c + + /* check if this daemon was a pending shrink target */ + { + prte_shrink_campaign_t *_camp, *_next; + int _t; + PMIX_LIST_FOREACH_SAFE(_camp, _next, + &prte_shrink_campaigns, prte_shrink_campaign_t) { + for (_t = 0; _t < _camp->ntargets; _t++) { + if (_camp->targets[_t] != proc->rank) continue; + /* stamp this slot so a repeated comm event for the same + * daemon cannot decrement the campaign twice */ + _camp->targets[_t] = PMIX_RANK_INVALID; + _camp->pending--; + prte_dvm_launch_fence--; + if (0 == _camp->pending) { + /* this request's shrink is complete — first let the + * active RAS modules release the freed resources back + * to their resource manager(s), then notify the + * requester that the DVM now reflects the new size */ + prte_ras_base_shrink_complete(_camp); + if (_camp->have_requester) { + /* success == true => PMIX_DVM_IS_READY */ + prte_plm_base_dvm_mod_notify(&_camp->requester, + _camp->alloc_id, + _camp->req_id, + true, PMIX_SUCCESS); + } + pmix_list_remove_item(&prte_shrink_campaigns, + &_camp->super); + PMIX_RELEASE(_camp); + } + if (0 == prte_dvm_launch_fence) { + prte_plm_base_fence_release(); + } + goto errmgr_shrink_done; + } + } + errmgr_shrink_done: ; + } + +Because the progress thread is single-threaded, the counter decrements and +the list manipulation are atomic with respect to all other state machine +callbacks. Stamping the matched slot ``PMIX_RANK_INVALID`` makes the +decrement idempotent: should the daemon generate more than one failure event, +only the first is counted. A daemon that crashes during a shrink is handled +identically to one that exits cleanly — the node was being removed anyway, +and jobs mapped to it are detected by ``prte_plm_base_job_needs_remap()`` and +re-routed to surviving nodes. + +Before the completion event is emitted, ``prte_ras_base_shrink_complete()`` +cycles across every active RAS module (``prte_ras_base.selected_modules``) and +invokes the optional ``shrink_complete`` entry point on each, passing the +completed ``prte_shrink_campaign_t``. This is the component's opportunity to +release the freed resources back to its resource manager; what it does with that +opportunity is up to the component. Unlike ``modify``, the cycle is not keyed +to a single component: a single ``PMIX_ALLOC_RELEASE`` may remove nodes drawn +from more than one allocation (see the "Resource release at shrink completion" +section of :ref:`elastic-dvm-spec-label`), so every module is offered the +campaign and each handles only the share that belongs to it. A module with no +``shrink_complete`` pointer, or with no stake in the operation, is a no-op. +The runtime guarantees only the ordering: the release cycle runs ahead of +``prte_plm_base_dvm_mod_notify()``, so by the time the requester sees +``PMIX_DVM_IS_READY`` every component has been given its chance to act — not +that any particular resource was in fact returned, which is the component's +decision. + +The ``PMIX_DVM_IS_READY`` notification is **per campaign**: it fires when *this* +request's last target departs, regardless of whether other (grow or shrink) +campaigns keep the shared fence nonzero. The fence-release of held jobs, by +contrast, waits for the global fence to reach zero. The failure counterpart +(``PMIX_ERR_DVM_MOD``) is emitted only on the xcast-failure cleanup in Step 1 — +once the shrink command is on the wire, every targeted daemon's departure is a +*success* for the campaign, since clean exit and crash are indistinguishable +and both remove the node as requested. + +Summary of Files Changed (Shrink Fence) +----------------------------------------- + +.. list-table:: + :widths: 50 50 + :header-rows: 1 + + * - File + - Change + * - ``src/runtime/prte_globals.h`` + - Declare ``prte_shrink_campaign_t`` (type + ``PMIX_CLASS_DECLARATION``, + including the requester fields) and ``prte_shrink_campaigns`` + (``pmix_list_t``). + * - ``src/runtime/prte_globals.c`` + - Define ``PMIX_CLASS_INSTANCE`` for ``prte_shrink_campaign_t`` + (destructor frees ``targets``, ``alloc_id``, ``req_id``). Define + ``prte_shrink_campaigns``. + * - ``src/runtime/prte_init.c`` + - ``PMIX_CONSTRUCT(&prte_shrink_campaigns, pmix_list_t)`` alongside + ``prte_held_jobs`` initialization. + * - ``src/runtime/prte_finalize.c`` + - ``PMIX_LIST_DESTRUCT(&prte_shrink_campaigns)``. + * - ``src/mca/ras/base/ras_base_allocate.c`` + - Add ``#include "src/mca/plm/base/plm_private.h"`` for + ``prte_plm_base_dvm_mod_notify()``. In the ``PMIX_ALLOC_RELEASE`` branch + of ``prte_ras_base_complete_request()``, guarded on ``0 < m``: create a + ``prte_shrink_campaign_t``, copy the rank array into it, record the + requester from ``req->tproc`` and ``PMIX_ALLOC_ID`` / ``PMIX_ALLOC_REQ_ID`` + from ``req->info``, append to ``prte_shrink_campaigns``, and increment + ``prte_dvm_launch_fence`` by ``m`` — all before ``free(ranks)``. Add + xcast-failure cleanup that removes the campaign, decrements the fence, + and emits ``PMIX_ERR_DVM_MOD`` (carrying ``prte_pmix_convert_rc(rc)``) to + the requester. + * - ``src/prted/prted_comm.c`` + - In ``PRTE_DAEMON_SHRINK_CMD`` handler: after the ``JOB_END`` + notification wait, activate ``PRTE_JOB_STATE_DAEMONS_TERMINATED`` and + exit. No acknowledgement is sent; the HNP detects departure via the + comm-failure path. + * - ``src/mca/plm/base/plm_base_launch_support.c`` + - Add ``prte_plm_base_job_needs_remap()`` and + ``prte_plm_base_reset_proc_map()``. Add hold check in + ``prte_plm_base_launch_apps()`` on + ``!pmix_list_is_empty(&prte_shrink_campaigns)``. + * - ``src/mca/plm/base/plm_private.h`` + - Declare the two remap helpers. + * - ``src/mca/ras/ras.h`` + - Add the ``shrink_complete`` module entry point: a + ``prte_ras_base_module_shrink_complete_fn_t`` taking the completed + ``prte_shrink_campaign_t``, and a field for it in + ``prte_ras_base_module_t`` (after ``modify``). Components that hand + resources back to a scheduler implement it; others leave it ``NULL``. + * - ``src/mca/ras/base/base.h`` + - Declare ``prte_ras_base_shrink_complete(prte_shrink_campaign_t *)``. + * - ``src/mca/ras/base/ras_base_allocate.c`` + - Define ``prte_ras_base_shrink_complete()``: cycle across + ``prte_ras_base.selected_modules`` and invoke each module's + ``shrink_complete`` (when non-NULL), passing the campaign. Not keyed to + one component, since a release may span multiple allocations. + * - ``src/mca/errmgr/dvm/errmgr_dvm.c`` + - Add ``#include "src/mca/ras/base/base.h"``. In ``proc_errors()``, + daemon-comm-failure block: search ``prte_shrink_campaigns`` for the dead + daemon's rank; if found, stamp the matched target slot + ``PMIX_RANK_INVALID``, decrement campaign ``pending`` and fence; when + ``pending`` hits zero call ``prte_ras_base_shrink_complete()`` to release + the resources RM-side, then emit ``PMIX_DVM_IS_READY`` to the requester + and remove the campaign; call ``prte_plm_base_fence_release()`` when the + fence hits zero. This is the sole shrink-completion trigger. + +The shared infrastructure this path relies on — the fence counter, held-job +arrays, ``VM_READY → MAP`` hold point, ``prte_plm_base_fence_release()``, and +the ``prte_plm_base_dvm_mod_notify()`` completion-event helper — is listed in +the "Shared Fence Infrastructure" table in :ref:`elastic-dvm-plan-label`. + +Design Invariants +----------------- + +* ``prte_shrink_campaigns`` is a ``pmix_list_t``; each entry covers exactly + one ``PMIX_ALLOC_RELEASE`` request. Multiple concurrent shrink campaigns + are supported. +* The fence is incremented by exactly ``m`` at campaign creation and + decremented by 1 for each targeted daemon whose departure is detected on + the errmgr comm-failure path (clean exit and crash are indistinguishable + and handled identically). Each target slot is stamped ``PMIX_RANK_INVALID`` + once counted, so a repeated comm event cannot decrement twice. A campaign + is removed from the list when its ``pending`` count reaches zero. +* The ``LAUNCH_APPS`` hold uses ``!pmix_list_is_empty(&prte_shrink_campaigns)``, + not ``prte_dvm_launch_fence > 0``, so a concurrent grow does not stall + already-mapped jobs on surviving nodes. +* ``prte_shrink_campaigns`` is stable throughout each campaign: the targets + array for a given campaign is valid from creation through removal, so + ``prte_plm_base_job_needs_remap()`` can safely iterate it during release. +* Jobs in ``prte_prelaunch_held_jobs`` hold a ``PMIX_RETAIN`` reference; + ``prte_plm_base_fence_release()`` releases it after re-activating the job. + These jobs wait only on a shrink and are never aborted by a concurrent + grow failure (the grow-failure abort touches only ``prte_held_jobs``); since + shrink completion is success-only, they are always re-activated, not failed. +* The completion event is per campaign and fires from the campaign-removal + point (``pending == 0``), so each accepted release yields exactly one + ``PMIX_DVM_IS_READY`` (success) or, on an xcast failure at creation, exactly + one ``PMIX_ERR_DVM_MOD`` — never both, and never for a scheduler-driven + release with no requester. +* The RM-side release cycle runs strictly *before* the completion event. At + ``pending == 0`` the campaign-removal point calls + ``prte_ras_base_shrink_complete()``, which offers the campaign to every active + RAS module, and only then emits ``PMIX_DVM_IS_READY``. Because the departing + nodes may span multiple allocations, all modules are cycled — each handling + only its own share. The invariant is the *ordering* (every component is + offered the campaign before the event fires), not that any resource was + actually returned: that is the component's decision, outside the runtime's + control. + +Follow-up — collective shrink completion +----------------------------------------- + +A possible optimization — repairing the routing tree once per shrink campaign +(a collective completion scheme) rather than once per departing daemon — has +been deferred out of the launch-fence work and tracked separately as +`openpmix/prrte#2492 `_. diff --git a/docs/plans/index.rst b/docs/plans/index.rst index d3415e2e21..e5d237e849 100644 --- a/docs/plans/index.rst +++ b/docs/plans/index.rst @@ -8,3 +8,4 @@ Detailed implementation plans for in-progress and upcoming features. per_app_mapping/index.rst node_reservation/index.rst + elastic_dvm/index.rst diff --git a/src/mca/errmgr/dvm/errmgr_dvm.c b/src/mca/errmgr/dvm/errmgr_dvm.c index 7a07a81ad4..8e775a2ecf 100644 --- a/src/mca/errmgr/dvm/errmgr_dvm.c +++ b/src/mca/errmgr/dvm/errmgr_dvm.c @@ -43,6 +43,7 @@ #include "src/mca/iof/base/base.h" #include "src/mca/odls/base/base.h" #include "src/mca/plm/base/base.h" +#include "src/mca/ras/base/base.h" #include "src/mca/rmaps/rmaps_types.h" #include "src/rml/rml.h" #include "src/mca/state/base/base.h" @@ -60,6 +61,7 @@ #include "src/mca/errmgr/base/base.h" #include "src/mca/errmgr/base/errmgr_private.h" #include "src/mca/errmgr/errmgr.h" +#include "src/mca/plm/base/plm_private.h" #include "errmgr_dvm.h" @@ -270,6 +272,55 @@ static void proc_errors(int fd, short args, void *cbdata) pptr->state = state; /* adjust our num_procs */ --prte_process_info.num_daemons; + /* if this daemon was the target of an in-progress grow campaign, + * resolve its rank against the launch fence (failure). Only the + * specific ranks being launched affect the fence, so an unrelated + * daemon loss during a grow no longer consumes the fence. When it + * was a grow target the campaign is rolled back out of the DVM and + * the loss is fully handled here, so skip the general daemon-loss + * handling below — that path would otherwise abort the whole DVM + * over a failure the grow rollback has already absorbed. */ + if (prte_plm_base_grow_target_failed(proc->rank)) { + goto cleanup; + } + /* check if this daemon was a pending shrink target */ + { + prte_shrink_campaign_t *_camp, *_next; + int _t; + PMIX_LIST_FOREACH_SAFE(_camp, _next, + &prte_shrink_campaigns, prte_shrink_campaign_t) { + for (_t = 0; _t < _camp->ntargets; _t++) { + if (_camp->targets[_t] != proc->rank) continue; + /* stamp this slot so a repeated comm event for the + * same daemon cannot decrement the campaign twice */ + _camp->targets[_t] = PMIX_RANK_INVALID; + _camp->pending--; + prte_dvm_launch_fence--; + if (0 == _camp->pending) { + /* this request's shrink is complete — first let the + * active RAS modules release the freed resources + * back to the scheduler, then notify the requester + * that the DVM now reflects the new size (clean exit + * and crash are both successes for the campaign, so + * this drain is always success) */ + prte_ras_base_shrink_complete(_camp); + if (_camp->have_requester) { + prte_plm_base_dvm_mod_notify(&_camp->requester, + _camp->alloc_id, + _camp->req_id, + true, PMIX_SUCCESS); + } + pmix_list_remove_item(&prte_shrink_campaigns, &_camp->super); + PMIX_RELEASE(_camp); + } + if (0 == prte_dvm_launch_fence) { + prte_plm_base_fence_release(); + } + goto errmgr_shrink_done; + } + } + errmgr_shrink_done: ; + } /* if we have ordered prteds to terminate or abort * is in progress, record it */ if (prte_prteds_term_ordered || prte_abnormal_term_ordered) { diff --git a/src/mca/plm/base/plm_base_launch_support.c b/src/mca/plm/base/plm_base_launch_support.c index 4c8df79704..8c70beaefb 100644 --- a/src/mca/plm/base/plm_base_launch_support.c +++ b/src/mca/plm/base/plm_base_launch_support.c @@ -837,6 +837,18 @@ void prte_plm_base_launch_apps(int fd, short args, void *cbdata) /* update job state */ caddy->jdata->state = caddy->job_state; + /* if a shrink campaign is active, hold this job until all targeted + * daemons have departed the DVM to avoid sending launch data to a dying + * daemon (shrink campaigns are only ever created in elastic mode, so the + * explicit guard keeps the non-elastic launch path identical) */ + if (prte_elastic_mode && !pmix_list_is_empty(&prte_shrink_campaigns)) { + jdata->state = PRTE_JOB_STATE_WAITING_FOR_DAEMONS; + PMIX_RETAIN(jdata); + pmix_pointer_array_add(prte_prelaunch_held_jobs, jdata); + PMIX_RELEASE(caddy); + return; + } + PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output, "%s plm:base:launch_apps for job %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace))); @@ -2388,5 +2400,420 @@ int prte_plm_base_setup_virtual_machine(prte_job_t *jdata) } } + /* The launch fence only operates when the DVM is permitted to grow/shrink. + * Outside elastic mode the DVM is fixed-size, so no campaign is recorded + * and the fence is never raised — leaving the normal launch path, and the + * normal daemon-failure handling, completely unchanged. */ + if (prte_elastic_mode && 0 < map->num_new_daemons) { + prte_grow_campaign_t *gcamp; + pmix_rank_t gr; + int gk; + + /* Record this launch campaign so the launch fence can be resolved on + * a per-daemon basis: each new daemon either reports home (success) + * or its launch fails (comm-failure / failed-to-start), and only + * those specific ranks affect this campaign. This avoids an + * unrelated daemon loss consuming the fence, and lets concurrent + * campaigns be tracked independently. The new daemons were assigned + * consecutive vpids starting at map->daemon_vpid_start (see the + * daemon-creation loop above). */ + gcamp = PMIX_NEW(prte_grow_campaign_t); + gcamp->ntargets = map->num_new_daemons; + gcamp->targets = (pmix_rank_t *) malloc(gcamp->ntargets * sizeof(pmix_rank_t)); + for (gk = 0, gr = map->daemon_vpid_start; gk < gcamp->ntargets; gk++, gr++) { + gcamp->targets[gk] = gr; + } + /* Record the requester for the spec's phase-two completion event. The + * RAS reservation machinery sets each reserved node's ->session + * backpointer (add_nodes_to_session), and that session carries the + * requestor and the allocation ids; take them from the first new + * daemon's node. The initial DVM bring-up and a scheduler-driven push + * have no such requestor (the default session, or an invalid requestor + * rank), so have_requester stays false and grow_drain() emits no event + * for them. */ + { + prte_proc_t *dproc = (prte_proc_t *) + pmix_pointer_array_get_item(daemons->procs, map->daemon_vpid_start); + prte_session_t *sess = + (NULL != dproc && NULL != dproc->node) ? dproc->node->session : NULL; + if (NULL != sess && PMIX_RANK_INVALID != sess->requestor.rank) { + PMIX_XFER_PROCID(&gcamp->requester, &sess->requestor); + gcamp->alloc_id = (NULL != sess->alloc_refid) ? strdup(sess->alloc_refid) : NULL; + gcamp->req_id = (NULL != sess->user_refid) ? strdup(sess->user_refid) : NULL; + gcamp->have_requester = true; + } + } + pmix_list_append(&prte_grow_campaigns, &gcamp->super); + prte_dvm_launch_fence += map->num_new_daemons; + } + return PRTE_SUCCESS; } + +void prte_plm_base_dvm_mod_notify(const pmix_proc_t *requester, + const char *alloc_id, + const char *req_id, + bool success, + pmix_status_t cause) +{ +#if PRTE_HAVE_DVM_MOD_EVENTS + pmix_status_t code = success ? PMIX_DVM_IS_READY : PMIX_ERR_DVM_MOD; + pmix_info_t *rinfo; + pmix_data_array_t parray; + pmix_proc_t ptarg; + size_t nrinfo, idx; + pmix_status_t rc; + + /* Assemble a directed (custom-range) notification to the requester only, + * mirroring the PMIX_ALLOC_TIMEOUT_WARNING delivery: custom range plus the + * allocation id, the requester's own request id when one was supplied, and + * — on failure — the underlying cause so the requester can distinguish + * what went wrong rather than only that something did. */ + nrinfo = 2; + if (NULL != req_id) { + nrinfo++; + } + if (!success) { + nrinfo++; + } + PMIX_INFO_CREATE(rinfo, nrinfo); + idx = 0; + PMIX_LOAD_PROCID(&ptarg, requester->nspace, requester->rank); + parray.type = PMIX_PROC; + parray.size = 1; + parray.array = &ptarg; + /* PMIX_INFO_LOAD deep-copies the data, so the stack copies are fine */ + PMIX_INFO_LOAD(&rinfo[idx++], PMIX_EVENT_CUSTOM_RANGE, &parray, PMIX_DATA_ARRAY); + PMIX_INFO_LOAD(&rinfo[idx++], PMIX_ALLOC_ID, (void *) alloc_id, PMIX_STRING); + if (NULL != req_id) { + PMIX_INFO_LOAD(&rinfo[idx++], PMIX_ALLOC_REQ_ID, (void *) req_id, PMIX_STRING); + } + if (!success) { + /* carry the underlying failure status; PMIX_JOB_TERM_STATUS is the + * standard pmix_status_t-typed info key (reconcile the carrier with + * PMIx PR openpmix#3917 should it define a dedicated DVM-mod cause + * key) */ + PMIX_INFO_LOAD(&rinfo[idx++], PMIX_JOB_TERM_STATUS, &cause, PMIX_STATUS); + } + rc = PMIx_Notify_event(code, PRTE_PROC_MY_NAME, PMIX_RANGE_CUSTOM, + rinfo, nrinfo, NULL, NULL); + if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) { + PMIX_ERROR_LOG(rc); + } + PMIX_INFO_FREE(rinfo, nrinfo); +#else + /* The installed PMIx defines neither DVM modification event code, so the + * completion notification is compiled out per the spec's backward- + * compatibility clause. The DVM still grows/shrinks; only the event is + * omitted. */ + PRTE_HIDE_UNUSED_PARAMS(requester, alloc_id, req_id, success, cause); +#endif +} + +void prte_plm_base_fence_release(void) +{ + int hi; + prte_job_t *held; + prte_shrink_campaign_t *scamp, *snext; + prte_grow_campaign_t *gcamp, *gnext; + + /* SUCCESS release — reached only when the global fence has dropped to + * zero, i.e. every grow and shrink campaign has completed successfully. + * Both classes of held job are admitted. The only path that *fails* a + * held job is prte_plm_base_abort_premap_held() on a grow failure. */ + for (hi = 0; hi < prte_held_jobs->size; hi++) { + held = (prte_job_t *) pmix_pointer_array_get_item(prte_held_jobs, hi); + if (NULL == held) { + continue; + } + pmix_pointer_array_set_item(prte_held_jobs, hi, NULL); + PRTE_ACTIVATE_JOB_STATE(held, PRTE_JOB_STATE_VM_READY); + PMIX_RELEASE(held); + } + + for (hi = 0; hi < prte_prelaunch_held_jobs->size; hi++) { + held = (prte_job_t *) pmix_pointer_array_get_item(prte_prelaunch_held_jobs, hi); + if (NULL == held) { + continue; + } + pmix_pointer_array_set_item(prte_prelaunch_held_jobs, hi, NULL); + if (prte_plm_base_job_needs_remap(held)) { + prte_plm_base_reset_proc_map(held); + PRTE_ACTIVATE_JOB_STATE(held, PRTE_JOB_STATE_MAP); + } else { + PRTE_ACTIVATE_JOB_STATE(held, PRTE_JOB_STATE_LAUNCH_APPS); + } + PMIX_RELEASE(held); + } + + /* Campaigns are removed individually as they drain, so both lists should + * be empty here. Sweep each defensively anyway — and sweep both kinds, + * not just shrink — so a future change that can leave a residual campaign + * behind cannot wedge the fence. */ + PMIX_LIST_FOREACH_SAFE(scamp, snext, &prte_shrink_campaigns, prte_shrink_campaign_t) { + pmix_list_remove_item(&prte_shrink_campaigns, &scamp->super); + PMIX_RELEASE(scamp); + } + PMIX_LIST_FOREACH_SAFE(gcamp, gnext, &prte_grow_campaigns, prte_grow_campaign_t) { + pmix_list_remove_item(&prte_grow_campaigns, &gcamp->super); + PMIX_RELEASE(gcamp); + } +} + +void prte_plm_base_abort_premap_held(void) +{ + int hi; + prte_job_t *held; + + /* GROW-FAILURE abort — fail every job parked at the VM_READY -> MAP + * boundary, immediately and independent of the fence value, so a grow + * failure aborts its pre-map waiters even while a concurrent shrink keeps + * the fence nonzero. The pre-launch held jobs are deliberately left + * untouched: they wait only on a shrink, never on the grow (spec + * conformance #4). */ + for (hi = 0; hi < prte_held_jobs->size; hi++) { + held = (prte_job_t *) pmix_pointer_array_get_item(prte_held_jobs, hi); + if (NULL == held) { + continue; + } + pmix_pointer_array_set_item(prte_held_jobs, hi, NULL); + PRTE_ACTIVATE_JOB_STATE(held, PRTE_JOB_STATE_NEVER_LAUNCHED); + PMIX_RELEASE(held); + } +} + +void prte_plm_base_grow_drain(bool success) +{ + prte_grow_campaign_t *camp; + + /* Resolve all in-progress grow campaigns at once and drop their entire + * contribution from the launch fence. This is called from exactly the + * two safe points: from vm_ready on the all-success path (after the + * WIREUP xcast, so held jobs are only admitted once the new daemons are + * wired up), and from the failure path when one of the launched daemons + * dies. Each drained campaign emits its phase-two completion event to its + * requester — PMIX_DVM_IS_READY on success, PMIX_ERR_DVM_MOD on failure. */ + while (NULL != (camp = (prte_grow_campaign_t *) + pmix_list_remove_first(&prte_grow_campaigns))) { + prte_dvm_launch_fence -= camp->ntargets; + if (camp->have_requester) { + /* the specific daemon-failure status is not threaded down to this + * layer yet, so report a generic cause on failure (reconcile by + * passing the dying daemon's status as a follow-up) */ + prte_plm_base_dvm_mod_notify(&camp->requester, camp->alloc_id, + camp->req_id, success, + success ? PMIX_SUCCESS : PMIX_ERROR); + } + PMIX_RELEASE(camp); + } + if (success) { + /* admit the held jobs only once the *global* fence is clear — a + * concurrent shrink may still hold it nonzero */ + if (0 == prte_dvm_launch_fence) { + prte_plm_base_fence_release(); + } + } else { + /* a grow failure fails the whole pre-map held-job set immediately, + * regardless of the fence value, so a concurrent shrink cannot later + * admit a job whose grow dependency has failed; the shrink-only + * pre-launch held jobs are left parked */ + prte_plm_base_abort_premap_held(); + } +} + +/* Roll a failed grow campaign back out of the DVM so a failed grow leaves the + * DVM at its exact pre-grow membership rather than half-extended (spec + * conformance #5). `trigger` is the rank whose death triggered the failure; + * the errmgr has already marked it not-alive and decremented num_daemons, and + * its routing is repaired here. Every *other* target is handled by whether a + * daemon actually came up: + * + * - a target that started (PRTE_PROC_FLAG_ALIVE) is terminated using the same + * PRTE_DAEMON_SHRINK_CMD machinery the DVM shrink path uses; its departure + * is then reconciled on the normal daemon-loss path as it exits; + * - a target that never started has no daemon to signal, so its launch-time + * daemon-count bump is reverted here (no comm-failure event will arrive for + * it). + * + * In all cases the node's daemon backpointer is cleared so the mapper can no + * longer place a later job on it. The new nodes carry no application procs — + * the jobs that would have used them were held at the fence, never launched — + * so clearing ``node->daemon`` is sufficient to remove the node from the DVM's + * usable set. The teardown is strictly campaign-scoped: it touches only the + * ranks in this campaign's ``targets`` array. */ +static void grow_rollback(prte_grow_campaign_t *camp, pmix_rank_t trigger) +{ + prte_job_t *daemons; + prte_proc_t *dproc; + prte_node_t *node; + pmix_rank_t *kill; + int32_t nkill = 0; + int t; + + daemons = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace); + if (NULL == daemons) { + return; + } + kill = (pmix_rank_t *) malloc(camp->ntargets * sizeof(pmix_rank_t)); + if (NULL == kill) { + return; + } + + /* repair routing for the daemon whose loss triggered the failure — the + * errmgr's own route_lost call is skipped because grow_target_failed() + * reports the death as handled (so the errmgr does not abort the DVM) */ + prte_rml_route_lost(trigger); + + for (t = 0; t < camp->ntargets; t++) { + pmix_rank_t r = camp->targets[t]; + if (PMIX_RANK_INVALID == r) { + continue; + } + dproc = (prte_proc_t *) pmix_pointer_array_get_item(daemons->procs, r); + if (NULL == dproc) { + continue; + } + node = dproc->node; + if (r != trigger) { + if (PRTE_FLAG_TEST(dproc, PRTE_PROC_FLAG_ALIVE)) { + /* a started daemon — terminate it via the shrink command */ + kill[nkill++] = r; + } else { + /* never started: no comm-failure event will arrive, so revert + * its launch-time daemon-count bump here */ + if (0 < prte_process_info.num_daemons) { + --prte_process_info.num_daemons; + } + } + } + /* detach the node from the DVM's usable set */ + if (NULL != node) { + if (NULL != node->session && node->session != prte_default_session) { + node->session = NULL; + } + if (node->daemon == dproc) { + node->daemon = NULL; + PMIX_RELEASE(dproc); + } + } + } + + if (0 < nkill) { + pmix_data_buffer_t msg; + prte_daemon_cmd_flag_t cmd = PRTE_DAEMON_SHRINK_CMD; + pmix_status_t rc; + + PMIX_DATA_BUFFER_CONSTRUCT(&msg); + rc = PMIx_Data_pack(NULL, &msg, &cmd, 1, PMIX_UINT8); + if (PMIX_SUCCESS == rc) { + rc = PMIx_Data_pack(NULL, &msg, &nkill, 1, PMIX_INT32); + } + if (PMIX_SUCCESS == rc) { + rc = PMIx_Data_pack(NULL, &msg, kill, nkill, PMIX_PROC_RANK); + } + if (PMIX_SUCCESS == rc) { + if (PRTE_SUCCESS != (rc = prte_grpcomm.xcast(PRTE_RML_TAG_DAEMON, &msg))) { + PRTE_ERROR_LOG(rc); + } + } else { + PMIX_ERROR_LOG(rc); + } + PMIX_DATA_BUFFER_DESTRUCT(&msg); + } + free(kill); +} + +bool prte_plm_base_grow_target_failed(pmix_rank_t rank) +{ + prte_grow_campaign_t *camp; + int t; + + /* Outside elastic mode there are no grow campaigns and the daemon loss is + * the errmgr's to handle exactly as it always has — never report it as + * handled here, so the normal DVM-failure path is preserved. */ + if (!prte_elastic_mode) { + return false; + } + + /* A daemon has died. Only act if it was actually the target of an + * in-progress grow campaign — an unrelated daemon loss must not consume + * the launch fence (and must be left to the errmgr's normal handling). + * If it was a grow target, that campaign is compromised: roll it back out + * of the DVM, drop its fence contribution, notify its requester of the + * failure, and abort the pre-map held jobs. The teardown is scoped to the + * matched campaign, so any concurrent grow keeps its own daemons and + * completes normally (spec conformance #5). */ + PMIX_LIST_FOREACH(camp, &prte_grow_campaigns, prte_grow_campaign_t) { + for (t = 0; t < camp->ntargets; t++) { + if (camp->targets[t] != rank) { + continue; + } + pmix_list_remove_item(&prte_grow_campaigns, &camp->super); + prte_dvm_launch_fence -= camp->ntargets; + grow_rollback(camp, rank); + if (camp->have_requester) { + /* the specific daemon-failure status is not threaded down to + * this layer yet, so report a generic cause */ + prte_plm_base_dvm_mod_notify(&camp->requester, camp->alloc_id, + camp->req_id, false, PMIX_ERROR); + } + PMIX_RELEASE(camp); + /* any grow failure fails the whole pre-map held-job set */ + prte_plm_base_abort_premap_held(); + return true; + } + } + return false; +} + +bool prte_plm_base_job_needs_remap(prte_job_t *jdata) +{ + prte_shrink_campaign_t *camp; + prte_proc_t *proc; + int p, t; + + PMIX_LIST_FOREACH(camp, &prte_shrink_campaigns, prte_shrink_campaign_t) { + for (p = 0; p < jdata->procs->size; p++) { + proc = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, p); + if (NULL == proc || NULL == proc->node || NULL == proc->node->daemon) continue; + for (t = 0; t < camp->ntargets; t++) { + if (camp->targets[t] == proc->node->daemon->name.rank) { + return true; + } + } + } + } + return false; +} + +void prte_plm_base_reset_proc_map(prte_job_t *jdata) +{ + int p, np; + prte_proc_t *proc; + prte_node_t *node; + prte_app_context_t *app; + + for (p = 0; p < jdata->procs->size; p++) { + proc = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, p); + if (NULL == proc) continue; + node = proc->node; + if (NULL != node) { + for (np = 0; np < node->procs->size; np++) { + if (pmix_pointer_array_get_item(node->procs, np) == proc) { + pmix_pointer_array_set_item(node->procs, np, NULL); + node->num_procs--; + app = (prte_app_context_t *) + pmix_pointer_array_get_item(jdata->apps, proc->app_idx); + if (NULL == app || !PRTE_FLAG_TEST(app, PRTE_APP_FLAG_TOOL)) { + node->slots_inuse--; + } + break; + } + } + } + pmix_pointer_array_set_item(jdata->procs, p, NULL); + PMIX_RELEASE(proc); + } + jdata->num_procs = 0; + jdata->num_launched = 0; +} diff --git a/src/mca/plm/base/plm_private.h b/src/mca/plm/base/plm_private.h index fdbace30fd..c5bbf7b4cc 100644 --- a/src/mca/plm/base/plm_private.h +++ b/src/mca/plm/base/plm_private.h @@ -94,6 +94,20 @@ PRTE_EXPORT void prte_plm_base_reset_job(prte_job_t *jdata); PRTE_EXPORT int prte_plm_base_setup_prted_cmd(int *argc, char ***argv); PRTE_EXPORT void prte_plm_base_check_all_complete(int fd, short args, void *cbdata); PRTE_EXPORT int prte_plm_base_setup_virtual_machine(prte_job_t *jdata); +PRTE_EXPORT void prte_plm_base_fence_release(void); +PRTE_EXPORT void prte_plm_base_abort_premap_held(void); +PRTE_EXPORT void prte_plm_base_grow_drain(bool success); +PRTE_EXPORT bool prte_plm_base_grow_target_failed(pmix_rank_t rank); +PRTE_EXPORT bool prte_plm_base_job_needs_remap(prte_job_t *jdata); +PRTE_EXPORT void prte_plm_base_reset_proc_map(prte_job_t *jdata); +/* emit the spec's phase-two completion event to the size-change requester: + * PMIX_DVM_IS_READY when success, else PMIX_ERR_DVM_MOD carrying `cause`. + * Compiles to a no-op when PMIx lacks the DVM modification event codes. */ +PRTE_EXPORT void prte_plm_base_dvm_mod_notify(const pmix_proc_t *requester, + const char *alloc_id, + const char *req_id, + bool success, + pmix_status_t cause); /** * Utilities for plm components that use proxy daemons diff --git a/src/mca/plm/plm_types.h b/src/mca/plm/plm_types.h index 3dc5753cf1..154aaacb7e 100644 --- a/src/mca/plm/plm_types.h +++ b/src/mca/plm/plm_types.h @@ -130,6 +130,7 @@ typedef int32_t prte_job_state_t; #define PRTE_JOB_STATE_RUNNING 14 /* all procs have been fork'd */ #define PRTE_JOB_STATE_SUSPENDED 15 /* job has been suspended */ #define PRTE_JOB_STATE_REGISTERED 16 /* all procs registered for sync */ +#define PRTE_JOB_STATE_WAITING_FOR_DAEMONS 17 /* parked: daemon launch/shrink campaign in progress */ #define PRTE_JOB_STATE_LOCAL_LAUNCH_COMPLETE 18 /* all local procs have attempted launch */ #define PRTE_JOB_STATE_READY_FOR_DEBUG 19 /* all local procs report ready for debug */ #define PRTE_JOB_STATE_STARTED 20 /* first process has been started */ diff --git a/src/mca/ras/base/base.h b/src/mca/ras/base/base.h index 94322b52df..c36759fb5f 100644 --- a/src/mca/ras/base/base.h +++ b/src/mca/ras/base/base.h @@ -88,6 +88,12 @@ PRTE_EXPORT void prte_ras_base_allocate(int fd, short args, void *cbdata); PRTE_EXPORT void prte_ras_base_modify(int fd, short args, void *cbdata); +/* Notify the active RAS modules that a shrink campaign has completed so they + * can release the freed resources back to the scheduler. Cycles across every + * selected module, calling the shrink_complete entry point on each that + * provides one. */ +PRTE_EXPORT void prte_ras_base_shrink_complete(prte_shrink_campaign_t *campaign); + PRTE_EXPORT void prte_ras_base_release_allocation(prte_session_t *session); /* Tear down a reservation: drop its hold on its nodes (clearing the diff --git a/src/mca/ras/base/ras_base_allocate.c b/src/mca/ras/base/ras_base_allocate.c index 953cf9f8c9..e7372ce94f 100644 --- a/src/mca/ras/base/ras_base_allocate.c +++ b/src/mca/ras/base/ras_base_allocate.c @@ -42,6 +42,7 @@ #include "src/mca/errmgr/errmgr.h" #include "src/mca/iof/base/base.h" #include "src/mca/odls/odls_types.h" +#include "src/mca/plm/base/plm_private.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/state/state.h" #include "src/runtime/prte_globals.h" @@ -616,6 +617,21 @@ void prte_ras_base_modify(int fd, short args, void *cbdata) PMIX_RELEASE(req); } +void prte_ras_base_shrink_complete(prte_shrink_campaign_t *campaign) +{ + prte_ras_base_selected_module_t *mod; + + /* cycle across the active modules and give each that supports the + * entry point a chance to release the freed resources back to the + * scheduler. Unlike modify, a shrink completion is not keyed to a + * single component, so every module is offered the campaign. */ + PMIX_LIST_FOREACH(mod, &prte_ras_base.selected_modules, prte_ras_base_selected_module_t) { + if (NULL != mod->module->shrink_complete) { + mod->module->shrink_complete(campaign); + } + } +} + /* monotonic counter used to mint unique session ids for reservations that * the host (rather than a scheduler) must identify on its own. */ static uint32_t prte_ras_reservation_counter = 0; @@ -675,20 +691,23 @@ static prte_session_t *create_reservation(const char *nspace, uint8_t inherit, return s; } -/* Register the nodes named in ndlist with the destination reservation: set - * each node's session backpointer and store a retained reference in the - * reservation. The node objects themselves live in the global pool. */ -static void add_nodes_to_session(pmix_list_t *ndlist, prte_session_t *dest) +/* Register the named nodes with the destination reservation: set each node's + * session backpointer and store a retained reference in the reservation. The + * node objects themselves live in the global pool, which is why this takes the + * node *names* and resolves them with prte_node_match: the caller's working + * list has already been drained into the pool by prte_ras_base_node_insert, so + * the names must be snapshotted before that insert and handed in here. */ +static void add_nodes_to_session(char **names, prte_session_t *dest) { - prte_node_t *nd, *gnode; - int k; + prte_node_t *gnode; + int k, i; bool present; - if (NULL == dest || dest == prte_default_session) { + if (NULL == dest || dest == prte_default_session || NULL == names) { return; } - PMIX_LIST_FOREACH(nd, ndlist, prte_node_t) { - gnode = prte_node_match(NULL, nd->name); + for (i = 0; NULL != names[i]; i++) { + gnode = prte_node_match(NULL, names[i]); if (NULL == gnode) { continue; } @@ -945,6 +964,7 @@ void prte_ras_base_complete_request(prte_pmix_server_req_t *req) prte_daemon_cmd_flag_t cmd = PRTE_DAEMON_SHRINK_CMD; size_t n; char **nodes, *ndstring; + char **rsv_names = NULL; int32_t cnt=0, m; int ret; prte_node_t *node; @@ -1099,16 +1119,30 @@ void prte_ras_base_complete_request(prte_pmix_server_req_t *req) return; } free(ndstring); + /* prte_ras_base_node_insert() drains ndlist into the global + * pool, so snapshot the node names first: add_nodes_to_session() + * below needs them to re-find the pool objects and set their + * session backpointer (which carries the requestor for the + * phase-two completion event). */ + { + prte_node_t *snap; + PMIX_LIST_FOREACH(snap, &ndlist, prte_node_t) { + PMIx_Argv_append_nosize(&rsv_names, snap->name); + } + } ret = prte_ras_base_node_insert(&ndlist, NULL); if (PRTE_SUCCESS != ret) { PRTE_ERROR_LOG(ret); PMIX_LIST_DESTRUCT(&ndlist); + PMIx_Argv_free(rsv_names); req->pstatus = prte_pmix_convert_rc(ret); return; } /* when reserving, withhold these nodes from the default pool by * registering them with the destination session */ - add_nodes_to_session(&ndlist, dest); + add_nodes_to_session(rsv_names, dest); + PMIx_Argv_free(rsv_names); + rsv_names = NULL; PMIX_LIST_DESTRUCT(&ndlist); found = true; } @@ -1261,11 +1295,54 @@ void prte_ras_base_complete_request(prte_pmix_server_req_t *req) req->pstatus = rc; return; } + /* Record the shrink campaign before freeing the ranks array. Only in + * elastic mode: outside it the DVM is fixed-size, so the release still + * xcasts the shrink command below but no campaign is recorded and the + * fence is never raised — the legacy fire-and-forget behavior. Also + * skip when the release removes no daemons (m == 0): an empty campaign + * would never drain (no target ever departs on the comm-failure path), + * so prte_shrink_campaigns would stay non-empty forever and stall every + * later job at the LAUNCH_APPS hold, and no completion event would fire. + * This mirrors the grow path's num_new_daemons > 0 guard and the spec's + * "no event when nothing changes" clause. */ + if (prte_elastic_mode && 0 < m) { + prte_shrink_campaign_t *_camp = PMIX_NEW(prte_shrink_campaign_t); + _camp->targets = (pmix_rank_t *) malloc(m * sizeof(pmix_rank_t)); + memcpy(_camp->targets, ranks, m * sizeof(pmix_rank_t)); + _camp->ntargets = m; + _camp->pending = m; + /* record the requester so the phase-two completion event can be + * directed at the process that issued this PMIX_ALLOC_RELEASE */ + PMIX_XFER_PROCID(&_camp->requester, &req->tproc); + for (n = 0; n < req->ninfo; n++) { + if (PMIx_Check_key(req->info[n].key, PMIX_ALLOC_ID)) { + _camp->alloc_id = strdup(req->info[n].value.data.string); + } else if (PMIx_Check_key(req->info[n].key, PMIX_ALLOC_REQ_ID)) { + _camp->req_id = strdup(req->info[n].value.data.string); + } + } + _camp->have_requester = true; + pmix_list_append(&prte_shrink_campaigns, &_camp->super); + prte_dvm_launch_fence += m; + } free(ranks); /* goes to all daemons */ if (PRTE_SUCCESS != (rc = prte_grpcomm.xcast(PRTE_RML_TAG_DAEMON, &msg))) { PRTE_ERROR_LOG(rc); + /* undo the campaign we just added (only if one was created), and + * tell the requester the DVM modification failed */ + if (prte_elastic_mode && 0 < m) { + prte_shrink_campaign_t *_camp = + (prte_shrink_campaign_t *) pmix_list_remove_last(&prte_shrink_campaigns); + prte_dvm_launch_fence -= _camp->pending; + if (_camp->have_requester) { + prte_plm_base_dvm_mod_notify(&_camp->requester, _camp->alloc_id, + _camp->req_id, false, + prte_pmix_convert_rc(rc)); + } + PMIX_RELEASE(_camp); + } } PMIX_DATA_BUFFER_DESTRUCT(&msg); } diff --git a/src/mca/ras/hosts/ras_hosts.c b/src/mca/ras/hosts/ras_hosts.c index eeff98b650..419f572e25 100644 --- a/src/mca/ras/hosts/ras_hosts.c +++ b/src/mca/ras/hosts/ras_hosts.c @@ -343,6 +343,7 @@ static pmix_status_t modify(prte_pmix_server_req_t *req) pmix_list_t nodes; size_t n, k; char **hostfiles; + bool handled = false; PMIX_CONSTRUCT(&nodes, pmix_list_t); @@ -361,6 +362,7 @@ static pmix_status_t modify(prte_pmix_server_req_t *req) } } PMIx_Argv_free(hostfiles); + handled = true; } if (PMIx_Check_key(req->info[n].key, PMIX_ADD_HOST)) { // comma-delimited list of hosts to add or delete @@ -371,6 +373,7 @@ static pmix_status_t modify(prte_pmix_server_req_t *req) req->pstatus = prte_pmix_convert_rc(rc); return req->pstatus; } + handled = true; } } @@ -384,5 +387,32 @@ static pmix_status_t modify(prte_pmix_server_req_t *req) return req->pstatus; } } - return PMIX_OPERATION_SUCCEEDED; + PMIX_LIST_DESTRUCT(&nodes); + + /* When no external scheduler is present, this component is the DVM's local + * resource authority for elastic operations. Claim the size-change + * directives so the base prte_ras_base_complete_request() logic runs with + * the ORIGINAL request info intact: + * - PMIX_ALLOC_NEW / PMIX_ALLOC_EXTEND carrying PMIX_ALLOC_NODE_LIST add + * the named nodes and extend the DVM; + * - PMIX_ALLOC_RELEASE removes the named nodes (PMIX_ALLOC_NODE_LIST) or + * tears down a whole reservation (PMIX_ALLOC_ID). + * Keeping the original request info intact preserves the node list and the + * allocation ids, so a release can target a specific reservation. */ + switch (req->allocdir) { + case PMIX_ALLOC_NEW: + case PMIX_ALLOC_EXTEND: + case PMIX_ALLOC_RELEASE: + handled = true; + break; + default: + break; + } + + /* If we satisfied something, let the base layer finish it; otherwise defer + * to the next module (this component is the lowest-priority RAS). */ + if (handled) { + return PMIX_OPERATION_SUCCEEDED; + } + return PMIX_ERR_TAKE_NEXT_OPTION; } diff --git a/src/mca/ras/pmix/ras_pmix.c b/src/mca/ras/pmix/ras_pmix.c index 8c5954ab4f..4b458af86e 100644 --- a/src/mca/ras/pmix/ras_pmix.c +++ b/src/mca/ras/pmix/ras_pmix.c @@ -112,24 +112,16 @@ static pmix_status_t modify(prte_pmix_server_req_t *req) pmix_info_t *xfer; size_t n; - if (prte_mca_ras_pmix_component.simulate) { - // pretend we are attached to a scheduler - xfer = PMIx_Info_create(1); - PMIX_INFO_LOAD(xfer, PMIX_ALLOC_NODE_LIST, prte_mca_ras_pmix_component.simulate_nodelist, PMIX_STRING); - req->pstatus = PMIX_SUCCESS; - req->info = xfer; - req->ninfo = 1; - prte_event_set(prte_event_base, &req->ev, -1, PRTE_EV_WRITE, passthru, req); - PMIX_POST_OBJECT(req); - prte_event_active(&req->ev, PRTE_EV_WRITE, 1); - return PMIX_SUCCESS; - } - // check if scheduler is attached and try to // attach if not rc = prte_pmix_set_scheduler(); if (PMIX_SUCCESS != rc) { - return rc; + /* No scheduler is reachable, so we cannot forward this request. Defer + * to the next RAS module rather than failing the whole request: in a + * schedulerless DVM the ras/hosts component handles node-list grow and + * shrink locally. Returning a hard error here (e.g. PMIX_ERR_UNREACH) + * would instead abort the modify loop before hosts is consulted. */ + return PMIX_ERR_TAKE_NEXT_OPTION; } // we need to pass the request on to the scheduler diff --git a/src/mca/ras/pmix/ras_pmix.h b/src/mca/ras/pmix/ras_pmix.h index 17f98a233e..bcbfd8954b 100644 --- a/src/mca/ras/pmix/ras_pmix.h +++ b/src/mca/ras/pmix/ras_pmix.h @@ -22,8 +22,6 @@ BEGIN_C_DECLS struct prte_ras_pmix_component_t { prte_ras_base_component_t super; - bool simulate; - char *simulate_nodelist; bool connect_to_system_scheduler; pmix_proc_t server; char *uri; diff --git a/src/mca/ras/pmix/ras_pmix_component.c b/src/mca/ras/pmix/ras_pmix_component.c index 41fd39b03c..bd8d1f3387 100644 --- a/src/mca/ras/pmix/ras_pmix_component.c +++ b/src/mca/ras/pmix/ras_pmix_component.c @@ -67,18 +67,6 @@ static int ras_pmix_register(void) { pmix_mca_base_component_t *component = &prte_mca_ras_pmix_component.super; - prte_mca_ras_pmix_component.simulate = false; - (void) pmix_mca_base_component_var_register(component, "simulate", - "Simulate a scheduler interaction", - PMIX_MCA_BASE_VAR_TYPE_BOOL, - &prte_mca_ras_pmix_component.simulate); - - prte_mca_ras_pmix_component.simulate_nodelist = NULL; - (void) pmix_mca_base_component_var_register(component, "simulate_nodelist", - "Comma-delimited list of nodes to use in simulation", - PMIX_MCA_BASE_VAR_TYPE_STRING, - &prte_mca_ras_pmix_component.simulate_nodelist); - prte_mca_ras_pmix_component.uri = NULL; (void) pmix_mca_base_component_var_register(component, "uri", "Specify the URI of the scheduler to which we are to connect, " diff --git a/src/mca/ras/ras.h b/src/mca/ras/ras.h index 0ba4f4e70d..fa7bf20922 100644 --- a/src/mca/ras/ras.h +++ b/src/mca/ras/ras.h @@ -89,6 +89,17 @@ typedef int (*prte_ras_base_module_allocate_fn_t)(prte_job_t *jdata, pmix_list_t /* modify allocation - includes deallocation */ typedef pmix_status_t (*prte_ras_base_module_modify_fn_t)(prte_pmix_server_req_t *req); +/** + * Notify the resource manager that a shrink campaign has completed. + * + * Called once the campaign's targeted daemons have departed and before + * the final completion notification is emitted to the requester, so the + * module can release the freed resources back to the scheduler. The + * campaign object is passed so the module can identify which shrink + * completed. + */ +typedef void (*prte_ras_base_module_shrink_complete_fn_t)(prte_shrink_campaign_t *campaign); + /** * Release an allocation associated with the given session. * Called when a prte_session_t is destructed. Returns PRTE_SUCCESS @@ -112,6 +123,8 @@ struct prte_ras_base_module_2_0_0_t { prte_ras_base_module_allocate_fn_t allocate; // modify function pointer prte_ras_base_module_modify_fn_t modify; + /** Notify the RM that a shrink campaign has completed */ + prte_ras_base_module_shrink_complete_fn_t shrink_complete; /** Release an allocation when its session is destructed */ prte_ras_base_module_release_fn_t release_allocation; /** Finalization function pointer */ diff --git a/src/mca/state/dvm/state_dvm.c b/src/mca/state/dvm/state_dvm.c index 1606f63cdc..2f3cc6caf1 100644 --- a/src/mca/state/dvm/state_dvm.c +++ b/src/mca/state/dvm/state_dvm.c @@ -38,6 +38,7 @@ #include "src/mca/iof/base/base.h" #include "src/mca/odls/odls_types.h" #include "src/mca/plm/base/base.h" +#include "src/mca/plm/base/plm_private.h" #include "src/mca/ras/base/base.h" #include "src/mca/rmaps/base/base.h" #include "src/rml/rml.h" @@ -285,6 +286,9 @@ static void vm_ready(int fd, short args, void *cbdata) if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&buf); + /* the whole DVM is being torn down; held jobs will be + * failed as part of that teardown, so leave the fence + * untouched here */ PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); return; } @@ -299,6 +303,9 @@ static void vm_ready(int fd, short args, void *cbdata) NULL == val) { PMIX_ERROR_LOG(ret); PMIX_DATA_BUFFER_DESTRUCT(&buf); + /* the whole DVM is being torn down; held jobs will be + * failed as part of that teardown, so leave the fence + * untouched here */ PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); return; } @@ -306,6 +313,9 @@ static void vm_ready(int fd, short args, void *cbdata) if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(ret); PMIX_DATA_BUFFER_DESTRUCT(&buf); + /* the whole DVM is being torn down; held jobs will be + * failed as part of that teardown, so leave the fence + * untouched here */ PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); return; } @@ -314,6 +324,9 @@ static void vm_ready(int fd, short args, void *cbdata) PMIX_ERROR_LOG(ret); PMIX_DATA_BUFFER_DESTRUCT(&buf); PMIX_VALUE_RELEASE(val); + /* the whole DVM is being torn down; held jobs will be + * failed as part of that teardown, so leave the fence + * untouched here */ PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); return; } @@ -324,11 +337,25 @@ static void vm_ready(int fd, short args, void *cbdata) if (PRTE_SUCCESS != (rc = prte_grpcomm.xcast(PRTE_RML_TAG_WIREUP, &buf))) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&buf); + /* the whole DVM is being torn down; held jobs will be + * failed as part of that teardown, so leave the fence + * untouched here */ PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); return; } PMIX_DATA_BUFFER_DESTRUCT(&buf); } + /* success path (and DO_NOT_LAUNCH / single-daemon path): the new + * daemons (if any) are now wired up. This callback only fires once + * every expected daemon has reported (num_reported == num_procs), so + * any in-progress grow campaigns have fully succeeded. Drain them, + * dropping their fence contribution, and release any held jobs. + * Doing the release here — after the WIREUP xcast above — guarantees + * held jobs are only admitted once the new daemons are wired up. + * Nothing to drain outside elastic mode (no campaign was ever made). */ + if (prte_elastic_mode) { + prte_plm_base_grow_drain(true); + } } if (PMIX_CHECK_NSPACE(PRTE_PROC_MY_NAME->nspace, caddy->jdata->nspace)) { prte_dvm_ready = true; @@ -357,6 +384,17 @@ static void vm_ready(int fd, short args, void *cbdata) return; } + /* if a daemon launch campaign is active, park this app job (only possible + * in elastic mode, where the fence is raised; the explicit guard keeps the + * non-elastic path identical even if the fence were ever left nonzero) */ + if (prte_elastic_mode && 0 < prte_dvm_launch_fence) { + caddy->jdata->state = PRTE_JOB_STATE_WAITING_FOR_DAEMONS; + PMIX_RETAIN(caddy->jdata); + pmix_pointer_array_add(prte_held_jobs, caddy->jdata); + PMIX_RELEASE(caddy); + return; + } + /* position any required files */ if (PRTE_SUCCESS != prte_filem.preposition_files(caddy->jdata, files_ready, caddy->jdata)) { PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_FILES_POSN_FAILED); @@ -542,6 +580,12 @@ static void check_complete(int fd, short args, void *cbdata) (2, prte_state_base_framework.framework_output, "%s state:dvm:check_job_complete - received NULL job, checking daemons", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); + /* safety net: if grow campaigns were still pending when the daemon + * job reached this point, drain them so held jobs are not parked + * indefinitely */ + if (!pmix_list_is_empty(&prte_grow_campaigns)) { + prte_plm_base_grow_drain(false); + } if (0 == prte_rml_base.n_children) { /* orteds are done! */ PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output, diff --git a/src/prted/prted_comm.c b/src/prted/prted_comm.c index 854245a541..860915ca4a 100644 --- a/src/prted/prted_comm.c +++ b/src/prted/prted_comm.c @@ -506,7 +506,9 @@ void prte_daemon_recv(int status, pmix_proc_t *sender, PRTE_PMIX_DESTRUCT_LOCK(&lk); // mark abnormal exit status PRTE_UPDATE_EXIT_STATUS(-1); - // do a clean exit + /* do a clean exit; the HNP detects our departure via the + * normal daemon-loss (comm-failure) path and completes the + * shrink campaign there */ PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_DAEMONS_TERMINATED); break; } diff --git a/src/runtime/prte_finalize.c b/src/runtime/prte_finalize.c index 6f086cc6ac..838200f039 100644 --- a/src/runtime/prte_finalize.c +++ b/src/runtime/prte_finalize.c @@ -89,6 +89,10 @@ int prte_finalize(void) #ifdef PRTE_PICKY_COMPILERS /* release the cache */ PMIX_RELEASE(prte_cache); + PMIX_RELEASE(prte_held_jobs); + PMIX_RELEASE(prte_prelaunch_held_jobs); + PMIX_LIST_DESTRUCT(&prte_shrink_campaigns); + PMIX_LIST_DESTRUCT(&prte_grow_campaigns); /* call the finalize function for this environment */ if (PRTE_SUCCESS != (rc = prte_ess.finalize())) { diff --git a/src/runtime/prte_globals.c b/src/runtime/prte_globals.c index 46fd9a8a92..aa2376bc99 100644 --- a/src/runtime/prte_globals.c +++ b/src/runtime/prte_globals.c @@ -74,6 +74,52 @@ char *prte_tool_basename = NULL; char *prte_tool_actual = NULL; bool prte_dvm_ready = false; pmix_pointer_array_t *prte_cache = NULL; + +int prte_dvm_launch_fence = 0; +pmix_pointer_array_t *prte_held_jobs = NULL; +pmix_pointer_array_t *prte_prelaunch_held_jobs = NULL; +pmix_list_t prte_shrink_campaigns; +pmix_list_t prte_grow_campaigns; + +static void campaign_construct(prte_shrink_campaign_t *p) +{ + /* zero the pointer/count fields: PMIX_NEW does not zero the object, and the + * creators set alloc_id/req_id only when the corresponding key is present, + * so without this the destructor would free() uninitialized garbage. */ + p->targets = NULL; + p->ntargets = 0; + p->pending = 0; + PMIX_PROC_LOAD(&p->requester, NULL, PMIX_RANK_INVALID); + p->alloc_id = NULL; + p->req_id = NULL; + p->have_requester = false; +} +static void campaign_destruct(prte_shrink_campaign_t *p) +{ + free(p->targets); + free(p->alloc_id); + free(p->req_id); +} +PMIX_CLASS_INSTANCE(prte_shrink_campaign_t, pmix_list_item_t, + campaign_construct, campaign_destruct); + +static void grow_campaign_construct(prte_grow_campaign_t *p) +{ + p->targets = NULL; + p->ntargets = 0; + PMIX_PROC_LOAD(&p->requester, NULL, PMIX_RANK_INVALID); + p->alloc_id = NULL; + p->req_id = NULL; + p->have_requester = false; +} +static void grow_campaign_destruct(prte_grow_campaign_t *p) +{ + free(p->targets); + free(p->alloc_id); + free(p->req_id); +} +PMIX_CLASS_INSTANCE(prte_grow_campaign_t, pmix_list_item_t, + grow_campaign_construct, grow_campaign_destruct); bool prte_persistent = true; bool prte_allow_run_as_root = false; bool prte_fwd_environment = false; diff --git a/src/runtime/prte_globals.h b/src/runtime/prte_globals.h index 1523c6b200..0122007cb9 100644 --- a/src/runtime/prte_globals.h +++ b/src/runtime/prte_globals.h @@ -596,6 +596,63 @@ PRTE_EXPORT extern char *prte_data_server_uri; PRTE_EXPORT extern bool prte_dvm_ready; PRTE_EXPORT extern pmix_pointer_array_t *prte_cache; PRTE_EXPORT extern bool prte_persistent; + +/* --- DVM launch fence --- */ + +/* tracks in-progress daemon launch campaigns (grow and shrink combined) */ +PRTE_EXPORT extern int prte_dvm_launch_fence; + +/* app jobs parked at VM_READY → MAP while a campaign is active */ +PRTE_EXPORT extern pmix_pointer_array_t *prte_held_jobs; + +/* app jobs parked at LAUNCH_APPS while a shrink campaign is active */ +PRTE_EXPORT extern pmix_pointer_array_t *prte_prelaunch_held_jobs; + +/* one entry per in-progress shrink campaign */ +typedef struct { + pmix_list_item_t super; + pmix_rank_t *targets; /* daemon ranks being terminated */ + int ntargets; /* initial count */ + int pending; /* targets not yet known to have departed */ + /* requester recorded so the spec's phase-two completion event can be + * directed at the process that issued the PMIX_ALLOC_RELEASE; left unset + * (have_requester == false) for a scheduler-driven release */ + pmix_proc_t requester; + char *alloc_id; /* PMIX_ALLOC_ID of the allocation, or NULL */ + char *req_id; /* requester's PMIX_ALLOC_REQ_ID, or NULL */ + bool have_requester; +} prte_shrink_campaign_t; +PMIX_CLASS_DECLARATION(prte_shrink_campaign_t); + +/* list of active shrink campaigns */ +PRTE_EXPORT extern pmix_list_t prte_shrink_campaigns; + +/* one entry per in-progress grow (daemon-launch) campaign. The campaign + * records the specific daemon ranks being launched so that the launch fence + * is only affected by those ranks: an unrelated daemon loss during a grow + * must not consume the fence, and concurrent campaigns must be tracked + * independently. The fence contribution (ntargets) is held in full until + * the campaign is drained at a safe point — on success in vm_ready, after + * the WIREUP xcast, or on failure when one of the targets dies — so that + * jobs held at the VM_READY → MAP boundary are not admitted before the new + * daemons are wired up. */ +typedef struct { + pmix_list_item_t super; + pmix_rank_t *targets; /* daemon ranks being launched */ + int ntargets; /* count, == this campaign's fence contribution */ + /* requester recorded so the spec's phase-two completion event can be + * directed at the process that drove the grow; left unset + * (have_requester == false) for a scheduler-driven push */ + pmix_proc_t requester; + char *alloc_id; /* PMIX_ALLOC_ID of the allocation, or NULL */ + char *req_id; /* requester's PMIX_ALLOC_REQ_ID, or NULL */ + bool have_requester; +} prte_grow_campaign_t; +PMIX_CLASS_DECLARATION(prte_grow_campaign_t); + +/* list of active grow campaigns */ +PRTE_EXPORT extern pmix_list_t prte_grow_campaigns; + PRTE_EXPORT extern bool prte_allow_run_as_root; PRTE_EXPORT extern bool prte_fwd_environment; PRTE_EXPORT extern bool prte_xml_output; diff --git a/src/runtime/prte_init.c b/src/runtime/prte_init.c index ea9e673285..a9312d92dc 100644 --- a/src/runtime/prte_init.c +++ b/src/runtime/prte_init.c @@ -491,6 +491,14 @@ int prte_init(int *pargc, char ***pargv, prte_proc_type_t flags) prte_cache = PMIX_NEW(pmix_pointer_array_t); pmix_pointer_array_init(prte_cache, 1, INT_MAX, 1); + /* initialize launch-fence held-job arrays */ + prte_held_jobs = PMIX_NEW(pmix_pointer_array_t); + pmix_pointer_array_init(prte_held_jobs, 1, INT_MAX, 1); + prte_prelaunch_held_jobs = PMIX_NEW(pmix_pointer_array_t); + pmix_pointer_array_init(prte_prelaunch_held_jobs, 1, INT_MAX, 1); + PMIX_CONSTRUCT(&prte_shrink_campaigns, pmix_list_t); + PMIX_CONSTRUCT(&prte_grow_campaigns, pmix_list_t); + /* All done */ PMIX_ACQUIRE_THREAD(&prte_init_lock); prte_initialized = true; diff --git a/src/util/error_strings.c b/src/util/error_strings.c index 669dfa44c1..ce9dd0e261 100644 --- a/src/util/error_strings.c +++ b/src/util/error_strings.c @@ -110,6 +110,8 @@ const char *prte_job_state_to_str(prte_job_state_t state) return "PROC CALLED ABORT"; case PRTE_JOB_STATE_HEARTBEAT_FAILED: return "HEARTBEAT FAILED"; + case PRTE_JOB_STATE_WAITING_FOR_DAEMONS: + return "WAITING FOR DAEMON CAMPAIGN"; case PRTE_JOB_STATE_NEVER_LAUNCHED: return "NEVER LAUNCHED"; case PRTE_JOB_STATE_ABORT_ORDERED: