From e1616a63b1671b0f210c21ab59d0bd6c6076b745 Mon Sep 17 00:00:00 2001 From: Paul Menzel Date: Mon, 25 May 2026 10:44:15 +0200 Subject: [PATCH 1/4] mxqsub: add --callback/-c option to run an executable when a job finishes mxqsub accepts --callback=EXECUTABLE (absolute path). The daemon runs the executable after every terminal outcome (finished, failed, killed, unknown), as the submitting user in the job workdir, with MXQ_JOB_ID, MXQ_GROUP_ID, MXQ_JOB_STATUS, and MXQ_JOB_WORKDIR in the environment. The callback is double-forked so it does not block the daemon, and runs under RLIMIT_CPU=60s, RLIMIT_AS=256MiB, RLIMIT_CORE=0. Adds job_callback column to mxq_job; migration in migrate_019. Also fixes mxq_set_job_status_unknown not updating job->job_status in-struct, inconsistent with the other status-update functions. Co-Authored-By: Claude Sonnet 4.6 --- mxq_job.c | 7 ++- mxq_job.h | 1 + mxqd.c | 59 ++++++++++++++++++++++++++ mxqsub.c | 26 +++++++++++- mysql/create_tables.sql | 1 + mysql/migrate_019_add_job_callback.sql | 2 + 6 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 mysql/migrate_019_add_job_callback.sql diff --git a/mxq_job.c b/mxq_job.c index daf4061e..3d02665c 100644 --- a/mxq_job.c +++ b/mxq_job.c @@ -18,7 +18,7 @@ #include "mxq_group.h" #include "mxq_job.h" -#define JOB_FIELDS_CNT 37 +#define JOB_FIELDS_CNT 38 #define JOB_FIELDS \ " job_id, " \ " job_status, " \ @@ -29,6 +29,7 @@ " job_argv, " \ " job_stdout, " \ " job_stderr, " \ + " job_callback, " \ " job_umask, " \ " host_submit, " \ " host_id, " \ @@ -73,6 +74,7 @@ static void bind_result_job_fields(struct mx_mysql_bind *result, struct mxq_job mx_mysql_bind_var(result, idx++, string, &(j->job_argv_str)); mx_mysql_bind_var(result, idx++, string, &(j->job_stdout)); mx_mysql_bind_var(result, idx++, string, &(j->job_stderr)); + mx_mysql_bind_var(result, idx++, string, &(j->job_callback)); mx_mysql_bind_var(result, idx++, uint32, &(j->job_umask)); mx_mysql_bind_var(result, idx++, string, &(j->host_submit)); mx_mysql_bind_var(result, idx++, string, &(j->host_id)); @@ -136,6 +138,7 @@ void mxq_job_free_content(struct mxq_job *j) mx_free_null(j->job_argv_str); mx_free_null(j->job_stdout); mx_free_null(j->job_stderr); + mx_free_null(j->job_callback); if (j->tmp_stderr == j->tmp_stdout) { j->tmp_stdout = NULL; @@ -605,6 +608,8 @@ int mxq_set_job_status_unknown(struct mx_mysql *mysql, struct mxq_job *job) return res; } + job->job_status = MXQ_JOB_STATUS_UNKNOWN; + return res; } diff --git a/mxq_job.h b/mxq_job.h index d73561cb..c008a026 100644 --- a/mxq_job.h +++ b/mxq_job.h @@ -29,6 +29,7 @@ struct mxq_job { char * job_stdout; char * job_stderr; + char * job_callback; char * tmp_stdout; char * tmp_stderr; diff --git a/mxqd.c b/mxqd.c index 3f572a8e..8c776e49 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1928,6 +1928,61 @@ static void release_gpu(struct mxq_server *server, struct mxq_group *group, stru } } +static void run_job_callback(struct mxq_group *group, struct mxq_job *job) +{ + if (!job->job_callback || !*job->job_callback) + return; + + mx_log_info("job=%s(%d):%lu:%lu :: running callback: %s", + group->user_name, group->user_uid, group->group_id, job->job_id, + job->job_callback); + + pid_t pid = fork(); + if (pid < 0) { + mx_log_err("job=%s(%d):%lu:%lu callback fork(): %m", + group->user_name, group->user_uid, group->group_id, job->job_id); + return; + } + + if (pid == 0) { + /* Double-fork: this middle process exits immediately so the daemon's + * waitpid() returns without blocking on the callback's runtime. */ + pid_t cpid = fork(); + if (cpid == 0) { + if (initgroups(group->user_name, group->user_gid) == -1) + _exit(1); + if (setregid(group->user_gid, group->user_gid) == -1) + _exit(1); + if (setreuid(group->user_uid, group->user_uid) == -1) + _exit(1); + if (chdir(job->job_workdir) == -1) + _exit(1); + + struct rlimit rlim; + rlim.rlim_cur = rlim.rlim_max = 60; + setrlimit(RLIMIT_CPU, &rlim); + rlim.rlim_cur = rlim.rlim_max = 256*1024*1024; + setrlimit(RLIMIT_AS, &rlim); + rlim.rlim_cur = rlim.rlim_max = 0; + setrlimit(RLIMIT_CORE, &rlim); + + mx_setenvf_forever("MXQ_JOB_ID", "%lu", job->job_id); + mx_setenvf_forever("MXQ_GROUP_ID", "%lu", job->group_id); + mx_setenv_forever("MXQ_JOB_STATUS", mxq_job_status_to_name(job->job_status)); + mx_setenv_forever("MXQ_JOB_WORKDIR", job->job_workdir); + + execl(job->job_callback, job->job_callback, NULL); + _exit(1); + } + _exit(cpid < 0 ? 1 : 0); + } + + int status; + if (waitpid(pid, &status, 0) == -1) + mx_log_err("job=%s(%d):%lu:%lu callback waitpid(): %m", + group->user_name, group->user_uid, group->group_id, job->job_id); +} + static int job_has_finished(struct mxq_server *server, struct mxq_group *group, struct mxq_job_list *jlist) { int cnt; @@ -1942,6 +1997,8 @@ static int job_has_finished(struct mxq_server *server, struct mxq_group *group, rename_outfiles(server, group, job); + run_job_callback(group, job); + cnt = jlist->group->slots_per_job; cpuset_clear_running(&server->cpu_set_running, &job->host_cpu_set); release_gpu(server, group, job); @@ -1966,6 +2023,8 @@ static int job_is_lost(struct mxq_server *server,struct mxq_group *group, struct rename_outfiles(server, group, job); + run_job_callback(group, job); + cnt = jlist->group->slots_per_job; cpuset_clear_running(&server->cpu_set_running, &job->host_cpu_set); release_gpu(server, group, job); diff --git a/mxqsub.c b/mxqsub.c index 72893be2..c0194d81 100644 --- a/mxqsub.c +++ b/mxqsub.c @@ -60,6 +60,10 @@ static void print_usage(void) " -e, --stderr=FILE set file to capture stderr (default: )\n" " -u, --umask=MASK set mode to use as umask (default: current umask)\n" " -p, --priority=PRIORITY set priority (default: 127)\n" + " -c, --callback=EXECUTABLE run EXECUTABLE when the job finishes (any outcome)\n" + " runs as the submitting user in the job workdir;\n" + " MXQ_JOB_ID, MXQ_GROUP_ID, MXQ_JOB_STATUS, and\n" + " MXQ_JOB_WORKDIR are set in the environment\n" "\n" "Job resource information:\n" " Scheduling is done based on the resources a job needs and\n" @@ -517,6 +521,7 @@ static int add_job(struct mx_mysql *mysql, struct mxq_job *j) " job_stdout = ?," " job_stderr = ?," + " job_callback = ?," " job_umask = ?," @@ -535,8 +540,9 @@ static int add_job(struct mx_mysql *mysql, struct mxq_job *j) mx_mysql_statement_param_bind(stmt, 4, string, &(j->job_argv_str)); mx_mysql_statement_param_bind(stmt, 5, string, &(j->job_stdout)); mx_mysql_statement_param_bind(stmt, 6, string, &(j->job_stderr)); - mx_mysql_statement_param_bind(stmt, 7, uint32, &(j->job_umask)); - mx_mysql_statement_param_bind(stmt, 8, string, &(j->host_submit)); + mx_mysql_statement_param_bind(stmt, 7, string, &(j->job_callback)); + mx_mysql_statement_param_bind(stmt, 8, uint32, &(j->job_umask)); + mx_mysql_statement_param_bind(stmt, 9, string, &(j->host_submit)); res = mx_mysql_statement_execute(stmt, &num_rows); if (res < 0) { @@ -705,6 +711,7 @@ int main(int argc, char *argv[]) char arg_debug; u_int32_t arg_tmpdir; u_int16_t arg_gpu; + char *arg_callback; _mx_cleanup_free_ char *current_workdir = NULL; _mx_cleanup_free_ char *arg_stdout_absolute = NULL; @@ -769,6 +776,7 @@ int main(int argc, char *argv[]) MX_OPTION_REQUIRED_ARG("prerequisites", 10), MX_OPTION_REQUIRED_ARG("tags", 11), MX_OPTION_NO_ARG("gpu", 12), + MX_OPTION_REQUIRED_ARG("callback", 'c'), MX_OPTION_END }; @@ -798,6 +806,7 @@ int main(int argc, char *argv[]) arg_prerequisites = ""; arg_tags = NULL; arg_gpu = 0; + arg_callback = ""; arg_mysql_default_group = getenv("MXQ_MYSQL_DEFAULT_GROUP"); if (!arg_mysql_default_group) @@ -1021,6 +1030,18 @@ int main(int argc, char *argv[]) case 12: arg_gpu = 1; break; + + case 'c': + if (!(*optctl.optarg)) { + mx_log_crit("--callback '%s': String is empty.", optctl.optarg); + exit(EX_CONFIG); + } + if (optctl.optarg[0] != '/') { + mx_log_crit("--callback '%s': must be an absolute path.", optctl.optarg); + exit(EX_CONFIG); + } + arg_callback = optctl.optarg; + break; } } @@ -1133,6 +1154,7 @@ int main(int argc, char *argv[]) job.job_argc = argc; job.job_argv = argv; job.job_argv_str = arg_args; + job.job_callback = arg_callback; /******************************************************************/ diff --git a/mysql/create_tables.sql b/mysql/create_tables.sql index 41585f76..d73d34a0 100644 --- a/mysql/create_tables.sql +++ b/mysql/create_tables.sql @@ -83,6 +83,7 @@ CREATE TABLE IF NOT EXISTS mxq_job ( job_stdout VARCHAR(4096) NOT NULL DEFAULT '/dev/null', job_stderr VARCHAR(4096) NOT NULL DEFAULT '/dev/null', + job_callback VARCHAR(4096) NOT NULL DEFAULT '', job_umask INT4 NOT NULL, diff --git a/mysql/migrate_019_add_job_callback.sql b/mysql/migrate_019_add_job_callback.sql new file mode 100644 index 00000000..c411aba7 --- /dev/null +++ b/mysql/migrate_019_add_job_callback.sql @@ -0,0 +1,2 @@ +ALTER TABLE mxq_job + ADD COLUMN job_callback VARCHAR(4096) NOT NULL DEFAULT '' AFTER job_stderr; From 7565dba8cdb019c75f2f11dd58cbb7857ef216b1 Mon Sep 17 00:00:00 2001 From: Paul Menzel Date: Mon, 25 May 2026 10:46:08 +0200 Subject: [PATCH 2/4] mxqsub.1: document --callback option Co-Authored-By: Claude Sonnet 4.6 --- manpages/mxqsub.1 | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/manpages/mxqsub.1 b/manpages/mxqsub.1 index 0f34d749..039c1516 100644 --- a/manpages/mxqsub.1 +++ b/manpages/mxqsub.1 @@ -34,6 +34,43 @@ is used to queue a job to be executed on a cluster node\&. [arguments] .RS 4 specify the estimated runtime\&. .RE +.PP +\fB\-c \fR\fB\fI\fR\fR, \fB\-\-callback=\fR\fB\fI\fR\fR +.RS 4 +run \fI\fR after the job reaches any terminal state (finished, failed, killed, or unknown)\&. +The executable must be given as an absolute path\&. +It is run directly via \fBexecl\fR(3) (not through a shell) as the submitting user in the job working directory\&. +The following environment variables are set: +.sp +.RS 4 +\fBMXQ_JOB_ID\fR \- the job id +.br +\fBMXQ_GROUP_ID\fR \- the job group id +.br +\fBMXQ_JOB_STATUS\fR \- one of \fIfinished\fR, \fIfailed\fR, \fIkilled\fR, or \fIunknown\fR +.br +\fBMXQ_JOB_WORKDIR\fR \- the job working directory +.RE +.RE +.SH "EXAMPLES" +.PP +Send an email when a job finishes\&. Save the following as e\&.g\&. +\fI~/bin/mxq\-notify\fR and make it executable: +.sp +.RS 4 +.nf +#!/bin/sh +mail \-s "MXQ job $MXQ_JOB_ID $MXQ_JOB_STATUS" "$USER" +.fi +.RE +.sp +Then submit a job with: +.sp +.RS 4 +.nf +mxqsub \-\-callback=$HOME/bin/mxq\-notify myjob +.fi +.RE .SH "ENVIRONMENT" .PP \fBMXQ_MYSQL_DEFAULTFILE\fR From 5958cae6d2d2b10c7d3fbb54fe92b002d9b35397 Mon Sep 17 00:00:00 2001 From: Paul Menzel Date: Sun, 19 Jul 2026 10:56:50 +0200 Subject: [PATCH 3/4] mxq-powerd: add external node power manager (suspend idle / resume on demand) Add mxq-powerd, a self-contained power manager for the MXQ cluster that powers idle compute nodes down to save energy and powers them back up when queued work needs them, oldest/least-efficient hardware off first. It runs on the management host on a fixed interval (systemd timer or cron). The manager is entirely external to MXQ: it only READS the MySQL database (through the stock mysql client) and actuates nodes with stock CLIs (ipmitool, wakeonlan, ssh). It issues no DB writes and requires no changes to mxqd, the schema, or mxqadmin. Implements docs/power-management-implementation.md: - static inventory /etc/mxq/nodes.conf (capability + transport + priority) - global policy /etc/mxq/powerd.conf (dry_run defaults to true) - IPMI credentials in /etc/mxq/powerd-secrets (0600; password via ipmitool -E, never on the command line) - per-node state machine in /var/lib/mxq/powerd-state.json, advanced each tick from the DB snapshot (pending demand, live capacity/liveness, inferred idle duration) - resume: wake lowest-gen nodes on core deficit / capability gap / warm pool shortfall, rate-limited - suspend: drain highest-gen idle nodes (ssh mxqdctl-hostconfig stop), wait for EXITED to close the claim race, then poweroff, with BMC soft/hard-off escalation; honours the surplus test, warm pool floor and rate limits - bounded transitions: stuck nodes land in FAILED, are excluded, and are left for an operator (mxq-powerd --clear HOST) Ships example configs, systemd service + timer, an mxq-powerd.8 man page, a README, and an opt-in `make install-powerd` target (not wired into the default install). DB access reuses MXQ's read-only defaults file (/etc/mxq/mysql_ro.cnf, group mxqclient) via my_print_defaults rather than adding a Python MySQL driver dependency; selectable in powerd.conf. Installs the program to /usr/libexec/mxq to match the existing helper convention. Co-Authored-By: Claude Opus 4.8 --- Makefile | 17 + docs/power-management-design.md | 225 +++++ docs/power-management-implementation.md | 308 +++++++ manpages/mxq-powerd.8 | 115 +++ powerd/.gitignore | 2 + powerd/README.md | 62 ++ powerd/mxq-powerd | 1085 +++++++++++++++++++++++ powerd/mxq-powerd.service | 11 + powerd/mxq-powerd.timer | 14 + powerd/nodes.conf.example | 25 + powerd/powerd-secrets.example | 16 + powerd/powerd.conf.example | 52 ++ 12 files changed, 1932 insertions(+) create mode 100644 docs/power-management-design.md create mode 100644 docs/power-management-implementation.md create mode 100644 manpages/mxq-powerd.8 create mode 100644 powerd/.gitignore create mode 100644 powerd/README.md create mode 100755 powerd/mxq-powerd create mode 100644 powerd/mxq-powerd.service create mode 100644 powerd/mxq-powerd.timer create mode 100644 powerd/nodes.conf.example create mode 100644 powerd/powerd-secrets.example create mode 100644 powerd/powerd.conf.example diff --git a/Makefile b/Makefile index ccd85f87..253fcb00 100644 --- a/Makefile +++ b/Makefile @@ -781,3 +781,20 @@ clean: CLEAN += test_parser.o test_parser: test_parser.o parser.tab.o keywordset.o test_parser.o: parser.tab.h keywordset.h clean: CLEAN += parser.tab.o + +######################################################################## +### mxq-powerd -- node power manager (opt-in; not part of `make install`) + +MAN8DIR := ${MANDIR}/man8 + +.PHONY: install-powerd +install-powerd: powerd/mxq-powerd manpages/mxq-powerd.8 + $(call quiet-installdir,0755,${DESTDIR}${LIBEXECDIR}/mxq) + $(call quiet-installdir,0755,${DESTDIR}${SYSCONFDIR}/mxq) + $(call quiet-installdir,0755,${DESTDIR}${MAN8DIR}) + $(call quiet-install,0755,powerd/mxq-powerd,${DESTDIR}${LIBEXECDIR}/mxq/mxq-powerd) + $(call quiet-install,0644,manpages/mxq-powerd.8,${DESTDIR}${MAN8DIR}/mxq-powerd.8) + $(call quiet-install,0644,powerd/powerd.conf.example,${DESTDIR}${SYSCONFDIR}/mxq/powerd.conf.example) + $(call quiet-install,0644,powerd/nodes.conf.example,${DESTDIR}${SYSCONFDIR}/mxq/nodes.conf.example) + $(call quiet-install,0600,powerd/powerd-secrets.example,${DESTDIR}${SYSCONFDIR}/mxq/powerd-secrets.example) + @echo " NOTE copy powerd/mxq-powerd.{service,timer} to /etc/systemd/system/ to enable the timer" diff --git a/docs/power-management-design.md b/docs/power-management-design.md new file mode 100644 index 00000000..a972cf77 --- /dev/null +++ b/docs/power-management-design.md @@ -0,0 +1,225 @@ +# Powering off idle MXQ nodes to save energy — design notes + +Status: **design exploration** (no code written yet). Feasibility study requested by +Paul Menzel, 2026-07-18. + +## 1. The question + +Can MXQ power down compute nodes when the cluster is idle and power them back on +when work arrives, to save energy? If so, what has to be built? + +Short answer: **yes, and mostly as a new, self-contained central component** — MXQ +does not need to be rewritten. But the piece that Slurm gets "for free" (a central +controller that already knows and commands every node) does **not exist in MXQ** and +has to be created. Paul's instinct in the request is correct: an independent +timer/cron that (a) picks idle nodes to switch off, oldest generations first, and +(b) a second loop that watches the queue and switches nodes back on. + +## 2. How MXQ works today (the relevant parts) + +MXQ is a **pull-based** scheduler. There is a central MySQL database and autonomous +per-node daemons; nothing ever *commands* a node. + +- **Three tables** (`mysql/create_tables.sql`): + - `mxq_group` — one row per array of identical jobs. Carries the per-job resource + requirements (`job_threads`, `job_memory`, `job_time`, `job_gpu`, `job_tmpdir_size`) + and trigger-maintained counters, notably `group_jobs_inq` (pending) and + `group_jobs_running`. + - `mxq_job` — individual jobs; placement columns `daemon_id`, `host_hostname`, + `job_status`, and timestamps `date_start` / `date_end`. + - `mxq_daemon` — **the per-node table**. One row per *daemon instance* (a fresh row + is INSERTed every time a daemon starts). Holds `hostname`, capacity + (`daemon_slots`, `daemon_memory`, `daemon_gpus_max`), live load + (`daemon_slots_running`, `daemon_memory_used`, …), `status`, `tags`, + `prerequisites`, and timestamps `mtime` / `daemon_start` / `daemon_stop`. + +- **The daemon (`mxqd`)** — `mxqd.c`, `mxq_daemon.c`: + - On start: `mxq_daemon_register()` INSERTs its row and gets an auto-increment + `daemon_id` (`mxq_daemon.c:48`). Stable *logical* node identity is the pair + `(hostname, daemon_name)`. + - Main loop (`mxqd.c:2715`): `sigtimedwait()` with a **20 s** poll interval. Each + pass it reads the global queue via `mxq_load_running_groups()` + (`SELECT … FROM mxq_group WHERE group_jobs_inq>0 OR group_jobs_running>0`, + `mxq_group.c:292`), then race-safely claims INQ jobs + (`UPDATE mxq_job … WHERE job_id=? AND job_status=INQ`, `mxq_job.c:311`). + - **Idle signal**: `update_status()` sets `mxq_daemon.status = IDLE (0)` exactly when + `slots_running == 0` (`mxqd.c:2595`). Status enum in `mxq_daemon.h:9`: + IDLE=0, RUNNING=10, WAITING=20, FULL=30, BACKFILL=40, CPUOPTIMAL=50, + TERMINATING=200, EXITED=250, CRASHED=255. + - **Liveness / heartbeat**: `mtime` is refreshed on every stats/status write, and at + least every 5 min even when idle (rate-limited in `mxq_daemon.c:235`). There is + **no fast heartbeat and no central watchdog**. A crashed node is only flagged + CRASHED when a *replacement* daemon starts (`recover_from_previous_crash`, + `mxqd.c:2487`). + - **Clean shutdown**: SIGTERM exits the main loop and enters a **drain loop** + (`mxqd.c:2762`) that starts no new jobs and waits for running jobs to finish, then + `mxq_daemon_shutdown()` sets `status=EXITED, daemon_stop=NOW()` (`mxq_daemon.c:151`). + SIGINT additionally kills running jobs; SIGQUIT/SIGUSR1 leave jobs running. + +- **Host bring-up today** — `mxqdctl-hostconfig.sh`: reads `/etc/hostconfig`, starts + one `mxqd` per matching line, and controls local daemons via signals. It **assumes + the host is already powered on**. There is no PXE/IPMI/WoL hook anywhere. + +- **Admin surface** — `mxqadmin.c` only closes/reopens groups. There is **no** + drain-node / remove-daemon / power command. No cron, systemd unit, or scheduler + callback exists in the repo (the recent `--callback` is a *per-job*, runs-as-user, + on-node hook — not suitable for privileged node power control). + +### What this gives us for free + +Everything a power decision needs is already observable in the DB *without touching +the nodes*: + +| Question | Where the answer is | +|---|---| +| Is there pending work, and how much? | `SUM(group_jobs_inq * job_threads)`, `job_memory`, `job_gpu`, `tags`/`prerequisites` over `mxq_group WHERE group_jobs_inq>0` | +| Is node H idle right now? | `mxq_daemon.status=0` and `daemon_slots_running=0`; cross-check `mxq_job` has no rows for H in status (100,150,200) | +| Is node H alive? | freshness of `mxq_daemon.mtime` (stale ⇒ gone) | +| How long has H been idle? | **not stored** — infer from `NOW() - MAX(mxq_job.date_end)` for H, else from `daemon_start` | +| What can H do (once it's off)? | last `mxq_daemon` row's capacity columns, but see §5 — better kept in a static inventory | + +## 3. Why we can't just copy Slurm + +Slurm's `slurmctld` is a central controller that already tracks every node and issues +commands to them; power-save is just two site scripts it invokes — +`SuspendProgram` (node idle > `SuspendTime` ⇒ power down) and `ResumeProgram` +(queued work needs a node ⇒ power up), rate-limited by `SuspendRate`/`ResumeRate`, +with `SuspendExcNodes` protecting infrastructure. Power-*on* is done out-of-band via +**IPMI `chassis power on`** or **Wake-on-LAN**; power-*off* by an ordered shutdown. + +MXQ has no `slurmctld`. So the equivalent of "the controller that decides and acts" +is exactly the new component we must add. The good news: because nodes *pull*, the +resume path is trivial — we only have to get the box powered on; `mxqd` then boots, +registers, and starts pulling with **zero scheduler involvement**. + +## 4. Proposed design — a central "power manager" + +A single new component on the management host (where `mxqdump`/`mxqadmin` already +run), driven by a **systemd timer or cron** every 1–2 min. It never runs on the +compute nodes. Two independent decision loops (they can be one script): + +``` + ┌─────────────────── management host ───────────────────┐ + │ mxq-powerd (cron/systemd timer, every ~1–2 min) │ + │ │ + reads │ ┌── suspend loop ──┐ ┌── resume loop ──┐ │ + ◄────────┤ │ idle > T & safe? │ │ pending demand │ │ + MySQL │ │ → drain+off │ │ > free capacity?│ │ + (mxq_*) │ └────────┬─────────┘ └────────┬────────┘ │ + │ │ ssh: mxqdctl stop; poweroff│ IPMI power on / │ + └───────────┼────────────────────────────┼─WoL────────────┘ + ▼ ▼ + compute node (in-band, compute node (out-of-band, + graceful, deregisters) node is off → BMC/WoL only) +``` + +### 4a. Suspend loop (power **off**) + +1. **Find safe idle candidates.** Per hostname (aggregating *all* live daemon rows, + since a host may run several daemons): every daemon `status=IDLE(0)` and + `daemon_slots_running=0`, `mtime` fresh (node alive), and no `mxq_job` rows for the + host in status (100,150,200). +2. **Idle-long-enough gate** (`SuspendTime`, e.g. 30 min): `NOW() - MAX(date_end)` + for the host exceeds the threshold (fallback `daemon_start` if it never ran a job). + This is the one signal MXQ doesn't store directly — see §6 for an optional column. +3. **Policy gates:** exclude-list (login/storage/infra nodes — the `SuspendExcNodes` + analog); keep a **minimum warm pool** of idle nodes on so short bursts don't wait + for a boot; rate-limit (`SuspendRate`) to avoid a sudden power drop. +4. **Ordering: oldest generations first.** Rank candidates by a per-node + power-priority from the inventory (§5) so the least efficient hardware powers off + first. +5. **Graceful power-off (in-band):** for each chosen node + `ssh HOST 'mxqdctl-hostconfig stop'` (SIGTERM ⇒ stops claiming, drains, sets + `status=EXITED`), **wait until its `mxq_daemon` row shows EXITED**, then + `ssh HOST 'systemctl poweroff'`. Waiting for EXITED closes the claim race: the + daemon only marks EXITED *after* draining, so once EXITED no job can be running or + half-claimed on that node. In-band shutdown (not IPMI power-off) lets the daemon + deregister cleanly — no stale RUNNING rows to reconcile. + +### 4b. Resume loop (power **on**) + +1. **Pending demand** from `mxq_group WHERE group_jobs_inq>0`: total pending cores + `SUM(group_jobs_inq*job_threads)`, plus memory / GPU / time / tag & prerequisite + constraints. +2. **Free capacity** of nodes already on: + `SUM(daemon_slots - daemon_slots_running)` over live daemons (respecting the same + tag/GPU/memory constraints). +3. **Deficit ⇒ power on.** Also power on when a *specific* pending job can only run on + a class of node that is currently all-off (e.g. GPU or high-memory / a required + tag), even if raw core capacity looks sufficient. +4. **Ordering: newest / most efficient first** (inverse of suspend), and pick nodes + whose inventoried capabilities actually satisfy the waiting jobs. +5. **Power on out-of-band:** IPMI `chassis power on` (preferred, reliable) or a WoL + magic packet. Rate-limit (`ResumeRate`) to avoid inrush current. +6. **Boot ⇒ self-heal:** the node's init/systemd runs `mxqdctl-hostconfig.sh`, `mxqd` + registers a fresh `mxq_daemon` row and starts pulling. Nothing else to do. +7. **Timeout:** if no fresh `mxq_daemon` row appears within `ResumeTimeout` + (e.g. 10 min), flag the node as failed, exclude it, and try the next candidate. + +### 4c. Anti-thrash + +Boot takes minutes, so hysteresis matters: generous `SuspendTime`, a warm pool, +independent rate limits both directions, and never suspend below the warm-pool floor. + +## 5. The one new piece of persistent state: a node inventory + +The live DB describes daemons *that are currently up*. To decide which **powered-off** +node to wake — and to reach its BMC — we need static per-node facts the DB can't give +once the node is off: + +`hostname, power_priority(generation), slots, memory, gpus, tags, bmc_address, mac` + +Recommended: a **flat site config file** (e.g. `/etc/mxq/nodes.conf`), matching the +existing `/etc/hostconfig` style, rather than a new DB table. It is simplest, keeps +site/hardware facts out of the runtime DB, and MXQ is already comfortable with flat +config + shell. A DB table (`mxq_node`) is a possible later step if the inventory +needs to be queried by other tools. + +## 6. Optional, later changes inside MXQ (not needed for v1) + +v1 can be **100% external** — no changes to `mxqd`, the schema, or `mxqadmin`. Cleaner +follow-ups if this graduates from experiment to production: + +- **A real DRAIN state.** Today "stop taking work but stay up" is only achievable by + SIGTERM (which also exits). A `daemon_flags` DRAINING bit that `mxqd` checks before + claiming (in the `mxq_load_running_groups` path) would let the manager mark a node + "don't take work", pause, and *cancel* cheaply if demand spikes — removing the + SIGTERM-then-wait dance and the residual race window entirely. +- **An `idle_since` timestamp** on `mxq_daemon`, set when `slots_running` hits 0, so + idle duration is read directly instead of inferred from `MAX(mxq_job.date_end)`. +- **`mxqadmin` verbs** `--drain HOST` / `--poweroff HOST` / `--poweron HOST`, so + operators share one tested code path with the automated manager. +- **A central watchdog** that sweeps stale `mtime` rows to CRASHED (today only a + restarting daemon does this) — useful independent of power management, and needed so + a node that fails to power on/off is noticed. + +## 7. Suggested phasing + +1. **v1 — external, read-only-ish:** the `mxq-powerd` script (suspend + resume loops) + + `nodes.conf` inventory + IPMI/WoL + `ssh poweroff`, on a timer. Start in + **dry-run/log-only** mode; verify decisions against a real idle/busy cluster before + arming actuation. No MXQ code touched. +2. **v2 — coordinate with the daemon:** add the DRAINING flag + `idle_since` + + `mxqadmin` verbs to remove the race window and simplify the manager. +3. **v3 — policy:** smarter warm-pool sizing / predictive wake based on submission + patterns (cf. the queueing-penalty research on ill-timed transitions). + +## 8. Open questions for Paul + +- Power-*on* transport available in the fleet: **IPMI/BMC**, **WoL**, both, or + something site-specific (redfish, PDU)? +- Is `ssh HOST poweroff` acceptable for power-*off*, or must that also go through the + BMC? +- Where should node generation / power-priority and BMC addresses live — a new + `/etc/mxq/nodes.conf`, or is that data already in an existing inventory/CMDB we + should read instead? +- Which nodes must **never** be suspended (login, storage, license servers)? +- Target `SuspendTime`, warm-pool size, and rate limits. + +## References + +- Slurm Power Saving Guide — https://slurm.schedmd.com/power_save.html +- OleHolmNielsen/Slurm_tools power_save (example IPMI scripts) — https://github.com/OleHolmNielsen/Slurm_tools/tree/master/power_save +- GSI Slurm power-management notes (WoL/IPMI resume) — https://web-docs.gsi.de/~vpenso/notes/posts/hpc/cluster/slurm/power-management.html +- Thinkbox Deadline "Machine Startup" (WoL/IPMI wake on submit) — https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/power-management.html +- SPARS: RL simulator for HPC power management (transition-timing penalties) — https://arxiv.org/pdf/2512.13268 diff --git a/docs/power-management-implementation.md b/docs/power-management-implementation.md new file mode 100644 index 00000000..6b15cb31 --- /dev/null +++ b/docs/power-management-implementation.md @@ -0,0 +1,308 @@ +# MXQ node power management — implementation design (BMC/IPMI + Wake-on-LAN) + +Status: **implementation design**, ready to build v1. Companion to +`power-management-design.md` (the feasibility study / architecture rationale). Read +that first for *why*; this doc is *how*. + +## 0. Assumptions (chosen where §8 of the study was unanswered — please correct) + +1. **Both transports exist**: nodes are reachable by **IPMI/BMC** (`ipmitool -I lanplus`) + and/or **Wake-on-LAN**. Power-*on* method is selectable **per node** in the + inventory; WoL is the cheap default, IPMI the reliable one and the only option when + the node's NIC/segment can't carry a magic packet. +2. **Power-*off* is graceful and in-band** (`ssh HOST poweroff` after the daemon has + drained), with a **BMC soft-off** (`ipmitool … chassis power soft`) fallback and a + **BMC hard-off** last resort only if soft-off also times out. +3. **New static inventory** `/etc/mxq/nodes.conf` (no existing CMDB integration in v1). +4. **Language: Python 3** for the manager (readable SQL + state machine + subprocess + actuation); the actuators themselves are stock CLIs (`ipmitool`, `wakeonlan`/ + `ether-wake`, `ssh`). Bash is viable but the demand/capacity/ordering/rate-limit + logic is beyond comfortable bash. Recording this as a decision, not a hard commit. +5. **No changes to `mxqd`, the schema, or `mxqadmin` in v1** — the manager is entirely + external and read-only against the DB except it issues no writes at all. + +## 1. Components + +``` +/usr/lib/mxq/mxq-powerd the manager (Python 3, one tick per invocation) +/etc/mxq/powerd.conf global policy (thresholds, rates, DB group, exclude) +/etc/mxq/nodes.conf per-node inventory (transport, capabilities, priority) +/etc/mxq/powerd-secrets optional: IPMI credentials, mode 0600, root-only +/var/lib/mxq/powerd-state.json manager state (in-flight transitions + timestamps) +/var/log/mxq/powerd.log actions + decisions +systemd: mxq-powerd.service + .timer (or a root cron entry) — runs on the mgmt host +``` + +The manager runs **on the management host** (where `mxqdump`/`mxqadmin` already run), +as **root** (needs to `ssh` to nodes and drive BMCs), on a **fixed interval** +(default every 60 s via a systemd timer — the analog of Slurm's `power_save_interval`). +One invocation = one evaluation tick; it is safe to run back-to-back and safe to miss +a tick. + +## 2. Inventory file — `/etc/mxq/nodes.conf` + +Flat, `#`-comment, whitespace-separated, matching the `/etc/hostconfig` house style. +One line per **host** (not per daemon). Columns: + +``` +# hostname gen slots mem_mb gpus tags on_method bmc_or_mac +node001 g1 64 257000 0 xeon,highmem ipmi 10.1.0.11 +node002 g1 64 257000 0 xeon,highmem ipmi 10.1.0.12 +node101 g3 256 980000 4 epyc,gpu,a100 wol e4:3d:1a:00:11:22 +login01 - - - - infra none - +``` + +- `gen` — generation / **power-priority** token. Suspend picks **highest `gen` numbers + (oldest, least efficient) first**; resume picks **lowest (newest, most efficient) + first**. This is the MXQ analog of Slurm node `Weight`. `-` ⇒ never auto-managed. +- `slots`/`mem_mb`/`gpus`/`tags` — capability of a **powered-off** node (the live DB + can't tell us this once it's dark; §5 of the study). Used by resume to match waiting + jobs' requirements (`job_threads`, `job_memory`, `job_gpu`, group `tags`). +- `on_method` ∈ `ipmi | wol | none`. `none` ⇒ excluded from all automation + (login/storage/infra). +- `bmc_or_mac` — BMC IP/hostname for `ipmi`, MAC for `wol`. + +IPMI credentials are **not** in this file. They live in `/etc/mxq/powerd-secrets` +(`0600`, root) — a global `user`/`password`, overridable per host. Keeping secrets out +of the world-readable inventory is deliberate. + +## 3. Global policy — `/etc/mxq/powerd.conf` + +```ini +[db] +# reuse MXQ's MySQL config-group convention; new read-only account recommended +defaults_group = mxqpowerd # section in ~/.my.cnf / /etc/mysql/… with a SELECT-only user + +[policy] +suspend_time = 1800 # sec idle before a node may power down (Slurm SuspendTime) +warm_pool = 4 # min idle managed nodes to keep ON (Slurm SuspendExcNodes :count) +min_uptime = 900 # sec a node must be up before it's suspend-eligible (anti-thrash) +suspend_rate = 4 # max power-downs per tick (Slurm SuspendRate) +resume_rate = 8 # max power-ups per tick (Slurm ResumeRate) +suspend_timeout = 300 # sec for a shutdown to complete (Slurm SuspendTimeout) +resume_timeout = 600 # sec for a boot+register (Slurm ResumeTimeout) +mtime_stale = 180 # sec: mxq_daemon.mtime older than this ⇒ node not alive +exclude = login01,storage0,storage1 # never manage (also: gen '-' / on_method none) + +[mode] +dry_run = true # v1 default: log decisions, take NO action. Flip to false to arm. +``` + +## 4. State the manager reads from the DB (no writes) + +All read-only. Aggregated **per hostname** because a host can run several `mxqd` +daemons and can carry stale EXITED/CRASHED rows. + +**A. Pending demand** (do we need more nodes, and of what shape): +```sql +SELECT SUM(group_jobs_inq) AS pend_jobs, + SUM(group_jobs_inq * job_threads) AS pend_cores, + MAX(job_memory) AS max_job_mem_kib, + MAX(job_gpu) AS max_job_gpu +FROM mxq_group +WHERE group_jobs_inq > 0 + AND (group_flags & 1) = 0; -- ignore CLOSED groups +``` +Per-group rows (with `job_threads,job_memory,job_gpu,job_time,tags`) are also pulled so +resume can match specific constraints, not just totals. + +**B. Live capacity & liveness** (what's on, and free): +```sql +SELECT hostname, + MAX(status) AS status, + SUM(daemon_slots) AS slots, + SUM(daemon_slots_running) AS slots_running, + SUM(daemon_slots - daemon_slots_running) AS slots_free, + MAX(UNIX_TIMESTAMP(mtime)) AS mtime +FROM mxq_daemon +WHERE status NOT IN (250, 255) -- exclude EXITED / CRASHED rows +GROUP BY hostname; +``` +A host is **alive** iff `NOW() - mtime <= mtime_stale`. Free capacity for resume = +`SUM(slots_free)` over alive hosts (filtered by the constraints from A). + +**C. Idle-duration** (no `idle_since` column exists — infer it): +```sql +SELECT host_hostname AS hostname, MAX(date_end) AS last_end +FROM mxq_job +WHERE host_hostname <> '' +GROUP BY host_hostname; +``` +Idle duration ≈ `NOW() - last_end`; if a host has no finished job since boot, fall back +to `daemon_start` from table B. A host is **safely idle** iff `status = 0 (IDLE)`, +`slots_running = 0`, and **no active jobs**: +```sql +SELECT COUNT(*) FROM mxq_job +WHERE host_hostname = ? AND job_status IN (100,150,200); -- ASSIGNED/LOADED/RUNNING +``` + +## 5. Per-node state machine + +The manager holds each managed host in one state in `powerd-state.json`, updated each +tick by combining that file with DB tables B/C. + +``` + demand needs it / not enough warm + POWERED_OFF ───────────────► POWERING_UP ──(fresh daemon row)──► UP_IDLE + ▲ │ │ + │ (no row by resume_timeout) │ job claimed + (host dark by ▼ ▼ + suspend_timeout) FAILED ◄───(off by …timeout)──── UP_BUSY + │ ▲ │ + POWERING_DOWN ◄──(EXITED seen)── DRAINING ◄──(idle>suspend_time & │ + │ poweroff issued │ surplus & rate ok) ◄──┘ + └──────────────────────────► (returns to POWERED_OFF on confirmed dark) +``` + +- **POWERED_OFF → POWERING_UP** (resume, §6): send wake, stamp `t_resume`. +- **POWERING_UP → UP_IDLE**: a fresh `mxq_daemon` row for the host appears (alive). + If not by `t_resume + resume_timeout` → **FAILED** (log, run resume-fail action, + exclude until operator clears — the `ResumeFailProgram` analog). +- **UP_IDLE → DRAINING** (suspend, §7): host safely idle > `suspend_time`, up > + `min_uptime`, is surplus (see §7), rate budget available, not in warm pool → send + `ssh HOST 'mxqdctl-hostconfig stop'`, stamp `t_drain`. +- **DRAINING → POWERING_DOWN**: host's daemon rows all show `status=EXITED(250)` → issue + power-off. **Waiting for EXITED is the race-closer**: `mxqd` only marks EXITED after + its drain loop finds nothing running, so once EXITED no job is running or half-claimed + on that host (`mxqd.c:2762`, `mxq_daemon.c:151`). If EXITED not seen by + `t_drain + suspend_timeout/2`, escalate to BMC soft-off. +- **POWERING_DOWN → POWERED_OFF**: confirmed dark (no fresh row / BMC `power status` + = off). If still not dark by `t_drain + suspend_timeout` → BMC hard-off, else FAILED. +- **UP_BUSY**: any active job → never a suspend candidate; a DRAINING host that somehow + claimed work is impossible after EXITED, but if a *new* daemon registers busy during + POWERING_UP we simply land in UP_BUSY. + +Cancellation: if demand spikes while a host is DRAINING (before EXITED), v1 lets it +finish exiting and immediately re-resumes it — simple and rare. (v2's DRAINING flag in +`mxqd`, study §6, makes this a cheap in-place cancel.) + +## 6. Resume (power ON) — the actuators + +Selection each tick: +1. `deficit_cores = pend_cores - free_cores_on_alive_hosts` (and analogous checks for a + pending job that needs GPU / memory / a tag no alive host provides). +2. Also ensure `warm_pool` idle nodes exist; if fewer, that's demand too. +3. If deficit > 0: from `POWERED_OFF` managed hosts **whose inventoried capability + satisfies the unmet requirement**, pick **lowest `gen` first**, up to `resume_rate`. + +Actuation per node: +```bash +# WoL +wakeonlan e4:3d:1a:00:11:22 # or: ether-wake -i +# IPMI +ipmitool -I lanplus -H 10.1.0.11 -U "$U" -P "$P" chassis power on +``` +Then move to POWERING_UP. Verification is **via the DB** — the node boots, its +init/systemd runs `mxqdctl-hostconfig.sh`, `mxqd` registers a fresh `mxq_daemon` row and +starts pulling. No scheduler action; the manager just watches for the row. + +Notes: +- **WoL prerequisites**: BIOS "Wake on LAN" enabled; the magic packet must reach the + node's L2 segment (a sender on each VLAN, or directed broadcast). If a node's segment + can't carry it, set `on_method = ipmi`. +- **IPMI**: BMC on a management network reachable from the mgmt host; credentials from + `powerd-secrets`. `ipmitool … chassis power status` gives an independent on/off check. +- **`resume_rate`** throttles inrush current (Slurm `ResumeRate`). + +## 7. Suspend (power OFF) — the actuators + +A host is a **suspend candidate** when: managed (`gen != -`, `on_method != none`, not in +`exclude`), **safely idle** (§4C) for `> suspend_time`, up `> min_uptime`, and alive. + +**Surplus test** (don't suspend nodes we'll immediately need, and honour the warm pool): +``` +surplus_cores = free_cores_on_alive_hosts - pend_cores +``` +Suspend a candidate only while `surplus_cores` stays ≥ 0 after removing it **and** the +count of remaining idle managed nodes stays ≥ `warm_pool`. Order candidates **highest +`gen` first** (oldest hardware off first), cap at `suspend_rate` per tick. + +Actuation per node (graceful, in-band): +```bash +ssh -o BatchMode=yes HOST 'mxqdctl-hostconfig stop' # SIGTERM: stop claiming, drain +# → manager waits for status=EXITED in mxq_daemon, then: +ssh -o BatchMode=yes HOST 'systemctl poweroff' # or: /sbin/poweroff +# fallback if EXITED / dark not reached in time: +ipmitool -I lanplus -H -U "$U" -P "$P" chassis power soft # ACPI soft-off +ipmitool … chassis power off # hard, last resort +``` + +## 8. Safety & operability + +- **`dry_run = true` is the v1 default.** The manager logs every decision + ("would power off node047: idle 41m, surplus 512c") and touches nothing until armed. + Run it against a live cluster for a few days and eyeball the log first. +- **Exclude is belt-and-suspenders**: `gen '-'`, `on_method none`, and the `exclude` + list all protect a node; infra must match at least one. +- **Rate limits** both directions; **warm pool** + **`suspend_time`** + **`min_uptime`** + give hysteresis so the cluster doesn't flap. +- **Timeouts** (`resume_timeout`, `suspend_timeout`) bound every transition; a stuck + node lands in **FAILED**, is excluded, logged, and left for an operator — never + retried blindly. +- **Idempotent & crash-safe**: all durable state is `powerd-state.json` + the DB; a + killed manager just re-derives on the next tick. +- **Manual override** (v1): edit `exclude` / set `on_method none` to pin a node on; + `systemctl stop mxq-powerd.timer` to disable globally. (v2: `mxqadmin --poweron/ + --poweroff/--drain`, study §6.) + +## 9. systemd units + +```ini +# /etc/systemd/system/mxq-powerd.service +[Unit] +Description=MXQ node power manager (one evaluation tick) +After=network-online.target +[Service] +Type=oneshot +ExecStart=/usr/lib/mxq/mxq-powerd --config /etc/mxq/powerd.conf + +# /etc/systemd/system/mxq-powerd.timer +[Unit] +Description=Run MXQ power manager every minute +[Timer] +OnBootSec=2min +OnUnitActiveSec=60s +AccuracySec=5s +[Install] +WantedBy=timers.target +``` +(A `* * * * * root /usr/lib/mxq/mxq-powerd` cron line is an equivalent fallback.) + +## 10. Verification plan + +Build order and how to prove each step **end-to-end**, not just unit tests: + +1. **Read-only sanity (dry-run, no actuation):** point at the production DB, confirm the + three queries (§4) return sane numbers, and that the log's idle/busy/demand + classification matches `mxqdump --running` / `mxqdump --inq` for the same instant. +2. **Single-node suspend on a test host:** pick one disposable node, arm only that node + (exclude all others), watch: `mxqdctl stop` → row goes `EXITED(250)` → `poweroff` → + host dark (`ipmitool … power status` / ping). Confirm **no** job was running and no + stale RUNNING rows remain (`mxqdump --running`). +3. **Single-node resume:** submit a job that only that node can satisfy (tag/GPU/mem), + confirm the manager wakes it (WoL and IPMI paths **tested separately**), a fresh + `mxq_daemon` row appears within `resume_timeout`, and the job runs. +4. **Race probe:** in a loop, submit a burst just as a node crosses `suspend_time`; + confirm the wait-for-EXITED gate never cuts power on a host that grabbed a job. +5. **Failure paths:** block the BMC / drop a NIC and confirm the node lands in FAILED, + is excluded, and logged — not retried into a boot loop. +6. **Load test on the real cluster:** the MXQ cluster is available for a soak run — + `mxqsub -m 956G -t 90m --processors=256 -o ~/powerd-soak.log ./submit.sh` — drive it + idle→busy→idle cycles and inspect the log for flapping and correct rate limiting. +7. Only after 1–6 pass in dry-run/single-node: flip `dry_run = false` cluster-wide with + a conservative `suspend_time` (e.g. 60 min) and a large `warm_pool`, then tighten. + +## 11. Open items for Paul + +- Confirm the transport per node class (which are IPMI-capable, which WoL-capable, any + cross-VLAN WoL relay needed). +- Confirm `ssh HOST poweroff` is acceptable, or if power-off must go via BMC soft-off. +- Confirm the infra/never-suspend set and initial `suspend_time` / `warm_pool`. +- OK to add a read-only `[mxqpowerd]` MySQL account/config group? +- Python 3 acceptable, or prefer bash to match `mxqdctl-hostconfig.sh`? + +## References +- Slurm Power Saving Guide — https://slurm.schedmd.com/power_save.html +- OleHolmNielsen/Slurm_tools power_save (IPMI example scripts) — https://github.com/OleHolmNielsen/Slurm_tools/tree/master/power_save +- GSI Slurm power-management notes (WoL/IPMI resume) — https://web-docs.gsi.de/~vpenso/notes/posts/hpc/cluster/slurm/power-management.html +- Thinkbox Deadline Machine Startup (WoL/IPMI wake) — https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/power-management.html diff --git a/manpages/mxq-powerd.8 b/manpages/mxq-powerd.8 new file mode 100644 index 00000000..fe64131e --- /dev/null +++ b/manpages/mxq-powerd.8 @@ -0,0 +1,115 @@ +.TH MXQ-POWERD 8 "2026-07-18" "MXQ" "MXQ System Administration" +.SH NAME +mxq-powerd \- MXQ node power manager (suspend idle nodes, resume on demand) +.SH SYNOPSIS +.B mxq-powerd +.RI [ options ] +.SH DESCRIPTION +.B mxq-powerd +powers idle MXQ compute nodes down to save energy and powers them back up +when queued work needs them. It runs on the management host (where +.BR mxqdump (1) +and +.BR mxqadmin (1) +run), as root, on a fixed interval (a systemd timer or cron). One invocation +performs one evaluation tick; it is safe to run back-to-back and safe to miss +a tick. +.PP +The manager is entirely external to MXQ: it only +.B reads +the MySQL database (through the stock +.BR mysql (1) +client) and actuates nodes with stock CLIs +.RB ( ipmitool ", " wakeonlan ", " ssh ). +It issues no database writes and requires no changes to +.BR mxqd ", the schema, or " mxqadmin . +.PP +Each tick it: aggregates pending demand and live free capacity from the +database; classifies every managed host through a per-node state machine +(kept in the state file); wakes powered-off nodes when there is a core +deficit, an unmet capability (GPU/memory/tag), or the warm pool is short; +and gracefully drains and powers off surplus idle nodes. Suspend prefers the +oldest, least efficient hardware (highest +.IR gen ); +resume prefers the newest (lowest +.IR gen ). +.SH OPTIONS +.TP +.BR \-c ", " \-\-config " " \fIFILE\fR +Configuration file (default: +.IR /etc/mxq/powerd.conf ). +.TP +.B \-\-arm +Override the config and actually actuate this run +.RI ( dry_run=false ). +.TP +.B \-\-dry\-run +Override the config and log only, never actuate. +.TP +.B \-\-stderr +Also log to standard error. +.TP +.B \-\-status +Print the per-host state and exit without touching anything. +.TP +.BR \-\-clear " " \fIHOST\fR +Clear a FAILED or stuck host from the state file so automation may manage it +again, and exit. +.SH FILES +.TP +.I /etc/mxq/powerd.conf +Global policy: thresholds, rate limits, warm pool, exclude list, DB access, +and the +.B dry_run +switch. See +.IR powerd.conf.example . +.TP +.I /etc/mxq/nodes.conf +Static per-host inventory: generation/priority, capability +(slots/memory/GPUs/tags), power-on transport and BMC address or MAC. See +.IR nodes.conf.example . +.TP +.I /etc/mxq/powerd-secrets +IPMI/BMC credentials (mode 0600, root only). Kept out of the world-readable +inventory. The password reaches +.B ipmitool +via +.B $IPMI_PASSWORD +.RB ( ipmitool " " \-E ), +never on the command line. +.TP +.I /var/lib/mxq/powerd-state.json +Durable manager state: in-flight transitions and timestamps. A flock on this +file also serialises overlapping ticks. +.TP +.I /var/log/mxq/powerd.log +Actions and decisions. +.SH SAFETY +.B dry_run = true +is the default: the manager logs every decision and touches nothing until +armed. A host is protected from automation by any of: an excluded hostname, +.I gen +of +.BR \- , +or +.I on_method +of +.BR none . +Rate limits, the warm pool, +.IR suspend_time ", and " min_uptime +provide hysteresis. Every transition is bounded by a timeout; a stuck node +lands in +.BR FAILED , +is excluded, logged, and left for an operator rather than retried blindly. +Clear it with +.BR "mxq-powerd \-\-clear " \fIHOST\fR . +.SH EXIT STATUS +0 on success, non-zero on a fatal configuration or database error. +.SH SEE ALSO +.BR mxqdump (1), +.BR mxqadmin (1), +.BR ipmitool (1), +.BR wakeonlan (1) +.PP +.I docs/power-management-implementation.md +in the MXQ source tree documents the design this implements. diff --git a/powerd/.gitignore b/powerd/.gitignore new file mode 100644 index 00000000..7a60b85e --- /dev/null +++ b/powerd/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/powerd/README.md b/powerd/README.md new file mode 100644 index 00000000..01e1b295 --- /dev/null +++ b/powerd/README.md @@ -0,0 +1,62 @@ +# mxq-powerd — MXQ node power management + +`mxq-powerd` powers idle MXQ compute nodes down and powers them back up when +work arrives, oldest/least-efficient hardware off first. It is an **external** +add-on: it only *reads* the MySQL database (via the stock `mysql` client) and +actuates nodes with stock CLIs (`ipmitool`, `wakeonlan`, `ssh`). It touches +neither `mxqd`, the schema, nor `mxqadmin`. + +See [`docs/power-management-implementation.md`](../docs/power-management-implementation.md) +for the full design and [`docs/power-management-design.md`](../docs/power-management-design.md) +for the *why*. + +## Files + +| File | Installed to | Purpose | +|---|---|---| +| `mxq-powerd` | `/usr/libexec/mxq/mxq-powerd` | the manager (Python 3) | +| `powerd.conf.example` | `/etc/mxq/powerd.conf` | global policy | +| `nodes.conf.example` | `/etc/mxq/nodes.conf` | per-host inventory | +| `powerd-secrets.example`| `/etc/mxq/powerd-secrets` | IPMI credentials (0600) | +| `mxq-powerd.service` | `/etc/systemd/system/` | one-tick oneshot unit | +| `mxq-powerd.timer` | `/etc/systemd/system/` | runs it every 60 s | +| `manpages/mxq-powerd.8` | `${MANDIR}/man8/` | manual page | + +## Install + +```sh +make install-powerd # installs program + manpage + example configs +# then, once, as root: +cp /etc/mxq/powerd.conf.example /etc/mxq/powerd.conf +cp /etc/mxq/nodes.conf.example /etc/mxq/nodes.conf +install -m 0600 /etc/mxq/powerd-secrets.example /etc/mxq/powerd-secrets +$EDITOR /etc/mxq/nodes.conf /etc/mxq/powerd.conf /etc/mxq/powerd-secrets +systemctl enable --now mxq-powerd.timer +``` + +The systemd units are provided as examples under `powerd/`; copy them to +`/etc/systemd/system/` and `systemctl daemon-reload`. + +## Runbook + +```sh +mxq-powerd --dry-run --stderr # run one tick, log decisions, actuate nothing +mxq-powerd --status # show the recorded per-host state +mxq-powerd --clear node047 # clear a FAILED/stuck host +mxq-powerd --arm # actuate (or set dry_run=false in the config) +``` + +`dry_run = true` is the default. Run it against the live cluster for a few days +and read `/var/log/mxq/powerd.log` before arming. Then flip `dry_run = false` +with a conservative `suspend_time` and a large `warm_pool`, and tighten from +there. `systemctl stop mxq-powerd.timer` disables the whole thing. + +## DB access + +`mxq-powerd` connects the same way the C tools do — through a MySQL defaults +file and config group. By default it reuses MXQ's read-only file +(`/etc/mxq/mysql_ro.cnf`, group `mxqclient`); a dedicated read-only account is +recommended. Because the `mysql` client only reads the `[client]`/`[mysql]` +groups, an arbitrary named group is bridged with `my_print_defaults`. To point +at a file whose credentials live under a plain `[client]` section instead, +leave `defaults_group` empty in `powerd.conf`. diff --git a/powerd/mxq-powerd b/powerd/mxq-powerd new file mode 100755 index 00000000..da393ce2 --- /dev/null +++ b/powerd/mxq-powerd @@ -0,0 +1,1085 @@ +#!/usr/bin/env python3 +# mxq-powerd -- MXQ node power manager (one evaluation tick per invocation) +# +# Powers idle MXQ compute nodes down and powers them back up when work +# arrives, oldest/least-efficient hardware off first, newest on first. +# +# It runs on the management host (where mxqdump/mxqadmin run), as root, on a +# fixed interval (systemd timer or cron). It is *external* to MXQ: it only +# READS the MySQL database (via the stock `mysql` client) and actuates nodes +# with stock CLIs (`ipmitool`, `wakeonlan`, `ssh`). It never writes the DB +# and requires no changes to mxqd, the schema or mxqadmin. +# +# See docs/power-management-implementation.md for the design this implements. + +import argparse +import configparser +import fcntl +import json +import logging +import os +import shlex +import subprocess +import sys +import time + +# --------------------------------------------------------------------------- +# Constants from the MXQ schema (mxq_daemon.h / mxq_job.h). +# --------------------------------------------------------------------------- + +DAEMON_STATUS_IDLE = 0 +DAEMON_STATUS_EXITED = 250 +DAEMON_STATUS_CRASHED = 255 + +# job_status values that mean "this job is on a host right now" +JOB_ACTIVE_STATUSES = (100, 150, 200) # ASSIGNED, LOADED, RUNNING + +MXQ_GROUP_FLAG_CLOSED = 1 # (1<<0) + +# --------------------------------------------------------------------------- +# Per-node lifecycle states (powerd-state.json). +# --------------------------------------------------------------------------- + +S_POWERED_OFF = "POWERED_OFF" +S_POWERING_UP = "POWERING_UP" +S_UP_IDLE = "UP_IDLE" +S_UP_BUSY = "UP_BUSY" +S_DRAINING = "DRAINING" +S_POWERING_DOWN = "POWERING_DOWN" +S_FAILED = "FAILED" + +STATE_VERSION = 1 + +log = logging.getLogger("mxq-powerd") + + +# =========================================================================== +# Configuration +# =========================================================================== + +CONFIG_DEFAULTS = { + "db": { + # How to reach the MySQL server. We reuse MXQ's read-only defaults + # file. If `defaults_group` is set we extract the connection options + # from that group with `my_print_defaults` (the `mysql` client itself + # only reads the [client]/[mysql] groups); otherwise `mysql` reads + # `defaults_file` directly. + "mysql": "mysql", + "my_print_defaults": "my_print_defaults", + "defaults_file": "/etc/mxq/mysql_ro.cnf", + "defaults_group": "mxqclient", + "database": "mxq", + }, + "policy": { + "suspend_time": "1800", # sec idle before a node may power down + "warm_pool": "4", # min idle managed nodes to keep ON + "min_uptime": "900", # sec up before suspend-eligible (anti-thrash) + "suspend_rate": "4", # max power-downs initiated per tick + "resume_rate": "8", # max power-ups initiated per tick + "suspend_timeout": "300", # sec for a shutdown to complete + "resume_timeout": "600", # sec for a boot+register + "mtime_stale": "180", # sec: mxq_daemon.mtime older than this => not alive + "exclude": "", # comma-separated hostnames never managed + }, + "mode": { + "dry_run": "true", # v1 default: log decisions, take NO action + }, + "paths": { + "nodes_conf": "/etc/mxq/nodes.conf", + "secrets_file": "/etc/mxq/powerd-secrets", + "state_file": "/var/lib/mxq/powerd-state.json", + "log_file": "/var/log/mxq/powerd.log", + }, + "actuation": { + "ssh": "ssh", + "ssh_opts": "-o BatchMode=yes -o ConnectTimeout=10", + "ipmitool": "ipmitool", + "wakeonlan": "wakeonlan", + "drain_cmd": "mxqdctl-hostconfig stop", + "poweroff_cmd": "systemctl poweroff", + }, +} + + +class Config: + def __init__(self, path=None): + self.cp = configparser.ConfigParser() + # seed defaults + self.cp.read_dict(CONFIG_DEFAULTS) + self.path = path + if path: + if not self.cp.read(path): + raise SystemExit("mxq-powerd: cannot read config file: %s" % path) + + def get(self, section, key): + return self.cp.get(section, key) + + def getint(self, section, key): + return self.cp.getint(section, key) + + def getbool(self, section, key): + return self.cp.getboolean(section, key) + + def exclude_set(self): + raw = self.get("policy", "exclude") + return {h.strip() for h in raw.split(",") if h.strip()} + + +# =========================================================================== +# Inventory (/etc/mxq/nodes.conf) +# =========================================================================== + +class Node: + """One managed host, from the static inventory.""" + + def __init__(self, hostname, gen, slots, mem_mb, gpus, tags, + on_method, bmc_or_mac): + self.hostname = hostname + self.gen = gen # None => never auto-managed + self.slots = slots # inventoried capacity (node powered off) + self.mem_mb = mem_mb + self.gpus = gpus + self.tags = tags # set of strings + self.on_method = on_method # 'ipmi' | 'wol' | 'none' + self.bmc_or_mac = bmc_or_mac + + @property + def managed(self): + return self.gen is not None and self.on_method in ("ipmi", "wol") + + +def _parse_int(tok): + if tok in ("-", ""): + return None + return int(tok) + + +def load_inventory(path): + """Parse the whitespace-separated /etc/mxq/nodes.conf. + + Columns: hostname gen slots mem_mb gpus tags on_method bmc_or_mac + A `-` in `gen` (or on_method `none`) marks a host as never auto-managed. + """ + nodes = {} + if not os.path.exists(path): + raise SystemExit("mxq-powerd: inventory not found: %s" % path) + with open(path) as fh: + for lineno, line in enumerate(fh, 1): + line = line.split("#", 1)[0].strip() + if not line: + continue + cols = line.split() + if len(cols) < 8: + log.warning("nodes.conf:%d: expected 8 columns, got %d -- skipped", + lineno, len(cols)) + continue + hostname, gen, slots, mem_mb, gpus, tags, on_method, bmc_or_mac = cols[:8] + gen_val = None if gen == "-" else gen + tag_set = {t for t in tags.split(",") if t and t != "-"} + try: + node = Node( + hostname=hostname, + gen=gen_val, + slots=_parse_int(slots) or 0, + mem_mb=_parse_int(mem_mb) or 0, + gpus=_parse_int(gpus) or 0, + tags=tag_set, + on_method=on_method, + bmc_or_mac=bmc_or_mac, + ) + except ValueError as exc: + log.warning("nodes.conf:%d: %s -- skipped", lineno, exc) + continue + nodes[hostname] = node + return nodes + + +# =========================================================================== +# Secrets (/etc/mxq/powerd-secrets) -- IPMI credentials, 0600 root-only +# =========================================================================== + +class Secrets: + """[ipmi] user/password with optional per-host [ipmi.HOST] overrides.""" + + def __init__(self, path): + self.cp = configparser.ConfigParser() + self.path = path + if path and os.path.exists(path): + mode = os.stat(path).st_mode & 0o077 + if mode: + log.warning("%s is group/world accessible (mode %o); " + "IPMI credentials should be 0600", + path, os.stat(path).st_mode & 0o777) + self.cp.read(path) + + def ipmi(self, hostname): + """Return (user, password) for a host, or (None, None).""" + user = password = None + if self.cp.has_section("ipmi"): + user = self.cp.get("ipmi", "user", fallback=None) + password = self.cp.get("ipmi", "password", fallback=None) + sect = "ipmi.%s" % hostname + if self.cp.has_section(sect): + user = self.cp.get(sect, "user", fallback=user) + password = self.cp.get(sect, "password", fallback=password) + return user, password + + +# =========================================================================== +# Durable state (/var/lib/mxq/powerd-state.json) +# =========================================================================== + +class StateStore: + """In-flight transitions + timestamps, persisted as JSON. + + A single flock on the file also serialises overlapping ticks. + """ + + def __init__(self, path): + self.path = path + self.hosts = {} + self._fh = None + + def lock(self): + d = os.path.dirname(self.path) + if d: + os.makedirs(d, exist_ok=True) + # open (creating if needed) and take an exclusive, non-blocking lock + self._fh = open(self.path, "a+") + try: + fcntl.flock(self._fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + raise SystemExit("mxq-powerd: another tick holds the lock on %s" + % self.path) + self._fh.seek(0) + data = self._fh.read().strip() + if data: + try: + doc = json.loads(data) + self.hosts = doc.get("hosts", {}) + except json.JSONDecodeError as exc: + log.error("state file %s is corrupt (%s); starting empty", + self.path, exc) + self.hosts = {} + + def save(self): + doc = {"version": STATE_VERSION, "hosts": self.hosts} + tmp = self.path + ".tmp" + with open(tmp, "w") as fh: + json.dump(doc, fh, indent=2, sort_keys=True) + fh.write("\n") + os.replace(tmp, self.path) + + def unlock(self): + if self._fh: + fcntl.flock(self._fh, fcntl.LOCK_UN) + self._fh.close() + self._fh = None + + # -- per-host helpers -------------------------------------------------- + + def host(self, hostname): + return self.hosts.get(hostname) + + def set_state(self, hostname, state, now, note=None, **stamps): + rec = self.hosts.setdefault(hostname, {}) + if rec.get("state") != state: + rec["since"] = now + rec["state"] = state + if note is not None: + rec["note"] = note + rec.update(stamps) + return rec + + def get_state(self, hostname): + rec = self.hosts.get(hostname) + return rec.get("state") if rec else None + + +# =========================================================================== +# Database access (read-only, via the stock `mysql` client) +# =========================================================================== + +class Database: + def __init__(self, cfg): + self.cfg = cfg + self._base = self._build_base_cmd() + + def _build_base_cmd(self): + c = self.cfg + mysql = c.get("db", "mysql") + dfile = c.get("db", "defaults_file") + dgroup = c.get("db", "defaults_group") + database = c.get("db", "database") + + if dgroup: + # Extract connection options from an arbitrary named group. The + # `mysql` client itself only reads [client]/[mysql], so we bridge + # via my_print_defaults. + opts = self._print_defaults(dfile, dgroup) + cmd = [mysql, "--no-defaults"] + opts + elif dfile: + cmd = [mysql, "--defaults-file=" + dfile] + else: + cmd = [mysql] + cmd += ["--batch", "--skip-column-names"] + if database: + cmd += ["--database=" + database] + return cmd + + def _print_defaults(self, dfile, group): + mpd = self.cfg.get("db", "my_print_defaults") + cmd = [mpd] + if dfile: + cmd.append("--defaults-file=" + dfile) + cmd.append(group) + try: + out = subprocess.run(cmd, capture_output=True, text=True, check=True) + except (OSError, subprocess.CalledProcessError) as exc: + raise SystemExit("mxq-powerd: cannot read DB defaults group [%s] " + "from %s: %s" % (group, dfile, exc)) + return [ln.strip() for ln in out.stdout.splitlines() if ln.strip()] + + def query(self, sql): + """Run SQL, return list of rows; each row is a list of str/None.""" + cmd = self._base + ["--execute=" + sql] + try: + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + except OSError as exc: + raise SystemExit("mxq-powerd: cannot run mysql: %s" % exc) + except subprocess.CalledProcessError as exc: + raise SystemExit("mxq-powerd: DB query failed: %s\nSQL: %s" + % (exc.stderr.strip(), sql)) + rows = [] + for line in res.stdout.splitlines(): + if not line: + continue # `mysql --batch` emits no blank data lines + fields = line.split("\t") + rows.append([None if f == "NULL" else f for f in fields]) + return rows + + def scalar(self, sql): + rows = self.query(sql) + if not rows or not rows[0]: + return None + return rows[0][0] + + +def _to_int(val, default=0): + if val is None: + return default + try: + return int(val) + except (TypeError, ValueError): + return default + + +# =========================================================================== +# Cluster snapshot (everything one tick needs from the DB) +# =========================================================================== + +class PendingGroup: + def __init__(self, jobs_inq, threads, memory_mb, gpu, tags): + self.jobs_inq = jobs_inq + self.threads = threads + self.memory_mb = memory_mb + self.gpu = gpu + self.tags = tags + + +class LiveHost: + def __init__(self, hostname, status, slots, slots_running, slots_free, + mtime, daemon_start): + self.hostname = hostname + self.status = status + self.slots = slots + self.slots_running = slots_running + self.slots_free = slots_free + self.mtime = mtime + self.daemon_start = daemon_start + self.active_jobs = 0 + self.last_end = None + + +class Snapshot: + def __init__(self, db, cfg): + self.now = _to_int(db.scalar("SELECT UNIX_TIMESTAMP()"), int(time.time())) + self.mtime_stale = cfg.getint("policy", "mtime_stale") + self._read_demand(db) + self._read_live(db) + self._read_activity(db) + + # -- A. pending demand ------------------------------------------------- + + def _read_demand(self, db): + # aggregate totals + row = db.query( + "SELECT COALESCE(SUM(group_jobs_inq),0), " + "COALESCE(SUM(group_jobs_inq*job_threads),0), " + "COALESCE(MAX(job_memory),0), COALESCE(MAX(job_gpu),0) " + "FROM mxq_group " + "WHERE group_jobs_inq > 0 AND (group_flags & %d) = 0" + % MXQ_GROUP_FLAG_CLOSED) + agg = row[0] if row else [0, 0, 0, 0] + self.pend_jobs = _to_int(agg[0]) + self.pend_cores = _to_int(agg[1]) + self.max_job_mem_mb = _to_int(agg[2]) + self.max_job_gpu = _to_int(agg[3]) + + # per-group rows so resume can match specific constraints + self.pending_groups = [] + for r in db.query( + "SELECT group_jobs_inq, job_threads, job_memory, job_gpu, tags " + "FROM mxq_group " + "WHERE group_jobs_inq > 0 AND (group_flags & %d) = 0" + % MXQ_GROUP_FLAG_CLOSED): + tags = {t for t in (r[4] or "").split(",") if t} + self.pending_groups.append(PendingGroup( + jobs_inq=_to_int(r[0]), threads=_to_int(r[1]), + memory_mb=_to_int(r[2]), gpu=_to_int(r[3]), tags=tags)) + + # -- B. live capacity & liveness -------------------------------------- + + def _read_live(self, db): + self.live = {} + for r in db.query( + "SELECT hostname, MAX(status), SUM(daemon_slots), " + "SUM(daemon_slots_running), " + "SUM(daemon_slots - daemon_slots_running), " + "MAX(UNIX_TIMESTAMP(mtime)), MIN(UNIX_TIMESTAMP(daemon_start)) " + "FROM mxq_daemon " + "WHERE status NOT IN (%d, %d) " + "GROUP BY hostname" + % (DAEMON_STATUS_EXITED, DAEMON_STATUS_CRASHED)): + host = LiveHost( + hostname=r[0], status=_to_int(r[1]), + slots=_to_int(r[2]), slots_running=_to_int(r[3]), + slots_free=_to_int(r[4]), mtime=_to_int(r[5]), + daemon_start=_to_int(r[6])) + self.live[host.hostname] = host + + # -- C. idle-duration & active jobs ----------------------------------- + + def _read_activity(self, db): + for r in db.query( + "SELECT host_hostname, MAX(UNIX_TIMESTAMP(date_end)) " + "FROM mxq_job WHERE host_hostname <> '' " + "GROUP BY host_hostname"): + h = self.live.get(r[0]) + if h is not None: + h.last_end = _to_int(r[1], None) if r[1] is not None else None + + active_sql = ( + "SELECT host_hostname, COUNT(*) FROM mxq_job " + "WHERE host_hostname <> '' AND job_status IN (%s) " + "GROUP BY host_hostname" + % ",".join(str(s) for s in JOB_ACTIVE_STATUSES)) + for r in db.query(active_sql): + h = self.live.get(r[0]) + if h is not None: + h.active_jobs = _to_int(r[1]) + + # -- derived helpers --------------------------------------------------- + + def is_alive(self, hostname): + h = self.live.get(hostname) + return h is not None and (self.now - h.mtime) <= self.mtime_stale + + def is_present(self, hostname): + """Has any non-EXITED/CRASHED daemon row (may be stale).""" + return hostname in self.live + + def is_busy(self, hostname): + h = self.live.get(hostname) + if h is None: + return False + return h.slots_running > 0 or h.active_jobs > 0 or h.status != DAEMON_STATUS_IDLE + + def is_idle(self, hostname): + """Safely idle: alive, all daemons IDLE, nothing running/claimed.""" + h = self.live.get(hostname) + if h is None or not self.is_alive(hostname): + return False + return (h.status == DAEMON_STATUS_IDLE and h.slots_running == 0 + and h.active_jobs == 0) + + def idle_secs(self, hostname): + """Seconds since the host last finished a job (fallback: daemon_start).""" + h = self.live.get(hostname) + if h is None: + return 0 + ref = h.last_end if h.last_end else h.daemon_start + if not ref: + return 0 + return max(0, self.now - ref) + + def uptime_secs(self, hostname): + h = self.live.get(hostname) + if h is None or not h.daemon_start: + return 0 + return max(0, self.now - h.daemon_start) + + +# =========================================================================== +# Actuators (all no-ops under dry_run) +# =========================================================================== + +class Actuator: + def __init__(self, cfg, secrets, dry_run): + self.cfg = cfg + self.secrets = secrets + self.dry_run = dry_run + + def _run(self, cmd, what, host, env=None): + printable = " ".join(shlex.quote(c) for c in cmd) + if self.dry_run: + log.info("[dry-run] %s %s: would run: %s", what, host, printable) + return True + log.info("%s %s: running: %s", what, host, printable) + try: + res = subprocess.run(cmd, capture_output=True, text=True, + env=env, timeout=60) + except (OSError, subprocess.TimeoutExpired) as exc: + log.error("%s %s: command error: %s", what, host, exc) + return False + if res.returncode != 0: + log.error("%s %s: exit %d: %s", what, host, res.returncode, + (res.stderr or res.stdout).strip()) + return False + return True + + # -- resume ------------------------------------------------------------ + + def power_on(self, node): + if node.on_method == "wol": + return self._wol(node) + if node.on_method == "ipmi": + return self._ipmi(node, ["chassis", "power", "on"], "power-on") + log.error("power_on %s: unknown on_method %r", node.hostname, + node.on_method) + return False + + def _wol(self, node): + cmd = [self.cfg.get("actuation", "wakeonlan"), node.bmc_or_mac] + return self._run(cmd, "wol", node.hostname) + + def _ipmi_cmd(self, node, sub): + user, password = self.secrets.ipmi(node.hostname) + cmd = [self.cfg.get("actuation", "ipmitool"), + "-I", "lanplus", "-H", node.bmc_or_mac] + if user: + cmd += ["-U", user] + # -E reads the password from $IPMI_PASSWORD, keeping it out of argv/ps + env = None + if password: + cmd += ["-E"] + env = dict(os.environ, IPMI_PASSWORD=password) + return cmd + list(sub), env + + def _ipmi(self, node, sub, what): + if node.on_method != "ipmi": + log.error("%s %s: not an ipmi node", what, node.hostname) + return False + cmd, env = self._ipmi_cmd(node, sub) + return self._run(cmd, what, node.hostname, env=env) + + # -- suspend ----------------------------------------------------------- + + def ssh_drain(self, node): + return self._ssh(node, self.cfg.get("actuation", "drain_cmd"), "drain") + + def ssh_poweroff(self, node): + return self._ssh(node, self.cfg.get("actuation", "poweroff_cmd"), + "poweroff") + + def _ssh(self, node, remote_cmd, what): + opts = shlex.split(self.cfg.get("actuation", "ssh_opts")) + cmd = [self.cfg.get("actuation", "ssh")] + opts + \ + [node.hostname, remote_cmd] + return self._run(cmd, what, node.hostname) + + def ipmi_soft_off(self, node): + return self._ipmi(node, ["chassis", "power", "soft"], "soft-off") + + def ipmi_hard_off(self, node): + return self._ipmi(node, ["chassis", "power", "off"], "hard-off") + + def ipmi_is_off(self, node): + """Return True if BMC reports chassis power off, None if unknown.""" + if node.on_method != "ipmi" or self.dry_run: + return None + cmd, env = self._ipmi_cmd(node, ["chassis", "power", "status"]) + try: + res = subprocess.run(cmd, capture_output=True, text=True, + env=env, timeout=30) + except (OSError, subprocess.TimeoutExpired): + return None + if res.returncode != 0: + return None + return "off" in res.stdout.lower() + + +# =========================================================================== +# The manager: one evaluation tick +# =========================================================================== + +class Manager: + def __init__(self, cfg, inventory, secrets, state, snap, actuator): + self.cfg = cfg + self.inventory = inventory + self.secrets = secrets + self.state = state + self.snap = snap + self.act = actuator + self.now = snap.now + + self.suspend_time = cfg.getint("policy", "suspend_time") + self.warm_pool = cfg.getint("policy", "warm_pool") + self.min_uptime = cfg.getint("policy", "min_uptime") + self.suspend_rate = cfg.getint("policy", "suspend_rate") + self.resume_rate = cfg.getint("policy", "resume_rate") + self.suspend_timeout = cfg.getint("policy", "suspend_timeout") + self.resume_timeout = cfg.getint("policy", "resume_timeout") + self.exclude = cfg.exclude_set() + + def managed_nodes(self): + for node in self.inventory.values(): + if node.managed and node.hostname not in self.exclude: + yield node + + # -- state reconciliation --------------------------------------------- + + def reconcile(self): + """Advance each managed host's state machine from observed reality.""" + for node in self.managed_nodes(): + h = node.hostname + cur = self.state.get_state(h) + if cur is None: + cur = self._infer_initial(h) + self.state.set_state(h, cur, self.now, + note="initial classification") + handler = getattr(self, "_st_" + cur.lower(), None) + if handler: + handler(node) + + def _infer_initial(self, hostname): + if self.snap.is_alive(hostname): + return S_UP_BUSY if self.snap.is_busy(hostname) else S_UP_IDLE + return S_POWERED_OFF + + def _st_powered_off(self, node): + # A host we believe is off but that reappeared alive: someone (or a + # boot we did not initiate) brought it up. Adopt reality. + if self.snap.is_alive(node.hostname): + new = S_UP_BUSY if self.snap.is_busy(node.hostname) else S_UP_IDLE + self.state.set_state(node.hostname, new, self.now, + note="observed alive") + + def _st_powering_up(self, node): + h = node.hostname + rec = self.state.host(h) + if self.snap.is_alive(h): + new = S_UP_BUSY if self.snap.is_busy(h) else S_UP_IDLE + self.state.set_state(h, new, self.now, note="resume complete") + log.info("%s: resume complete -> %s", h, new) + return + t_resume = rec.get("t_resume", rec.get("since", self.now)) + if self.now - t_resume > self.resume_timeout: + self.state.set_state(h, S_FAILED, self.now, + note="resume timeout (%ds)" % self.resume_timeout) + log.error("%s: FAILED -- no daemon row within resume_timeout=%ds; " + "excluding until operator clears", + h, self.resume_timeout) + + def _st_up_idle(self, node): + h = node.hostname + if not self.snap.is_present(h): + self.state.set_state(h, S_POWERED_OFF, self.now, + note="disappeared while idle") + log.warning("%s: was UP_IDLE but daemon vanished -> POWERED_OFF", h) + return + if self.snap.is_busy(h): + self.state.set_state(h, S_UP_BUSY, self.now, note="job claimed") + + def _st_up_busy(self, node): + h = node.hostname + if not self.snap.is_present(h): + self.state.set_state(h, S_POWERED_OFF, self.now, + note="disappeared while busy") + log.warning("%s: was UP_BUSY but daemon vanished -> POWERED_OFF", h) + return + if self.snap.is_idle(h): + self.state.set_state(h, S_UP_IDLE, self.now, note="became idle") + + def _st_draining(self, node): + h = node.hostname + rec = self.state.host(h) + t_drain = rec.get("t_drain", rec.get("since", self.now)) + # DRAINING -> POWERING_DOWN once all daemon rows are gone (EXITED): + # waiting for EXITED is the race-closer -- mxqd only exits after its + # drain loop finds nothing running. + if not self.snap.is_present(h): + log.info("%s: drained (all daemons exited); issuing power-off", h) + if self.act.ssh_poweroff(node): + self.state.set_state(h, S_POWERING_DOWN, self.now, + t_off=self.now, note="poweroff issued") + else: + # graceful poweroff failed -> escalate to BMC soft-off + self._escalate_soft_off(node, "ssh poweroff failed") + return + # not drained yet: escalate to BMC soft-off at half the suspend timeout + if self.now - t_drain > self.suspend_timeout // 2: + self._escalate_soft_off(node, "EXITED not seen in time") + + def _escalate_soft_off(self, node, why): + h = node.hostname + if node.on_method == "ipmi": + log.warning("%s: %s -- escalating to BMC soft-off", h, why) + self.act.ipmi_soft_off(node) + self.state.set_state(h, S_POWERING_DOWN, self.now, + t_off=self.now, note="soft-off (%s)" % why) + else: + log.warning("%s: %s -- no BMC (on_method=%s); leaving DRAINING", + h, why, node.on_method) + + def _st_powering_down(self, node): + h = node.hostname + rec = self.state.host(h) + t_off = rec.get("t_off", rec.get("t_drain", rec.get("since", self.now))) + t_drain = rec.get("t_drain", t_off) + + if self._confirmed_dark(node): + self.state.set_state(h, S_POWERED_OFF, self.now, + note="confirmed dark") + log.info("%s: confirmed powered off", h) + return + # bounded by suspend_timeout from the drain: hard-off, then FAILED + if self.now - t_drain > self.suspend_timeout: + if node.on_method == "ipmi": + log.warning("%s: still up after suspend_timeout=%ds -- BMC hard-off", + h, self.suspend_timeout) + self.act.ipmi_hard_off(node) + # give the hard-off one more tick to confirm before FAILED + if self._confirmed_dark(node): + self.state.set_state(h, S_POWERED_OFF, self.now, + note="dark after hard-off") + return + self.state.set_state(h, S_FAILED, self.now, + note="did not power down in time") + log.error("%s: FAILED -- did not power down within suspend_timeout; " + "excluding until operator clears", h) + + def _confirmed_dark(self, node): + # In dry-run nothing is actually powered off; treat absence as dark so + # the state machine can progress for observation. + bmc = self.act.ipmi_is_off(node) + if bmc is True: + return True + if bmc is False: + return False + # No BMC answer (WoL node, or dry-run): fall back to DB liveness. + return not self.snap.is_alive(node.hostname) + + # -- resume (power ON) ------------------------------------------------- + + def _free_cores_alive(self): + total = 0 + for node in self.managed_nodes(): + if self.snap.is_alive(node.hostname): + total += self.snap.live[node.hostname].slots_free + return total + + def _idle_on_count(self): + n = 0 + for node in self.managed_nodes(): + if self.state.get_state(node.hostname) == S_UP_IDLE \ + and self.snap.is_alive(node.hostname): + n += 1 + return n + + def _host_satisfies(self, node, group): + if node.slots < group.threads: + return False + if node.mem_mb and group.memory_mb and node.mem_mb < group.memory_mb: + return False + if node.gpus < group.gpu: + return False + if group.tags and not group.tags.issubset(node.tags): + return False + return True + + def _alive_can_serve(self, group): + """Can any alive managed host with free slots run this group's jobs?""" + for node in self.managed_nodes(): + if not self.snap.is_alive(node.hostname): + continue + live = self.snap.live[node.hostname] + if live.slots_free < group.threads: + continue + if node.gpus < group.gpu: + continue + if group.tags and not group.tags.issubset(node.tags): + continue + if node.mem_mb and group.memory_mb and node.mem_mb < group.memory_mb: + continue + return True + return False + + def resume(self): + snap = self.snap + free_cores = self._free_cores_alive() + deficit = snap.pend_cores - free_cores + idle_on = self._idle_on_count() + warm_deficit = max(0, self.warm_pool - idle_on) + + # groups no alive host can currently place (GPU/mem/tag gaps) + unmet = [g for g in snap.pending_groups if not self._alive_can_serve(g)] + + need = deficit > 0 or unmet or warm_deficit > 0 + if not need: + log.info("resume: no deficit (pend_cores=%d free_cores=%d " + "idle_on=%d/warm_pool=%d)", + snap.pend_cores, free_cores, idle_on, self.warm_pool) + return + + log.info("resume: deficit_cores=%d unmet_groups=%d warm_deficit=%d", + deficit, len(unmet), warm_deficit) + + candidates = [n for n in self.managed_nodes() + if self.state.get_state(n.hostname) == S_POWERED_OFF] + # newest / most efficient first == lowest gen first + candidates.sort(key=lambda n: (self._gen_key(n), -n.slots)) + + woken = 0 + for node in candidates: + if woken >= self.resume_rate: + log.info("resume: hit resume_rate=%d", self.resume_rate) + break + if not (deficit > 0 or unmet or warm_deficit > 0): + break + helps_unmet = [g for g in unmet if self._host_satisfies(node, g)] + # if the only remaining need is capability gaps, skip nodes that + # cannot close any of them + if deficit <= 0 and warm_deficit <= 0 and not helps_unmet: + continue + reason = self._resume_reason(node, deficit, helps_unmet, warm_deficit) + log.info("resume: waking %s (gen=%s slots=%d) -- %s", + node.hostname, node.gen, node.slots, reason) + if self.act.power_on(node): + self.state.set_state(node.hostname, S_POWERING_UP, self.now, + t_resume=self.now, note=reason) + woken += 1 + deficit -= node.slots + if warm_deficit > 0: + warm_deficit -= 1 + for g in helps_unmet: + if g in unmet: + unmet.remove(g) + + @staticmethod + def _resume_reason(node, deficit, helps_unmet, warm_deficit): + bits = [] + if deficit > 0: + bits.append("core deficit") + if helps_unmet: + bits.append("capability gap (%d groups)" % len(helps_unmet)) + if warm_deficit > 0: + bits.append("warm pool") + return ", ".join(bits) or "warm pool" + + # -- suspend (power OFF) ---------------------------------------------- + + def _gen_key(self, node): + """Sort key from the `gen` token; numeric part if present.""" + g = node.gen or "" + digits = "".join(c for c in g if c.isdigit()) + return (int(digits) if digits else 0, g) + + def suspend(self): + snap = self.snap + free_cores = self._free_cores_alive() + surplus = free_cores - snap.pend_cores + idle_on = self._idle_on_count() + + candidates = [] + for node in self.managed_nodes(): + h = node.hostname + if self.state.get_state(h) != S_UP_IDLE: + continue + if not snap.is_idle(h): + continue + if snap.idle_secs(h) <= self.suspend_time: + continue + if snap.uptime_secs(h) <= self.min_uptime: + continue + candidates.append(node) + + # oldest / least efficient first == highest gen first + candidates.sort(key=lambda n: self._gen_key(n), reverse=True) + + if not candidates: + log.info("suspend: no eligible idle candidates " + "(surplus_cores=%d idle_on=%d)", surplus, idle_on) + return + + drained = 0 + for node in candidates: + h = node.hostname + if drained >= self.suspend_rate: + log.info("suspend: hit suspend_rate=%d", self.suspend_rate) + break + free = snap.live[h].slots_free + if surplus - free < 0: + log.info("suspend: keeping %s -- surplus would go negative " + "(surplus=%d, host_free=%d)", h, surplus, free) + continue + if idle_on - 1 < self.warm_pool: + log.info("suspend: stopping -- warm_pool floor (idle_on=%d, " + "warm_pool=%d)", idle_on, self.warm_pool) + break + log.info("suspend: draining %s (gen=%s idle=%dm surplus=%dc) ", + h, node.gen, snap.idle_secs(h) // 60, surplus) + if self.act.ssh_drain(node): + self.state.set_state(h, S_DRAINING, self.now, + t_drain=self.now, note="drain issued") + drained += 1 + surplus -= free + idle_on -= 1 + + # -- one full tick ----------------------------------------------------- + + def tick(self): + log.info("tick: now=%d dry_run=%s pend_jobs=%d pend_cores=%d " + "live_hosts=%d", + self.now, self.act.dry_run, self.snap.pend_jobs, + self.snap.pend_cores, len(self.snap.live)) + self.reconcile() + self.resume() + self.suspend() + + +# =========================================================================== +# CLI +# =========================================================================== + +def setup_logging(cfg, to_stderr): + handlers = [] + logfile = cfg.get("paths", "log_file") + if logfile: + try: + d = os.path.dirname(logfile) + if d: + os.makedirs(d, exist_ok=True) + handlers.append(logging.FileHandler(logfile)) + except OSError as exc: + print("mxq-powerd: cannot open log file %s: %s" % (logfile, exc), + file=sys.stderr) + if to_stderr or not handlers: + handlers.append(logging.StreamHandler(sys.stderr)) + fmt = logging.Formatter("%(asctime)s %(levelname)s %(message)s", + "%Y-%m-%d %H:%M:%S") + for h in handlers: + h.setFormatter(fmt) + log.setLevel(logging.INFO) + for h in handlers: + log.addHandler(h) + + +def cmd_status(cfg, args): + """Print the current per-host state without touching anything.""" + state = StateStore(cfg.get("paths", "state_file")) + state.lock() + try: + if not state.hosts: + print("(no state recorded yet)") + return 0 + width = max(len(h) for h in state.hosts) + for host in sorted(state.hosts): + rec = state.hosts[host] + note = rec.get("note", "") + since = rec.get("since") + age = "" + if since: + age = " since=%s" % time.strftime("%Y-%m-%d %H:%M:%S", + time.localtime(since)) + print("%-*s %-14s%s %s" % (width, host, rec.get("state", "?"), + age, note)) + finally: + state.unlock() + return 0 + + +def cmd_clear(cfg, args): + """Clear a FAILED host so automation may manage it again.""" + state = StateStore(cfg.get("paths", "state_file")) + state.lock() + try: + rec = state.hosts.get(args.clear) + if not rec: + print("mxq-powerd: no state for host %s" % args.clear, + file=sys.stderr) + return 1 + old = rec.get("state") + state.hosts.pop(args.clear, None) + state.save() + print("cleared %s (was %s); it will be re-classified next tick" + % (args.clear, old)) + finally: + state.unlock() + return 0 + + +def cmd_tick(cfg, args): + inventory = load_inventory(cfg.get("paths", "nodes_conf")) + secrets = Secrets(cfg.get("paths", "secrets_file")) + dry_run = cfg.getbool("mode", "dry_run") + if args.arm: + dry_run = False + if args.dry_run: + dry_run = True + + db = Database(cfg) + snap = Snapshot(db, cfg) + actuator = Actuator(cfg, secrets, dry_run) + + state = StateStore(cfg.get("paths", "state_file")) + state.lock() + try: + mgr = Manager(cfg, inventory, secrets, state, snap, actuator) + mgr.tick() + state.save() + finally: + state.unlock() + return 0 + + +def main(argv=None): + p = argparse.ArgumentParser( + prog="mxq-powerd", + description="MXQ node power manager -- one evaluation tick per run.") + p.add_argument("-c", "--config", default="/etc/mxq/powerd.conf", + help="config file (default: /etc/mxq/powerd.conf)") + p.add_argument("--arm", action="store_true", + help="override config: actually actuate (dry_run=false)") + p.add_argument("--dry-run", action="store_true", + help="override config: log only, never actuate") + p.add_argument("--stderr", action="store_true", + help="also log to stderr") + p.add_argument("--status", action="store_true", + help="print per-host state and exit") + p.add_argument("--clear", metavar="HOST", + help="clear a FAILED/stuck host from the state and exit") + args = p.parse_args(argv) + + cfg = Config(args.config if os.path.exists(args.config) else None) + setup_logging(cfg, to_stderr=args.stderr or args.status or bool(args.clear)) + + if args.status: + return cmd_status(cfg, args) + if args.clear: + return cmd_clear(cfg, args) + return cmd_tick(cfg, args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/powerd/mxq-powerd.service b/powerd/mxq-powerd.service new file mode 100644 index 00000000..41ea6b5e --- /dev/null +++ b/powerd/mxq-powerd.service @@ -0,0 +1,11 @@ +# /etc/systemd/system/mxq-powerd.service +[Unit] +Description=MXQ node power manager (one evaluation tick) +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/libexec/mxq/mxq-powerd --config /etc/mxq/powerd.conf +# The manager needs root to ssh to nodes and drive BMCs. +User=root diff --git a/powerd/mxq-powerd.timer b/powerd/mxq-powerd.timer new file mode 100644 index 00000000..36696211 --- /dev/null +++ b/powerd/mxq-powerd.timer @@ -0,0 +1,14 @@ +# /etc/systemd/system/mxq-powerd.timer +# +# Enable with: systemctl enable --now mxq-powerd.timer +# Disable power management globally with: systemctl stop mxq-powerd.timer +[Unit] +Description=Run MXQ power manager every minute + +[Timer] +OnBootSec=2min +OnUnitActiveSec=60s +AccuracySec=5s + +[Install] +WantedBy=timers.target diff --git a/powerd/nodes.conf.example b/powerd/nodes.conf.example new file mode 100644 index 00000000..ce568c4d --- /dev/null +++ b/powerd/nodes.conf.example @@ -0,0 +1,25 @@ +# /etc/mxq/nodes.conf -- static per-host inventory for mxq-powerd. +# +# Flat, '#'-comment, whitespace-separated (the /etc/hostconfig house style). +# One line per HOST (not per daemon). This describes what a powered-OFF node +# can do -- the live DB cannot tell us once the node is dark. +# +# Columns: +# hostname the short hostname as it appears in mxq_daemon.hostname +# gen generation / power-priority token. Suspend picks the HIGHEST +# gen (oldest, least efficient) first; resume the LOWEST +# (newest) first. '-' => never auto-managed. +# slots CPU slots the node offers +# mem_mb memory in MB +# gpus number of GPUs +# tags comma-separated capability tags ('-' for none) +# on_method ipmi | wol | none ('none' => excluded from all automation) +# bmc_or_mac BMC IP/hostname for ipmi, MAC address for wol ('-' if none) +# +# IPMI credentials do NOT go here -- see /etc/mxq/powerd-secrets. + +# hostname gen slots mem_mb gpus tags on_method bmc_or_mac +node001 g1 64 257000 0 xeon,highmem ipmi 10.1.0.11 +node002 g1 64 257000 0 xeon,highmem ipmi 10.1.0.12 +node101 g3 256 980000 4 epyc,gpu,a100 wol e4:3d:1a:00:11:22 +login01 - - - - infra none - diff --git a/powerd/powerd-secrets.example b/powerd/powerd-secrets.example new file mode 100644 index 00000000..28e9704c --- /dev/null +++ b/powerd/powerd-secrets.example @@ -0,0 +1,16 @@ +# /etc/mxq/powerd-secrets -- IPMI/BMC credentials for mxq-powerd. +# +# Install as root, mode 0600: +# install -m 0600 -o root -g root powerd-secrets.example /etc/mxq/powerd-secrets +# +# Kept out of the world-readable nodes.conf on purpose. The password is passed +# to ipmitool via $IPMI_PASSWORD (ipmitool -E), never on the command line. + +[ipmi] +user = ADMIN +password = changeme + +# Optional per-host overrides (section name is ipmi. ): +#[ipmi.node001] +#user = root +#password = another-secret diff --git a/powerd/powerd.conf.example b/powerd/powerd.conf.example new file mode 100644 index 00000000..7bb061a9 --- /dev/null +++ b/powerd/powerd.conf.example @@ -0,0 +1,52 @@ +# /etc/mxq/powerd.conf -- global policy for mxq-powerd. +# +# Copy to /etc/mxq/powerd.conf and edit. All keys shown are the built-in +# defaults; you only need to list the ones you change. + +[db] +# How mxq-powerd reaches the MySQL server. It only ever runs SELECTs. +# +# By default it reuses MXQ's read-only client config file and extracts the +# connection options from the named group with `my_print_defaults` (the +# `mysql` client itself only reads the [client]/[mysql] groups). A dedicated +# read-only account is recommended. +mysql = mysql +my_print_defaults = my_print_defaults +defaults_file = /etc/mxq/mysql_ro.cnf +defaults_group = mxqclient +database = mxq +# To use a file whose credentials live under a plain [client] section instead, +# leave defaults_group empty: +# defaults_group = + +[policy] +suspend_time = 1800 # sec a node must be idle before it may power down +warm_pool = 4 # min idle managed nodes to keep powered ON +min_uptime = 900 # sec a node must be up before it is suspend-eligible +suspend_rate = 4 # max power-downs initiated per tick +resume_rate = 8 # max power-ups initiated per tick +suspend_timeout = 300 # sec for a shutdown to complete before escalation +resume_timeout = 600 # sec for a boot+register before the node is FAILED +mtime_stale = 180 # sec: mxq_daemon.mtime older than this => not alive +# Hostnames never managed (login/storage/infra). Belt-and-suspenders: a node +# is also protected by gen '-' or on_method 'none' in nodes.conf. +exclude = login01,storage0,storage1 + +[mode] +# v1 default: log every decision, take NO action. Run for a few days against +# a live cluster and eyeball the log before flipping this to false. +dry_run = true + +[paths] +nodes_conf = /etc/mxq/nodes.conf +secrets_file = /etc/mxq/powerd-secrets +state_file = /var/lib/mxq/powerd-state.json +log_file = /var/log/mxq/powerd.log + +[actuation] +ssh = ssh +ssh_opts = -o BatchMode=yes -o ConnectTimeout=10 +ipmitool = ipmitool +wakeonlan = wakeonlan +drain_cmd = mxqdctl-hostconfig stop +poweroff_cmd = systemctl poweroff From b3d008920b83f1e257458a95bd3b9204d61519b9 Mon Sep 17 00:00:00 2001 From: Paul Menzel Date: Mon, 20 Jul 2026 09:50:45 +0200 Subject: [PATCH 4/4] mxq-powerd: match jobs to hosts the way MXQ does (PR #173 review) Reviewer feedback (@donald) on the RFC: the job/host match was wrong. It tested group tags as a subset of node tags, but MXQ (server_is_qualified() in mxqd_control.c) does something quite different, and the host white/blacklist was missing entirely. Reproduce the real model for powered-off nodes: * group_whitelist / group_blacklist gate on the hostname; * the group's `prerequisites` EXPRESSION is evaluated against the host's tags, and the host's own `prerequisites` against the group's tags; * a GPU group needs a GPU host. - Add a small evaluator mirroring parser.y (tags as identifiers, `! & |`, parens, precedence ! > & > |; empty => true, syntax error => false). - Pull tags, prerequisites, group_whitelist and group_blacklist per pending group; tokenise tag/white/blacklist strings on whitespace like keywordset_new(). - Give nodes.conf a `prerequisites` column and treat each node's tags as including the implicit `true`, hostname and short hostname that mxqd adds. - Fold qualification + a capacity check into _host_satisfies()/_alive_can_serve(). Also document the operational points raised in review: enable the timer on a single management node, mxqd must start at boot for resume to work, use an ssh key restricted to the drain/poweroff commands, and the free-tmpdir blind spot. README, man page and design doc updated to match. Co-Authored-By: Claude Opus 4.8 --- docs/power-management-implementation.md | 53 ++++-- manpages/mxq-powerd.8 | 35 ++++ powerd/README.md | 24 +++ powerd/mxq-powerd | 211 +++++++++++++++++++++--- powerd/mxq-powerd.timer | 6 + powerd/nodes.conf.example | 42 +++-- 6 files changed, 326 insertions(+), 45 deletions(-) diff --git a/docs/power-management-implementation.md b/docs/power-management-implementation.md index 6b15cb31..fe5df8e8 100644 --- a/docs/power-management-implementation.md +++ b/docs/power-management-implementation.md @@ -45,23 +45,38 @@ Flat, `#`-comment, whitespace-separated, matching the `/etc/hostconfig` house st One line per **host** (not per daemon). Columns: ``` -# hostname gen slots mem_mb gpus tags on_method bmc_or_mac -node001 g1 64 257000 0 xeon,highmem ipmi 10.1.0.11 -node002 g1 64 257000 0 xeon,highmem ipmi 10.1.0.12 -node101 g3 256 980000 4 epyc,gpu,a100 wol e4:3d:1a:00:11:22 -login01 - - - - infra none - +# hostname gen slots mem_mb gpus tags prerequisites on_method bmc_or_mac +node001 g1 64 257000 0 xeon,highmem - ipmi 10.1.0.11 +node002 g1 64 257000 0 xeon,highmem - ipmi 10.1.0.12 +node101 g3 256 980000 4 epyc,gpu,a100 - wol e4:3d:1a:00:11:22 +login01 - - - - infra - none - ``` - `gen` — generation / **power-priority** token. Suspend picks **highest `gen` numbers (oldest, least efficient) first**; resume picks **lowest (newest, most efficient) first**. This is the MXQ analog of Slurm node `Weight`. `-` ⇒ never auto-managed. -- `slots`/`mem_mb`/`gpus`/`tags` — capability of a **powered-off** node (the live DB - can't tell us this once it's dark; §5 of the study). Used by resume to match waiting - jobs' requirements (`job_threads`, `job_memory`, `job_gpu`, group `tags`). +- `slots`/`mem_mb`/`gpus`/`tags`/`prerequisites` — capability and match attributes of a + **powered-off** node (the live DB can't tell us this once it's dark; §5 of the study). + `tags` and `prerequisites` mirror what the host's `mxqd` publishes and let resume + reproduce MXQ's job/host match (see below); `mxqd` implicitly adds `true`, the hostname + and the short hostname, so those need not be listed. Both take `-` for none. - `on_method` ∈ `ipmi | wol | none`. `none` ⇒ excluded from all automation (login/storage/infra). - `bmc_or_mac` — BMC IP/hostname for `ipmi`, MAC for `wol`. +**Matching model.** Deciding whether a powered-off node could accept a pending group +mirrors `server_is_qualified()` in `mxqd_control.c` — it is **not** a tag-set +intersection: + +- the group's `group_whitelist` / `group_blacklist` gate on the hostname; +- the group's `prerequisites` **expression** is evaluated against the **host's** tags; +- the host's own `prerequisites` expression is evaluated against the **group's** tags; +- a GPU group needs a GPU host. + +The expression grammar (`parser.y`: identifiers as tags, operators `!` `&` `|`, parens, +precedence `! > & > |`) is reproduced in the manager. On top of qualification, resume adds +a capacity check (the node is big enough for at least one of the group's jobs). + IPMI credentials are **not** in this file. They live in `/etc/mxq/powerd-secrets` (`0600`, root) — a global `user`/`password`, overridable per host. Keeping secrets out of the world-readable inventory is deliberate. @@ -103,8 +118,9 @@ FROM mxq_group WHERE group_jobs_inq > 0 AND (group_flags & 1) = 0; -- ignore CLOSED groups ``` -Per-group rows (with `job_threads,job_memory,job_gpu,job_time,tags`) are also pulled so -resume can match specific constraints, not just totals. +Per-group rows (with `job_threads,job_memory,job_gpu,tags,prerequisites,group_whitelist, +group_blacklist`) are also pulled so resume can reproduce the full job/host match +(§2, *Matching model*), not just core totals. **B. Live capacity & liveness** (what's on, and free): ```sql @@ -245,6 +261,20 @@ ipmitool … chassis power off # hard, las `systemctl stop mxq-powerd.timer` to disable globally. (v2: `mxqadmin --poweron/ --poweroff/--drain`, study §6.) +**Operational prerequisites** (raised in review): + +- **Single instance.** The timer must be enabled on **exactly one** management node + (e.g. `afk`); the manager takes no cluster-wide lock, so two instances would fight. +- **`mxqd` must start at boot.** Resume relies on a booted node bringing `mxqd` up so it + re-registers; MXQ does **not** currently start `mxqd` automatically after boot, so that + must be arranged first — otherwise woken nodes never become usable and trip + `resume_timeout` → FAILED. +- **Restricted ssh key.** The management node needs a dedicated key whose + `authorized_keys` entry is locked (`command="…",restrict`) to only `mxqdctl-hostconfig + stop` and the power-off command. +- **tmpdir blind spot.** MXQ cannot see free per-node tmpdir (`/scratch/local2`) space and + users can consume it outside `mxqd`'s control, so it cannot be a wake/keep criterion. + ## 9. systemd units ```ini @@ -300,6 +330,9 @@ Build order and how to prove each step **end-to-end**, not just unit tests: - Confirm the infra/never-suspend set and initial `suspend_time` / `warm_pool`. - OK to add a read-only `[mxqpowerd]` MySQL account/config group? - Python 3 acceptable, or prefer bash to match `mxqdctl-hostconfig.sh`? +- **Arrange `mxqd`-at-boot** (see §8) — a hard dependency for resume, currently missing. +- **Pick the single management node** the timer runs on, and provision its restricted ssh + key (§8). ## References - Slurm Power Saving Guide — https://slurm.schedmd.com/power_save.html diff --git a/manpages/mxq-powerd.8 b/manpages/mxq-powerd.8 index fe64131e..8ef6b129 100644 --- a/manpages/mxq-powerd.8 +++ b/manpages/mxq-powerd.8 @@ -103,6 +103,41 @@ lands in is excluded, logged, and left for an operator rather than retried blindly. Clear it with .BR "mxq-powerd \-\-clear " \fIHOST\fR . +.SH CAVEATS +.B mxq-powerd +takes no cluster-wide lock, so its timer must be enabled on +.B exactly one +management node. +.PP +Resume assumes that powering a node on brings its +.BR mxqd (8) +back up so it re-registers; the daemon must therefore be started at boot. +.PP +The drain and power-off actuators use +.BR ssh (1); +give the manager a key whose +.I authorized_keys +entry is restricted +.RB ( command= "...) to only " "mxqdctl-hostconfig stop" +and the power-off command. +.PP +Free per-node tmpdir space (for example +.IR /scratch/local2 ) +is not visible to MXQ and can be consumed outside +.BR mxqd 's +control, so it cannot be used as a wake or keep criterion. +.PP +Matching mirrors +.B server_is_qualified +in +.IR mxqd_control.c : +a group's +.I prerequisites +expression is evaluated against a host's tags, the host's own +.I prerequisites +against the group's tags, and the group white/blacklist against the hostname. +The per-host inventory must carry each node's tags and prerequisites so this +match can be reproduced while the node is powered off. .SH EXIT STATUS 0 on success, non-zero on a fatal configuration or database error. .SH SEE ALSO diff --git a/powerd/README.md b/powerd/README.md index 01e1b295..fbf7246f 100644 --- a/powerd/README.md +++ b/powerd/README.md @@ -37,6 +37,30 @@ systemctl enable --now mxq-powerd.timer The systemd units are provided as examples under `powerd/`; copy them to `/etc/systemd/system/` and `systemctl daemon-reload`. +## Operational prerequisites + +- **Enable the timer on exactly one node.** `mxq-powerd` takes no cluster-wide + lock (the flock only serialises ticks on one host), so two instances would + fight over the same nodes. Pick a single management node (e.g. `afk`). +- **`mxqd` must start at boot.** Resume assumes that powering a node on brings + its `mxqd` up so it registers in `mxq_daemon` again; otherwise a woken node + never becomes usable and eventually trips `resume_timeout` → `FAILED`. MXQ + today does *not* start `mxqd` automatically after boot — that has to be + arranged before arming. +- **Restrict the ssh key.** The drain/poweroff actuators ssh into each node. + Give the manager a dedicated key whose `authorized_keys` entry is locked to + just those commands, e.g.: + + ``` + command="mxqdctl-hostconfig stop",restrict ssh-ed25519 AAAA... mxq-powerd-drain + ``` + + (or a small wrapper that permits only `mxqdctl-hostconfig stop` and + `systemctl poweroff`). Point `ssh_opts`/keys in `powerd.conf` at it. +- **tmpdir is a blind spot.** MXQ cannot see free `/scratch/local2` space, and + users can consume it outside `mxqd`'s control. `mxq-powerd` inherits that + limitation: it cannot use free tmpdir as a wake/keep criterion. + ## Runbook ```sh diff --git a/powerd/mxq-powerd b/powerd/mxq-powerd index da393ce2..209dbeaf 100755 --- a/powerd/mxq-powerd +++ b/powerd/mxq-powerd @@ -53,6 +53,128 @@ STATE_VERSION = 1 log = logging.getLogger("mxq-powerd") +# =========================================================================== +# Prerequisite expression evaluation +# +# MXQ matches a job group to a host with two boolean expressions plus a +# host white/blacklist -- see server_is_qualified() in mxqd_control.c: +# +# * group.group_whitelist / group.group_blacklist gate on the hostname; +# * group.prerequisites is an expression evaluated against the HOST's tags; +# * the host daemon's prerequisites is an expression evaluated against the +# GROUP's tags; +# * a GPU group needs a GPU host. +# +# It is NOT a tag-set-vs-tag-set intersection. The expression grammar +# (parser.y) is: identifiers -- a tag, true iff present in the tag set -- with +# the operators `!` `&` `|` and parentheses, precedence ! > & > |. An empty +# expression means "always true"; a syntactically invalid one makes yyparse() +# fail, which server_is_qualified() treats as "not qualified" (i.e. false). +# =========================================================================== + +class _ExprError(Exception): + pass + + +def _tokenize_expr(expr): + toks = [] + i, n = 0, len(expr) + while i < n: + c = expr[i] + if c in " \t": + i += 1 + elif c in "&|!()": + toks.append(c) + i += 1 + elif c.isalpha(): + j = i + 1 + while j < n and (expr[j].isalnum() or expr[j] == "_"): + j += 1 + toks.append(("TAG", expr[i:j])) + i = j + else: + # In parser.y any other character is its own token, which the + # grammar then rejects as a syntax error. + raise _ExprError("unexpected character %r" % c) + return toks + + +class _ExprParser: + """Recursive-descent evaluator mirroring parser.y (! > & > |).""" + + def __init__(self, toks, tags): + self.toks = toks + self.pos = 0 + self.tags = tags + + def _peek(self): + return self.toks[self.pos] if self.pos < len(self.toks) else None + + def parse(self): + val = self._or() + if self.pos != len(self.toks): + raise _ExprError("trailing tokens") + return val + + def _or(self): + val = self._and() + while self._peek() == "|": + self.pos += 1 + rhs = self._and() + val = bool(val or rhs) + return val + + def _and(self): + val = self._not() + while self._peek() == "&": + self.pos += 1 + rhs = self._not() + val = bool(val and rhs) + return val + + def _not(self): + if self._peek() == "!": + self.pos += 1 + return not self._not() + return self._atom() + + def _atom(self): + tok = self._peek() + if tok == "(": + self.pos += 1 + val = self._or() + if self._peek() != ")": + raise _ExprError("missing )") + self.pos += 1 + return val + if isinstance(tok, tuple) and tok[0] == "TAG": + self.pos += 1 + return tok[1] in self.tags + raise _ExprError("unexpected token %r" % (tok,)) + + +def eval_prerequisites(expr, tags): + """Evaluate an MXQ prerequisites expression against a set of tags. + + Empty expression -> True. A syntax error -> False (matching a failed + yyparse() in server_is_qualified()). + """ + expr = (expr or "").strip() + if not expr: + return True + try: + return bool(_ExprParser(_tokenize_expr(expr), tags).parse()) + except _ExprError as exc: + log.warning("invalid prerequisites expression %r: %s", expr, exc) + return False + + +def keyword_set(raw): + """Split a tag/white/blacklist string into a set the way keywordset_new() + does -- on any whitespace (see keywordset.c).""" + return {t for t in (raw or "").split() if t} + + # =========================================================================== # Configuration # =========================================================================== @@ -133,13 +255,14 @@ class Node: """One managed host, from the static inventory.""" def __init__(self, hostname, gen, slots, mem_mb, gpus, tags, - on_method, bmc_or_mac): + prerequisites, on_method, bmc_or_mac): self.hostname = hostname self.gen = gen # None => never auto-managed self.slots = slots # inventoried capacity (node powered off) self.mem_mb = mem_mb self.gpus = gpus - self.tags = tags # set of strings + self.tags = tags # set of strings (hostconfig/cpufeatures) + self.prerequisites = prerequisites # host's own mxqd --prerequisites expr self.on_method = on_method # 'ipmi' | 'wol' | 'none' self.bmc_or_mac = bmc_or_mac @@ -147,6 +270,18 @@ class Node: def managed(self): return self.gen is not None and self.on_method in ("ipmi", "wol") + @property + def hostname_short(self): + return self.hostname.split(".", 1)[0] + + @property + def match_tags(self): + """Tag set a booted mxqd would expose for this host: the inventoried + tags plus the implicit `true`, hostname and short hostname that mxqd.c + always adds. A group's prerequisites expression is matched against + this set.""" + return set(self.tags) | {"true", self.hostname, self.hostname_short} + def _parse_int(tok): if tok in ("-", ""): @@ -157,7 +292,14 @@ def _parse_int(tok): def load_inventory(path): """Parse the whitespace-separated /etc/mxq/nodes.conf. - Columns: hostname gen slots mem_mb gpus tags on_method bmc_or_mac + Columns: hostname gen slots mem_mb gpus tags prerequisites on_method bmc_or_mac + + `tags` and `prerequisites` mirror what this host's mxqd publishes (its + hostconfig/cpufeature tags and its --prerequisites expression); they let + the manager reproduce MXQ's job/host match while the node is powered off. + Both take `-` for "none". Because columns are whitespace-separated, the + tags column is comma-separated internally and the prerequisites expression + must be written without spaces (e.g. `gpu&!maintenance`). A `-` in `gen` (or on_method `none`) marks a host as never auto-managed. """ nodes = {} @@ -169,13 +311,15 @@ def load_inventory(path): if not line: continue cols = line.split() - if len(cols) < 8: - log.warning("nodes.conf:%d: expected 8 columns, got %d -- skipped", + if len(cols) < 9: + log.warning("nodes.conf:%d: expected 9 columns, got %d -- skipped", lineno, len(cols)) continue - hostname, gen, slots, mem_mb, gpus, tags, on_method, bmc_or_mac = cols[:8] + (hostname, gen, slots, mem_mb, gpus, tags, prereq, + on_method, bmc_or_mac) = cols[:9] gen_val = None if gen == "-" else gen tag_set = {t for t in tags.split(",") if t and t != "-"} + prereq_val = "" if prereq == "-" else prereq try: node = Node( hostname=hostname, @@ -184,6 +328,7 @@ def load_inventory(path): mem_mb=_parse_int(mem_mb) or 0, gpus=_parse_int(gpus) or 0, tags=tag_set, + prerequisites=prereq_val, on_method=on_method, bmc_or_mac=bmc_or_mac, ) @@ -379,12 +524,16 @@ def _to_int(val, default=0): # =========================================================================== class PendingGroup: - def __init__(self, jobs_inq, threads, memory_mb, gpu, tags): + def __init__(self, jobs_inq, threads, memory_mb, gpu, tags, + prerequisites, whitelist, blacklist): self.jobs_inq = jobs_inq self.threads = threads self.memory_mb = memory_mb self.gpu = gpu - self.tags = tags + self.tags = tags # group tags (matched by host prereq) + self.prerequisites = prerequisites # expr matched against host tags + self.whitelist = whitelist # allowed hostnames (set); empty=any + self.blacklist = blacklist # denied hostnames (set) class LiveHost: @@ -426,17 +575,19 @@ class Snapshot: self.max_job_mem_mb = _to_int(agg[2]) self.max_job_gpu = _to_int(agg[3]) - # per-group rows so resume can match specific constraints + # per-group rows so resume can reproduce MXQ's job/host match self.pending_groups = [] for r in db.query( - "SELECT group_jobs_inq, job_threads, job_memory, job_gpu, tags " + "SELECT group_jobs_inq, job_threads, job_memory, job_gpu, " + "tags, prerequisites, group_whitelist, group_blacklist " "FROM mxq_group " "WHERE group_jobs_inq > 0 AND (group_flags & %d) = 0" % MXQ_GROUP_FLAG_CLOSED): - tags = {t for t in (r[4] or "").split(",") if t} self.pending_groups.append(PendingGroup( jobs_inq=_to_int(r[0]), threads=_to_int(r[1]), - memory_mb=_to_int(r[2]), gpu=_to_int(r[3]), tags=tags)) + memory_mb=_to_int(r[2]), gpu=_to_int(r[3]), + tags=keyword_set(r[4]), prerequisites=(r[5] or ""), + whitelist=keyword_set(r[6]), blacklist=keyword_set(r[7]))) # -- B. live capacity & liveness -------------------------------------- @@ -797,15 +948,37 @@ class Manager: n += 1 return n + def _qualified(self, node, group): + """Reproduce server_is_qualified() (mxqd_control.c): may a job from + `group` ever run on `node`? Uses the node's declared tags and + prerequisites from the inventory (a powered-off node cannot report its + own), which the operator keeps in step with the host's mxqd config.""" + # group white/blacklist gate on the hostname (short or full) + names = {node.hostname, node.hostname_short} + if group.whitelist and names.isdisjoint(group.whitelist): + return False + if group.blacklist and not names.isdisjoint(group.blacklist): + return False + # group's prerequisites expression, evaluated against the host's tags + if not eval_prerequisites(group.prerequisites, node.match_tags): + return False + # host's own prerequisites expression, evaluated against group tags + if not eval_prerequisites(node.prerequisites, group.tags): + return False + # a GPU group needs a GPU host (job_gpu && !daemon.gpus_max) + if group.gpu and node.gpus <= 0: + return False + return True + def _host_satisfies(self, node, group): + """Is this powered-off node worth waking for `group`? Qualification + (above) plus a capacity check that it can hold at least one job.""" + if not self._qualified(node, group): + return False if node.slots < group.threads: return False if node.mem_mb and group.memory_mb and node.mem_mb < group.memory_mb: return False - if node.gpus < group.gpu: - return False - if group.tags and not group.tags.issubset(node.tags): - return False return True def _alive_can_serve(self, group): @@ -816,12 +989,10 @@ class Manager: live = self.snap.live[node.hostname] if live.slots_free < group.threads: continue - if node.gpus < group.gpu: - continue - if group.tags and not group.tags.issubset(node.tags): - continue if node.mem_mb and group.memory_mb and node.mem_mb < group.memory_mb: continue + if not self._qualified(node, group): + continue return True return False diff --git a/powerd/mxq-powerd.timer b/powerd/mxq-powerd.timer index 36696211..830ca506 100644 --- a/powerd/mxq-powerd.timer +++ b/powerd/mxq-powerd.timer @@ -2,6 +2,12 @@ # # Enable with: systemctl enable --now mxq-powerd.timer # Disable power management globally with: systemctl stop mxq-powerd.timer +# +# IMPORTANT: enable this timer on EXACTLY ONE management node (e.g. afk). +# mxq-powerd holds no cross-node lock; two instances would fight over the same +# nodes. That node needs an ssh key/account whose authorized_keys entry is +# restricted (command="...") to just the drain and poweroff commands the +# manager runs -- see powerd/README.md. [Unit] Description=Run MXQ power manager every minute diff --git a/powerd/nodes.conf.example b/powerd/nodes.conf.example index ce568c4d..f81d2c2a 100644 --- a/powerd/nodes.conf.example +++ b/powerd/nodes.conf.example @@ -5,21 +5,33 @@ # can do -- the live DB cannot tell us once the node is dark. # # Columns: -# hostname the short hostname as it appears in mxq_daemon.hostname -# gen generation / power-priority token. Suspend picks the HIGHEST -# gen (oldest, least efficient) first; resume the LOWEST -# (newest) first. '-' => never auto-managed. -# slots CPU slots the node offers -# mem_mb memory in MB -# gpus number of GPUs -# tags comma-separated capability tags ('-' for none) -# on_method ipmi | wol | none ('none' => excluded from all automation) -# bmc_or_mac BMC IP/hostname for ipmi, MAC address for wol ('-' if none) +# hostname the short hostname as it appears in mxq_daemon.hostname +# gen generation / power-priority token. Suspend picks the +# HIGHEST gen (oldest, least efficient) first; resume the +# LOWEST (newest) first. '-' => never auto-managed. +# slots CPU slots the node offers +# mem_mb memory in MB +# gpus number of GPUs +# tags comma-separated capability tags ('-' for none). Mirror the +# hostconfig/cpufeature tags this host's mxqd publishes; a +# job group's `prerequisites` expression is matched against +# them. (mxqd also adds `true`, the hostname and short +# hostname implicitly -- do not list those.) +# prerequisites the host's own mxqd --prerequisites expression, matched +# against a group's tags, or '-' for none. Because columns +# are whitespace-separated, write it without spaces, e.g. +# `gpu&!maintenance`. +# on_method ipmi | wol | none ('none' => excluded from all automation) +# bmc_or_mac BMC IP/hostname for ipmi, MAC address for wol ('-' if none) +# +# `tags`/`prerequisites` let mxq-powerd reproduce MXQ's job/host match (see +# server_is_qualified() in mxqd_control.c) while a node is off, so it only +# wakes a node that could actually accept the pending work. # # IPMI credentials do NOT go here -- see /etc/mxq/powerd-secrets. -# hostname gen slots mem_mb gpus tags on_method bmc_or_mac -node001 g1 64 257000 0 xeon,highmem ipmi 10.1.0.11 -node002 g1 64 257000 0 xeon,highmem ipmi 10.1.0.12 -node101 g3 256 980000 4 epyc,gpu,a100 wol e4:3d:1a:00:11:22 -login01 - - - - infra none - +# hostname gen slots mem_mb gpus tags prerequisites on_method bmc_or_mac +node001 g1 64 257000 0 xeon,highmem - ipmi 10.1.0.11 +node002 g1 64 257000 0 xeon,highmem - ipmi 10.1.0.12 +node101 g3 256 980000 4 epyc,gpu,a100 - wol e4:3d:1a:00:11:22 +login01 - - - - infra - none -