diff --git a/Makefile b/Makefile index ccd85f87..253fcb00 100644 --- a/Makefile +++ b/Makefile @@ -781,3 +781,20 @@ clean: CLEAN += test_parser.o test_parser: test_parser.o parser.tab.o keywordset.o test_parser.o: parser.tab.h keywordset.h clean: CLEAN += parser.tab.o + +######################################################################## +### mxq-powerd -- node power manager (opt-in; not part of `make install`) + +MAN8DIR := ${MANDIR}/man8 + +.PHONY: install-powerd +install-powerd: powerd/mxq-powerd manpages/mxq-powerd.8 + $(call quiet-installdir,0755,${DESTDIR}${LIBEXECDIR}/mxq) + $(call quiet-installdir,0755,${DESTDIR}${SYSCONFDIR}/mxq) + $(call quiet-installdir,0755,${DESTDIR}${MAN8DIR}) + $(call quiet-install,0755,powerd/mxq-powerd,${DESTDIR}${LIBEXECDIR}/mxq/mxq-powerd) + $(call quiet-install,0644,manpages/mxq-powerd.8,${DESTDIR}${MAN8DIR}/mxq-powerd.8) + $(call quiet-install,0644,powerd/powerd.conf.example,${DESTDIR}${SYSCONFDIR}/mxq/powerd.conf.example) + $(call quiet-install,0644,powerd/nodes.conf.example,${DESTDIR}${SYSCONFDIR}/mxq/nodes.conf.example) + $(call quiet-install,0600,powerd/powerd-secrets.example,${DESTDIR}${SYSCONFDIR}/mxq/powerd-secrets.example) + @echo " NOTE copy powerd/mxq-powerd.{service,timer} to /etc/systemd/system/ to enable the timer" diff --git a/docs/power-management-design.md b/docs/power-management-design.md new file mode 100644 index 00000000..a972cf77 --- /dev/null +++ b/docs/power-management-design.md @@ -0,0 +1,225 @@ +# Powering off idle MXQ nodes to save energy — design notes + +Status: **design exploration** (no code written yet). Feasibility study requested by +Paul Menzel, 2026-07-18. + +## 1. The question + +Can MXQ power down compute nodes when the cluster is idle and power them back on +when work arrives, to save energy? If so, what has to be built? + +Short answer: **yes, and mostly as a new, self-contained central component** — MXQ +does not need to be rewritten. But the piece that Slurm gets "for free" (a central +controller that already knows and commands every node) does **not exist in MXQ** and +has to be created. Paul's instinct in the request is correct: an independent +timer/cron that (a) picks idle nodes to switch off, oldest generations first, and +(b) a second loop that watches the queue and switches nodes back on. + +## 2. How MXQ works today (the relevant parts) + +MXQ is a **pull-based** scheduler. There is a central MySQL database and autonomous +per-node daemons; nothing ever *commands* a node. + +- **Three tables** (`mysql/create_tables.sql`): + - `mxq_group` — one row per array of identical jobs. Carries the per-job resource + requirements (`job_threads`, `job_memory`, `job_time`, `job_gpu`, `job_tmpdir_size`) + and trigger-maintained counters, notably `group_jobs_inq` (pending) and + `group_jobs_running`. + - `mxq_job` — individual jobs; placement columns `daemon_id`, `host_hostname`, + `job_status`, and timestamps `date_start` / `date_end`. + - `mxq_daemon` — **the per-node table**. One row per *daemon instance* (a fresh row + is INSERTed every time a daemon starts). Holds `hostname`, capacity + (`daemon_slots`, `daemon_memory`, `daemon_gpus_max`), live load + (`daemon_slots_running`, `daemon_memory_used`, …), `status`, `tags`, + `prerequisites`, and timestamps `mtime` / `daemon_start` / `daemon_stop`. + +- **The daemon (`mxqd`)** — `mxqd.c`, `mxq_daemon.c`: + - On start: `mxq_daemon_register()` INSERTs its row and gets an auto-increment + `daemon_id` (`mxq_daemon.c:48`). Stable *logical* node identity is the pair + `(hostname, daemon_name)`. + - Main loop (`mxqd.c:2715`): `sigtimedwait()` with a **20 s** poll interval. Each + pass it reads the global queue via `mxq_load_running_groups()` + (`SELECT … FROM mxq_group WHERE group_jobs_inq>0 OR group_jobs_running>0`, + `mxq_group.c:292`), then race-safely claims INQ jobs + (`UPDATE mxq_job … WHERE job_id=? AND job_status=INQ`, `mxq_job.c:311`). + - **Idle signal**: `update_status()` sets `mxq_daemon.status = IDLE (0)` exactly when + `slots_running == 0` (`mxqd.c:2595`). Status enum in `mxq_daemon.h:9`: + IDLE=0, RUNNING=10, WAITING=20, FULL=30, BACKFILL=40, CPUOPTIMAL=50, + TERMINATING=200, EXITED=250, CRASHED=255. + - **Liveness / heartbeat**: `mtime` is refreshed on every stats/status write, and at + least every 5 min even when idle (rate-limited in `mxq_daemon.c:235`). There is + **no fast heartbeat and no central watchdog**. A crashed node is only flagged + CRASHED when a *replacement* daemon starts (`recover_from_previous_crash`, + `mxqd.c:2487`). + - **Clean shutdown**: SIGTERM exits the main loop and enters a **drain loop** + (`mxqd.c:2762`) that starts no new jobs and waits for running jobs to finish, then + `mxq_daemon_shutdown()` sets `status=EXITED, daemon_stop=NOW()` (`mxq_daemon.c:151`). + SIGINT additionally kills running jobs; SIGQUIT/SIGUSR1 leave jobs running. + +- **Host bring-up today** — `mxqdctl-hostconfig.sh`: reads `/etc/hostconfig`, starts + one `mxqd` per matching line, and controls local daemons via signals. It **assumes + the host is already powered on**. There is no PXE/IPMI/WoL hook anywhere. + +- **Admin surface** — `mxqadmin.c` only closes/reopens groups. There is **no** + drain-node / remove-daemon / power command. No cron, systemd unit, or scheduler + callback exists in the repo (the recent `--callback` is a *per-job*, runs-as-user, + on-node hook — not suitable for privileged node power control). + +### What this gives us for free + +Everything a power decision needs is already observable in the DB *without touching +the nodes*: + +| Question | Where the answer is | +|---|---| +| Is there pending work, and how much? | `SUM(group_jobs_inq * job_threads)`, `job_memory`, `job_gpu`, `tags`/`prerequisites` over `mxq_group WHERE group_jobs_inq>0` | +| Is node H idle right now? | `mxq_daemon.status=0` and `daemon_slots_running=0`; cross-check `mxq_job` has no rows for H in status (100,150,200) | +| Is node H alive? | freshness of `mxq_daemon.mtime` (stale ⇒ gone) | +| How long has H been idle? | **not stored** — infer from `NOW() - MAX(mxq_job.date_end)` for H, else from `daemon_start` | +| What can H do (once it's off)? | last `mxq_daemon` row's capacity columns, but see §5 — better kept in a static inventory | + +## 3. Why we can't just copy Slurm + +Slurm's `slurmctld` is a central controller that already tracks every node and issues +commands to them; power-save is just two site scripts it invokes — +`SuspendProgram` (node idle > `SuspendTime` ⇒ power down) and `ResumeProgram` +(queued work needs a node ⇒ power up), rate-limited by `SuspendRate`/`ResumeRate`, +with `SuspendExcNodes` protecting infrastructure. Power-*on* is done out-of-band via +**IPMI `chassis power on`** or **Wake-on-LAN**; power-*off* by an ordered shutdown. + +MXQ has no `slurmctld`. So the equivalent of "the controller that decides and acts" +is exactly the new component we must add. The good news: because nodes *pull*, the +resume path is trivial — we only have to get the box powered on; `mxqd` then boots, +registers, and starts pulling with **zero scheduler involvement**. + +## 4. Proposed design — a central "power manager" + +A single new component on the management host (where `mxqdump`/`mxqadmin` already +run), driven by a **systemd timer or cron** every 1–2 min. It never runs on the +compute nodes. Two independent decision loops (they can be one script): + +``` + ┌─────────────────── management host ───────────────────┐ + │ mxq-powerd (cron/systemd timer, every ~1–2 min) │ + │ │ + reads │ ┌── suspend loop ──┐ ┌── resume loop ──┐ │ + ◄────────┤ │ idle > T & safe? │ │ pending demand │ │ + MySQL │ │ → drain+off │ │ > free capacity?│ │ + (mxq_*) │ └────────┬─────────┘ └────────┬────────┘ │ + │ │ ssh: mxqdctl stop; poweroff│ IPMI power on / │ + └───────────┼────────────────────────────┼─WoL────────────┘ + ▼ ▼ + compute node (in-band, compute node (out-of-band, + graceful, deregisters) node is off → BMC/WoL only) +``` + +### 4a. Suspend loop (power **off**) + +1. **Find safe idle candidates.** Per hostname (aggregating *all* live daemon rows, + since a host may run several daemons): every daemon `status=IDLE(0)` and + `daemon_slots_running=0`, `mtime` fresh (node alive), and no `mxq_job` rows for the + host in status (100,150,200). +2. **Idle-long-enough gate** (`SuspendTime`, e.g. 30 min): `NOW() - MAX(date_end)` + for the host exceeds the threshold (fallback `daemon_start` if it never ran a job). + This is the one signal MXQ doesn't store directly — see §6 for an optional column. +3. **Policy gates:** exclude-list (login/storage/infra nodes — the `SuspendExcNodes` + analog); keep a **minimum warm pool** of idle nodes on so short bursts don't wait + for a boot; rate-limit (`SuspendRate`) to avoid a sudden power drop. +4. **Ordering: oldest generations first.** Rank candidates by a per-node + power-priority from the inventory (§5) so the least efficient hardware powers off + first. +5. **Graceful power-off (in-band):** for each chosen node + `ssh HOST 'mxqdctl-hostconfig stop'` (SIGTERM ⇒ stops claiming, drains, sets + `status=EXITED`), **wait until its `mxq_daemon` row shows EXITED**, then + `ssh HOST 'systemctl poweroff'`. Waiting for EXITED closes the claim race: the + daemon only marks EXITED *after* draining, so once EXITED no job can be running or + half-claimed on that node. In-band shutdown (not IPMI power-off) lets the daemon + deregister cleanly — no stale RUNNING rows to reconcile. + +### 4b. Resume loop (power **on**) + +1. **Pending demand** from `mxq_group WHERE group_jobs_inq>0`: total pending cores + `SUM(group_jobs_inq*job_threads)`, plus memory / GPU / time / tag & prerequisite + constraints. +2. **Free capacity** of nodes already on: + `SUM(daemon_slots - daemon_slots_running)` over live daemons (respecting the same + tag/GPU/memory constraints). +3. **Deficit ⇒ power on.** Also power on when a *specific* pending job can only run on + a class of node that is currently all-off (e.g. GPU or high-memory / a required + tag), even if raw core capacity looks sufficient. +4. **Ordering: newest / most efficient first** (inverse of suspend), and pick nodes + whose inventoried capabilities actually satisfy the waiting jobs. +5. **Power on out-of-band:** IPMI `chassis power on` (preferred, reliable) or a WoL + magic packet. Rate-limit (`ResumeRate`) to avoid inrush current. +6. **Boot ⇒ self-heal:** the node's init/systemd runs `mxqdctl-hostconfig.sh`, `mxqd` + registers a fresh `mxq_daemon` row and starts pulling. Nothing else to do. +7. **Timeout:** if no fresh `mxq_daemon` row appears within `ResumeTimeout` + (e.g. 10 min), flag the node as failed, exclude it, and try the next candidate. + +### 4c. Anti-thrash + +Boot takes minutes, so hysteresis matters: generous `SuspendTime`, a warm pool, +independent rate limits both directions, and never suspend below the warm-pool floor. + +## 5. The one new piece of persistent state: a node inventory + +The live DB describes daemons *that are currently up*. To decide which **powered-off** +node to wake — and to reach its BMC — we need static per-node facts the DB can't give +once the node is off: + +`hostname, power_priority(generation), slots, memory, gpus, tags, bmc_address, mac` + +Recommended: a **flat site config file** (e.g. `/etc/mxq/nodes.conf`), matching the +existing `/etc/hostconfig` style, rather than a new DB table. It is simplest, keeps +site/hardware facts out of the runtime DB, and MXQ is already comfortable with flat +config + shell. A DB table (`mxq_node`) is a possible later step if the inventory +needs to be queried by other tools. + +## 6. Optional, later changes inside MXQ (not needed for v1) + +v1 can be **100% external** — no changes to `mxqd`, the schema, or `mxqadmin`. Cleaner +follow-ups if this graduates from experiment to production: + +- **A real DRAIN state.** Today "stop taking work but stay up" is only achievable by + SIGTERM (which also exits). A `daemon_flags` DRAINING bit that `mxqd` checks before + claiming (in the `mxq_load_running_groups` path) would let the manager mark a node + "don't take work", pause, and *cancel* cheaply if demand spikes — removing the + SIGTERM-then-wait dance and the residual race window entirely. +- **An `idle_since` timestamp** on `mxq_daemon`, set when `slots_running` hits 0, so + idle duration is read directly instead of inferred from `MAX(mxq_job.date_end)`. +- **`mxqadmin` verbs** `--drain HOST` / `--poweroff HOST` / `--poweron HOST`, so + operators share one tested code path with the automated manager. +- **A central watchdog** that sweeps stale `mtime` rows to CRASHED (today only a + restarting daemon does this) — useful independent of power management, and needed so + a node that fails to power on/off is noticed. + +## 7. Suggested phasing + +1. **v1 — external, read-only-ish:** the `mxq-powerd` script (suspend + resume loops) + + `nodes.conf` inventory + IPMI/WoL + `ssh poweroff`, on a timer. Start in + **dry-run/log-only** mode; verify decisions against a real idle/busy cluster before + arming actuation. No MXQ code touched. +2. **v2 — coordinate with the daemon:** add the DRAINING flag + `idle_since` + + `mxqadmin` verbs to remove the race window and simplify the manager. +3. **v3 — policy:** smarter warm-pool sizing / predictive wake based on submission + patterns (cf. the queueing-penalty research on ill-timed transitions). + +## 8. Open questions for Paul + +- Power-*on* transport available in the fleet: **IPMI/BMC**, **WoL**, both, or + something site-specific (redfish, PDU)? +- Is `ssh HOST poweroff` acceptable for power-*off*, or must that also go through the + BMC? +- Where should node generation / power-priority and BMC addresses live — a new + `/etc/mxq/nodes.conf`, or is that data already in an existing inventory/CMDB we + should read instead? +- Which nodes must **never** be suspended (login, storage, license servers)? +- Target `SuspendTime`, warm-pool size, and rate limits. + +## References + +- Slurm Power Saving Guide — https://slurm.schedmd.com/power_save.html +- OleHolmNielsen/Slurm_tools power_save (example IPMI scripts) — https://github.com/OleHolmNielsen/Slurm_tools/tree/master/power_save +- GSI Slurm power-management notes (WoL/IPMI resume) — https://web-docs.gsi.de/~vpenso/notes/posts/hpc/cluster/slurm/power-management.html +- Thinkbox Deadline "Machine Startup" (WoL/IPMI wake on submit) — https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/power-management.html +- SPARS: RL simulator for HPC power management (transition-timing penalties) — https://arxiv.org/pdf/2512.13268 diff --git a/docs/power-management-implementation.md b/docs/power-management-implementation.md new file mode 100644 index 00000000..fe5df8e8 --- /dev/null +++ b/docs/power-management-implementation.md @@ -0,0 +1,341 @@ +# MXQ node power management — implementation design (BMC/IPMI + Wake-on-LAN) + +Status: **implementation design**, ready to build v1. Companion to +`power-management-design.md` (the feasibility study / architecture rationale). Read +that first for *why*; this doc is *how*. + +## 0. Assumptions (chosen where §8 of the study was unanswered — please correct) + +1. **Both transports exist**: nodes are reachable by **IPMI/BMC** (`ipmitool -I lanplus`) + and/or **Wake-on-LAN**. Power-*on* method is selectable **per node** in the + inventory; WoL is the cheap default, IPMI the reliable one and the only option when + the node's NIC/segment can't carry a magic packet. +2. **Power-*off* is graceful and in-band** (`ssh HOST poweroff` after the daemon has + drained), with a **BMC soft-off** (`ipmitool … chassis power soft`) fallback and a + **BMC hard-off** last resort only if soft-off also times out. +3. **New static inventory** `/etc/mxq/nodes.conf` (no existing CMDB integration in v1). +4. **Language: Python 3** for the manager (readable SQL + state machine + subprocess + actuation); the actuators themselves are stock CLIs (`ipmitool`, `wakeonlan`/ + `ether-wake`, `ssh`). Bash is viable but the demand/capacity/ordering/rate-limit + logic is beyond comfortable bash. Recording this as a decision, not a hard commit. +5. **No changes to `mxqd`, the schema, or `mxqadmin` in v1** — the manager is entirely + external and read-only against the DB except it issues no writes at all. + +## 1. Components + +``` +/usr/lib/mxq/mxq-powerd the manager (Python 3, one tick per invocation) +/etc/mxq/powerd.conf global policy (thresholds, rates, DB group, exclude) +/etc/mxq/nodes.conf per-node inventory (transport, capabilities, priority) +/etc/mxq/powerd-secrets optional: IPMI credentials, mode 0600, root-only +/var/lib/mxq/powerd-state.json manager state (in-flight transitions + timestamps) +/var/log/mxq/powerd.log actions + decisions +systemd: mxq-powerd.service + .timer (or a root cron entry) — runs on the mgmt host +``` + +The manager runs **on the management host** (where `mxqdump`/`mxqadmin` already run), +as **root** (needs to `ssh` to nodes and drive BMCs), on a **fixed interval** +(default every 60 s via a systemd timer — the analog of Slurm's `power_save_interval`). +One invocation = one evaluation tick; it is safe to run back-to-back and safe to miss +a tick. + +## 2. Inventory file — `/etc/mxq/nodes.conf` + +Flat, `#`-comment, whitespace-separated, matching the `/etc/hostconfig` house style. +One line per **host** (not per daemon). Columns: + +``` +# hostname gen slots mem_mb gpus tags prerequisites on_method bmc_or_mac +node001 g1 64 257000 0 xeon,highmem - ipmi 10.1.0.11 +node002 g1 64 257000 0 xeon,highmem - ipmi 10.1.0.12 +node101 g3 256 980000 4 epyc,gpu,a100 - wol e4:3d:1a:00:11:22 +login01 - - - - infra - none - +``` + +- `gen` — generation / **power-priority** token. Suspend picks **highest `gen` numbers + (oldest, least efficient) first**; resume picks **lowest (newest, most efficient) + first**. This is the MXQ analog of Slurm node `Weight`. `-` ⇒ never auto-managed. +- `slots`/`mem_mb`/`gpus`/`tags`/`prerequisites` — capability and match attributes of a + **powered-off** node (the live DB can't tell us this once it's dark; §5 of the study). + `tags` and `prerequisites` mirror what the host's `mxqd` publishes and let resume + reproduce MXQ's job/host match (see below); `mxqd` implicitly adds `true`, the hostname + and the short hostname, so those need not be listed. Both take `-` for none. +- `on_method` ∈ `ipmi | wol | none`. `none` ⇒ excluded from all automation + (login/storage/infra). +- `bmc_or_mac` — BMC IP/hostname for `ipmi`, MAC for `wol`. + +**Matching model.** Deciding whether a powered-off node could accept a pending group +mirrors `server_is_qualified()` in `mxqd_control.c` — it is **not** a tag-set +intersection: + +- the group's `group_whitelist` / `group_blacklist` gate on the hostname; +- the group's `prerequisites` **expression** is evaluated against the **host's** tags; +- the host's own `prerequisites` expression is evaluated against the **group's** tags; +- a GPU group needs a GPU host. + +The expression grammar (`parser.y`: identifiers as tags, operators `!` `&` `|`, parens, +precedence `! > & > |`) is reproduced in the manager. On top of qualification, resume adds +a capacity check (the node is big enough for at least one of the group's jobs). + +IPMI credentials are **not** in this file. They live in `/etc/mxq/powerd-secrets` +(`0600`, root) — a global `user`/`password`, overridable per host. Keeping secrets out +of the world-readable inventory is deliberate. + +## 3. Global policy — `/etc/mxq/powerd.conf` + +```ini +[db] +# reuse MXQ's MySQL config-group convention; new read-only account recommended +defaults_group = mxqpowerd # section in ~/.my.cnf / /etc/mysql/… with a SELECT-only user + +[policy] +suspend_time = 1800 # sec idle before a node may power down (Slurm SuspendTime) +warm_pool = 4 # min idle managed nodes to keep ON (Slurm SuspendExcNodes :count) +min_uptime = 900 # sec a node must be up before it's suspend-eligible (anti-thrash) +suspend_rate = 4 # max power-downs per tick (Slurm SuspendRate) +resume_rate = 8 # max power-ups per tick (Slurm ResumeRate) +suspend_timeout = 300 # sec for a shutdown to complete (Slurm SuspendTimeout) +resume_timeout = 600 # sec for a boot+register (Slurm ResumeTimeout) +mtime_stale = 180 # sec: mxq_daemon.mtime older than this ⇒ node not alive +exclude = login01,storage0,storage1 # never manage (also: gen '-' / on_method none) + +[mode] +dry_run = true # v1 default: log decisions, take NO action. Flip to false to arm. +``` + +## 4. State the manager reads from the DB (no writes) + +All read-only. Aggregated **per hostname** because a host can run several `mxqd` +daemons and can carry stale EXITED/CRASHED rows. + +**A. Pending demand** (do we need more nodes, and of what shape): +```sql +SELECT SUM(group_jobs_inq) AS pend_jobs, + SUM(group_jobs_inq * job_threads) AS pend_cores, + MAX(job_memory) AS max_job_mem_kib, + MAX(job_gpu) AS max_job_gpu +FROM mxq_group +WHERE group_jobs_inq > 0 + AND (group_flags & 1) = 0; -- ignore CLOSED groups +``` +Per-group rows (with `job_threads,job_memory,job_gpu,tags,prerequisites,group_whitelist, +group_blacklist`) are also pulled so resume can reproduce the full job/host match +(§2, *Matching model*), not just core totals. + +**B. Live capacity & liveness** (what's on, and free): +```sql +SELECT hostname, + MAX(status) AS status, + SUM(daemon_slots) AS slots, + SUM(daemon_slots_running) AS slots_running, + SUM(daemon_slots - daemon_slots_running) AS slots_free, + MAX(UNIX_TIMESTAMP(mtime)) AS mtime +FROM mxq_daemon +WHERE status NOT IN (250, 255) -- exclude EXITED / CRASHED rows +GROUP BY hostname; +``` +A host is **alive** iff `NOW() - mtime <= mtime_stale`. Free capacity for resume = +`SUM(slots_free)` over alive hosts (filtered by the constraints from A). + +**C. Idle-duration** (no `idle_since` column exists — infer it): +```sql +SELECT host_hostname AS hostname, MAX(date_end) AS last_end +FROM mxq_job +WHERE host_hostname <> '' +GROUP BY host_hostname; +``` +Idle duration ≈ `NOW() - last_end`; if a host has no finished job since boot, fall back +to `daemon_start` from table B. A host is **safely idle** iff `status = 0 (IDLE)`, +`slots_running = 0`, and **no active jobs**: +```sql +SELECT COUNT(*) FROM mxq_job +WHERE host_hostname = ? AND job_status IN (100,150,200); -- ASSIGNED/LOADED/RUNNING +``` + +## 5. Per-node state machine + +The manager holds each managed host in one state in `powerd-state.json`, updated each +tick by combining that file with DB tables B/C. + +``` + demand needs it / not enough warm + POWERED_OFF ───────────────► POWERING_UP ──(fresh daemon row)──► UP_IDLE + ▲ │ │ + │ (no row by resume_timeout) │ job claimed + (host dark by ▼ ▼ + suspend_timeout) FAILED ◄───(off by …timeout)──── UP_BUSY + │ ▲ │ + POWERING_DOWN ◄──(EXITED seen)── DRAINING ◄──(idle>suspend_time & │ + │ poweroff issued │ surplus & rate ok) ◄──┘ + └──────────────────────────► (returns to POWERED_OFF on confirmed dark) +``` + +- **POWERED_OFF → POWERING_UP** (resume, §6): send wake, stamp `t_resume`. +- **POWERING_UP → UP_IDLE**: a fresh `mxq_daemon` row for the host appears (alive). + If not by `t_resume + resume_timeout` → **FAILED** (log, run resume-fail action, + exclude until operator clears — the `ResumeFailProgram` analog). +- **UP_IDLE → DRAINING** (suspend, §7): host safely idle > `suspend_time`, up > + `min_uptime`, is surplus (see §7), rate budget available, not in warm pool → send + `ssh HOST 'mxqdctl-hostconfig stop'`, stamp `t_drain`. +- **DRAINING → POWERING_DOWN**: host's daemon rows all show `status=EXITED(250)` → issue + power-off. **Waiting for EXITED is the race-closer**: `mxqd` only marks EXITED after + its drain loop finds nothing running, so once EXITED no job is running or half-claimed + on that host (`mxqd.c:2762`, `mxq_daemon.c:151`). If EXITED not seen by + `t_drain + suspend_timeout/2`, escalate to BMC soft-off. +- **POWERING_DOWN → POWERED_OFF**: confirmed dark (no fresh row / BMC `power status` + = off). If still not dark by `t_drain + suspend_timeout` → BMC hard-off, else FAILED. +- **UP_BUSY**: any active job → never a suspend candidate; a DRAINING host that somehow + claimed work is impossible after EXITED, but if a *new* daemon registers busy during + POWERING_UP we simply land in UP_BUSY. + +Cancellation: if demand spikes while a host is DRAINING (before EXITED), v1 lets it +finish exiting and immediately re-resumes it — simple and rare. (v2's DRAINING flag in +`mxqd`, study §6, makes this a cheap in-place cancel.) + +## 6. Resume (power ON) — the actuators + +Selection each tick: +1. `deficit_cores = pend_cores - free_cores_on_alive_hosts` (and analogous checks for a + pending job that needs GPU / memory / a tag no alive host provides). +2. Also ensure `warm_pool` idle nodes exist; if fewer, that's demand too. +3. If deficit > 0: from `POWERED_OFF` managed hosts **whose inventoried capability + satisfies the unmet requirement**, pick **lowest `gen` first**, up to `resume_rate`. + +Actuation per node: +```bash +# WoL +wakeonlan e4:3d:1a:00:11:22 # or: ether-wake -i +# IPMI +ipmitool -I lanplus -H 10.1.0.11 -U "$U" -P "$P" chassis power on +``` +Then move to POWERING_UP. Verification is **via the DB** — the node boots, its +init/systemd runs `mxqdctl-hostconfig.sh`, `mxqd` registers a fresh `mxq_daemon` row and +starts pulling. No scheduler action; the manager just watches for the row. + +Notes: +- **WoL prerequisites**: BIOS "Wake on LAN" enabled; the magic packet must reach the + node's L2 segment (a sender on each VLAN, or directed broadcast). If a node's segment + can't carry it, set `on_method = ipmi`. +- **IPMI**: BMC on a management network reachable from the mgmt host; credentials from + `powerd-secrets`. `ipmitool … chassis power status` gives an independent on/off check. +- **`resume_rate`** throttles inrush current (Slurm `ResumeRate`). + +## 7. Suspend (power OFF) — the actuators + +A host is a **suspend candidate** when: managed (`gen != -`, `on_method != none`, not in +`exclude`), **safely idle** (§4C) for `> suspend_time`, up `> min_uptime`, and alive. + +**Surplus test** (don't suspend nodes we'll immediately need, and honour the warm pool): +``` +surplus_cores = free_cores_on_alive_hosts - pend_cores +``` +Suspend a candidate only while `surplus_cores` stays ≥ 0 after removing it **and** the +count of remaining idle managed nodes stays ≥ `warm_pool`. Order candidates **highest +`gen` first** (oldest hardware off first), cap at `suspend_rate` per tick. + +Actuation per node (graceful, in-band): +```bash +ssh -o BatchMode=yes HOST 'mxqdctl-hostconfig stop' # SIGTERM: stop claiming, drain +# → manager waits for status=EXITED in mxq_daemon, then: +ssh -o BatchMode=yes HOST 'systemctl poweroff' # or: /sbin/poweroff +# fallback if EXITED / dark not reached in time: +ipmitool -I lanplus -H -U "$U" -P "$P" chassis power soft # ACPI soft-off +ipmitool … chassis power off # hard, last resort +``` + +## 8. Safety & operability + +- **`dry_run = true` is the v1 default.** The manager logs every decision + ("would power off node047: idle 41m, surplus 512c") and touches nothing until armed. + Run it against a live cluster for a few days and eyeball the log first. +- **Exclude is belt-and-suspenders**: `gen '-'`, `on_method none`, and the `exclude` + list all protect a node; infra must match at least one. +- **Rate limits** both directions; **warm pool** + **`suspend_time`** + **`min_uptime`** + give hysteresis so the cluster doesn't flap. +- **Timeouts** (`resume_timeout`, `suspend_timeout`) bound every transition; a stuck + node lands in **FAILED**, is excluded, logged, and left for an operator — never + retried blindly. +- **Idempotent & crash-safe**: all durable state is `powerd-state.json` + the DB; a + killed manager just re-derives on the next tick. +- **Manual override** (v1): edit `exclude` / set `on_method none` to pin a node on; + `systemctl stop mxq-powerd.timer` to disable globally. (v2: `mxqadmin --poweron/ + --poweroff/--drain`, study §6.) + +**Operational prerequisites** (raised in review): + +- **Single instance.** The timer must be enabled on **exactly one** management node + (e.g. `afk`); the manager takes no cluster-wide lock, so two instances would fight. +- **`mxqd` must start at boot.** Resume relies on a booted node bringing `mxqd` up so it + re-registers; MXQ does **not** currently start `mxqd` automatically after boot, so that + must be arranged first — otherwise woken nodes never become usable and trip + `resume_timeout` → FAILED. +- **Restricted ssh key.** The management node needs a dedicated key whose + `authorized_keys` entry is locked (`command="…",restrict`) to only `mxqdctl-hostconfig + stop` and the power-off command. +- **tmpdir blind spot.** MXQ cannot see free per-node tmpdir (`/scratch/local2`) space and + users can consume it outside `mxqd`'s control, so it cannot be a wake/keep criterion. + +## 9. systemd units + +```ini +# /etc/systemd/system/mxq-powerd.service +[Unit] +Description=MXQ node power manager (one evaluation tick) +After=network-online.target +[Service] +Type=oneshot +ExecStart=/usr/lib/mxq/mxq-powerd --config /etc/mxq/powerd.conf + +# /etc/systemd/system/mxq-powerd.timer +[Unit] +Description=Run MXQ power manager every minute +[Timer] +OnBootSec=2min +OnUnitActiveSec=60s +AccuracySec=5s +[Install] +WantedBy=timers.target +``` +(A `* * * * * root /usr/lib/mxq/mxq-powerd` cron line is an equivalent fallback.) + +## 10. Verification plan + +Build order and how to prove each step **end-to-end**, not just unit tests: + +1. **Read-only sanity (dry-run, no actuation):** point at the production DB, confirm the + three queries (§4) return sane numbers, and that the log's idle/busy/demand + classification matches `mxqdump --running` / `mxqdump --inq` for the same instant. +2. **Single-node suspend on a test host:** pick one disposable node, arm only that node + (exclude all others), watch: `mxqdctl stop` → row goes `EXITED(250)` → `poweroff` → + host dark (`ipmitool … power status` / ping). Confirm **no** job was running and no + stale RUNNING rows remain (`mxqdump --running`). +3. **Single-node resume:** submit a job that only that node can satisfy (tag/GPU/mem), + confirm the manager wakes it (WoL and IPMI paths **tested separately**), a fresh + `mxq_daemon` row appears within `resume_timeout`, and the job runs. +4. **Race probe:** in a loop, submit a burst just as a node crosses `suspend_time`; + confirm the wait-for-EXITED gate never cuts power on a host that grabbed a job. +5. **Failure paths:** block the BMC / drop a NIC and confirm the node lands in FAILED, + is excluded, and logged — not retried into a boot loop. +6. **Load test on the real cluster:** the MXQ cluster is available for a soak run — + `mxqsub -m 956G -t 90m --processors=256 -o ~/powerd-soak.log ./submit.sh` — drive it + idle→busy→idle cycles and inspect the log for flapping and correct rate limiting. +7. Only after 1–6 pass in dry-run/single-node: flip `dry_run = false` cluster-wide with + a conservative `suspend_time` (e.g. 60 min) and a large `warm_pool`, then tighten. + +## 11. Open items for Paul + +- Confirm the transport per node class (which are IPMI-capable, which WoL-capable, any + cross-VLAN WoL relay needed). +- Confirm `ssh HOST poweroff` is acceptable, or if power-off must go via BMC soft-off. +- Confirm the infra/never-suspend set and initial `suspend_time` / `warm_pool`. +- OK to add a read-only `[mxqpowerd]` MySQL account/config group? +- Python 3 acceptable, or prefer bash to match `mxqdctl-hostconfig.sh`? +- **Arrange `mxqd`-at-boot** (see §8) — a hard dependency for resume, currently missing. +- **Pick the single management node** the timer runs on, and provision its restricted ssh + key (§8). + +## References +- Slurm Power Saving Guide — https://slurm.schedmd.com/power_save.html +- OleHolmNielsen/Slurm_tools power_save (IPMI example scripts) — https://github.com/OleHolmNielsen/Slurm_tools/tree/master/power_save +- GSI Slurm power-management notes (WoL/IPMI resume) — https://web-docs.gsi.de/~vpenso/notes/posts/hpc/cluster/slurm/power-management.html +- Thinkbox Deadline Machine Startup (WoL/IPMI wake) — https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/power-management.html diff --git a/manpages/mxq-powerd.8 b/manpages/mxq-powerd.8 new file mode 100644 index 00000000..8ef6b129 --- /dev/null +++ b/manpages/mxq-powerd.8 @@ -0,0 +1,150 @@ +.TH MXQ-POWERD 8 "2026-07-18" "MXQ" "MXQ System Administration" +.SH NAME +mxq-powerd \- MXQ node power manager (suspend idle nodes, resume on demand) +.SH SYNOPSIS +.B mxq-powerd +.RI [ options ] +.SH DESCRIPTION +.B mxq-powerd +powers idle MXQ compute nodes down to save energy and powers them back up +when queued work needs them. It runs on the management host (where +.BR mxqdump (1) +and +.BR mxqadmin (1) +run), as root, on a fixed interval (a systemd timer or cron). One invocation +performs one evaluation tick; it is safe to run back-to-back and safe to miss +a tick. +.PP +The manager is entirely external to MXQ: it only +.B reads +the MySQL database (through the stock +.BR mysql (1) +client) and actuates nodes with stock CLIs +.RB ( ipmitool ", " wakeonlan ", " ssh ). +It issues no database writes and requires no changes to +.BR mxqd ", the schema, or " mxqadmin . +.PP +Each tick it: aggregates pending demand and live free capacity from the +database; classifies every managed host through a per-node state machine +(kept in the state file); wakes powered-off nodes when there is a core +deficit, an unmet capability (GPU/memory/tag), or the warm pool is short; +and gracefully drains and powers off surplus idle nodes. Suspend prefers the +oldest, least efficient hardware (highest +.IR gen ); +resume prefers the newest (lowest +.IR gen ). +.SH OPTIONS +.TP +.BR \-c ", " \-\-config " " \fIFILE\fR +Configuration file (default: +.IR /etc/mxq/powerd.conf ). +.TP +.B \-\-arm +Override the config and actually actuate this run +.RI ( dry_run=false ). +.TP +.B \-\-dry\-run +Override the config and log only, never actuate. +.TP +.B \-\-stderr +Also log to standard error. +.TP +.B \-\-status +Print the per-host state and exit without touching anything. +.TP +.BR \-\-clear " " \fIHOST\fR +Clear a FAILED or stuck host from the state file so automation may manage it +again, and exit. +.SH FILES +.TP +.I /etc/mxq/powerd.conf +Global policy: thresholds, rate limits, warm pool, exclude list, DB access, +and the +.B dry_run +switch. See +.IR powerd.conf.example . +.TP +.I /etc/mxq/nodes.conf +Static per-host inventory: generation/priority, capability +(slots/memory/GPUs/tags), power-on transport and BMC address or MAC. See +.IR nodes.conf.example . +.TP +.I /etc/mxq/powerd-secrets +IPMI/BMC credentials (mode 0600, root only). Kept out of the world-readable +inventory. The password reaches +.B ipmitool +via +.B $IPMI_PASSWORD +.RB ( ipmitool " " \-E ), +never on the command line. +.TP +.I /var/lib/mxq/powerd-state.json +Durable manager state: in-flight transitions and timestamps. A flock on this +file also serialises overlapping ticks. +.TP +.I /var/log/mxq/powerd.log +Actions and decisions. +.SH SAFETY +.B dry_run = true +is the default: the manager logs every decision and touches nothing until +armed. A host is protected from automation by any of: an excluded hostname, +.I gen +of +.BR \- , +or +.I on_method +of +.BR none . +Rate limits, the warm pool, +.IR suspend_time ", and " min_uptime +provide hysteresis. Every transition is bounded by a timeout; a stuck node +lands in +.BR FAILED , +is excluded, logged, and left for an operator rather than retried blindly. +Clear it with +.BR "mxq-powerd \-\-clear " \fIHOST\fR . +.SH CAVEATS +.B mxq-powerd +takes no cluster-wide lock, so its timer must be enabled on +.B exactly one +management node. +.PP +Resume assumes that powering a node on brings its +.BR mxqd (8) +back up so it re-registers; the daemon must therefore be started at boot. +.PP +The drain and power-off actuators use +.BR ssh (1); +give the manager a key whose +.I authorized_keys +entry is restricted +.RB ( command= "...) to only " "mxqdctl-hostconfig stop" +and the power-off command. +.PP +Free per-node tmpdir space (for example +.IR /scratch/local2 ) +is not visible to MXQ and can be consumed outside +.BR mxqd 's +control, so it cannot be used as a wake or keep criterion. +.PP +Matching mirrors +.B server_is_qualified +in +.IR mxqd_control.c : +a group's +.I prerequisites +expression is evaluated against a host's tags, the host's own +.I prerequisites +against the group's tags, and the group white/blacklist against the hostname. +The per-host inventory must carry each node's tags and prerequisites so this +match can be reproduced while the node is powered off. +.SH EXIT STATUS +0 on success, non-zero on a fatal configuration or database error. +.SH SEE ALSO +.BR mxqdump (1), +.BR mxqadmin (1), +.BR ipmitool (1), +.BR wakeonlan (1) +.PP +.I docs/power-management-implementation.md +in the MXQ source tree documents the design this implements. diff --git a/manpages/mxqsub.1 b/manpages/mxqsub.1 index 0f34d749..039c1516 100644 --- a/manpages/mxqsub.1 +++ b/manpages/mxqsub.1 @@ -34,6 +34,43 @@ is used to queue a job to be executed on a cluster node\&. [arguments] .RS 4 specify the estimated runtime\&. .RE +.PP +\fB\-c \fR\fB\fI\fR\fR, \fB\-\-callback=\fR\fB\fI\fR\fR +.RS 4 +run \fI\fR after the job reaches any terminal state (finished, failed, killed, or unknown)\&. +The executable must be given as an absolute path\&. +It is run directly via \fBexecl\fR(3) (not through a shell) as the submitting user in the job working directory\&. +The following environment variables are set: +.sp +.RS 4 +\fBMXQ_JOB_ID\fR \- the job id +.br +\fBMXQ_GROUP_ID\fR \- the job group id +.br +\fBMXQ_JOB_STATUS\fR \- one of \fIfinished\fR, \fIfailed\fR, \fIkilled\fR, or \fIunknown\fR +.br +\fBMXQ_JOB_WORKDIR\fR \- the job working directory +.RE +.RE +.SH "EXAMPLES" +.PP +Send an email when a job finishes\&. Save the following as e\&.g\&. +\fI~/bin/mxq\-notify\fR and make it executable: +.sp +.RS 4 +.nf +#!/bin/sh +mail \-s "MXQ job $MXQ_JOB_ID $MXQ_JOB_STATUS" "$USER" +.fi +.RE +.sp +Then submit a job with: +.sp +.RS 4 +.nf +mxqsub \-\-callback=$HOME/bin/mxq\-notify myjob +.fi +.RE .SH "ENVIRONMENT" .PP \fBMXQ_MYSQL_DEFAULTFILE\fR diff --git a/mxq_job.c b/mxq_job.c index daf4061e..3d02665c 100644 --- a/mxq_job.c +++ b/mxq_job.c @@ -18,7 +18,7 @@ #include "mxq_group.h" #include "mxq_job.h" -#define JOB_FIELDS_CNT 37 +#define JOB_FIELDS_CNT 38 #define JOB_FIELDS \ " job_id, " \ " job_status, " \ @@ -29,6 +29,7 @@ " job_argv, " \ " job_stdout, " \ " job_stderr, " \ + " job_callback, " \ " job_umask, " \ " host_submit, " \ " host_id, " \ @@ -73,6 +74,7 @@ static void bind_result_job_fields(struct mx_mysql_bind *result, struct mxq_job mx_mysql_bind_var(result, idx++, string, &(j->job_argv_str)); mx_mysql_bind_var(result, idx++, string, &(j->job_stdout)); mx_mysql_bind_var(result, idx++, string, &(j->job_stderr)); + mx_mysql_bind_var(result, idx++, string, &(j->job_callback)); mx_mysql_bind_var(result, idx++, uint32, &(j->job_umask)); mx_mysql_bind_var(result, idx++, string, &(j->host_submit)); mx_mysql_bind_var(result, idx++, string, &(j->host_id)); @@ -136,6 +138,7 @@ void mxq_job_free_content(struct mxq_job *j) mx_free_null(j->job_argv_str); mx_free_null(j->job_stdout); mx_free_null(j->job_stderr); + mx_free_null(j->job_callback); if (j->tmp_stderr == j->tmp_stdout) { j->tmp_stdout = NULL; @@ -605,6 +608,8 @@ int mxq_set_job_status_unknown(struct mx_mysql *mysql, struct mxq_job *job) return res; } + job->job_status = MXQ_JOB_STATUS_UNKNOWN; + return res; } diff --git a/mxq_job.h b/mxq_job.h index d73561cb..c008a026 100644 --- a/mxq_job.h +++ b/mxq_job.h @@ -29,6 +29,7 @@ struct mxq_job { char * job_stdout; char * job_stderr; + char * job_callback; char * tmp_stdout; char * tmp_stderr; diff --git a/mxqd.c b/mxqd.c index 3f572a8e..8c776e49 100644 --- a/mxqd.c +++ b/mxqd.c @@ -1928,6 +1928,61 @@ static void release_gpu(struct mxq_server *server, struct mxq_group *group, stru } } +static void run_job_callback(struct mxq_group *group, struct mxq_job *job) +{ + if (!job->job_callback || !*job->job_callback) + return; + + mx_log_info("job=%s(%d):%lu:%lu :: running callback: %s", + group->user_name, group->user_uid, group->group_id, job->job_id, + job->job_callback); + + pid_t pid = fork(); + if (pid < 0) { + mx_log_err("job=%s(%d):%lu:%lu callback fork(): %m", + group->user_name, group->user_uid, group->group_id, job->job_id); + return; + } + + if (pid == 0) { + /* Double-fork: this middle process exits immediately so the daemon's + * waitpid() returns without blocking on the callback's runtime. */ + pid_t cpid = fork(); + if (cpid == 0) { + if (initgroups(group->user_name, group->user_gid) == -1) + _exit(1); + if (setregid(group->user_gid, group->user_gid) == -1) + _exit(1); + if (setreuid(group->user_uid, group->user_uid) == -1) + _exit(1); + if (chdir(job->job_workdir) == -1) + _exit(1); + + struct rlimit rlim; + rlim.rlim_cur = rlim.rlim_max = 60; + setrlimit(RLIMIT_CPU, &rlim); + rlim.rlim_cur = rlim.rlim_max = 256*1024*1024; + setrlimit(RLIMIT_AS, &rlim); + rlim.rlim_cur = rlim.rlim_max = 0; + setrlimit(RLIMIT_CORE, &rlim); + + mx_setenvf_forever("MXQ_JOB_ID", "%lu", job->job_id); + mx_setenvf_forever("MXQ_GROUP_ID", "%lu", job->group_id); + mx_setenv_forever("MXQ_JOB_STATUS", mxq_job_status_to_name(job->job_status)); + mx_setenv_forever("MXQ_JOB_WORKDIR", job->job_workdir); + + execl(job->job_callback, job->job_callback, NULL); + _exit(1); + } + _exit(cpid < 0 ? 1 : 0); + } + + int status; + if (waitpid(pid, &status, 0) == -1) + mx_log_err("job=%s(%d):%lu:%lu callback waitpid(): %m", + group->user_name, group->user_uid, group->group_id, job->job_id); +} + static int job_has_finished(struct mxq_server *server, struct mxq_group *group, struct mxq_job_list *jlist) { int cnt; @@ -1942,6 +1997,8 @@ static int job_has_finished(struct mxq_server *server, struct mxq_group *group, rename_outfiles(server, group, job); + run_job_callback(group, job); + cnt = jlist->group->slots_per_job; cpuset_clear_running(&server->cpu_set_running, &job->host_cpu_set); release_gpu(server, group, job); @@ -1966,6 +2023,8 @@ static int job_is_lost(struct mxq_server *server,struct mxq_group *group, struct rename_outfiles(server, group, job); + run_job_callback(group, job); + cnt = jlist->group->slots_per_job; cpuset_clear_running(&server->cpu_set_running, &job->host_cpu_set); release_gpu(server, group, job); diff --git a/mxqsub.c b/mxqsub.c index 72893be2..c0194d81 100644 --- a/mxqsub.c +++ b/mxqsub.c @@ -60,6 +60,10 @@ static void print_usage(void) " -e, --stderr=FILE set file to capture stderr (default: )\n" " -u, --umask=MASK set mode to use as umask (default: current umask)\n" " -p, --priority=PRIORITY set priority (default: 127)\n" + " -c, --callback=EXECUTABLE run EXECUTABLE when the job finishes (any outcome)\n" + " runs as the submitting user in the job workdir;\n" + " MXQ_JOB_ID, MXQ_GROUP_ID, MXQ_JOB_STATUS, and\n" + " MXQ_JOB_WORKDIR are set in the environment\n" "\n" "Job resource information:\n" " Scheduling is done based on the resources a job needs and\n" @@ -517,6 +521,7 @@ static int add_job(struct mx_mysql *mysql, struct mxq_job *j) " job_stdout = ?," " job_stderr = ?," + " job_callback = ?," " job_umask = ?," @@ -535,8 +540,9 @@ static int add_job(struct mx_mysql *mysql, struct mxq_job *j) mx_mysql_statement_param_bind(stmt, 4, string, &(j->job_argv_str)); mx_mysql_statement_param_bind(stmt, 5, string, &(j->job_stdout)); mx_mysql_statement_param_bind(stmt, 6, string, &(j->job_stderr)); - mx_mysql_statement_param_bind(stmt, 7, uint32, &(j->job_umask)); - mx_mysql_statement_param_bind(stmt, 8, string, &(j->host_submit)); + mx_mysql_statement_param_bind(stmt, 7, string, &(j->job_callback)); + mx_mysql_statement_param_bind(stmt, 8, uint32, &(j->job_umask)); + mx_mysql_statement_param_bind(stmt, 9, string, &(j->host_submit)); res = mx_mysql_statement_execute(stmt, &num_rows); if (res < 0) { @@ -705,6 +711,7 @@ int main(int argc, char *argv[]) char arg_debug; u_int32_t arg_tmpdir; u_int16_t arg_gpu; + char *arg_callback; _mx_cleanup_free_ char *current_workdir = NULL; _mx_cleanup_free_ char *arg_stdout_absolute = NULL; @@ -769,6 +776,7 @@ int main(int argc, char *argv[]) MX_OPTION_REQUIRED_ARG("prerequisites", 10), MX_OPTION_REQUIRED_ARG("tags", 11), MX_OPTION_NO_ARG("gpu", 12), + MX_OPTION_REQUIRED_ARG("callback", 'c'), MX_OPTION_END }; @@ -798,6 +806,7 @@ int main(int argc, char *argv[]) arg_prerequisites = ""; arg_tags = NULL; arg_gpu = 0; + arg_callback = ""; arg_mysql_default_group = getenv("MXQ_MYSQL_DEFAULT_GROUP"); if (!arg_mysql_default_group) @@ -1021,6 +1030,18 @@ int main(int argc, char *argv[]) case 12: arg_gpu = 1; break; + + case 'c': + if (!(*optctl.optarg)) { + mx_log_crit("--callback '%s': String is empty.", optctl.optarg); + exit(EX_CONFIG); + } + if (optctl.optarg[0] != '/') { + mx_log_crit("--callback '%s': must be an absolute path.", optctl.optarg); + exit(EX_CONFIG); + } + arg_callback = optctl.optarg; + break; } } @@ -1133,6 +1154,7 @@ int main(int argc, char *argv[]) job.job_argc = argc; job.job_argv = argv; job.job_argv_str = arg_args; + job.job_callback = arg_callback; /******************************************************************/ diff --git a/mysql/create_tables.sql b/mysql/create_tables.sql index 41585f76..d73d34a0 100644 --- a/mysql/create_tables.sql +++ b/mysql/create_tables.sql @@ -83,6 +83,7 @@ CREATE TABLE IF NOT EXISTS mxq_job ( job_stdout VARCHAR(4096) NOT NULL DEFAULT '/dev/null', job_stderr VARCHAR(4096) NOT NULL DEFAULT '/dev/null', + job_callback VARCHAR(4096) NOT NULL DEFAULT '', job_umask INT4 NOT NULL, diff --git a/mysql/migrate_019_add_job_callback.sql b/mysql/migrate_019_add_job_callback.sql new file mode 100644 index 00000000..c411aba7 --- /dev/null +++ b/mysql/migrate_019_add_job_callback.sql @@ -0,0 +1,2 @@ +ALTER TABLE mxq_job + ADD COLUMN job_callback VARCHAR(4096) NOT NULL DEFAULT '' AFTER job_stderr; diff --git a/powerd/.gitignore b/powerd/.gitignore new file mode 100644 index 00000000..7a60b85e --- /dev/null +++ b/powerd/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/powerd/README.md b/powerd/README.md new file mode 100644 index 00000000..fbf7246f --- /dev/null +++ b/powerd/README.md @@ -0,0 +1,86 @@ +# mxq-powerd — MXQ node power management + +`mxq-powerd` powers idle MXQ compute nodes down and powers them back up when +work arrives, oldest/least-efficient hardware off first. It is an **external** +add-on: it only *reads* the MySQL database (via the stock `mysql` client) and +actuates nodes with stock CLIs (`ipmitool`, `wakeonlan`, `ssh`). It touches +neither `mxqd`, the schema, nor `mxqadmin`. + +See [`docs/power-management-implementation.md`](../docs/power-management-implementation.md) +for the full design and [`docs/power-management-design.md`](../docs/power-management-design.md) +for the *why*. + +## Files + +| File | Installed to | Purpose | +|---|---|---| +| `mxq-powerd` | `/usr/libexec/mxq/mxq-powerd` | the manager (Python 3) | +| `powerd.conf.example` | `/etc/mxq/powerd.conf` | global policy | +| `nodes.conf.example` | `/etc/mxq/nodes.conf` | per-host inventory | +| `powerd-secrets.example`| `/etc/mxq/powerd-secrets` | IPMI credentials (0600) | +| `mxq-powerd.service` | `/etc/systemd/system/` | one-tick oneshot unit | +| `mxq-powerd.timer` | `/etc/systemd/system/` | runs it every 60 s | +| `manpages/mxq-powerd.8` | `${MANDIR}/man8/` | manual page | + +## Install + +```sh +make install-powerd # installs program + manpage + example configs +# then, once, as root: +cp /etc/mxq/powerd.conf.example /etc/mxq/powerd.conf +cp /etc/mxq/nodes.conf.example /etc/mxq/nodes.conf +install -m 0600 /etc/mxq/powerd-secrets.example /etc/mxq/powerd-secrets +$EDITOR /etc/mxq/nodes.conf /etc/mxq/powerd.conf /etc/mxq/powerd-secrets +systemctl enable --now mxq-powerd.timer +``` + +The systemd units are provided as examples under `powerd/`; copy them to +`/etc/systemd/system/` and `systemctl daemon-reload`. + +## Operational prerequisites + +- **Enable the timer on exactly one node.** `mxq-powerd` takes no cluster-wide + lock (the flock only serialises ticks on one host), so two instances would + fight over the same nodes. Pick a single management node (e.g. `afk`). +- **`mxqd` must start at boot.** Resume assumes that powering a node on brings + its `mxqd` up so it registers in `mxq_daemon` again; otherwise a woken node + never becomes usable and eventually trips `resume_timeout` → `FAILED`. MXQ + today does *not* start `mxqd` automatically after boot — that has to be + arranged before arming. +- **Restrict the ssh key.** The drain/poweroff actuators ssh into each node. + Give the manager a dedicated key whose `authorized_keys` entry is locked to + just those commands, e.g.: + + ``` + command="mxqdctl-hostconfig stop",restrict ssh-ed25519 AAAA... mxq-powerd-drain + ``` + + (or a small wrapper that permits only `mxqdctl-hostconfig stop` and + `systemctl poweroff`). Point `ssh_opts`/keys in `powerd.conf` at it. +- **tmpdir is a blind spot.** MXQ cannot see free `/scratch/local2` space, and + users can consume it outside `mxqd`'s control. `mxq-powerd` inherits that + limitation: it cannot use free tmpdir as a wake/keep criterion. + +## Runbook + +```sh +mxq-powerd --dry-run --stderr # run one tick, log decisions, actuate nothing +mxq-powerd --status # show the recorded per-host state +mxq-powerd --clear node047 # clear a FAILED/stuck host +mxq-powerd --arm # actuate (or set dry_run=false in the config) +``` + +`dry_run = true` is the default. Run it against the live cluster for a few days +and read `/var/log/mxq/powerd.log` before arming. Then flip `dry_run = false` +with a conservative `suspend_time` and a large `warm_pool`, and tighten from +there. `systemctl stop mxq-powerd.timer` disables the whole thing. + +## DB access + +`mxq-powerd` connects the same way the C tools do — through a MySQL defaults +file and config group. By default it reuses MXQ's read-only file +(`/etc/mxq/mysql_ro.cnf`, group `mxqclient`); a dedicated read-only account is +recommended. Because the `mysql` client only reads the `[client]`/`[mysql]` +groups, an arbitrary named group is bridged with `my_print_defaults`. To point +at a file whose credentials live under a plain `[client]` section instead, +leave `defaults_group` empty in `powerd.conf`. diff --git a/powerd/mxq-powerd b/powerd/mxq-powerd new file mode 100755 index 00000000..209dbeaf --- /dev/null +++ b/powerd/mxq-powerd @@ -0,0 +1,1256 @@ +#!/usr/bin/env python3 +# mxq-powerd -- MXQ node power manager (one evaluation tick per invocation) +# +# Powers idle MXQ compute nodes down and powers them back up when work +# arrives, oldest/least-efficient hardware off first, newest on first. +# +# It runs on the management host (where mxqdump/mxqadmin run), as root, on a +# fixed interval (systemd timer or cron). It is *external* to MXQ: it only +# READS the MySQL database (via the stock `mysql` client) and actuates nodes +# with stock CLIs (`ipmitool`, `wakeonlan`, `ssh`). It never writes the DB +# and requires no changes to mxqd, the schema or mxqadmin. +# +# See docs/power-management-implementation.md for the design this implements. + +import argparse +import configparser +import fcntl +import json +import logging +import os +import shlex +import subprocess +import sys +import time + +# --------------------------------------------------------------------------- +# Constants from the MXQ schema (mxq_daemon.h / mxq_job.h). +# --------------------------------------------------------------------------- + +DAEMON_STATUS_IDLE = 0 +DAEMON_STATUS_EXITED = 250 +DAEMON_STATUS_CRASHED = 255 + +# job_status values that mean "this job is on a host right now" +JOB_ACTIVE_STATUSES = (100, 150, 200) # ASSIGNED, LOADED, RUNNING + +MXQ_GROUP_FLAG_CLOSED = 1 # (1<<0) + +# --------------------------------------------------------------------------- +# Per-node lifecycle states (powerd-state.json). +# --------------------------------------------------------------------------- + +S_POWERED_OFF = "POWERED_OFF" +S_POWERING_UP = "POWERING_UP" +S_UP_IDLE = "UP_IDLE" +S_UP_BUSY = "UP_BUSY" +S_DRAINING = "DRAINING" +S_POWERING_DOWN = "POWERING_DOWN" +S_FAILED = "FAILED" + +STATE_VERSION = 1 + +log = logging.getLogger("mxq-powerd") + + +# =========================================================================== +# Prerequisite expression evaluation +# +# MXQ matches a job group to a host with two boolean expressions plus a +# host white/blacklist -- see server_is_qualified() in mxqd_control.c: +# +# * group.group_whitelist / group.group_blacklist gate on the hostname; +# * group.prerequisites is an expression evaluated against the HOST's tags; +# * the host daemon's prerequisites is an expression evaluated against the +# GROUP's tags; +# * a GPU group needs a GPU host. +# +# It is NOT a tag-set-vs-tag-set intersection. The expression grammar +# (parser.y) is: identifiers -- a tag, true iff present in the tag set -- with +# the operators `!` `&` `|` and parentheses, precedence ! > & > |. An empty +# expression means "always true"; a syntactically invalid one makes yyparse() +# fail, which server_is_qualified() treats as "not qualified" (i.e. false). +# =========================================================================== + +class _ExprError(Exception): + pass + + +def _tokenize_expr(expr): + toks = [] + i, n = 0, len(expr) + while i < n: + c = expr[i] + if c in " \t": + i += 1 + elif c in "&|!()": + toks.append(c) + i += 1 + elif c.isalpha(): + j = i + 1 + while j < n and (expr[j].isalnum() or expr[j] == "_"): + j += 1 + toks.append(("TAG", expr[i:j])) + i = j + else: + # In parser.y any other character is its own token, which the + # grammar then rejects as a syntax error. + raise _ExprError("unexpected character %r" % c) + return toks + + +class _ExprParser: + """Recursive-descent evaluator mirroring parser.y (! > & > |).""" + + def __init__(self, toks, tags): + self.toks = toks + self.pos = 0 + self.tags = tags + + def _peek(self): + return self.toks[self.pos] if self.pos < len(self.toks) else None + + def parse(self): + val = self._or() + if self.pos != len(self.toks): + raise _ExprError("trailing tokens") + return val + + def _or(self): + val = self._and() + while self._peek() == "|": + self.pos += 1 + rhs = self._and() + val = bool(val or rhs) + return val + + def _and(self): + val = self._not() + while self._peek() == "&": + self.pos += 1 + rhs = self._not() + val = bool(val and rhs) + return val + + def _not(self): + if self._peek() == "!": + self.pos += 1 + return not self._not() + return self._atom() + + def _atom(self): + tok = self._peek() + if tok == "(": + self.pos += 1 + val = self._or() + if self._peek() != ")": + raise _ExprError("missing )") + self.pos += 1 + return val + if isinstance(tok, tuple) and tok[0] == "TAG": + self.pos += 1 + return tok[1] in self.tags + raise _ExprError("unexpected token %r" % (tok,)) + + +def eval_prerequisites(expr, tags): + """Evaluate an MXQ prerequisites expression against a set of tags. + + Empty expression -> True. A syntax error -> False (matching a failed + yyparse() in server_is_qualified()). + """ + expr = (expr or "").strip() + if not expr: + return True + try: + return bool(_ExprParser(_tokenize_expr(expr), tags).parse()) + except _ExprError as exc: + log.warning("invalid prerequisites expression %r: %s", expr, exc) + return False + + +def keyword_set(raw): + """Split a tag/white/blacklist string into a set the way keywordset_new() + does -- on any whitespace (see keywordset.c).""" + return {t for t in (raw or "").split() if t} + + +# =========================================================================== +# Configuration +# =========================================================================== + +CONFIG_DEFAULTS = { + "db": { + # How to reach the MySQL server. We reuse MXQ's read-only defaults + # file. If `defaults_group` is set we extract the connection options + # from that group with `my_print_defaults` (the `mysql` client itself + # only reads the [client]/[mysql] groups); otherwise `mysql` reads + # `defaults_file` directly. + "mysql": "mysql", + "my_print_defaults": "my_print_defaults", + "defaults_file": "/etc/mxq/mysql_ro.cnf", + "defaults_group": "mxqclient", + "database": "mxq", + }, + "policy": { + "suspend_time": "1800", # sec idle before a node may power down + "warm_pool": "4", # min idle managed nodes to keep ON + "min_uptime": "900", # sec up before suspend-eligible (anti-thrash) + "suspend_rate": "4", # max power-downs initiated per tick + "resume_rate": "8", # max power-ups initiated per tick + "suspend_timeout": "300", # sec for a shutdown to complete + "resume_timeout": "600", # sec for a boot+register + "mtime_stale": "180", # sec: mxq_daemon.mtime older than this => not alive + "exclude": "", # comma-separated hostnames never managed + }, + "mode": { + "dry_run": "true", # v1 default: log decisions, take NO action + }, + "paths": { + "nodes_conf": "/etc/mxq/nodes.conf", + "secrets_file": "/etc/mxq/powerd-secrets", + "state_file": "/var/lib/mxq/powerd-state.json", + "log_file": "/var/log/mxq/powerd.log", + }, + "actuation": { + "ssh": "ssh", + "ssh_opts": "-o BatchMode=yes -o ConnectTimeout=10", + "ipmitool": "ipmitool", + "wakeonlan": "wakeonlan", + "drain_cmd": "mxqdctl-hostconfig stop", + "poweroff_cmd": "systemctl poweroff", + }, +} + + +class Config: + def __init__(self, path=None): + self.cp = configparser.ConfigParser() + # seed defaults + self.cp.read_dict(CONFIG_DEFAULTS) + self.path = path + if path: + if not self.cp.read(path): + raise SystemExit("mxq-powerd: cannot read config file: %s" % path) + + def get(self, section, key): + return self.cp.get(section, key) + + def getint(self, section, key): + return self.cp.getint(section, key) + + def getbool(self, section, key): + return self.cp.getboolean(section, key) + + def exclude_set(self): + raw = self.get("policy", "exclude") + return {h.strip() for h in raw.split(",") if h.strip()} + + +# =========================================================================== +# Inventory (/etc/mxq/nodes.conf) +# =========================================================================== + +class Node: + """One managed host, from the static inventory.""" + + def __init__(self, hostname, gen, slots, mem_mb, gpus, tags, + prerequisites, on_method, bmc_or_mac): + self.hostname = hostname + self.gen = gen # None => never auto-managed + self.slots = slots # inventoried capacity (node powered off) + self.mem_mb = mem_mb + self.gpus = gpus + self.tags = tags # set of strings (hostconfig/cpufeatures) + self.prerequisites = prerequisites # host's own mxqd --prerequisites expr + self.on_method = on_method # 'ipmi' | 'wol' | 'none' + self.bmc_or_mac = bmc_or_mac + + @property + def managed(self): + return self.gen is not None and self.on_method in ("ipmi", "wol") + + @property + def hostname_short(self): + return self.hostname.split(".", 1)[0] + + @property + def match_tags(self): + """Tag set a booted mxqd would expose for this host: the inventoried + tags plus the implicit `true`, hostname and short hostname that mxqd.c + always adds. A group's prerequisites expression is matched against + this set.""" + return set(self.tags) | {"true", self.hostname, self.hostname_short} + + +def _parse_int(tok): + if tok in ("-", ""): + return None + return int(tok) + + +def load_inventory(path): + """Parse the whitespace-separated /etc/mxq/nodes.conf. + + Columns: hostname gen slots mem_mb gpus tags prerequisites on_method bmc_or_mac + + `tags` and `prerequisites` mirror what this host's mxqd publishes (its + hostconfig/cpufeature tags and its --prerequisites expression); they let + the manager reproduce MXQ's job/host match while the node is powered off. + Both take `-` for "none". Because columns are whitespace-separated, the + tags column is comma-separated internally and the prerequisites expression + must be written without spaces (e.g. `gpu&!maintenance`). + A `-` in `gen` (or on_method `none`) marks a host as never auto-managed. + """ + nodes = {} + if not os.path.exists(path): + raise SystemExit("mxq-powerd: inventory not found: %s" % path) + with open(path) as fh: + for lineno, line in enumerate(fh, 1): + line = line.split("#", 1)[0].strip() + if not line: + continue + cols = line.split() + if len(cols) < 9: + log.warning("nodes.conf:%d: expected 9 columns, got %d -- skipped", + lineno, len(cols)) + continue + (hostname, gen, slots, mem_mb, gpus, tags, prereq, + on_method, bmc_or_mac) = cols[:9] + gen_val = None if gen == "-" else gen + tag_set = {t for t in tags.split(",") if t and t != "-"} + prereq_val = "" if prereq == "-" else prereq + try: + node = Node( + hostname=hostname, + gen=gen_val, + slots=_parse_int(slots) or 0, + mem_mb=_parse_int(mem_mb) or 0, + gpus=_parse_int(gpus) or 0, + tags=tag_set, + prerequisites=prereq_val, + on_method=on_method, + bmc_or_mac=bmc_or_mac, + ) + except ValueError as exc: + log.warning("nodes.conf:%d: %s -- skipped", lineno, exc) + continue + nodes[hostname] = node + return nodes + + +# =========================================================================== +# Secrets (/etc/mxq/powerd-secrets) -- IPMI credentials, 0600 root-only +# =========================================================================== + +class Secrets: + """[ipmi] user/password with optional per-host [ipmi.HOST] overrides.""" + + def __init__(self, path): + self.cp = configparser.ConfigParser() + self.path = path + if path and os.path.exists(path): + mode = os.stat(path).st_mode & 0o077 + if mode: + log.warning("%s is group/world accessible (mode %o); " + "IPMI credentials should be 0600", + path, os.stat(path).st_mode & 0o777) + self.cp.read(path) + + def ipmi(self, hostname): + """Return (user, password) for a host, or (None, None).""" + user = password = None + if self.cp.has_section("ipmi"): + user = self.cp.get("ipmi", "user", fallback=None) + password = self.cp.get("ipmi", "password", fallback=None) + sect = "ipmi.%s" % hostname + if self.cp.has_section(sect): + user = self.cp.get(sect, "user", fallback=user) + password = self.cp.get(sect, "password", fallback=password) + return user, password + + +# =========================================================================== +# Durable state (/var/lib/mxq/powerd-state.json) +# =========================================================================== + +class StateStore: + """In-flight transitions + timestamps, persisted as JSON. + + A single flock on the file also serialises overlapping ticks. + """ + + def __init__(self, path): + self.path = path + self.hosts = {} + self._fh = None + + def lock(self): + d = os.path.dirname(self.path) + if d: + os.makedirs(d, exist_ok=True) + # open (creating if needed) and take an exclusive, non-blocking lock + self._fh = open(self.path, "a+") + try: + fcntl.flock(self._fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + raise SystemExit("mxq-powerd: another tick holds the lock on %s" + % self.path) + self._fh.seek(0) + data = self._fh.read().strip() + if data: + try: + doc = json.loads(data) + self.hosts = doc.get("hosts", {}) + except json.JSONDecodeError as exc: + log.error("state file %s is corrupt (%s); starting empty", + self.path, exc) + self.hosts = {} + + def save(self): + doc = {"version": STATE_VERSION, "hosts": self.hosts} + tmp = self.path + ".tmp" + with open(tmp, "w") as fh: + json.dump(doc, fh, indent=2, sort_keys=True) + fh.write("\n") + os.replace(tmp, self.path) + + def unlock(self): + if self._fh: + fcntl.flock(self._fh, fcntl.LOCK_UN) + self._fh.close() + self._fh = None + + # -- per-host helpers -------------------------------------------------- + + def host(self, hostname): + return self.hosts.get(hostname) + + def set_state(self, hostname, state, now, note=None, **stamps): + rec = self.hosts.setdefault(hostname, {}) + if rec.get("state") != state: + rec["since"] = now + rec["state"] = state + if note is not None: + rec["note"] = note + rec.update(stamps) + return rec + + def get_state(self, hostname): + rec = self.hosts.get(hostname) + return rec.get("state") if rec else None + + +# =========================================================================== +# Database access (read-only, via the stock `mysql` client) +# =========================================================================== + +class Database: + def __init__(self, cfg): + self.cfg = cfg + self._base = self._build_base_cmd() + + def _build_base_cmd(self): + c = self.cfg + mysql = c.get("db", "mysql") + dfile = c.get("db", "defaults_file") + dgroup = c.get("db", "defaults_group") + database = c.get("db", "database") + + if dgroup: + # Extract connection options from an arbitrary named group. The + # `mysql` client itself only reads [client]/[mysql], so we bridge + # via my_print_defaults. + opts = self._print_defaults(dfile, dgroup) + cmd = [mysql, "--no-defaults"] + opts + elif dfile: + cmd = [mysql, "--defaults-file=" + dfile] + else: + cmd = [mysql] + cmd += ["--batch", "--skip-column-names"] + if database: + cmd += ["--database=" + database] + return cmd + + def _print_defaults(self, dfile, group): + mpd = self.cfg.get("db", "my_print_defaults") + cmd = [mpd] + if dfile: + cmd.append("--defaults-file=" + dfile) + cmd.append(group) + try: + out = subprocess.run(cmd, capture_output=True, text=True, check=True) + except (OSError, subprocess.CalledProcessError) as exc: + raise SystemExit("mxq-powerd: cannot read DB defaults group [%s] " + "from %s: %s" % (group, dfile, exc)) + return [ln.strip() for ln in out.stdout.splitlines() if ln.strip()] + + def query(self, sql): + """Run SQL, return list of rows; each row is a list of str/None.""" + cmd = self._base + ["--execute=" + sql] + try: + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + except OSError as exc: + raise SystemExit("mxq-powerd: cannot run mysql: %s" % exc) + except subprocess.CalledProcessError as exc: + raise SystemExit("mxq-powerd: DB query failed: %s\nSQL: %s" + % (exc.stderr.strip(), sql)) + rows = [] + for line in res.stdout.splitlines(): + if not line: + continue # `mysql --batch` emits no blank data lines + fields = line.split("\t") + rows.append([None if f == "NULL" else f for f in fields]) + return rows + + def scalar(self, sql): + rows = self.query(sql) + if not rows or not rows[0]: + return None + return rows[0][0] + + +def _to_int(val, default=0): + if val is None: + return default + try: + return int(val) + except (TypeError, ValueError): + return default + + +# =========================================================================== +# Cluster snapshot (everything one tick needs from the DB) +# =========================================================================== + +class PendingGroup: + def __init__(self, jobs_inq, threads, memory_mb, gpu, tags, + prerequisites, whitelist, blacklist): + self.jobs_inq = jobs_inq + self.threads = threads + self.memory_mb = memory_mb + self.gpu = gpu + self.tags = tags # group tags (matched by host prereq) + self.prerequisites = prerequisites # expr matched against host tags + self.whitelist = whitelist # allowed hostnames (set); empty=any + self.blacklist = blacklist # denied hostnames (set) + + +class LiveHost: + def __init__(self, hostname, status, slots, slots_running, slots_free, + mtime, daemon_start): + self.hostname = hostname + self.status = status + self.slots = slots + self.slots_running = slots_running + self.slots_free = slots_free + self.mtime = mtime + self.daemon_start = daemon_start + self.active_jobs = 0 + self.last_end = None + + +class Snapshot: + def __init__(self, db, cfg): + self.now = _to_int(db.scalar("SELECT UNIX_TIMESTAMP()"), int(time.time())) + self.mtime_stale = cfg.getint("policy", "mtime_stale") + self._read_demand(db) + self._read_live(db) + self._read_activity(db) + + # -- A. pending demand ------------------------------------------------- + + def _read_demand(self, db): + # aggregate totals + row = db.query( + "SELECT COALESCE(SUM(group_jobs_inq),0), " + "COALESCE(SUM(group_jobs_inq*job_threads),0), " + "COALESCE(MAX(job_memory),0), COALESCE(MAX(job_gpu),0) " + "FROM mxq_group " + "WHERE group_jobs_inq > 0 AND (group_flags & %d) = 0" + % MXQ_GROUP_FLAG_CLOSED) + agg = row[0] if row else [0, 0, 0, 0] + self.pend_jobs = _to_int(agg[0]) + self.pend_cores = _to_int(agg[1]) + self.max_job_mem_mb = _to_int(agg[2]) + self.max_job_gpu = _to_int(agg[3]) + + # per-group rows so resume can reproduce MXQ's job/host match + self.pending_groups = [] + for r in db.query( + "SELECT group_jobs_inq, job_threads, job_memory, job_gpu, " + "tags, prerequisites, group_whitelist, group_blacklist " + "FROM mxq_group " + "WHERE group_jobs_inq > 0 AND (group_flags & %d) = 0" + % MXQ_GROUP_FLAG_CLOSED): + self.pending_groups.append(PendingGroup( + jobs_inq=_to_int(r[0]), threads=_to_int(r[1]), + memory_mb=_to_int(r[2]), gpu=_to_int(r[3]), + tags=keyword_set(r[4]), prerequisites=(r[5] or ""), + whitelist=keyword_set(r[6]), blacklist=keyword_set(r[7]))) + + # -- B. live capacity & liveness -------------------------------------- + + def _read_live(self, db): + self.live = {} + for r in db.query( + "SELECT hostname, MAX(status), SUM(daemon_slots), " + "SUM(daemon_slots_running), " + "SUM(daemon_slots - daemon_slots_running), " + "MAX(UNIX_TIMESTAMP(mtime)), MIN(UNIX_TIMESTAMP(daemon_start)) " + "FROM mxq_daemon " + "WHERE status NOT IN (%d, %d) " + "GROUP BY hostname" + % (DAEMON_STATUS_EXITED, DAEMON_STATUS_CRASHED)): + host = LiveHost( + hostname=r[0], status=_to_int(r[1]), + slots=_to_int(r[2]), slots_running=_to_int(r[3]), + slots_free=_to_int(r[4]), mtime=_to_int(r[5]), + daemon_start=_to_int(r[6])) + self.live[host.hostname] = host + + # -- C. idle-duration & active jobs ----------------------------------- + + def _read_activity(self, db): + for r in db.query( + "SELECT host_hostname, MAX(UNIX_TIMESTAMP(date_end)) " + "FROM mxq_job WHERE host_hostname <> '' " + "GROUP BY host_hostname"): + h = self.live.get(r[0]) + if h is not None: + h.last_end = _to_int(r[1], None) if r[1] is not None else None + + active_sql = ( + "SELECT host_hostname, COUNT(*) FROM mxq_job " + "WHERE host_hostname <> '' AND job_status IN (%s) " + "GROUP BY host_hostname" + % ",".join(str(s) for s in JOB_ACTIVE_STATUSES)) + for r in db.query(active_sql): + h = self.live.get(r[0]) + if h is not None: + h.active_jobs = _to_int(r[1]) + + # -- derived helpers --------------------------------------------------- + + def is_alive(self, hostname): + h = self.live.get(hostname) + return h is not None and (self.now - h.mtime) <= self.mtime_stale + + def is_present(self, hostname): + """Has any non-EXITED/CRASHED daemon row (may be stale).""" + return hostname in self.live + + def is_busy(self, hostname): + h = self.live.get(hostname) + if h is None: + return False + return h.slots_running > 0 or h.active_jobs > 0 or h.status != DAEMON_STATUS_IDLE + + def is_idle(self, hostname): + """Safely idle: alive, all daemons IDLE, nothing running/claimed.""" + h = self.live.get(hostname) + if h is None or not self.is_alive(hostname): + return False + return (h.status == DAEMON_STATUS_IDLE and h.slots_running == 0 + and h.active_jobs == 0) + + def idle_secs(self, hostname): + """Seconds since the host last finished a job (fallback: daemon_start).""" + h = self.live.get(hostname) + if h is None: + return 0 + ref = h.last_end if h.last_end else h.daemon_start + if not ref: + return 0 + return max(0, self.now - ref) + + def uptime_secs(self, hostname): + h = self.live.get(hostname) + if h is None or not h.daemon_start: + return 0 + return max(0, self.now - h.daemon_start) + + +# =========================================================================== +# Actuators (all no-ops under dry_run) +# =========================================================================== + +class Actuator: + def __init__(self, cfg, secrets, dry_run): + self.cfg = cfg + self.secrets = secrets + self.dry_run = dry_run + + def _run(self, cmd, what, host, env=None): + printable = " ".join(shlex.quote(c) for c in cmd) + if self.dry_run: + log.info("[dry-run] %s %s: would run: %s", what, host, printable) + return True + log.info("%s %s: running: %s", what, host, printable) + try: + res = subprocess.run(cmd, capture_output=True, text=True, + env=env, timeout=60) + except (OSError, subprocess.TimeoutExpired) as exc: + log.error("%s %s: command error: %s", what, host, exc) + return False + if res.returncode != 0: + log.error("%s %s: exit %d: %s", what, host, res.returncode, + (res.stderr or res.stdout).strip()) + return False + return True + + # -- resume ------------------------------------------------------------ + + def power_on(self, node): + if node.on_method == "wol": + return self._wol(node) + if node.on_method == "ipmi": + return self._ipmi(node, ["chassis", "power", "on"], "power-on") + log.error("power_on %s: unknown on_method %r", node.hostname, + node.on_method) + return False + + def _wol(self, node): + cmd = [self.cfg.get("actuation", "wakeonlan"), node.bmc_or_mac] + return self._run(cmd, "wol", node.hostname) + + def _ipmi_cmd(self, node, sub): + user, password = self.secrets.ipmi(node.hostname) + cmd = [self.cfg.get("actuation", "ipmitool"), + "-I", "lanplus", "-H", node.bmc_or_mac] + if user: + cmd += ["-U", user] + # -E reads the password from $IPMI_PASSWORD, keeping it out of argv/ps + env = None + if password: + cmd += ["-E"] + env = dict(os.environ, IPMI_PASSWORD=password) + return cmd + list(sub), env + + def _ipmi(self, node, sub, what): + if node.on_method != "ipmi": + log.error("%s %s: not an ipmi node", what, node.hostname) + return False + cmd, env = self._ipmi_cmd(node, sub) + return self._run(cmd, what, node.hostname, env=env) + + # -- suspend ----------------------------------------------------------- + + def ssh_drain(self, node): + return self._ssh(node, self.cfg.get("actuation", "drain_cmd"), "drain") + + def ssh_poweroff(self, node): + return self._ssh(node, self.cfg.get("actuation", "poweroff_cmd"), + "poweroff") + + def _ssh(self, node, remote_cmd, what): + opts = shlex.split(self.cfg.get("actuation", "ssh_opts")) + cmd = [self.cfg.get("actuation", "ssh")] + opts + \ + [node.hostname, remote_cmd] + return self._run(cmd, what, node.hostname) + + def ipmi_soft_off(self, node): + return self._ipmi(node, ["chassis", "power", "soft"], "soft-off") + + def ipmi_hard_off(self, node): + return self._ipmi(node, ["chassis", "power", "off"], "hard-off") + + def ipmi_is_off(self, node): + """Return True if BMC reports chassis power off, None if unknown.""" + if node.on_method != "ipmi" or self.dry_run: + return None + cmd, env = self._ipmi_cmd(node, ["chassis", "power", "status"]) + try: + res = subprocess.run(cmd, capture_output=True, text=True, + env=env, timeout=30) + except (OSError, subprocess.TimeoutExpired): + return None + if res.returncode != 0: + return None + return "off" in res.stdout.lower() + + +# =========================================================================== +# The manager: one evaluation tick +# =========================================================================== + +class Manager: + def __init__(self, cfg, inventory, secrets, state, snap, actuator): + self.cfg = cfg + self.inventory = inventory + self.secrets = secrets + self.state = state + self.snap = snap + self.act = actuator + self.now = snap.now + + self.suspend_time = cfg.getint("policy", "suspend_time") + self.warm_pool = cfg.getint("policy", "warm_pool") + self.min_uptime = cfg.getint("policy", "min_uptime") + self.suspend_rate = cfg.getint("policy", "suspend_rate") + self.resume_rate = cfg.getint("policy", "resume_rate") + self.suspend_timeout = cfg.getint("policy", "suspend_timeout") + self.resume_timeout = cfg.getint("policy", "resume_timeout") + self.exclude = cfg.exclude_set() + + def managed_nodes(self): + for node in self.inventory.values(): + if node.managed and node.hostname not in self.exclude: + yield node + + # -- state reconciliation --------------------------------------------- + + def reconcile(self): + """Advance each managed host's state machine from observed reality.""" + for node in self.managed_nodes(): + h = node.hostname + cur = self.state.get_state(h) + if cur is None: + cur = self._infer_initial(h) + self.state.set_state(h, cur, self.now, + note="initial classification") + handler = getattr(self, "_st_" + cur.lower(), None) + if handler: + handler(node) + + def _infer_initial(self, hostname): + if self.snap.is_alive(hostname): + return S_UP_BUSY if self.snap.is_busy(hostname) else S_UP_IDLE + return S_POWERED_OFF + + def _st_powered_off(self, node): + # A host we believe is off but that reappeared alive: someone (or a + # boot we did not initiate) brought it up. Adopt reality. + if self.snap.is_alive(node.hostname): + new = S_UP_BUSY if self.snap.is_busy(node.hostname) else S_UP_IDLE + self.state.set_state(node.hostname, new, self.now, + note="observed alive") + + def _st_powering_up(self, node): + h = node.hostname + rec = self.state.host(h) + if self.snap.is_alive(h): + new = S_UP_BUSY if self.snap.is_busy(h) else S_UP_IDLE + self.state.set_state(h, new, self.now, note="resume complete") + log.info("%s: resume complete -> %s", h, new) + return + t_resume = rec.get("t_resume", rec.get("since", self.now)) + if self.now - t_resume > self.resume_timeout: + self.state.set_state(h, S_FAILED, self.now, + note="resume timeout (%ds)" % self.resume_timeout) + log.error("%s: FAILED -- no daemon row within resume_timeout=%ds; " + "excluding until operator clears", + h, self.resume_timeout) + + def _st_up_idle(self, node): + h = node.hostname + if not self.snap.is_present(h): + self.state.set_state(h, S_POWERED_OFF, self.now, + note="disappeared while idle") + log.warning("%s: was UP_IDLE but daemon vanished -> POWERED_OFF", h) + return + if self.snap.is_busy(h): + self.state.set_state(h, S_UP_BUSY, self.now, note="job claimed") + + def _st_up_busy(self, node): + h = node.hostname + if not self.snap.is_present(h): + self.state.set_state(h, S_POWERED_OFF, self.now, + note="disappeared while busy") + log.warning("%s: was UP_BUSY but daemon vanished -> POWERED_OFF", h) + return + if self.snap.is_idle(h): + self.state.set_state(h, S_UP_IDLE, self.now, note="became idle") + + def _st_draining(self, node): + h = node.hostname + rec = self.state.host(h) + t_drain = rec.get("t_drain", rec.get("since", self.now)) + # DRAINING -> POWERING_DOWN once all daemon rows are gone (EXITED): + # waiting for EXITED is the race-closer -- mxqd only exits after its + # drain loop finds nothing running. + if not self.snap.is_present(h): + log.info("%s: drained (all daemons exited); issuing power-off", h) + if self.act.ssh_poweroff(node): + self.state.set_state(h, S_POWERING_DOWN, self.now, + t_off=self.now, note="poweroff issued") + else: + # graceful poweroff failed -> escalate to BMC soft-off + self._escalate_soft_off(node, "ssh poweroff failed") + return + # not drained yet: escalate to BMC soft-off at half the suspend timeout + if self.now - t_drain > self.suspend_timeout // 2: + self._escalate_soft_off(node, "EXITED not seen in time") + + def _escalate_soft_off(self, node, why): + h = node.hostname + if node.on_method == "ipmi": + log.warning("%s: %s -- escalating to BMC soft-off", h, why) + self.act.ipmi_soft_off(node) + self.state.set_state(h, S_POWERING_DOWN, self.now, + t_off=self.now, note="soft-off (%s)" % why) + else: + log.warning("%s: %s -- no BMC (on_method=%s); leaving DRAINING", + h, why, node.on_method) + + def _st_powering_down(self, node): + h = node.hostname + rec = self.state.host(h) + t_off = rec.get("t_off", rec.get("t_drain", rec.get("since", self.now))) + t_drain = rec.get("t_drain", t_off) + + if self._confirmed_dark(node): + self.state.set_state(h, S_POWERED_OFF, self.now, + note="confirmed dark") + log.info("%s: confirmed powered off", h) + return + # bounded by suspend_timeout from the drain: hard-off, then FAILED + if self.now - t_drain > self.suspend_timeout: + if node.on_method == "ipmi": + log.warning("%s: still up after suspend_timeout=%ds -- BMC hard-off", + h, self.suspend_timeout) + self.act.ipmi_hard_off(node) + # give the hard-off one more tick to confirm before FAILED + if self._confirmed_dark(node): + self.state.set_state(h, S_POWERED_OFF, self.now, + note="dark after hard-off") + return + self.state.set_state(h, S_FAILED, self.now, + note="did not power down in time") + log.error("%s: FAILED -- did not power down within suspend_timeout; " + "excluding until operator clears", h) + + def _confirmed_dark(self, node): + # In dry-run nothing is actually powered off; treat absence as dark so + # the state machine can progress for observation. + bmc = self.act.ipmi_is_off(node) + if bmc is True: + return True + if bmc is False: + return False + # No BMC answer (WoL node, or dry-run): fall back to DB liveness. + return not self.snap.is_alive(node.hostname) + + # -- resume (power ON) ------------------------------------------------- + + def _free_cores_alive(self): + total = 0 + for node in self.managed_nodes(): + if self.snap.is_alive(node.hostname): + total += self.snap.live[node.hostname].slots_free + return total + + def _idle_on_count(self): + n = 0 + for node in self.managed_nodes(): + if self.state.get_state(node.hostname) == S_UP_IDLE \ + and self.snap.is_alive(node.hostname): + n += 1 + return n + + def _qualified(self, node, group): + """Reproduce server_is_qualified() (mxqd_control.c): may a job from + `group` ever run on `node`? Uses the node's declared tags and + prerequisites from the inventory (a powered-off node cannot report its + own), which the operator keeps in step with the host's mxqd config.""" + # group white/blacklist gate on the hostname (short or full) + names = {node.hostname, node.hostname_short} + if group.whitelist and names.isdisjoint(group.whitelist): + return False + if group.blacklist and not names.isdisjoint(group.blacklist): + return False + # group's prerequisites expression, evaluated against the host's tags + if not eval_prerequisites(group.prerequisites, node.match_tags): + return False + # host's own prerequisites expression, evaluated against group tags + if not eval_prerequisites(node.prerequisites, group.tags): + return False + # a GPU group needs a GPU host (job_gpu && !daemon.gpus_max) + if group.gpu and node.gpus <= 0: + return False + return True + + def _host_satisfies(self, node, group): + """Is this powered-off node worth waking for `group`? Qualification + (above) plus a capacity check that it can hold at least one job.""" + if not self._qualified(node, group): + return False + if node.slots < group.threads: + return False + if node.mem_mb and group.memory_mb and node.mem_mb < group.memory_mb: + return False + return True + + def _alive_can_serve(self, group): + """Can any alive managed host with free slots run this group's jobs?""" + for node in self.managed_nodes(): + if not self.snap.is_alive(node.hostname): + continue + live = self.snap.live[node.hostname] + if live.slots_free < group.threads: + continue + if node.mem_mb and group.memory_mb and node.mem_mb < group.memory_mb: + continue + if not self._qualified(node, group): + continue + return True + return False + + def resume(self): + snap = self.snap + free_cores = self._free_cores_alive() + deficit = snap.pend_cores - free_cores + idle_on = self._idle_on_count() + warm_deficit = max(0, self.warm_pool - idle_on) + + # groups no alive host can currently place (GPU/mem/tag gaps) + unmet = [g for g in snap.pending_groups if not self._alive_can_serve(g)] + + need = deficit > 0 or unmet or warm_deficit > 0 + if not need: + log.info("resume: no deficit (pend_cores=%d free_cores=%d " + "idle_on=%d/warm_pool=%d)", + snap.pend_cores, free_cores, idle_on, self.warm_pool) + return + + log.info("resume: deficit_cores=%d unmet_groups=%d warm_deficit=%d", + deficit, len(unmet), warm_deficit) + + candidates = [n for n in self.managed_nodes() + if self.state.get_state(n.hostname) == S_POWERED_OFF] + # newest / most efficient first == lowest gen first + candidates.sort(key=lambda n: (self._gen_key(n), -n.slots)) + + woken = 0 + for node in candidates: + if woken >= self.resume_rate: + log.info("resume: hit resume_rate=%d", self.resume_rate) + break + if not (deficit > 0 or unmet or warm_deficit > 0): + break + helps_unmet = [g for g in unmet if self._host_satisfies(node, g)] + # if the only remaining need is capability gaps, skip nodes that + # cannot close any of them + if deficit <= 0 and warm_deficit <= 0 and not helps_unmet: + continue + reason = self._resume_reason(node, deficit, helps_unmet, warm_deficit) + log.info("resume: waking %s (gen=%s slots=%d) -- %s", + node.hostname, node.gen, node.slots, reason) + if self.act.power_on(node): + self.state.set_state(node.hostname, S_POWERING_UP, self.now, + t_resume=self.now, note=reason) + woken += 1 + deficit -= node.slots + if warm_deficit > 0: + warm_deficit -= 1 + for g in helps_unmet: + if g in unmet: + unmet.remove(g) + + @staticmethod + def _resume_reason(node, deficit, helps_unmet, warm_deficit): + bits = [] + if deficit > 0: + bits.append("core deficit") + if helps_unmet: + bits.append("capability gap (%d groups)" % len(helps_unmet)) + if warm_deficit > 0: + bits.append("warm pool") + return ", ".join(bits) or "warm pool" + + # -- suspend (power OFF) ---------------------------------------------- + + def _gen_key(self, node): + """Sort key from the `gen` token; numeric part if present.""" + g = node.gen or "" + digits = "".join(c for c in g if c.isdigit()) + return (int(digits) if digits else 0, g) + + def suspend(self): + snap = self.snap + free_cores = self._free_cores_alive() + surplus = free_cores - snap.pend_cores + idle_on = self._idle_on_count() + + candidates = [] + for node in self.managed_nodes(): + h = node.hostname + if self.state.get_state(h) != S_UP_IDLE: + continue + if not snap.is_idle(h): + continue + if snap.idle_secs(h) <= self.suspend_time: + continue + if snap.uptime_secs(h) <= self.min_uptime: + continue + candidates.append(node) + + # oldest / least efficient first == highest gen first + candidates.sort(key=lambda n: self._gen_key(n), reverse=True) + + if not candidates: + log.info("suspend: no eligible idle candidates " + "(surplus_cores=%d idle_on=%d)", surplus, idle_on) + return + + drained = 0 + for node in candidates: + h = node.hostname + if drained >= self.suspend_rate: + log.info("suspend: hit suspend_rate=%d", self.suspend_rate) + break + free = snap.live[h].slots_free + if surplus - free < 0: + log.info("suspend: keeping %s -- surplus would go negative " + "(surplus=%d, host_free=%d)", h, surplus, free) + continue + if idle_on - 1 < self.warm_pool: + log.info("suspend: stopping -- warm_pool floor (idle_on=%d, " + "warm_pool=%d)", idle_on, self.warm_pool) + break + log.info("suspend: draining %s (gen=%s idle=%dm surplus=%dc) ", + h, node.gen, snap.idle_secs(h) // 60, surplus) + if self.act.ssh_drain(node): + self.state.set_state(h, S_DRAINING, self.now, + t_drain=self.now, note="drain issued") + drained += 1 + surplus -= free + idle_on -= 1 + + # -- one full tick ----------------------------------------------------- + + def tick(self): + log.info("tick: now=%d dry_run=%s pend_jobs=%d pend_cores=%d " + "live_hosts=%d", + self.now, self.act.dry_run, self.snap.pend_jobs, + self.snap.pend_cores, len(self.snap.live)) + self.reconcile() + self.resume() + self.suspend() + + +# =========================================================================== +# CLI +# =========================================================================== + +def setup_logging(cfg, to_stderr): + handlers = [] + logfile = cfg.get("paths", "log_file") + if logfile: + try: + d = os.path.dirname(logfile) + if d: + os.makedirs(d, exist_ok=True) + handlers.append(logging.FileHandler(logfile)) + except OSError as exc: + print("mxq-powerd: cannot open log file %s: %s" % (logfile, exc), + file=sys.stderr) + if to_stderr or not handlers: + handlers.append(logging.StreamHandler(sys.stderr)) + fmt = logging.Formatter("%(asctime)s %(levelname)s %(message)s", + "%Y-%m-%d %H:%M:%S") + for h in handlers: + h.setFormatter(fmt) + log.setLevel(logging.INFO) + for h in handlers: + log.addHandler(h) + + +def cmd_status(cfg, args): + """Print the current per-host state without touching anything.""" + state = StateStore(cfg.get("paths", "state_file")) + state.lock() + try: + if not state.hosts: + print("(no state recorded yet)") + return 0 + width = max(len(h) for h in state.hosts) + for host in sorted(state.hosts): + rec = state.hosts[host] + note = rec.get("note", "") + since = rec.get("since") + age = "" + if since: + age = " since=%s" % time.strftime("%Y-%m-%d %H:%M:%S", + time.localtime(since)) + print("%-*s %-14s%s %s" % (width, host, rec.get("state", "?"), + age, note)) + finally: + state.unlock() + return 0 + + +def cmd_clear(cfg, args): + """Clear a FAILED host so automation may manage it again.""" + state = StateStore(cfg.get("paths", "state_file")) + state.lock() + try: + rec = state.hosts.get(args.clear) + if not rec: + print("mxq-powerd: no state for host %s" % args.clear, + file=sys.stderr) + return 1 + old = rec.get("state") + state.hosts.pop(args.clear, None) + state.save() + print("cleared %s (was %s); it will be re-classified next tick" + % (args.clear, old)) + finally: + state.unlock() + return 0 + + +def cmd_tick(cfg, args): + inventory = load_inventory(cfg.get("paths", "nodes_conf")) + secrets = Secrets(cfg.get("paths", "secrets_file")) + dry_run = cfg.getbool("mode", "dry_run") + if args.arm: + dry_run = False + if args.dry_run: + dry_run = True + + db = Database(cfg) + snap = Snapshot(db, cfg) + actuator = Actuator(cfg, secrets, dry_run) + + state = StateStore(cfg.get("paths", "state_file")) + state.lock() + try: + mgr = Manager(cfg, inventory, secrets, state, snap, actuator) + mgr.tick() + state.save() + finally: + state.unlock() + return 0 + + +def main(argv=None): + p = argparse.ArgumentParser( + prog="mxq-powerd", + description="MXQ node power manager -- one evaluation tick per run.") + p.add_argument("-c", "--config", default="/etc/mxq/powerd.conf", + help="config file (default: /etc/mxq/powerd.conf)") + p.add_argument("--arm", action="store_true", + help="override config: actually actuate (dry_run=false)") + p.add_argument("--dry-run", action="store_true", + help="override config: log only, never actuate") + p.add_argument("--stderr", action="store_true", + help="also log to stderr") + p.add_argument("--status", action="store_true", + help="print per-host state and exit") + p.add_argument("--clear", metavar="HOST", + help="clear a FAILED/stuck host from the state and exit") + args = p.parse_args(argv) + + cfg = Config(args.config if os.path.exists(args.config) else None) + setup_logging(cfg, to_stderr=args.stderr or args.status or bool(args.clear)) + + if args.status: + return cmd_status(cfg, args) + if args.clear: + return cmd_clear(cfg, args) + return cmd_tick(cfg, args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/powerd/mxq-powerd.service b/powerd/mxq-powerd.service new file mode 100644 index 00000000..41ea6b5e --- /dev/null +++ b/powerd/mxq-powerd.service @@ -0,0 +1,11 @@ +# /etc/systemd/system/mxq-powerd.service +[Unit] +Description=MXQ node power manager (one evaluation tick) +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/libexec/mxq/mxq-powerd --config /etc/mxq/powerd.conf +# The manager needs root to ssh to nodes and drive BMCs. +User=root diff --git a/powerd/mxq-powerd.timer b/powerd/mxq-powerd.timer new file mode 100644 index 00000000..830ca506 --- /dev/null +++ b/powerd/mxq-powerd.timer @@ -0,0 +1,20 @@ +# /etc/systemd/system/mxq-powerd.timer +# +# Enable with: systemctl enable --now mxq-powerd.timer +# Disable power management globally with: systemctl stop mxq-powerd.timer +# +# IMPORTANT: enable this timer on EXACTLY ONE management node (e.g. afk). +# mxq-powerd holds no cross-node lock; two instances would fight over the same +# nodes. That node needs an ssh key/account whose authorized_keys entry is +# restricted (command="...") to just the drain and poweroff commands the +# manager runs -- see powerd/README.md. +[Unit] +Description=Run MXQ power manager every minute + +[Timer] +OnBootSec=2min +OnUnitActiveSec=60s +AccuracySec=5s + +[Install] +WantedBy=timers.target diff --git a/powerd/nodes.conf.example b/powerd/nodes.conf.example new file mode 100644 index 00000000..f81d2c2a --- /dev/null +++ b/powerd/nodes.conf.example @@ -0,0 +1,37 @@ +# /etc/mxq/nodes.conf -- static per-host inventory for mxq-powerd. +# +# Flat, '#'-comment, whitespace-separated (the /etc/hostconfig house style). +# One line per HOST (not per daemon). This describes what a powered-OFF node +# can do -- the live DB cannot tell us once the node is dark. +# +# Columns: +# hostname the short hostname as it appears in mxq_daemon.hostname +# gen generation / power-priority token. Suspend picks the +# HIGHEST gen (oldest, least efficient) first; resume the +# LOWEST (newest) first. '-' => never auto-managed. +# slots CPU slots the node offers +# mem_mb memory in MB +# gpus number of GPUs +# tags comma-separated capability tags ('-' for none). Mirror the +# hostconfig/cpufeature tags this host's mxqd publishes; a +# job group's `prerequisites` expression is matched against +# them. (mxqd also adds `true`, the hostname and short +# hostname implicitly -- do not list those.) +# prerequisites the host's own mxqd --prerequisites expression, matched +# against a group's tags, or '-' for none. Because columns +# are whitespace-separated, write it without spaces, e.g. +# `gpu&!maintenance`. +# on_method ipmi | wol | none ('none' => excluded from all automation) +# bmc_or_mac BMC IP/hostname for ipmi, MAC address for wol ('-' if none) +# +# `tags`/`prerequisites` let mxq-powerd reproduce MXQ's job/host match (see +# server_is_qualified() in mxqd_control.c) while a node is off, so it only +# wakes a node that could actually accept the pending work. +# +# IPMI credentials do NOT go here -- see /etc/mxq/powerd-secrets. + +# hostname gen slots mem_mb gpus tags prerequisites on_method bmc_or_mac +node001 g1 64 257000 0 xeon,highmem - ipmi 10.1.0.11 +node002 g1 64 257000 0 xeon,highmem - ipmi 10.1.0.12 +node101 g3 256 980000 4 epyc,gpu,a100 - wol e4:3d:1a:00:11:22 +login01 - - - - infra - none - diff --git a/powerd/powerd-secrets.example b/powerd/powerd-secrets.example new file mode 100644 index 00000000..28e9704c --- /dev/null +++ b/powerd/powerd-secrets.example @@ -0,0 +1,16 @@ +# /etc/mxq/powerd-secrets -- IPMI/BMC credentials for mxq-powerd. +# +# Install as root, mode 0600: +# install -m 0600 -o root -g root powerd-secrets.example /etc/mxq/powerd-secrets +# +# Kept out of the world-readable nodes.conf on purpose. The password is passed +# to ipmitool via $IPMI_PASSWORD (ipmitool -E), never on the command line. + +[ipmi] +user = ADMIN +password = changeme + +# Optional per-host overrides (section name is ipmi. ): +#[ipmi.node001] +#user = root +#password = another-secret diff --git a/powerd/powerd.conf.example b/powerd/powerd.conf.example new file mode 100644 index 00000000..7bb061a9 --- /dev/null +++ b/powerd/powerd.conf.example @@ -0,0 +1,52 @@ +# /etc/mxq/powerd.conf -- global policy for mxq-powerd. +# +# Copy to /etc/mxq/powerd.conf and edit. All keys shown are the built-in +# defaults; you only need to list the ones you change. + +[db] +# How mxq-powerd reaches the MySQL server. It only ever runs SELECTs. +# +# By default it reuses MXQ's read-only client config file and extracts the +# connection options from the named group with `my_print_defaults` (the +# `mysql` client itself only reads the [client]/[mysql] groups). A dedicated +# read-only account is recommended. +mysql = mysql +my_print_defaults = my_print_defaults +defaults_file = /etc/mxq/mysql_ro.cnf +defaults_group = mxqclient +database = mxq +# To use a file whose credentials live under a plain [client] section instead, +# leave defaults_group empty: +# defaults_group = + +[policy] +suspend_time = 1800 # sec a node must be idle before it may power down +warm_pool = 4 # min idle managed nodes to keep powered ON +min_uptime = 900 # sec a node must be up before it is suspend-eligible +suspend_rate = 4 # max power-downs initiated per tick +resume_rate = 8 # max power-ups initiated per tick +suspend_timeout = 300 # sec for a shutdown to complete before escalation +resume_timeout = 600 # sec for a boot+register before the node is FAILED +mtime_stale = 180 # sec: mxq_daemon.mtime older than this => not alive +# Hostnames never managed (login/storage/infra). Belt-and-suspenders: a node +# is also protected by gen '-' or on_method 'none' in nodes.conf. +exclude = login01,storage0,storage1 + +[mode] +# v1 default: log every decision, take NO action. Run for a few days against +# a live cluster and eyeball the log before flipping this to false. +dry_run = true + +[paths] +nodes_conf = /etc/mxq/nodes.conf +secrets_file = /etc/mxq/powerd-secrets +state_file = /var/lib/mxq/powerd-state.json +log_file = /var/log/mxq/powerd.log + +[actuation] +ssh = ssh +ssh_opts = -o BatchMode=yes -o ConnectTimeout=10 +ipmitool = ipmitool +wakeonlan = wakeonlan +drain_cmd = mxqdctl-hostconfig stop +poweroff_cmd = systemctl poweroff