92 lines
4.5 KiB
Python
92 lines
4.5 KiB
Python
"""Run a heavy command once it holds one of the machine's heavy slots, so sessions running at once cannot starve the machine.
|
|
|
|
usage: run-heavy <command> [<arg>...]
|
|
|
|
A heavy command is one that loads the machine for minutes: a full gate, mutation tests, an e2e run,
|
|
a docker build. The machine has one heavy slot for every 4 CPUs that run-heavy may use, and at least one, shared by every
|
|
session that runs run-heavy. Each slot is a file locked with flock, under
|
|
$XDG_STATE_HOME/run-heavy (default ~/.local/state/run-heavy). While every slot is held, run-heavy
|
|
says so once on stderr and waits. It then replaces itself with the command. The slot's lock stays held
|
|
until the command, and every process that inherited the lock, exits. Its exit code is the command's.
|
|
|
|
On Linux, run-heavy also caps the memory of each container the command makes with docker run or
|
|
docker create. The cap is the slot's share: three quarters of the machine's memory, split evenly
|
|
across the slots, handed to the command in RUN_HEAVY_MEMORY, in MiB. run-heavy puts its own docker
|
|
first on the command's PATH. That docker adds the limits a container lacks, then runs the real one:
|
|
|
|
- A container that sets neither limit gets a memory limit and a swap limit, both the share.
|
|
- A container that sets one of them keeps it, and gets the other at the same size.
|
|
- A --memory-swap of -1, unlimited swap, is dropped and replaced.
|
|
- A --memory of 0, unlimited memory, leaves the call as it is.
|
|
|
|
Every other docker call goes through unchanged. A docker compose service, and a docker named by its
|
|
full path, get no cap: give those containers their own limits.
|
|
|
|
A command it runs carries the slot's number in RUN_HEAVY_SLOT, and run-heavy called with that set
|
|
runs its command at once, so a gate that wraps its own steps cannot wait on itself.
|
|
|
|
As GNU env, nice and timeout do, it exits 125, saying why, when no command is given or the slot
|
|
folder or a slot file cannot be made or locked; 126 when the command is found but cannot run; and 127 when it is not found.
|
|
"""
|
|
import os
|
|
import pathlib
|
|
import shutil
|
|
import sys
|
|
import typing
|
|
|
|
from run_heavy.capped_docker import CANNOT_RUN, NOT_FOUND, PATH_ENTRY, SHARE_VARIABLE, without_path_entry
|
|
from failure import Failure
|
|
from machine_load.memory import Memory
|
|
from machine_load.slots import slot_count, slot_folder, waited_slot
|
|
|
|
OWN_FAILURE = 125
|
|
HELD = "RUN_HEAVY_SLOT"
|
|
USAGE = "usage: run-heavy <command> [<arg>...]"
|
|
|
|
|
|
def main(argv: list[str], env: typing.Mapping[str, str], cpus: int | None, memory: Memory | None) -> int:
|
|
if argv[:1] in (["-h"], ["--help"]):
|
|
print(__doc__.strip())
|
|
return 0
|
|
if not argv:
|
|
print(f"run-heavy: no command given; {USAGE}", file=sys.stderr)
|
|
return OWN_FAILURE
|
|
if HELD not in env:
|
|
slot = held_slot(slot_folder(env), slot_count(cpus))
|
|
if isinstance(slot, Failure):
|
|
print(f"run-heavy: {slot.reason}", file=sys.stderr)
|
|
return OWN_FAILURE
|
|
env = {**env, HELD: str(slot), **capping(memory, slot_count(cpus), env)}
|
|
return run(argv, env)
|
|
|
|
|
|
def capping(memory: Memory | None, slots: int, env: typing.Mapping[str, str]) -> dict[str, str]:
|
|
"""What the command's environment needs for its docker to cap each container it makes at the slot's share of memory; none where the memory is unknown, as off Linux, or where PATH finds no docker to cap."""
|
|
path = env.get("PATH", "")
|
|
if memory is None or shutil.which("docker", path=without_path_entry(path)) is None:
|
|
return {}
|
|
return {SHARE_VARIABLE: str(memory.slot_share(slots)), "PATH": os.pathsep.join([str(PATH_ENTRY), path])}
|
|
|
|
|
|
def held_slot(folder: pathlib.Path, slots: int) -> int | Failure:
|
|
"""The number of the slot this call holds, its lock left for the command to inherit, or why no slot could be taken."""
|
|
try:
|
|
folder.mkdir(parents=True, exist_ok=True)
|
|
slot, fd = waited_slot(folder, slots, lambda: print(f"run-heavy: waiting for a heavy slot, of {slots} on this machine, in {folder}; `fuser {folder}/slot-*` names the processes that hold them", file=sys.stderr, flush=True))
|
|
except OSError as failure:
|
|
return Failure(f"cannot take a heavy slot in {folder}: {failure.strerror}")
|
|
os.set_inheritable(fd, True)
|
|
return slot
|
|
|
|
|
|
def run(argv: list[str], env: typing.Mapping[str, str]) -> int:
|
|
"""Replaces this process with the command; returns only where the command cannot start."""
|
|
try:
|
|
os.execvpe(argv[0], argv, env)
|
|
except FileNotFoundError:
|
|
print(f"run-heavy: {argv[0]}: command not found", file=sys.stderr)
|
|
return NOT_FOUND
|
|
except OSError as failure:
|
|
print(f"run-heavy: cannot run {argv[0]}: {failure.strerror}", file=sys.stderr)
|
|
return CANNOT_RUN
|