Files

93 lines
4.7 KiB
Python

"""Run a heavy command once it holds one of the machine's heavy slots, so sessions running at once cannot starve the machine.
usage: run-heavy <command> [<arg>...]
A heavy command is one that loads the machine for minutes: a full gate, mutation tests, an e2e run,
a docker build. The machine has one heavy slot for every 4 CPUs that run-heavy may use, and at least one, shared by every
session that runs run-heavy. Each slot is a file locked with flock, under
$XDG_STATE_HOME/run-heavy (default ~/.local/state/run-heavy). While every slot is held, run-heavy
says so once on stderr and waits. It then replaces itself with the command. The slot's lock stays held
until the command, and every process that inherited the lock, exits. Its exit code is the command's.
On Linux, run-heavy also caps the memory of each container the command makes with docker run or
docker create. The cap is the slot's share: three quarters of the machine's memory, split evenly
across the slots, handed to the command in RUN_HEAVY_MEMORY, in MiB. run-heavy puts its own docker
first on the command's PATH. That docker adds the limits a container lacks, then runs the real one:
- A container that sets neither limit gets a memory limit and a swap limit, both the share.
- A container that sets one of them keeps it, and gets the other at the same size.
- A --memory-swap of -1, unlimited swap, is dropped and replaced.
- A --memory of 0, unlimited memory, leaves the call as it is.
Every other docker call goes through unchanged. A docker compose service, and a docker named by its
full path, get no cap: give those containers their own limits.
A command it runs carries the slot's number in RUN_HEAVY_SLOT, and run-heavy called with that set
runs its command at once, so a gate that wraps its own steps cannot wait on itself.
As GNU env, nice and timeout do, it exits 125, saying why, when no command is given or the slot
folder or a slot file cannot be made or locked; 126 when the command is found but cannot run; and 127 when it is not found.
"""
import fcntl
import os
import pathlib
import shutil
import sys
import time
import typing
from run_heavy.capped_docker import CANNOT_RUN, NOT_FOUND, PATH_ENTRY, SHARE_VARIABLE, without_path_entry
from failure import Failure
from machine_load.slots import HELD, Lock, slot_count, slot_folder, waited_slot
from machine_load.memory import Memory
OWN_FAILURE = 125
USAGE = "usage: run-heavy <command> [<arg>...]"
def main(argv: list[str], env: typing.Mapping[str, str], cpus: int | None, memory: Memory | None) -> int:
if argv[:1] in (["-h"], ["--help"]):
print(__doc__.strip())
return 0
if not argv:
print(f"run-heavy: no command given; {USAGE}", file=sys.stderr)
return OWN_FAILURE
if HELD not in env:
slot = held_slot(slot_folder(env, pathlib.Path.home()), slot_count(cpus), time.sleep, fcntl.flock)
if isinstance(slot, Failure):
print(f"run-heavy: {slot.reason}", file=sys.stderr)
return OWN_FAILURE
env = {**env, HELD: str(slot), **capping(memory, slot_count(cpus), env)}
return run(argv, env)
def capping(memory: Memory | None, slots: int, env: typing.Mapping[str, str]) -> dict[str, str]:
"""What the command's environment needs for its docker to cap each container it makes at the slot's share of memory; none where the memory is unknown, as off Linux, or where PATH finds no docker to cap."""
path = env.get("PATH", "")
if memory is None or shutil.which("docker", path=without_path_entry(path)) is None:
return {}
return {SHARE_VARIABLE: str(memory.slot_share(slots)), "PATH": os.pathsep.join([str(PATH_ENTRY), path])}
def held_slot(folder: pathlib.Path, slots: int, sleep: typing.Callable[[float], None], lock: Lock) -> int | Failure:
"""The number of the slot this call holds, its lock left for the command to inherit, or why no slot could be taken; it waits for one with sleep, and locks it with lock."""
try:
folder.mkdir(parents=True, exist_ok=True)
slot, fd = waited_slot(folder, slots, lambda: print(f"run-heavy: waiting for a heavy slot, of {slots} on this machine, in {folder}; `fuser {folder}/slot-*` names the processes that hold them", file=sys.stderr, flush=True), sleep, lock)
except OSError as failure:
return Failure(f"cannot take a heavy slot in {folder}: {failure.strerror}")
os.set_inheritable(fd, True)
return slot
def run(argv: list[str], env: typing.Mapping[str, str]) -> int:
"""Replaces this process with the command; returns only where the command cannot start."""
try:
os.execvpe(argv[0], argv, env)
except FileNotFoundError:
print(f"run-heavy: {argv[0]}: command not found", file=sys.stderr)
return NOT_FOUND
except OSError as failure:
print(f"run-heavy: cannot run {argv[0]}: {failure.strerror}", file=sys.stderr)
return CANNOT_RUN