Files
claude-config/bin/do_epic/scheduler.py
T

223 lines
12 KiB
Python

"""Which of an epic's issues run as chunks now, each in its own window of the epic's tmux session (docs/decisions.md → `do-epic: drives one Jira epic …`).
Three limits hold a chunk back: the epic's cap of chunks at once, its repo's `parallel-chunks`, counted across every epic, and a loop-claude that runs chunks in its repo. A parked chunk (`parked`) counts toward none of them: its window only polls the forge, and `resume` gives it a running slot under the same limits once its worker has something to do. do-epic checks a repo and starts a chunk in it while holding the lock loop-claude holds when it starts one (`chunk_driver.claims.starting`), so the two never start chunks in one checkout together."""
import dataclasses
import pathlib
import typing
from do_epic.epic_record import CAP
from chunk_driver.claims import CHUNK_FLAG, epic_chunks, HANDED, LOCK, loop_chunks, project_lock, starting
from do_epic.backend.protocol import EpicIssue
from failure import Failure
from do_epic.repo_pick import HumanTask, recorded, Repo
from project_config.settings import PARALLEL_CHUNKS
from do_epic.parked import parked_chunks
from chunk_pr.approval import PARKS
from do_epic.split import SPLIT_LABEL
from agent_cycle import tmux
# Under the home folder, a folder per epic key holds the epic's LOCK. A chunk without a repo works there.
STATE = ".local/state/do-epic"
# Why a human's task with a live window waits, as the epic's status says.
HUMAN_WAIT = "it is a human's task; do-epic polls Jira until you close it"
@dataclasses.dataclass(frozen=True)
class Chunk:
"""An issue to work as one chunk, in the window named for its key."""
issue: str
# The name of the repo it changes; None for work without a repo.
repo: str | None
# Whether it is a human's task that holds up another open issue: its window only waits for the human to close the issue (`human_tasks`), and doesn't take a running slot.
human: bool = False
@dataclasses.dataclass(frozen=True)
class Waits:
"""An open issue that doesn't run now, and why."""
issue: str
reason: str
@dataclasses.dataclass(frozen=True)
class Started:
# The issues whose chunks started, and the epic's key where its finishing did.
started: list[str]
waits: list[Waits]
# Whether the epic is closed, so that not even its finishing starts.
closed: bool = False
@dataclasses.dataclass(frozen=True)
class RepoRoom:
"""What limits the chunks in one repo's checkout."""
# The repo's `parallel-chunks`.
limit: int
# How many chunks of any epic run there.
running: int
# Whether a loop-claude runs chunks there.
loop_claude: bool
@dataclasses.dataclass(frozen=True)
class Epic:
"""The epic whose chunks start."""
key: str
# How many of its chunks may run at once.
cap: int
# The checkout of the repo with this name and its `parallel-chunks`, or why the repo cannot be used.
locate: typing.Callable[[str], tuple[pathlib.Path, int] | Failure]
home: pathlib.Path
# The do-epic script each window runs.
launcher: pathlib.Path
env: typing.Mapping[str, str]
@property
def folder(self) -> pathlib.Path:
return self.home / STATE / self.key
def open_blockers(issue: EpicIssue, by_key: dict[str, EpicIssue]) -> list[str]:
"""The keys of the issues that block issue, or its parent, that are not done. A blocker outside the epic counts, since its state is unknown."""
parent = by_key.get(issue.parent)
return [key for key in (*issue.blockers, *(parent.blockers if parent else ())) if key not in by_key or not by_key[key].done]
def held_up(issues: list[EpicIssue]) -> dict[str, list[str]]:
"""Each blocker's key, mapped to the keys of the open issues it holds up, in the tracker's rank order. A split item's blocker holds up the item's sub-tasks."""
by_key = {issue.key: issue for issue in issues}
held: dict[str, list[str]] = {}
for issue in issues:
if not issue.done and SPLIT_LABEL not in issue.labels:
for key in open_blockers(issue, by_key):
held.setdefault(key, []).append(issue.key)
return held
def chunks(issues: list[EpicIssue]) -> tuple[list[Chunk], list[Waits]]:
"""The open issues to work as chunks, in the tracker's rank order, and those that wait. An item split by repo is worked through its sub-tasks, and what blocks it blocks them. A blocker outside the epic blocks until the human removes the link, since its state is unknown. A human's task gets a window only while it holds up another issue and nothing holds it up; that window waits for the human."""
by_key = {issue.key: issue for issue in issues}
holding = held_up(issues)
found: list[Chunk] = []
waits: list[Waits] = []
for issue in issues:
if issue.done or SPLIT_LABEL in issue.labels:
continue
pick = recorded(issue)
blocking = open_blockers(issue, by_key)
outside = [key for key in blocking if key not in by_key]
if isinstance(pick, Failure):
waits.append(Waits(issue.key, pick.reason))
elif pick is None:
waits.append(Waits(issue.key, "no label names its repo"))
elif isinstance(pick, HumanTask) and not blocking and issue.key in holding:
found.append(Chunk(issue.key, None, human=True))
elif isinstance(pick, HumanTask):
waits.append(Waits(issue.key, "it is a human's task"))
elif blocking:
waits.append(Waits(issue.key, f"{' and '.join(blocking)} block{'s' if len(blocking) == 1 else ''} it" + (f"; do-epic cannot see {' and '.join(outside)}, outside the epic, finish: remove such a link in Jira once its issue is done" if outside else "")))
else:
found.append(Chunk(issue.key, pick.name if isinstance(pick, Repo) else None))
return found, waits
def refusal(repo: str, room: RepoRoom) -> str | None:
"""Why the repo takes no further chunk; None where it takes one."""
if room.loop_claude:
return f"loop-claude runs chunks in {repo}"
if room.running >= room.limit:
return f"{repo} runs as many chunks as its `{PARALLEL_CHUNKS}` allows: {room.limit}"
return None
def start_chunks(epic: Epic, issues: list[EpicIssue], finishing: bool, own: str | None, run: tmux.Tmux) -> Started | Failure:
"""Starts a do-epic in a new window for each of the epic's chunks that may start now. Where finishing is set, no issue is open, and the one chunk to start is the epic's finishing (finish.py), in a window named for the epic's key. own is the key of the chunk that asks, which has finished or parked: it counts as running nowhere, and no stopped chunk starts again. Where own is None, the human's `do-epic <EPIC-KEY>` asks, and each stopped chunk starts again. A parked chunk waits in its window; one without a live window, as after a reboot, starts again as a stopped chunk does, whatever the limits, since it goes on parked. A Failure says why a window could not start; those before it run."""
session = tmux.session_name(tmux.EPIC, epic.key)
found, waits = chunks(issues)
if finishing:
found.append(Chunk(epic.key, None))
started: list[str] = []
parked = parked_chunks(epic.folder)
if isinstance(parked, Failure):
return parked
with starting(epic.folder) as unlocked:
if unlocked:
return unlocked
windows = tmux.listed(session, run)
live = {window.name: window for window in windows if not window.dead and window.name != own}
running = {window.name for window in windows if not window.dead and not window.parked} - {own}
stopped = {window.name for window in windows if window.dead} - set(live)
# The windows this call started that take a running slot.
taken = 0
for chunk in found:
if chunk.issue in running or chunk.issue == own:
continue
slot = chunk.issue not in parked and not chunk.human
if chunk.human and chunk.issue in live:
waits.append(Waits(chunk.issue, HUMAN_WAIT))
elif chunk.issue in parked and chunk.issue in live:
waits.append(Waits(chunk.issue, parked[chunk.issue].waits(live[chunk.issue].awaits)))
elif own and (chunk.issue in stopped or chunk.issue in parked):
waits.append(Waits(chunk.issue, f"its window stopped; run `do-epic {epic.key}` to start it again"))
elif slot and len(running) + taken >= epic.cap:
waits.append(Waits(chunk.issue, f"the epic runs as many chunks as its `{CAP}` line allows: {epic.cap}"))
else:
now = start_chunk(epic, chunk if chunk.issue not in parked else Chunk(chunk.issue, None), own, windows, run)
if isinstance(now, Failure):
return now
if isinstance(now, str):
waits.append(Waits(chunk.issue, now))
else:
windows = now
started.append(chunk.issue)
taken += slot
# The epic's finishing is no issue: its wait is listed after every issue's.
order = [*(issue.key for issue in issues), epic.key]
return Started(started, sorted(waits, key=lambda wait: order.index(wait.issue)))
def start_chunk(epic: Epic, chunk: Chunk, own: str | None, windows: list[tmux.Listed], run: tmux.Tmux) -> list[tmux.Listed] | str | Failure:
"""Starts chunk's window beside windows, the epic's session's, and returns the session's windows afterwards; the reason where its repo takes no chunk now, and a Failure where tmux started none, or where the repo's lock cannot be taken. It holds that lock from the check to the start. A chunk without a repo starts in the epic's folder. So does a parked chunk, since it takes no room in its repo."""
session = tmux.session_name(tmux.EPIC, epic.key)
handed = {**{name: value for name, value in epic.env.items() if name.startswith(HANDED)}, PARKS: "1"}
command = [str(epic.launcher), CHUNK_FLAG, chunk.issue, epic.key]
if chunk.repo is None:
return tmux.start_chunk(tmux.Window(session, chunk.issue, ""), str(epic.folder), handed, command, windows, False, run)
located = epic.locate(chunk.repo)
if isinstance(located, Failure):
return located.reason
checkout, limit = located
with starting(project_lock(str(checkout), epic.home)) as unlocked:
if unlocked:
return unlocked
if reason := refusal(chunk.repo, room_in(str(checkout), limit, (session, own), epic.home, run)):
return reason
return tmux.start_chunk(tmux.Window(session, chunk.issue, ""), str(checkout), handed, command, windows, False, run)
def room_in(checkout: str, limit: int, leaving: tuple[str, str | None], home: pathlib.Path, run: tmux.Tmux) -> RepoRoom:
"""The room in a repo's checkout under its limit. leaving is the session and name of the window that asks, which doesn't count: its chunk finished or is parked."""
theirs = [placed for placed in epic_chunks(checkout, run) if (placed.session, placed.window.name) != leaving]
return RepoRoom(limit, len(theirs), bool(loop_chunks(checkout, home, run)))
def resume(epic: Epic, repo: str, window: tmux.Window, run: tmux.Tmux) -> str | None:
"""Takes a running slot for the parked chunk in window, whose worker has something to do again in the repo called repo: None once the window counts as a running chunk, else why the chunk still waits. The limits are those a starting chunk meets, checked under the same locks. The window itself never counts against them: one started again for a parked chunk, as after a reboot, isn't named as parked until its first read of the gate."""
with starting(epic.folder) as unlocked:
if unlocked:
return unlocked.reason
running = [listed for listed in tmux.listed(window.session, run) if not listed.dead and not listed.parked and listed.name != window.name]
if len(running) >= epic.cap:
return f"the epic runs as many chunks as its `{CAP}` line allows: {epic.cap}"
located = epic.locate(repo)
if isinstance(located, Failure):
return located.reason
checkout, limit = located
with starting(project_lock(str(checkout), epic.home)) as unlocked:
if unlocked:
return unlocked.reason
if reason := refusal(repo, room_in(str(checkout), limit, (window.session, window.name), epic.home, run)):
return reason
tmux.unpark(window, str(checkout), run)
return None