Files
claude-config/bin/do_epic/scheduler.py
T

254 lines
14 KiB
Python

"""Which of an epic's issues run as chunks now, each in its own window of the epic's tmux session (docs/decisions.md → `do-epic: drives one Jira epic …`).
Three limits hold a chunk back: the epic's cap of chunks at once, its repo's `parallel-chunks`, counted across every epic, and a loop-claude that runs chunks in its repo. A parked chunk (`parked`) counts toward none of them: its window only polls the forge, and `resume` gives it a running slot under the same limits once its worker has something to do. do-epic checks a repo and starts a chunk in it while holding the lock loop-claude holds when it starts one (`chunk_driver.claims.starting`), so the two never start chunks in one checkout together."""
import dataclasses
import pathlib
import typing
from do_epic.epic_record import CAP
from chunk_driver.claims import CHUNK_FLAG, epic_chunks, HANDED, loop_chunks, project_lock, starting
from do_epic.backend.protocol import EpicIssue
from failure import Failure
from do_epic.split import finished, is_split, SPLIT_LABEL
from do_epic.repo_pick import HumanTask, LabelRepos, Pick, recorded, Repo
from text_files.markdown_text import listed
from project_config.settings import PARALLEL_CHUNKS
from do_epic.parked import parked_chunks
from chunk_pr.approval import PARKS
from agent_cycle import tmux
# Under the home folder, a folder per epic key holds the epic's lock file, which `claims.LOCK` names. A chunk without a repo works there.
STATE = ".local/state/do-epic"
# Why a human's task with a live window waits, as the epic's status says.
HUMAN_WAIT = "it is a human's task; do-epic polls Jira until you close it"
@dataclasses.dataclass(frozen=True)
class Chunk:
"""An issue to work as one chunk, in the window named for its key."""
issue: str
# The name of the repo it changes; None for work without a repo.
repo: str | None
# Whether it is a human's task that holds up another open issue: its window only waits for the human to close the issue (`human_tasks`), and doesn't take a running slot.
human: bool = False
@dataclasses.dataclass(frozen=True)
class Waits:
"""An open issue that doesn't run now, and why."""
issue: str
reason: str
@dataclasses.dataclass(frozen=True)
class Started:
# The issues whose chunks started, and the epic's key where its finishing did.
started: list[str]
waits: list[Waits]
# Whether the epic is closed, so that not even its finishing starts.
closed: bool = False
@dataclasses.dataclass(frozen=True)
class RepoRoom:
"""What limits the chunks in one repo's checkout."""
# The repo's `parallel-chunks`.
limit: int
# How many chunks of any epic run there.
running: int
# Whether a loop-claude runs chunks there.
loop_claude: bool
@dataclasses.dataclass(frozen=True)
class Epic:
"""The epic whose chunks start."""
key: str
# How many of its chunks may run at once.
cap: int
# The checkout of the repo with this name and its `parallel-chunks`, or why the repo cannot be used.
locate: typing.Callable[[str], tuple[pathlib.Path, int] | Failure]
home: pathlib.Path
# The do-epic script each window runs.
launcher: pathlib.Path
env: typing.Mapping[str, str]
# The team's own labels for its repos.
team: LabelRepos
# The statuses `jira-unblocking-statuses` names; `settled` says what they do.
unblocking: tuple[str, ...] = ()
@property
def folder(self) -> pathlib.Path:
return self.home / STATE / self.key
def unblocks(status: str, unblocking: tuple[str, ...]) -> bool:
"""Whether status is one of the unblocking statuses, whatever its case, so a name typed in another case than Jira shows still matches."""
return status.casefold() in {name.casefold() for name in unblocking}
def settled(issue: EpicIssue, by_key: dict[str, EpicIssue], unblocking: tuple[str, ...]) -> bool:
"""Whether the issue's work has landed, so that it takes no chunk and unblocks what it blocks: the tracker counts it done, or its status is one of unblocking, such as one the team's automation gives an issue whose pull request merged. An item marked SPLIT_LABEL has landed once each of its sub-tasks has, since nobody moves it before they are done."""
tasks = [task for task in by_key.values() if task.parent == issue.key] if SPLIT_LABEL in issue.labels else []
return issue.done or unblocks(issue.status, unblocking) or bool(tasks) and all(settled(task, by_key, unblocking) for task in tasks)
def open_blockers(issue: EpicIssue, by_key: dict[str, EpicIssue], unblocking: tuple[str, ...] = ()) -> list[str]:
"""The keys of the issues that block issue, or its parent, that are not settled. A blocker outside the epic counts, since its state is unknown."""
parent = by_key.get(issue.parent)
return [key for key in (*issue.blockers, *(parent.blockers if parent else ())) if key not in by_key or not settled(by_key[key], by_key, unblocking)]
def held_up(issues: list[EpicIssue], unblocking: tuple[str, ...] = ()) -> dict[str, list[str]]:
"""Each blocker's key, mapped to the keys of the open issues it holds up, in the tracker's rank order. A split item's blocker holds up the item's sub-tasks."""
by_key = {issue.key: issue for issue in issues}
held: dict[str, list[str]] = {}
for issue in issues:
if not settled(issue, by_key, unblocking) and not is_split(issue, issues):
for key in open_blockers(issue, by_key, unblocking):
held.setdefault(key, []).append(issue.key)
return held
def chunks(issues: list[EpicIssue], team: LabelRepos, unblocking: tuple[str, ...] = ()) -> tuple[list[Chunk], list[Waits]]:
"""The open issues to work as chunks, in the tracker's rank order, and those that wait. A split item is worked through its sub-tasks, and what blocks it blocks them. An issue in one of the unblocking statuses is worked already: it waits until Jira counts it done."""
by_key = {issue.key: issue for issue in issues}
holding = held_up(issues, unblocking)
found: list[Chunk] = []
waits: list[Waits] = []
for issue in issues:
if issue.done:
continue
if unblocks(issue.status, unblocking):
waits.append(Waits(issue.key, f"its work has landed: it is {issue.status}, and the epic's finishing waits until Jira counts it done"))
continue
if is_split(issue, issues):
if all(task.done for task in issues if task.parent == issue.key) and not finished(issues, issue.key):
waits.append(Waits(issue.key, "its sub-tasks are done; close it in Jira"))
continue
turn = turn_of(issue, recorded(issue, team), open_blockers(issue, by_key, unblocking), by_key, issue.key in holding, unblocking)
if isinstance(turn, Chunk):
found.append(turn)
else:
waits.append(turn)
return found, waits
def turn_of(issue: EpicIssue, pick: Pick | None | Failure, blocking: list[str], by_key: dict[str, EpicIssue], holds: bool, unblocking: tuple[str, ...] = ()) -> Chunk | Waits:
"""The open issue's chunk, or why it waits. blocking are its open blockers, and holds says it holds up another open issue. A human's task gets a window only while it holds up another issue and nothing holds it up; that window waits for the human. A blocker outside the epic blocks until the human removes the link, since its state is unknown."""
if isinstance(pick, Failure):
return Waits(issue.key, pick.reason)
if pick is None:
return Waits(issue.key, "no label names its repo" + ("; label it in Jira, since do-epic plans no sub-task" if issue.parent in by_key else ""))
if isinstance(pick, HumanTask):
return Chunk(issue.key, None, human=True) if holds and not blocking else Waits(issue.key, "it is a human's task")
if blocking:
outside = [key for key in blocking if key not in by_key]
unseen = f"; do-epic cannot see {' and '.join(outside)}, outside the epic, finish: remove such a link in Jira once its issue is done" if outside else ""
inside = [key for key in blocking if key in by_key]
whom = ("it is" if len(blocking) == 1 else "each is") if not outside else f"{' and '.join(inside)} {'is' if len(inside) == 1 else 'are'}"
until = f", until {whom} {listed((*unblocking, 'done'), 'or')}" if unblocking and inside else ""
return Waits(issue.key, f"{' and '.join(blocking)} block{'s' if len(blocking) == 1 else ''} it{until}{unseen}")
return Chunk(issue.key, pick.name if isinstance(pick, Repo) else None)
def refusal(repo: str, room: RepoRoom) -> str | None:
"""Why the repo takes no further chunk; None where it takes one."""
if room.loop_claude:
return f"loop-claude runs chunks in {repo}"
if room.running >= room.limit:
return f"{repo} runs as many chunks as its `{PARALLEL_CHUNKS}` allows: {room.limit}"
return None
def start_chunks(epic: Epic, issues: list[EpicIssue], finishing: bool, own: str | None, run: tmux.Tmux) -> Started | Failure:
"""Starts a do-epic in a new window for each of the epic's chunks that may start now. Where finishing is set, no issue is open, and the one chunk to start is the epic's finishing (finish.py), in a window named for the epic's key. own is the key of the chunk that asks, which has finished or parked: it counts as running nowhere, and no stopped chunk starts again. Where own is None, the human's `do-epic <EPIC-KEY>` asks, and each stopped chunk starts again. A parked chunk waits in its window; one without a live window, as after a reboot, starts again as a stopped chunk does, whatever the limits, since it goes on parked. A Failure says why a window could not start; those before it run."""
session = tmux.session_name(tmux.EPIC, epic.key)
found, waits = chunks(issues, epic.team, epic.unblocking)
if finishing:
found.append(Chunk(epic.key, None))
started: list[str] = []
parked = parked_chunks(epic.folder)
if isinstance(parked, Failure):
return parked
with starting(epic.folder) as unlocked:
if unlocked:
return unlocked
windows = tmux.listed(session, run)
live = {window.name: window for window in windows if not window.dead and window.name != own}
running = {window.name for window in windows if not window.dead and not window.parked} - {own}
stopped = {window.name for window in windows if window.dead} - set(live)
# The windows this call started that take a running slot.
taken = 0
for chunk in found:
if chunk.issue in running or chunk.issue == own:
continue
slot = chunk.issue not in parked and not chunk.human
if chunk.human and chunk.issue in live:
waits.append(Waits(chunk.issue, HUMAN_WAIT))
elif chunk.issue in parked and chunk.issue in live:
waits.append(Waits(chunk.issue, parked[chunk.issue].waits(live[chunk.issue].awaits)))
elif own and (chunk.issue in stopped or chunk.issue in parked):
waits.append(Waits(chunk.issue, f"its window stopped; run `do-epic {epic.key}` to start it again"))
elif slot and len(running) + taken >= epic.cap:
waits.append(Waits(chunk.issue, f"the epic runs as many chunks as its `{CAP}` line allows: {epic.cap}"))
else:
now = start_chunk(epic, chunk if chunk.issue not in parked else Chunk(chunk.issue, None), own, windows, run)
if isinstance(now, Failure):
return now
if isinstance(now, Waits):
waits.append(now)
else:
windows = now
started.append(chunk.issue)
taken += slot
# The epic's finishing is no issue: its wait is listed after every issue's.
order = [*(issue.key for issue in issues), epic.key]
return Started(started, sorted(waits, key=lambda wait: order.index(wait.issue)))
def start_chunk(epic: Epic, chunk: Chunk, own: str | None, windows: list[tmux.Listed], run: tmux.Tmux) -> list[tmux.Listed] | Waits | Failure:
"""Starts chunk's window beside windows, the epic's session's, and returns the session's windows afterwards; a Waits with the reason where its repo takes no chunk now, and a Failure where tmux started none, or where the repo's lock cannot be taken. It holds that lock from the check to the start. A chunk without a repo starts in the epic's folder. So does a parked chunk, since it takes no room in its repo."""
session = tmux.session_name(tmux.EPIC, epic.key)
handed = {**{name: value for name, value in epic.env.items() if name.startswith(HANDED)}, PARKS: "1"}
command = [str(epic.launcher), CHUNK_FLAG, chunk.issue, epic.key]
if chunk.repo is None:
return tmux.start_chunk(tmux.Window(session, chunk.issue, ""), str(epic.folder), handed, command, windows, False, run)
located = epic.locate(chunk.repo)
if isinstance(located, Failure):
return Waits(chunk.issue, located.reason)
checkout, limit = located
with starting(project_lock(str(checkout), epic.home)) as unlocked:
if unlocked:
return unlocked
if reason := refusal(chunk.repo, room_in(str(checkout), limit, (session, own), epic.home, run)):
return Waits(chunk.issue, reason)
return tmux.start_chunk(tmux.Window(session, chunk.issue, ""), str(checkout), handed, command, windows, False, run)
def room_in(checkout: str, limit: int, leaving: tuple[str, str | None], home: pathlib.Path, run: tmux.Tmux) -> RepoRoom:
"""The room in a repo's checkout under its limit. leaving is the session and name of the window that asks, which doesn't count: its chunk finished or is parked."""
theirs = [placed for placed in epic_chunks(checkout, run) if (placed.session, placed.window.name) != leaving]
return RepoRoom(limit, len(theirs), bool(loop_chunks(checkout, home, run)))
def resume(epic: Epic, repo: str, window: tmux.Window, run: tmux.Tmux) -> str | None:
"""Takes a running slot for the parked chunk in window, whose worker has something to do again in the repo called repo: None once the window counts as a running chunk, else why the chunk still waits. The limits are those a starting chunk meets, checked under the same locks. The window itself never counts against them: one started again for a parked chunk, as after a reboot, isn't named as parked until its first read of the gate."""
with starting(epic.folder) as unlocked:
if unlocked:
return unlocked.reason
running = [listed for listed in tmux.listed(window.session, run) if not listed.dead and not listed.parked and listed.name != window.name]
if len(running) >= epic.cap:
return f"the epic runs as many chunks as its `{CAP}` line allows: {epic.cap}"
located = epic.locate(repo)
if isinstance(located, Failure):
return located.reason
checkout, limit = located
with starting(project_lock(str(checkout), epic.home)) as unlocked:
if unlocked:
return unlocked.reason
if reason := refusal(repo, room_in(str(checkout), limit, (window.session, window.name), epic.home, run)):
return reason
tmux.unpark(window, str(checkout), run)
return None