96 lines
4.8 KiB
Python
96 lines
4.8 KiB
Python
import dataclasses
|
|
import typing
|
|
|
|
from failure import Failure
|
|
|
|
JUDGE_RULES = (
|
|
"Decide whether the human must now be interrupted. interrupt=true when any of these appear:\n"
|
|
"- the agent asks the human anything, seeks approval, or says a decision is the human's, and neither "
|
|
"README.md nor the project's records settle the matter\n"
|
|
"- it asks again a question a [loop] line already answered\n"
|
|
"- it carried on under an assumption that decides the deliverable, that the human could "
|
|
"reasonably reject and that neither README.md nor the project's records settle; a caveat about "
|
|
"transient state it then handled is not one\n"
|
|
"- it is blocked on something only the human can supply: a credential, a version bump, a choice "
|
|
"outside the repo, or a release ready to cut, which only the human may publish\n"
|
|
"- it flags that the goals, the plan in todo.md or the task itself is unclear, missing or in need of change\n"
|
|
"- it chose between alternatives that neither README.md nor the project's records discriminate "
|
|
"between - neither rules the rejected one out - and the choice materially changes the deliverable\n"
|
|
"interrupt=false otherwise. Bookkeeping the agent settled silently - what to document, wording, "
|
|
"where a note goes - is not on its own a reason to interrupt. The human may have replied inside the transcript, on a [human] line: judge only what "
|
|
"is still open, never a question they already answered there. A [loop] line is what loop-claude told "
|
|
"the agent, such as an answer quoted from README.md or the project's records.\n"
|
|
"Where the agent asked or flagged something itself, the question is that one, in the agent's own "
|
|
"wording; never construct another from facts it stated.\n"
|
|
"Where README.md or the project's records answer the agent's own question and no [loop] line has "
|
|
"quoted that answer yet, set interrupt=false and quote the passage that answers it; the agent is "
|
|
"then given that passage in place of the human's reply. An interrupt outranks a quoted answer, and either "
|
|
"outranks finished.\n"
|
|
"The chunk is the piece of work the agent is on, shipped as one pull request. "
|
|
"finished=true when the chunk is complete: merged, or nothing remains that the agent could do on "
|
|
"its own. finished=false when work remains that a further run can continue: an open PR, a review "
|
|
"not yet run, CI not yet reported."
|
|
)
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class Reply:
|
|
answer: str = dataclasses.field(metadata={"description": "With interrupt false, the passage of README.md or the project's records that answers the agent's question, quoted verbatim, without elisions. Empty otherwise."})
|
|
evidence: str = dataclasses.field(metadata={"description": "A verbatim quote of the one transcript passage the verdict rests on, without elisions: what triggers the interrupt; for a quoted answer, the agent's question; else what shows the chunk finished or the work remaining."})
|
|
finished: bool = dataclasses.field(metadata={"description": "Whether the chunk is finished."})
|
|
interrupt: bool = dataclasses.field(metadata={"description": "Whether the human must now be interrupted."})
|
|
question: str = dataclasses.field(metadata={"description": "With interrupt true, the single question to put to the human. With a quoted answer, the agent's question that the answer field answers. Empty otherwise."})
|
|
|
|
|
|
REPLY_TYPES = typing.get_type_hints(Reply)
|
|
JSON_TYPES = {bool: "boolean", str: "string"}
|
|
JUDGE_SCHEMA = {
|
|
"properties": {field.name: {**field.metadata, "type": JSON_TYPES[REPLY_TYPES[field.name]]} for field in dataclasses.fields(Reply)},
|
|
"required": list(REPLY_TYPES),
|
|
"type": "object",
|
|
}
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class Interrupt:
|
|
evidence: str
|
|
question: str
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class QuotedAnswer:
|
|
"""The agent asked a question that README.md or the project's records answer. The answer field quotes the passage."""
|
|
evidence: str
|
|
question: str
|
|
answer: str
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class Finished:
|
|
evidence: str
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class Unfinished:
|
|
evidence: str
|
|
|
|
|
|
Verdict = Interrupt | QuotedAnswer | Finished | Unfinished
|
|
|
|
|
|
def verdict_from(output: object) -> Verdict | Failure:
|
|
if not isinstance(output, dict):
|
|
return Failure("the judge returned no reply")
|
|
for name, kind in REPLY_TYPES.items():
|
|
if not isinstance(output.get(name), kind):
|
|
return Failure(f"the judge's reply has no {name} of type {JSON_TYPES[kind]}")
|
|
if output["interrupt"]:
|
|
if not output["question"].strip():
|
|
return Failure("the judge interrupted without a question")
|
|
return Interrupt(output["evidence"], output["question"])
|
|
if output["answer"].strip():
|
|
if not output["question"].strip():
|
|
return Failure("the judge quoted an answer without a question")
|
|
return QuotedAnswer(output["evidence"], output["question"], output["answer"])
|
|
return Finished(output["evidence"]) if output["finished"] else Unfinished(output["evidence"])
|