Answer stability round 1: 1 🔴, 3 🟡 (0 misread); sort the new files' imports, sync the gzip before removing the file, read the plain file then its gzip without a check between, and age a file by the later of its mtime and ctime

This commit is contained in:
2026-11-17 02:33:15 +01:00
parent abe30ab893
commit 04904180d5
4 changed files with 86 additions and 36 deletions
+24 -18
View File
@@ -1,29 +1,32 @@
"""Compress every file of loop-claude's archive and the review folders that no tool writes to any more, keeping each one.
"""Gzip each file of loop-claude's archive and the review folders that no tool changes any more, keeping all of them.
usage: compress-archives
It gzips each file last written over 30 days ago, beside its plain name as <name>.gz with the same
times: in loop-claude's archive, $LOOP_CLAUDE_TRANSCRIPTS (default ~/.claude/loop-claude-transcripts),
each transcript (*.jsonl); in the review folders, ~/.local/state/larv-review, every file. It writes
the gzip, reads it back, and removes the plain file only once the gzip reads back its bytes. Every
tool that reads these folders reads a gzipped file by its plain name. Install's crontab line runs it
daily.
It gzips each file that last changed over 30 days ago: in loop-claude's archive,
$LOOP_CLAUDE_TRANSCRIPTS (default ~/.claude/loop-claude-transcripts), each transcript, *.jsonl;
in the review folders, ~/.local/state/larv-review, every file. A file's last change is the later
of its mtime and ctime, so a copied or unpacked file counts from when it landed.
It prints how many files it compressed. Exit 1, saying why, at the first file it cannot compress,
which it keeps plain with every file after it, or on any argument but --help.
The gzip lands beside the file as <name>.gz, with the same times. The file is removed only once
its gzip reads back the same bytes. The archive's readers and review-replay read a gzipped file by
its plain name. README.md → Install has the crontab line that runs it daily.
It prints how many files it compressed. It exits 1, saying why, on any argument but --help or -h,
and at the first file it cannot compress, leaving that file and every one after it plain.
"""
import os
import pathlib
import stat
import sys
import time
import typing
from transcript_archive.location import archive_root
from text_files.compressed import compress, SUFFIX
from failure import Failure
from review_rules.review_folder import REPORTS
from text_files.compressed import compress, SUFFIX
from transcript_archive.location import archive_root
# Why 30 days: docs/decisions.md → `archives: loop-claude's archive and the review folders keep every file, and gzip each one 30 days after it was last written`.
# Why 30 days: docs/decisions.md → `archives: gzip each transcript in loop-claude's archive and each file in the review folders 30 days after it last changed, keeping all of them`.
AGE_DAYS = 30
@@ -35,9 +38,9 @@ def main(argv: list[str], env: typing.Mapping[str, str] = os.environ, home: path
print(f"unknown arguments {' '.join(argv)}; usage: compress-archives", file=sys.stderr)
return 1
home = home or pathlib.Path.home()
before = (time.time() if now is None else now) - AGE_DAYS * 24 * 60 * 60
cutoff = (time.time() if now is None else now) - AGE_DAYS * 24 * 60 * 60
count = 0
for path in [*old_files(archive_root(home, env), "*.jsonl", before), *old_files(home / REPORTS, "*", before)]:
for path in [*old_files(archive_root(home, env), "*.jsonl", cutoff), *old_files(home / REPORTS, "*", cutoff)]:
if failed := compress(path):
print(failed.reason, file=sys.stderr)
return 1
@@ -46,14 +49,17 @@ def main(argv: list[str], env: typing.Mapping[str, str] = os.environ, home: path
return 0
def old_files(root: pathlib.Path, pattern: str, before: float) -> list[pathlib.Path]:
"""Each plain file under root that pattern matches, last written before the time before, sorted; a link is left alone, since its target lies elsewhere."""
def old_files(root: pathlib.Path, pattern: str, cutoff: float) -> list[pathlib.Path]:
"""Each plain file under root that pattern matches and that last changed before the cutoff, sorted; a link is left alone, since its target lies elsewhere."""
found = []
for path in root.rglob(pattern):
if path.name.endswith(SUFFIX) or path.is_symlink():
continue
try:
if not path.name.endswith(SUFFIX) and path.is_file() and not path.is_symlink() and path.stat().st_mtime < before:
found.append(path)
status = path.stat()
# Removed while the walk ran, such as a file of review-replay's copy of a review folder.
except FileNotFoundError:
continue
if stat.S_ISREG(status.st_mode) and max(status.st_mtime, status.st_ctime) < cutoff:
found.append(path)
return sorted(found)
@@ -1,26 +1,28 @@
import contextlib
import gzip
import io
from unittest import mock
import os
import pathlib
import sys
import tempfile
import time
import unittest
from unittest import mock
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from compress_archives import main
from failure import Failure
from compress_archives import main
NOW = 1790000000
DAY = 24 * 60 * 60
# A month ahead, so a file's ctime, which is when the test wrote it, lies over 30 days before NOW.
NOW = time.time() + 31 * DAY
OLD = 30 * DAY + 60
NEW = 30 * DAY - 60
class Main(unittest.TestCase):
"""A home whose archive and review folders each hold files last written a minute over 30 days ago, and one a minute under."""
"""A home whose archive and review folders each hold files that last changed a minute over 30 days ago, and one that changed a minute under."""
def setUp(self):
self.home = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory()))
@@ -72,12 +74,16 @@ class Main(unittest.TestCase):
self.run_main()
self.assertTrue((self.archive / "b.jsonl").is_file())
def test_keeps_a_file_copied_today_with_its_old_times_plain(self):
self.assertEqual(self.run_main(now=time.time()), (0, "Compressed 0 files.\n", ""))
self.assertTrue((self.archive / "a.jsonl").is_file())
def test_measures_the_30_days_from_the_clock_by_default(self):
with mock.patch.object(main.time, "time", return_value=NOW):
self.assertEqual(self.run_main(now=None), (0, "Compressed 4 files.\n", ""))
def test_skips_a_file_removed_while_it_walks(self):
gone = mock.Mock(**{"is_file.return_value": True, "is_symlink.return_value": False, "stat.side_effect": FileNotFoundError})
gone = mock.Mock(**{"is_symlink.return_value": False, "stat.side_effect": FileNotFoundError})
# Mock(name=...) names the mock itself, so the path's name is set after.
gone.name = "gone.jsonl"
kept = self.archive / "a.jsonl"
@@ -106,7 +112,7 @@ class Main(unittest.TestCase):
for flag in ("--help", "-h"):
code, out, _ = self.run_main([flag])
self.assertEqual(code, 0)
self.assertTrue(out.startswith("Compress every file"), out)
self.assertTrue(out.startswith("Gzip each file"), out)
self.assertEqual(self.run_main(["x", "y"]), (1, "", "unknown arguments x y; usage: compress-archives\n"))
+26 -10
View File
@@ -1,6 +1,6 @@
"""A file kept plain, or gzipped beside its plain name as <name>.gz: reading it by its plain name, and compressing it.
Why gzip and when a file is compressed: docs/decisions.md → `archives: loop-claude's archive and the review folders keep every file, and gzip each one 30 days after it was last written`.
Why gzip and when a file is compressed: docs/decisions.md → `archives: gzip each transcript in loop-claude's archive and each file in the review folders 30 days after it last changed, keeping all of them`.
"""
import errno
import gzip
@@ -16,21 +16,26 @@ SUFFIX = ".gz"
def stored(path: pathlib.Path) -> pathlib.Path | None:
"""Where the file named path is kept: path itself, else its gzip; None where neither is."""
# A plain file beside its gzip was written after it was compressed, so it holds the newer bytes. Whatever else stands at path, such as a folder, is read as path, so the read names it.
# A plain file beside its gzip was written after it was compressed, so it holds the newer bytes. Anything else at path, such as a folder, counts as path, so reading it fails naming path.
return next((kept for kept in (path, gzipped(path)) if kept.exists()), None)
def read_stored(path: pathlib.Path) -> bytes:
"""The bytes of the file named path, read from its gzip where it is kept that way. Raises OSError, FileNotFoundError where neither is kept."""
kept = stored(path)
if kept is None:
raise FileNotFoundError(2, "No such file or directory", str(path))
if kept == path:
return path.read_bytes()
# Tried in turn rather than checked first: compress-archives may remove the plain file once its gzip is whole, between a check and the read.
try:
return gzip.decompress(kept.read_bytes())
except (EOFError, zlib.error) as failure:
raise OSError(errno.EIO, f"not a whole gzip: {failure}", str(kept)) from failure
return path.read_bytes()
except FileNotFoundError:
pass
packed = gzipped(path)
try:
data = packed.read_bytes()
except FileNotFoundError:
raise FileNotFoundError(errno.ENOENT, "No such file or directory", str(path)) from None
try:
return gzip.decompress(data)
except (EOFError, zlib.error, gzip.BadGzipFile) as failure:
raise OSError(errno.EIO, f"not a whole gzip: {failure}", str(packed)) from failure
def stored_glob(folder: pathlib.Path, pattern: str) -> list[pathlib.Path]:
@@ -49,6 +54,9 @@ def compress(path: pathlib.Path) -> Failure | None:
if gzip.decompress(packed.read_bytes()) != data:
return Failure(f"could not compress {path}: {packed} reads back other bytes")
os.utime(packed, ns=(before.st_atime_ns, before.st_mtime_ns))
# On disk before the plain file goes, so a crash cannot leave an empty gzip in its place.
synced(packed)
synced(packed.parent)
after = path.stat()
# Written while it compressed: the plain file keeps the newer bytes, which readers prefer, and a later run compresses them.
if (after.st_size, after.st_mtime_ns) == (before.st_size, before.st_mtime_ns):
@@ -58,5 +66,13 @@ def compress(path: pathlib.Path) -> Failure | None:
return None
def synced(path: pathlib.Path) -> None:
handle = os.open(path, os.O_RDONLY)
try:
os.fsync(handle)
finally:
os.close(handle)
def gzipped(path: pathlib.Path) -> pathlib.Path:
return path.with_name(path.name + SUFFIX)
+24 -2
View File
@@ -1,16 +1,16 @@
import errno
import gzip
from unittest import mock
import os
import pathlib
import sys
import tempfile
import unittest
from unittest import mock
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from failure import Failure
from text_files import compressed
from failure import Failure
class Compress(unittest.TestCase):
@@ -26,6 +26,21 @@ class Compress(unittest.TestCase):
self.assertEqual(packed.read_bytes(), gzip.compress(b'{"a": 1}\n', mtime=0))
self.assertEqual(packed.stat().st_mtime, 2000)
def test_syncs_the_gzip_and_its_folder_before_removing_the_file(self):
with tempfile.TemporaryDirectory() as root:
plain = pathlib.Path(root, "t.jsonl")
plain.write_bytes(b"a\n")
synced = []
fsync = os.fsync
def record(handle):
synced.append((os.readlink(f"/proc/self/fd/{handle}"), plain.exists()))
fsync(handle)
with mock.patch.object(compressed.os, "fsync", record):
self.assertIsNone(compressed.compress(plain))
self.assertEqual(synced, [(os.path.realpath(f"{plain}.gz"), True), (os.path.realpath(root), True)])
def test_a_gzip_it_cannot_write_keeps_the_original_and_says_why(self):
with tempfile.TemporaryDirectory() as root:
plain = pathlib.Path(root, "t.jsonl")
@@ -96,6 +111,13 @@ class Read(unittest.TestCase):
self.assertEqual((raised.exception.errno, raised.exception.filename), (errno.EIO, str(pathlib.Path(root, "a.jsonl.gz"))))
self.assertTrue(raised.exception.strerror.startswith("not a whole gzip: "), raised.exception.strerror)
def test_a_file_that_is_no_gzip_fails_as_an_os_error(self):
with tempfile.TemporaryDirectory() as root:
pathlib.Path(root, "a.jsonl.gz").write_bytes(b"plain text\n")
with self.assertRaises(OSError) as raised:
compressed.read_stored(pathlib.Path(root, "a.jsonl"))
self.assertEqual((raised.exception.errno, raised.exception.filename), (errno.EIO, str(pathlib.Path(root, "a.jsonl.gz"))))
def test_globs_each_file_once_by_its_plain_name(self):
with tempfile.TemporaryDirectory() as root:
for name in ("a.jsonl", "b.jsonl.gz", "c.jsonl", "c.jsonl.gz", "d.md"):