Answer stability round 1: 1 🔴, 3 🟡 (0 misread); sort the new files' imports, sync the gzip before removing the file, read the plain file then its gzip without a check between, and age a file by the later of its mtime and ctime
This commit is contained in:
@@ -1,29 +1,32 @@
|
||||
"""Compress every file of loop-claude's archive and the review folders that no tool writes to any more, keeping each one.
|
||||
"""Gzip each file of loop-claude's archive and the review folders that no tool changes any more, keeping all of them.
|
||||
|
||||
usage: compress-archives
|
||||
|
||||
It gzips each file last written over 30 days ago, beside its plain name as <name>.gz with the same
|
||||
times: in loop-claude's archive, $LOOP_CLAUDE_TRANSCRIPTS (default ~/.claude/loop-claude-transcripts),
|
||||
each transcript (*.jsonl); in the review folders, ~/.local/state/larv-review, every file. It writes
|
||||
the gzip, reads it back, and removes the plain file only once the gzip reads back its bytes. Every
|
||||
tool that reads these folders reads a gzipped file by its plain name. Install's crontab line runs it
|
||||
daily.
|
||||
It gzips each file that last changed over 30 days ago: in loop-claude's archive,
|
||||
$LOOP_CLAUDE_TRANSCRIPTS (default ~/.claude/loop-claude-transcripts), each transcript, *.jsonl;
|
||||
in the review folders, ~/.local/state/larv-review, every file. A file's last change is the later
|
||||
of its mtime and ctime, so a copied or unpacked file counts from when it landed.
|
||||
|
||||
It prints how many files it compressed. Exit 1, saying why, at the first file it cannot compress,
|
||||
which it keeps plain with every file after it, or on any argument but --help.
|
||||
The gzip lands beside the file as <name>.gz, with the same times. The file is removed only once
|
||||
its gzip reads back the same bytes. The archive's readers and review-replay read a gzipped file by
|
||||
its plain name. README.md → Install has the crontab line that runs it daily.
|
||||
|
||||
It prints how many files it compressed. It exits 1, saying why, on any argument but --help or -h,
|
||||
and at the first file it cannot compress, leaving that file and every one after it plain.
|
||||
"""
|
||||
import os
|
||||
import pathlib
|
||||
import stat
|
||||
import sys
|
||||
import time
|
||||
import typing
|
||||
|
||||
from transcript_archive.location import archive_root
|
||||
from text_files.compressed import compress, SUFFIX
|
||||
from failure import Failure
|
||||
from review_rules.review_folder import REPORTS
|
||||
from text_files.compressed import compress, SUFFIX
|
||||
from transcript_archive.location import archive_root
|
||||
|
||||
# Why 30 days: docs/decisions.md → `archives: loop-claude's archive and the review folders keep every file, and gzip each one 30 days after it was last written`.
|
||||
# Why 30 days: docs/decisions.md → `archives: gzip each transcript in loop-claude's archive and each file in the review folders 30 days after it last changed, keeping all of them`.
|
||||
AGE_DAYS = 30
|
||||
|
||||
|
||||
@@ -35,9 +38,9 @@ def main(argv: list[str], env: typing.Mapping[str, str] = os.environ, home: path
|
||||
print(f"unknown arguments {' '.join(argv)}; usage: compress-archives", file=sys.stderr)
|
||||
return 1
|
||||
home = home or pathlib.Path.home()
|
||||
before = (time.time() if now is None else now) - AGE_DAYS * 24 * 60 * 60
|
||||
cutoff = (time.time() if now is None else now) - AGE_DAYS * 24 * 60 * 60
|
||||
count = 0
|
||||
for path in [*old_files(archive_root(home, env), "*.jsonl", before), *old_files(home / REPORTS, "*", before)]:
|
||||
for path in [*old_files(archive_root(home, env), "*.jsonl", cutoff), *old_files(home / REPORTS, "*", cutoff)]:
|
||||
if failed := compress(path):
|
||||
print(failed.reason, file=sys.stderr)
|
||||
return 1
|
||||
@@ -46,14 +49,17 @@ def main(argv: list[str], env: typing.Mapping[str, str] = os.environ, home: path
|
||||
return 0
|
||||
|
||||
|
||||
def old_files(root: pathlib.Path, pattern: str, before: float) -> list[pathlib.Path]:
|
||||
"""Each plain file under root that pattern matches, last written before the time before, sorted; a link is left alone, since its target lies elsewhere."""
|
||||
def old_files(root: pathlib.Path, pattern: str, cutoff: float) -> list[pathlib.Path]:
|
||||
"""Each plain file under root that pattern matches and that last changed before the cutoff, sorted; a link is left alone, since its target lies elsewhere."""
|
||||
found = []
|
||||
for path in root.rglob(pattern):
|
||||
if path.name.endswith(SUFFIX) or path.is_symlink():
|
||||
continue
|
||||
try:
|
||||
if not path.name.endswith(SUFFIX) and path.is_file() and not path.is_symlink() and path.stat().st_mtime < before:
|
||||
found.append(path)
|
||||
status = path.stat()
|
||||
# Removed while the walk ran, such as a file of review-replay's copy of a review folder.
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
if stat.S_ISREG(status.st_mode) and max(status.st_mtime, status.st_ctime) < cutoff:
|
||||
found.append(path)
|
||||
return sorted(found)
|
||||
|
||||
@@ -1,26 +1,28 @@
|
||||
import contextlib
|
||||
import gzip
|
||||
import io
|
||||
from unittest import mock
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from compress_archives import main
|
||||
from failure import Failure
|
||||
from compress_archives import main
|
||||
|
||||
NOW = 1790000000
|
||||
DAY = 24 * 60 * 60
|
||||
# A month ahead, so a file's ctime, which is when the test wrote it, lies over 30 days before NOW.
|
||||
NOW = time.time() + 31 * DAY
|
||||
OLD = 30 * DAY + 60
|
||||
NEW = 30 * DAY - 60
|
||||
|
||||
|
||||
class Main(unittest.TestCase):
|
||||
"""A home whose archive and review folders each hold files last written a minute over 30 days ago, and one a minute under."""
|
||||
"""A home whose archive and review folders each hold files that last changed a minute over 30 days ago, and one that changed a minute under."""
|
||||
|
||||
def setUp(self):
|
||||
self.home = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory()))
|
||||
@@ -72,12 +74,16 @@ class Main(unittest.TestCase):
|
||||
self.run_main()
|
||||
self.assertTrue((self.archive / "b.jsonl").is_file())
|
||||
|
||||
def test_keeps_a_file_copied_today_with_its_old_times_plain(self):
|
||||
self.assertEqual(self.run_main(now=time.time()), (0, "Compressed 0 files.\n", ""))
|
||||
self.assertTrue((self.archive / "a.jsonl").is_file())
|
||||
|
||||
def test_measures_the_30_days_from_the_clock_by_default(self):
|
||||
with mock.patch.object(main.time, "time", return_value=NOW):
|
||||
self.assertEqual(self.run_main(now=None), (0, "Compressed 4 files.\n", ""))
|
||||
|
||||
def test_skips_a_file_removed_while_it_walks(self):
|
||||
gone = mock.Mock(**{"is_file.return_value": True, "is_symlink.return_value": False, "stat.side_effect": FileNotFoundError})
|
||||
gone = mock.Mock(**{"is_symlink.return_value": False, "stat.side_effect": FileNotFoundError})
|
||||
# Mock(name=...) names the mock itself, so the path's name is set after.
|
||||
gone.name = "gone.jsonl"
|
||||
kept = self.archive / "a.jsonl"
|
||||
@@ -106,7 +112,7 @@ class Main(unittest.TestCase):
|
||||
for flag in ("--help", "-h"):
|
||||
code, out, _ = self.run_main([flag])
|
||||
self.assertEqual(code, 0)
|
||||
self.assertTrue(out.startswith("Compress every file"), out)
|
||||
self.assertTrue(out.startswith("Gzip each file"), out)
|
||||
self.assertEqual(self.run_main(["x", "y"]), (1, "", "unknown arguments x y; usage: compress-archives\n"))
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""A file kept plain, or gzipped beside its plain name as <name>.gz: reading it by its plain name, and compressing it.
|
||||
|
||||
Why gzip and when a file is compressed: docs/decisions.md → `archives: loop-claude's archive and the review folders keep every file, and gzip each one 30 days after it was last written`.
|
||||
Why gzip and when a file is compressed: docs/decisions.md → `archives: gzip each transcript in loop-claude's archive and each file in the review folders 30 days after it last changed, keeping all of them`.
|
||||
"""
|
||||
import errno
|
||||
import gzip
|
||||
@@ -16,21 +16,26 @@ SUFFIX = ".gz"
|
||||
|
||||
def stored(path: pathlib.Path) -> pathlib.Path | None:
|
||||
"""Where the file named path is kept: path itself, else its gzip; None where neither is."""
|
||||
# A plain file beside its gzip was written after it was compressed, so it holds the newer bytes. Whatever else stands at path, such as a folder, is read as path, so the read names it.
|
||||
# A plain file beside its gzip was written after it was compressed, so it holds the newer bytes. Anything else at path, such as a folder, counts as path, so reading it fails naming path.
|
||||
return next((kept for kept in (path, gzipped(path)) if kept.exists()), None)
|
||||
|
||||
|
||||
def read_stored(path: pathlib.Path) -> bytes:
|
||||
"""The bytes of the file named path, read from its gzip where it is kept that way. Raises OSError, FileNotFoundError where neither is kept."""
|
||||
kept = stored(path)
|
||||
if kept is None:
|
||||
raise FileNotFoundError(2, "No such file or directory", str(path))
|
||||
if kept == path:
|
||||
return path.read_bytes()
|
||||
# Tried in turn rather than checked first: compress-archives may remove the plain file once its gzip is whole, between a check and the read.
|
||||
try:
|
||||
return gzip.decompress(kept.read_bytes())
|
||||
except (EOFError, zlib.error) as failure:
|
||||
raise OSError(errno.EIO, f"not a whole gzip: {failure}", str(kept)) from failure
|
||||
return path.read_bytes()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
packed = gzipped(path)
|
||||
try:
|
||||
data = packed.read_bytes()
|
||||
except FileNotFoundError:
|
||||
raise FileNotFoundError(errno.ENOENT, "No such file or directory", str(path)) from None
|
||||
try:
|
||||
return gzip.decompress(data)
|
||||
except (EOFError, zlib.error, gzip.BadGzipFile) as failure:
|
||||
raise OSError(errno.EIO, f"not a whole gzip: {failure}", str(packed)) from failure
|
||||
|
||||
|
||||
def stored_glob(folder: pathlib.Path, pattern: str) -> list[pathlib.Path]:
|
||||
@@ -49,6 +54,9 @@ def compress(path: pathlib.Path) -> Failure | None:
|
||||
if gzip.decompress(packed.read_bytes()) != data:
|
||||
return Failure(f"could not compress {path}: {packed} reads back other bytes")
|
||||
os.utime(packed, ns=(before.st_atime_ns, before.st_mtime_ns))
|
||||
# On disk before the plain file goes, so a crash cannot leave an empty gzip in its place.
|
||||
synced(packed)
|
||||
synced(packed.parent)
|
||||
after = path.stat()
|
||||
# Written while it compressed: the plain file keeps the newer bytes, which readers prefer, and a later run compresses them.
|
||||
if (after.st_size, after.st_mtime_ns) == (before.st_size, before.st_mtime_ns):
|
||||
@@ -58,5 +66,13 @@ def compress(path: pathlib.Path) -> Failure | None:
|
||||
return None
|
||||
|
||||
|
||||
def synced(path: pathlib.Path) -> None:
|
||||
handle = os.open(path, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(handle)
|
||||
finally:
|
||||
os.close(handle)
|
||||
|
||||
|
||||
def gzipped(path: pathlib.Path) -> pathlib.Path:
|
||||
return path.with_name(path.name + SUFFIX)
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
import errno
|
||||
import gzip
|
||||
from unittest import mock
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from failure import Failure
|
||||
from text_files import compressed
|
||||
from failure import Failure
|
||||
|
||||
|
||||
class Compress(unittest.TestCase):
|
||||
@@ -26,6 +26,21 @@ class Compress(unittest.TestCase):
|
||||
self.assertEqual(packed.read_bytes(), gzip.compress(b'{"a": 1}\n', mtime=0))
|
||||
self.assertEqual(packed.stat().st_mtime, 2000)
|
||||
|
||||
def test_syncs_the_gzip_and_its_folder_before_removing_the_file(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
plain = pathlib.Path(root, "t.jsonl")
|
||||
plain.write_bytes(b"a\n")
|
||||
synced = []
|
||||
fsync = os.fsync
|
||||
|
||||
def record(handle):
|
||||
synced.append((os.readlink(f"/proc/self/fd/{handle}"), plain.exists()))
|
||||
fsync(handle)
|
||||
|
||||
with mock.patch.object(compressed.os, "fsync", record):
|
||||
self.assertIsNone(compressed.compress(plain))
|
||||
self.assertEqual(synced, [(os.path.realpath(f"{plain}.gz"), True), (os.path.realpath(root), True)])
|
||||
|
||||
def test_a_gzip_it_cannot_write_keeps_the_original_and_says_why(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
plain = pathlib.Path(root, "t.jsonl")
|
||||
@@ -96,6 +111,13 @@ class Read(unittest.TestCase):
|
||||
self.assertEqual((raised.exception.errno, raised.exception.filename), (errno.EIO, str(pathlib.Path(root, "a.jsonl.gz"))))
|
||||
self.assertTrue(raised.exception.strerror.startswith("not a whole gzip: "), raised.exception.strerror)
|
||||
|
||||
def test_a_file_that_is_no_gzip_fails_as_an_os_error(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
pathlib.Path(root, "a.jsonl.gz").write_bytes(b"plain text\n")
|
||||
with self.assertRaises(OSError) as raised:
|
||||
compressed.read_stored(pathlib.Path(root, "a.jsonl"))
|
||||
self.assertEqual((raised.exception.errno, raised.exception.filename), (errno.EIO, str(pathlib.Path(root, "a.jsonl.gz"))))
|
||||
|
||||
def test_globs_each_file_once_by_its_plain_name(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
for name in ("a.jsonl", "b.jsonl.gz", "c.jsonl", "c.jsonl.gz", "d.md"):
|
||||
|
||||
Reference in New Issue
Block a user