Files

79 lines
3.5 KiB
Python

"""A file kept plain, or gzipped beside its plain name as <name>.gz: reading it by its plain name, and compressing it.
Why gzip and when a file is compressed: docs/decisions.md → `archives: gzip each transcript in loop-claude's archive and each file in the review folders 30 days after it last changed, keeping all of them`.
"""
import errno
import gzip
import os
import pathlib
import typing
import zlib
from failure import Failure
SUFFIX = ".gz"
def stored(path: pathlib.Path) -> pathlib.Path | None:
"""Where the file named path is kept: path itself, else its gzip; None where neither is."""
# A plain file beside its gzip was written after it was compressed, so it holds the newer bytes. Anything else at path, such as a folder, counts as path, so reading it fails naming path.
return next((kept for kept in (path, gzipped(path)) if kept.exists()), None)
def read_stored(path: pathlib.Path) -> bytes:
"""The bytes of the file named path, read from its gzip where it is kept that way. Raises OSError, FileNotFoundError where neither is kept."""
# Tried in turn rather than checked first: compress-archives may remove the plain file once its gzip is whole, between a check and the read.
try:
return path.read_bytes()
except FileNotFoundError:
pass
packed = gzipped(path)
try:
data = packed.read_bytes()
except FileNotFoundError:
raise FileNotFoundError(errno.ENOENT, "No such file or directory", str(path)) from None
try:
return gzip.decompress(data)
except (EOFError, zlib.error, gzip.BadGzipFile) as failure:
raise OSError(errno.EIO, f"not a whole gzip: {failure}", str(packed)) from failure
def stored_glob(folder: pathlib.Path, pattern: str) -> list[pathlib.Path]:
"""The plain name of each file in folder that pattern matches, whether kept plain or gzipped, sorted."""
return sorted({*folder.glob(pattern), *(path.with_name(path.name.removesuffix(SUFFIX)) for path in folder.glob(pattern + SUFFIX))})
def compress(path: pathlib.Path, *, pack: typing.Callable[..., bytes], replace: typing.Callable[[pathlib.Path, bytes], Failure | None], sync: typing.Callable[[int], None]) -> Failure | None:
"""Replaces path with its gzip, which keeps path's times: writes the gzip, reads it back, and removes path only once it reads back path's bytes. A failure keeps path. pack compresses as gzip.compress does, replace writes as replace_file does, and sync flushes as os.fsync does."""
packed = gzipped(path)
try:
before = path.stat()
data = path.read_bytes()
if unwritten := replace(packed, pack(data, mtime=0)):
return Failure(f"could not compress {path}: {unwritten.reason}")
if gzip.decompress(packed.read_bytes()) != data:
return Failure(f"could not compress {path}: {packed} reads back other bytes")
os.utime(packed, ns=(before.st_atime_ns, before.st_mtime_ns))
# On disk before the plain file goes, so a crash cannot leave an empty gzip in its place.
synced(packed, sync)
synced(packed.parent, sync)
after = path.stat()
# Written while it compressed: the plain file keeps the newer bytes, which readers prefer, and a later run compresses them.
if (after.st_size, after.st_mtime_ns) == (before.st_size, before.st_mtime_ns):
path.unlink()
except (OSError, EOFError, zlib.error) as failure:
return Failure(f"could not compress {path}: {failure}")
return None
def synced(path: pathlib.Path, sync: typing.Callable[[int], None]) -> None:
handle = os.open(path, os.O_RDONLY)
try:
sync(handle)
finally:
os.close(handle)
def gzipped(path: pathlib.Path) -> pathlib.Path:
return path.with_name(path.name + SUFFIX)