79 lines
3.5 KiB
Python
79 lines
3.5 KiB
Python
"""A file kept plain, or gzipped beside its plain name as <name>.gz: reading it by its plain name, and compressing it.
|
|
|
|
Why gzip and when a file is compressed: docs/decisions.md → `archives: gzip each transcript in loop-claude's archive and each file in the review folders 30 days after it last changed, keeping all of them`.
|
|
"""
|
|
import errno
|
|
import gzip
|
|
import os
|
|
import pathlib
|
|
import typing
|
|
import zlib
|
|
|
|
from failure import Failure
|
|
|
|
SUFFIX = ".gz"
|
|
|
|
|
|
def stored(path: pathlib.Path) -> pathlib.Path | None:
|
|
"""Where the file named path is kept: path itself, else its gzip; None where neither is."""
|
|
# A plain file beside its gzip was written after it was compressed, so it holds the newer bytes. Anything else at path, such as a folder, counts as path, so reading it fails naming path.
|
|
return next((kept for kept in (path, gzipped(path)) if kept.exists()), None)
|
|
|
|
|
|
def read_stored(path: pathlib.Path) -> bytes:
|
|
"""The bytes of the file named path, read from its gzip where it is kept that way. Raises OSError, FileNotFoundError where neither is kept."""
|
|
# Tried in turn rather than checked first: compress-archives may remove the plain file once its gzip is whole, between a check and the read.
|
|
try:
|
|
return path.read_bytes()
|
|
except FileNotFoundError:
|
|
pass
|
|
packed = gzipped(path)
|
|
try:
|
|
data = packed.read_bytes()
|
|
except FileNotFoundError:
|
|
raise FileNotFoundError(errno.ENOENT, "No such file or directory", str(path)) from None
|
|
try:
|
|
return gzip.decompress(data)
|
|
except (EOFError, zlib.error, gzip.BadGzipFile) as failure:
|
|
raise OSError(errno.EIO, f"not a whole gzip: {failure}", str(packed)) from failure
|
|
|
|
|
|
def stored_glob(folder: pathlib.Path, pattern: str) -> list[pathlib.Path]:
|
|
"""The plain name of each file in folder that pattern matches, whether kept plain or gzipped, sorted."""
|
|
return sorted({*folder.glob(pattern), *(path.with_name(path.name.removesuffix(SUFFIX)) for path in folder.glob(pattern + SUFFIX))})
|
|
|
|
|
|
def compress(path: pathlib.Path, *, pack: typing.Callable[..., bytes], replace: typing.Callable[[pathlib.Path, bytes], Failure | None], sync: typing.Callable[[int], None]) -> Failure | None:
|
|
"""Replaces path with its gzip, which keeps path's times: writes the gzip, reads it back, and removes path only once it reads back path's bytes. A failure keeps path. pack compresses as gzip.compress does, replace writes as replace_file does, and sync flushes as os.fsync does."""
|
|
packed = gzipped(path)
|
|
try:
|
|
before = path.stat()
|
|
data = path.read_bytes()
|
|
if unwritten := replace(packed, pack(data, mtime=0)):
|
|
return Failure(f"could not compress {path}: {unwritten.reason}")
|
|
if gzip.decompress(packed.read_bytes()) != data:
|
|
return Failure(f"could not compress {path}: {packed} reads back other bytes")
|
|
os.utime(packed, ns=(before.st_atime_ns, before.st_mtime_ns))
|
|
# On disk before the plain file goes, so a crash cannot leave an empty gzip in its place.
|
|
synced(packed, sync)
|
|
synced(packed.parent, sync)
|
|
after = path.stat()
|
|
# Written while it compressed: the plain file keeps the newer bytes, which readers prefer, and a later run compresses them.
|
|
if (after.st_size, after.st_mtime_ns) == (before.st_size, before.st_mtime_ns):
|
|
path.unlink()
|
|
except (OSError, EOFError, zlib.error) as failure:
|
|
return Failure(f"could not compress {path}: {failure}")
|
|
return None
|
|
|
|
|
|
def synced(path: pathlib.Path, sync: typing.Callable[[int], None]) -> None:
|
|
handle = os.open(path, os.O_RDONLY)
|
|
try:
|
|
sync(handle)
|
|
finally:
|
|
os.close(handle)
|
|
|
|
|
|
def gzipped(path: pathlib.Path) -> pathlib.Path:
|
|
return path.with_name(path.name + SUFFIX)
|