Files

137 lines
5.8 KiB
Python

import errno
import gzip
import os
import pathlib
import sys
import tempfile
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from text_files.doubles import compress
from text_files import compressed
from failure import Failure
class Compress(unittest.TestCase):
def test_replaces_the_file_with_a_gzip_of_the_same_bytes_and_times(self):
with tempfile.TemporaryDirectory() as root:
plain = pathlib.Path(root, "t.jsonl")
plain.write_bytes(b'{"a": 1}\n')
os.utime(plain, (1000, 2000))
self.assertIsNone(compress(plain))
self.assertFalse(plain.exists())
packed = pathlib.Path(root, "t.jsonl.gz")
# A gzip header's own time is 0, so compressing the same bytes twice writes the same file.
self.assertEqual(packed.read_bytes(), gzip.compress(b'{"a": 1}\n', mtime=0))
self.assertEqual(packed.stat().st_mtime, 2000)
def test_asks_for_a_gzip_whose_header_holds_no_time_whatever_the_packer_would_write_unasked(self):
with tempfile.TemporaryDirectory() as root:
plain = pathlib.Path(root, "t.jsonl")
plain.write_bytes(b"a\n")
asked = []
def pack(data, **kwargs):
asked.append(kwargs)
return gzip.compress(data, **kwargs)
self.assertIsNone(compress(plain, pack=pack))
self.assertEqual(asked, [{"mtime": 0}])
def test_syncs_the_gzip_and_its_folder_before_removing_the_file(self):
with tempfile.TemporaryDirectory() as root:
plain = pathlib.Path(root, "t.jsonl")
plain.write_bytes(b"a\n")
synced = []
fsync = os.fsync
def record(handle):
synced.append((os.readlink(f"/proc/self/fd/{handle}"), plain.exists()))
fsync(handle)
self.assertIsNone(compress(plain, sync=record))
self.assertEqual(synced, [(os.path.realpath(f"{plain}.gz"), True), (os.path.realpath(root), True)])
def test_a_gzip_it_cannot_write_keeps_the_original_and_says_why(self):
with tempfile.TemporaryDirectory() as root:
plain = pathlib.Path(root, "t.jsonl")
plain.write_bytes(b"kept\n")
self.assertEqual(compress(plain, replace=lambda path, data: Failure("disk full")), Failure(f"could not compress {plain}: disk full"))
self.assertEqual(plain.read_bytes(), b"kept\n")
def test_a_file_it_cannot_read_fails_naming_it(self):
with tempfile.TemporaryDirectory() as root:
folder = pathlib.Path(root, "t.jsonl")
folder.mkdir()
failed = compress(folder)
self.assertTrue(failed.reason.startswith(f"could not compress {folder}: [Errno {errno.EISDIR}]"), failed.reason)
def test_a_copy_that_reads_back_other_bytes_keeps_the_original_and_fails(self):
with tempfile.TemporaryDirectory() as root:
plain = pathlib.Path(root, "t.jsonl")
plain.write_bytes(b"kept\n")
failed = compress(plain, pack=lambda data, **kwargs: gzip.compress(b"other\n"))
self.assertIsInstance(failed, Failure)
self.assertIn(str(plain), failed.reason)
self.assertEqual(plain.read_bytes(), b"kept\n")
def test_a_file_written_while_it_compressed_stays_plain(self):
with tempfile.TemporaryDirectory() as root:
plain = pathlib.Path(root, "t.jsonl")
plain.write_bytes(b"one\n")
def grow_meanwhile(data, **kwargs):
with open(plain, "ab") as handle:
handle.write(b"two\n")
return gzip.compress(data, **kwargs)
self.assertIsNone(compress(plain, pack=grow_meanwhile))
self.assertEqual(plain.read_bytes(), b"one\ntwo\n")
self.assertEqual(compressed.read_stored(plain), b"one\ntwo\n")
class Read(unittest.TestCase):
def test_reads_a_file_kept_plain_or_gzipped_by_its_plain_name(self):
with tempfile.TemporaryDirectory() as root:
plain = pathlib.Path(root, "a.jsonl")
plain.write_bytes(b"a\n")
pathlib.Path(root, "b.jsonl.gz").write_bytes(gzip.compress(b"b\n"))
self.assertEqual(compressed.read_stored(plain), b"a\n")
self.assertEqual(compressed.read_stored(pathlib.Path(root, "b.jsonl")), b"b\n")
self.assertEqual(compressed.stored(pathlib.Path(root, "b.jsonl")), pathlib.Path(root, "b.jsonl.gz"))
self.assertIsNone(compressed.stored(pathlib.Path(root, "c.jsonl")))
with self.assertRaises(FileNotFoundError) as raised:
compressed.read_stored(pathlib.Path(root, "c.jsonl"))
self.assertEqual((raised.exception.errno, raised.exception.strerror, raised.exception.filename), (errno.ENOENT, "No such file or directory", str(pathlib.Path(root, "c.jsonl"))))
def test_the_plain_file_wins_over_an_older_gzip_beside_it(self):
with tempfile.TemporaryDirectory() as root:
pathlib.Path(root, "a.jsonl").write_bytes(b"a\nmore\n")
pathlib.Path(root, "a.jsonl.gz").write_bytes(gzip.compress(b"a\n"))
self.assertEqual(compressed.read_stored(pathlib.Path(root, "a.jsonl")), b"a\nmore\n")
def test_a_truncated_gzip_fails_as_an_os_error(self):
with tempfile.TemporaryDirectory() as root:
pathlib.Path(root, "a.jsonl.gz").write_bytes(gzip.compress(b"a\n" * 100)[:20])
with self.assertRaises(OSError) as raised:
compressed.read_stored(pathlib.Path(root, "a.jsonl"))
self.assertEqual((raised.exception.errno, raised.exception.filename), (errno.EIO, str(pathlib.Path(root, "a.jsonl.gz"))))
self.assertTrue(raised.exception.strerror.startswith("not a whole gzip: "), raised.exception.strerror)
def test_a_file_that_is_no_gzip_fails_as_an_os_error(self):
with tempfile.TemporaryDirectory() as root:
pathlib.Path(root, "a.jsonl.gz").write_bytes(b"plain text\n")
with self.assertRaises(OSError) as raised:
compressed.read_stored(pathlib.Path(root, "a.jsonl"))
self.assertEqual((raised.exception.errno, raised.exception.filename), (errno.EIO, str(pathlib.Path(root, "a.jsonl.gz"))))
def test_globs_each_file_once_by_its_plain_name(self):
with tempfile.TemporaryDirectory() as root:
for name in ("a.jsonl", "b.jsonl.gz", "c.jsonl", "c.jsonl.gz", "d.md"):
pathlib.Path(root, name).write_bytes(b"")
self.assertEqual(compressed.stored_glob(pathlib.Path(root), "*.jsonl"), [pathlib.Path(root, name) for name in ("a.jsonl", "b.jsonl", "c.jsonl")])
if __name__ == "__main__":
unittest.main()