137 lines
5.8 KiB
Python
137 lines
5.8 KiB
Python
import errno
|
|
import gzip
|
|
import os
|
|
import pathlib
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from text_files.doubles import compress
|
|
from text_files import compressed
|
|
from failure import Failure
|
|
|
|
|
|
class Compress(unittest.TestCase):
|
|
def test_replaces_the_file_with_a_gzip_of_the_same_bytes_and_times(self):
|
|
with tempfile.TemporaryDirectory() as root:
|
|
plain = pathlib.Path(root, "t.jsonl")
|
|
plain.write_bytes(b'{"a": 1}\n')
|
|
os.utime(plain, (1000, 2000))
|
|
self.assertIsNone(compress(plain))
|
|
self.assertFalse(plain.exists())
|
|
packed = pathlib.Path(root, "t.jsonl.gz")
|
|
# A gzip header's own time is 0, so compressing the same bytes twice writes the same file.
|
|
self.assertEqual(packed.read_bytes(), gzip.compress(b'{"a": 1}\n', mtime=0))
|
|
self.assertEqual(packed.stat().st_mtime, 2000)
|
|
|
|
def test_asks_for_a_gzip_whose_header_holds_no_time_whatever_the_packer_would_write_unasked(self):
|
|
with tempfile.TemporaryDirectory() as root:
|
|
plain = pathlib.Path(root, "t.jsonl")
|
|
plain.write_bytes(b"a\n")
|
|
asked = []
|
|
|
|
def pack(data, **kwargs):
|
|
asked.append(kwargs)
|
|
return gzip.compress(data, **kwargs)
|
|
|
|
self.assertIsNone(compress(plain, pack=pack))
|
|
self.assertEqual(asked, [{"mtime": 0}])
|
|
|
|
def test_syncs_the_gzip_and_its_folder_before_removing_the_file(self):
|
|
with tempfile.TemporaryDirectory() as root:
|
|
plain = pathlib.Path(root, "t.jsonl")
|
|
plain.write_bytes(b"a\n")
|
|
synced = []
|
|
fsync = os.fsync
|
|
|
|
def record(handle):
|
|
synced.append((os.readlink(f"/proc/self/fd/{handle}"), plain.exists()))
|
|
fsync(handle)
|
|
|
|
self.assertIsNone(compress(plain, sync=record))
|
|
self.assertEqual(synced, [(os.path.realpath(f"{plain}.gz"), True), (os.path.realpath(root), True)])
|
|
|
|
def test_a_gzip_it_cannot_write_keeps_the_original_and_says_why(self):
|
|
with tempfile.TemporaryDirectory() as root:
|
|
plain = pathlib.Path(root, "t.jsonl")
|
|
plain.write_bytes(b"kept\n")
|
|
self.assertEqual(compress(plain, replace=lambda path, data: Failure("disk full")), Failure(f"could not compress {plain}: disk full"))
|
|
self.assertEqual(plain.read_bytes(), b"kept\n")
|
|
|
|
def test_a_file_it_cannot_read_fails_naming_it(self):
|
|
with tempfile.TemporaryDirectory() as root:
|
|
folder = pathlib.Path(root, "t.jsonl")
|
|
folder.mkdir()
|
|
failed = compress(folder)
|
|
self.assertTrue(failed.reason.startswith(f"could not compress {folder}: [Errno {errno.EISDIR}]"), failed.reason)
|
|
|
|
def test_a_copy_that_reads_back_other_bytes_keeps_the_original_and_fails(self):
|
|
with tempfile.TemporaryDirectory() as root:
|
|
plain = pathlib.Path(root, "t.jsonl")
|
|
plain.write_bytes(b"kept\n")
|
|
failed = compress(plain, pack=lambda data, **kwargs: gzip.compress(b"other\n"))
|
|
self.assertIsInstance(failed, Failure)
|
|
self.assertIn(str(plain), failed.reason)
|
|
self.assertEqual(plain.read_bytes(), b"kept\n")
|
|
|
|
def test_a_file_written_while_it_compressed_stays_plain(self):
|
|
with tempfile.TemporaryDirectory() as root:
|
|
plain = pathlib.Path(root, "t.jsonl")
|
|
plain.write_bytes(b"one\n")
|
|
def grow_meanwhile(data, **kwargs):
|
|
with open(plain, "ab") as handle:
|
|
handle.write(b"two\n")
|
|
return gzip.compress(data, **kwargs)
|
|
|
|
self.assertIsNone(compress(plain, pack=grow_meanwhile))
|
|
self.assertEqual(plain.read_bytes(), b"one\ntwo\n")
|
|
self.assertEqual(compressed.read_stored(plain), b"one\ntwo\n")
|
|
|
|
|
|
class Read(unittest.TestCase):
|
|
def test_reads_a_file_kept_plain_or_gzipped_by_its_plain_name(self):
|
|
with tempfile.TemporaryDirectory() as root:
|
|
plain = pathlib.Path(root, "a.jsonl")
|
|
plain.write_bytes(b"a\n")
|
|
pathlib.Path(root, "b.jsonl.gz").write_bytes(gzip.compress(b"b\n"))
|
|
self.assertEqual(compressed.read_stored(plain), b"a\n")
|
|
self.assertEqual(compressed.read_stored(pathlib.Path(root, "b.jsonl")), b"b\n")
|
|
self.assertEqual(compressed.stored(pathlib.Path(root, "b.jsonl")), pathlib.Path(root, "b.jsonl.gz"))
|
|
self.assertIsNone(compressed.stored(pathlib.Path(root, "c.jsonl")))
|
|
with self.assertRaises(FileNotFoundError) as raised:
|
|
compressed.read_stored(pathlib.Path(root, "c.jsonl"))
|
|
self.assertEqual((raised.exception.errno, raised.exception.strerror, raised.exception.filename), (errno.ENOENT, "No such file or directory", str(pathlib.Path(root, "c.jsonl"))))
|
|
|
|
def test_the_plain_file_wins_over_an_older_gzip_beside_it(self):
|
|
with tempfile.TemporaryDirectory() as root:
|
|
pathlib.Path(root, "a.jsonl").write_bytes(b"a\nmore\n")
|
|
pathlib.Path(root, "a.jsonl.gz").write_bytes(gzip.compress(b"a\n"))
|
|
self.assertEqual(compressed.read_stored(pathlib.Path(root, "a.jsonl")), b"a\nmore\n")
|
|
|
|
def test_a_truncated_gzip_fails_as_an_os_error(self):
|
|
with tempfile.TemporaryDirectory() as root:
|
|
pathlib.Path(root, "a.jsonl.gz").write_bytes(gzip.compress(b"a\n" * 100)[:20])
|
|
with self.assertRaises(OSError) as raised:
|
|
compressed.read_stored(pathlib.Path(root, "a.jsonl"))
|
|
self.assertEqual((raised.exception.errno, raised.exception.filename), (errno.EIO, str(pathlib.Path(root, "a.jsonl.gz"))))
|
|
self.assertTrue(raised.exception.strerror.startswith("not a whole gzip: "), raised.exception.strerror)
|
|
|
|
def test_a_file_that_is_no_gzip_fails_as_an_os_error(self):
|
|
with tempfile.TemporaryDirectory() as root:
|
|
pathlib.Path(root, "a.jsonl.gz").write_bytes(b"plain text\n")
|
|
with self.assertRaises(OSError) as raised:
|
|
compressed.read_stored(pathlib.Path(root, "a.jsonl"))
|
|
self.assertEqual((raised.exception.errno, raised.exception.filename), (errno.EIO, str(pathlib.Path(root, "a.jsonl.gz"))))
|
|
|
|
def test_globs_each_file_once_by_its_plain_name(self):
|
|
with tempfile.TemporaryDirectory() as root:
|
|
for name in ("a.jsonl", "b.jsonl.gz", "c.jsonl", "c.jsonl.gz", "d.md"):
|
|
pathlib.Path(root, name).write_bytes(b"")
|
|
self.assertEqual(compressed.stored_glob(pathlib.Path(root), "*.jsonl"), [pathlib.Path(root, name) for name in ("a.jsonl", "b.jsonl", "c.jsonl")])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|