Hold off resuming while the host is short on memory or swapping

test-resume-stalled-sessions.py

@@ -109,6 +109,45 @@ class ResetMoment(unittest.TestCase):
         self.assertIsNone(watchdog.reset_moment("API error — see detail", self.blocked_at))
 
 
+class MemoryStrain(unittest.TestCase):
+    """A host with no room left must not be given another session to run."""
+
+    def meminfo(self, available_kb, swap_total_kb=4194300, swap_free_kb=4194300):
+        temp = tempfile.NamedTemporaryFile("w", suffix=".meminfo", delete=False)
+        self.addCleanup(os.unlink, temp.name)
+        temp.write(
+            "MemTotal:       32877496 kB\n"
+            "MemFree:         1048576 kB\n"
+            f"MemAvailable:   {available_kb} kB\n"
+            f"SwapTotal:      {swap_total_kb} kB\n"
+            f"SwapFree:       {swap_free_kb} kB\n"
+        )
+        temp.close()
+        return Path(temp.name)
+
+    def test_healthy_host_takes_another_session(self):
+        self.assertIsNone(watchdog.memory_strain(2048, 50, self.meminfo(30083284)))
+
+    def test_low_memory_holds_off(self):
+        strain = watchdog.memory_strain(2048, 50, self.meminfo(1048576))
+        self.assertIn("1024 MB available", strain)
+
+    def test_swap_in_heavy_use_holds_off(self):
+        strain = watchdog.memory_strain(2048, 50, self.meminfo(30083284, swap_free_kb=1048575))
+        self.assertIn("swap 75% used", strain)
+
+    def test_swapless_host_is_judged_on_memory_alone(self):
+        self.assertIsNone(
+            watchdog.memory_strain(2048, 50, self.meminfo(30083284, swap_total_kb=0, swap_free_kb=0))
+        )
+
+    def test_host_that_does_not_report_memory_is_not_blocked(self):
+        self.assertIsNone(watchdog.memory_strain(2048, 50, Path("/nonexistent/meminfo")))
+        kernel_without_memavailable = self.meminfo(0)
+        kernel_without_memavailable.write_text("MemTotal:       32877496 kB\n")
+        self.assertIsNone(watchdog.memory_strain(2048, 50, kernel_without_memavailable))
+
+
 class Candidates(unittest.TestCase):
     """The whole path: a spend-limited background job with no process left."""