feat: add durable leased task engine

tests/integration/test_tasks.py

@@ -0,0 +1,81 @@
+"""Durable task concurrency and recovery tests."""
+
+import asyncio
+import os
+import uuid
+
+import asyncpg  # type: ignore[import-untyped]
+import pytest
+from asyncpg import Pool
+
+from app.db.migrations import migrate
+from app.tasks.repository import TaskRepository
+
+pytestmark = [
+    pytest.mark.integration,
+    pytest.mark.skipif(not os.getenv("TEST_DATABASE_URL"), reason="TEST_DATABASE_URL is required"),
+]
+
+
+async def repository() -> tuple[Pool, TaskRepository]:
+    pool = await asyncpg.create_pool(os.environ["TEST_DATABASE_URL"], min_size=1, max_size=4)
+    async with pool.acquire() as connection:
+        await migrate(connection)
+    return pool, TaskRepository(pool)
+
+
+async def test_two_worker_exclusion_idempotency_and_logs() -> None:
+    pool, tasks = await repository()
+    key = uuid.uuid4().hex
+    try:
+        created = await tasks.create(
+            upid=f"UPID:pve1:00000001:00000001:00000001:test:{key}:root@pam:",
+            task_type="test",
+            payload={"value": 1},
+            resource_key=f"vm:{key}",
+            idempotency_key=key,
+        )
+        repeated = await tasks.create(
+            upid=f"ignored-{key}", task_type="test", payload={}, idempotency_key=key
+        )
+        assert repeated.id == created.id
+
+        first, second = await asyncio.gather(
+            tasks.claim("worker-a", 30), tasks.claim("worker-b", 30)
+        )
+        claimed = first or second
+        assert claimed is not None
+        assert (first is None) != (second is None)
+        worker = "worker-a" if first is not None else "worker-b"
+        await tasks.append_log(claimed.id, "started")
+        await tasks.progress(claimed.id, worker, 50)
+        await tasks.finish(claimed.id, worker, status="success", result={"ok": True})
+        assert await tasks.logs(claimed.id) == ("started",)
+        finished = await tasks.get(claimed.id)
+        assert finished is not None
+        assert finished.status == "success"
+    finally:
+        await pool.close()
+
+
+async def test_expired_lease_is_reclaimed_after_restart() -> None:
+    pool, tasks = await repository()
+    key = uuid.uuid4().hex
+    try:
+        created = await tasks.create(
+            upid=f"UPID:pve1:00000001:00000001:00000001:test:{key}:root@pam:",
+            task_type="test",
+            payload={},
+        )
+        assert await tasks.claim("dead-worker", 0) is not None
+        recovered = await tasks.claim("new-worker", 30)
+        assert recovered is not None
+        assert recovered.id == created.id
+        assert recovered.attempt == 2
+        await tasks.request_cancel(recovered.id)
+        cancelled = await tasks.get(recovered.id)
+        assert cancelled is not None
+        assert cancelled.cancel_requested
+        await tasks.finish(recovered.id, "new-worker", status="cancelled")
+    finally:
+        await pool.close()

tests/unit/test_health.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 from typing import Self
 
 import pytest
@@ -56,3 +57,27 @@ async def test_health_endpoints(database_ready: bool, status_code: int) -> None:
     assert ready.headers["X-Request-ID"] == "test-request"
     assert database.connected
     assert database.closed
+
+
+async def test_lifespan_starts_and_stops_injected_workers() -> None:
+    database = FakeDatabase(True)
+    started = asyncio.Event()
+    stopping = asyncio.Event()
+
+    class Worker:
+        async def run(self) -> None:
+            started.set()
+            await stopping.wait()
+
+        def stop(self) -> None:
+            stopping.set()
+
+    application = create_app(
+        Settings(),
+        lambda _settings: database,
+        worker_factories=(lambda _database: Worker(),),
+    )
+    async with application.router.lifespan_context(application):
+        await started.wait()
+
+    assert stopping.is_set()

tests/unit/test_task_worker.py

@@ -0,0 +1,74 @@
+"""Bounded task worker outcome tests."""
+
+import uuid
+from typing import cast
+
+from app.tasks.repository import Task, TaskRepository
+from app.tasks.worker import TaskWorker
+
+
+class FakeRepository:
+    def __init__(self, task: Task) -> None:
+        self.task = task
+        self.finishes: list[tuple[str, str | None]] = []
+
+    async def get(self, _task_id: uuid.UUID) -> Task:
+        return self.task
+
+    async def finish(
+        self,
+        _task_id: uuid.UUID,
+        _worker_id: str,
+        *,
+        status: str,
+        result: dict[str, object] | None = None,
+        error: str | None = None,
+    ) -> None:
+        del result
+        self.finishes.append((status, error))
+
+
+def make_task(*, task_type: str = "test", cancelled: bool = False) -> Task:
+    return Task(uuid.uuid4(), "UPID:test", task_type, "running", {}, 0, cancelled, 1)
+
+
+async def test_worker_persists_success_error_and_unsupported() -> None:
+    task = make_task()
+    repository = FakeRepository(task)
+
+    async def success(_task: Task) -> dict[str, object]:
+        return {"ok": True}
+
+    worker = TaskWorker(cast(TaskRepository, repository), "worker", {"test": success})
+    await worker._execute(task)
+    assert repository.finishes == [("success", None)]
+
+    unsupported = make_task(task_type="missing")
+    repository.task = unsupported
+    await worker._execute(unsupported)
+    assert repository.finishes[-1] == ("error", "unsupported task type")
+
+    async def failure(_task: Task) -> None:
+        raise RuntimeError("private detail")
+
+    failed = make_task()
+    repository.task = failed
+    worker.handlers["test"] = failure
+    await worker._execute(failed)
+    assert repository.finishes[-1] == ("error", "RuntimeError")
+
+
+async def test_worker_honors_persisted_cancellation() -> None:
+    task = make_task(cancelled=True)
+    repository = FakeRepository(task)
+    called = False
+
+    async def handler(_task: Task) -> None:
+        nonlocal called
+        called = True
+
+    worker = TaskWorker(cast(TaskRepository, repository), "worker", {"test": handler})
+    await worker._execute(task)
+
+    assert not called
+    assert repository.finishes == [("cancelled", None)]

tests/unit/test_upid.py

@@ -0,0 +1,71 @@
+"""UPID examples and round-trip properties."""
+
+import pytest
+from hypothesis import given
+from hypothesis import strategies as st
+
+from app.tasks.upid import Upid
+
+SAFE = st.from_regex(r"[a-z0-9][a-z0-9_-]{0,19}", fullmatch=True)
+
+
+@given(
+    node=SAFE,
+    pid=st.integers(min_value=0, max_value=0xFFFFFFFF),
+    process_start=st.integers(min_value=0, max_value=0xFFFFFFFF),
+    start_time=st.integers(min_value=0, max_value=0xFFFFFFFF),
+    task_type=SAFE,
+    task_id=st.text(alphabet="abcdefghijklmnopqrstuvwxyz0123456789_-", max_size=20),
+    user=SAFE,
+)
+def test_upid_round_trip(
+    node: str,
+    pid: int,
+    process_start: int,
+    start_time: int,
+    task_type: str,
+    task_id: str,
+    user: str,
+) -> None:
+    upid = Upid(node, pid, process_start, start_time, task_type, task_id, user)
+
+    assert Upid.parse(str(upid)) == upid
+
+
+def test_known_upid_shape() -> None:
+    value = "UPID:pve1:0000002A:00000010:65A1B2C3:qmstart:100:root@pam:"
+
+    parsed = Upid.parse(value)
+
+    assert parsed.pid == 42
+    assert parsed.task_id == "100"
+    assert str(parsed) == value
+
+
+@pytest.mark.parametrize("value", ["", "UPID:broken", "UPID:pve:GGGGGGGG:00000000:00000000:x::u:"])
+def test_invalid_upids_are_rejected(value: str) -> None:
+    with pytest.raises(ValueError):
+        Upid.parse(value)
+
+
+@pytest.mark.parametrize(
+    "kwargs",
+    [
+        {"pid": -1},
+        {"node": "bad:node"},
+        {"task_id": "bad:id"},
+    ],
+)
+def test_invalid_upid_components_are_rejected(kwargs: dict[str, object]) -> None:
+    values: dict[str, object] = {
+        "node": "pve1",
+        "pid": 1,
+        "process_start": 1,
+        "start_time": 1,
+        "task_type": "test",
+        "task_id": "100",
+        "user": "root@pam",
+    }
+    values.update(kwargs)
+    with pytest.raises(ValueError):
+        Upid(**values)  # type: ignore[arg-type]