feat: add simulation clocks and VM transitions

app/simulation/clock.py

@@ -0,0 +1,61 @@
+"""Injectable simulation clocks; task leases deliberately do not use these."""
+
+from __future__ import annotations
+
+import asyncio
+from datetime import UTC, datetime, timedelta
+from typing import Protocol
+
+
+class Clock(Protocol):
+    async def now(self) -> datetime: ...
+
+    async def sleep(self, seconds: float) -> None: ...
+
+
+class RealClock:
+    async def now(self) -> datetime:
+        return datetime.now(UTC)
+
+    async def sleep(self, seconds: float) -> None:
+        await asyncio.sleep(seconds)
+
+
+class AcceleratedClock:
+    def __init__(self, scale: float) -> None:
+        if scale <= 0:
+            raise ValueError("clock scale must be positive")
+        self._scale = scale
+
+    async def now(self) -> datetime:
+        return datetime.now(UTC)
+
+    async def sleep(self, seconds: float) -> None:
+        await asyncio.sleep(seconds / self._scale)
+
+
+class ManualClock:
+    def __init__(self, initial: datetime) -> None:
+        if initial.tzinfo is None:
+            raise ValueError("manual clock requires timezone-aware time")
+        self._now = initial
+        self._condition = asyncio.Condition()
+
+    async def now(self) -> datetime:
+        async with self._condition:
+            return self._now
+
+    async def sleep(self, seconds: float) -> None:
+        if seconds < 0:
+            raise ValueError("sleep duration cannot be negative")
+        async with self._condition:
+            target = self._now + timedelta(seconds=seconds)
+            await self._condition.wait_for(lambda: self._now >= target)
+
+    async def advance(self, seconds: float) -> datetime:
+        if seconds < 0:
+            raise ValueError("clock cannot move backwards")
+        async with self._condition:
+            self._now += timedelta(seconds=seconds)
+            self._condition.notify_all()
+            return self._now

app/simulation/scenarios.py

@@ -0,0 +1,49 @@
+"""Seeded deterministic fault-rule evaluation."""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class FaultContext:
+    method: str
+    path: str
+    principal: str | None = None
+    node: str | None = None
+    vmid: str | None = None
+    call_number: int = 1
+
+
+@dataclass(frozen=True, slots=True)
+class FaultRule:
+    kind: str
+    probability: float = 1.0
+    method: str | None = None
+    path_prefix: str | None = None
+    principal: str | None = None
+    node: str | None = None
+    vmid: str | None = None
+    call_number: int | None = None
+
+    def __post_init__(self) -> None:
+        if not 0 <= self.probability <= 1:
+            raise ValueError("fault probability must be between zero and one")
+
+
+def matches(rule: FaultRule, context: FaultContext, seed: int) -> bool:
+    filters = (
+        (rule.method, context.method),
+        (rule.principal, context.principal),
+        (rule.node, context.node),
+        (rule.vmid, context.vmid),
+        (rule.call_number, context.call_number),
+    )
+    if any(expected is not None and expected != actual for expected, actual in filters):
+        return False
+    if rule.path_prefix is not None and not context.path.startswith(rule.path_prefix):
+        return False
+    material = f"{seed}:{rule.kind}:{context.method}:{context.path}:{context.call_number}"
+    sample = int.from_bytes(hashlib.sha256(material.encode()).digest()[:8], "big") / 2**64
+    return sample < rule.probability

app/simulation/transitions.py

@@ -0,0 +1,63 @@
+"""Explicit virtual-machine state machine."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+
+from app.simulation.clock import Clock
+
+
+class VmState(StrEnum):
+    STOPPED = "stopped"
+    STARTING = "starting"
+    RUNNING = "running"
+    PAUSING = "pausing"
+    PAUSED = "paused"
+    RESUMING = "resuming"
+    STOPPING = "stopping"
+    MIGRATING = "migrating"
+    SNAPSHOTTING = "snapshotting"
+    BACKING_UP = "backing_up"
+    ERROR = "error"
+
+
+class InvalidTransitionError(ValueError):
+    pass
+
+
+TRANSITIONS: dict[tuple[VmState, str], tuple[VmState, VmState]] = {
+    (VmState.STOPPED, "start"): (VmState.STARTING, VmState.RUNNING),
+    (VmState.RUNNING, "stop"): (VmState.STOPPING, VmState.STOPPED),
+    (VmState.RUNNING, "pause"): (VmState.PAUSING, VmState.PAUSED),
+    (VmState.PAUSED, "resume"): (VmState.RESUMING, VmState.RUNNING),
+    (VmState.RUNNING, "migrate"): (VmState.MIGRATING, VmState.RUNNING),
+    (VmState.STOPPED, "migrate"): (VmState.MIGRATING, VmState.STOPPED),
+    (VmState.RUNNING, "snapshot"): (VmState.SNAPSHOTTING, VmState.RUNNING),
+    (VmState.STOPPED, "snapshot"): (VmState.SNAPSHOTTING, VmState.STOPPED),
+    (VmState.RUNNING, "backup"): (VmState.BACKING_UP, VmState.RUNNING),
+    (VmState.STOPPED, "backup"): (VmState.BACKING_UP, VmState.STOPPED),
+}
+
+
+@dataclass(frozen=True, slots=True)
+class Transition:
+    operation: str
+    before: VmState
+    intermediate: VmState
+    after: VmState
+
+
+def plan_transition(state: VmState, operation: str) -> Transition:
+    states = TRANSITIONS.get((state, operation))
+    if states is None:
+        raise InvalidTransitionError(f"cannot {operation} VM while it is {state}")
+    return Transition(operation, state, states[0], states[1])
+
+
+async def execute_transition(
+    state: VmState, operation: str, clock: Clock, duration_seconds: float
+) -> tuple[VmState, VmState]:
+    transition = plan_transition(state, operation)
+    await clock.sleep(duration_seconds)
+    return transition.intermediate, transition.after

docs/architecture.md

@@ -97,6 +97,12 @@ expired work to be reclaimed after process failure. Lifespan owns a bounded set
 of asyncio workers and waits for orderly shutdown; PostgreSQL remains the queue
 and source of truth across replicas.
 
+Simulation durations use injected real, accelerated, or manually advanced
+clocks. VM operations are explicit state-machine transitions, and seeded fault
+rules evaluate deterministically. Worker leases are intentionally excluded from
+virtual time: they use PostgreSQL wall time and process monotonic sleeps so a
+paused or accelerated scenario cannot invalidate distributed-worker safety.
+
 Authentication secrets use salted scrypt hashes. Session tickets are signed and
 expiring; mutation requests use ticket-bound CSRF tokens. API-token privileges
 are intersected with their owning principal's effective propagated ACLs, so a

tests/unit/test_clock.py

@@ -0,0 +1,28 @@
+"""Simulation clock behavior."""
+
+import asyncio
+from datetime import UTC, datetime
+
+import pytest
+
+from app.simulation.clock import AcceleratedClock, ManualClock
+
+
+async def test_manual_clock_releases_sleep_only_after_advance() -> None:
+    clock = ManualClock(datetime(2026, 1, 1, tzinfo=UTC))
+    sleeper = asyncio.create_task(clock.sleep(10))
+    await asyncio.sleep(0)
+    assert not sleeper.done()
+
+    await clock.advance(9)
+    assert not sleeper.done()
+    await clock.advance(1)
+    await sleeper
+    assert await clock.now() == datetime(2026, 1, 1, 0, 0, 10, tzinfo=UTC)
+
+
+def test_clocks_reject_invalid_configuration() -> None:
+    with pytest.raises(ValueError):
+        AcceleratedClock(0)
+    with pytest.raises(ValueError):
+        ManualClock(datetime(2026, 1, 1))

tests/unit/test_transitions.py

@@ -0,0 +1,46 @@
+"""VM state-machine and deterministic fault properties."""
+
+import pytest
+from hypothesis import given
+from hypothesis import strategies as st
+
+from app.simulation.scenarios import FaultContext, FaultRule, matches
+from app.simulation.transitions import InvalidTransitionError, VmState, plan_transition
+
+
+@pytest.mark.parametrize(
+    ("state", "operation", "final"),
+    [
+        (VmState.STOPPED, "start", VmState.RUNNING),
+        (VmState.RUNNING, "stop", VmState.STOPPED),
+        (VmState.RUNNING, "pause", VmState.PAUSED),
+        (VmState.PAUSED, "resume", VmState.RUNNING),
+        (VmState.RUNNING, "snapshot", VmState.RUNNING),
+        (VmState.STOPPED, "migrate", VmState.STOPPED),
+    ],
+)
+def test_valid_transitions(state: VmState, operation: str, final: VmState) -> None:
+    transition = plan_transition(state, operation)
+    assert transition.before is state
+    assert transition.after is final
+    assert transition.intermediate is not state
+
+
+@given(st.sampled_from(tuple(VmState)), st.text(min_size=1, max_size=12))
+def test_transition_result_is_declared_or_rejected(state: VmState, operation: str) -> None:
+    try:
+        transition = plan_transition(state, operation)
+    except InvalidTransitionError:
+        return
+    assert transition.before is state
+
+
+def test_fault_evaluation_is_seeded_and_filtered() -> None:
+    context = FaultContext("POST", "/nodes/pve1/qemu/100/status/start", node="pve1")
+    certain = FaultRule("task-failure", method="POST", node="pve1")
+    impossible = FaultRule("task-failure", probability=0)
+
+    assert matches(certain, context, seed=42)
+    assert not matches(impossible, context, seed=42)
+    probabilistic = FaultRule("task-failure", probability=0.5)
+    assert matches(probabilistic, context, 42) == matches(probabilistic, context, 42)