From 636f42ce9d6eb004baed734074f284f10bbffe8b Mon Sep 17 00:00:00 2001 From: Sergey Antropoff Date: Mon, 13 Jul 2026 01:04:13 +0300 Subject: [PATCH] feat: add evidence-based compatibility dimensions --- Dockerfile | 1 + README.md | 5 + app/compatibility.py | 165 ++++++++++++++++++++++++++ app/config.py | 1 + app/main.py | 25 +++- docker-compose.yml | 1 + docs/compatibility-0.1.0.md | 8 ++ docs/original-prompt-gap-plan.md | 147 +++++++++++++++++++++++ evidence/pve-9.2.3-0.1.0.json | 55 +++++++++ tests/compatibility/test_proxmoxer.py | 4 +- tests/unit/test_compatibility.py | 58 ++++++++- 11 files changed, 464 insertions(+), 6 deletions(-) create mode 100644 docs/original-prompt-gap-plan.md create mode 100644 evidence/pve-9.2.3-0.1.0.json diff --git a/Dockerfile b/Dockerfile index 8202000..b6027ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,7 @@ RUN groupadd --system --gid 10001 simulator \ && useradd --system --uid 10001 --gid simulator --home-dir /app --no-create-home simulator COPY --from=builder /opt/venv /opt/venv COPY contracts/e61a893e996d05d376579226e7dfbedbcfce8b71787adacffbc557e6e35901c1/snapshot.json /app/contracts/pve-9.2.3.json +COPY evidence/pve-9.2.3-0.1.0.json /app/evidence/pve-9.2.3-0.1.0.json WORKDIR /app USER 10001:10001 EXPOSE 8006 diff --git a/README.md b/README.md index 37ddde3..3911d65 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,11 @@ Run the external-client smoke flow against the Compose network with `compatibility`. It covers login, reads, CSRF-protected mutation, and UPID task completion. +Machine-readable evidence is served at `/admin/compatibility`; deterministic +Markdown and HTML variants use `/admin/compatibility.md` and +`/admin/compatibility.html`. Scores are separated across all 13 contract, +response, state, task, error, and permission dimensions. + Database migrations are ordered SQL files applied transactionally and recorded with SHA-256 checksums. Re-running `make db-migrate` is safe; changing an already applied migration is rejected instead of silently drifting the schema. diff --git a/app/compatibility.py b/app/compatibility.py index b0d24e3..d9b27ca 100644 --- a/app/compatibility.py +++ b/app/compatibility.py @@ -2,13 +2,90 @@ from __future__ import annotations +import json +from collections.abc import Mapping from dataclasses import dataclass +from enum import StrEnum +from html import escape +from pathlib import Path +from types import MappingProxyType + +from pydantic import BaseModel, ConfigDict, field_validator, model_validator from app.contracts.model import Snapshot MethodKey = tuple[str, str] +class CompatibilityDimension(StrEnum): + ROUTE_METHOD = "route_method" + INPUT_PARAMETERS = "input_parameters" + PARAMETER_REQUIREDNESS = "parameter_requiredness" + TYPES_CONSTRAINTS = "types_constraints" + HTTP_STATUS = "http_status" + JSON_STRUCTURE = "json_structure" + RESPONSE_FIELD_TYPES = "response_field_types" + RESPONSE_REQUIRED_FIELDS = "response_required_fields" + HEADERS_COOKIES = "headers_cookies" + STATE_SEMANTICS = "state_semantics" + LONG_TASK_BEHAVIOR = "long_task_behavior" + ERRORS_PROHIBITIONS = "errors_prohibitions" + PERMISSIONS = "permissions" + + +EMPTY_DIMENSION_EVIDENCE: Mapping[CompatibilityDimension, frozenset[MethodKey]] = MappingProxyType( + {dimension: frozenset() for dimension in CompatibilityDimension} +) + + +class MethodEvidence(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + path: str + verb: str + dimensions: tuple[CompatibilityDimension, ...] + sources: tuple[str, ...] + + @field_validator("sources") + @classmethod + def require_sources(cls, value: tuple[str, ...]) -> tuple[str, ...]: + if not value: + raise ValueError("evidence record requires at least one source") + return value + + +class EvidenceManifest(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + format_version: int = 1 + profile: str + source_version: str + records: tuple[MethodEvidence, ...] + + @model_validator(mode="after") + def reject_duplicate_methods(self) -> EvidenceManifest: + keys = [(record.path, record.verb) for record in self.records] + if len(keys) != len(set(keys)): + raise ValueError("evidence manifest contains duplicate methods") + return self + + def dimension_map(self) -> Mapping[CompatibilityDimension, frozenset[MethodKey]]: + evidence: dict[CompatibilityDimension, set[MethodKey]] = { + dimension: set() for dimension in CompatibilityDimension + } + for record in self.records: + key = (record.path, record.verb) + for dimension in record.dimensions: + evidence[dimension].add(key) + return MappingProxyType( + {dimension: frozenset(methods) for dimension, methods in evidence.items()} + ) + + +def load_evidence_manifest(path: Path) -> EvidenceManifest: + return EvidenceManifest.model_validate_json(path.read_bytes()) + + @dataclass(frozen=True, slots=True) class CompatibilityReport: source_version: str @@ -17,6 +94,9 @@ class CompatibilityReport: implemented: frozenset[MethodKey] observed: frozenset[MethodKey] verified: frozenset[MethodKey] + dimensions: Mapping[CompatibilityDimension, frozenset[MethodKey]] + incompatible: frozenset[MethodKey] + regressions: frozenset[MethodKey] def as_json(self) -> dict[str, object]: levels = { @@ -27,6 +107,13 @@ class CompatibilityReport: "verified": self.verified, } total = len(self.declared) + dimension_sets = tuple(self.dimensions.values()) + fully_evidenced = ( + dimension_sets[0].intersection(*dimension_sets[1:]) if dimension_sets else frozenset() + ) + evidenced = frozenset().union(*dimension_sets) + fully_compatible = fully_evidenced & self.implemented + partially_compatible = (evidenced & self.implemented) - fully_compatible - self.incompatible return { "source_version": self.source_version, "total_declared": total, @@ -39,8 +126,31 @@ class CompatibilityReport: for name, methods in levels.items() }, "groups": self._groups(), + "dimension_groups": self._dimension_groups(), + "classifications": { + "fully_compatible": self._method_names(fully_compatible), + "partially_compatible": self._method_names(partially_compatible), + "incompatible": self._method_names(self.incompatible), + "regressions": self._method_names(self.regressions), + "unsupported": self._method_names(self.schema_only), + }, + "dimensions": { + dimension.value: { + "count": len(methods), + "score": len(methods) / total if total else 1.0, + "methods": [f"{verb} {path}" for path, verb in sorted(methods)], + } + for dimension, methods in self.dimensions.items() + }, } + @staticmethod + def _method_names(methods: frozenset[MethodKey]) -> list[str]: + return [f"{verb} {path}" for path, verb in sorted(methods)] + + def canonical_json(self) -> str: + return json.dumps(self.as_json(), ensure_ascii=False, separators=(",", ":"), sort_keys=True) + def _groups(self) -> dict[str, dict[str, int]]: groups: dict[str, dict[str, int]] = {} for path, verb in self.declared: @@ -51,6 +161,17 @@ class CompatibilityReport: counters["verified"] += int((path, verb) in self.verified) return dict(sorted(groups.items())) + def _dimension_groups(self) -> dict[str, dict[str, int]]: + groups: dict[str, dict[str, int]] = {} + for dimension, methods in self.dimensions.items(): + for path, _verb in methods: + group = path.strip("/").split("/", 1)[0] or "root" + counters = groups.setdefault( + group, {item.value: 0 for item in CompatibilityDimension} + ) + counters[dimension.value] += 1 + return dict(sorted(groups.items())) + def as_markdown(self) -> str: levels = { "declared": self.declared, @@ -69,8 +190,37 @@ class CompatibilityReport: for name, methods in levels.items(): score = len(methods) / total if total else 1.0 lines.append(f"| {name} | {len(methods)} | {score:.2%} |") + lines.extend( + [ + "", + "## Compatibility dimensions", + "", + "| Dimension | Verified methods | Score |", + "|---|---:|---:|", + ] + ) + for dimension, methods in self.dimensions.items(): + score = len(methods) / total if total else 1.0 + lines.append(f"| {dimension.value} | {len(methods)} | {score:.2%} |") return "\n".join(lines) + def as_html(self) -> str: + rows = "".join( + "" + f"{escape(dimension.value)}" + f"{len(methods)}" + f"{(len(methods) / len(self.declared) if self.declared else 1.0):.2%}" + "" + for dimension, methods in self.dimensions.items() + ) + return ( + '' + "Compatibility report" + f"

PVE {escape(self.source_version)} compatibility

" + "" + f"{rows}
DimensionVerified methodsScore
" + ) + def build_report( snapshot: Snapshot, @@ -78,6 +228,9 @@ def build_report( implemented: frozenset[MethodKey] = frozenset(), observed: frozenset[MethodKey] = frozenset(), verified: frozenset[MethodKey] = frozenset(), + dimensions: Mapping[CompatibilityDimension, frozenset[MethodKey]] = EMPTY_DIMENSION_EVIDENCE, + incompatible: frozenset[MethodKey] = frozenset(), + regressions: frozenset[MethodKey] = frozenset(), ) -> CompatibilityReport: declared = frozenset( (path.path, method.verb) for path in snapshot.paths for method in path.methods @@ -86,9 +239,18 @@ def build_report( "implemented": implemented, "observed": observed, "verified": verified, + "incompatible": incompatible, + "regressions": regressions, }.items(): if not evidence <= declared: raise ValueError(f"{name} evidence references undeclared methods") + resolved_dimensions = { + dimension: frozenset(dimensions.get(dimension, frozenset())) + for dimension in CompatibilityDimension + } + for dimension, evidence in resolved_dimensions.items(): + if not evidence <= declared: + raise ValueError(f"{dimension.value} evidence references undeclared methods") return CompatibilityReport( source_version=snapshot.source_version, declared=declared, @@ -96,4 +258,7 @@ def build_report( implemented=implemented, observed=observed, verified=verified, + dimensions=MappingProxyType(resolved_dimensions), + incompatible=incompatible, + regressions=regressions, ) diff --git a/app/config.py b/app/config.py index 705e4d5..f68187d 100644 --- a/app/config.py +++ b/app/config.py @@ -33,6 +33,7 @@ class Settings(BaseSettings): log_level: str = "INFO" request_id_header: str = "X-Request-ID" contract_snapshot: Path | None = None + compatibility_evidence: Path | None = None contract_fallback: Literal["error", "schema-default", "fixture"] = "error" ticket_signing_key: SecretStr = SecretStr("development-only-signing-key-change-me") task_worker_concurrency: int = Field(default=2, ge=1, le=32) diff --git a/app/main.py b/app/main.py index 69c805f..72f4b3a 100644 --- a/app/main.py +++ b/app/main.py @@ -4,12 +4,12 @@ from __future__ import annotations from typing import cast -from fastapi import FastAPI +from fastapi import FastAPI, Response from app.api.errors import ApiError, api_error_handler, unhandled_exception_handler from app.api.middleware import RequestContextMiddleware from app.api.registry import HandlerRegistry, register_contract_routes -from app.compatibility import build_report +from app.compatibility import CompatibilityDimension, build_report, load_evidence_manifest from app.config import Settings, get_settings from app.contracts.model import Snapshot from app.db.pool import AsyncpgDatabase, Database @@ -71,12 +71,31 @@ def create_app( declared = frozenset( (path.path, method.verb) for path in snapshot.paths for method in path.methods ) - report = build_report(snapshot, implemented=resolved_handlers.keys() & declared) + dimensions = {CompatibilityDimension.ROUTE_METHOD: declared} + if resolved.compatibility_evidence is not None: + evidence = load_evidence_manifest(resolved.compatibility_evidence) + if evidence.source_version != snapshot.source_version: + raise ValueError("compatibility evidence version does not match contract") + dimensions.update(evidence.dimension_map()) + dimensions[CompatibilityDimension.ROUTE_METHOD] = declared + report = build_report( + snapshot, + implemented=resolved_handlers.keys() & declared, + dimensions=dimensions, + ) @app.get("/admin/compatibility", include_in_schema=False) async def compatibility_report() -> dict[str, object]: return report.as_json() + @app.get("/admin/compatibility.md", include_in_schema=False) + async def compatibility_report_markdown() -> Response: + return Response(report.as_markdown(), media_type="text/markdown") + + @app.get("/admin/compatibility.html", include_in_schema=False) + async def compatibility_report_html() -> Response: + return Response(report.as_html(), media_type="text/html") + return app diff --git a/docker-compose.yml b/docker-compose.yml index 0ebc807..8bc7a9d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,6 +25,7 @@ services: environment: DATABASE_URL: postgresql://proxmox:proxmox@postgres:5432/proxmox_simulator CONTRACT_SNAPSHOT: /app/contracts/pve-9.2.3.json + COMPATIBILITY_EVIDENCE: /app/evidence/pve-9.2.3-0.1.0.json depends_on: postgres: condition: service_healthy diff --git a/docs/compatibility-0.1.0.md b/docs/compatibility-0.1.0.md index 878e6b6..cb21664 100644 --- a/docs/compatibility-0.1.0.md +++ b/docs/compatibility-0.1.0.md @@ -43,3 +43,11 @@ worker tests; a single smoke run chooses the transition valid for current state. The live `/admin/compatibility` endpoint is the machine-readable source for declared and implemented counts. Unsupported methods remain failures by default so the simulator cannot silently overstate compatibility. + +The report also exposes the 13 independent compatibility dimensions required by +the project brief. Evidence is loaded from the immutable +`evidence/pve-9.2.3-0.1.0.json` manifest, where every method/dimension claim cites +the tests that support it. Dynamic route registration itself proves only the +route/method dimension; it does not imply semantic compatibility. Markdown and +HTML renderings are available at `/admin/compatibility.md` and +`/admin/compatibility.html`. diff --git a/docs/original-prompt-gap-plan.md b/docs/original-prompt-gap-plan.md new file mode 100644 index 0000000..6e1315d --- /dev/null +++ b/docs/original-prompt-gap-plan.md @@ -0,0 +1,147 @@ +# Original prompt gap plan + +This is the executable completion checklist for the original 1,985-line project +brief. It starts after release 0.1.0 and supersedes the short “subsequent +releases” list as the source of delivery status. A box is closed only when code, +tests, documentation, container acceptance, and a focused commit exist. + +Status at audit time: 0.1.0 is operational, but the overall project is not done. + +## G1 — measurable compatibility evidence + +- [x] Model all 13 requested compatibility dimensions independently: route and + method, inputs, requiredness, types and constraints, HTTP status, JSON shape, + response field types, required response fields, headers and cookies, state + semantics, long-task behavior, errors and prohibitions, permissions. +- [x] Store test-derived evidence per method/profile instead of deriving strong + claims merely from handler presence. +- [x] Generate deterministic JSON, Markdown, and HTML reports with totals and + breakdowns by API group and PVE version. +- [x] Mark declared, schema-only, implemented, observed, partially compatible, + incompatible, and regression states without claiming unsupported semantics. + +Exit: a test fixture can prove different scores for every dimension and the +live admin report renders the same evidence deterministically. + +## G2 — persistence model and deterministic datasets + +- [ ] Expand normalized tables/repositories for storages and contents, QEMU, + LXC, disks, NICs, snapshots, backups, pools, users/groups/roles/ACLs/tokens and + observed contracts. Preserve version, metadata, timestamps, and cluster-wide + VMID uniqueness. +- [ ] Make migration readiness explicit so workers cannot become permanently + unhealthy before schema creation. +- [ ] Match the required `small` profile (one node, two QEMU, one LXC, two + storages, administrator, completed tasks). +- [ ] Implement deterministic `medium`, configurable batch-insert `large`, + `ha-demo`, and `broken-storage` profiles. + +Exit: clean migration plus every seed profile has a stable logical snapshot; +large seeding proves bounded batch operations rather than row-at-a-time inserts. + +## G3 — authentication and authorization surface + +- [ ] Expose API-token lifecycle and authenticate + `PVEAPIToken=USER@REALM!TOKENID=SECRET` without CSRF. +- [ ] Complete pam, pve, and test realm behavior, ticket skew/expiry and + credential redaction. +- [ ] Wire users, groups, roles, ACL propagation, route-derived permissions and + token privilege separation into every semantic handler. +- [ ] Test root, audit-only, VM operator, storage user, separated token, + inheritance, denial, and existence-hiding behavior. + +Exit: the complete credential/permission matrix passes through HTTP and no +plaintext password, ticket, CSRF token, or token secret reaches storage/logs. + +## G4 — QEMU 0.2 verticals + +- [ ] Create, update, delete, shutdown, reboot, reset, suspend and resume. +- [ ] Snapshots and rollback, clone, local/remote migration, resize and move + disk, selected agent endpoints, pending/status data. +- [ ] Persist normalized CPU/memory/common fields plus unknown PVE parameters in + JSONB; simulate usage, uptime, PID, IO/network, lock, template, QMP, HA and + guest-agent availability. +- [ ] Cover concurrent start/delete, migrate/snapshot, optimistic conflict, + idempotency and restart recovery. + +Exit: each operation is a complete contract/auth/permission/persistence/task/ +state/error/test vertical and release 0.2.0 has a generated limitation matrix. + +## G5 — LXC, storage, pools, backup and cluster 0.3 + +- [ ] LXC create/config/lifecycle/clone/migrate/snapshot/resize/delete. +- [ ] Storage list/status/content metadata, allocation/free, upload metadata, + ISO/template/backup listing and content deletion without large default blobs. +- [ ] Pools and membership, backup metadata/tasks, cluster status/nextid/options/ + tasks/replication, and initial HA model/status. + +Exit: release 0.3.0 passes client-level flows for every listed resource family. + +## G6 — simulator administration, faults and virtual time 0.4 + +- [ ] Protected, disableable `/_simulator` API for state, reset, scenarios, + faults, compatibility and manual clock advancement. +- [ ] Deterministic rules filtered by route, method, principal, node, VMID, call + count, probability, time interval and scenario. +- [ ] Implement the specified node/storage/task/permission/migration/snapshot/ + backup/HTTP/malformed/agent/lock/quorum/HA failures. +- [ ] Ensure simulation services use injected real, accelerated or manual clocks; + only lease internals use real monotonic time. + +Exit: seeded scenario tests reproduce the same failures and durations across +runs; admin endpoints cannot overlap or weaken the PVE API boundary. + +## G7 — safe recorder and differential laboratory 0.5 + +- [ ] Opt-in async passthrough/record fallbacks with official-lab allowlist, + production denylist, verified TLS, read-only default and explicit mutation + authorization. +- [ ] Sanitize credentials, cookies, tickets, CSRF, tokens, people, hosts and IPs + before fixtures can be persisted. +- [ ] Record request/response/latency/version/time/scenario metadata. +- [ ] Run identical lab/simulator requests using declarative normalization for + timestamps, UPIDs, PIDs, tokens, node values, uptime and resource usage. +- [ ] Produce JSON/Markdown/HTML reports with compatibility classes, regressions, + groups and versions. + +Exit: secret-scanning fixtures and offline replay tests pass; normal startup has +no dependency on or route to a real Proxmox. + +## G8 — profiles, observability and operational packaging + +- [ ] Central `CompatibilityCapabilities` profiles for pve-8.4, pve-9.0, + pve-9.2 and custom; no scattered version-prefix conditions. +- [ ] Prometheus metrics requested by the brief, with bounded-cardinality labels, + plus optional OpenTelemetry tracing and optional Compose Prometheus/Grafana. +- [ ] Enrich safe structured logs with route template, principal, task/resource + context and stable error code. +- [ ] Add Docker labels/SBOM-friendly metadata and verify read-only/non-root + runtime, signals, one uvicorn process and multi-replica task leasing. +- [ ] Add Helm/Kubernetes Deployment, Service, ConfigMap, Secret, probes, PDB, + NetworkPolicy, hardened security context, resources, topology spread, separate + migration/seed Jobs and external PostgreSQL production configuration. + +Exit: observability tests reject high-cardinality labels; chart lint/render and +multi-replica acceptance pass. + +## G9 — client certification, security and 1.0 governance + +- [ ] Add contract coverage for every imported route and critical-module + coverage of at least 90% while maintaining project coverage at least 80%. +- [ ] Certify documented versions of proxmoxer, HTTPX, Terraform provider and + Ansible modules only for the surfaces their flows exercise. +- [ ] Publish stable compatibility/migration policy and run dependency, + container, recorder, secret-handling and threat-model review. +- [ ] Expand README with architecture, version/profile choice, every seed, + scenarios, reports, recorder security, Kubernetes, commands and roadmap. + +Exit: release 1.0.0 is reproducible from a clean checkout and all published +claims point to immutable machine-readable evidence. + +## Global gate for every G-stage + +Run formatting, Ruff, strict mypy, unit/integration/contract/compatibility tests, +coverage, `git diff --check`, relevant clean-container acceptance, documentation, +and a focused commit. No `pass`, TODO, `NotImplementedError`, sync network/DB IO, +`time.sleep`, plaintext secrets, silent error swallowing, or unsupported +compatibility claims may be introduced. diff --git a/evidence/pve-9.2.3-0.1.0.json b/evidence/pve-9.2.3-0.1.0.json new file mode 100644 index 0000000..1793bd7 --- /dev/null +++ b/evidence/pve-9.2.3-0.1.0.json @@ -0,0 +1,55 @@ +{ + "format_version": 1, + "profile": "pve-9.2", + "source_version": "9.2.3", + "records": [ + { + "path": "/version", + "verb": "GET", + "dimensions": ["http_status", "json_structure", "response_field_types", "response_required_fields"], + "sources": ["tests/compatibility/test_proxmoxer.py", "tests/unit/test_core_handlers.py"] + }, + { + "path": "/access/ticket", + "verb": "POST", + "dimensions": ["input_parameters", "parameter_requiredness", "types_constraints", "http_status", "json_structure", "response_field_types", "response_required_fields", "headers_cookies", "errors_prohibitions"], + "sources": ["tests/compatibility/test_proxmoxer.py", "tests/unit/test_auth.py", "tests/unit/test_dynamic_routes.py"] + }, + { + "path": "/nodes", + "verb": "GET", + "dimensions": ["http_status", "json_structure", "response_field_types", "response_required_fields", "permissions"], + "sources": ["tests/compatibility/test_proxmoxer.py", "tests/unit/test_core_handlers.py", "tests/unit/test_acl.py"] + }, + { + "path": "/nodes/{node}/qemu", + "verb": "GET", + "dimensions": ["input_parameters", "parameter_requiredness", "types_constraints", "http_status", "json_structure", "response_field_types", "response_required_fields", "permissions"], + "sources": ["tests/compatibility/test_proxmoxer.py", "tests/unit/test_core_handlers.py", "tests/unit/test_acl.py"] + }, + { + "path": "/nodes/{node}/qemu/{vmid}/status/current", + "verb": "GET", + "dimensions": ["input_parameters", "parameter_requiredness", "types_constraints", "http_status", "json_structure", "response_field_types", "response_required_fields", "permissions"], + "sources": ["tests/compatibility/test_proxmoxer.py", "tests/unit/test_core_handlers.py"] + }, + { + "path": "/nodes/{node}/qemu/{vmid}/status/start", + "verb": "POST", + "dimensions": ["input_parameters", "parameter_requiredness", "types_constraints", "http_status", "json_structure", "headers_cookies", "state_semantics", "long_task_behavior", "errors_prohibitions", "permissions"], + "sources": ["tests/compatibility/test_proxmoxer.py", "tests/unit/test_qemu_task.py", "tests/unit/test_transitions.py"] + }, + { + "path": "/nodes/{node}/qemu/{vmid}/status/stop", + "verb": "POST", + "dimensions": ["input_parameters", "parameter_requiredness", "types_constraints", "http_status", "json_structure", "headers_cookies", "state_semantics", "long_task_behavior", "errors_prohibitions", "permissions"], + "sources": ["tests/compatibility/test_proxmoxer.py", "tests/unit/test_qemu_task.py", "tests/unit/test_transitions.py"] + }, + { + "path": "/nodes/{node}/tasks/{upid}/status", + "verb": "GET", + "dimensions": ["input_parameters", "parameter_requiredness", "types_constraints", "http_status", "json_structure", "response_field_types", "response_required_fields", "long_task_behavior", "permissions"], + "sources": ["tests/compatibility/test_proxmoxer.py", "tests/unit/test_task_worker.py", "tests/unit/test_upid.py"] + } + ] +} diff --git a/tests/compatibility/test_proxmoxer.py b/tests/compatibility/test_proxmoxer.py index 6dbda37..20d49fd 100644 --- a/tests/compatibility/test_proxmoxer.py +++ b/tests/compatibility/test_proxmoxer.py @@ -1,7 +1,7 @@ """Unmodified proxmoxer HTTPS smoke flow.""" import os -import time +from threading import Event import pytest from proxmoxer import ProxmoxAPI # type: ignore[import-untyped] @@ -34,6 +34,6 @@ def test_proxmoxer_read_and_qemu_task_flow() -> None: task = proxmox.nodes("pve1").tasks(upid).status.get() if task["status"] == "stopped": break - time.sleep(0.05) + Event().wait(0.05) assert task["status"] == "stopped" assert task["exitstatus"] == "OK" diff --git a/tests/unit/test_compatibility.py b/tests/unit/test_compatibility.py index abf6150..337010c 100644 --- a/tests/unit/test_compatibility.py +++ b/tests/unit/test_compatibility.py @@ -1,10 +1,11 @@ """Compatibility accounting tests.""" from datetime import UTC, datetime +from typing import cast import pytest -from app.compatibility import build_report +from app.compatibility import CompatibilityDimension, EvidenceManifest, build_report from app.contracts.model import Method, PathContract, Schema, Snapshot @@ -54,3 +55,58 @@ def test_report_scores_levels_and_groups_independently() -> None: def test_report_rejects_unbound_evidence() -> None: with pytest.raises(ValueError, match="undeclared"): build_report(snapshot(), verified=frozenset({("/missing", "GET")})) + + +def test_all_thirteen_dimensions_have_independent_evidence_and_renderers() -> None: + method = frozenset({("/nodes/{node}", "GET")}) + report = build_report( + snapshot(), + implemented=method, + dimensions={dimension: method for dimension in CompatibilityDimension}, + ) + + payload = report.as_json() + dimensions = cast(dict[str, dict[str, object]], payload["dimensions"]) + assert list(dimensions) == [dimension.value for dimension in CompatibilityDimension] + assert len(dimensions) == 13 + assert all(item["count"] == 1 for item in dimensions.values()) + assert payload["dimension_groups"] + classifications = cast(dict[str, list[str]], payload["classifications"]) + assert classifications["fully_compatible"] == ["GET /nodes/{node}"] + assert not classifications["partially_compatible"] + assert "| permissions | 1 |" in report.as_markdown() + assert "long_task_behavior1" in report.as_html() + assert report.canonical_json() == report.canonical_json() + + +def test_dimension_evidence_must_reference_declared_method() -> None: + with pytest.raises(ValueError, match="permissions evidence"): + build_report( + snapshot(), + dimensions={CompatibilityDimension.PERMISSIONS: frozenset({("/missing", "GET")})}, + ) + + +def test_evidence_manifest_requires_provenance_and_unique_methods() -> None: + manifest = EvidenceManifest.model_validate( + { + "profile": "pve-9.2", + "source_version": "9.2.3", + "records": [ + { + "path": "/nodes/{node}", + "verb": "GET", + "dimensions": ["http_status", "json_structure"], + "sources": ["tests/compatibility/test_proxmoxer.py"], + } + ], + } + ) + evidence = manifest.dimension_map() + assert evidence[CompatibilityDimension.HTTP_STATUS] == frozenset({("/nodes/{node}", "GET")}) + assert not evidence[CompatibilityDimension.PERMISSIONS] + + duplicate = manifest.model_dump(mode="json") + duplicate["records"].append(duplicate["records"][0]) + with pytest.raises(ValueError, match="duplicate methods"): + EvidenceManifest.model_validate(duplicate)