diff --git a/Makefile b/Makefile index fc1dded..df11c1d 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ db-reset: ## Recreate the local database volume $(COMPOSE) up -d postgres api-import: ## Import an API snapshot - @echo "API import is scheduled for milestone B4" >&2; exit 2 + $(BIN)/proxmox-api-contract import $(ARGS) api-diff: ## Compare API snapshots @echo "API diff is scheduled for milestone B5" >&2; exit 2 @@ -82,4 +82,3 @@ clean: ## Remove generated local artifacts rm -rf $(VENV) .coverage coverage.xml htmlcov .mypy_cache .pytest_cache .ruff_cache ci: format lint typecheck coverage ## Run the complete local quality gate - diff --git a/README.md b/README.md index 4524a33..2582f9e 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,20 @@ curl http://localhost:8006/health/live curl http://localhost:8006/health/ready ``` +Contract artifacts can be validated and imported into immutable local revisions: + +```bash +.venv/bin/proxmox-api-contract validate tests/fixtures/api-viewer/pve-9.2.3-version.json +.venv/bin/proxmox-api-contract --store contracts import \ + --file tests/fixtures/api-viewer/pve-9.2.3-version.json --version 9.2.3 +.venv/bin/proxmox-api-contract --store contracts list +``` + +Remote imports accept HTTPS URLs on the explicit official-domain allowlist and +reject private address resolution, unsafe redirects, oversized responses, and +unbounded retries. Imported revisions are addressed by their normalized +snapshot checksum and are never overwritten. + See [the architecture](docs/architecture.md) for component boundaries and durability decisions. Commands for not-yet-implemented milestones intentionally return a non-zero status instead of pretending to succeed. - diff --git a/app/contracts/cli.py b/app/contracts/cli.py new file mode 100644 index 0000000..375488f --- /dev/null +++ b/app/contracts/cli.py @@ -0,0 +1,62 @@ +"""Command-line interface for contract imports and inspection.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +from datetime import UTC, datetime +from pathlib import Path + +from app.contracts.importer import RemoteSourceImporter +from app.contracts.normalize import normalize_snapshot +from app.contracts.source import ApiViewerParser, LocalFileImporter, SourceImporter +from app.contracts.store import RevisionStore + + +def parser() -> argparse.ArgumentParser: + root = argparse.ArgumentParser(prog="proxmox-api-contract") + root.add_argument("--store", type=Path, default=Path("contracts")) + commands = root.add_subparsers(dest="command", required=True) + import_command = commands.add_parser("import") + source = import_command.add_mutually_exclusive_group(required=True) + source.add_argument("--file", type=Path) + source.add_argument("--url") + import_command.add_argument("--version", required=True) + validate = commands.add_parser("validate") + validate.add_argument("file", type=Path) + commands.add_parser("list") + show = commands.add_parser("show") + show.add_argument("revision") + return root + + +async def run(arguments: argparse.Namespace) -> int: + store = RevisionStore(arguments.store) + if arguments.command == "list": + for revision in store.list(): + print(revision) + return 0 + if arguments.command == "show": + print(json.dumps(store.manifest(arguments.revision).model_dump(mode="json"), indent=2)) + return 0 + if arguments.command == "validate": + parsed = ApiViewerParser().parse(arguments.file.read_bytes()) + print(json.dumps({"nodes": len(parsed.nodes), "warnings": len(parsed.warnings)})) + return 0 + importer: SourceImporter + if arguments.file is not None: + importer = LocalFileImporter(arguments.file) + else: + importer = RemoteSourceImporter(arguments.url) + raw = await importer.load() + parsed = ApiViewerParser().parse(raw) + snapshot, manifest = normalize_snapshot( + parsed, raw=raw, source_version=arguments.version, retrieved_at=datetime.now(UTC) + ) + print(store.save(raw, snapshot, manifest)) + return 0 + + +def main() -> None: + raise SystemExit(asyncio.run(run(parser().parse_args()))) diff --git a/app/contracts/importer.py b/app/contracts/importer.py new file mode 100644 index 0000000..5622802 --- /dev/null +++ b/app/contracts/importer.py @@ -0,0 +1,95 @@ +"""Network-constrained remote contract retrieval.""" + +from __future__ import annotations + +import asyncio +import ipaddress +import socket +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from urllib.parse import urljoin, urlsplit + +import httpx + +from app.contracts.source import SourceError + +Resolver = Callable[[str], Awaitable[tuple[str, ...]]] + + +async def resolve_host(host: str) -> tuple[str, ...]: + loop = asyncio.get_running_loop() + results = await loop.getaddrinfo(host, 443, type=socket.SOCK_STREAM) + return tuple(sorted({str(result[4][0]) for result in results})) + + +def validate_remote_url(url: str, allowed_hosts: frozenset[str]) -> str: + parsed = urlsplit(url) + if parsed.scheme != "https": + raise SourceError("remote imports require HTTPS") + if parsed.username or parsed.password or parsed.port not in (None, 443): + raise SourceError("remote URL contains forbidden authority components") + host = (parsed.hostname or "").rstrip(".").lower() + if host not in allowed_hosts: + raise SourceError("remote host is not in the official-domain allowlist") + if parsed.fragment: + raise SourceError("remote URL fragments are not allowed") + return host + + +def validate_public_addresses(addresses: tuple[str, ...]) -> None: + if not addresses: + raise SourceError("remote host did not resolve") + for value in addresses: + address = ipaddress.ip_address(value) + if not address.is_global: + raise SourceError(f"remote host resolved to a non-public address: {value}") + + +@dataclass(frozen=True, slots=True) +class RemoteSourceImporter: + url: str + allowed_hosts: frozenset[str] = frozenset({"pve.proxmox.com"}) + max_bytes: int = 16 * 1024 * 1024 + max_redirects: int = 3 + retries: int = 2 + timeout_seconds: float = 20.0 + resolver: Resolver = resolve_host + transport: httpx.AsyncBaseTransport | None = None + + async def load(self) -> bytes: + current = self.url + timeout = httpx.Timeout(self.timeout_seconds) + async with httpx.AsyncClient( + follow_redirects=False, timeout=timeout, transport=self.transport + ) as client: + for redirect_count in range(self.max_redirects + 1): + host = validate_remote_url(current, self.allowed_hosts) + validate_public_addresses(await self.resolver(host)) + response = await self._request(client, current) + if response.is_redirect: + if redirect_count == self.max_redirects: + raise SourceError("remote import exceeded redirect limit") + location = response.headers.get("location") + if not location: + raise SourceError("remote redirect has no location") + current = urljoin(current, location) + continue + response.raise_for_status() + content_length = response.headers.get("content-length") + if content_length and int(content_length) > self.max_bytes: + raise SourceError("remote artifact exceeds size limit") + content = response.content + if len(content) > self.max_bytes: + raise SourceError("remote artifact exceeds size limit") + return content + raise SourceError("remote import failed") + + async def _request(self, client: httpx.AsyncClient, url: str) -> httpx.Response: + for attempt in range(self.retries + 1): + try: + return await client.get(url) + except (httpx.TimeoutException, httpx.NetworkError): + if attempt == self.retries: + raise + await asyncio.sleep(0.1 * (2**attempt)) + raise SourceError("remote import retry loop exhausted") diff --git a/app/contracts/store.py b/app/contracts/store.py new file mode 100644 index 0000000..a1bd4d0 --- /dev/null +++ b/app/contracts/store.py @@ -0,0 +1,48 @@ +"""Immutable filesystem storage for imported contract revisions.""" + +from __future__ import annotations + +import os +import tempfile +from dataclasses import dataclass +from pathlib import Path + +from app.contracts.model import Manifest, Snapshot, canonical_json + + +@dataclass(frozen=True, slots=True) +class RevisionStore: + root: Path + + def save(self, raw: bytes, snapshot: Snapshot, manifest: Manifest) -> Path: + revision = self.root / manifest.snapshot_sha256 + if revision.exists(): + return revision + self.root.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix=".import-", dir=self.root)) + try: + self._write(temporary / "raw.js", raw) + self._write(temporary / "snapshot.json", snapshot.canonical_bytes()) + self._write(temporary / "manifest.json", canonical_json(manifest)) + os.replace(temporary, revision) + except BaseException: + for child in temporary.iterdir(): + child.unlink() + temporary.rmdir() + raise + return revision + + def list(self) -> tuple[str, ...]: + if not self.root.exists(): + return () + return tuple(sorted(path.name for path in self.root.iterdir() if path.is_dir())) + + def manifest(self, revision: str) -> Manifest: + return Manifest.model_validate_json((self.root / revision / "manifest.json").read_bytes()) + + @staticmethod + def _write(path: Path, content: bytes) -> None: + with path.open("xb") as stream: + stream.write(content) + stream.flush() + os.fsync(stream.fileno()) diff --git a/pyproject.toml b/pyproject.toml index b8b5887..3953099 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ authors = [{ name = "proxmox-api-simulator contributors" }] dependencies = [ "asyncpg>=0.30,<0.31", "fastapi>=0.116,<0.117", + "httpx>=0.28,<0.29", "pydantic>=2.11,<3", "pydantic-settings>=2.10,<3", "uvicorn[standard]>=0.35,<0.36", @@ -21,7 +22,6 @@ dependencies = [ [project.optional-dependencies] dev = [ "hypothesis>=6.135,<7", - "httpx>=0.28,<0.29", "mypy>=1.17,<1.18", "pytest>=8.4,<9", "pytest-asyncio>=1.1,<2", @@ -29,6 +29,9 @@ dev = [ "ruff>=0.12,<0.13", ] +[project.scripts] +proxmox-api-contract = "app.contracts.cli:main" + [tool.hatch.build.targets.wheel] packages = ["app"] diff --git a/tests/unit/test_contract_cli.py b/tests/unit/test_contract_cli.py new file mode 100644 index 0000000..9d57fee --- /dev/null +++ b/tests/unit/test_contract_cli.py @@ -0,0 +1,52 @@ +"""Offline command workflows for contract management.""" + +import argparse +import json +from pathlib import Path + +import pytest + +from app.contracts.cli import parser, run + +FIXTURE = Path(__file__).parents[1] / "fixtures" / "api-viewer" / "pve-9.2.3-version.json" + + +async def test_validate_command_reports_source_counts( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + arguments = argparse.Namespace(command="validate", store=tmp_path, file=FIXTURE) + + assert await run(arguments) == 0 + output = capsys.readouterr().out + assert json.loads(output) == {"nodes": 1, "warnings": 0} + + +async def test_local_import_list_and_show( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + import_arguments = argparse.Namespace( + command="import", + store=tmp_path, + file=FIXTURE, + url=None, + version="9.2.3", + ) + assert await run(import_arguments) == 0 + revision = Path(capsys.readouterr().out.strip()).name + + assert await run(argparse.Namespace(command="list", store=tmp_path)) == 0 + assert capsys.readouterr().out.strip() == revision + + assert await run(argparse.Namespace(command="show", store=tmp_path, revision=revision)) == 0 + manifest = json.loads(capsys.readouterr().out) + assert manifest["source_version"] == "9.2.3" + assert manifest["snapshot_sha256"] == revision + + +def test_cli_parser_accepts_local_import() -> None: + arguments = parser().parse_args( + ["--store", "saved", "import", "--file", str(FIXTURE), "--version", "9.2.3"] + ) + + assert arguments.command == "import" + assert arguments.store == Path("saved") diff --git a/tests/unit/test_contract_importer.py b/tests/unit/test_contract_importer.py new file mode 100644 index 0000000..715a98b --- /dev/null +++ b/tests/unit/test_contract_importer.py @@ -0,0 +1,90 @@ +"""Security and idempotency tests for contract imports.""" + +from datetime import UTC, datetime +from pathlib import Path + +import httpx +import pytest + +from app.contracts.importer import RemoteSourceImporter, validate_remote_url +from app.contracts.normalize import normalize_snapshot +from app.contracts.source import ApiViewerParser, SourceError +from app.contracts.store import RevisionStore + + +async def public_resolver(_host: str) -> tuple[str, ...]: + return ("93.184.216.34",) + + +@pytest.mark.parametrize( + "url", + [ + "http://pve.proxmox.com/apidoc.js", + "https://evil.example/apidoc.js", + "https://pve.proxmox.com.evil.example/apidoc.js", + "https://user@pve.proxmox.com/apidoc.js", + "https://pve.proxmox.com:444/apidoc.js", + ], +) +def test_remote_url_policy_rejects_unsafe_urls(url: str) -> None: + with pytest.raises(SourceError): + validate_remote_url(url, frozenset({"pve.proxmox.com"})) + + +async def test_remote_import_rejects_private_resolution() -> None: + async def private_resolver(_host: str) -> tuple[str, ...]: + return ("127.0.0.1",) + + importer = RemoteSourceImporter( + "https://pve.proxmox.com/apidoc.js", + resolver=private_resolver, + transport=httpx.MockTransport(lambda request: httpx.Response(200, content=b"[]")), + ) + + with pytest.raises(SourceError, match="non-public"): + await importer.load() + + +async def test_redirect_is_revalidated() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(302, headers={"location": "https://evil.example/private"}) + + importer = RemoteSourceImporter( + "https://pve.proxmox.com/apidoc.js", + resolver=public_resolver, + transport=httpx.MockTransport(handler), + ) + + with pytest.raises(SourceError, match="allowlist"): + await importer.load() + + +async def test_remote_import_enforces_size_limit() -> None: + importer = RemoteSourceImporter( + "https://pve.proxmox.com/apidoc.js", + max_bytes=2, + resolver=public_resolver, + transport=httpx.MockTransport(lambda request: httpx.Response(200, content=b"[]\n")), + ) + + with pytest.raises(SourceError, match="size"): + await importer.load() + + +def test_revision_store_is_idempotent(tmp_path: Path) -> None: + raw = b'[{"path":"/version","info":{}}]' + parsed = ApiViewerParser().parse(raw) + snapshot, manifest = normalize_snapshot( + parsed, + raw=raw, + source_version="test", + retrieved_at=datetime(2026, 1, 1, tzinfo=UTC), + ) + store = RevisionStore(tmp_path) + + first = store.save(raw, snapshot, manifest) + second = store.save(raw, snapshot, manifest) + + assert first == second + assert store.list() == (manifest.snapshot_sha256,) + assert store.manifest(manifest.snapshot_sha256) == manifest