feat: add secure contract import CLI
This commit is contained in:
@@ -70,7 +70,7 @@ db-reset: ## Recreate the local database volume
|
||||
$(COMPOSE) up -d postgres
|
||||
|
||||
api-import: ## Import an API snapshot
|
||||
@echo "API import is scheduled for milestone B4" >&2; exit 2
|
||||
$(BIN)/proxmox-api-contract import $(ARGS)
|
||||
|
||||
api-diff: ## Compare API snapshots
|
||||
@echo "API diff is scheduled for milestone B5" >&2; exit 2
|
||||
@@ -82,4 +82,3 @@ clean: ## Remove generated local artifacts
|
||||
rm -rf $(VENV) .coverage coverage.xml htmlcov .mypy_cache .pytest_cache .ruff_cache
|
||||
|
||||
ci: format lint typecheck coverage ## Run the complete local quality gate
|
||||
|
||||
|
||||
@@ -26,7 +26,20 @@ curl http://localhost:8006/health/live
|
||||
curl http://localhost:8006/health/ready
|
||||
```
|
||||
|
||||
Contract artifacts can be validated and imported into immutable local revisions:
|
||||
|
||||
```bash
|
||||
.venv/bin/proxmox-api-contract validate tests/fixtures/api-viewer/pve-9.2.3-version.json
|
||||
.venv/bin/proxmox-api-contract --store contracts import \
|
||||
--file tests/fixtures/api-viewer/pve-9.2.3-version.json --version 9.2.3
|
||||
.venv/bin/proxmox-api-contract --store contracts list
|
||||
```
|
||||
|
||||
Remote imports accept HTTPS URLs on the explicit official-domain allowlist and
|
||||
reject private address resolution, unsafe redirects, oversized responses, and
|
||||
unbounded retries. Imported revisions are addressed by their normalized
|
||||
snapshot checksum and are never overwritten.
|
||||
|
||||
See [the architecture](docs/architecture.md) for component boundaries and
|
||||
durability decisions. Commands for not-yet-implemented milestones intentionally
|
||||
return a non-zero status instead of pretending to succeed.
|
||||
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Command-line interface for contract imports and inspection."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from app.contracts.importer import RemoteSourceImporter
|
||||
from app.contracts.normalize import normalize_snapshot
|
||||
from app.contracts.source import ApiViewerParser, LocalFileImporter, SourceImporter
|
||||
from app.contracts.store import RevisionStore
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
root = argparse.ArgumentParser(prog="proxmox-api-contract")
|
||||
root.add_argument("--store", type=Path, default=Path("contracts"))
|
||||
commands = root.add_subparsers(dest="command", required=True)
|
||||
import_command = commands.add_parser("import")
|
||||
source = import_command.add_mutually_exclusive_group(required=True)
|
||||
source.add_argument("--file", type=Path)
|
||||
source.add_argument("--url")
|
||||
import_command.add_argument("--version", required=True)
|
||||
validate = commands.add_parser("validate")
|
||||
validate.add_argument("file", type=Path)
|
||||
commands.add_parser("list")
|
||||
show = commands.add_parser("show")
|
||||
show.add_argument("revision")
|
||||
return root
|
||||
|
||||
|
||||
async def run(arguments: argparse.Namespace) -> int:
|
||||
store = RevisionStore(arguments.store)
|
||||
if arguments.command == "list":
|
||||
for revision in store.list():
|
||||
print(revision)
|
||||
return 0
|
||||
if arguments.command == "show":
|
||||
print(json.dumps(store.manifest(arguments.revision).model_dump(mode="json"), indent=2))
|
||||
return 0
|
||||
if arguments.command == "validate":
|
||||
parsed = ApiViewerParser().parse(arguments.file.read_bytes())
|
||||
print(json.dumps({"nodes": len(parsed.nodes), "warnings": len(parsed.warnings)}))
|
||||
return 0
|
||||
importer: SourceImporter
|
||||
if arguments.file is not None:
|
||||
importer = LocalFileImporter(arguments.file)
|
||||
else:
|
||||
importer = RemoteSourceImporter(arguments.url)
|
||||
raw = await importer.load()
|
||||
parsed = ApiViewerParser().parse(raw)
|
||||
snapshot, manifest = normalize_snapshot(
|
||||
parsed, raw=raw, source_version=arguments.version, retrieved_at=datetime.now(UTC)
|
||||
)
|
||||
print(store.save(raw, snapshot, manifest))
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> None:
|
||||
raise SystemExit(asyncio.run(run(parser().parse_args())))
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Network-constrained remote contract retrieval."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import socket
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
import httpx
|
||||
|
||||
from app.contracts.source import SourceError
|
||||
|
||||
Resolver = Callable[[str], Awaitable[tuple[str, ...]]]
|
||||
|
||||
|
||||
async def resolve_host(host: str) -> tuple[str, ...]:
|
||||
loop = asyncio.get_running_loop()
|
||||
results = await loop.getaddrinfo(host, 443, type=socket.SOCK_STREAM)
|
||||
return tuple(sorted({str(result[4][0]) for result in results}))
|
||||
|
||||
|
||||
def validate_remote_url(url: str, allowed_hosts: frozenset[str]) -> str:
|
||||
parsed = urlsplit(url)
|
||||
if parsed.scheme != "https":
|
||||
raise SourceError("remote imports require HTTPS")
|
||||
if parsed.username or parsed.password or parsed.port not in (None, 443):
|
||||
raise SourceError("remote URL contains forbidden authority components")
|
||||
host = (parsed.hostname or "").rstrip(".").lower()
|
||||
if host not in allowed_hosts:
|
||||
raise SourceError("remote host is not in the official-domain allowlist")
|
||||
if parsed.fragment:
|
||||
raise SourceError("remote URL fragments are not allowed")
|
||||
return host
|
||||
|
||||
|
||||
def validate_public_addresses(addresses: tuple[str, ...]) -> None:
|
||||
if not addresses:
|
||||
raise SourceError("remote host did not resolve")
|
||||
for value in addresses:
|
||||
address = ipaddress.ip_address(value)
|
||||
if not address.is_global:
|
||||
raise SourceError(f"remote host resolved to a non-public address: {value}")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RemoteSourceImporter:
|
||||
url: str
|
||||
allowed_hosts: frozenset[str] = frozenset({"pve.proxmox.com"})
|
||||
max_bytes: int = 16 * 1024 * 1024
|
||||
max_redirects: int = 3
|
||||
retries: int = 2
|
||||
timeout_seconds: float = 20.0
|
||||
resolver: Resolver = resolve_host
|
||||
transport: httpx.AsyncBaseTransport | None = None
|
||||
|
||||
async def load(self) -> bytes:
|
||||
current = self.url
|
||||
timeout = httpx.Timeout(self.timeout_seconds)
|
||||
async with httpx.AsyncClient(
|
||||
follow_redirects=False, timeout=timeout, transport=self.transport
|
||||
) as client:
|
||||
for redirect_count in range(self.max_redirects + 1):
|
||||
host = validate_remote_url(current, self.allowed_hosts)
|
||||
validate_public_addresses(await self.resolver(host))
|
||||
response = await self._request(client, current)
|
||||
if response.is_redirect:
|
||||
if redirect_count == self.max_redirects:
|
||||
raise SourceError("remote import exceeded redirect limit")
|
||||
location = response.headers.get("location")
|
||||
if not location:
|
||||
raise SourceError("remote redirect has no location")
|
||||
current = urljoin(current, location)
|
||||
continue
|
||||
response.raise_for_status()
|
||||
content_length = response.headers.get("content-length")
|
||||
if content_length and int(content_length) > self.max_bytes:
|
||||
raise SourceError("remote artifact exceeds size limit")
|
||||
content = response.content
|
||||
if len(content) > self.max_bytes:
|
||||
raise SourceError("remote artifact exceeds size limit")
|
||||
return content
|
||||
raise SourceError("remote import failed")
|
||||
|
||||
async def _request(self, client: httpx.AsyncClient, url: str) -> httpx.Response:
|
||||
for attempt in range(self.retries + 1):
|
||||
try:
|
||||
return await client.get(url)
|
||||
except (httpx.TimeoutException, httpx.NetworkError):
|
||||
if attempt == self.retries:
|
||||
raise
|
||||
await asyncio.sleep(0.1 * (2**attempt))
|
||||
raise SourceError("remote import retry loop exhausted")
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Immutable filesystem storage for imported contract revisions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from app.contracts.model import Manifest, Snapshot, canonical_json
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RevisionStore:
|
||||
root: Path
|
||||
|
||||
def save(self, raw: bytes, snapshot: Snapshot, manifest: Manifest) -> Path:
|
||||
revision = self.root / manifest.snapshot_sha256
|
||||
if revision.exists():
|
||||
return revision
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
temporary = Path(tempfile.mkdtemp(prefix=".import-", dir=self.root))
|
||||
try:
|
||||
self._write(temporary / "raw.js", raw)
|
||||
self._write(temporary / "snapshot.json", snapshot.canonical_bytes())
|
||||
self._write(temporary / "manifest.json", canonical_json(manifest))
|
||||
os.replace(temporary, revision)
|
||||
except BaseException:
|
||||
for child in temporary.iterdir():
|
||||
child.unlink()
|
||||
temporary.rmdir()
|
||||
raise
|
||||
return revision
|
||||
|
||||
def list(self) -> tuple[str, ...]:
|
||||
if not self.root.exists():
|
||||
return ()
|
||||
return tuple(sorted(path.name for path in self.root.iterdir() if path.is_dir()))
|
||||
|
||||
def manifest(self, revision: str) -> Manifest:
|
||||
return Manifest.model_validate_json((self.root / revision / "manifest.json").read_bytes())
|
||||
|
||||
@staticmethod
|
||||
def _write(path: Path, content: bytes) -> None:
|
||||
with path.open("xb") as stream:
|
||||
stream.write(content)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
+4
-1
@@ -13,6 +13,7 @@ authors = [{ name = "proxmox-api-simulator contributors" }]
|
||||
dependencies = [
|
||||
"asyncpg>=0.30,<0.31",
|
||||
"fastapi>=0.116,<0.117",
|
||||
"httpx>=0.28,<0.29",
|
||||
"pydantic>=2.11,<3",
|
||||
"pydantic-settings>=2.10,<3",
|
||||
"uvicorn[standard]>=0.35,<0.36",
|
||||
@@ -21,7 +22,6 @@ dependencies = [
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"hypothesis>=6.135,<7",
|
||||
"httpx>=0.28,<0.29",
|
||||
"mypy>=1.17,<1.18",
|
||||
"pytest>=8.4,<9",
|
||||
"pytest-asyncio>=1.1,<2",
|
||||
@@ -29,6 +29,9 @@ dev = [
|
||||
"ruff>=0.12,<0.13",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
proxmox-api-contract = "app.contracts.cli:main"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["app"]
|
||||
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Offline command workflows for contract management."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.contracts.cli import parser, run
|
||||
|
||||
FIXTURE = Path(__file__).parents[1] / "fixtures" / "api-viewer" / "pve-9.2.3-version.json"
|
||||
|
||||
|
||||
async def test_validate_command_reports_source_counts(
|
||||
tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
arguments = argparse.Namespace(command="validate", store=tmp_path, file=FIXTURE)
|
||||
|
||||
assert await run(arguments) == 0
|
||||
output = capsys.readouterr().out
|
||||
assert json.loads(output) == {"nodes": 1, "warnings": 0}
|
||||
|
||||
|
||||
async def test_local_import_list_and_show(
|
||||
tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
import_arguments = argparse.Namespace(
|
||||
command="import",
|
||||
store=tmp_path,
|
||||
file=FIXTURE,
|
||||
url=None,
|
||||
version="9.2.3",
|
||||
)
|
||||
assert await run(import_arguments) == 0
|
||||
revision = Path(capsys.readouterr().out.strip()).name
|
||||
|
||||
assert await run(argparse.Namespace(command="list", store=tmp_path)) == 0
|
||||
assert capsys.readouterr().out.strip() == revision
|
||||
|
||||
assert await run(argparse.Namespace(command="show", store=tmp_path, revision=revision)) == 0
|
||||
manifest = json.loads(capsys.readouterr().out)
|
||||
assert manifest["source_version"] == "9.2.3"
|
||||
assert manifest["snapshot_sha256"] == revision
|
||||
|
||||
|
||||
def test_cli_parser_accepts_local_import() -> None:
|
||||
arguments = parser().parse_args(
|
||||
["--store", "saved", "import", "--file", str(FIXTURE), "--version", "9.2.3"]
|
||||
)
|
||||
|
||||
assert arguments.command == "import"
|
||||
assert arguments.store == Path("saved")
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Security and idempotency tests for contract imports."""
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.contracts.importer import RemoteSourceImporter, validate_remote_url
|
||||
from app.contracts.normalize import normalize_snapshot
|
||||
from app.contracts.source import ApiViewerParser, SourceError
|
||||
from app.contracts.store import RevisionStore
|
||||
|
||||
|
||||
async def public_resolver(_host: str) -> tuple[str, ...]:
|
||||
return ("93.184.216.34",)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url",
|
||||
[
|
||||
"http://pve.proxmox.com/apidoc.js",
|
||||
"https://evil.example/apidoc.js",
|
||||
"https://pve.proxmox.com.evil.example/apidoc.js",
|
||||
"https://user@pve.proxmox.com/apidoc.js",
|
||||
"https://pve.proxmox.com:444/apidoc.js",
|
||||
],
|
||||
)
|
||||
def test_remote_url_policy_rejects_unsafe_urls(url: str) -> None:
|
||||
with pytest.raises(SourceError):
|
||||
validate_remote_url(url, frozenset({"pve.proxmox.com"}))
|
||||
|
||||
|
||||
async def test_remote_import_rejects_private_resolution() -> None:
|
||||
async def private_resolver(_host: str) -> tuple[str, ...]:
|
||||
return ("127.0.0.1",)
|
||||
|
||||
importer = RemoteSourceImporter(
|
||||
"https://pve.proxmox.com/apidoc.js",
|
||||
resolver=private_resolver,
|
||||
transport=httpx.MockTransport(lambda request: httpx.Response(200, content=b"[]")),
|
||||
)
|
||||
|
||||
with pytest.raises(SourceError, match="non-public"):
|
||||
await importer.load()
|
||||
|
||||
|
||||
async def test_redirect_is_revalidated() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(302, headers={"location": "https://evil.example/private"})
|
||||
|
||||
importer = RemoteSourceImporter(
|
||||
"https://pve.proxmox.com/apidoc.js",
|
||||
resolver=public_resolver,
|
||||
transport=httpx.MockTransport(handler),
|
||||
)
|
||||
|
||||
with pytest.raises(SourceError, match="allowlist"):
|
||||
await importer.load()
|
||||
|
||||
|
||||
async def test_remote_import_enforces_size_limit() -> None:
|
||||
importer = RemoteSourceImporter(
|
||||
"https://pve.proxmox.com/apidoc.js",
|
||||
max_bytes=2,
|
||||
resolver=public_resolver,
|
||||
transport=httpx.MockTransport(lambda request: httpx.Response(200, content=b"[]\n")),
|
||||
)
|
||||
|
||||
with pytest.raises(SourceError, match="size"):
|
||||
await importer.load()
|
||||
|
||||
|
||||
def test_revision_store_is_idempotent(tmp_path: Path) -> None:
|
||||
raw = b'[{"path":"/version","info":{}}]'
|
||||
parsed = ApiViewerParser().parse(raw)
|
||||
snapshot, manifest = normalize_snapshot(
|
||||
parsed,
|
||||
raw=raw,
|
||||
source_version="test",
|
||||
retrieved_at=datetime(2026, 1, 1, tzinfo=UTC),
|
||||
)
|
||||
store = RevisionStore(tmp_path)
|
||||
|
||||
first = store.save(raw, snapshot, manifest)
|
||||
second = store.save(raw, snapshot, manifest)
|
||||
|
||||
assert first == second
|
||||
assert store.list() == (manifest.snapshot_sha256,)
|
||||
assert store.manifest(manifest.snapshot_sha256) == manifest
|
||||
Reference in New Issue
Block a user