feat: add secure contract import CLI
This commit is contained in:
@@ -70,7 +70,7 @@ db-reset: ## Recreate the local database volume
|
|||||||
$(COMPOSE) up -d postgres
|
$(COMPOSE) up -d postgres
|
||||||
|
|
||||||
api-import: ## Import an API snapshot
|
api-import: ## Import an API snapshot
|
||||||
@echo "API import is scheduled for milestone B4" >&2; exit 2
|
$(BIN)/proxmox-api-contract import $(ARGS)
|
||||||
|
|
||||||
api-diff: ## Compare API snapshots
|
api-diff: ## Compare API snapshots
|
||||||
@echo "API diff is scheduled for milestone B5" >&2; exit 2
|
@echo "API diff is scheduled for milestone B5" >&2; exit 2
|
||||||
@@ -82,4 +82,3 @@ clean: ## Remove generated local artifacts
|
|||||||
rm -rf $(VENV) .coverage coverage.xml htmlcov .mypy_cache .pytest_cache .ruff_cache
|
rm -rf $(VENV) .coverage coverage.xml htmlcov .mypy_cache .pytest_cache .ruff_cache
|
||||||
|
|
||||||
ci: format lint typecheck coverage ## Run the complete local quality gate
|
ci: format lint typecheck coverage ## Run the complete local quality gate
|
||||||
|
|
||||||
|
|||||||
@@ -26,7 +26,20 @@ curl http://localhost:8006/health/live
|
|||||||
curl http://localhost:8006/health/ready
|
curl http://localhost:8006/health/ready
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Contract artifacts can be validated and imported into immutable local revisions:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
.venv/bin/proxmox-api-contract validate tests/fixtures/api-viewer/pve-9.2.3-version.json
|
||||||
|
.venv/bin/proxmox-api-contract --store contracts import \
|
||||||
|
--file tests/fixtures/api-viewer/pve-9.2.3-version.json --version 9.2.3
|
||||||
|
.venv/bin/proxmox-api-contract --store contracts list
|
||||||
|
```
|
||||||
|
|
||||||
|
Remote imports accept HTTPS URLs on the explicit official-domain allowlist and
|
||||||
|
reject private address resolution, unsafe redirects, oversized responses, and
|
||||||
|
unbounded retries. Imported revisions are addressed by their normalized
|
||||||
|
snapshot checksum and are never overwritten.
|
||||||
|
|
||||||
See [the architecture](docs/architecture.md) for component boundaries and
|
See [the architecture](docs/architecture.md) for component boundaries and
|
||||||
durability decisions. Commands for not-yet-implemented milestones intentionally
|
durability decisions. Commands for not-yet-implemented milestones intentionally
|
||||||
return a non-zero status instead of pretending to succeed.
|
return a non-zero status instead of pretending to succeed.
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,62 @@
|
|||||||
|
"""Command-line interface for contract imports and inspection."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app.contracts.importer import RemoteSourceImporter
|
||||||
|
from app.contracts.normalize import normalize_snapshot
|
||||||
|
from app.contracts.source import ApiViewerParser, LocalFileImporter, SourceImporter
|
||||||
|
from app.contracts.store import RevisionStore
|
||||||
|
|
||||||
|
|
||||||
|
def parser() -> argparse.ArgumentParser:
|
||||||
|
root = argparse.ArgumentParser(prog="proxmox-api-contract")
|
||||||
|
root.add_argument("--store", type=Path, default=Path("contracts"))
|
||||||
|
commands = root.add_subparsers(dest="command", required=True)
|
||||||
|
import_command = commands.add_parser("import")
|
||||||
|
source = import_command.add_mutually_exclusive_group(required=True)
|
||||||
|
source.add_argument("--file", type=Path)
|
||||||
|
source.add_argument("--url")
|
||||||
|
import_command.add_argument("--version", required=True)
|
||||||
|
validate = commands.add_parser("validate")
|
||||||
|
validate.add_argument("file", type=Path)
|
||||||
|
commands.add_parser("list")
|
||||||
|
show = commands.add_parser("show")
|
||||||
|
show.add_argument("revision")
|
||||||
|
return root
|
||||||
|
|
||||||
|
|
||||||
|
async def run(arguments: argparse.Namespace) -> int:
|
||||||
|
store = RevisionStore(arguments.store)
|
||||||
|
if arguments.command == "list":
|
||||||
|
for revision in store.list():
|
||||||
|
print(revision)
|
||||||
|
return 0
|
||||||
|
if arguments.command == "show":
|
||||||
|
print(json.dumps(store.manifest(arguments.revision).model_dump(mode="json"), indent=2))
|
||||||
|
return 0
|
||||||
|
if arguments.command == "validate":
|
||||||
|
parsed = ApiViewerParser().parse(arguments.file.read_bytes())
|
||||||
|
print(json.dumps({"nodes": len(parsed.nodes), "warnings": len(parsed.warnings)}))
|
||||||
|
return 0
|
||||||
|
importer: SourceImporter
|
||||||
|
if arguments.file is not None:
|
||||||
|
importer = LocalFileImporter(arguments.file)
|
||||||
|
else:
|
||||||
|
importer = RemoteSourceImporter(arguments.url)
|
||||||
|
raw = await importer.load()
|
||||||
|
parsed = ApiViewerParser().parse(raw)
|
||||||
|
snapshot, manifest = normalize_snapshot(
|
||||||
|
parsed, raw=raw, source_version=arguments.version, retrieved_at=datetime.now(UTC)
|
||||||
|
)
|
||||||
|
print(store.save(raw, snapshot, manifest))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
raise SystemExit(asyncio.run(run(parser().parse_args())))
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
"""Network-constrained remote contract retrieval."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import ipaddress
|
||||||
|
import socket
|
||||||
|
from collections.abc import Awaitable, Callable
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from urllib.parse import urljoin, urlsplit
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from app.contracts.source import SourceError
|
||||||
|
|
||||||
|
Resolver = Callable[[str], Awaitable[tuple[str, ...]]]
|
||||||
|
|
||||||
|
|
||||||
|
async def resolve_host(host: str) -> tuple[str, ...]:
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
results = await loop.getaddrinfo(host, 443, type=socket.SOCK_STREAM)
|
||||||
|
return tuple(sorted({str(result[4][0]) for result in results}))
|
||||||
|
|
||||||
|
|
||||||
|
def validate_remote_url(url: str, allowed_hosts: frozenset[str]) -> str:
|
||||||
|
parsed = urlsplit(url)
|
||||||
|
if parsed.scheme != "https":
|
||||||
|
raise SourceError("remote imports require HTTPS")
|
||||||
|
if parsed.username or parsed.password or parsed.port not in (None, 443):
|
||||||
|
raise SourceError("remote URL contains forbidden authority components")
|
||||||
|
host = (parsed.hostname or "").rstrip(".").lower()
|
||||||
|
if host not in allowed_hosts:
|
||||||
|
raise SourceError("remote host is not in the official-domain allowlist")
|
||||||
|
if parsed.fragment:
|
||||||
|
raise SourceError("remote URL fragments are not allowed")
|
||||||
|
return host
|
||||||
|
|
||||||
|
|
||||||
|
def validate_public_addresses(addresses: tuple[str, ...]) -> None:
|
||||||
|
if not addresses:
|
||||||
|
raise SourceError("remote host did not resolve")
|
||||||
|
for value in addresses:
|
||||||
|
address = ipaddress.ip_address(value)
|
||||||
|
if not address.is_global:
|
||||||
|
raise SourceError(f"remote host resolved to a non-public address: {value}")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class RemoteSourceImporter:
|
||||||
|
url: str
|
||||||
|
allowed_hosts: frozenset[str] = frozenset({"pve.proxmox.com"})
|
||||||
|
max_bytes: int = 16 * 1024 * 1024
|
||||||
|
max_redirects: int = 3
|
||||||
|
retries: int = 2
|
||||||
|
timeout_seconds: float = 20.0
|
||||||
|
resolver: Resolver = resolve_host
|
||||||
|
transport: httpx.AsyncBaseTransport | None = None
|
||||||
|
|
||||||
|
async def load(self) -> bytes:
|
||||||
|
current = self.url
|
||||||
|
timeout = httpx.Timeout(self.timeout_seconds)
|
||||||
|
async with httpx.AsyncClient(
|
||||||
|
follow_redirects=False, timeout=timeout, transport=self.transport
|
||||||
|
) as client:
|
||||||
|
for redirect_count in range(self.max_redirects + 1):
|
||||||
|
host = validate_remote_url(current, self.allowed_hosts)
|
||||||
|
validate_public_addresses(await self.resolver(host))
|
||||||
|
response = await self._request(client, current)
|
||||||
|
if response.is_redirect:
|
||||||
|
if redirect_count == self.max_redirects:
|
||||||
|
raise SourceError("remote import exceeded redirect limit")
|
||||||
|
location = response.headers.get("location")
|
||||||
|
if not location:
|
||||||
|
raise SourceError("remote redirect has no location")
|
||||||
|
current = urljoin(current, location)
|
||||||
|
continue
|
||||||
|
response.raise_for_status()
|
||||||
|
content_length = response.headers.get("content-length")
|
||||||
|
if content_length and int(content_length) > self.max_bytes:
|
||||||
|
raise SourceError("remote artifact exceeds size limit")
|
||||||
|
content = response.content
|
||||||
|
if len(content) > self.max_bytes:
|
||||||
|
raise SourceError("remote artifact exceeds size limit")
|
||||||
|
return content
|
||||||
|
raise SourceError("remote import failed")
|
||||||
|
|
||||||
|
async def _request(self, client: httpx.AsyncClient, url: str) -> httpx.Response:
|
||||||
|
for attempt in range(self.retries + 1):
|
||||||
|
try:
|
||||||
|
return await client.get(url)
|
||||||
|
except (httpx.TimeoutException, httpx.NetworkError):
|
||||||
|
if attempt == self.retries:
|
||||||
|
raise
|
||||||
|
await asyncio.sleep(0.1 * (2**attempt))
|
||||||
|
raise SourceError("remote import retry loop exhausted")
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
"""Immutable filesystem storage for imported contract revisions."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app.contracts.model import Manifest, Snapshot, canonical_json
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class RevisionStore:
|
||||||
|
root: Path
|
||||||
|
|
||||||
|
def save(self, raw: bytes, snapshot: Snapshot, manifest: Manifest) -> Path:
|
||||||
|
revision = self.root / manifest.snapshot_sha256
|
||||||
|
if revision.exists():
|
||||||
|
return revision
|
||||||
|
self.root.mkdir(parents=True, exist_ok=True)
|
||||||
|
temporary = Path(tempfile.mkdtemp(prefix=".import-", dir=self.root))
|
||||||
|
try:
|
||||||
|
self._write(temporary / "raw.js", raw)
|
||||||
|
self._write(temporary / "snapshot.json", snapshot.canonical_bytes())
|
||||||
|
self._write(temporary / "manifest.json", canonical_json(manifest))
|
||||||
|
os.replace(temporary, revision)
|
||||||
|
except BaseException:
|
||||||
|
for child in temporary.iterdir():
|
||||||
|
child.unlink()
|
||||||
|
temporary.rmdir()
|
||||||
|
raise
|
||||||
|
return revision
|
||||||
|
|
||||||
|
def list(self) -> tuple[str, ...]:
|
||||||
|
if not self.root.exists():
|
||||||
|
return ()
|
||||||
|
return tuple(sorted(path.name for path in self.root.iterdir() if path.is_dir()))
|
||||||
|
|
||||||
|
def manifest(self, revision: str) -> Manifest:
|
||||||
|
return Manifest.model_validate_json((self.root / revision / "manifest.json").read_bytes())
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _write(path: Path, content: bytes) -> None:
|
||||||
|
with path.open("xb") as stream:
|
||||||
|
stream.write(content)
|
||||||
|
stream.flush()
|
||||||
|
os.fsync(stream.fileno())
|
||||||
+4
-1
@@ -13,6 +13,7 @@ authors = [{ name = "proxmox-api-simulator contributors" }]
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"asyncpg>=0.30,<0.31",
|
"asyncpg>=0.30,<0.31",
|
||||||
"fastapi>=0.116,<0.117",
|
"fastapi>=0.116,<0.117",
|
||||||
|
"httpx>=0.28,<0.29",
|
||||||
"pydantic>=2.11,<3",
|
"pydantic>=2.11,<3",
|
||||||
"pydantic-settings>=2.10,<3",
|
"pydantic-settings>=2.10,<3",
|
||||||
"uvicorn[standard]>=0.35,<0.36",
|
"uvicorn[standard]>=0.35,<0.36",
|
||||||
@@ -21,7 +22,6 @@ dependencies = [
|
|||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
dev = [
|
dev = [
|
||||||
"hypothesis>=6.135,<7",
|
"hypothesis>=6.135,<7",
|
||||||
"httpx>=0.28,<0.29",
|
|
||||||
"mypy>=1.17,<1.18",
|
"mypy>=1.17,<1.18",
|
||||||
"pytest>=8.4,<9",
|
"pytest>=8.4,<9",
|
||||||
"pytest-asyncio>=1.1,<2",
|
"pytest-asyncio>=1.1,<2",
|
||||||
@@ -29,6 +29,9 @@ dev = [
|
|||||||
"ruff>=0.12,<0.13",
|
"ruff>=0.12,<0.13",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[project.scripts]
|
||||||
|
proxmox-api-contract = "app.contracts.cli:main"
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
[tool.hatch.build.targets.wheel]
|
||||||
packages = ["app"]
|
packages = ["app"]
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
"""Offline command workflows for contract management."""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.contracts.cli import parser, run
|
||||||
|
|
||||||
|
FIXTURE = Path(__file__).parents[1] / "fixtures" / "api-viewer" / "pve-9.2.3-version.json"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_validate_command_reports_source_counts(
|
||||||
|
tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
||||||
|
) -> None:
|
||||||
|
arguments = argparse.Namespace(command="validate", store=tmp_path, file=FIXTURE)
|
||||||
|
|
||||||
|
assert await run(arguments) == 0
|
||||||
|
output = capsys.readouterr().out
|
||||||
|
assert json.loads(output) == {"nodes": 1, "warnings": 0}
|
||||||
|
|
||||||
|
|
||||||
|
async def test_local_import_list_and_show(
|
||||||
|
tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
||||||
|
) -> None:
|
||||||
|
import_arguments = argparse.Namespace(
|
||||||
|
command="import",
|
||||||
|
store=tmp_path,
|
||||||
|
file=FIXTURE,
|
||||||
|
url=None,
|
||||||
|
version="9.2.3",
|
||||||
|
)
|
||||||
|
assert await run(import_arguments) == 0
|
||||||
|
revision = Path(capsys.readouterr().out.strip()).name
|
||||||
|
|
||||||
|
assert await run(argparse.Namespace(command="list", store=tmp_path)) == 0
|
||||||
|
assert capsys.readouterr().out.strip() == revision
|
||||||
|
|
||||||
|
assert await run(argparse.Namespace(command="show", store=tmp_path, revision=revision)) == 0
|
||||||
|
manifest = json.loads(capsys.readouterr().out)
|
||||||
|
assert manifest["source_version"] == "9.2.3"
|
||||||
|
assert manifest["snapshot_sha256"] == revision
|
||||||
|
|
||||||
|
|
||||||
|
def test_cli_parser_accepts_local_import() -> None:
|
||||||
|
arguments = parser().parse_args(
|
||||||
|
["--store", "saved", "import", "--file", str(FIXTURE), "--version", "9.2.3"]
|
||||||
|
)
|
||||||
|
|
||||||
|
assert arguments.command == "import"
|
||||||
|
assert arguments.store == Path("saved")
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
"""Security and idempotency tests for contract imports."""
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.contracts.importer import RemoteSourceImporter, validate_remote_url
|
||||||
|
from app.contracts.normalize import normalize_snapshot
|
||||||
|
from app.contracts.source import ApiViewerParser, SourceError
|
||||||
|
from app.contracts.store import RevisionStore
|
||||||
|
|
||||||
|
|
||||||
|
async def public_resolver(_host: str) -> tuple[str, ...]:
|
||||||
|
return ("93.184.216.34",)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"url",
|
||||||
|
[
|
||||||
|
"http://pve.proxmox.com/apidoc.js",
|
||||||
|
"https://evil.example/apidoc.js",
|
||||||
|
"https://pve.proxmox.com.evil.example/apidoc.js",
|
||||||
|
"https://user@pve.proxmox.com/apidoc.js",
|
||||||
|
"https://pve.proxmox.com:444/apidoc.js",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_remote_url_policy_rejects_unsafe_urls(url: str) -> None:
|
||||||
|
with pytest.raises(SourceError):
|
||||||
|
validate_remote_url(url, frozenset({"pve.proxmox.com"}))
|
||||||
|
|
||||||
|
|
||||||
|
async def test_remote_import_rejects_private_resolution() -> None:
|
||||||
|
async def private_resolver(_host: str) -> tuple[str, ...]:
|
||||||
|
return ("127.0.0.1",)
|
||||||
|
|
||||||
|
importer = RemoteSourceImporter(
|
||||||
|
"https://pve.proxmox.com/apidoc.js",
|
||||||
|
resolver=private_resolver,
|
||||||
|
transport=httpx.MockTransport(lambda request: httpx.Response(200, content=b"[]")),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(SourceError, match="non-public"):
|
||||||
|
await importer.load()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_redirect_is_revalidated() -> None:
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
return httpx.Response(302, headers={"location": "https://evil.example/private"})
|
||||||
|
|
||||||
|
importer = RemoteSourceImporter(
|
||||||
|
"https://pve.proxmox.com/apidoc.js",
|
||||||
|
resolver=public_resolver,
|
||||||
|
transport=httpx.MockTransport(handler),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(SourceError, match="allowlist"):
|
||||||
|
await importer.load()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_remote_import_enforces_size_limit() -> None:
|
||||||
|
importer = RemoteSourceImporter(
|
||||||
|
"https://pve.proxmox.com/apidoc.js",
|
||||||
|
max_bytes=2,
|
||||||
|
resolver=public_resolver,
|
||||||
|
transport=httpx.MockTransport(lambda request: httpx.Response(200, content=b"[]\n")),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(SourceError, match="size"):
|
||||||
|
await importer.load()
|
||||||
|
|
||||||
|
|
||||||
|
def test_revision_store_is_idempotent(tmp_path: Path) -> None:
|
||||||
|
raw = b'[{"path":"/version","info":{}}]'
|
||||||
|
parsed = ApiViewerParser().parse(raw)
|
||||||
|
snapshot, manifest = normalize_snapshot(
|
||||||
|
parsed,
|
||||||
|
raw=raw,
|
||||||
|
source_version="test",
|
||||||
|
retrieved_at=datetime(2026, 1, 1, tzinfo=UTC),
|
||||||
|
)
|
||||||
|
store = RevisionStore(tmp_path)
|
||||||
|
|
||||||
|
first = store.save(raw, snapshot, manifest)
|
||||||
|
second = store.save(raw, snapshot, manifest)
|
||||||
|
|
||||||
|
assert first == second
|
||||||
|
assert store.list() == (manifest.snapshot_sha256,)
|
||||||
|
assert store.manifest(manifest.snapshot_sha256) == manifest
|
||||||
Reference in New Issue
Block a user