diff --git a/.gitignore b/.gitignore index efb7eac..3c83d6e 100644 --- a/.gitignore +++ b/.gitignore @@ -67,7 +67,7 @@ hypothesis/ *.sublime-project *.sublime-workspace -# Docker / local runtime +# Docker / local runtime (`make up-local` writes port remaps here) *.log docker-compose.override.yml diff --git a/Makefile b/Makefile index 84a45a6..7c14cf8 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ PUSH_LATEST ?= 1 COMPOSE_RELEASE ?= $(COMPOSE) -f docker-compose.release.yml HELM_CHART ?= ./helm/proxmox-api-simulator -.PHONY: help install format lint typecheck test test-unit test-integration test-contract test-compatibility test-surface evidence coverage run dev up down restart logs docker-build docker-up docker-down docker-logs docker-restart db-up db-down db-migrate db-reset api-import api-diff seed clean ci ci-all shell release release-build release-up release-down release-seed helm-deps helm-template helm-lint pulumi-tests +.PHONY: help install format lint typecheck test test-unit test-integration test-contract test-compatibility test-surface evidence coverage run dev up up-local down down-local restart logs docker-build docker-up docker-down docker-logs docker-restart db-up db-down db-migrate db-reset api-import api-diff seed clean ci ci-all shell release release-build release-up release-down release-seed helm-deps helm-template helm-lint pulumi-tests push help: ## Show available commands @awk 'BEGIN {FS = ":.*## "}; /^[a-zA-Z0-9_-]+:.*## / {printf "%-22s %s\n", $$1, $$2}' $(MAKEFILE_LIST) @@ -75,9 +75,17 @@ up: ## Start PostgreSQL and simulator (plain HTTP :8006) @test -f .env || cp .env.example .env $(COMPOSE) up -d --build --wait +up-local: ## Start locally; remap busy ports via gitignored docker-compose.override.yml + @COMPOSE="$(COMPOSE)" ./scripts/up-local.sh + down: ## Stop local services $(COMPOSE) down +down-local: ## Stop local stack and remove gitignored docker-compose.override.yml + @$(COMPOSE) down + @rm -f docker-compose.override.yml + @echo "Stopped local stack and removed docker-compose.override.yml" + restart: ## Rebuild and restart the stack @test -f .env || cp .env.example .env $(COMPOSE) up -d --build --force-recreate --wait @@ -127,7 +135,11 @@ api-diff: ## Compare API snapshots seed: ## Seed simulation data @test -f .env || cp .env.example .env - SEED_PROFILE="$${PROFILE:-small}" $(COMPOSE) run --rm --entrypoint python $(SERVICE_SIM) -m app.simulation.seed_cli + $(COMPOSE) run --rm \ + -e SEED_PROFILE="$${PROFILE:-small}" \ + -e SEED_LARGE_NODES="$${SEED_LARGE_NODES:-10}" \ + -e SEED_LARGE_RESOURCES="$${SEED_LARGE_RESOURCES:-1000}" \ + --entrypoint python $(SERVICE_SIM) -m app.simulation.seed_cli shell: ## Open an interactive shell in the development container $(COMPOSE) run --rm --no-deps $(SERVICE_DEV) bash @@ -175,8 +187,12 @@ release-down: ## Stop the published Hub stack $(COMPOSE_RELEASE) down release-seed: ## Seed the published Hub stack (PROFILE=small by default) - SEED_PROFILE="$${PROFILE:-small}" IMAGE_TAG="$${IMAGE_TAG:-$(VERSION)}" DOCKER_IMAGE="$(DOCKER_IMAGE)" \ - $(COMPOSE_RELEASE) run --rm --entrypoint python simulator -m app.simulation.seed_cli + IMAGE_TAG="$${IMAGE_TAG:-$(VERSION)}" DOCKER_IMAGE="$(DOCKER_IMAGE)" \ + $(COMPOSE_RELEASE) run --rm \ + -e SEED_PROFILE="$${PROFILE:-small}" \ + -e SEED_LARGE_NODES="$${SEED_LARGE_NODES:-10}" \ + -e SEED_LARGE_RESOURCES="$${SEED_LARGE_RESOURCES:-1000}" \ + --entrypoint python simulator -m app.simulation.seed_cli helm-deps: ## No-op placeholder (chart has no OCI dependencies) @echo "Chart $(HELM_CHART) vendors PostgreSQL templates; no helm dependency update required." @@ -196,3 +212,21 @@ helm-template: ## Render Helm manifests locally (requires helm) pulumi-tests: ## Full Pulumi suite (surface majors 6–9 + lifecycle, HTML report) $(MAKE) -C pulumi-tests up $(MAKE) -C pulumi-tests test + +push: ## git add ., prompt for commit message, push origin (GitHub + antropoff.ru) + @set -e; \ + git add .; \ + echo "=== staged ==="; \ + git status --short; \ + echo; \ + if git diff --cached --quiet; then \ + echo "Nothing to commit — pushing current branch."; \ + else \ + echo "Enter commit message, then Ctrl-D:"; \ + msg=$$(cat &2; exit 1; fi; \ + git commit -m "$$msg"; \ + fi; \ + echo "Pushing to both remotes:"; \ + git remote get-url --push --all origin | sed 's/^/ - /'; \ + git push origin HEAD diff --git a/README.md b/README.md index 22d4012..c9e8020 100644 --- a/README.md +++ b/README.md @@ -113,9 +113,9 @@ Interactive console with light/dark themes, endpoint catalog for PVE 6–9, runtime contract hot-swap, and a UPID task monitor. More detail: [Web UI](docs/web-ui.md). -![Web UI light theme](docs/images/web-ui-light.png) +![Web UI main](docs/images/web-ui-main.png) -![Web UI dark theme](docs/images/web-ui-dark.png) +![Web UI API catalog](docs/images/web-ui-api-catalog.png) ## Documentation diff --git a/README.ru.md b/README.ru.md index c8d2b1b..142a1c9 100644 --- a/README.ru.md +++ b/README.ru.md @@ -113,9 +113,9 @@ curl -sS -X POST -d 'username=root@pam&password=secret' \ горячей сменой runtime-контракта и монитором задач UPID. Подробнее: [Web UI](docs/ru/web-ui.md). -![Web UI, светлая тема](docs/images/web-ui-light.png) +![Web UI, главный экран](docs/images/web-ui-main.png) -![Web UI, тёмная тема](docs/images/web-ui-dark.png) +![Web UI, каталог API](docs/images/web-ui-api-catalog.png) ## Документация diff --git a/app/contracts/examples.py b/app/contracts/examples.py index ce09127..34cc96e 100644 --- a/app/contracts/examples.py +++ b/app/contracts/examples.py @@ -1,7 +1,15 @@ -"""Generate example values from Proxmox contract schemas.""" +"""Generate example values from Proxmox contract schemas. + +Examples follow the official API-viewer dialect: named ``format`` tokens, +property-string ``format`` objects (``key=value,key2=value2`` / bare +``default_key``), and common path-parameter placeholders. +""" from __future__ import annotations +from collections.abc import Mapping +from typing import Any + from app.contracts.model import Schema _PATH_PARAM_EXAMPLES: dict[str, object] = { @@ -22,6 +30,53 @@ _PATH_PARAM_EXAMPLES: dict[str, object] = { "key": "cpu", "digest": "00000000", "name": "example", + "clustername": "example", +} + +_FORMAT_EXAMPLES: dict[str, object] = { + "pve-node": "pve01", + "pve-node-list": "pve01,pve02", + "pve-vmid": 100, + "pve-vmid-list": "100,101", + "pve-storage-id": "local", + "pve-storage-server": "192.168.0.10", + "pve-poolid": "testpool", + "pve-userid": "root@pam", + "pve-userid-list": "root@pam,admin@pve", + "pve-realm": "pam", + "pve-groupid": "admins", + "pve-groupid-list": "admins,ops", + "pve-roleid": "Administrator", + "pve-roleid-list": "Administrator,PVEAdmin", + "pve-tokenid": "automation", + "pve-tokenid-list": "automation", + "pve-priv-list": "VM.Allocate,VM.Config.Options", + "pve-iface": "net0", + "pve-iface-list": "net0,net1", + "pve-configid": "default", + "pve-configid-list": "default", + "pve-sdn-vnet-id": "vnet0", + "pve-replication-job-id": "100-0", + "pve-ha-resource-or-vm-id": "vm:100", + "pve-ha-group-node-list": "pve01:1,pve02:1", + "pve-day-of-week-list": "mon,tue,wed", + "pve-fw-addr-spec": "192.168.0.0/24", + "pve-fw-dport-spec": "22", + "pve-fw-sport-spec": "1024:65535", + "pve-fw-protocol-spec": "tcp", + "pve-fw-icmp-type-spec": "echo-request", + "address": "192.168.0.1", + "ip": "192.168.0.1", + "IPorCIDR": "192.168.0.0/24", + "IPorCIDRorAlias": "192.168.0.0/24", + "dns-name": "example.local", + "email-opt": "admin@example.local", + "email-list": "admin@example.local", + "email-or-username-list": "admin@example.local", + "ldap-simple-attr": "uid", + "ldap-simple-attr-list": "posixGroup,groupOfNames", + "string-alist": "/var/lib/vz", + "pve-tfa-secret": "ABCDEFGHIJKLMNOP", } @@ -31,6 +86,14 @@ def path_param_example(name: str) -> object | None: return _PATH_PARAM_EXAMPLES.get(name) +def wire_param_name(name: str, *, index: int = 0) -> str: + """Map contract ``foo[n]`` names to a concrete wire key (``foo0``).""" + + if "[n]" in name: + return name.replace("[n]", str(index)) + return name + + def schema_example(schema: Schema, *, name: str | None = None) -> object: """Build a representative example value for a contract schema.""" @@ -47,6 +110,19 @@ def schema_example(schema: Schema, *, name: str | None = None) -> object: hinted = path_param_example(indexed.rstrip("0123456789")) if hinted is not None: return hinted + base = indexed.rstrip("0123456789") + if base in _PATH_PARAM_EXAMPLES: + return _PATH_PARAM_EXAMPLES[base] + if isinstance(schema.format, Mapping): + return property_string_example(schema.format) + if isinstance(schema.format, str): + formatted = _named_format_example(schema.format, name=name) + if formatted is not None: + return formatted + if schema.format == "email": + return "user@example.com" + if schema.format == "uri": + return "https://example.com" if schema.type == "array": if schema.items is not None: return [schema_example(schema.items)] @@ -68,9 +144,105 @@ def schema_example(schema: Schema, *, name: str | None = None) -> object: return float(schema.minimum) return 1.0 if schema.type == "string" or schema.type is None: - if schema.format == "email": - return "user@example.com" - if schema.format == "uri": - return "https://example.com" return "example" return None + + +def property_string_example(fmt: Mapping[str, Any]) -> str: + """Render a Proxmox property-string example from a ``format`` object. + + Mirrors the API viewer dialect: a ``default_key`` may appear as a bare + value; other required keys use ``key=value``; optional keys are omitted + from the minimal example (they appear in ``typetext`` as ``[,key=…]``). + """ + + bare: str | None = None + keyed: list[str] = [] + for key, raw in sorted(fmt.items()): + if not isinstance(raw, Mapping): + continue + value = _format_key_example(key, raw) + rendered = _stringify_property_value(value) + if raw.get("default_key"): + bare = rendered + continue + if _is_optional_flag(raw.get("optional")): + continue + keyed.append(f"{key}={rendered}") + + if bare is not None and not keyed: + return bare + if bare is not None: + return ",".join([bare, *keyed]) + if keyed: + return ",".join(keyed) + # All keys optional — include defaults / default_key style extras for a + # usable stub that still matches typetext shape. + fallback: list[str] = [] + for key, raw in sorted(fmt.items()): + if not isinstance(raw, Mapping): + continue + value = _format_key_example(key, raw) + rendered = _stringify_property_value(value) + if raw.get("default_key"): + return rendered + fallback.append(f"{key}={rendered}") + return ",".join(fallback) if fallback else "example" + + +def _named_format_example(fmt: str, *, name: str | None) -> object | None: + if fmt in _FORMAT_EXAMPLES: + return _FORMAT_EXAMPLES[fmt] + if fmt.endswith("-list") and fmt[: -len("-list")] in _FORMAT_EXAMPLES: + base = _FORMAT_EXAMPLES[fmt[: -len("-list")]] + return str(base) + if name and name in _FORMAT_EXAMPLES: + return _FORMAT_EXAMPLES[name] + return None + + +def _format_key_example(key: str, raw: Mapping[str, Any]) -> object: + if "default" in raw and raw["default"] is not None: + return raw["default"] + enum = raw.get("enum") + if isinstance(enum, (list, tuple)) and enum: + return enum[0] + nested_format = raw.get("format") + if isinstance(nested_format, Mapping): + return property_string_example(nested_format) + if isinstance(nested_format, str): + named = _named_format_example(nested_format, name=key) + if named is not None: + return named + if nested_format == "email": + return "user@example.com" + if nested_format == "uri": + return "https://example.com" + hinted = path_param_example(key) + if hinted is not None: + return hinted + typ = raw.get("type") + if typ == "boolean": + return 1 + if typ == "integer": + minimum = raw.get("minimum") + return int(minimum) if isinstance(minimum, (int, float)) else 1 + if typ == "number": + minimum = raw.get("minimum") + return float(minimum) if isinstance(minimum, (int, float)) else 1.0 + pattern = raw.get("pattern") + if isinstance(pattern, str) and "second|minute|hour|day" in pattern: + return "1/second" + return "example" + + +def _stringify_property_value(value: object) -> str: + if isinstance(value, bool): + return "1" if value else "0" + return str(value) + + +def _is_optional_flag(value: object) -> bool: + if value is True or value == 1 or value == "1": + return True + return False diff --git a/app/handlers/access.py b/app/handlers/access.py index eb3d967..f118ec2 100644 --- a/app/handlers/access.py +++ b/app/handlers/access.py @@ -125,7 +125,7 @@ def register_access_handlers(registry: HandlerRegistry) -> None: return [ { "userid": str(row["name"]), - "enable": 1 if row["enabled"] else 0, + "enable": bool(row["enabled"]), "realm-type": str(row["realm_kind"]), } for row in rows @@ -168,10 +168,25 @@ def register_access_handlers(registry: HandlerRegistry) -> None: ) if row is None: raise ApiError(404, "user does not exist") + groups = await database(request).pool.fetch( + """SELECT g.group_id FROM identity_group_members m + JOIN identity_groups g ON g.id = m.group_id + JOIN principals p ON p.id = m.principal_id + WHERE p.name=$1 ORDER BY g.group_id""", + userid, + ) + tokens = await database(request).pool.fetch( + """SELECT t.token_id FROM api_tokens t + JOIN principals p ON p.id = t.principal_id + WHERE p.name=$1 ORDER BY t.token_id""", + userid, + ) return { "userid": str(row["name"]), - "enable": 1 if row["enabled"] else 0, + "enable": bool(row["enabled"]), "realm-type": str(row["realm_kind"]), + "groups": [str(item["group_id"]) for item in groups], + "tokens": [{"tokenid": str(item["token_id"]), "privsep": 1} for item in tokens], } async def user_update(request: Request, inputs: dict[str, Any]) -> None: @@ -397,10 +412,18 @@ def register_access_handlers(registry: HandlerRegistry) -> None: path = str(payload["path"]) roleid = str(payload["roles"]) propagate = bool(int(payload.get("propagate", 1))) + delete = bool(int(payload.get("delete", 0))) users = [item.strip() for item in str(payload.get("users", "")).split(",") if item.strip()] groups = [ item.strip() for item in str(payload.get("groups", "")).split(",") if item.strip() ] + tokens = [ + item.strip() for item in str(payload.get("tokens", "")).split(",") if item.strip() + ] + await database(request).pool.execute( + """INSERT INTO roles(name) VALUES($1) ON CONFLICT DO NOTHING""", + roleid, + ) for userid in users: principal_id = await database(request).pool.fetchval( "SELECT id FROM principals WHERE name=$1", @@ -408,10 +431,15 @@ def register_access_handlers(registry: HandlerRegistry) -> None: ) if principal_id is None: raise ApiError(404, f"user {userid} does not exist") - await database(request).pool.execute( - """INSERT INTO roles(name) VALUES($1) ON CONFLICT DO NOTHING""", - roleid, - ) + if delete: + await database(request).pool.execute( + """DELETE FROM acl_entries + WHERE principal_id=$1 AND role_name=$2 AND path=$3""", + principal_id, + roleid, + path, + ) + continue await database(request).pool.execute( """INSERT INTO acl_entries(principal_id, role_name, path, propagate) VALUES($1, $2, $3, $4) @@ -429,10 +457,15 @@ def register_access_handlers(registry: HandlerRegistry) -> None: ) if group_id is None: raise ApiError(404, f"group {groupid} does not exist") - await database(request).pool.execute( - """INSERT INTO roles(name) VALUES($1) ON CONFLICT DO NOTHING""", - roleid, - ) + if delete: + await database(request).pool.execute( + """DELETE FROM group_acl_entries + WHERE group_id=$1 AND role_name=$2 AND path=$3""", + group_id, + roleid, + path, + ) + continue await database(request).pool.execute( """INSERT INTO group_acl_entries(group_id, role_name, path, propagate) VALUES($1, $2, $3, $4) @@ -443,6 +476,46 @@ def register_access_handlers(registry: HandlerRegistry) -> None: path, propagate, ) + for token_ref in tokens: + # Wire form: userid!tokenid + userid, _, tokenid = token_ref.partition("!") + if not userid or not tokenid: + raise ApiError(400, f"invalid token ACL subject '{token_ref}'") + token_row = await database(request).pool.fetchrow( + """SELECT t.id FROM api_tokens t + JOIN principals p ON p.id = t.principal_id + WHERE p.name=$1 AND t.token_id=$2""", + userid, + tokenid, + ) + if token_row is None: + raise ApiError(404, f"token {token_ref} does not exist") + # Persist as principal ACL under the owning user with token scope in path metadata. + # Keep durable by tagging path; GET ACL still lists principal entries. + scoped_path = f"{path}#token:{tokenid}" if path != "/" else f"/#token:{tokenid}" + principal_id = await database(request).pool.fetchval( + "SELECT id FROM principals WHERE name=$1", + userid, + ) + if delete: + await database(request).pool.execute( + """DELETE FROM acl_entries + WHERE principal_id=$1 AND role_name=$2 AND path=$3""", + principal_id, + roleid, + scoped_path, + ) + else: + await database(request).pool.execute( + """INSERT INTO acl_entries(principal_id, role_name, path, propagate) + VALUES($1, $2, $3, $4) + ON CONFLICT (principal_id, role_name, path) DO UPDATE + SET propagate=EXCLUDED.propagate""", + principal_id, + roleid, + scoped_path, + propagate, + ) async def token_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: userid = str(values(inputs)["userid"]) @@ -562,10 +635,8 @@ def register_access_handlers(registry: HandlerRegistry) -> None: ) if row is None: raise ApiError(404, "role does not exist") - return { - "roleid": str(row["name"]), - "privs": ",".join(str(item) for item in row["privileges"]), - } + # Contract returns privilege→boolean map (additionalProperties: 0 on named privs). + return {str(priv): 1 for priv in row["privileges"]} async def role_create(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) diff --git a/app/handlers/acme.py b/app/handlers/acme.py index 4074390..e7d5332 100644 --- a/app/handlers/acme.py +++ b/app/handlers/acme.py @@ -2,6 +2,8 @@ from __future__ import annotations +import hashlib +import json from typing import Any from fastapi import Request @@ -35,7 +37,7 @@ def _default_directory(acme: dict[str, Any]) -> str: for item in directories: if isinstance(item, dict) and item.get("url"): return str(item["url"]) - return "" + return "https://acme-v02.api.letsencrypt.org/directory" def register_acme_handlers(registry: HandlerRegistry) -> None: @@ -61,7 +63,7 @@ def register_acme_handlers(registry: HandlerRegistry) -> None: for name, item in sorted(accounts.items()) ] - async def account_create(request: Request, inputs: dict[str, Any]) -> None: + async def account_create(request: Request, inputs: dict[str, Any]) -> str: payload = values(inputs) name = str(payload.get("name") or "default") metadata = await cluster_metadata(request) @@ -80,6 +82,7 @@ def register_acme_handlers(registry: HandlerRegistry) -> None: "location": f"https://acme.example.local/acct/{name}", } await save_cluster_metadata(request, metadata) + return name async def account_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: name = str(values(inputs)["name"]) @@ -88,8 +91,10 @@ def register_acme_handlers(registry: HandlerRegistry) -> None: if not isinstance(account, dict): raise ApiError(404, "ACME account does not exist") return { - "name": name, - "contact": account.get("contact"), + "account": { + "name": name, + "contact": account.get("contact"), + }, "directory": account.get("directory"), "tos": account.get("tos_url"), "location": account.get("location"), @@ -123,7 +128,13 @@ def register_acme_handlers(registry: HandlerRegistry) -> None: for plugin_id, item in sorted(plugins.items()): if plugin_type and item.get("type") != plugin_type: continue - result.append({"plugin": plugin_id, **{k: v for k, v in item.items() if k != "data"}}) + entry = {"plugin": plugin_id, **{k: v for k, v in item.items() if k != "data"}} + if "digest" not in entry: + material = {k: v for k, v in entry.items() if k != "digest"} + entry["digest"] = hashlib.sha1( + json.dumps(material, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + result.append(entry) return result async def plugins_create(request: Request, inputs: dict[str, Any]) -> None: diff --git a/app/handlers/backup.py b/app/handlers/backup.py index 3d23ace..eecdd91 100644 --- a/app/handlers/backup.py +++ b/app/handlers/backup.py @@ -1,8 +1,8 @@ -"""Cluster backup and vzdump handlers.""" +"""Cluster backup jobs and node vzdump handlers.""" from __future__ import annotations -import json +import secrets from typing import Any from fastapi import Request @@ -10,149 +10,198 @@ from fastapi import Request from app.api.errors import ApiError from app.api.registry import HandlerRegistry from app.db.primitives import ConflictError -from app.handlers.common import database, require_node, state, values +from app.handlers.common import ( + cluster_metadata, + database, + require_node, + save_cluster_metadata, + state, + subdirs, + values, +) from app.tasks.repository import TaskRepository from app.tasks.upid import Upid +def _backup_jobs(metadata: dict[str, Any]) -> dict[str, dict[str, Any]]: + jobs = metadata.get("backup_jobs") + if not isinstance(jobs, dict): + return {} + return { + str(job_id): dict(payload) for job_id, payload in jobs.items() if isinstance(payload, dict) + } + + +def _job_view(job_id: str, payload: dict[str, Any]) -> dict[str, Any]: + view = dict(payload) + view["id"] = job_id + enabled = view.get("enabled", 1) + view["enabled"] = bool(int(enabled)) if not isinstance(enabled, bool) else enabled + return view + + def register_backup_handlers(registry: HandlerRegistry) -> None: - async def backup_list(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: - rows = await database(_request).pool.fetch( - """SELECT b.id, b.volume_id, b.size_bytes, b.metadata, b.created_at, - r.external_id AS vmid, n.name AS node, s.storage_id - FROM backups b - LEFT JOIN resources r ON r.id = b.resource_id - LEFT JOIN nodes n ON n.id = r.node_id - JOIN storages s ON s.resource_id = b.storage_resource_id - ORDER BY b.created_at DESC LIMIT 2000""" + async def backup_list(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: + metadata = await cluster_metadata(request) + jobs = _backup_jobs(metadata) + return [_job_view(job_id, payload) for job_id, payload in sorted(jobs.items())] + + async def backup_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: + job_id = str(values(inputs)["id"]) + metadata = await cluster_metadata(request) + jobs = _backup_jobs(metadata) + payload = jobs.get(job_id) + if payload is None: + raise ApiError(404, "backup job does not exist") + return _job_view(job_id, payload) + + async def backup_create(request: Request, inputs: dict[str, Any]) -> None: + payload = values(inputs) + metadata = await cluster_metadata(request) + jobs = _backup_jobs(metadata) + job_id = str(payload.get("id") or f"backup-{secrets.token_hex(3)}") + if job_id in jobs: + raise ApiError(400, f"backup job '{job_id}' already exists") + entry = { + key: value + for key, value in payload.items() + if key not in {"delete", "digest", "node", "target"} + } + entry["id"] = job_id + entry.setdefault("enabled", 1) + entry.setdefault("storage", payload.get("storage") or "local") + jobs[job_id] = entry + metadata["backup_jobs"] = jobs + await save_cluster_metadata(request, metadata) + + async def backup_update(request: Request, inputs: dict[str, Any]) -> None: + job_id = str(values(inputs)["id"]) + payload = values(inputs) + metadata = await cluster_metadata(request) + jobs = _backup_jobs(metadata) + current = jobs.get(job_id) + if current is None: + raise ApiError(404, "backup job does not exist") + updated = dict(current) + for key, value in payload.items(): + if key in {"id", "delete", "digest"}: + continue + updated[key] = value + updated["id"] = job_id + jobs[job_id] = updated + metadata["backup_jobs"] = jobs + await save_cluster_metadata(request, metadata) + + async def backup_delete(request: Request, inputs: dict[str, Any]) -> None: + job_id = str(values(inputs)["id"]) + metadata = await cluster_metadata(request) + jobs = _backup_jobs(metadata) + if job_id not in jobs: + raise ApiError(404, "backup job does not exist") + del jobs[job_id] + metadata["backup_jobs"] = jobs + await save_cluster_metadata(request, metadata) + + async def backup_info(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, str]]: + return subdirs("not-backed-up") + + async def backupinfo_stub(_request: Request, _inputs: dict[str, Any]) -> str: + # PVE 6.x `/cluster/backupinfo` returns a stub string; 7+ uses `/backup-info` index. + return "Please use the 'not-backed-up' API" + + async def backup_not_backed_up( + request: Request, _inputs: dict[str, Any] + ) -> list[dict[str, Any]]: + metadata = await cluster_metadata(request) + jobs = _backup_jobs(metadata) + covered: set[str] = set() + for payload in jobs.values(): + if int(payload.get("all") or 0): + covered.add("*") + for item in str(payload.get("vmid") or "").split(","): + item = item.strip() + if item: + covered.add(item) + rows = await database(request).pool.fetch( + """SELECT r.external_id, r.kind, r.state FROM resources r + WHERE r.kind IN ('qemu', 'lxc') + ORDER BY r.external_id::integer""" ) result: list[dict[str, Any]] = [] + if "*" in covered: + return result for row in rows: - metadata = state(row["metadata"]) + vmid = str(row["external_id"]) + if vmid in covered: + continue + guest_state = state(row["state"]) result.append( { - "id": str(row["id"]), - "volid": str(row["volume_id"]), - "size": int(row["size_bytes"]), - "vmid": int(row["vmid"]) if row["vmid"] is not None else None, - "node": str(row["node"]) if row["node"] is not None else None, - "storage": str(row["storage_id"]), - "starttime": int(row["created_at"].timestamp()), - "mode": metadata.get("mode", "snapshot"), - "type": metadata.get("type", "vzdump"), + "vmid": int(vmid), + "type": "qemu" if row["kind"] == "qemu" else "lxc", + "name": str(guest_state.get("name") or f"{row['kind']}-{vmid}"), } ) return result - async def backup_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: - backup_id = str(values(inputs)["id"]) - row = await database(request).pool.fetchrow( - """SELECT b.id, b.volume_id, b.size_bytes, b.metadata, b.created_at, - r.external_id AS vmid, n.name AS node, s.storage_id - FROM backups b - LEFT JOIN resources r ON r.id = b.resource_id - LEFT JOIN nodes n ON n.id = r.node_id - JOIN storages s ON s.resource_id = b.storage_resource_id - WHERE b.id::text = $1 OR b.volume_id = $1""", - backup_id, - ) - if row is None: - raise ApiError(404, "backup does not exist") - metadata = state(row["metadata"]) - return { - "id": str(row["id"]), - "volid": str(row["volume_id"]), - "size": int(row["size_bytes"]), - "vmid": int(row["vmid"]) if row["vmid"] is not None else None, - "node": str(row["node"]) if row["node"] is not None else None, - "storage": str(row["storage_id"]), - "starttime": int(row["created_at"].timestamp()), - "notes": metadata.get("notes-template"), - **metadata, - } - - async def backup_update(request: Request, inputs: dict[str, Any]) -> None: - backup_id = str(values(inputs)["id"]) - row = await database(request).pool.fetchrow( - "SELECT id, metadata FROM backups WHERE id::text = $1", - backup_id, - ) - if row is None: - raise ApiError(404, "backup does not exist") - metadata = state(row["metadata"]) - payload = values(inputs) - if "notes" in payload: - metadata["notes-template"] = payload["notes"] - await database(request).pool.execute( - "UPDATE backups SET metadata=$2::jsonb WHERE id=$1", - row["id"], - json.dumps(metadata, sort_keys=True), - ) - - async def backup_delete(request: Request, inputs: dict[str, Any]) -> None: - backup_id = str(values(inputs)["id"]) - status = await database(request).pool.execute( - "DELETE FROM backups WHERE id::text = $1", - backup_id, - ) - if status != "DELETE 1": - raise ApiError(404, "backup does not exist") - - async def backup_create(request: Request, inputs: dict[str, Any]) -> str: - payload = values(inputs) - node = str(payload.get("node") or payload.get("target") or "pve01") - await require_node(request, node) - vmid = payload.get("vmid") - return await _schedule_vzdump( - request, - node=node, - vmids=[str(vmid)] if vmid is not None else None, - payload=payload, - ) - - async def backup_info(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: - rows = await database(_request).pool.fetch( - """SELECT r.external_id AS vmid, n.name AS node, max(b.created_at) AS last_backup + async def backup_included_volumes(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: + job_id = str(values(inputs)["id"]) + metadata = await cluster_metadata(request) + jobs = _backup_jobs(metadata) + job = jobs.get(job_id) + if job is None: + raise ApiError(404, "backup job does not exist") + vmids = [item.strip() for item in str(job.get("vmid") or "").split(",") if item.strip()] + guest_select = """SELECT r.external_id, r.kind, r.state, + v.config AS qemu_config, c.config AS lxc_config FROM resources r - JOIN nodes n ON n.id = r.node_id - LEFT JOIN backups b ON b.resource_id = r.id - WHERE r.kind = 'qemu' - GROUP BY r.external_id, n.name - ORDER BY r.external_id::integer - LIMIT 5000""" - ) - return [ - { - "vmid": int(row["vmid"]), - "node": str(row["node"]), - "lastbackup": int(row["last_backup"].timestamp()) if row["last_backup"] else 0, - "protected": 0, - } - for row in rows - ] - - async def backup_not_backed_up(_request: Request, _inputs: dict[str, Any]) -> list[int]: - rows = await database(_request).pool.fetch( - """SELECT r.external_id::integer AS vmid - FROM resources r - LEFT JOIN backups b ON b.resource_id = r.id - WHERE r.kind = 'qemu' AND b.id IS NULL - ORDER BY r.external_id::integer""" - ) - return [int(row["vmid"]) for row in rows] - - async def backup_included_volumes(request: Request, inputs: dict[str, Any]) -> list[str]: - backup_id = str(values(inputs)["id"]) - row = await database(request).pool.fetchrow( - """SELECT b.volume_id, r.external_id AS vmid - FROM backups b LEFT JOIN resources r ON r.id = b.resource_id - WHERE b.id::text = $1""", - backup_id, - ) - if row is None: - raise ApiError(404, "backup does not exist") - vmid = row["vmid"] - return [f"qemu/{vmid}"] if vmid is not None else [str(row["volume_id"])] + LEFT JOIN virtual_machines v ON v.resource_id = r.id + LEFT JOIN containers c ON c.resource_id = r.id""" + if int(job.get("all") or 0): + rows = await database(request).pool.fetch( + f"""{guest_select} + WHERE r.kind IN ('qemu', 'lxc') + ORDER BY r.external_id::integer""" + ) + else: + rows = await database(request).pool.fetch( + f"""{guest_select} + WHERE r.kind IN ('qemu', 'lxc') AND r.external_id = ANY($1::text[]) + ORDER BY r.external_id::integer""", + vmids, + ) + children: list[dict[str, Any]] = [] + for row in rows: + guest_state = state(row["state"]) + config_raw = row["qemu_config"] if row["kind"] == "qemu" else row["lxc_config"] + config = state(config_raw) if config_raw is not None else {} + volumes: list[dict[str, Any]] = [] + for key, value in sorted(config.items()): + if not ( + key.startswith("scsi") + or key.startswith("virtio") + or key.startswith("sata") + or key.startswith("ide") + or key in {"rootfs", "mp0", "mp1", "mp2", "mp3"} + ): + continue + volumes.append( + { + "id": key, + "name": str(value).split(",", 1)[0], + "included": True, + "reason": "included by default", + } + ) + children.append( + { + "id": int(row["external_id"]), + "name": str(guest_state.get("name") or f"{row['kind']}-{row['external_id']}"), + "type": "qemu" if row["kind"] == "qemu" else "lxc", + "children": volumes, + } + ) + return {"children": children} async def vzdump_defaults(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: from app.handlers.nodes import load_node_ops @@ -193,6 +242,7 @@ def register_backup_handlers(registry: HandlerRegistry) -> None: registry.register("/cluster/backup", "GET", backup_list) registry.register("/cluster/backup", "POST", backup_create) registry.register("/cluster/backup-info", "GET", backup_info) + registry.register("/cluster/backupinfo", "GET", backupinfo_stub) registry.register("/cluster/backup-info/not-backed-up", "GET", backup_not_backed_up) registry.register("/cluster/backup/{id}", "GET", backup_get) registry.register("/cluster/backup/{id}", "PUT", backup_update) @@ -229,11 +279,11 @@ async def _schedule_vzdump( payload={ "node": node, "vmids": vmids, - "storage": str(payload.get("storage") or "nfs-backup"), + "storage": str(payload.get("storage") or "local"), "mode": str(payload.get("mode") or "snapshot"), "compress": str(payload.get("compress") or "zstd"), }, - resource_key=f"backup:{node}", + resource_key=f"backup:{node}:{secrets.token_hex(4)}", idempotency_key=request.headers.get("Idempotency-Key"), ) except ConflictError as error: diff --git a/app/handlers/ceph.py b/app/handlers/ceph.py index 1435c4d..d4c5116 100644 --- a/app/handlers/ceph.py +++ b/app/handlers/ceph.py @@ -242,11 +242,45 @@ def register_ceph_handlers(registry: HandlerRegistry) -> None: await _save_cluster_ceph(request, ceph) return _upid(node, "cephrestart") + def _wire_ceph_pool(name: str, item: dict[str, Any], *, index: int) -> dict[str, Any]: + pool_id = item.get("pool_id", item.get("pool")) + try: + pool_num = int(pool_id) + except (TypeError, ValueError): + pool_num = index + crush_raw = item.get("crush_rule", 0) + try: + crush_id = int(crush_raw) + crush_name = str(item.get("crush_rule_name") or "replicated_rule") + except (TypeError, ValueError): + crush_name = str(crush_raw or "replicated_rule") + crush_id = 0 + application = str(item.get("application") or "rbd") + return { + "pool": pool_num, + "pool_name": str(item.get("pool_name") or name), + "size": int(item.get("size") or 3), + "min_size": int(item.get("min_size") or 2), + "pg_num": int(item.get("pg_num") or 128), + "crush_rule": crush_id, + "crush_rule_name": crush_name, + "application_metadata": item.get("application_metadata") + if isinstance(item.get("application_metadata"), dict) + else {application: {}}, + "bytes_used": int(item.get("bytes_used") or 0), + "percent_used": float(item.get("percent_used") or 0.0), + "healthy": bool(item.get("healthy", True)), + } + async def pool_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: await require_node(request, str(values(inputs)["node"])) ceph = await _load_cluster_ceph(request) pools = ceph.get("pools") or {} - return [dict(item) for _, item in sorted(pools.items()) if isinstance(item, dict)] + return [ + _wire_ceph_pool(name, item, index=index) + for index, (name, item) in enumerate(sorted(pools.items())) + if isinstance(item, dict) + ] async def pool_create(request: Request, inputs: dict[str, Any]) -> str: payload = values(inputs) @@ -278,7 +312,10 @@ def register_ceph_handlers(registry: HandlerRegistry) -> None: pool = (ceph.get("pools") or {}).get(name) if not isinstance(pool, dict): raise ApiError(404, "pool does not exist") - return [dict(pool)] + # Reuse list wire shape for detail GETs. + names = sorted((ceph.get("pools") or {}).keys()) + index = names.index(name) if name in names else 0 + return [_wire_ceph_pool(name, pool, index=index)] async def pool_update(request: Request, inputs: dict[str, Any]) -> str: payload = values(inputs) @@ -482,30 +519,63 @@ def register_ceph_handlers(registry: HandlerRegistry) -> None: await _save_node_ceph(request, node, ceph) return _upid(node, "cephdestroymon") - async def osd_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: + async def osd_list(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: node = str(values(inputs)["node"]) await require_node(request, node) rows = await database(request).pool.fetch( - """SELECT r.external_id, r.state + """SELECT r.external_id, r.state, n.name AS node FROM resources r JOIN nodes n ON n.id=r.node_id - WHERE n.name=$1 AND r.kind='ceph-osd' - ORDER BY r.external_id""", - node, + WHERE r.kind='ceph-osd' + ORDER BY n.name, r.external_id""" ) - result: list[dict[str, Any]] = [] + hosts: dict[str, dict[str, Any]] = {} + host_id = -2 for row in rows: + host_name = str(row["node"]) + host = hosts.get(host_name) + if host is None: + host = { + "id": host_id, + "name": host_name, + "type": "host", + "type_id": 1, + "children": [], + } + hosts[host_name] = host + host_id -= 1 payload = state(row["state"]) - osd_id = payload.get("osd_id", row["external_id"]) - result.append( + osd_raw = payload.get("osd_id", row["external_id"]) + try: + osd_id = int(osd_raw) + except (TypeError, ValueError): + digits = "".join(ch for ch in str(osd_raw) if ch.isdigit()) + osd_id = int(digits) if digits else 0 + host["children"].append( { - "osd": int(osd_id) if str(osd_id).isdigit() else osd_id, + "id": osd_id, + "name": f"osd.{osd_id}", + "type": "osd", + "type_id": 0, "status": payload.get("status", "up"), "in": 1 if payload.get("in", True) else 0, - "weight": payload.get("weight", 1.0), + "weight": float(payload.get("weight", 1.0)), "device_class": payload.get("device_class", "hdd"), + "host": host_name, } ) - return result + root_children = list(hosts.values()) + # Prefer the requested node first in the tree for local dumps. + root_children.sort(key=lambda item: (0 if item["name"] == node else 1, item["name"])) + return { + "flags": "", + "root": { + "id": -1, + "name": "default", + "type": "root", + "type_id": 10, + "children": root_children, + }, + } async def osd_create(request: Request, inputs: dict[str, Any]) -> str: payload = values(inputs) @@ -677,16 +747,36 @@ def register_ceph_handlers(registry: HandlerRegistry) -> None: health = ceph.get("health") if not isinstance(health, dict): health = {"status": "HEALTH_OK" if ceph.get("running") else "HEALTH_WARN"} + node_count = int(await database(_request).pool.fetchval("SELECT COUNT(*) FROM nodes") or 0) return { - "version": version_str, + "version": { + "version": version_str, + "release": "reef", + "epoch": 0, + }, "health": health, "osdmap": { + "epoch": 1, "num_osds": num_osds, "num_up_osds": num_up, "num_in_osds": num_in, + "num_remapped_pgs": 0, }, - "pgmap": {"bytes_used": used, "bytes_total": total}, - "fsmap": {"filesystems": list((ceph.get("fs") or {}).keys())}, + "pgmap": { + "bytes_used": used, + "bytes_total": total, + "bytes_avail": max(total - used, 0), + "data_bytes": used, + "num_pgs": max(num_osds * 32, 32), + }, + "fsmap": { + "epoch": 1, + "by_rank": [], + "filesystems": list((ceph.get("fs") or {}).keys()), + }, + "monmap": {"num_mons": max(node_count, 1)}, + "mgrmap": {"available": True, "num_standbys": 0}, + "servicemap": {"epoch": 1, "services": {}}, } base = "/nodes/{node}/ceph" diff --git a/app/handlers/cluster.py b/app/handlers/cluster.py index 0bc4d1f..7a4180b 100644 --- a/app/handlers/cluster.py +++ b/app/handlers/cluster.py @@ -24,7 +24,43 @@ def _replication_jobs(metadata: dict[str, Any]) -> list[dict[str, Any]]: jobs = metadata.get("replication", []) if not isinstance(jobs, list): return [] - return [dict(item) for item in jobs if isinstance(item, dict)] + result: list[dict[str, Any]] = [] + for item in jobs: + if not isinstance(item, dict): + continue + guest_raw = item.get("guest", 0) + try: + guest = int(guest_raw) + except (TypeError, ValueError): + guest = int(str(guest_raw).split(":")[-1] or 0) + jobnum_raw = item.get("jobnum") + if jobnum_raw is None: + id_text = str(item.get("id") or "0") + tail = id_text.rsplit("-", 1)[-1] + jobnum = int(tail) if tail.isdigit() else 0 + else: + jobnum = int(jobnum_raw) + disable = item.get("disable") + if disable is None: + disable = 0 if int(item.get("enabled", 1) or 1) else 1 + entry = { + "id": str(item.get("id") or f"{guest}-{jobnum}"), + "guest": guest, + "jobnum": jobnum, + "target": str(item.get("target") or ""), + "type": str(item.get("type") or "local"), + "disable": bool(int(disable)), + } + if item.get("schedule") is not None: + entry["schedule"] = str(item["schedule"]) + if item.get("rate") is not None: + entry["rate"] = int(item["rate"]) + if item.get("comment") is not None: + entry["comment"] = str(item["comment"]) + if item.get("source") is not None: + entry["source"] = str(item["source"]) + result.append(entry) + return result def register_cluster_handlers(registry: HandlerRegistry) -> None: @@ -53,24 +89,41 @@ def register_cluster_handlers(registry: HandlerRegistry) -> None: """SELECT id, name, status FROM nodes ORDER BY name""" ) metadata = await cluster_metadata(request) - quorate = metadata.get("quorate") - result: list[dict[str, Any]] = [] + quorate = bool(int(metadata.get("quorate", 1) or 1)) + cluster_name = metadata.get("cluster_config") + if isinstance(cluster_name, dict): + name = str(cluster_name.get("clustername") or "pve-simulator") + version = int(cluster_name.get("version") or 2) + else: + name = "pve-simulator" + version = 2 + result: list[dict[str, Any]] = [ + { + "type": "cluster", + "id": "cluster", + "name": name, + "nodes": len(rows), + "quorate": quorate, + "version": version, + } + ] for index, row in enumerate(rows): online = str(row["status"]) == "online" ops = await load_node_ops(request, str(row["name"])) cluster_node = ops.get("cluster_status") entry = dict(cluster_node) if isinstance(cluster_node, dict) else {} + local_raw = entry.get("local", index == 0) + node_name = str(row["name"]) result.append( { - "id": str(row["id"]), - "name": str(row["name"]), - "nodeid": entry.get("nodeid", index), - "online": 1 if online else 0, - "local": entry.get("local", 1 if index == 0 else 0), - "ip": entry.get("ip") or ops.get("ip") or "", - "level": entry.get("level", "c"), - "type": entry.get("type", "node"), - "quorate": entry.get("quorate", quorate if quorate is not None else 1), + "type": "node", + "id": f"node/{node_name}", + "name": node_name, + "nodeid": int(entry.get("nodeid", index + 1)), + "online": online, + "local": bool(int(local_raw)) if not isinstance(local_raw, bool) else local_raw, + "ip": str(entry.get("ip") or ops.get("ip") or ""), + "level": str(entry.get("level", "")), } ) return result diff --git a/app/handlers/cluster_config.py b/app/handlers/cluster_config.py index 233ac6d..6a52b66 100644 --- a/app/handlers/cluster_config.py +++ b/app/handlers/cluster_config.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import secrets from typing import Any @@ -12,10 +13,12 @@ from app.api.registry import HandlerRegistry from app.handlers.common import ( cluster_metadata, database, + require_value, save_cluster_metadata, - subdirs, values, ) +from app.tasks.repository import TaskRepository +from app.tasks.upid import Upid def _config(metadata: dict[str, Any]) -> dict[str, Any]: @@ -23,19 +26,132 @@ def _config(metadata: dict[str, Any]) -> dict[str, Any]: return current if isinstance(current, dict) else {} +def _principal(request: Request) -> str: + return str(getattr(request.state, "principal", None) or "root@pam") + + +def _fingerprint_sha256() -> str: + return ":".join(f"{byte:02X}" for byte in secrets.token_bytes(32)) + + +def _stable_digest(material: str) -> str: + return hashlib.sha256(material.encode("utf-8")).hexdigest() + + +def _node_addr(node: str, index: int, entry: dict[str, Any]) -> str: + for key in ("pve_addr", "new_node_ip", "ring0_addr"): + value = entry.get(key) + if isinstance(value, str) and value and not value.endswith(".local"): + # Prefer literal IPs; hostnames are still accepted as ring addresses. + if key != "ring0_addr" or all(part.isdigit() for part in value.split(".")): + return value + return f"10.0.0.{index}" + + +def _ring0_addr(node: str, index: int, entry: dict[str, Any], links: dict[str, Any]) -> str: + if entry.get("ring0_addr"): + return str(entry["ring0_addr"]) + link0 = links.get("link0") + if isinstance(link0, str) and link0: + # Property string may be bare IP or address=IP[,priority=N]. + if link0.startswith("address="): + return link0.removeprefix("address=").split(",", 1)[0] + return link0.split(",", 1)[0] + if entry.get("new_node_ip"): + return str(entry["new_node_ip"]) + return f"10.0.0.{index}" + + +def _render_corosync_conf(config: dict[str, Any], node_names: list[str]) -> str: + clustername = str(config.get("clustername") or "proxmox") + totem = dict(config.get("totem") or {}) + added_raw = config.get("added_nodes") + added: dict[str, Any] = dict(added_raw) if isinstance(added_raw, dict) else {} + links = dict(config.get("links") or {}) + lines = [ + "totem {", + " version: 2", + f" cluster_name: {totem.get('cluster_name', clustername)}", + f" secauth: {totem.get('secauth', 'on')}", + "}", + "nodelist {", + ] + for index, name in enumerate(node_names, start=1): + entry_raw = added.get(name) + entry: dict[str, Any] = dict(entry_raw) if isinstance(entry_raw, dict) else {} + nodeid = int(entry.get("nodeid") or index) + votes = int(entry.get("quorum_votes") or entry.get("votes") or config.get("votes") or 1) + ring0 = _ring0_addr(name, index, entry, links) + lines.extend( + [ + " node {", + f" name: {name}", + f" nodeid: {nodeid}", + f" quorum_votes: {votes}", + f" ring0_addr: {ring0}", + " }", + ] + ) + lines.append("}") + lines.extend(["quorum {", " provider: corosync_votequorum", "}"]) + return "\n".join(lines) + "\n" + + +def _ensure_corosync_materials( + config: dict[str, Any], node_names: list[str] +) -> tuple[str, str, str]: + authkey = config.get("corosync_authkey") + if not isinstance(authkey, str) or not authkey: + authkey = secrets.token_hex(32) + config["corosync_authkey"] = authkey + conf = _render_corosync_conf(config, node_names) + config["corosync_conf"] = conf + digest = _stable_digest(conf) + config["config_digest"] = digest + return authkey, conf, digest + + +async def _cluster_task( + request: Request, *, task_type: str, worker: str, task_id: str = "0" +) -> str: + from app.db.primitives import ConflictError + + pool = database(request).pool + node = await pool.fetchval("SELECT name FROM nodes ORDER BY name LIMIT 1") or "localhost" + upid = str(Upid.allocate(str(node), worker, task_id, _principal(request))) + try: + task = await TaskRepository(pool).create( + upid=upid, + task_type=task_type, + payload={"cluster": True, "worker": worker}, + resource_key=f"cluster:{task_type}:{secrets.token_hex(4)}", + ) + except ConflictError as error: + raise ApiError(409, str(error)) from error + return task.upid + + def register_cluster_config_handlers(registry: HandlerRegistry) -> None: async def index(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, str]]: - return subdirs("apiversion", "join", "nodes", "qdevice", "totem") + # PVE uses {name}; contract child link is href="{name}". + return [{"name": name} for name in ("nodes", "totem", "join", "qdevice", "apiversion")] - async def create(request: Request, inputs: dict[str, Any]) -> None: + async def create(request: Request, inputs: dict[str, Any]) -> str: payload = values(inputs) + clustername = str(require_value(payload, "clustername")) metadata = await cluster_metadata(request) config = dict(_config(metadata)) - if payload.get("clustername"): - config["clustername"] = str(payload["clustername"]) - totem = dict(config.get("totem") or {}) - totem["cluster_name"] = str(payload["clustername"]) - config["totem"] = totem + # Match PVE: refuse when corosync materials already exist (seeded or prior create). + if config.get("corosync_conf") or config.get("config_digest"): + raise ApiError(400, "cluster config already exists") + config["clustername"] = clustername + totem = dict(config.get("totem") or {}) + totem["version"] = int(totem.get("version") or 2) + totem["secauth"] = totem.get("secauth") or "on" + totem["cluster_name"] = clustername + # PVE defaults token-coefficient to 125 when omitted (majors that expose it). + totem["token_coefficient"] = payload.get("token-coefficient", 125) + config["totem"] = totem if "votes" in payload: config["votes"] = payload["votes"] if "nodeid" in payload: @@ -44,13 +160,43 @@ def register_cluster_config_handlers(registry: HandlerRegistry) -> None: if links: config["links"] = links config["token"] = secrets.token_hex(16) + + rows = await database(request).pool.fetch("SELECT name FROM nodes ORDER BY name") + node_names = [str(row["name"]) for row in rows] + if not node_names: + node_names = ["localhost"] + creator = node_names[0] + added = dict(config.get("added_nodes") or {}) + creator_entry = dict(added.get(creator) or {}) + creator_entry.update( + { + "node": creator, + "nodeid": payload.get("nodeid") or creator_entry.get("nodeid") or 1, + "quorum_votes": payload.get("votes") or creator_entry.get("quorum_votes") or 1, + "ring0_addr": _ring0_addr( + creator, 1, creator_entry, dict(config.get("links") or {}) + ), + "pve_addr": creator_entry.get("pve_addr") or _node_addr(creator, 1, creator_entry), + "pve_fp": creator_entry.get("pve_fp") or _fingerprint_sha256(), + } + ) + added[creator] = creator_entry + config["added_nodes"] = added + _ensure_corosync_materials(config, node_names) + metadata["cluster_config"] = config await save_cluster_metadata(request, metadata) await database(request).pool.execute( """UPDATE clusters SET name=$1, updated_at=now() WHERE id=(SELECT id FROM clusters LIMIT 1)""", - str(config["clustername"]), + clustername, + ) + return await _cluster_task( + request, + task_type="cluster-create", + worker="clustercreate", + task_id=clustername, ) async def apiversion(_request: Request, _inputs: dict[str, Any]) -> int: @@ -60,28 +206,69 @@ def register_cluster_config_handlers(registry: HandlerRegistry) -> None: async def join_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: metadata = await cluster_metadata(request) config = _config(metadata) - node = values(inputs).get("node") + preferred_raw = values(inputs).get("node") + # Contract documents the omitted-node default as this phrase; PVE uses local nodename. + if preferred_raw in {None, "", "current connected node"}: + preferred = None + else: + preferred = str(preferred_raw) rows = await database(request).pool.fetch("SELECT name, status FROM nodes ORDER BY name") - nodelist = [ - {"name": str(row["name"]), "online": 1 if row["status"] == "online" else 0} - for row in rows - ] + added_raw = config.get("added_nodes") + added: dict[str, Any] = dict(added_raw) if isinstance(added_raw, dict) else {} + links = dict(config.get("links") or {}) + nodelist: list[dict[str, Any]] = [] + for index, row in enumerate(rows, start=1): + name = str(row["name"]) + entry_raw = added.get(name) + entry: dict[str, Any] = dict(entry_raw) if isinstance(entry_raw, dict) else {} + if "pve_fp" not in entry: + entry["pve_fp"] = _fingerprint_sha256() + added[name] = entry + if "pve_addr" not in entry: + entry["pve_addr"] = _node_addr(name, index, entry) + added[name] = entry + nodelist.append( + { + "name": name, + "nodeid": int(entry.get("nodeid") or index), + "quorum_votes": int( + entry.get("quorum_votes") or entry.get("votes") or config.get("votes") or 1 + ), + "ring0_addr": _ring0_addr(name, index, entry, links), + "pve_addr": str(entry["pve_addr"]), + "pve_fp": str(entry["pve_fp"]), + } + ) + if added != dict(added_raw or {}): + updated = dict(config) + updated["added_nodes"] = added + if not updated.get("config_digest"): + _ensure_corosync_materials(updated, [item["name"] for item in nodelist]) + metadata["cluster_config"] = updated + await save_cluster_metadata(request, metadata) + config = updated + preferred_node = preferred or (nodelist[0]["name"] if nodelist else None) + digest = config.get("config_digest") + if not isinstance(digest, str) or not digest: + updated = dict(config) + _, _, digest = _ensure_corosync_materials(updated, [item["name"] for item in nodelist]) + metadata["cluster_config"] = updated + await save_cluster_metadata(request, metadata) + config = updated return { - "clustername": config.get("clustername"), - "config_digest": secrets.token_hex(8), "nodelist": nodelist, - "preferred_node": node or (nodelist[0]["name"] if nodelist else None), - "totem": config.get("totem", {}), - "links": config.get("links", {}), + "preferred_node": preferred_node, + "totem": dict(config.get("totem") or {}), + "config_digest": digest, } - async def join_post(request: Request, inputs: dict[str, Any]) -> None: + async def join_post(request: Request, inputs: dict[str, Any]) -> str: payload = values(inputs) + hostname = str(require_value(payload, "hostname")) + require_value(payload, "fingerprint") + require_value(payload, "password") metadata = await cluster_metadata(request) config = dict(_config(metadata)) - hostname = str(payload.get("hostname") or payload.get("node") or "") - if not hostname: - raise ApiError(400, "parameter verification failed - 'hostname' missing") joins = dict(config.get("join_info") or {}) joins[hostname] = { "hostname": hostname, @@ -89,23 +276,22 @@ def register_cluster_config_handlers(registry: HandlerRegistry) -> None: "nodeid": payload.get("nodeid"), "votes": payload.get("votes", 1), "force": payload.get("force"), + "password_set": True, } - # password accepted but not stored in clear form - if payload.get("password"): - joins[hostname]["password_set"] = True + links = {key: value for key, value in payload.items() if key.startswith("link")} + if links: + joins[hostname]["links"] = links config["join_info"] = joins + rows = await database(request).pool.fetch("SELECT name FROM nodes ORDER BY name") + node_names = [str(row["name"]) for row in rows] + _ensure_corosync_materials(config, node_names or ["localhost"]) metadata["cluster_config"] = config - exists = await database(request).pool.fetchval( - "SELECT EXISTS(SELECT 1 FROM nodes WHERE name=$1)", - hostname, - ) - if not exists: - await database(request).pool.execute( - """INSERT INTO nodes(id, name, status, metadata) - VALUES(gen_random_uuid(), $1, 'online', '{}'::jsonb)""", - hostname, - ) await save_cluster_metadata(request, metadata) + return await _cluster_task( + request, + task_type="cluster-join", + worker="clusterjoin", + ) async def nodes_list(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: rows = await database(request).pool.fetch("SELECT name, status FROM nodes ORDER BY name") @@ -113,6 +299,7 @@ def register_cluster_config_handlers(registry: HandlerRegistry) -> None: config = _config(metadata) added_raw = config.get("added_nodes") added: dict[str, Any] = dict(added_raw) if isinstance(added_raw, dict) else {} + links = dict(config.get("links") or {}) result = [] for index, row in enumerate(rows, start=1): name = str(row["name"]) @@ -122,33 +309,40 @@ def register_cluster_config_handlers(registry: HandlerRegistry) -> None: { "node": name, "nodeid": entry.get("nodeid", index), - "ring0_addr": entry.get("ring0_addr", ""), + "ring0_addr": _ring0_addr(name, index, entry, links), "quorum_votes": entry.get("quorum_votes", config.get("votes", 0)), } ) return result - async def nodes_add(request: Request, inputs: dict[str, Any]) -> None: + async def nodes_add(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: payload = values(inputs) - node = str(payload["node"]) + node = str(require_value(payload, "node")) metadata = await cluster_metadata(request) config = dict(_config(metadata)) added = dict(config.get("added_nodes") or {}) - added[node] = { + if node in added and not payload.get("force"): + raise ApiError(400, f"can't add existing node '{node}'") + index = len(added) + 1 + links = {key: value for key, value in payload.items() if key.startswith("link")} + entry = { "node": node, - "nodeid": payload.get("nodeid"), + "nodeid": payload.get("nodeid") or index, "new_node_ip": payload.get("new_node_ip"), "votes": payload.get("votes", 1), "apiversion": payload.get("apiversion"), "force": payload.get("force"), - "ring0_addr": payload.get("ring0_addr") or f"{node}.local", + "ring0_addr": payload.get("ring0_addr") + or payload.get("new_node_ip") + or _ring0_addr(node, index, {}, links or dict(config.get("links") or {})), "quorum_votes": payload.get("quorum_votes", payload.get("votes", 1)), + "pve_addr": payload.get("new_node_ip") or _node_addr(node, index, {}), + "pve_fp": _fingerprint_sha256(), } - links = {key: value for key, value in payload.items() if key.startswith("link")} if links: - added[node]["links"] = links + entry["links"] = links + added[node] = entry config["added_nodes"] = added - metadata["cluster_config"] = config exists = await database(request).pool.fetchval( "SELECT EXISTS(SELECT 1 FROM nodes WHERE name=$1)", node, @@ -159,7 +353,18 @@ def register_cluster_config_handlers(registry: HandlerRegistry) -> None: VALUES(gen_random_uuid(), $1, 'online', '{}'::jsonb)""", node, ) + rows = await database(request).pool.fetch("SELECT name FROM nodes ORDER BY name") + node_names = [str(row["name"]) for row in rows] + if node not in node_names: + node_names.append(node) + authkey, conf, _digest = _ensure_corosync_materials(config, node_names) + metadata["cluster_config"] = config await save_cluster_metadata(request, metadata) + return { + "corosync_authkey": authkey, + "corosync_conf": conf, + "warnings": [], + } async def nodes_delete(request: Request, inputs: dict[str, Any]) -> None: node = str(values(inputs)["node"]) @@ -171,6 +376,9 @@ def register_cluster_config_handlers(registry: HandlerRegistry) -> None: joins.pop(node, None) config["added_nodes"] = added config["join_info"] = joins + rows = await database(request).pool.fetch("SELECT name FROM nodes ORDER BY name") + node_names = [str(row["name"]) for row in rows if str(row["name"]) != node] + _ensure_corosync_materials(config, node_names or ["localhost"]) metadata["cluster_config"] = config await save_cluster_metadata(request, metadata) # Keep node row; mark offline to avoid cascading guest deletes. diff --git a/app/handlers/cluster_extra.py b/app/handlers/cluster_extra.py index d5dfa46..53c1f0c 100644 --- a/app/handlers/cluster_extra.py +++ b/app/handlers/cluster_extra.py @@ -15,9 +15,11 @@ from app.handlers.common import ( database, require_node, save_cluster_metadata, + state, subdirs, values, ) +from app.simulation.ceph_defaults import CEPH_FLAG_DESCRIPTIONS, default_ceph_flags from app.tasks.repository import TaskRepository from app.tasks.upid import Upid @@ -55,10 +57,9 @@ def _save_ha_rules(metadata: dict[str, Any], rules: list[dict[str, Any]]) -> Non def _replication_jobs(metadata: dict[str, Any]) -> list[dict[str, Any]]: - jobs = metadata.get("replication", []) - if not isinstance(jobs, list): - return [] - return [dict(item) for item in jobs if isinstance(item, dict)] + from app.handlers.cluster import _replication_jobs as normalize + + return normalize(metadata) def _ceph(metadata: dict[str, Any]) -> dict[str, Any]: @@ -189,10 +190,42 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: async def metrics_export(request: Request, _inputs: dict[str, Any]) -> dict[str, Any]: metadata = await cluster_metadata(request) metrics = _metrics(metadata) - return { - "data": str(metrics.get("export_data") or ""), - "timestamp": int(time.time()), - } + now = int(time.time()) + seeded = metrics.get("export_series") + if isinstance(seeded, list) and seeded: + series = [dict(item) for item in seeded if isinstance(item, dict)] + return {"data": series, "timestamp": now} + # Build a PVE-shaped metric array from live inventory (scales with seed). + rows = await database(request).pool.fetch("SELECT name FROM nodes ORDER BY name") + series: list[dict[str, Any]] = [] + for row in rows: + node = str(row["name"]) + series.append( + { + "id": f"node/{node}", + "metric": "up", + "timestamp": now, + "type": "gauge", + "value": 1, + } + ) + guests = await database(request).pool.fetch( + """SELECT r.kind, r.external_id, n.name AS node + FROM resources r JOIN nodes n ON n.id=r.node_id + WHERE r.kind IN ('qemu', 'lxc') + ORDER BY r.external_id::integer""" + ) + for guest in guests: + series.append( + { + "id": f"{guest['kind']}/{guest['external_id']}", + "metric": "up", + "timestamp": now, + "type": "gauge", + "value": 1, + } + ) + return {"data": series, "timestamp": now} async def metrics_server_list( request: Request, _inputs: dict[str, Any] @@ -398,43 +431,69 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: ) -> list[dict[str, str]]: return subdirs("flags", "metadata", "status") - async def ceph_flags_get(request: Request, _inputs: dict[str, Any]) -> dict[str, int]: + async def ceph_flags_get(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: metadata = await cluster_metadata(request) ceph = _ceph(metadata) - flags = ceph.get("flags") - if not isinstance(flags, dict): - return {} - return {str(key): int(value) for key, value in flags.items()} + stored = ceph.get("flags") + flags = default_ceph_flags() + if isinstance(stored, dict): + for name, value in stored.items(): + if name in flags: + flags[name] = int(value) + return [ + { + "name": name, + "value": bool(flags[name]), + "description": description, + } + for name, description in sorted(CEPH_FLAG_DESCRIPTIONS.items()) + ] - async def ceph_flags_put(request: Request, inputs: dict[str, Any]) -> dict[str, int]: + async def ceph_flags_put(request: Request, inputs: dict[str, Any]) -> str: payload = values(inputs) metadata = await cluster_metadata(request) ceph = dict(_ceph(metadata)) - flags = dict(ceph.get("flags") or {}) + flags = default_ceph_flags() + stored = ceph.get("flags") + if isinstance(stored, dict): + for name, value in stored.items(): + if name in flags: + flags[name] = int(value) for key, value in payload.items(): if key in {"delete", "digest"}: continue + if key not in CEPH_FLAG_DESCRIPTIONS: + raise ApiError(400, f"unknown ceph flag '{key}'") flags[str(key)] = int(value) ceph["flags"] = flags metadata["ceph"] = ceph await save_cluster_metadata(request, metadata) - return {str(key): int(value) for key, value in flags.items()} + return await _cluster_task(request, task_type="ceph-flags", worker="cephflags") - async def ceph_flag_get(request: Request, inputs: dict[str, Any]) -> dict[str, int]: + async def ceph_flag_get(request: Request, inputs: dict[str, Any]) -> bool: flag = str(values(inputs)["flag"]) + if flag not in CEPH_FLAG_DESCRIPTIONS: + raise ApiError(404, "ceph flag does not exist") metadata = await cluster_metadata(request) ceph = _ceph(metadata) flags = ceph.get("flags") - if not isinstance(flags, dict) or flag not in flags: - raise ApiError(404, "ceph flag does not exist") - return {flag: int(flags[flag])} + if isinstance(flags, dict) and flag in flags: + return bool(int(flags[flag])) + return False - async def ceph_flag_put(request: Request, inputs: dict[str, Any]) -> dict[str, int]: + async def ceph_flag_put(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) flag = str(payload["flag"]) + if flag not in CEPH_FLAG_DESCRIPTIONS: + raise ApiError(404, "ceph flag does not exist") metadata = await cluster_metadata(request) ceph = dict(_ceph(metadata)) - flags = dict(ceph.get("flags") or {}) + flags = default_ceph_flags() + stored = ceph.get("flags") + if isinstance(stored, dict): + for name, value in stored.items(): + if name in flags: + flags[name] = int(value) if "value" in payload: flags[flag] = int(payload["value"]) elif flag in payload: @@ -444,7 +503,6 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: ceph["flags"] = flags metadata["ceph"] = ceph await save_cluster_metadata(request, metadata) - return {flag: int(flags[flag])} async def ceph_metadata(request: Request, _inputs: dict[str, Any]) -> dict[str, Any]: metadata = await cluster_metadata(request) @@ -452,12 +510,77 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: config_raw = ceph.get("config") config: dict[str, Any] = dict(config_raw) if isinstance(config_raw, dict) else {} version = ceph.get("version") - flags = ceph.get("flags") + nodes = await database(request).pool.fetch("SELECT name FROM nodes ORDER BY name") + node_names = [str(row["name"]) for row in nodes] + version_str = ( + str(version.get("str")) + if isinstance(version, dict) and version.get("str") + else "18.2.2" + ) + version_short = ".".join(version_str.split(".")[:2]) if version_str else "18.2" + osd_rows = await database(request).pool.fetch( + """SELECT r.external_id, r.state, n.name AS node + FROM resources r JOIN nodes n ON n.id=r.node_id + WHERE r.kind='ceph-osd' ORDER BY r.external_id""" + ) + osd_map: dict[str, Any] = {} + for row in osd_rows: + payload = state(row["state"]) + external = str(row["external_id"]) + osd_id = str(payload.get("osd_id", external.removeprefix("osd."))) + hostname = str(row["node"]) + osd_map[osd_id] = { + "id": int(osd_id) if osd_id.isdigit() else osd_id, + "hostname": hostname, + "name": f"osd.{osd_id}", + "ceph_version": version_str, + "ceph_version_short": version_short, + "ceph_release": "reef", + "device_class": payload.get("device_class", "hdd"), + "dev": payload.get("dev", ""), + } + if not osd_map: + fallback_host = node_names[0] if node_names else "pve01" + osd_map["0"] = { + "id": 0, + "hostname": fallback_host, + "name": "osd.0", + "ceph_version": version_str, + "ceph_version_short": version_short, + "ceph_release": "reef", + } + + def _service_entry(name: str, index: int) -> dict[str, Any]: + return { + "name": name, + "hostname": name, + "addr": f"10.0.0.{index}", + "ceph_version": version_str, + "ceph_version_short": version_short, + "ceph_release": "reef", + } + return { - "version": dict(version) if isinstance(version, dict) else {}, + "mds": {}, + "mgr": { + f"mgr@{node}": _service_entry(node, index) + for index, node in enumerate(node_names[:1] or ["pve01"], start=1) + }, + "mon": { + f"mon@{node}": _service_entry(node, index) + for index, node in enumerate(node_names[:3] or ["pve01"], start=1) + }, + "node": { + node: { + "name": node, + "hostname": node, + "ceph_version": version_str, + "ceph_version_short": version_short, + } + for node in (node_names or ["pve01"]) + }, + "osd": list(osd_map.values()), "fsid": str(config.get("fsid") or ""), - "initialized": int(bool(ceph.get("initialized"))), - "flags": dict(flags) if isinstance(flags, dict) else {}, } async def ha_rule_create(request: Request, inputs: dict[str, Any]) -> None: diff --git a/app/handlers/common.py b/app/handlers/common.py index 31f8f0c..bd489c7 100644 --- a/app/handlers/common.py +++ b/app/handlers/common.py @@ -129,13 +129,24 @@ def storage_payload(row: Any) -> dict[str, Any]: total = int(row["capacity_bytes"] or 0) used = int(row["used_bytes"] or 0) avail = max(total - used, 0) + storage_type = str(row["storage_type"]) + formats = config.get("formats") + if not isinstance(formats, str) or not formats: + formats = { + "dir": "raw,qcow2,vmdk", + "lvmthin": "raw", + "rbd": "raw", + "zfspool": "raw,subvol", + }.get(storage_type, "raw") payload: dict[str, Any] = { "storage": str(row["storage_id"]), - "type": str(row["storage_type"]), + "type": storage_type, "shared": int(bool(row["shared"])), "content": content_str, - "active": 1, - "enabled": 1, + "active": int(bool(config.get("active", True))), + "enabled": int(bool(config.get("enabled", True))), + "formats": formats, + "select_existing": int(bool(config.get("select_existing", False))), "total": total, "used": used, "avail": avail, diff --git a/app/handlers/core.py b/app/handlers/core.py index 8192c2c..f61e849 100644 --- a/app/handlers/core.py +++ b/app/handlers/core.py @@ -19,7 +19,7 @@ from app.handlers.ceph import register_ceph_handlers from app.handlers.cluster import register_cluster_handlers from app.handlers.cluster_config import register_cluster_config_handlers from app.handlers.cluster_extra import register_cluster_extra_handlers -from app.handlers.common import require_node, subdirs +from app.handlers.common import cluster_metadata, require_node, subdirs from app.handlers.firewall import register_firewall_handlers from app.handlers.ha import register_ha_handlers from app.handlers.legacy_aliases import register_legacy_aliases @@ -60,10 +60,14 @@ def build_core_handlers(settings: Settings) -> HandlerRegistry: raise ApiError(401, "authentication failure") key = settings.ticket_signing_key.get_secret_value().encode() ticket = issue_ticket(username, key) + cluster_name = await _database(request).pool.fetchval( + "SELECT name FROM clusters ORDER BY created_at LIMIT 1" + ) return { "username": username, "ticket": ticket, "CSRFPreventionToken": csrf_token(ticket, key), + "clustername": str(cluster_name or "pve-simulator"), "cap": {"vms": {"VM.Audit": 1, "VM.PowerMgmt": 1}}, } @@ -81,7 +85,9 @@ def build_core_handlers(settings: Settings) -> HandlerRegistry: status_dict = dict(status_payload) if isinstance(status_payload, dict) else {} fingerprint = status_dict.get("ssl_fingerprint") or status_dict.get("fingerprint") if fingerprint in (None, "", 0) or isinstance(fingerprint, dict | list): - fingerprint = ":".join(["00"] * 32) + from app.simulation.seed import _seed_fingerprint + + fingerprint = _seed_fingerprint(name) def _as_float(value: object, default: float) -> float: if isinstance(value, bool) or value is None or isinstance(value, dict | list): @@ -126,10 +132,62 @@ def build_core_handlers(settings: Settings) -> HandlerRegistry: ops = await load_node_ops(request, node) status = ops.get("status") payload = dict(status) if isinstance(status, dict) else {} + mem_used = int(payload.get("mem") or 0) + mem_total = int(payload.get("maxmem") or 8 * 1024**3) + memory = payload.get("memory") + if isinstance(memory, dict): + mem_used = int(memory.get("used") or mem_used) + mem_total = int(memory.get("total") or mem_total) + rootfs = payload.get("rootfs") + if not isinstance(rootfs, dict): + rootfs = { + "used": int(payload.get("rootfs_used") or mem_used // 8), + "total": int(payload.get("rootfs_total") or 100 * 1024**3), + "avail": int(payload.get("rootfs_avail") or 80 * 1024**3), + "free": int(payload.get("rootfs_free") or 80 * 1024**3), + } + cpuinfo = payload.get("cpuinfo") + if not isinstance(cpuinfo, dict): + cpuinfo = { + "cpus": int(payload.get("maxcpu") or 4), + "cores": int(payload.get("maxcpu") or 4), + "sockets": 1, + "model": str(payload.get("cpu_model") or "QEMU Virtual CPU"), + "flags": "", + } + boot_info = payload.get("boot-info") + if not isinstance(boot_info, dict): + boot_info = {"mode": "efi", "secureboot": 0} + current_kernel = payload.get("current-kernel") + if not isinstance(current_kernel, dict): + kversion = str(payload.get("kversion") or "6.8.12-1-pve") + current_kernel = { + "sysname": "Linux", + "release": kversion, + "version": f"#1 SMP PREEMPT_DYNAMIC {kversion}", + "machine": "x86_64", + } return { - "status": str(row["status"]), - "node": str(row["name"]), - **payload, + "uptime": int(payload.get("uptime") or 0), + "wait": float(payload.get("wait") or 0.0), + "idle": float(payload.get("idle") or 0.95), + "cpu": float(payload.get("cpu") or 0.0), + "loadavg": list(payload.get("loadavg") or ["0.00", "0.00", "0.00"]), + "memory": { + "used": mem_used, + "total": mem_total, + "free": max(mem_total - mem_used, 0), + "available": max(mem_total - mem_used, 0), + }, + "rootfs": rootfs, + "swap": payload.get("swap") + if isinstance(payload.get("swap"), dict) + else {"used": 0, "total": 0, "free": 0}, + "cpuinfo": cpuinfo, + "kversion": str(payload.get("kversion") or current_kernel.get("release") or ""), + "pveversion": str(payload.get("pveversion") or "pve-manager/9.2.3"), + "current-kernel": current_kernel, + "boot-info": boot_info, } async def resources(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: @@ -197,6 +255,15 @@ def build_core_handlers(settings: Settings) -> HandlerRegistry: ops = await load_node_ops(request, name) status_payload = ops.get("status") status_dict = dict(status_payload) if isinstance(status_payload, dict) else {} + mem_raw = status_dict.get("mem", status_dict.get("memory")) + if isinstance(mem_raw, dict): + mem_val = int(_num(mem_raw.get("used"), 0)) + maxmem_val = int( + _num(status_dict.get("maxmem", mem_raw.get("total")), 8 * 1024**3) + ) + else: + mem_val = int(_num(mem_raw, 0)) + maxmem_val = int(_num(status_dict.get("maxmem"), 8 * 1024**3)) result.append( { "type": "node", @@ -205,10 +272,8 @@ def build_core_handlers(settings: Settings) -> HandlerRegistry: "status": str(row["status"]), "cpu": float(_num(status_dict.get("cpu"), 0.0)), "maxcpu": int(_num(status_dict.get("maxcpu"), 4)), - "mem": int( - _num(status_dict.get("mem"), _num(status_dict.get("memory"), 0)) - ), - "maxmem": int(_num(status_dict.get("maxmem"), 8 * 1024**3)), + "mem": mem_val, + "maxmem": maxmem_val, "uptime": int(_num(status_dict.get("uptime"), 0)), "level": str(status_dict.get("level") or ""), } @@ -220,12 +285,40 @@ def build_core_handlers(settings: Settings) -> HandlerRegistry: elif type_filter == "storage": kind_filter = ("storage",) elif type_filter in (None,): - kind_filter = ("qemu", "lxc", "storage") + kind_filter = ("qemu", "lxc", "storage", "pool") elif type_filter in {"qemu", "lxc", "storage", "pool", "sdn"}: kind_filter = (str(type_filter),) else: kind_filter = () + ha_by_guest: dict[str, str] = {} + if kind_filter and ("qemu" in kind_filter or "lxc" in kind_filter): + ha_rows = await _database(request).pool.fetch( + "SELECT external_id, state FROM resources WHERE kind='ha'" + ) + for ha_row in ha_rows: + sid = str(ha_row["external_id"]) + ha_state = _as_dict(ha_row["state"]) + ha_by_guest[sid] = str(ha_state.get("state") or "started") + + pool_by_vmid: dict[str, str] = {} + if kind_filter and ("qemu" in kind_filter or "lxc" in kind_filter or "pool" in kind_filter): + membership = await _database(request).pool.fetch( + """SELECT p.pool_id AS pool, r.external_id AS vmid + FROM pool_members pm + JOIN pools p ON p.id=pm.pool_id + JOIN resources r ON r.id=pm.resource_id""" + ) + for row in membership: + pool_by_vmid[str(row["vmid"])] = str(row["pool"]) + + node_names = [ + str(item["node"]) + for item in await _database(request).pool.fetch( + "SELECT name AS node FROM nodes ORDER BY name" + ) + ] + if kind_filter: rows = await _database(request).pool.fetch( """SELECT r.kind AS type, r.external_id, r.state, n.name AS node @@ -243,6 +336,7 @@ def build_core_handlers(settings: Settings) -> HandlerRegistry: status = str(state.get("status") or "stopped") running = status in {"running", "paused"} vmid = int(external_id) + sid = f"{'vm' if kind == 'qemu' else 'ct'}:{external_id}" item: dict[str, Any] = { "type": kind, "id": f"{kind}/{external_id}", @@ -250,7 +344,7 @@ def build_core_handlers(settings: Settings) -> HandlerRegistry: "vmid": vmid, "name": str(state.get("name") or f"{kind}-{external_id}"), "status": status, - "template": 1 if state.get("template") in {True, "1"} else 0, + "template": bool(state.get("template") in {True, "1", 1}), "cpu": _cpu_util(state, running=running), "maxcpu": _maxcpu(state), "mem": int(_num(state.get("mem"), 0)) if running else 0, @@ -259,27 +353,64 @@ def build_core_handlers(settings: Settings) -> HandlerRegistry: "maxdisk": int(_num(state.get("maxdisk"), 0)), "uptime": int(_num(state.get("uptime"), 0)) if running else 0, } + if sid in ha_by_guest: + item["hastate"] = ha_by_guest[sid] + if external_id in pool_by_vmid: + item["pool"] = pool_by_vmid[external_id] result.append(item) elif kind == "storage": content = state.get("content") if isinstance(content, list): - content_text = ",".join(str(item) for item in content) + content_text = ",".join(str(part) for part in content) else: content_text = str(content or "") + shared = bool(_num(state.get("shared"), 0)) + # Schema allows one storages row per storage_id; expand to every + # node like a real PVE dump (local + shared storages). + targets = node_names if (shared or len(node_names) > 1) else [node] + if not shared and external_id not in {"local", "local-lvm", "ceph"}: + targets = [node] + for target in targets: + result.append( + { + "type": "storage", + "id": f"storage/{target}/{external_id}", + "node": target, + "storage": external_id, + "status": str(state.get("status") or "available"), + "content": content_text, + "disk": int(_num(state.get("disk"), 0)), + "maxdisk": int(_num(state.get("maxdisk"), 1 * 1024**3)), + "shared": shared, + "plugintype": str( + state.get("plugintype") or state.get("type") or "dir" + ), + } + ) + elif kind == "pool": result.append( { - "type": "storage", - "id": f"storage/{node}/{external_id}", - "node": node, - "storage": external_id, - "status": str(state.get("status") or "available"), - "content": content_text, - "disk": int(_num(state.get("disk"), 0)), - "maxdisk": int(_num(state.get("maxdisk"), 1 * 1024**3)), - "shared": int(_num(state.get("shared"), 0)), - "plugintype": str( - state.get("plugintype") or state.get("type") or "dir" - ), + "type": "pool", + "id": f"pool/{external_id}", + "pool": external_id, + "comment": str(state.get("comment") or ""), + } + ) + + if type_filter in (None, "sdn"): + metadata = await cluster_metadata(request) + sdn = metadata.get("sdn") + zones = sdn.get("zones") if isinstance(sdn, dict) else None + if isinstance(zones, dict): + for zone_name, zone in sorted(zones.items()): + if not isinstance(zone, dict): + continue + result.append( + { + "type": "sdn", + "id": f"sdn/{zone_name}", + "sdn": zone_name, + "status": str(zone.get("status") or "ok"), } ) return result diff --git a/app/handlers/firewall.py b/app/handlers/firewall.py index 69649d1..ee73d47 100644 --- a/app/handlers/firewall.py +++ b/app/handlers/firewall.py @@ -112,7 +112,7 @@ def register_firewall_handlers(registry: HandlerRegistry) -> None: options = section.get("options", {}) return dict(options) if isinstance(options, dict) else {} - async def options_put(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: + async def options_put(request: Request, inputs: dict[str, Any]) -> None: payload = await _ready(request, inputs) firewall = await _load_firewall(request) section = _ensure_scope(firewall, scope_fn(payload)) @@ -123,7 +123,6 @@ def register_firewall_handlers(registry: HandlerRegistry) -> None: current[key] = value section["options"] = current await _save_firewall(request, firewall) - return current async def rules_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: payload = await _ready(request, inputs) diff --git a/app/handlers/ha.py b/app/handlers/ha.py index 28ef7ce..7108196 100644 --- a/app/handlers/ha.py +++ b/app/handlers/ha.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json from typing import Any @@ -27,6 +28,13 @@ def _ha_groups(metadata: dict[str, Any]) -> dict[str, dict[str, Any]]: return {str(key): dict(value) for key, value in groups.items() if isinstance(value, dict)} +def _ha_digest(payload: dict[str, Any]) -> str: + material = {key: value for key, value in payload.items() if key != "digest"} + return hashlib.sha1( + json.dumps(material, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + def register_ha_handlers(registry: HandlerRegistry) -> None: async def ha_index(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, str]]: return subdirs("groups", "resources", "rules", "status") @@ -41,17 +49,17 @@ def register_ha_handlers(registry: HandlerRegistry) -> None: for row in rows: payload = state(row["state"]) sid = str(row["external_id"]) - result.append( - { - "sid": sid, - "type": "vm" if sid.startswith("vm:") else "ct", - "state": payload.get("state", "started"), - "group": payload.get("group"), - "node": str(row["node"]), - "max_relocate": payload.get("max_relocate", 1), - "max_restart": payload.get("max_restart", 1), - } - ) + entry = { + "sid": sid, + "type": "vm" if sid.startswith("vm:") else "ct", + "state": payload.get("state", "started"), + "group": payload.get("group"), + "node": str(row["node"]), + "max_relocate": payload.get("max_relocate", 1), + "max_restart": payload.get("max_restart", 1), + } + entry["digest"] = _ha_digest(entry) + result.append(entry) return result async def ha_resource_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: @@ -208,29 +216,75 @@ def register_ha_handlers(registry: HandlerRegistry) -> None: async def ha_status(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, str]]: return subdirs("current", "manager_status") - async def ha_status_current(request: Request, _inputs: dict[str, Any]) -> dict[str, Any]: - row = await database(request).pool.fetchrow( - """SELECT count(*) FILTER (WHERE state->>'state' = 'started') AS started, - count(*) AS total - FROM resources WHERE kind='ha'""" - ) + async def ha_status_current(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: master = await database(request).pool.fetchval( "SELECT name FROM nodes WHERE status='online' ORDER BY name LIMIT 1" ) metadata = await cluster_metadata(request) ha_raw = metadata.get("ha") ha: dict[str, Any] = dict(ha_raw) if isinstance(ha_raw, dict) else {} + armed = bool(ha.get("armed")) if "armed" in ha else False status_raw = ha.get("status_current") status: dict[str, Any] = dict(status_raw) if isinstance(status_raw, dict) else {} - armed = bool(ha.get("armed")) if "armed" in ha else False - return { - "quorate": status.get("quorate", metadata.get("quorate", 0)), - "mode": status.get("mode", "active" if armed else "disabled"), - "master_node": status.get("master_node", str(master or "")), - "ha_started": int(row["started"] or 0), - "ha_total": int(row["total"] or 0), - "armed": 1 if armed else 0, - } + quorate = bool(int(status.get("quorate", metadata.get("quorate", 1)) or 1)) + master_node = str(status.get("master_node") or master or "") + rows = await database(request).pool.fetch( + """SELECT r.external_id, r.state, n.name AS node + FROM resources r JOIN nodes n ON n.id=r.node_id + WHERE r.kind='ha' ORDER BY r.external_id""" + ) + nodes = await database(request).pool.fetch("SELECT name FROM nodes ORDER BY name") + result: list[dict[str, Any]] = [ + { + "id": "quorum", + "type": "quorum", + "node": master_node, + "status": "OK" if quorate else "offline", + "quorate": quorate, + }, + { + "id": "master", + "type": "master", + "node": master_node, + "status": "active" if armed else "disabled", + }, + { + "id": "fencing", + "type": "fencing", + "node": master_node, + "status": "armed" if armed else "disarmed", + "armed-state": "armed" if armed else "disarmed", + }, + ] + for node_row in nodes: + node_name = str(node_row["name"]) + result.append( + { + "id": f"lrm:{node_name}", + "type": "lrm", + "node": node_name, + "status": "active" if quorate else "offline", + } + ) + for row in rows: + payload = state(row["state"]) + sid = str(row["external_id"]) + crm = str(payload.get("state") or "started") + result.append( + { + "id": f"service:{sid}", + "type": "service", + "sid": sid, + "node": str(row["node"]), + "crm_state": crm, + "state": crm, + "status": crm, + "request_state": str(payload.get("request_state") or crm), + "max_relocate": int(payload.get("max_relocate") or 1), + "max_restart": int(payload.get("max_restart") or 1), + } + ) + return result async def ha_manager_status(request: Request, _inputs: dict[str, Any]) -> dict[str, Any]: metadata = await cluster_metadata(request) @@ -252,7 +306,7 @@ def register_ha_handlers(registry: HandlerRegistry) -> None: return [dict(item) for item in rules if isinstance(item, dict)] return [] - async def ha_relocate(request: Request, inputs: dict[str, Any]) -> None: + async def ha_relocate(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: payload = values(inputs) sid = str(payload["sid"]) target = str(payload.get("node") or payload.get("target") or "") @@ -292,9 +346,14 @@ def register_ha_handlers(registry: HandlerRegistry) -> None: ha_row["id"], json.dumps(current, sort_keys=True), ) + return { + "sid": sid, + "requested-node": target, + "comigrated-resources": [], + } - async def ha_migrate(request: Request, inputs: dict[str, Any]) -> None: - await ha_relocate(request, inputs) + async def ha_migrate(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: + return await ha_relocate(request, inputs) async def ha_arm(request: Request, _inputs: dict[str, Any]) -> None: metadata = await cluster_metadata(request) diff --git a/app/handlers/legacy_aliases.py b/app/handlers/legacy_aliases.py index 0da8164..76c6554 100644 --- a/app/handlers/legacy_aliases.py +++ b/app/handlers/legacy_aliases.py @@ -19,7 +19,7 @@ def register_legacy_aliases(registry: HandlerRegistry) -> None: alias("/access/tfa", "POST", "/access/tfa/{userid}", "POST") alias("/access/tfa", "PUT", "/access/tfa/{userid}/{id}", "PUT") - alias("/cluster/backupinfo", "GET", "/cluster/backup-info", "GET") + # /cluster/backupinfo is registered as a PVE 6 stub string in backup.py alias( "/cluster/backupinfo/not_backed_up", "GET", diff --git a/app/handlers/lxc.py b/app/handlers/lxc.py index 79bf824..9d17bb8 100644 --- a/app/handlers/lxc.py +++ b/app/handlers/lxc.py @@ -38,27 +38,120 @@ def _state(value: object) -> dict[str, Any]: return dict(cast(Mapping[str, Any], value)) +# Runtime-only keys stored in ``resources.state``; never echo on GET /config. +_LXC_CONFIG_INTERNAL = frozenset( + { + "agent", + "cloudinit_dump", + "config", + "consoles", + "interfaces", + "migrate_preconditions", + "name", + "rrd", + "rrddata", + "status", + "volume_moves", + } +) + + +def _public_lxc_config( + config: dict[str, Any], state: dict[str, Any], *, vmid: int +) -> dict[str, Any]: + import hashlib + + merged = {**state, **config} + payload: dict[str, Any] = {"vmid": vmid} + for key, value in merged.items(): + if key in _LXC_CONFIG_INTERNAL: + continue + if isinstance(value, dict | list): + continue + payload[key] = value + digest_src = {key: value for key, value in payload.items() if key != "digest"} + payload["digest"] = hashlib.sha1( + json.dumps(digest_src, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + return payload + + +def _lxc_index_item(vmid: int, vm_state: dict[str, Any], config: dict[str, Any]) -> dict[str, Any]: + status = str(vm_state.get("status", "stopped")) + running = status in {"running", "paused"} + memory_mb = int(config.get("memory", config.get("mem", 512))) + maxmem = memory_mb * 2**20 + return { + "vmid": vmid, + "name": str(config.get("hostname", config.get("name", f"ct-{vmid}"))), + "status": status, + "cpus": int(config.get("cores", config.get("cpus", 1))), + "cpu": float(vm_state.get("cpu", 0.05 if running else 0.0)), + "maxmem": maxmem, + "mem": int(vm_state.get("mem", maxmem // 2 if running else 0)), + "maxdisk": int(vm_state.get("maxdisk", config.get("maxdisk", 8 * 1024**3))), + "disk": int(vm_state.get("disk", 0)), + "diskread": int(vm_state.get("diskread", 0)), + "diskwrite": int(vm_state.get("diskwrite", 0)), + "netin": int(vm_state.get("netin", 0)), + "netout": int(vm_state.get("netout", 0)), + "maxswap": int(config.get("swap", 512)) * 2**20, + "uptime": int(vm_state.get("uptime", 0)) if running else 0, + "lock": str(vm_state.get("lock", "")), + "tags": str(config.get("tags", "")), + "template": int(bool(vm_state.get("template", False))), + "pressurecpusome": float(vm_state.get("pressurecpusome", 0.0)), + "pressurecpufull": float(vm_state.get("pressurecpufull", 0.0)), + "pressureiosome": float(vm_state.get("pressureiosome", 0.0)), + "pressureiofull": float(vm_state.get("pressureiofull", 0.0)), + "pressurememorysome": float(vm_state.get("pressurememorysome", 0.0)), + "pressurememoryfull": float(vm_state.get("pressurememoryfull", 0.0)), + } + + def register_lxc_handlers(registry: HandlerRegistry) -> None: async def lxc_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: node = str(_values(inputs)["node"]) rows = await _database(request).pool.fetch( - """SELECT r.external_id::integer AS vmid, r.state - FROM resources r JOIN nodes n ON n.id=r.node_id + """SELECT r.external_id::integer AS vmid, r.state, c.config + FROM resources r + JOIN nodes n ON n.id=r.node_id + JOIN containers c ON c.resource_id=r.id WHERE n.name=$1 AND r.kind='lxc' ORDER BY r.external_id::integer""", node, ) - return [{"vmid": int(row["vmid"]), **_state(row["state"])} for row in rows] + return [ + _lxc_index_item(int(row["vmid"]), _state(row["state"]), _state(row["config"])) + for row in rows + ] async def lxc_config(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: node, vmid = str(_values(inputs)["node"]), str(_values(inputs)["vmid"]) row = await _lxc_resource(request, node, vmid) - return {"vmid": int(vmid), **_state(row["config"]), **_state(row["state"])} + return _public_lxc_config(_state(row["config"]), _state(row["state"]), vmid=int(vmid)) async def lxc_current(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: - return await lxc_config(request, inputs) + payload = _values(inputs) + node, vmid = str(payload["node"]), int(payload["vmid"]) + row = await _lxc_resource(request, node, str(vmid)) + vm_state = _state(row["state"]) + config = _state(row["config"]) + item = _lxc_index_item(vmid, vm_state, config) + item["ha"] = {"managed": int(vm_state.get("ha_managed", 0))} + return item - async def lxc_status(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: - return await lxc_config(request, inputs) + async def lxc_status(request: Request, inputs: dict[str, Any]) -> list[dict[str, str]]: + payload = _values(inputs) + await _lxc_resource(request, str(payload["node"]), str(payload["vmid"])) + return subdirs( + "current", + "reboot", + "resume", + "shutdown", + "start", + "stop", + "suspend", + ) async def mutate(operation: str, request: Request, inputs: dict[str, Any]) -> str: values = _values(inputs) @@ -421,7 +514,7 @@ def register_lxc_handlers(registry: HandlerRegistry) -> None: } } - async def lxc_resize(request: Request, inputs: dict[str, Any]) -> None: + async def lxc_resize(request: Request, inputs: dict[str, Any]) -> str: payload = _values(inputs) node, vmid = str(payload["node"]), str(payload["vmid"]) disk = str(payload.get("disk") or "rootfs") @@ -446,6 +539,13 @@ def register_lxc_handlers(registry: HandlerRegistry) -> None: resource["id"], json.dumps({disk: config[disk]}, sort_keys=True), ) + return await _create_task( + request, + node=node, + vmid=vmid, + task_type="lxc-resize", + payload={"resource_id": str(resource["id"]), "disk": disk, "size": size}, + ) async def lxc_template(request: Request, inputs: dict[str, Any]) -> None: payload = _values(inputs) @@ -537,6 +637,8 @@ async def _create_task( "lxc-clone": "pctclone", "lxc-migrate": "pctmigrate", "lxc-remote-migrate": "pctremote", + "lxc-resize": "pctresize", + "lxc-move-volume": "pctmove", }[task_type] upid = str(Upid.allocate(node, worker_type, vmid, str(request.state.principal))) try: diff --git a/app/handlers/lxc_extra.py b/app/handlers/lxc_extra.py index ed66e84..3eeecba 100644 --- a/app/handlers/lxc_extra.py +++ b/app/handlers/lxc_extra.py @@ -43,8 +43,11 @@ def register_lxc_extra_handlers(registry: HandlerRegistry) -> None: return [dict(item) for item in ifaces] if isinstance(ifaces, list) else [] async def move_volume(request: Request, inputs: dict[str, Any]) -> str: + from app.handlers.lxc import _create_task + values = _values(inputs) - resource = await _lxc_resource(request, str(values["node"]), str(values["vmid"])) + node, vmid = str(values["node"]), str(values["vmid"]) + resource = await _lxc_resource(request, node, vmid) volume = str(values.get("volume") or values.get("disk") or "rootfs") storage = str(values.get("storage") or "local-lvm") config = _state(resource["config"]) @@ -60,7 +63,17 @@ def register_lxc_extra_handlers(registry: HandlerRegistry) -> None: moves = state["volume_moves"] = [] moves.append({"volume": volume, "storage": storage}) await _save_state(request, resource["id"], state) - return f"UPID:{values['node']}:lxc-move-volume:{values['vmid']}" + return await _create_task( + request, + node=node, + vmid=vmid, + task_type="lxc-move-volume", + payload={ + "resource_id": str(resource["id"]), + "volume": volume, + "storage": storage, + }, + ) async def rrd(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: values = _values(inputs) diff --git a/app/handlers/nodes.py b/app/handlers/nodes.py index f9d6cb1..ea7796d 100644 --- a/app/handlers/nodes.py +++ b/app/handlers/nodes.py @@ -43,13 +43,59 @@ def register_node_ops_handlers(registry: HandlerRegistry) -> None: node = str(values(inputs)["node"]) ops = await load_node_ops(request, node) packages = ops.get("apt", {}).get("packages", []) - return list(packages) if isinstance(packages, list) else [] + if not isinstance(packages, list): + return [] + result: list[dict[str, Any]] = [] + for item in packages: + if not isinstance(item, dict): + continue + entry = dict(item) + status = str(entry.pop("Status", entry.get("CurrentState", "Installed"))) + if status.lower() == "upgradable": + entry["CurrentState"] = "Installed" + entry.setdefault("OldVersion", entry.get("Version", "")) + else: + entry["CurrentState"] = "Installed" if status.lower() == "installed" else status + entry.setdefault("Package", entry.get("Package") or "unknown") + entry.setdefault("Version", entry.get("Version") or "") + entry.setdefault("Arch", "amd64") + entry.setdefault("Origin", "Proxmox") + result.append(entry) + return result - async def apt_repositories(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: + async def apt_repositories(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: node = str(values(inputs)["node"]) ops = await load_node_ops(request, node) - repositories = ops.get("apt", {}).get("repositories", []) - return list(repositories) if isinstance(repositories, list) else [] + repositories = ops.get("apt", {}).get("repositories") + if isinstance(repositories, dict) and "files" in repositories: + return { + "digest": str(repositories.get("digest") or "seedapt"), + "files": list(repositories.get("files") or []), + "errors": list(repositories.get("errors") or []), + "infos": list(repositories.get("infos") or []), + "standard-repos": list(repositories.get("standard-repos") or []), + } + legacy = list(repositories) if isinstance(repositories, list) else [] + return { + "digest": "seedapt", + "files": [ + { + "path": "/etc/apt/sources.list.d/pve-enterprise.list", + "file-type": "list", + "digest": "seedapt-file0", + "repositories": legacy, + } + ], + "errors": [], + "infos": [], + "standard-repos": [ + { + "handle": "no-subscription", + "name": "Proxmox VE No-Subscription Repository", + "status": 1, + } + ], + } async def apt_changelog(request: Request, inputs: dict[str, Any]) -> str: node = str(values(inputs)["node"]) @@ -89,7 +135,7 @@ def register_node_ops_handlers(registry: HandlerRegistry) -> None: return item raise ApiError(404, "interface does not exist") - async def network_mutate(request: Request, inputs: dict[str, Any]) -> None: + async def network_mutate(request: Request, inputs: dict[str, Any]) -> str | None: node = str(values(inputs)["node"]) payload = values(inputs) ops = await load_node_ops(request, node) @@ -140,8 +186,14 @@ def register_node_ops_handlers(registry: HandlerRegistry) -> None: raise ApiError(404, "interface does not exist") ops["network"] = updated else: + # PUT /nodes/{node}/network without iface → reload (returns UPID string). ops["network_applied"] = True + await save_node_ops(request, node, ops) + return await _node_task( + request, node=node, task_type="network-reload", worker="srvreload" + ) await save_node_ops(request, node, ops) + return None async def disks_index(request: Request, inputs: dict[str, Any]) -> list[dict[str, str]]: await require_node(request, str(values(inputs)["node"])) @@ -189,7 +241,7 @@ def register_node_ops_handlers(registry: HandlerRegistry) -> None: async def disks_zfs(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: return await disks_collection(request, inputs, "zfs") - async def disks_initgpt(request: Request, inputs: dict[str, Any]) -> None: + async def disks_initgpt(request: Request, inputs: dict[str, Any]) -> str: node = str(values(inputs)["node"]) disk = str(values(inputs).get("disk") or values(inputs).get("device") or "") if not disk: @@ -219,8 +271,9 @@ def register_node_ops_handlers(registry: HandlerRegistry) -> None: disks["list"] = items ops["disks"] = disks await save_node_ops(request, node, ops) + return await _node_task(request, node=node, task_type="disk-initgpt", worker="diskinit") - async def disks_wipedisk(request: Request, inputs: dict[str, Any]) -> None: + async def disks_wipedisk(request: Request, inputs: dict[str, Any]) -> str: node = str(values(inputs)["node"]) disk = str(values(inputs).get("disk") or values(inputs).get("device") or "") if not disk: @@ -243,14 +296,30 @@ def register_node_ops_handlers(registry: HandlerRegistry) -> None: smart.pop(disk, None) ops["disks"] = disks await save_node_ops(request, node, ops) + return await _node_task(request, node=node, task_type="disk-wipe", worker="diskwipe") - async def services_index(request: Request, inputs: dict[str, Any]) -> list[dict[str, str]]: + async def services_index(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: node = str(values(inputs)["node"]) ops = await load_node_ops(request, node) services = ops.get("services") or {} - return [{"subdir": name} for name in sorted(services)] + result: list[dict[str, Any]] = [] + for name in sorted(services): + payload = dict(services[name]) if isinstance(services[name], dict) else {} + state_name = str(payload.get("state", "stopped")) + enabled = bool(int(payload.get("enabled", 0) or 0)) + result.append( + { + "service": name, + "name": f"{name}.service", + "desc": str(payload.get("desc") or name), + "state": state_name, + "active-state": "active" if state_name == "running" else "inactive", + "unit-state": "enabled" if enabled else "disabled", + } + ) + return result - async def service_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: + async def service_get(request: Request, inputs: dict[str, Any]) -> list[dict[str, str]]: node = str(values(inputs)["node"]) service = str(values(inputs)["service"]) ops = await load_node_ops(request, node) @@ -258,12 +327,25 @@ def register_node_ops_handlers(registry: HandlerRegistry) -> None: if service not in services: services[service] = {"state": "stopped", "enabled": 0} await save_node_ops(request, node, ops) - payload = dict(services[service]) - payload["service"] = service - return payload + return subdirs("state", "start", "stop", "restart", "reload") async def service_state(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: - return await service_get(request, inputs) + node = str(values(inputs)["node"]) + service = str(values(inputs)["service"]) + ops = await load_node_ops(request, node) + services = ops.setdefault("services", {}) + if service not in services: + services[service] = {"state": "stopped", "enabled": 0} + await save_node_ops(request, node, ops) + payload = dict(services[service]) if isinstance(services[service], dict) else {} + state_name = str(payload.get("state", "stopped")) + enabled = bool(int(payload.get("enabled", 0) or 0)) + return { + "service": service, + "state": state_name, + "active-state": "active" if state_name == "running" else "inactive", + "unit-state": "enabled" if enabled else "disabled", + } async def service_action(request: Request, inputs: dict[str, Any]) -> str: node = str(values(inputs)["node"]) @@ -286,7 +368,12 @@ def register_node_ops_handlers(registry: HandlerRegistry) -> None: services[service] = current ops["services"] = services await save_node_ops(request, node, ops) - return "OK" + return await _node_task( + request, + node=node, + task_type=f"service-{action}", + worker=f"srv{action}", + ) registry.register("/nodes/{node}/apt", "GET", apt_index) registry.register("/nodes/{node}/apt/versions", "GET", apt_versions) @@ -319,6 +406,8 @@ def register_node_ops_handlers(registry: HandlerRegistry) -> None: async def _node_task(request: Request, *, node: str, task_type: str, worker: str) -> str: + import secrets + from app.api.errors import ApiError from app.db.primitives import ConflictError @@ -329,7 +418,7 @@ async def _node_task(request: Request, *, node: str, task_type: str, worker: str upid=upid, task_type=task_type, payload={"node": node}, - resource_key=f"node:{node}", + resource_key=f"node:{node}:{task_type}:{secrets.token_hex(4)}", ) except ConflictError as error: raise ApiError(409, str(error)) from error diff --git a/app/handlers/nodes_extra.py b/app/handlers/nodes_extra.py index 1323d9e..24e104b 100644 --- a/app/handlers/nodes_extra.py +++ b/app/handlers/nodes_extra.py @@ -108,7 +108,9 @@ async def _migrate_guests(request: Request, node: str, target: str) -> None: def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: - async def disks_create(request: Request, inputs: dict[str, Any], kind: str) -> dict[str, Any]: + async def disks_create(request: Request, inputs: dict[str, Any], kind: str) -> str: + from app.handlers.nodes import _node_task + payload = values(inputs) node = str(payload["node"]) await require_node(request, node) @@ -134,9 +136,13 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: ops["disks"] = disks disks[kind] = items await save_node_ops(request, node, ops) - return entry + return await _node_task( + request, node=node, task_type=f"disk-{kind}-create", worker=f"disk{kind}" + ) + + async def disks_delete(request: Request, inputs: dict[str, Any], kind: str) -> str: + from app.handlers.nodes import _node_task - async def disks_delete(request: Request, inputs: dict[str, Any], kind: str) -> None: payload = values(inputs) node = str(payload["node"]) name = str(payload["name"]) @@ -148,6 +154,9 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: raise ApiError(404, f"{kind} does not exist") ops.setdefault("disks", {})[kind] = remaining await save_node_ops(request, node, ops) + return await _node_task( + request, node=node, task_type=f"disk-{kind}-delete", worker=f"diskdel{kind}" + ) async def disks_zfs_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: payload = values(inputs) @@ -194,11 +203,11 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: certs["acme"] = acme ops["certificates"] = certs await save_node_ops(request, node, ops) - if method == "DELETE": - return None return await _node_task(request, node=node, task_type="acme", worker="acme") - async def certificates_custom(request: Request, inputs: dict[str, Any]) -> None: + async def certificates_custom( + request: Request, inputs: dict[str, Any] + ) -> dict[str, Any] | None: payload = values(inputs) node = str(payload["node"]) await require_node(request, node) @@ -206,35 +215,40 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: certs = _certificates(ops) if request.method.upper() == "DELETE": certs["custom"] = None - else: - certificates = str(payload.get("certificates") or payload.get("cert") or "") - if not certificates: - raise ApiError(400, "parameter verification failed - 'certificates' missing") - key = str(payload.get("key") or payload.get("private-key") or "") - certs["custom"] = { - "certificates": certificates, - "key": key, - "restart": int(payload.get("restart") or 0), - "filename": str(payload.get("filename") or "pveproxy-ssl.pem"), - } - info = list(certs.get("info") or []) - info = [item for item in info if item.get("filename") != certs["custom"]["filename"]] - info.append( - { - "filename": certs["custom"]["filename"], - "fingerprint": secrets.token_hex(20), - "issuer": "CN=Simulator", - "subject": "CN=pve.local", - "notbefore": int(time.time()) - 86_400, - "notafter": int(time.time()) + 365 * 86_400, - "san": ["DNS:pve.local"], - "public-key-type": "rsa", - "public-key-bits": 2048, - } - ) - certs["info"] = info + ops["certificates"] = certs + await save_node_ops(request, node, ops) + return None + certificates = str(payload.get("certificates") or payload.get("cert") or "") + if not certificates: + raise ApiError(400, "parameter verification failed - 'certificates' missing") + key = str(payload.get("key") or payload.get("private-key") or "") + filename = str(payload.get("filename") or "pveproxy-ssl.pem") + fingerprint = ":".join(secrets.token_hex(1) for _ in range(32)) # SHA-256 fingerprint style + summary = { + "filename": filename, + "fingerprint": fingerprint, + "issuer": "CN=Simulator", + "subject": "CN=pve.local", + "notbefore": int(time.time()) - 86_400, + "notafter": int(time.time()) + 365 * 86_400, + "san": ["DNS:pve.local"], + "public-key-type": "rsa", + "public-key-bits": 2048, + "pem": certificates, + } + certs["custom"] = { + "certificates": certificates, + "key": key, + "restart": int(payload.get("restart") or 0), + "filename": filename, + } + info = list(certs.get("info") or []) + info = [item for item in info if item.get("filename") != filename] + info.append({key: value for key, value in summary.items() if key != "pem"}) + certs["info"] = info ops["certificates"] = certs await save_node_ops(request, node, ops) + return summary async def certificates_info(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: node = str(values(inputs)["node"]) @@ -257,7 +271,39 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: certs["info"] = info ops["certificates"] = certs await save_node_ops(request, node, ops) - return [dict(item) for item in info] + if not info: + from app.simulation.seed import _seed_fingerprint + + info = [ + { + "filename": "pve-ssl.pem", + "fingerprint": _seed_fingerprint(node), + "issuer": f"CN={node}.proxmox.local", + "subject": f"CN={node}.proxmox.local", + "notbefore": 1_700_000_000, + "notafter": 2_000_000_000, + "pem": ( + "-----BEGIN CERTIFICATE-----\n" + "SIMULATOR-SEED-CERT\n" + "-----END CERTIFICATE-----\n" + ), + "san": [f"{node}.proxmox.local"], + "public-key-type": "rsa", + "public-key-bits": 4096, + } + ] + certs["info"] = info + ops["certificates"] = certs + await save_node_ops(request, node, ops) + result: list[dict[str, Any]] = [] + for item in info: + entry = dict(item) + entry.setdefault("pem", "-----BEGIN CERTIFICATE-----\nSIM\n-----END CERTIFICATE-----\n") + entry.setdefault("san", [str(entry.get("subject", "CN=pve.local")).removeprefix("CN=")]) + entry.setdefault("public-key-type", "rsa") + entry.setdefault("public-key-bits", 4096) + result.append(entry) + return result async def scan_index(request: Request, inputs: dict[str, Any]) -> list[dict[str, str]]: await require_node(request, str(values(inputs)["node"])) @@ -437,44 +483,136 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: if not isinstance(apt, dict): apt = {} ops["apt"] = apt - repositories = list(apt.get("repositories") or []) + envelope = apt.get("repositories") + if isinstance(envelope, dict) and "files" in envelope: + files = [ + dict(item) for item in list(envelope.get("files") or []) if isinstance(item, dict) + ] + standard = [ + dict(item) + for item in list(envelope.get("standard-repos") or []) + if isinstance(item, dict) + ] + else: + legacy = list(envelope) if isinstance(envelope, list) else [] + files = [ + { + "path": "/etc/apt/sources.list.d/pve-enterprise.list", + "file-type": "list", + "digest": "seedapt-file0", + "repositories": legacy, + } + ] + standard = [] method = request.method.upper() - if method == "POST": - entry = { + handle = payload.get("handle") + if handle is not None: + updated_standard: list[dict[str, Any]] = [] + found = False + for item in standard: + if item.get("handle") == handle: + found = True + merged = dict(item) + if "enabled" in payload: + merged["status"] = int(bool(payload.get("enabled"))) + updated_standard.append(merged) + else: + updated_standard.append(item) + if not found: + updated_standard.append( + { + "handle": str(handle), + "name": str(payload.get("name") or handle), + "status": int(bool(payload.get("enabled", 1))), + } + ) + standard = updated_standard + elif method == "POST": + repo_entry = { key: value for key, value in payload.items() - if key not in {"node", "delete", "digest"} + if key not in {"node", "delete", "digest", "path", "handle", "index", "file-type"} } - entry.setdefault("path", f"/etc/apt/sources.list.d/sim-{secrets.token_hex(2)}.list") - entry.setdefault("enabled", 1) - repositories.append(entry) + path = str( + payload.get("path") or f"/etc/apt/sources.list.d/sim-{secrets.token_hex(2)}.list" + ) + matched = False + for file_entry in files: + if file_entry.get("path") == path: + repos = list(file_entry.get("repositories") or []) + repos.append(repo_entry) + file_entry["repositories"] = repos + matched = True + break + if not matched: + files.append( + { + "path": path, + "file-type": str(payload.get("file-type") or "list"), + "digest": secrets.token_hex(4), + "repositories": [repo_entry], + } + ) else: path = payload.get("path") - handle = payload.get("handle") index = payload.get("index") - updated: list[dict[str, Any]] = [] - for idx, item in enumerate(repositories): - match = False - if path is not None and item.get("path") == path: - match = True - if handle is not None and item.get("handle") == handle: - match = True - if index is not None and idx == int(index): - match = True - if match or (path is None and handle is None and index is None and idx == 0): - merged = { - **item, - **{ - key: value - for key, value in payload.items() - if key not in {"node", "delete", "digest", "path", "handle", "index"} - }, - } - updated.append(merged) - else: - updated.append(item) - repositories = updated - apt["repositories"] = repositories + for file_idx, file_entry in enumerate(files): + if path is not None and file_entry.get("path") != path: + continue + repos = [ + dict(item) + for item in list(file_entry.get("repositories") or []) + if isinstance(item, dict) + ] + if index is not None: + idx = int(index) + if 0 <= idx < len(repos): + repos[idx] = { + **repos[idx], + **{ + key: value + for key, value in payload.items() + if key + not in { + "node", + "delete", + "digest", + "path", + "handle", + "index", + "file-type", + } + }, + } + elif path is not None or file_idx == 0: + if repos: + repos[0] = { + **repos[0], + **{ + key: value + for key, value in payload.items() + if key + not in { + "node", + "delete", + "digest", + "path", + "handle", + "index", + "file-type", + } + }, + } + file_entry["repositories"] = repos + if path is not None: + break + apt["repositories"] = { + "digest": secrets.token_hex(4), + "errors": [], + "infos": [], + "standard-repos": standard, + "files": files, + } ops["apt"] = apt await save_node_ops(request, node, ops) diff --git a/app/handlers/pools.py b/app/handlers/pools.py index 4863f5d..b3b0093 100644 --- a/app/handlers/pools.py +++ b/app/handlers/pools.py @@ -14,14 +14,31 @@ from app.handlers.common import database, require_value, state, values from app.simulation.seed import CLUSTER_ID, stable_id -async def _pool_members(request: Request, pool_id: uuid.UUID) -> list[str]: +async def _pool_members(request: Request, pool_id: uuid.UUID) -> list[dict[str, Any]]: rows = await database(request).pool.fetch( - """SELECT r.external_id FROM pool_members pm + """SELECT r.external_id, r.kind, r.state, n.name AS node + FROM pool_members pm JOIN resources r ON r.id = pm.resource_id - WHERE pm.pool_id=$1 ORDER BY r.external_id::integer""", + JOIN nodes n ON n.id = r.node_id + WHERE pm.pool_id=$1 + ORDER BY r.external_id::integer""", pool_id, ) - return [str(row["external_id"]) for row in rows] + members: list[dict[str, Any]] = [] + for row in rows: + guest_state = state(row["state"]) + kind = str(row["kind"]) + vmid = int(row["external_id"]) + members.append( + { + "id": f"{kind}/{vmid}", + "node": str(row["node"]), + "type": kind, + "vmid": vmid, + "name": str(guest_state.get("name") or f"{kind}-{vmid}"), + } + ) + return members def register_pool_handlers(registry: HandlerRegistry) -> None: @@ -39,16 +56,25 @@ def register_pool_handlers(registry: HandlerRegistry) -> None: metadata = state(row["metadata"]) members = await _pool_members(request, row["id"]) if not members and isinstance(metadata.get("members"), list): - members = [str(item) for item in metadata["members"]] - item: dict[str, Any] = { + # Fallback for metadata-only membership before pool_members rows exist. + for item in metadata["members"]: + members.append( + { + "id": f"qemu/{item}", + "node": "", + "type": "qemu", + "vmid": int(item) if str(item).isdigit() else 0, + } + ) + pool_item: dict[str, Any] = { "poolid": str(row["pool_id"]), "members": members, } if row["comment"] is not None: - item["comment"] = str(row["comment"]) + pool_item["comment"] = str(row["comment"]) elif metadata.get("comment"): - item["comment"] = str(metadata["comment"]) - result.append(item) + pool_item["comment"] = str(metadata["comment"]) + result.append(pool_item) return result async def pool_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: diff --git a/app/handlers/qemu.py b/app/handlers/qemu.py index 8b39090..b26e0d8 100644 --- a/app/handlers/qemu.py +++ b/app/handlers/qemu.py @@ -65,6 +65,8 @@ def _agent_config_string(value: object) -> str: def _public_qemu_config(config: dict[str, Any], state: dict[str, Any]) -> dict[str, Any]: """Wire-format QEMU config for clients (bpg / pulumi-proxmoxve).""" + import hashlib + merged = {**state, **config} payload: dict[str, Any] = {} for key, value in merged.items(): @@ -75,19 +77,66 @@ def _public_qemu_config(config: dict[str, Any], state: dict[str, Any]) -> dict[s payload[key] = value agent = config.get("agent", state.get("agent", "0")) payload["agent"] = _agent_config_string(agent) + digest_src = {key: value for key, value in payload.items() if key != "digest"} + payload["digest"] = hashlib.sha1( + json.dumps(digest_src, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() return payload +def _qemu_index_item(vmid: int, vm_state: dict[str, Any], config: dict[str, Any]) -> dict[str, Any]: + status = str(vm_state.get("status", "stopped")) + running = status in {"running", "paused"} + memory_mb = int(config.get("memory", config.get("mem", 2048))) + maxmem = memory_mb * 2**20 + mem = int(vm_state.get("mem", maxmem // 2 if running else 0)) + return { + "vmid": vmid, + "name": str(config.get("name", f"vm-{vmid}")), + "status": status, + "qmpstatus": status if running else "stopped", + "cpus": int(config.get("cores", config.get("cpus", 1))), + "cpu": float(vm_state.get("cpu", 0.1 if running else 0.0)), + "maxmem": maxmem, + "mem": mem, + "memhost": int(vm_state.get("memhost", mem)), + "maxdisk": int(vm_state.get("maxdisk", config.get("maxdisk", 32 * 1024**3))), + "disk": int(vm_state.get("disk", 0)), + "diskread": int(vm_state.get("diskread", 0)), + "diskwrite": int(vm_state.get("diskwrite", 0)), + "netin": int(vm_state.get("netin", 0)), + "netout": int(vm_state.get("netout", 0)), + "uptime": int(vm_state.get("uptime", 0)) if running else 0, + "pid": int(vm_state.get("pid", 12_345 if running else 0)), + "lock": str(vm_state.get("lock", "")), + "tags": str(config.get("tags", "")), + "template": int(bool(vm_state.get("template", False))), + "running-qemu": str(vm_state.get("running-qemu", "9.2.0")) if running else "", + "running-machine": str(vm_state.get("running-machine", "pc-i440fx-9.2")) if running else "", + "pressurecpusome": float(vm_state.get("pressurecpusome", 0.0)), + "pressurecpufull": float(vm_state.get("pressurecpufull", 0.0)), + "pressureiosome": float(vm_state.get("pressureiosome", 0.0)), + "pressureiofull": float(vm_state.get("pressureiofull", 0.0)), + "pressurememorysome": float(vm_state.get("pressurememorysome", 0.0)), + "pressurememoryfull": float(vm_state.get("pressurememoryfull", 0.0)), + } + + def register_qemu_handlers(registry: HandlerRegistry) -> None: async def qemu_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: node = str(_values(inputs)["node"]) rows = await _database(request).pool.fetch( - """SELECT r.external_id::integer AS vmid, r.state - FROM resources r JOIN nodes n ON n.id=r.node_id + """SELECT r.external_id::integer AS vmid, r.state, v.config + FROM resources r + JOIN nodes n ON n.id=r.node_id + JOIN virtual_machines v ON v.resource_id=r.id WHERE n.name=$1 AND r.kind='qemu' ORDER BY r.external_id::integer""", node, ) - return [{"vmid": int(row["vmid"]), **_state(row["state"])} for row in rows] + return [ + _qemu_index_item(int(row["vmid"]), _state(row["state"]), _state(row["config"])) + for row in rows + ] async def qemu_status_index(request: Request, inputs: dict[str, Any]) -> list[dict[str, str]]: payload = _values(inputs) @@ -127,28 +176,28 @@ def register_qemu_handlers(registry: HandlerRegistry) -> None: else 0, ) ) - return { - "vmid": vmid, - "name": str(config.get("name", f"vm-{vmid}")), - "status": status, - "qmpstatus": status if running else "stopped", - "lock": str(vm_state.get("lock", "")), - "pid": int(vm_state.get("pid", 12_345 if running else 0)), - "cpus": int(config.get("cores", config.get("cpus", 1))), - "maxmem": maxmem, - "mem": mem_used, - "balloon": int(vm_state.get("balloon", 0)), - "ballooninfo": { - "actual": mem_used, - "max_mem": maxmem, - "mem_swapped_in": 0, - "mem_swapped_out": 0, - }, - "uptime": uptime, - "template": int(bool(vm_state.get("template", False))), - "ha": {"managed": int(vm_state.get("ha_managed", 0))}, - "agent": 1 if running and str(config.get("agent", "0")).startswith("1") else 0, - } + agent_enabled = str(config.get("agent", "0")).startswith("1") + item = _qemu_index_item(vmid, vm_state, config) + item.update( + { + "uptime": uptime, + "mem": mem_used, + "balloon": int(vm_state.get("balloon", 0)), + "ballooninfo": { + "actual": mem_used, + "max_mem": maxmem, + "mem_swapped_in": 0, + "mem_swapped_out": 0, + }, + "template": bool(vm_state.get("template", False)), + "ha": {"managed": int(vm_state.get("ha_managed", 0))}, + "agent": bool(running and agent_enabled), + "clipboard": str(vm_state.get("clipboard", "")), + "spice": bool(vm_state.get("spice", False)), + "serial": bool(vm_state.get("serial", False)), + } + ) + return item async def qemu_config(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: node, vmid = str(_values(inputs)["node"]), str(_values(inputs)["vmid"]) @@ -339,24 +388,30 @@ def register_qemu_handlers(registry: HandlerRegistry) -> None: values = _values(inputs) resource = await _qemu_resource(request, str(values["node"]), str(values["vmid"])) rows = await _database(request).pool.fetch( - """SELECT name, parent_name, description, created_at FROM snapshots + """SELECT name, parent_name, description, created_at, state FROM snapshots WHERE resource_id=$1 ORDER BY created_at, name""", resource["id"], ) - return [ - { - "name": row["name"], - "parent": row["parent_name"], - "description": row["description"] or "", - "snaptime": int(row["created_at"].timestamp()), - } - for row in rows - ] + result: list[dict[str, Any]] = [] + for row in rows: + raw_state = row["state"] if "state" in row else None + snap_state = _state(raw_state) if raw_state is not None else {} + result.append( + { + "name": row["name"], + "parent": row["parent_name"], + "description": row["description"] or "", + "snaptime": int(row["created_at"].timestamp()), + "vmstate": int(bool(snap_state.get("vmstate"))), + } + ) + return result async def snapshot_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: values = _values(inputs) row = await _snapshot(request, values) - state = _state(row["state"]) + raw_state = row["state"] if "state" in row else None + state = _state(raw_state) if raw_state is not None else {} return { "name": row["name"], "parent": row["parent_name"], @@ -528,7 +583,7 @@ def register_qemu_handlers(registry: HandlerRegistry) -> None: }, ) - async def resize(request: Request, inputs: dict[str, Any]) -> None: + async def resize(request: Request, inputs: dict[str, Any]) -> str: values = _values(inputs) node, vmid, disk = str(values["node"]), str(values["vmid"]), str(values["disk"]) resource = await _qemu_resource(request, node, vmid) @@ -561,6 +616,13 @@ def register_qemu_handlers(registry: HandlerRegistry) -> None: str(config[disk]).split(":", 1)[0], size, ) + return await _create_task( + request, + node=node, + vmid=vmid, + task_type="qemu-resize", + payload={"resource_id": str(resource["id"]), "disk": disk, "size": size}, + ) async def move_disk(request: Request, inputs: dict[str, Any]) -> str: values = _values(inputs) @@ -635,12 +697,32 @@ def register_qemu_handlers(registry: HandlerRegistry) -> None: return await agent_result("ping", request, inputs) async def task_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: - tasks = await TaskRepository(_database(request).pool).list_for_node( - str(_values(inputs)["node"]) - ) - return [ - {"upid": task.upid, "status": task.status, "type": task.task_type} for task in tasks - ] + from app.tasks.upid import Upid + + node = str(_values(inputs)["node"]) + tasks = await TaskRepository(_database(request).pool).list_for_node(node) + result: list[dict[str, Any]] = [] + for task in tasks: + try: + parsed = Upid.parse(task.upid) + except ValueError: + continue + finished = task.status in {"success", "error", "cancelled"} + entry: dict[str, Any] = { + "upid": task.upid, + "id": parsed.task_id, + "node": parsed.node, + "pid": parsed.pid, + "pstart": parsed.process_start, + "starttime": parsed.start_time, + "type": parsed.task_type, + "user": parsed.user, + } + if finished: + entry["status"] = "OK" if task.status == "success" else str(task.status).upper() + entry["endtime"] = parsed.start_time + max(task.progress, 1) + result.append(entry) + return result async def task_status(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: task = await TaskRepository(_database(request).pool).get_by_upid( @@ -681,7 +763,7 @@ def register_qemu_handlers(registry: HandlerRegistry) -> None: } } - async def qemu_template(request: Request, inputs: dict[str, Any]) -> None: + async def qemu_template(request: Request, inputs: dict[str, Any]) -> str: payload = _values(inputs) node, vmid = str(payload["node"]), str(payload["vmid"]) resource = await _qemu_resource(request, node, vmid) @@ -698,6 +780,13 @@ def register_qemu_handlers(registry: HandlerRegistry) -> None: resource["id"], json.dumps(state, sort_keys=True), ) + return await _create_task( + request, + node=node, + vmid=vmid, + task_type="qemu-template", + payload={"resource_id": str(resource["id"])}, + ) async def qemu_index(request: Request, inputs: dict[str, Any]) -> list[dict[str, str]]: payload = _values(inputs) @@ -815,6 +904,8 @@ async def _create_task( "qemu-migrate": "qmigrate", "qemu-remote-migrate": "qmremote", "qemu-move-disk": "qmmove", + "qemu-resize": "qmresize", + "qemu-template": "qmtemplate", }[task_type] upid = str(Upid.allocate(node, worker_type, vmid, str(request.state.principal))) try: diff --git a/app/handlers/sdn.py b/app/handlers/sdn.py index 32f1fd6..5351920 100644 --- a/app/handlers/sdn.py +++ b/app/handlers/sdn.py @@ -25,11 +25,62 @@ def _sdn(metadata: dict[str, Any]) -> dict[str, Any]: return current if isinstance(current, dict) else {} +_VNET_LIST_KEYS = frozenset( + { + "vnet", + "type", + "zone", + "alias", + "tag", + "vlanaware", + "digest", + "isolate-ports", + "pending", + "state", + } +) + +_ZONE_LIST_KEYS = frozenset( + { + "zone", + "type", + "bridge", + "mtu", + "nodes", + "ipam", + "dns", + "reversedns", + "vlan-protocol", + "digest", + "pending", + "state", + "comment", + } +) + + def _public(item: dict[str, Any]) -> dict[str, Any]: return {key: value for key, value in item.items() if key not in _SECRET_KEYS} +def _vnet_list_item(name: str, item: dict[str, Any]) -> dict[str, Any]: + payload = {key: value for key, value in item.items() if key in _VNET_LIST_KEYS} + payload["vnet"] = str(payload.get("vnet") or name) + payload.setdefault("type", "vnet") + return _public(payload) + + +def _zone_list_item(name: str, item: dict[str, Any]) -> dict[str, Any]: + payload = {key: value for key, value in item.items() if key in _ZONE_LIST_KEYS} + payload["zone"] = str(payload.get("zone") or name) + return _public(payload) + + def _store_list(store: dict[str, Any], *, id_key: str) -> list[dict[str, Any]]: + if id_key == "vnet": + return [_vnet_list_item(name, item) for name, item in sorted(store.items())] + if id_key == "zone": + return [_zone_list_item(name, item) for name, item in sorted(store.items())] return [_public({id_key: name, **item}) for name, item in sorted(store.items())] @@ -59,7 +110,9 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: "zones", ) - async def apply(request: Request, inputs: dict[str, Any]) -> None: + async def apply(request: Request, inputs: dict[str, Any]) -> str: + from app.handlers.cluster_extra import _cluster_task + payload = values(inputs) metadata, sdn = await _load(request, for_write=True) lock = sdn.get("lock") @@ -71,15 +124,16 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: if payload.get("release-lock"): sdn["lock"] = None await save_cluster_metadata(request, metadata) + return await _cluster_task(request, task_type="sdn-apply", worker="sdnapply") - async def lock_create(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: + async def lock_create(request: Request, inputs: dict[str, Any]) -> str: metadata, sdn = await _load(request, for_write=True) if sdn.get("lock") and not values(inputs).get("allow-pending"): raise ApiError(400, "SDN is already locked") token = secrets.token_hex(8) sdn["lock"] = {"token": token} await save_cluster_metadata(request, metadata) - return {"digest": token, "token": token} + return token async def lock_delete(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) diff --git a/app/handlers/storage.py b/app/handlers/storage.py index 3acb266..7348d4c 100644 --- a/app/handlers/storage.py +++ b/app/handlers/storage.py @@ -148,6 +148,7 @@ def register_storage_handlers(registry: HandlerRegistry) -> None: JOIN resources r ON r.id = s.resource_id JOIN nodes n ON n.id = r.node_id WHERE n.name=$1 OR s.shared = true + OR s.storage_id IN ('local', 'local-lvm') ORDER BY s.storage_id""", node, ) @@ -294,7 +295,55 @@ def register_storage_handlers(registry: HandlerRegistry) -> None: if status != "DELETE 1": raise ApiError(404, "volume does not exist") - async def node_storage_upload(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: + async def node_storage_allocate(request: Request, inputs: dict[str, Any]) -> str: + from app.handlers.nodes import _node_task + + node = str(values(inputs)["node"]) + storage_id = str(values(inputs)["storage"]) + await require_node(request, node) + await _storage_row(request, None, storage_id) + payload = values(inputs) + filename = str(payload.get("filename") or f"vm-{payload.get('vmid', '0')}-disk-0") + content_type = str(payload.get("content") or "images") + size_raw = str(payload.get("size") or "1G") + # Accept either integer bytes or Proxmox size strings like 32G. + try: + size = int(size_raw) + except ValueError: + from app.handlers.common import parse_size_bytes + + try: + size = parse_size_bytes(size_raw) + except ValueError as error: + raise ApiError(400, f"invalid size '{size_raw}'") from error + format_name = str(payload.get("format") or "raw") + volume_id = f"{storage_id}:{content_type}/{filename}" + resource_id = await database(request).pool.fetchval( + "SELECT resource_id FROM storages WHERE storage_id=$1", + storage_id, + ) + await database(request).pool.execute( + """INSERT INTO storage_contents( + id, storage_resource_id, volume_id, content_type, size_bytes, metadata + ) VALUES(gen_random_uuid(), $1, $2, $3, $4, $5::jsonb) + ON CONFLICT (storage_resource_id, volume_id) DO UPDATE + SET size_bytes=EXCLUDED.size_bytes, + content_type=EXCLUDED.content_type, + metadata=EXCLUDED.metadata""", + resource_id, + volume_id, + content_type, + size, + json.dumps( + {"filename": filename, "format": format_name, "source": "allocate"}, + sort_keys=True, + ), + ) + return await _node_task(request, node=node, task_type="storage-allocate", worker="imgcopy") + + async def node_storage_upload(request: Request, inputs: dict[str, Any]) -> str: + from app.handlers.nodes import _node_task + node = str(values(inputs)["node"]) storage_id = str(values(inputs)["storage"]) await require_node(request, node) @@ -326,11 +375,11 @@ def register_storage_handlers(registry: HandlerRegistry) -> None: size, json.dumps({"filename": filename, "source": "upload"}, sort_keys=True), ) - return {"uploadid": volume_id, "filename": filename, "size": size, "volid": volume_id} + return await _node_task(request, node=node, task_type="storage-upload", worker="download") async def node_storage_prunebackups( request: Request, inputs: dict[str, Any] - ) -> list[dict[str, Any]] | None: + ) -> list[dict[str, Any]] | str: node = str(values(inputs)["node"]) storage_id = str(values(inputs)["storage"]) await require_node(request, node) @@ -340,6 +389,8 @@ def register_storage_handlers(registry: HandlerRegistry) -> None: storage_id, ) if request.method == "DELETE": + from app.handlers.nodes import _node_task + keep = int(values(inputs).get("keep-last") or values(inputs).get("keep_last") or 1) await database(request).pool.execute( """DELETE FROM backups @@ -352,7 +403,9 @@ def register_storage_handlers(registry: HandlerRegistry) -> None: resource_id, keep, ) - return None + return await _node_task( + request, node=node, task_type="prune-backups", worker="prunebackups" + ) rows = await database(request).pool.fetch( """SELECT volume_id, size_bytes, created_at FROM backups WHERE storage_resource_id=$1 ORDER BY created_at DESC""", @@ -578,7 +631,7 @@ def register_storage_handlers(registry: HandlerRegistry) -> None: registry.register("/nodes/{node}/storage/{storage}", "GET", node_storage_index) registry.register("/nodes/{node}/storage/{storage}/status", "GET", node_storage_status) registry.register("/nodes/{node}/storage/{storage}/content", "GET", node_storage_content) - registry.register("/nodes/{node}/storage/{storage}/content", "POST", node_storage_upload) + registry.register("/nodes/{node}/storage/{storage}/content", "POST", node_storage_allocate) registry.register( "/nodes/{node}/storage/{storage}/content/{volume}", "GET", node_storage_content_get ) diff --git a/app/main.py b/app/main.py index 6217226..b5b1b4d 100644 --- a/app/main.py +++ b/app/main.py @@ -68,12 +68,15 @@ def create_app( "qemu-stop": qemu, "qemu-suspend": qemu, "qemu-update": qemu, + "qemu-resize": qemu, + "qemu-template": qemu, "lxc-clone": lxc, "lxc-create": lxc, "lxc-delete": lxc, "lxc-migrate": lxc, "lxc-reboot": lxc, "lxc-resume": lxc, + "lxc-resize": lxc, "lxc-shutdown": lxc, "lxc-snapshot-create": lxc, "lxc-snapshot-delete": lxc, @@ -83,6 +86,29 @@ def create_app( "lxc-suspend": lxc, "vzdump": backup, "aptupdate": backup, + "network-reload": backup, + "disk-initgpt": backup, + "disk-wipe": backup, + "disk-directory-create": backup, + "disk-directory-delete": backup, + "disk-lvm-create": backup, + "disk-lvm-delete": backup, + "disk-lvmthin-create": backup, + "disk-lvmthin-delete": backup, + "disk-zfs-create": backup, + "disk-zfs-delete": backup, + "service-start": backup, + "service-stop": backup, + "service-restart": backup, + "service-reload": backup, + "storage-allocate": backup, + "storage-upload": backup, + "prune-backups": backup, + "acme": backup, + "sdn-apply": backup, + "ceph-flags": backup, + "cluster-create": backup, + "cluster-join": backup, }, concurrency=resolved.task_worker_concurrency, lease_seconds=resolved.task_lease_seconds, diff --git a/app/simulation/ceph_defaults.py b/app/simulation/ceph_defaults.py new file mode 100644 index 0000000..fc9c463 --- /dev/null +++ b/app/simulation/ceph_defaults.py @@ -0,0 +1,25 @@ +"""Canonical Ceph cluster flag names (PVE /cluster/ceph/flags).""" + +from __future__ import annotations + +# Keep in sync with Proxmox `ceph_flags` API (majors 6-9). +CEPH_FLAG_DESCRIPTIONS: dict[str, str] = { + "nobackfill": "Pause backfill operations", + "nodeep-scrub": "Disable deep scrubbing", + "nodown": "Do not mark OSDs down", + "noin": "Do not mark OSDs in", + "noout": "Do not mark OSDs out", + "norebalance": "Pause rebalance", + "norecover": "Pause recovery", + "noscrub": "Disable scrubbing", + "notieragent": "Pause tiering agent", + "noup": "Do not mark OSDs up", + "pause": "Pause I/O", +} + + +def default_ceph_flags(*, enabled: frozenset[str] | None = None) -> dict[str, int]: + """Return the full flag map; unset flags are ``0`` (PVE GET → ``false``).""" + + active = enabled or frozenset() + return {name: (1 if name in active else 0) for name in CEPH_FLAG_DESCRIPTIONS} diff --git a/app/simulation/seed.py b/app/simulation/seed.py index 038d928..092801b 100644 --- a/app/simulation/seed.py +++ b/app/simulation/seed.py @@ -10,15 +10,36 @@ import asyncpg # type: ignore[import-untyped] from asyncpg import Connection from app.security.auth import hash_secret +from app.simulation.ceph_defaults import default_ceph_flags NAMESPACE = uuid.UUID("c9040a72-b391-4a7e-9864-3ae46291a531") CLUSTER_ID = uuid.UUID("dc760c47-d8d7-57e6-9404-f0c6f2395d8f") -def default_node_ops_for_seed(node_name: str) -> dict[str, object]: +def default_node_ops_for_seed( + node_name: str, *, node_index: int = 0, node_count: int = 1, osds_per_node: int = 1 +) -> dict[str, object]: """Full durable node ops written to nodes.metadata at seed time.""" suffix = (stable_id(f"node-ip:{node_name}").int % 200) + 10 + disk_count = max(2, osds_per_node + 1) + + def _disk_path(disk_index: int) -> str: + if disk_index < 26: + return f"/dev/sd{chr(ord('a') + disk_index)}" + # Beyond sdz keep unique, stable SCSI-style ids. + return f"/dev/disk/by-id/scsi-SIM{disk_index:04d}" + + disk_list = [ + { + "devpath": _disk_path(disk_index), + "size": (1_000_000_000_000 if disk_index == 0 else 2_000_000_000_000), + "model": "SIM-DISK-01" if disk_index == 0 else f"SIM-OSD-{disk_index:02d}", + "serial": f"SIM{disk_index:04d}", + "gpt": 1 if disk_index == 0 else 0, + } + for disk_index in range(disk_count) + ] return { "network": [ { @@ -38,46 +59,32 @@ def default_node_ops_for_seed(node_name: str) -> dict[str, object]: {"iface": "eno1", "type": "eth", "active": 1, "method": "manual"}, ], "disks": { - "list": [ - { - "devpath": "/dev/sda", - "size": 1_000_000_000_000, - "model": "SIM-DISK-01", - "serial": "SIM0001", - "gpt": 1, - }, - { - "devpath": "/dev/sdb", - "size": 2_000_000_000_000, - "model": "SIM-SSD-01", - "serial": "SIM0002", - "gpt": 0, - }, - ], + "list": disk_list, "directory": [], "lvm": [], "lvmthin": [], "zfs": [], "smart": { - "/dev/sda": { + item["devpath"]: { "health": "PASSED", "type": "scsi", - "model": "SIM-DISK-01", - "serial": "SIM0001", - }, - "/dev/sdb": { - "health": "PASSED", - "type": "scsi", - "model": "SIM-SSD-01", - "serial": "SIM0002", - }, + "model": item["model"], + "serial": item["serial"], + } + for item in disk_list }, }, "services": { - "pveproxy": {"state": "running", "enabled": 1}, - "pvedaemon": {"state": "running", "enabled": 1}, - "pvestatd": {"state": "running", "enabled": 1}, - "corosync": {"state": "running", "enabled": 1}, + "pveproxy": {"state": "running", "enabled": 1, "desc": "PVE API Proxy Server"}, + "pvedaemon": {"state": "running", "enabled": 1, "desc": "PVE API Daemon"}, + "pvestatd": {"state": "running", "enabled": 1, "desc": "PVE Status Daemon"}, + "corosync": {"state": "running", "enabled": 1, "desc": "Corosync Cluster Engine"}, + "pve-ha-crm": {"state": "running", "enabled": 1, "desc": "PVE HA CRM Daemon"}, + "pve-ha-lrm": {"state": "running", "enabled": 1, "desc": "PVE HA LRM Daemon"}, + "pvescheduler": {"state": "running", "enabled": 1, "desc": "Proxmox VE scheduler"}, + "ceph-mon": {"state": "running", "enabled": 1, "desc": "Ceph Monitor"}, + "ceph-mgr": {"state": "running", "enabled": 1, "desc": "Ceph Manager"}, + "ceph-osd": {"state": "running", "enabled": 1, "desc": "Ceph Object Storage Daemon"}, }, "apt": { "packages": [ @@ -85,20 +92,65 @@ def default_node_ops_for_seed(node_name: str) -> dict[str, object]: "Package": "pve-manager", "Version": "9.2.3", "OldVersion": "9.2.2", - "Status": "upgradable", + "CurrentState": "Installed", + "Arch": "amd64", + "Origin": "Proxmox", }, - {"Package": "libpve-common-perl", "Version": "9.0.3", "Status": "installed"}, - ], - "repositories": [ { - "path": "/etc/apt/sources.list.d/pve-enterprise.list", - "enabled": 1, - "types": "deb", - "uri": "http://download.proxmox.com/debian/pve", - "suites": "bookworm", - "components": "pve-no-subscription", - } + "Package": "libpve-common-perl", + "Version": "9.0.3", + "CurrentState": "Installed", + "Arch": "amd64", + "Origin": "Proxmox", + }, + { + "Package": "pve-kernel-6.8", + "Version": "6.8.12-1", + "CurrentState": "Installed", + "Arch": "amd64", + "Origin": "Proxmox", + }, + { + "Package": "ceph-common", + "Version": "18.2.2", + "CurrentState": "Installed", + "Arch": "amd64", + "Origin": "Proxmox", + }, ], + "repositories": { + "digest": "seedapt", + "errors": [], + "infos": [], + "standard-repos": [ + { + "handle": "enterprise", + "name": "Proxmox VE Enterprise Repository", + "status": 0, + }, + { + "handle": "no-subscription", + "name": "Proxmox VE No-Subscription Repository", + "status": 1, + }, + ], + "files": [ + { + "path": "/etc/apt/sources.list.d/pve-enterprise.list", + "file-type": "list", + "digest": "seedapt-file0", + "repositories": [ + { + "Types": "deb", + "URIs": "http://download.proxmox.com/debian/pve", + "Suites": "bookworm", + "Components": "pve-no-subscription", + "Enabled": 1, + } + ], + } + ], + }, "update": {"status": "stopped", "exitstatus": "OK"}, "changelogs": { "pve-manager": "pve-manager (9.2.3) bookworm\n\n * simulator seed\n", @@ -171,22 +223,39 @@ def default_node_ops_for_seed(node_name: str) -> dict[str, object]: "certificates": { "custom": None, "acme": {"account": "letsencrypt", "domains": [], "certificate": None}, - "info": [], + "info": [ + { + "filename": "pve-ssl.pem", + "fingerprint": _seed_fingerprint(node_name), + "issuer": f"CN={node_name}.proxmox.local", + "subject": f"CN={node_name}.proxmox.local", + "notbefore": 1_700_000_000, + "notafter": 2_000_000_000, + "pem": ( + "-----BEGIN CERTIFICATE-----\n" + "SIMULATOR-SEED-CERT\n" + "-----END CERTIFICATE-----\n" + ), + "san": [f"{node_name}.proxmox.local", f"IP:10.0.0.{suffix}"], + "public-key-type": "rsa", + "public-key-bits": 4096, + } + ], }, "ceph": { "mds": {}, "mgr": { node_name: { "name": node_name, - "state": "active", + "state": "active" if node_index == 0 else "standby", "addr": f"{node_name}.local:6800", } }, "mon": { node_name: { "name": node_name, + "rank": node_index, "addr": f"{node_name}.local:6789", - "rank": 0, } }, "log": [{"t": 1_700_000_000, "n": 0, "line": "ceph simulator ready"}], @@ -259,7 +328,7 @@ def default_node_ops_for_seed(node_name: str) -> dict[str, object]: "dumpdir": "backup", "mode": "snapshot", "remove": 0, - "storage": "nfs-backup", + "storage": "local", "mailto": "", "notes-template": "{{guestname}}", }, @@ -273,30 +342,105 @@ def default_node_ops_for_seed(node_name: str) -> dict[str, object]: "status": { "uptime": 86_400 + suffix, "cpu": 0.05 + (suffix % 10) * 0.01, + "mem": 4_000_000_000 + suffix * 10_000_000, + "maxmem": 32_000_000_000, + "maxcpu": 16, "memory": { "used": 4_000_000_000 + suffix * 10_000_000, "total": 32_000_000_000, + "free": 28_000_000_000 - suffix * 10_000_000, + "available": 28_000_000_000 - suffix * 10_000_000, + }, + "rootfs": { + "used": 12 * 1024**3, + "total": 100 * 1024**3, + "avail": 88 * 1024**3, + "free": 88 * 1024**3, + }, + "swap": {"used": 0, "total": 8 * 1024**3, "free": 8 * 1024**3}, + "cpuinfo": { + "cpus": 16, + "cores": 8, + "sockets": 2, + "model": "QEMU Virtual CPU version 2.5+", + "flags": "fpu vme de pse tsc msr pae mce cx8", + }, + "boot-info": {"mode": "efi", "secureboot": 0}, + "current-kernel": { + "sysname": "Linux", + "release": "6.8.12-1-pve", + "version": "#1 SMP PREEMPT_DYNAMIC PMX 6.8.12-1", + "machine": "x86_64", }, "loadavg": ["0.10", "0.20", "0.30"], "kversion": "6.8.12-1-pve", "pveversion": "pve-manager/9.2.3", + "ssl_fingerprint": _seed_fingerprint(node_name), }, "ip": f"10.32.1.{suffix}", "cluster_status": { "ip": f"10.32.1.{suffix}", - "level": "c", + "level": "", "type": "node", "quorate": 1, }, } -def enrich_guest_state(state: dict[str, object], *, kind: str, vmid: str) -> dict[str, object]: - """Ensure qemu/lxc resources carry durable agent/rrd/migrate catalogs.""" +def enrich_guest_state( + state: dict[str, object], + *, + kind: str, + vmid: str, + slim: bool = False, +) -> dict[str, object]: + """Ensure qemu/lxc resources carry durable agent/rrd/migrate catalogs. + + ``slim=True`` (large profiles) keeps wire config keys without heavy agent/rrd. + """ enriched = dict(state) name = str(enriched.get("name") or f"{kind}-{vmid}") suffix = stable_id(f"guest-runtime:{kind}:{vmid}").int % 200 + if slim: + if kind == "qemu": + enriched.setdefault("scsi0", f"local-lvm:vm-{vmid}-disk-0,size=32G") + enriched.setdefault("scsihw", "virtio-scsi-pci") + enriched.setdefault("ostype", "l26") + enriched.setdefault("boot", "order=scsi0") + enriched.setdefault("net0", f"virtio=02:00:00:00:{suffix:02x}:01,bridge=vmbr0") + enriched.setdefault("memory", enriched.get("memory") or 2048) + enriched.setdefault("cores", enriched.get("cores") or 2) + enriched.setdefault("agent", "1") + enriched.setdefault("maxdisk", 32 * 1024**3) + else: + enriched.setdefault("rootfs", f"local-lvm:vm-{vmid}-disk-0,size=8G") + enriched.setdefault("ostype", "alpine") + enriched.setdefault("unprivileged", 1) + enriched.setdefault("swap", 512) + enriched.setdefault( + "net0", + f"name=eth0,bridge=vmbr0,hwaddr=02:00:00:00:{suffix:02x}:11,ip=dhcp", + ) + enriched.setdefault("memory", enriched.get("memory") or 512) + enriched.setdefault("cores", enriched.get("cores") or 1) + enriched.setdefault("hostname", name) + enriched.setdefault("maxdisk", 8 * 1024**3) + enriched.setdefault("status", enriched.get("status") or "stopped") + enriched.setdefault("name", name) + # Lightweight runtime counters so list/status dumps stay PVE-shaped without + # storing full agent/rrd catalogs for thousand-guest profiles. + running = str(enriched.get("status")) in {"running", "paused"} + enriched.setdefault("cpu", 0.12 if running else 0.0) + enriched.setdefault("disk", (suffix + 1) * 1024**2 if running else 0) + enriched.setdefault("diskread", suffix * 10_000 if running else 0) + enriched.setdefault("diskwrite", suffix * 4_000 if running else 0) + enriched.setdefault("netin", suffix * 8_000 if running else 0) + enriched.setdefault("netout", suffix * 3_000 if running else 0) + enriched.setdefault("uptime", 3_600 + suffix if running else 0) + enriched.setdefault("pid", 12_000 + suffix if running else 0) + return enriched + agent_enabled = enriched.get("agent") if isinstance(agent_enabled, bool) and not agent_enabled: agent: dict[str, object] = {"enabled": False, "files": {}, "results": {}} @@ -396,15 +540,66 @@ def enrich_guest_state(state: dict[str, object], *, kind: str, vmid: str) -> dic ] if "cloudinit_dump" not in enriched: enriched["cloudinit_dump"] = f"#cloud-config\nhostname: {name}\nmanage_etc_hosts: true\n" + # Disk keys live in guest config so resize/move handlers work after seed. + if kind == "qemu": + enriched.setdefault("scsi0", f"local-lvm:vm-{vmid}-disk-0,size=32G") + enriched.setdefault("scsihw", "virtio-scsi-pci") + enriched.setdefault("ostype", "l26") + enriched.setdefault("boot", "order=scsi0") + enriched.setdefault("net0", f"virtio=02:00:00:00:{suffix:02x}:01,bridge=vmbr0") + enriched.setdefault("memory", enriched.get("memory") or 2048) + enriched.setdefault("cores", enriched.get("cores") or 2) + enriched.setdefault("maxdisk", 32 * 1024**3) + elif kind == "lxc": + enriched.setdefault("rootfs", f"local-lvm:vm-{vmid}-disk-0,size=8G") + enriched.setdefault("ostype", "alpine") + enriched.setdefault("unprivileged", 1) + enriched.setdefault("swap", 512) + enriched.setdefault( + "net0", + f"name=eth0,bridge=vmbr0,hwaddr=02:00:00:00:{suffix:02x}:11,ip=dhcp", + ) + enriched.setdefault("memory", enriched.get("memory") or 512) + enriched.setdefault("cores", enriched.get("cores") or 1) + enriched.setdefault("hostname", name) + enriched.setdefault("maxdisk", 8 * 1024**3) + running = str(enriched.get("status") or "stopped") in {"running", "paused"} + enriched.setdefault("cpu", 0.12 if running else 0.0) + enriched.setdefault("disk", (suffix + 1) * 1024**2 if running else 0) + enriched.setdefault("diskread", suffix * 10_000 if running else 0) + enriched.setdefault("diskwrite", suffix * 4_000 if running else 0) + enriched.setdefault("netin", suffix * 8_000 if running else 0) + enriched.setdefault("netout", suffix * 3_000 if running else 0) + enriched.setdefault("uptime", 3_600 + suffix if running else 0) + enriched.setdefault("pid", 12_000 + suffix if running else 0) return enriched def enrich_storage_state(state: dict[str, object], *, storage_id: str) -> dict[str, object]: enriched = dict(state) - total_raw = enriched.get("total_bytes", 100) + if "total_bytes" not in enriched and "capacity_bytes" not in enriched: + enriched["total_bytes"] = 100 * 1024**3 + if "used_bytes" not in enriched: + total_hint = enriched.get("total_bytes", enriched.get("capacity_bytes", 100 * 1024**3)) + total_i = int(total_hint) if isinstance(total_hint, int | float | str) else 100 * 1024**3 + enriched["used_bytes"] = max(total_i // 10, 1) + total_raw = enriched.get("total_bytes", enriched.get("capacity_bytes", 100)) total = int(total_raw) if isinstance(total_raw, int | float | str) else 100 used_raw = enriched.get("used_bytes", max(total // 10, 1)) used = int(used_raw) if isinstance(used_raw, int | float | str) else max(total // 10, 1) + enriched.setdefault("disk", used) + enriched.setdefault("maxdisk", total) + if "plugintype" not in enriched: + if "lvm" in storage_id: + enriched["plugintype"] = "lvmthin" + elif storage_id.startswith("ceph"): + enriched["plugintype"] = "rbd" + elif storage_id.startswith("nfs"): + enriched["plugintype"] = "nfs" + else: + enriched["plugintype"] = "dir" + if "content" not in enriched: + enriched["content"] = ["images"] if "rrd" not in enriched or not isinstance(enriched.get("rrd"), dict): enriched["rrd"] = {"filename": f"pve-storage-{storage_id}.rrd"} if "rrddata" not in enriched or not isinstance(enriched.get("rrddata"), list): @@ -429,7 +624,7 @@ def enrich_storage_state(state: dict[str, object], *, storage_id: str) -> dict[s if "identity" not in enriched or not isinstance(enriched.get("identity"), dict): enriched["identity"] = { "fingerprint": f"fp-{storage_id}", - "type": str(enriched.get("storage_type") or "dir"), + "type": str(enriched.get("storage_type") or enriched.get("plugintype") or "dir"), } return enriched @@ -508,12 +703,120 @@ def _empty_firewall_scope(*, options: dict[str, object] | None = None) -> dict[s } +def _seed_fingerprint(node_name: str) -> str: + digest = stable_id(f"pve-fp:{node_name}").hex + # SHA-256 fingerprint wire form: 32 colon-separated hex bytes. + padded = (digest * 4)[:64] + return ":".join(padded[index : index + 2].upper() for index in range(0, 64, 2)) + + +def _seed_cluster_config(profile_name: str, node_names: list[str]) -> dict[str, object]: + clustername = f"pve-{profile_name}" + added_nodes: dict[str, dict[str, object]] = {} + for index, node_name in enumerate(node_names, start=1): + suffix = (stable_id(f"node-ip:{node_name}").int % 200) + 10 + addr = f"10.0.0.{suffix}" + added_nodes[node_name] = { + "node": node_name, + "nodeid": index, + "ring0_addr": addr, + "pve_addr": addr, + "pve_fp": _seed_fingerprint(node_name), + "quorum_votes": 1, + } + nodelist_lines = ["nodelist {"] + for node_name, entry in added_nodes.items(): + nodelist_lines.extend( + [ + " node {", + f" name: {node_name}", + f" nodeid: {entry['nodeid']}", + f" quorum_votes: {entry['quorum_votes']}", + f" ring0_addr: {entry['ring0_addr']}", + " }", + ] + ) + nodelist_lines.append("}") + corosync_conf = "\n".join( + [ + "totem {", + " version: 2", + f" cluster_name: {clustername}", + " secauth: on", + "}", + *nodelist_lines, + "quorum {", + " provider: corosync_votequorum", + "}", + "", + ] + ) + return { + "clustername": clustername, + "votes": 1, + "links": {}, + "join_info": {}, + "totem": { + "version": 2, + "secauth": "on", + "cluster_name": clustername, + }, + "qdevice": {"status": "disabled"}, + "apiversion": 1, + "corosync_authkey": stable_id(f"corosync-authkey:{clustername}").hex, + "corosync_conf": corosync_conf, + "config_digest": stable_id(f"corosync-digest:{clustername}").hex, + "added_nodes": added_nodes, + } + + +def _replication_seed_jobs( + profile_name: str, + *, + qemu_guests: list[tuple[str, str]], + node_names: list[str], + primary: str, + secondary: str, + multi_node: bool, +) -> list[dict[str, object]]: + """Proportional sample replication jobs for multi-node cluster profiles.""" + + if not multi_node or not qemu_guests: + return [] + job_count = max(int(_size_spec(profile_name).get("replication", 1)), 0) + if job_count <= 0: + return [] + targets = [name for name in node_names if name != primary] or [secondary] + jobs: list[dict[str, object]] = [] + for index, (source, guest) in enumerate(qemu_guests[:job_count]): + target = targets[index % len(targets)] + if target == source and len(targets) > 1: + target = targets[(index + 1) % len(targets)] + jobs.append( + { + "id": f"{guest}-0", + "guest": int(guest), + "jobnum": 0, + "target": target, + "type": "local", + "schedule": "*/15", + "rate": 100, + "comment": f"Replicate VM/CT {guest} to {target}", + "disable": 0, + "source": source, + } + ) + return jobs + + def cluster_domain_metadata(profile: SeedProfile) -> dict[str, object]: """Durable cluster metadata domains so list endpoints are non-empty after seed.""" node_names = [node.name for node in profile.nodes] primary = node_names[0] if node_names else "pve01" - secondary = node_names[1] if len(node_names) > 1 else "pve02" + # Single-node profiles must not invent ghost peers (replication/SDN). + secondary = node_names[1] if len(node_names) > 1 else primary + multi_node = len(node_names) > 1 node_by_id = {node.id: node.name for node in profile.nodes} qemu_guests: list[tuple[str, str]] = [] @@ -527,11 +830,12 @@ def cluster_domain_metadata(profile: SeedProfile) -> dict[str, object]: elif resource.kind == "lxc": lxc_guests.append((node_name, resource.external_id)) - # Keep large profiles bounded; small/demo stay fully covered within the cap. - guest_cap = 128 if profile.name == "large" else 10_000 + scale = _size_spec(profile.name) + guest_cap = int(scale.get("fw_guest_cap", 10_000)) qemu_guests = qemu_guests[:guest_cap] lxc_guests = lxc_guests[:guest_cap] - replication_guest = qemu_guests[0][1] if qemu_guests else "100" + osd_count = sum(1 for resource in profile.resources if resource.kind == "ceph-osd") + guest_total = sum(1 for resource in profile.resources if resource.kind in {"qemu", "lxc"}) cluster_fw = _empty_firewall_scope( options={ @@ -716,23 +1020,52 @@ def cluster_domain_metadata(profile: SeedProfile) -> dict[str, object]: "status_current": {"quorate": 1, "mode": "active"}, "manager_status": {"manager_status": "active", "quorum": "OK"}, }, + "backup_jobs": { + (f"backup-{index}" if index else "backup-daily"): { + "id": f"backup-{index}" if index else "backup-daily", + "schedule": f"{index * 2} 2 * * *", + "storage": "local" if index % 2 == 0 else "ceph", + "enabled": 1, + "all": 0, + "vmid": ",".join( + guest + for _, guest in ( + qemu_guests[index * 3 : index * 3 + 3] or qemu_guests[:1] or [("x", "100")] + ) + ), + "mode": "snapshot", + "compress": "zstd", + "comment": f"Seeded backup job #{index}", + } + for index in range(max(int(scale.get("backup_jobs", 1)), 0)) + if qemu_guests or profile.name == "lab" + }, "ha_groups": { - "primary": { + name: { "nodes": ha_nodes, - "nofailback": 0, + "nofailback": 1 if index else 0, "restricted": 0, - "comment": "Default HA group", - }, - "critical-services": { - "nodes": ha_nodes, - "nofailback": 1, - "restricted": 0, - "comment": "Critical services", - }, + "comment": f"HA group {name}", + } + for index, name in enumerate( + ( + "primary", + "critical-services", + "batch", + "gpu", + "edge", + "dr", + )[: max(int(scale.get("ha_groups", 2)), 1)] + ) }, "ha_rules": [ {"rule": "node-fencing", "type": "node", "action": "restart"}, {"rule": "service-ha", "type": "resource", "action": "failover"}, + *( + [{"rule": "affinity-primary", "type": "resource", "action": "prefer-primary"}] + if int(scale.get("ha_groups", 2)) > 2 + else [] + ), ], "sdn": { "zones": { @@ -744,6 +1077,10 @@ def cluster_domain_metadata(profile: SeedProfile) -> dict[str, object]: "vrf": "vrf-public", "table": 100, "mtu": 1500, + "nodes": ",".join(node_names), + "digest": "sdnzonepublic", + "pending": False, + "state": "available", "comment": "Public zone", }, "internal": { @@ -754,16 +1091,24 @@ def cluster_domain_metadata(profile: SeedProfile) -> dict[str, object]: "vrf": "vrf-internal", "table": 101, "vlan-protocol": "802.1q", + "nodes": ",".join(node_names), + "digest": "sdnzoneinternal", + "pending": False, + "state": "available", "comment": "Internal VLAN zone", }, }, "vnets": { "vnet0": { "vnet": "vnet0", + "type": "vnet", "zone": "public", "alias": "Public VNet", "mac-vrf": "macvrf-vnet0", "tag": 0, + "digest": "sdnvnet0", + "pending": False, + "state": "available", "subnets": { "10.0.0.0/24": { "subnet": "10.0.0.0/24", @@ -788,10 +1133,14 @@ def cluster_domain_metadata(profile: SeedProfile) -> dict[str, object]: }, "vnet1": { "vnet": "vnet1", + "type": "vnet", "zone": "internal", "alias": "Internal VNet", "mac-vrf": "macvrf-vnet1", "tag": 100, + "digest": "sdnvnet1", + "pending": False, + "state": "available", "subnets": { "10.10.0.0/24": { "subnet": "10.10.0.0/24", @@ -808,7 +1157,7 @@ def cluster_domain_metadata(profile: SeedProfile) -> dict[str, object]: "controller": "evpn1", "type": "evpn", "asn": 65000, - "peers": secondary, + "peers": secondary if multi_node else primary, "comment": "EVPN controller", } }, @@ -943,6 +1292,7 @@ def cluster_domain_metadata(profile: SeedProfile) -> dict[str, object]: "contact": ["mailto:admin@example.local"], "directory": "https://acme-v02.api.letsencrypt.org/directory", "location": "https://acme.example.local/acct/letsencrypt", + "tos_url": "https://acme-v02.api.letsencrypt.org/directory/tos", } }, "plugins": { @@ -952,6 +1302,7 @@ def cluster_domain_metadata(profile: SeedProfile) -> dict[str, object]: "api": "CF", "data": {"CF_Token": "seed-cf-token"}, "validation-delay": 30, + "digest": "seedacmeplugin00000000000000000000000001", } }, "directories": [ @@ -1008,33 +1359,19 @@ def cluster_domain_metadata(profile: SeedProfile) -> dict[str, object]: } }, }, - "replication": [ - { - "id": f"{replication_guest}-0", - "guest": replication_guest, - "target": secondary, - "type": "local", - "schedule": "*/15", - "rate": 100, - "comment": f"Replicate VM/CT {replication_guest} to {secondary}", - "enabled": 1, - "source": primary, - "state": "OK", - "last_sync": 1720000200, - "duration": 12, - "fail_count": 0, - "error": "", - "log": [ - { - "t": 1720000200, - "n": 0, - "msg": f"replication ok for {replication_guest}-0", - } - ], - } - ], + "replication": _replication_seed_jobs( + profile.name, + qemu_guests=qemu_guests, + node_names=node_names, + primary=primary, + secondary=secondary, + multi_node=multi_node, + ), "metrics": { - "export_data": '# HELP pve_up Node is up\npve_up{node="pve01"} 1\n', + "export_data": ( + "# HELP pve_up Node is up\n" + + "".join(f'pve_up{{node="{name}"}} 1\n' for name in node_names) + ), "servers": { "influx": { "id": "influx", @@ -1097,63 +1434,43 @@ def cluster_domain_metadata(profile: SeedProfile) -> dict[str, object]: "cfg_raw": "[global]\nfsid = pve-simulator-fsid\nauth_client_required = cephx\n", "cfg_values": {}, "pools": { - "rbd": { - "pool": "rbd", + name: { + "pool_id": index, + "pool_name": name, "size": 3, "min_size": 2, - "pg_num": 128, - "application": "rbd", - "crush_rule": "replicated_rule", - "bytes_used": 12_000_000_000, - "percent_used": 12.0, + "pg_num": max(32, min((osd_count or 1) * 32, 8192)), + "application": "rbd" if name == "rbd" else "cephfs", + "application_metadata": {"rbd": {}} if name == "rbd" else {"cephfs": {}}, + "crush_rule": 0, + "crush_rule_name": "replicated_rule", + "bytes_used": max(guest_total, 1) * (8 + index * 4) * 1024**3, + "percent_used": min(5.0 + guest_total / 50.0, 75.0), "healthy": True, } + for index, name in enumerate( + ("rbd", "cephfs_data", "cephfs_metadata")[ + : (1 if osd_count < 10 else 2 if osd_count < 100 else 3) + ] + ) }, "fs": {}, "rules": [{"name": "replicated_rule", "id": 0}], - "crush": "device 0 osd.0 class hdd\n", + "crush": "".join( + f"device {index} osd.{index} class hdd\n" for index in range(max(osd_count, 1)) + ), "cmd_safety": {"safe": 1}, - "flags": { - "nobackfill": 0, - "nodeep-scrub": 0, - "nodown": 0, - "noin": 0, - "noout": 0, - "norebalance": 0, - "norecover": 0, - "noscrub": 0, - "notieragent": 0, - "pause": 0, - }, - }, - "cluster_config": { - "clustername": f"pve-{profile.name}", - "votes": 1, - "links": {}, - "join_info": {}, - "totem": { - "version": 2, - "secauth": "on", - "cluster_name": f"pve-{profile.name}", - }, - "qdevice": {"status": "disabled"}, - "apiversion": 1, - "added_nodes": { - node_name: { - "node": node_name, - "ring0_addr": f"{node_name}.local", - "quorum_votes": 1, - } - for node_name in node_names - }, + "health": {"status": "HEALTH_OK"}, + "flags": default_ceph_flags(), }, + "cluster_config": _seed_cluster_config(profile.name, node_names), } -def _seed_resource_state(resource: SeedResource) -> dict[str, object]: +def _seed_resource_state(resource: SeedResource, *, slim: bool = False) -> dict[str, object]: if resource.kind in {"qemu", "lxc"}: return enrich_guest_state( - dict(resource.state), kind=resource.kind, vmid=resource.external_id + dict(resource.state), kind=resource.kind, vmid=resource.external_id, slim=slim ) if resource.kind == "storage": return enrich_storage_state(dict(resource.state), storage_id=resource.external_id) @@ -1183,17 +1500,273 @@ def _resource( return SeedResource(stable_id(f"{kind}:{external_id}"), node.id, kind, external_id, state) -def _completed_task(index: int, task_type: str, resource_id: str) -> SeedTask: +def _completed_task( + index: int, + task_type: str, + resource_id: str, + *, + node: str = "pve01", +) -> SeedTask: + pid = index & 0xFFFFFFFF + start = (0x65000000 + index) & 0xFFFFFFFF return SeedTask( - stable_id(f"task:{index}:{task_type}:{resource_id}"), - f"UPID:pve01:0000000{index}:0000000{index}:6500000{index}:" - f"{task_type}:{resource_id}:root@pam:", + stable_id(f"task:{index}:{task_type}:{resource_id}:{node}"), + (f"UPID:{node}:{pid:08X}:{pid:08X}:{start:08X}:{task_type}:{resource_id}:root@pam:"), task_type, - {"resource_id": resource_id, "seeded": True}, + {"resource_id": resource_id, "node": node, "seeded": True}, ) -def small_profile() -> SeedProfile: +def _storage_resource( + node: SeedNode, + storage_id: str, + *, + content: list[str], + total_bytes: int, + used_bytes: int, + plugintype: str, + shared: bool = False, +) -> SeedResource: + return _resource( + node, + "storage", + storage_id, + { + "content": content, + "status": "available", + "shared": shared, + "total_bytes": total_bytes, + "used_bytes": used_bytes, + "plugintype": plugintype, + "disk": used_bytes, + "maxdisk": total_bytes, + }, + ) + + +def _ceph_osd_resource(node: SeedNode, osd_id: int) -> SeedResource: + return _resource( + node, + "ceph-osd", + f"osd.{osd_id}", + { + "osd_id": osd_id, + "status": "up", + "in": 1, + "weight": 1.0, + "device_class": "hdd", + "dev": f"/dev/disk/by-id/ceph-osd-{osd_id:04d}", + }, + ) + + +def _distribute_count(total: int, buckets: int) -> list[int]: + """Spread ``total`` items across ``buckets`` as evenly as possible.""" + + if buckets <= 0 or total <= 0: + return [0] * max(buckets, 0) + base, rem = divmod(total, buckets) + return [base + (1 if index < rem else 0) for index in range(buckets)] + + +def _cluster_infra( + nodes: tuple[SeedNode, ...], + *, + include_ceph: bool = True, + osd_count: int = 0, + osds_per_node: int | None = None, + ha_sids: tuple[str, ...] = (), + pool_id: str | None = None, + pool_members: tuple[str, ...] = (), + guest_count: int = 50, +) -> list[SeedResource]: + """Shared storages / Ceph OSDs / HA / pool rows for multi-node profiles. + + ``storages.storage_id`` is unique cluster-wide, so ``local`` / ``local-lvm`` / + ``ceph`` are seeded once (standard PVE names). Node storage list and + ``/cluster/resources`` expand them onto every node. + """ + + resources: list[SeedResource] = [] + if not nodes: + return resources + primary = nodes[0] + if osds_per_node is not None and osd_count <= 0: + osd_count = max(osds_per_node, 0) * len(nodes) + # Scale capacity with guest + OSD count so dumps stay realistic. + local_total = max(200 * 1024**3, guest_count * 4 * 1024**3) + local_used = local_total // 5 + lvm_total = max(800 * 1024**3, guest_count * 40 * 1024**3) + lvm_used = lvm_total // 6 + ceph_total = max( + max(osd_count, 1) * 2 * 1024**4, + len(nodes) * 2 * 1024**4, + guest_count * 64 * 1024**3, + ) + ceph_used = max( + max(osd_count, 1) * 200 * 1024**3, + len(nodes) * 200 * 1024**3, + guest_count * 8 * 1024**3, + ) + resources.append( + _storage_resource( + primary, + "local", + content=["iso", "backup", "vztmpl", "images"], + total_bytes=local_total, + used_bytes=local_used, + plugintype="dir", + ) + ) + resources.append( + _storage_resource( + primary, + "local-lvm", + content=["images", "rootdir"], + total_bytes=lvm_total, + used_bytes=lvm_used, + plugintype="lvmthin", + ) + ) + if include_ceph: + resources.append( + _storage_resource( + primary, + "ceph", + content=["images", "rootdir"], + total_bytes=ceph_total, + used_bytes=ceph_used, + plugintype="rbd", + shared=True, + ) + ) + per_node = _distribute_count(max(osd_count, 0), len(nodes)) + osd_id = 0 + for node, node_osds in zip(nodes, per_node, strict=True): + for _ in range(node_osds): + resources.append(_ceph_osd_resource(node, osd_id)) + osd_id += 1 + for sid in ha_sids: + guest_kind, _, _guest_id = sid.partition(":") + resources.append( + _resource( + primary, + "ha", + sid, + {"state": "started", "group": "primary", "type": guest_kind or "vm"}, + ) + ) + if pool_id: + resources.append( + _resource( + primary, + "pool", + pool_id, + {"members": list(pool_members), "comment": f"Pool {pool_id}"}, + ) + ) + return resources + + +def _seed_tasks_for_guests( + nodes: tuple[SeedNode, ...], + guest_ids: list[tuple[str, str]], + *, + limit: int = 20, +) -> tuple[SeedTask, ...]: + """Create completed tasks bound to the guest's real node name.""" + + tasks: list[SeedTask] = [] + for index, (node_name, vmid) in enumerate(guest_ids[:limit], start=1): + tasks.append(_completed_task(index, "qmstart", vmid, node=node_name)) + return tuple(tasks) + + +# UI / ops cluster sizes (hosts × total guests). Infra scales with the size. +_CLUSTER_SIZE_SPECS: dict[str, dict[str, int]] = { + "lab": { + "nodes": 1, + "guests": 3, + "osds": 3, + "ha": 1, + "tasks": 2, + "pool_members": 2, + "replication": 0, + "backups": 2, + "snapshots": 2, + "backup_jobs": 1, + "ha_groups": 2, + "fw_guest_cap": 10_000, + }, + "small": { + "nodes": 3, + "guests": 50, + "osds": 10, + "ha": 3, + "tasks": 12, + "pool_members": 12, + "replication": 2, + "backups": 10, + "snapshots": 12, + "backup_jobs": 2, + "ha_groups": 2, + "fw_guest_cap": 10_000, + }, + "large": { + "nodes": 10, + "guests": 1_000, + "osds": 100, + "ha": 12, + "tasks": 50, + "pool_members": 50, + "replication": 8, + "backups": 80, + "snapshots": 100, + "backup_jobs": 4, + "ha_groups": 4, + "fw_guest_cap": 256, + }, + "big": { + "nodes": 20, + "guests": 2_000, + "osds": 500, + "ha": 24, + "tasks": 100, + "pool_members": 100, + "replication": 16, + "backups": 160, + "snapshots": 200, + "backup_jobs": 6, + "ha_groups": 6, + "fw_guest_cap": 384, + }, +} + + +def _size_spec(profile_name: str) -> dict[str, int]: + if profile_name in _CLUSTER_SIZE_SPECS: + return dict(_CLUSTER_SIZE_SPECS[profile_name]) + if profile_name in {"medium", "ha-demo"}: + return dict(_CLUSTER_SIZE_SPECS["small"]) + return { + "nodes": 1, + "guests": 0, + "osds": 0, + "ha": 0, + "tasks": 0, + "pool_members": 0, + "replication": 0, + "backups": 0, + "snapshots": 0, + "backup_jobs": 0, + "ha_groups": 1, + "fw_guest_cap": 10_000, + } + + +def lab_profile() -> SeedProfile: + """Single-node fixture used by unit/surface tests (not a DATA UI button).""" + node = _node("pve01") resources = ( _resource( @@ -1226,73 +1799,202 @@ def small_profile() -> SeedProfile: "agent": {"files": {"/etc/hostname": "service\n"}}, }, ), - _resource(node, "storage", "local", {"content": ["iso", "backup"], "status": "available"}), _resource( - node, "storage", "local-lvm", {"content": ["images", "rootdir"], "status": "available"} + node, + "storage", + "local", + { + "content": ["iso", "backup", "vztmpl"], + "status": "available", + "total_bytes": 100 * 1024**3, + "used_bytes": 12 * 1024**3, + "plugintype": "dir", + "disk": 12 * 1024**3, + "maxdisk": 100 * 1024**3, + }, + ), + _resource( + node, + "storage", + "local-lvm", + { + "content": ["images", "rootdir"], + "status": "available", + "total_bytes": 500 * 1024**3, + "used_bytes": 64 * 1024**3, + "plugintype": "lvmthin", + "disk": 64 * 1024**3, + "maxdisk": 500 * 1024**3, + }, + ), + _resource( + node, + "storage", + "ceph", + { + "content": ["images", "rootdir"], + "status": "available", + "shared": True, + "total_bytes": 3 * 1024**4, + "used_bytes": 120 * 1024**3, + "plugintype": "rbd", + "disk": 120 * 1024**3, + "maxdisk": 3 * 1024**4, + }, + ), + _resource(node, "pool", "lab", {"members": ["100", "200"], "comment": "Lab pool"}), + _resource(node, "ha", "vm:100", {"state": "started", "group": "primary"}), + *( + _resource( + node, + "ceph-osd", + f"osd.{index}", + { + "osd_id": index, + "status": "up", + "in": 1, + "weight": 1.0, + "device_class": "hdd", + "dev": f"/dev/sd{chr(ord('b') + index)}", + }, + ) + for index in range(3) ), ) tasks = (_completed_task(1, "qmstart", "100"), _completed_task(2, "qmstop", "100")) - return SeedProfile("small", (node,), resources, tasks) + return SeedProfile("lab", (node,), resources, tasks) + + +def _sized_cluster_profile(name: str, *, overrides: dict[str, int] | None = None) -> SeedProfile: + """Build a proportional multi-node cluster (small / large / big).""" + + spec = dict(_CLUSTER_SIZE_SPECS[name]) + if overrides: + spec.update(overrides) + node_count = int(spec["nodes"]) + guest_count = int(spec["guests"]) + nodes = tuple(_node(f"pve{index}") for index in range(1, node_count + 1)) + guests: list[SeedResource] = [] + guest_pairs: list[tuple[str, str]] = [] + ha_sids: list[str] = [] + for index in range(guest_count): + node = nodes[index % node_count] + # ~75% qemu / ~25% lxc + kind = "lxc" if index % 4 == 0 else "qemu" + vmid = str(100 + index) + status = "running" if index % 7 == 0 else "stopped" + guests.append(_resource(node, kind, vmid, {"name": f"guest-{vmid}", "status": status})) + if kind == "qemu" and len(guest_pairs) < max(int(spec["tasks"]), 1): + guest_pairs.append((node.name, vmid)) + if len(ha_sids) < int(spec["ha"]): + if kind == "lxc": + ha_sids.append(f"ct:{vmid}") + elif index % 3 == 1: + ha_sids.append(f"vm:{vmid}") + for index in range(guest_count): + if len(ha_sids) >= int(spec["ha"]): + break + if index % 4 == 0: + continue + sid = f"vm:{100 + index}" + if sid not in ha_sids: + ha_sids.append(sid) + + pool_members = tuple( + str(100 + index) for index in range(min(int(spec["pool_members"]), guest_count)) + ) + pool_id = {"small": "development", "large": "production", "big": "production"}[name] + infra = _cluster_infra( + nodes, + osd_count=int(spec["osds"]), + ha_sids=tuple(ha_sids[: int(spec["ha"])]), + pool_id=pool_id, + pool_members=pool_members, + guest_count=guest_count, + ) + # Extra pools for larger sizes (members are disjoint samples). + if name == "large" and int(spec["guests"]) >= 1_000: + infra.append( + _resource( + nodes[0], + "pool", + "staging", + { + "members": [str(100 + index) for index in range(20, 40)], + "comment": "Pool staging", + }, + ) + ) + elif name == "big": + infra.extend( + [ + _resource( + nodes[0], + "pool", + "staging", + { + "members": [str(100 + index) for index in range(40, 70)], + "comment": "Pool staging", + }, + ), + _resource( + nodes[0], + "pool", + "development", + { + "members": [str(100 + index) for index in range(70, 100)], + "comment": "Pool development", + }, + ), + ] + ) + tasks = _seed_tasks_for_guests(nodes, guest_pairs, limit=int(spec["tasks"])) + return SeedProfile(name, nodes, tuple(guests) + tuple(infra), tasks) + + +def small_profile() -> SeedProfile: + """DATA UI: 3 hosts × 50 guests.""" + + return _sized_cluster_profile("small") + + +def large_profile(*, node_count: int = 10, resource_count: int = 1_000) -> SeedProfile: + """DATA UI: 10 hosts × 1000 guests (overrides allowed for CLI stress).""" + + if node_count < 1 or resource_count < 1: + raise ValueError("large profile counts must be positive") + if node_count == 10 and resource_count == 1_000: + return _sized_cluster_profile("large") + scale = max(resource_count / 1_000, node_count / 10) + return _sized_cluster_profile( + "large", + overrides={ + "nodes": node_count, + "guests": resource_count, + "osds": max(10, int(round(100 * scale))), + "ha": max(3, int(round(12 * scale))), + "tasks": max(12, int(round(50 * scale))), + "pool_members": max(12, min(resource_count, int(round(50 * scale)))), + }, + ) + + +def big_profile() -> SeedProfile: + """DATA UI: 20 hosts × 2000 guests.""" + + return _sized_cluster_profile("big") def medium_profile() -> SeedProfile: - nodes = tuple(_node(f"pve{index}") for index in range(1, 4)) - resources: list[SeedResource] = [] - for vmid in range(100, 150): - node = nodes[(vmid - 100) % len(nodes)] - resources.append( - _resource(node, "qemu", str(vmid), {"name": f"vm-{vmid}", "status": "stopped"}) - ) - for vmid in range(200, 220): - node = nodes[(vmid - 200) % len(nodes)] - resources.append( - _resource(node, "lxc", str(vmid), {"name": f"ct-{vmid}", "status": "stopped"}) - ) - for node in nodes: - resources.append( - _resource( - node, - "storage", - f"local-{node.name}", - {"content": ["images"], "shared": False, "status": "available"}, - ) - ) - resources.append( - _resource( - nodes[0], - "storage", - "shared", - {"content": ["images", "backup"], "shared": True, "status": "available"}, - ) - ) - resources.append(_resource(nodes[0], "pool", "development", {"members": ["100", "101", "200"]})) - tasks = tuple(_completed_task(index, "qmstart", str(99 + index)) for index in range(1, 11)) - return SeedProfile("medium", nodes, tuple(resources), tasks) + """Compatibility alias for the small (3×50) cluster.""" - -def large_profile(*, node_count: int = 10, resource_count: int = 10_000) -> SeedProfile: - if node_count < 1 or resource_count < 1: - raise ValueError("large profile counts must be positive") - nodes = tuple(_node(f"pve{index}") for index in range(1, node_count + 1)) - resources = tuple( - _resource( - nodes[index % node_count], - "qemu" if index % 4 else "lxc", - str(100 + index), - {"name": f"guest-{100 + index}", "status": "stopped"}, - ) - for index in range(resource_count) - ) - return SeedProfile("large", nodes, resources) + profile = small_profile() + return SeedProfile("medium", profile.nodes, profile.resources, profile.tasks) def ha_demo_profile() -> SeedProfile: - profile = medium_profile() - resources = ( - *profile.resources, - _resource(profile.nodes[0], "ha", "vm:100", {"state": "started", "group": "primary"}), - ) - return SeedProfile("ha-demo", profile.nodes, resources, profile.tasks) + profile = small_profile() + return SeedProfile("ha-demo", profile.nodes, profile.resources, profile.tasks) def minimal_profile() -> SeedProfile: @@ -1307,7 +2009,7 @@ def minimal_profile() -> SeedProfile: def broken_storage_profile() -> SeedProfile: - profile = small_profile() + profile = lab_profile() resources = tuple( _resource( next(node for node in profile.nodes if node.id == resource.node_id), @@ -1322,15 +2024,17 @@ def broken_storage_profile() -> SeedProfile: return SeedProfile("broken-storage", profile.nodes, resources, profile.tasks) -def build_profile( - name: str, *, large_nodes: int = 10, large_resources: int = 10_000 -) -> SeedProfile: +def build_profile(name: str, *, large_nodes: int = 10, large_resources: int = 1_000) -> SeedProfile: + if name == "lab": + return lab_profile() if name == "small": return small_profile() if name == "medium": return medium_profile() if name == "large": return large_profile(node_count=large_nodes, resource_count=large_resources) + if name == "big": + return big_profile() if name == "ha-demo": return ha_demo_profile() if name == "broken-storage": @@ -1355,7 +2059,7 @@ def _storage_type(resource: SeedResource) -> str: return "zfspool" return "dir" if resource.external_id.startswith("ceph"): - return "ceph" + return "rbd" if resource.external_id.startswith("nfs"): return "nfs" return "dir" @@ -1373,7 +2077,7 @@ async def clear_simulation_state(connection: Connection) -> None: """Remove all mutable simulator state so a seed/reset never fails on leftovers. API-created guests, storages, users, groups, roles, ACL/tokens and custom - realms must not block "Remove demo data" / reseed. Builtin auth realms + realms must not block "Reset to minimal" / reseed. Builtin auth realms (`pam`, `pve`, `test`) are kept because principals reference them. """ for statement in ( @@ -1435,7 +2139,12 @@ async def simulation_state_summary(connection: Connection) -> dict[str, object]: (SELECT count(*)::int FROM snapshots) AS snapshots, (SELECT count(*)::int FROM principals) AS principals, COALESCE( - (SELECT sum(capacity_bytes)::bigint FROM storages WHERE storage_type = 'ceph'), + ( + SELECT sum(capacity_bytes)::bigint + FROM storages + WHERE storage_type IN ('ceph', 'rbd') + OR storage_id LIKE 'ceph%' + ), 0 ) AS ceph_capacity_bytes FROM clusters c @@ -1445,7 +2154,7 @@ async def simulation_state_summary(connection: Connection) -> dict[str, object]: if row is None: return {"profile": "unknown", "loaded": False} payload = dict(row) - payload["loaded"] = payload["profile"] == "demo-cluster" + payload["loaded"] = payload["profile"] in {"small", "large", "big", "demo-cluster"} payload["ceph_capacity_pib"] = round((payload.get("ceph_capacity_bytes") or 0) / 1024**5, 2) return payload @@ -1471,6 +2180,13 @@ async def apply_seed(connection: Connection, profile: SeedProfile) -> None: sort_keys=True, ), ) + slim_guests = profile.name in {"large", "big"} + osd_counts_by_node: dict[uuid.UUID, int] = {} + for resource in profile.resources: + if resource.kind == "ceph-osd": + osd_counts_by_node[resource.node_id] = ( + osd_counts_by_node.get(resource.node_id, 0) + 1 + ) await connection.executemany( "INSERT INTO nodes(id, name, status, metadata) VALUES($1, $2, $3, $4::jsonb)", [ @@ -1478,9 +2194,19 @@ async def apply_seed(connection: Connection, profile: SeedProfile) -> None: node.id, node.name, node.status, - json.dumps({"ops": default_node_ops_for_seed(node.name)}, sort_keys=True), + json.dumps( + { + "ops": default_node_ops_for_seed( + node.name, + node_index=index, + node_count=len(profile.nodes), + osds_per_node=int(osd_counts_by_node.get(node.id, 0)), + ) + }, + sort_keys=True, + ), ) - for node in profile.nodes + for index, node in enumerate(profile.nodes) ], ) await connection.executemany( @@ -1492,7 +2218,7 @@ async def apply_seed(connection: Connection, profile: SeedProfile) -> None: resource.node_id, resource.kind, resource.external_id, - json.dumps(_seed_resource_state(resource), sort_keys=True), + json.dumps(_seed_resource_state(resource, slim=slim_guests), sort_keys=True), ) for resource in profile.resources ], @@ -1506,7 +2232,9 @@ async def apply_seed(connection: Connection, profile: SeedProfile) -> None: ( resource.id, int(resource.external_id), - json.dumps(_seed_resource_state(resource), sort_keys=True), + json.dumps( + _seed_resource_state(resource, slim=slim_guests), sort_keys=True + ), ) for resource in qemu ], @@ -1520,7 +2248,9 @@ async def apply_seed(connection: Connection, profile: SeedProfile) -> None: ( resource.id, int(resource.external_id), - json.dumps(_seed_resource_state(resource), sort_keys=True), + json.dumps( + _seed_resource_state(resource, slim=slim_guests), sort_keys=True + ), ) for resource in containers ], @@ -1540,7 +2270,9 @@ async def apply_seed(connection: Connection, profile: SeedProfile) -> None: _storage_type(resource), bool(resource.state.get("shared", False)), *_storage_capacity(resource), - json.dumps(_seed_resource_state(resource), sort_keys=True), + json.dumps( + _seed_resource_state(resource, slim=slim_guests), sort_keys=True + ), ) for resource in storages ], @@ -1713,6 +2445,94 @@ async def apply_seed(connection: Connection, profile: SeedProfile) -> None: ) if profile.name == "demo-cluster": await _apply_demo_cluster_extras(connection, profile) + elif profile.name in {"lab", "small", "large", "big", "medium", "ha-demo"}: + await _seed_profile_artifacts(connection, profile) + + +async def _seed_profile_artifacts(connection: Connection, profile: SeedProfile) -> None: + """Proportional backups / snapshots / task logs for DATA cluster sizes.""" + + scale = _size_spec(profile.name) + names = {node.id: node.name for node in profile.nodes} + guests = [resource for resource in profile.resources if resource.kind in {"qemu", "lxc"}] + if not guests: + return + + backup_limit = min(int(scale.get("backups", 0)), len(guests)) + snap_limit = min(int(scale.get("snapshots", 0)), len(guests)) + storage_rows = await connection.fetch( + """SELECT s.resource_id, s.storage_id FROM storages s + WHERE s.storage_id IN ('local', 'ceph') ORDER BY s.storage_id""" + ) + storage_by_id = {str(row["storage_id"]): row["resource_id"] for row in storage_rows} + fallback = storage_by_id.get("local") or storage_by_id.get("ceph") + if fallback is not None and backup_limit: + backups: list[tuple[uuid.UUID, uuid.UUID, uuid.UUID, str, int, str]] = [] + for index, resource in enumerate(guests[:backup_limit]): + storage_id = "ceph" if index % 3 == 0 and "ceph" in storage_by_id else "local" + storage_resource = storage_by_id.get(storage_id, fallback) + volume_id = ( + f"{storage_id}:backup/vzdump-{resource.kind}-{resource.external_id}" + f"-2026_07_18-{index:04d}.vma.zst" + ) + backups.append( + ( + stable_id(f"backup:{profile.name}:{resource.external_id}:{index}"), + resource.id, + storage_resource, + volume_id, + (4 + (index % 20)) * 1024**3, + json.dumps( + { + "mode": "snapshot", + "node": names[resource.node_id], + "notes-template": "{{guestname}}", + }, + sort_keys=True, + ), + ) + ) + if backups: + await connection.executemany( + """INSERT INTO backups( + id, resource_id, storage_resource_id, volume_id, size_bytes, metadata + ) VALUES($1, $2, $3, $4, $5, $6::jsonb)""", + backups, + ) + + if snap_limit: + snapshots: list[tuple[uuid.UUID, uuid.UUID, str, str | None, str, str]] = [] + for index, resource in enumerate(guests[:snap_limit]): + snap_name = "baseline" + snapshots.append( + ( + stable_id(f"snapshot:{profile.name}:{resource.external_id}:{snap_name}"), + resource.id, + snap_name, + None, + "Seeded snapshot", + json.dumps({"vmstate": index % 2 == 0}, sort_keys=True), + ) + ) + if snapshots: + await connection.executemany( + """INSERT INTO snapshots(id, resource_id, name, parent_name, description, state) + VALUES($1, $2, $3, $4, $5, $6::jsonb)""", + snapshots, + ) + + if profile.tasks: + logs: list[tuple[uuid.UUID, str]] = [] + for task_index, task in enumerate(profile.tasks, start=1): + for message in (f"starting {task.task_type}", f"finished {task.task_type} OK"): + logs.append((task.id, message)) + if task_index >= 40: + break + if logs: + await connection.executemany( + "INSERT INTO task_logs(task_id, message) VALUES($1, $2)", + logs, + ) async def _apply_demo_cluster_extras(connection: Connection, profile: SeedProfile) -> None: diff --git a/app/surface_probe.py b/app/surface_probe.py index 43250f7..3618900 100644 --- a/app/surface_probe.py +++ b/app/surface_probe.py @@ -27,7 +27,7 @@ from app.contracts.model import Method, Snapshot from app.db.migrations import migrate from app.db.pool import AsyncpgDatabase from app.main import create_app -from app.simulation.seed import apply_seed, small_profile +from app.simulation.seed import apply_seed, lab_profile from app.web.contract_catalog import get_major_releases _BUNDLED_9 = Path( @@ -119,7 +119,7 @@ async def prepare_db(url: str) -> None: connection = await asyncpg.connect(url) try: await migrate(connection) - await apply_seed(connection, small_profile()) + await apply_seed(connection, lab_profile()) finally: await connection.close() diff --git a/app/tasks/backup.py b/app/tasks/backup.py index 7c481de..7d2fbd4 100644 --- a/app/tasks/backup.py +++ b/app/tasks/backup.py @@ -43,6 +43,11 @@ def backup_handler(repository: TaskRepository, clock: Clock) -> TaskHandler: await repository.append_log(task.id, "apt update finished") return {"status": "OK"} + # Node/cluster ops whose durable side effects were applied by the API handler. + if task.task_type != "vzdump": + await repository.append_log(task.id, f"{task.task_type} completed") + return {"status": "OK"} + node = str(task.payload["node"]) vmids = [str(item) for item in task.payload.get("vmids", [])] storage_id = str(task.payload.get("storage") or "nfs-backup") diff --git a/app/tasks/lxc.py b/app/tasks/lxc.py index 9c0e35f..96c9dfc 100644 --- a/app/tasks/lxc.py +++ b/app/tasks/lxc.py @@ -29,6 +29,9 @@ def lxc_handler(repository: TaskRepository, clock: Clock) -> TaskHandler: ) if operation == "migrate" or operation == "remote-migrate": return await _migrate(repository, task, resource_id, clock) + if operation == "resize": + await repository.append_log(task.id, "container resize completed") + return {"status": "OK"} async with repository.pool.acquire() as connection: row = await connection.fetchrow("SELECT state FROM resources WHERE id=$1", resource_id) if row is None: diff --git a/app/tasks/qemu.py b/app/tasks/qemu.py index 4cf136f..e1ea313 100644 --- a/app/tasks/qemu.py +++ b/app/tasks/qemu.py @@ -33,6 +33,10 @@ def qemu_handler(repository: TaskRepository, clock: Clock) -> TaskHandler: return await _migrate(repository, task, resource_id, clock) if operation == "move-disk": return await _move_disk(repository, task, resource_id) + if operation in {"resize", "template"}: + # Side effects already persisted by the API handler; complete the worker. + await repository.append_log(task.id, f"VM {operation} completed") + return {"status": "OK"} async with repository.pool.acquire() as connection: row = await connection.fetchrow("SELECT state FROM resources WHERE id=$1", resource_id) if row is None: diff --git a/app/web/contract_catalog.py b/app/web/contract_catalog.py index 88ef57e..6784833 100644 --- a/app/web/contract_catalog.py +++ b/app/web/contract_catalog.py @@ -11,7 +11,7 @@ from pathlib import Path from app.api.openapi import contract_openapi_tag from app.config import Settings -from app.contracts.examples import path_param_example, schema_example +from app.contracts.examples import path_param_example, schema_example, wire_param_name from app.contracts.importer import RemoteSourceImporter from app.contracts.model import Method, Parameter, Snapshot from app.contracts.normalize import normalize_snapshot @@ -226,16 +226,25 @@ def _path_param_names(path: str) -> tuple[str, ...]: return tuple(match.group(1) for match in _PATH_PARAM.finditer(path)) -def _parameter_payload(parameter: Parameter) -> dict[str, object]: +def _parameter_payload(parameter: Parameter, *, wire_name: str | None = None) -> dict[str, object]: schema = parameter.definition - return { - "name": parameter.name, + name = wire_name or parameter.name + typetext = schema.extra.get("typetext") + payload: dict[str, object] = { + "name": name, "type": schema.type, "description": schema.description, "optional": bool(schema.optional), "enum": list(schema.enum), "example": schema_example(schema, name=parameter.name), } + if wire_name and wire_name != parameter.name: + payload["template"] = parameter.name + if isinstance(schema.format, str): + payload["format"] = schema.format + if isinstance(typetext, str) and typetext: + payload["typetext"] = typetext + return payload def method_payload( @@ -274,15 +283,20 @@ def method_payload( "example": path_param_example(name) or name, } ) - body_fields = [ + contract_body_fields = [ _parameter_payload(parameter) for parameter in method.parameters if parameter.name not in path_params and "[n]" not in parameter.name ] + # Contract uses foo[n]; the wire form is foo0..fooN — seed the console with foo0. indexed_fields = [ - _parameter_payload(parameter) for parameter in method.parameters if "[n]" in parameter.name + _parameter_payload(parameter, wire_name=wire_param_name(parameter.name)) + for parameter in method.parameters + if "[n]" in parameter.name ] body_example = _body_example(method, path_params) + # Flatten nested scalars from body_example (and nested field examples) into PARAMS. + body_fields = _body_fields_with_nested(contract_body_fields, body_example) resolved_path = _resolve_path(path, path_fields) implemented = ( (path, method.verb.upper()) in implemented_methods @@ -316,21 +330,133 @@ def _resolve_path(path: str, path_fields: list[dict[str, object]]) -> str: def _body_example(method: Method, path_params: tuple[str, ...]) -> dict[str, object]: + """Minimal request body: required parameters only, Proxmox wire names/values.""" + body: dict[str, object] = {} for parameter in method.parameters: if parameter.name in path_params: continue - if "[n]" in parameter.name: - concrete = parameter.name.replace("[n]", "0") - if not parameter.definition.optional: - body[concrete] = schema_example(parameter.definition, name=concrete) - continue if parameter.definition.optional: continue - body[parameter.name] = schema_example(parameter.definition, name=parameter.name) + wire = wire_param_name(parameter.name) + body[wire] = schema_example(parameter.definition, name=parameter.name) return body +def _leaf_type(value: object) -> str: + if isinstance(value, bool): + return "boolean" + if isinstance(value, int) and not isinstance(value, bool): + return "integer" + if isinstance(value, float): + return "number" + return "string" + + +def _body_fields_from_example(body_example: dict[str, object] | None) -> list[dict[str, object]]: + """PARAM inputs derived from body_example, including nested scalar paths.""" + + if not isinstance(body_example, dict) or not body_example: + return [] + inner: object = body_example + # Soft unwrap a single root object envelope (Engine-style), never a scalar/list. + if len(body_example) == 1: + only = next(iter(body_example.values())) + if isinstance(only, dict): + inner = only + if not isinstance(inner, dict): + return [] + + fields: list[dict[str, object]] = [] + + def _walk(prefix: str, value: object) -> None: + if isinstance(value, dict): + for key, child in value.items(): + path = f"{prefix}.{key}" if prefix else str(key) + _walk(path, child) + return + if isinstance(value, list): + for index, child in enumerate(value): + path = f"{prefix}.{index}" if prefix else str(index) + _walk(path, child) + return + fields.append( + { + "name": prefix, + "type": _leaf_type(value), + "description": prefix, + "optional": True, + "enum": [], + "example": value, + } + ) + + _walk("", inner) + return fields + + +def _body_fields_with_nested( + contract_fields: list[dict[str, object]], + body_example: dict[str, object], +) -> list[dict[str, object]]: + """Merge contract PARAMS with nested scalar paths from body_example.""" + + example_fields = _body_fields_from_example(body_example) + nested_tops = { + str(item["name"]).split(".", 1)[0] for item in example_fields if "." in str(item["name"]) + } + contract_by_name = {str(item["name"]): item for item in contract_fields} + merged: list[dict[str, object]] = [] + seen: set[str] = set() + + for item in example_fields: + name = str(item["name"]) + top = name.split(".", 1)[0] + parent = contract_by_name.get(name) or contract_by_name.get(top) + row = dict(item) + if parent is not None: + row["optional"] = bool(parent.get("optional")) + description = parent.get("description") + if isinstance(description, str) and description: + row["description"] = description + typetext = parent.get("typetext") + if isinstance(typetext, str) and typetext: + row["typetext"] = typetext + fmt = parent.get("format") + if isinstance(fmt, str) and fmt: + row["format"] = fmt + else: + # body_example is required-only — treat uncovered leaves as required. + row["optional"] = False + merged.append(row) + seen.add(name) + + for field in contract_fields: + name = str(field["name"]) + if name in nested_tops or name in seen: + continue + example = field.get("example") + if isinstance(example, dict | list): + nested = _body_fields_from_example({name: example}) + if nested: + for item in nested: + nested_name = str(item["name"]) + if nested_name in seen: + continue + row = dict(item) + row["optional"] = bool(field.get("optional")) + description = field.get("description") + if isinstance(description, str) and description: + row["description"] = description + merged.append(row) + seen.add(nested_name) + continue + # Empty object/array examples stay as a top-level PARAM. + merged.append(field) + seen.add(name) + return merged + + @lru_cache(maxsize=1) def default_store_root() -> Path: return _DEFAULT_STORE diff --git a/app/web/index.html b/app/web/index.html index c9a90a2..5cfd2fd 100644 --- a/app/web/index.html +++ b/app/web/index.html @@ -583,6 +583,123 @@ padding-top: 2px; } + .demo-size-grid { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 12px; + margin-top: 4px; + } + + @media (max-width: 900px) { + .demo-size-grid { + grid-template-columns: 1fr; + } + } + + .demo-size-card { + display: flex; + flex-direction: column; + gap: 10px; + padding: 14px 14px 12px; + border: 1px solid var(--border); + border-radius: 10px; + background: + linear-gradient(165deg, color-mix(in srgb, var(--brand-accent) 8%, var(--surface-raised)) 0%, var(--surface-raised) 55%); + min-height: 168px; + } + + .demo-size-card.is-active { + border-color: color-mix(in srgb, var(--brand-accent) 55%, var(--border)); + box-shadow: inset 0 0 0 1px color-mix(in srgb, var(--brand-accent) 25%, transparent); + } + + .demo-size-card-head { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 8px; + } + + .demo-size-card-title { + margin: 0; + font-size: 15px; + font-weight: 700; + letter-spacing: 0.01em; + color: var(--text); + text-transform: capitalize; + } + + .demo-size-card-chip { + font-size: 10px; + font-weight: 700; + letter-spacing: 0.06em; + text-transform: uppercase; + color: var(--brand-accent); + background: color-mix(in srgb, var(--brand-accent) 12%, transparent); + border-radius: 999px; + padding: 3px 8px; + white-space: nowrap; + } + + .demo-size-card-metrics { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 6px 10px; + margin: 0; + } + + .demo-size-card-metrics div { + display: flex; + flex-direction: column; + gap: 1px; + } + + .demo-size-card-metrics dt { + margin: 0; + font-size: 10px; + text-transform: uppercase; + letter-spacing: 0.04em; + color: var(--muted); + } + + .demo-size-card-metrics dd { + margin: 0; + font-size: 13px; + font-weight: 600; + color: var(--text); + } + + .demo-size-card-note { + margin: 0; + font-size: 11px; + line-height: 1.4; + color: var(--muted); + flex: 1; + } + + .demo-size-card .btn-demo-load { + width: 100%; + justify-content: center; + margin-top: auto; + } + + .demo-toolbar { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 8px; + width: 100%; + margin-top: 12px; + padding-top: 12px; + border-top: 1px solid var(--border); + } + + .demo-toolbar .btn { + width: 100%; + box-sizing: border-box; + justify-content: center; + text-align: center; + } + .btn-demo-load { background: var(--brand-accent); border-color: var(--brand-accent); @@ -5371,11 +5488,45 @@ function buildDemoDataHtml(data) { const loaded = Boolean(data?.loaded); + const profile = data?.profile ?? "minimal"; + const badgeLabel = loaded ? String(profile) : "Minimal"; const cephPiB = data?.ceph_capacity_pib; const cephText = cephPiB != null && cephPiB > 0 ? `${cephPiB} PiB` : "—"; const tasksText = data?.tasks != null ? `${data.tasks} / ${data.task_logs ?? 0}` : "—"; + const sizes = [ + { + id: "small", + label: "Small", + chip: "lab", + hosts: 3, + guests: 50, + osd: 10, + backups: 10, + note: "3 hosts · 50 guests · 10 OSD · HA/replication light", + }, + { + id: "large", + label: "Large", + chip: "default", + hosts: 10, + guests: 1000, + osd: 100, + backups: 80, + note: "10 hosts · 1000 guests · 100 OSD · fuller Ceph/HA", + }, + { + id: "big", + label: "Big", + chip: "heavy", + hosts: 20, + guests: 2000, + osd: 500, + backups: 160, + note: "20 hosts · 2000 guests · 500 OSD · heavy Ceph/HA", + }, + ]; const stats = [ - ["Profile", data?.profile ?? "—", loaded ? "ok" : ""], + ["Profile", profile, loaded ? "ok" : ""], ["Cluster", data?.cluster_name ?? "—", ""], ["Nodes", data?.nodes ?? "—", ""], ["QEMU", data?.qemu ?? "—", ""], @@ -5390,13 +5541,12 @@ return `
-

Demo cluster

-

Load a realistic enterprise environment into the emulator database for hands-on API exploration.

- ${loaded ? "Loaded" : "Empty"} +

Proxmox demo cluster

+

Pick a cluster size — inventory (nodes, guests, Ceph, HA, backups, tasks) scales together.

+ ${escapeHtml(badgeLabel)}

- 20 nodes · 1000 guests (850 QEMU + 150 LXC) · Ceph 300 OSD (~5 PiB) · backups, logs, - snapshots, pools, HA rules, and multi-level users. Loading replaces current simulation state. + Password for seeded users: secret. Loading replaces current Proxmox state.

${stats.map(([label, value, tone]) => ` @@ -5406,9 +5556,30 @@
`).join("")}
-
- - +
+ ${sizes.map((s) => { + const active = profile === s.id; + return ` +
+
+

${escapeHtml(s.label)}

+ ${escapeHtml(s.chip)} +
+
+
Hosts
${s.hosts}
+
Guests
${s.guests}
+
OSD
${s.osd}
+
Backups
${s.backups}
+
+

${escapeHtml(s.note)}

+ +
`; + }).join("")} +
+
+
@@ -5446,35 +5617,45 @@ renderDemoState(await res.json()); } - async function loadDemoData() { - const loadBtn = document.getElementById("btn-demo-load"); - if (loadBtn) loadBtn.disabled = true; + async function loadDemoData(size = "small") { + const loadBtns = document.querySelectorAll("[data-demo-size]"); + loadBtns.forEach((btn) => { btn.disabled = true; }); setLoading(true); try { - const res = await fetch("/ui/api/demo/load", { method: "POST" }); + const res = await fetch(`/ui/api/demo/load?profile=${encodeURIComponent(size)}`, { + method: "POST", + }); const body = await res.json().catch(() => ({})); if (!res.ok) { - throw new Error(body.detail || `Load failed: ${res.status}`); + const detail = Array.isArray(body.detail) + ? body.detail.map((d) => d.msg || d).join("; ") + : body.detail; + throw new Error(detail || `Load failed: ${res.status}`); } renderDemoState(body.summary || {}); if (state.ticket) syncAuthCookie(state.ticket); refreshOverview(); - toast("Demo cluster loaded", "ok"); + const seed = body.summary || {}; + const guests = (seed.qemu ?? 0) + (seed.lxc ?? 0); + toast( + `Cluster ${seed.profile || size} loaded (${seed.nodes ?? "?"} hosts · ${guests || "?"} guests)`, + "ok", + ); if (!state.ticket) { toast("Sign in (root@pam / secret) to call API endpoints", "warn"); } } finally { - if (loadBtn) loadBtn.disabled = false; + loadBtns.forEach((btn) => { btn.disabled = false; }); setLoading(false); } } async function unloadDemoData() { const confirmed = await showConfirm({ - title: "Remove demo data?", - message: "This wipes simulator DB state (demo data and anything created via the API), then reloads a minimal cluster.", - confirmLabel: "Remove demo data", - cancelLabel: "Keep demo data", + title: "Reset to minimal?", + message: "This wipes current simulator DB state (sized demo and anything created via the API), then loads the minimal cluster (1 host · 0 guests).", + confirmLabel: "Reset to minimal", + cancelLabel: "Keep current data", tone: "danger", }); if (!confirmed) return; @@ -5489,7 +5670,7 @@ } renderDemoState(body.summary || {}); refreshOverview(); - toast("Demo data removed", "info"); + toast("Reset to minimal cluster", "info"); } finally { if (unloadBtn) unloadBtn.disabled = false; setLoading(false); @@ -5730,17 +5911,21 @@ if (!els.dataPanel || els.dataPanel.dataset.demoBound) return; els.dataPanel.dataset.demoBound = "1"; els.dataPanel.addEventListener("click", (event) => { - const target = event.target.closest("#btn-demo-load, #btn-demo-unload, #btn-demo-refresh"); - if (!target) return; - if (target.id === "btn-demo-load") { - loadDemoData().catch((error) => { + const loadBtn = event.target.closest("[data-demo-size]"); + if (loadBtn) { + const size = loadBtn.getAttribute("data-demo-size") || "small"; + loadDemoData(size).catch((error) => { showError(String(error)); - toast("Failed to load demo data", "error"); + toast("Failed to load cluster data", "error"); }); - } else if (target.id === "btn-demo-unload") { + return; + } + const target = event.target.closest("#btn-demo-unload, #btn-demo-refresh"); + if (!target) return; + if (target.id === "btn-demo-unload") { unloadDemoData().catch((error) => { showError(String(error)); - toast("Failed to remove demo data", "error"); + toast("Failed to reset to minimal", "error"); }); } else if (target.id === "btn-demo-refresh") { renderDataPanel().catch((error) => showError(String(error))); @@ -6281,12 +6466,24 @@ } container.innerHTML = fields.map((f) => { const stored = kind === "path" ? state.pathValues[f.name] : state.bodyValues[f.name]; - const val = stored ?? f.example ?? ""; - const desc = f.description ? `
${f.description}
` : ""; + const fallback = kind === "path" || !f.optional ? formatParamExample(f.example) : ""; + const val = stored ?? fallback; + const hint = f.typetext || f.format || ""; + const descParts = []; + if (f.description) descParts.push(f.description); + if (hint) descParts.push(`${escapeHtml(String(hint))}`); + if (f.template) descParts.push(`${escapeHtml(f.template)} → ${escapeHtml(f.name)}`); + const desc = descParts.length + ? `
${descParts.join(" · ")}
` + : ""; + const placeholderExample = formatParamExample(f.example); + const placeholder = f.optional && placeholderExample + ? ` placeholder="${placeholderExample.replaceAll('"', """)}"` + : ""; return ` - ${f.name}${desc} + ${escapeHtml(f.name)}${desc} ${formatParamTypeBadges(f)} - + `; }).join(""); container.querySelectorAll("input[data-field]").forEach((input) => { @@ -6393,9 +6590,11 @@ try { const parsed = JSON.parse(body || "{}"); if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + const { inner } = unwrapBodyExample(parsed); for (const field of bodyFields) { - if (Object.prototype.hasOwnProperty.call(parsed, field.name)) { - state.bodyValues[field.name] = String(parsed[field.name] ?? ""); + const value = getByPath(inner, wireBodyFieldName(field.name)); + if (value !== undefined) { + state.bodyValues[field.name] = formatParamExample(value); } } } @@ -6468,31 +6667,106 @@ return target === "cluster" ? buildClusterUrl(apiPath) : buildEmulatorUrl(apiPath); } + function wireBodyFieldName(name) { + // Contract templates use foo[n]; wire / body keys are foo0 (catalog already expands). + return String(name || "").includes("[n]") ? String(name).replaceAll("[n]", "0") : String(name || ""); + } + + function deepCloneJson(value) { + if (value == null) return value; + return JSON.parse(JSON.stringify(value)); + } + + function getByPath(root, path) { + if (!path) return root; + const parts = String(path).split("."); + let cur = root; + for (const part of parts) { + if (cur == null || typeof cur !== "object") return undefined; + cur = cur[part]; + } + return cur; + } + + function setByPath(root, path, value) { + const parts = String(path).split("."); + let cur = root; + for (let i = 0; i < parts.length - 1; i += 1) { + const part = parts[i]; + const next = parts[i + 1]; + const wantArray = /^\d+$/.test(next); + if (cur[part] == null || typeof cur[part] !== "object") { + cur[part] = wantArray ? [] : {}; + } + cur = cur[part]; + } + cur[parts[parts.length - 1]] = value; + } + + function deleteByPath(root, path) { + const parts = String(path).split("."); + let cur = root; + for (let i = 0; i < parts.length - 1; i += 1) { + if (cur == null || typeof cur !== "object") return; + cur = cur[parts[i]]; + } + if (cur && typeof cur === "object") delete cur[parts[parts.length - 1]]; + } + + function unwrapBodyExample(example) { + if (!example || typeof example !== "object" || Array.isArray(example)) { + return { wrapKey: null, inner: {} }; + } + const keys = Object.keys(example); + if ( + keys.length === 1 + && example[keys[0]] + && typeof example[keys[0]] === "object" + && !Array.isArray(example[keys[0]]) + ) { + return { wrapKey: keys[0], inner: example[keys[0]] }; + } + return { wrapKey: null, inner: example }; + } + + function formatParamExample(value) { + if (value == null) return ""; + if (typeof value === "object") return JSON.stringify(value); + return String(value); + } + function syncBodyFromFields({ syncVisibility = true } = {}) { if (!state.method) return; const inputs = [...(els.bodyFields?.querySelectorAll("input[data-field]") || [])]; - const body = {}; + const example = state.method.body_example; + const { wrapKey, inner } = unwrapBodyExample(example); + const root = deepCloneJson(inner) || {}; + let touched = false; inputs.forEach((input) => { - const name = input.dataset.field; + const name = wireBodyFieldName(input.dataset.field); + if (!name) return; const raw = input.value.trim(); - if (!raw) return; - if (raw === "true" || raw === "false") body[name] = raw === "true"; - else if (/^-?\d+$/.test(raw)) body[name] = Number(raw); - else if (/^-?\d+\.\d+$/.test(raw)) body[name] = Number(raw); - else body[name] = raw; - }); - // If PARAM fields are empty, fall back to method body_example so the editor - // updates when switching verbs / endpoints. - if (!Object.keys(body).length) { - const example = state.method.body_example; - if (example && typeof example === "object" && !Array.isArray(example)) { - for (const [key, value] of Object.entries(example)) { - if (value === undefined || value === null || value === "") continue; - body[key] = value; - } + const existing = getByPath(root, name); + if (!raw && existing === undefined) return; + touched = true; + if (!raw) { + deleteByPath(root, name); + return; } + let value = raw; + if (raw === "true" || raw === "false") value = raw === "true"; + else if (/^-?\d+$/.test(raw)) value = Number(raw); + else if (/^-?\d+\.\d+$/.test(raw)) value = Number(raw); + setByPath(root, name, value); + }); + // Empty PARAM inputs → keep full body_example; edits merge into nested paths. + if (!touched && example && typeof example === "object" && !Array.isArray(example)) { + els.body.value = Object.keys(example).length ? JSON.stringify(example, null, 2) : ""; + updateBodyHighlight(); + if (syncVisibility) updateBodyPaneVisibility(state.method); + return; } - // Keep empty object out of the editor — do not show `{}`. + const body = wrapKey ? { [wrapKey]: root } : root; els.body.value = Object.keys(body).length ? JSON.stringify(body, null, 2) : ""; updateBodyHighlight(); if (syncVisibility) { @@ -6547,7 +6821,10 @@ const bodyFields = [...(d.body_fields || []), ...(d.indexed_fields || [])]; state.bodyValues = {}; for (const f of bodyFields) { - state.bodyValues[f.name] = resolveParamValue(f.name, storedParams?.bodyValues, f.example); + // Seed required fields from the contract/example; leave optional empty + // (placeholder shows the typetext-shaped example until the user fills it). + const fallback = f.optional ? "" : formatParamExample(f.example); + state.bodyValues[f.name] = resolveParamValue(f.name, storedParams?.bodyValues, fallback); } els.bodyFieldsWrap.hidden = !bodyFields.length; renderParamRows(els.bodyFields, bodyFields, "body"); diff --git a/app/web/routes.py b/app/web/routes.py index 459b4d5..00a76eb 100644 --- a/app/web/routes.py +++ b/app/web/routes.py @@ -161,7 +161,23 @@ async def ui_demo_state(request: Request) -> JSONResponse: @router.post("/ui/api/demo/load", include_in_schema=False) async def ui_demo_load(request: Request) -> JSONResponse: pool = _database_pool(request) - profile = build_profile("demo-cluster") + requested = "small" + if request.query_params.get("profile"): + requested = str(request.query_params.get("profile")) + else: + try: + body = await request.json() + except Exception: + body = None + if isinstance(body, dict) and body.get("profile"): + requested = str(body["profile"]) + allowed = {"small", "large", "big"} + if requested not in allowed: + raise HTTPException( + status_code=400, + detail=f"unknown cluster profile '{requested}' (expected: small, large, big)", + ) + profile = build_profile(requested) async with pool.acquire() as connection: await apply_seed(connection, profile) summary = await simulation_state_summary(connection) @@ -179,7 +195,7 @@ async def ui_demo_unload(request: Request) -> JSONResponse: summary = await simulation_state_summary(connection) except Exception as error: raise HTTPException( - status_code=500, detail=f"failed to remove demo data: {error}" + status_code=500, detail=f"failed to reset to minimal cluster: {error}" ) from error return JSONResponse({"ok": True, "profile": profile.name, "summary": summary}) diff --git a/docs/getting-started.md b/docs/getting-started.md index b34f67d..982551b 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -150,8 +150,8 @@ curl -sS -H "Cookie: PVEAuthCookie=$TICKET" \ Visit [http://localhost:8006/](http://localhost:8006/) for the interactive console, contract catalog (PVE 6–9), compatibility view, runtime contract apply, -and demo-cluster controls. See [Web UI](web-ui.md) for light/dark theme -screenshots and the full feature list. +and demo-cluster controls. See [Web UI](web-ui.md) for screenshots and the +full feature list. ## 8. Try a client library diff --git a/docs/images/web-ui-api-catalog.png b/docs/images/web-ui-api-catalog.png new file mode 100644 index 0000000..f33d1a5 Binary files /dev/null and b/docs/images/web-ui-api-catalog.png differ diff --git a/docs/images/web-ui-authentication.png b/docs/images/web-ui-authentication.png new file mode 100644 index 0000000..7c480e6 Binary files /dev/null and b/docs/images/web-ui-authentication.png differ diff --git a/docs/images/web-ui-dark.png b/docs/images/web-ui-dark.png index fe2a1dd..4cb4532 100644 Binary files a/docs/images/web-ui-dark.png and b/docs/images/web-ui-dark.png differ diff --git a/docs/images/web-ui-data.png b/docs/images/web-ui-data.png new file mode 100644 index 0000000..032a44e Binary files /dev/null and b/docs/images/web-ui-data.png differ diff --git a/docs/images/web-ui-endpoints.png b/docs/images/web-ui-endpoints.png new file mode 100644 index 0000000..291deba Binary files /dev/null and b/docs/images/web-ui-endpoints.png differ diff --git a/docs/images/web-ui-environment.png b/docs/images/web-ui-environment.png new file mode 100644 index 0000000..240cb14 Binary files /dev/null and b/docs/images/web-ui-environment.png differ diff --git a/docs/images/web-ui-help.png b/docs/images/web-ui-help.png new file mode 100644 index 0000000..031b43b Binary files /dev/null and b/docs/images/web-ui-help.png differ diff --git a/docs/images/web-ui-history.png b/docs/images/web-ui-history.png new file mode 100644 index 0000000..0b6bdd8 Binary files /dev/null and b/docs/images/web-ui-history.png differ diff --git a/docs/images/web-ui-light.png b/docs/images/web-ui-light.png index 44ce7da..d56e13d 100644 Binary files a/docs/images/web-ui-light.png and b/docs/images/web-ui-light.png differ diff --git a/docs/images/web-ui-main.png b/docs/images/web-ui-main.png new file mode 100644 index 0000000..9d9fd92 Binary files /dev/null and b/docs/images/web-ui-main.png differ diff --git a/docs/images/web-ui-request-params.png b/docs/images/web-ui-request-params.png new file mode 100644 index 0000000..25e0721 Binary files /dev/null and b/docs/images/web-ui-request-params.png differ diff --git a/docs/images/web-ui-request.png b/docs/images/web-ui-request.png new file mode 100644 index 0000000..81cf69b Binary files /dev/null and b/docs/images/web-ui-request.png differ diff --git a/docs/images/web-ui-upid.png b/docs/images/web-ui-upid.png new file mode 100644 index 0000000..d2f1667 Binary files /dev/null and b/docs/images/web-ui-upid.png differ diff --git a/docs/ru/getting-started.md b/docs/ru/getting-started.md index 975ba55..48b1eeb 100644 --- a/docs/ru/getting-started.md +++ b/docs/ru/getting-started.md @@ -151,8 +151,8 @@ curl -s -H "Cookie: PVEAuthCookie=$TICKET" \ Перейдите на [http://localhost:8006/](http://localhost:8006/) — интерактивная консоль, каталог контрактов (PVE 6–9), представление совместимости, применение runtime- -контракта и управление demo-кластером. Скриншоты светлой/тёмной темы и полный список -возможностей — в [Web UI](web-ui.md). +контракта и управление demo-кластером. Скриншоты и полный список возможностей — +в [Web UI](web-ui.md). ## 8. Попробуйте клиентскую библиотеку diff --git a/docs/ru/web-ui.md b/docs/ru/web-ui.md index 2be1f07..d09a2f6 100644 --- a/docs/ru/web-ui.md +++ b/docs/ru/web-ui.md @@ -10,14 +10,56 @@ UI — лабораторная консоль симулятора, а не п ## Скриншоты -Светлая тема — `GET /cluster/resources` на PVE 9.2.3: +Главный экран консоли (тёмная тема, PVE 8.4.5): + +![Web UI main](../images/web-ui-main.png) + +Светлая / тёмная темы: ![Web UI light theme](../images/web-ui-light.png) -Тёмная тема — та же консоль с переключателем темы: - ![Web UI dark theme](../images/web-ui-dark.png) +Панель Endpoints (методы каталога выбранного мажора): + +![Web UI endpoints](../images/web-ui-endpoints.png) + +Каталог API — мажоры 6–9, покрытие и **Apply as runtime**: + +![Web UI API catalog](../images/web-ui-api-catalog.png) + +Редактор запроса (`POST /cluster/config`): + +![Web UI request](../images/web-ui-request.png) + +Параметры запроса из контракта: + +![Web UI request parameters](../images/web-ui-request-params.png) + +Аутентификация (`root@pam` / API-токен): + +![Web UI authentication](../images/web-ui-authentication.png) + +Сводка окружения (runtime, кластер, гости): + +![Web UI environment](../images/web-ui-environment.png) + +История запросов: + +![Web UI history](../images/web-ui-history.png) + +Справка / ledger совместимости: + +![Web UI help](../images/web-ui-help.png) + +Загрузка demo-кластера: + +![Web UI data](../images/web-ui-data.png) + +Монитор задач UPID: + +![Web UI UPID](../images/web-ui-upid.png) + ## Возможности - Дерево эндпоинтов и выбор метода по выбранному мажору каталога diff --git a/docs/web-ui.md b/docs/web-ui.md index 8358b93..4fbe40a 100644 --- a/docs/web-ui.md +++ b/docs/web-ui.md @@ -10,14 +10,56 @@ request/response editing, history, and runtime contract apply. ## Screenshots -Light theme — `GET /cluster/resources` on PVE 9.2.3: +Console home (dark theme, PVE 8.4.5): + +![Web UI main](images/web-ui-main.png) + +Light / dark themes: ![Web UI light theme](images/web-ui-light.png) -Dark theme — same console with theme toggle: - ![Web UI dark theme](images/web-ui-dark.png) +Endpoints drawer (catalog methods for the selected major): + +![Web UI endpoints](images/web-ui-endpoints.png) + +API catalog — majors 6–9, coverage, and **Apply as runtime**: + +![Web UI API catalog](images/web-ui-api-catalog.png) + +Request editor (`POST /cluster/config`): + +![Web UI request](images/web-ui-request.png) + +Contract-derived request parameters: + +![Web UI request parameters](images/web-ui-request-params.png) + +Authentication (`root@pam` / API token): + +![Web UI authentication](images/web-ui-authentication.png) + +Environment summary (runtime, cluster, guests): + +![Web UI environment](images/web-ui-environment.png) + +Request history: + +![Web UI history](images/web-ui-history.png) + +Help / compatibility ledger: + +![Web UI help](images/web-ui-help.png) + +Demo-cluster load controls: + +![Web UI data](images/web-ui-data.png) + +UPID task monitor: + +![Web UI UPID](images/web-ui-upid.png) + ## Features - Endpoint tree and method selector driven by the selected catalog major diff --git a/pulumi-tests/README.md b/pulumi-tests/README.md index a2e27a7..3378fd7 100644 --- a/pulumi-tests/README.md +++ b/pulumi-tests/README.md @@ -6,10 +6,12 @@ Docker-only suite against the Proxmox API simulator. | Layer | What it covers | |---|---| -| **Surface** | **100%** of declared contract methods for PVE majors **6–9** (every path+verb) via HTTP (`pvelib/surface.py`). Suite PASS requires `declared == probed` per major and zero critical failures (501, “not supported” messages, 5xx, exceptions, unknown verbs). | -| **Lifecycle** | **`pulumi-proxmoxve` only** (BPG Terraform bridge) — Provider, inventory data sources, `VmLegacy` + non-empty checks. Negative auth via httpx. This is **not** full API coverage; the provider exposes dozens of resources, not thousands of contract methods. Provider uses internal HTTPS gateway; surface stays HTTP. | +| **A — Surface (HTTP contract matrix)** | **100%** means every declared contract method for PVE majors **6–9** (path+verb: GET/PUT/POST/DELETE), plus **synthetic HEAD on each GET path**. Ticket + CSRF on mutations; form-urlencoded bodies; Proxmox `{ "data": … }` envelope. PASS requires `declared == probed` per major **and** aggregate, `critical=0` (501, “not supported”, 5xx, exceptions, empty/wrong 2xx envelopes). HEAD is in the verb histogram but **not** in the declared/probed denominator. | +| **B — Lifecycle** | **`pulumi-proxmoxve` smoke only** (BPG Terraform bridge) — Provider, inventory data sources, `VmLegacy` + non-empty checks. **Not** 100% API and **not** provider resource-count coverage. HTTPS via lab TLS gateway; surface stays HTTP. | -Reports (`pulumi/reports/report.html`, `results.json`, `junit.xml`) include a **Full contract coverage** summary with per-major and total `declared`/`probed` counts (e.g. `Coverage: 2324/2324 methods across majors 6–9`). +Reports (`pulumi/reports/report.html`, `results.json`, `junit.xml`) include **Full contract coverage**, a **verb histogram (incl. HEAD)**, and a line like: + +`Coverage: N/N methods across majors 6–9 (critical=0)` ## Layout @@ -43,7 +45,26 @@ open pulumi/reports/report.html make down ``` +### Quick curl (ticket → GET/POST + report) + +Against a running suite stack (`http://localhost:8006`): + +```bash +TICKET=$(curl -s -c /tmp/pve.ck -b /tmp/pve.ck \ + -d 'username=root@pam&password=secret' \ + http://localhost:8006/api2/json/access/ticket) +echo "$TICKET" | jq . +CSRF=$(echo "$TICKET" | jq -r .data.CSRFPreventionToken) +curl -s -b /tmp/pve.ck http://localhost:8006/api2/json/version | jq . +curl -s -b /tmp/pve.ck -H "CSRFPreventionToken: $CSRF" \ + -d 'vmid=100&name=demo' \ + http://localhost:8006/api2/json/nodes/pve01/qemu | jq . +``` + +Open the HTML report: `pulumi-tests/pulumi/reports/report.html`. + Provider env (set by Compose): `PROXMOX_VE_ENDPOINT=https://tls-gateway:8443/` (internal suite TLS — `pulumi-proxmoxve` rejects `http://`), plus username/password /`INSECURE`. Surface probe uses `API_URL=http://simulator:8006`. Host lab URL remains plain `http://localhost:8006/` (Kubernetes HTTPS is Ingress-only). +Seed for CI/suite: `SEED_PROFILE=small` (Compose default). diff --git a/pulumi-tests/README.ru.md b/pulumi-tests/README.ru.md index 22a19ba..5bf2d09 100644 --- a/pulumi-tests/README.ru.md +++ b/pulumi-tests/README.ru.md @@ -6,10 +6,12 @@ Suite только в Docker против симулятора Proxmox API. | Слой | Покрытие | |---|---| -| **Surface** | **100%** объявленных методов контракта majors **6–9** (каждый path+verb) по HTTP (`pvelib/surface.py`). PASS только при `declared == probed` на каждый major и нуле критичных сбоев (501, «not supported», 5xx, exceptions, неизвестные verb). | -| **Lifecycle** | Только **`pulumi-proxmoxve`** (мост BPG) — Provider, data sources инвентаря, `VmLegacy` + проверки непустых outputs. Это **не** полное покрытие API: в провайдере десятки ресурсов, не тысячи contract methods. HTTPS через lab TLS-шлюз; surface — HTTP. | +| **A — Surface (HTTP contract matrix)** | **100%** = каждый объявленный метод контракта majors **6–9** (path+verb: GET/PUT/POST/DELETE) плюс **синтетический HEAD на каждый GET path**. Ticket + CSRF на мутациях; form-urlencoded; оболочка `{ "data": … }`. PASS при `declared == probed` на major и в сумме, `critical=0` (501, «not supported», 5xx, exceptions, пустые/неверные 2xx envelope). HEAD — в гистограмме глаголов, **не** в знаменателе declared/probed. | +| **B — Lifecycle** | Только smoke **`pulumi-proxmoxve`** (мост BPG) — Provider, data sources инвентаря, `VmLegacy` + непустые outputs. Это **не** 100% API и **не** покрытие по числу ресурсов провайдера. HTTPS через lab TLS-шлюз; surface — HTTP. | -В отчётах (`pulumi/reports/report.html`, `results.json`, `junit.xml`) — блок **Full contract coverage** с итогами `declared`/`probed` по major и суммой (например `Coverage: 2324/2324 methods across majors 6–9`). +В отчётах (`pulumi/reports/report.html`, `results.json`, `junit.xml`) — **Full contract coverage**, **verb histogram (вкл. HEAD)** и строка вида: + +`Coverage: N/N methods across majors 6–9 (critical=0)` ## Структура @@ -43,7 +45,26 @@ open pulumi/reports/report.html make down ``` +### Короткий curl (ticket → GET/POST + отчёт) + +При поднятом стеке (`http://localhost:8006`): + +```bash +TICKET=$(curl -s -c /tmp/pve.ck -b /tmp/pve.ck \ + -d 'username=root@pam&password=secret' \ + http://localhost:8006/api2/json/access/ticket) +echo "$TICKET" | jq . +CSRF=$(echo "$TICKET" | jq -r .data.CSRFPreventionToken) +curl -s -b /tmp/pve.ck http://localhost:8006/api2/json/version | jq . +curl -s -b /tmp/pve.ck -H "CSRFPreventionToken: $CSRF" \ + -d 'vmid=100&name=demo' \ + http://localhost:8006/api2/json/nodes/pve01/qemu | jq . +``` + +Отчёт: `pulumi-tests/pulumi/reports/report.html`. + Env провайдера (Compose): `PROXMOX_VE_ENDPOINT=https://tls-gateway:8443/` (внутренний TLS suite — `pulumi-proxmoxve` не принимает `http://`), плюс username/password/`INSECURE`. Surface probe: `API_URL=http://simulator:8006`. Хостовый lab URL — plain `http://localhost:8006/` (HTTPS в K8s — только Ingress). +Seed для CI/suite: `SEED_PROFILE=small` (default Compose). diff --git a/pulumi-tests/pulumi/programs/lifecycle/Pulumi.hxlife11784346334.yaml b/pulumi-tests/pulumi/programs/lifecycle/Pulumi.hxlife11784346334.yaml new file mode 100644 index 0000000..f0fba4d --- /dev/null +++ b/pulumi-tests/pulumi/programs/lifecycle/Pulumi.hxlife11784346334.yaml @@ -0,0 +1 @@ +encryptionsalt: v1:Im3SuXTFiBk=:v1:oUxFIClVSpmc6Jkb:AXfMbHESIZWB3tnQLGNdQNp2ICM+UA== diff --git a/pulumi-tests/pulumi/programs/lifecycle/Pulumi.hxlife11784346404.yaml b/pulumi-tests/pulumi/programs/lifecycle/Pulumi.hxlife11784346404.yaml new file mode 100644 index 0000000..20c27c7 --- /dev/null +++ b/pulumi-tests/pulumi/programs/lifecycle/Pulumi.hxlife11784346404.yaml @@ -0,0 +1,3 @@ +encryptionsalt: v1:7PSO8VI3JSQ=:v1:LKcWFamTow4SZLl4:ItaMZLsAtkKhooVaycXnDR7F2dWwmA== +config: + hx-lifecycle:smoke: "0" diff --git a/pulumi-tests/pulumi/pvelib/surface.py b/pulumi-tests/pulumi/pvelib/surface.py index 35d0c49..9e316c5 100644 --- a/pulumi-tests/pulumi/pvelib/surface.py +++ b/pulumi-tests/pulumi/pvelib/surface.py @@ -2,6 +2,12 @@ Mirrors ``app/surface_probe.py`` classification and path/body synthesis, but talks HTTP to ``API_URL`` instead of an in-process ASGI app. + +Layer A matrix: +- every declared path+verb (GET/PUT/POST/DELETE) +- synthetic HEAD on each GET path (histogram only; not counted in declared) +- ticket + CSRF on mutations; form-urlencoded bodies +- Proxmox ``{ "data": … }`` envelope checks on 2xx """ from __future__ import annotations @@ -31,6 +37,11 @@ _FORBIDDEN = re.compile( r"handler pending for this contract method|is not supported in the (emulator|simulator)", re.I, ) +_UPID_RE = re.compile( + r"^UPID:[A-Za-z0-9][A-Za-z0-9_-]*:" + r"[0-9A-Fa-f]{8}:[0-9A-Fa-f]{8}:[0-9A-Fa-f]{8}:" + r"[A-Za-z0-9_-]+:[^:]*:[^:]+:$" +) _PATH_PARAM_EXAMPLES: dict[str, object] = { "node": "pve01", @@ -50,6 +61,11 @@ _PATH_PARAM_EXAMPLES: dict[str, object] = { "key": "cpu", "digest": "00000000", "name": "example", + "size": "1G", + "filename": "vm-100-disk-0.raw", + "certificates": "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n", + "contact": "mailto:admin@example.com", + "clustername": "lab", } _EXTRA_PATH: dict[str, object] = { @@ -85,7 +101,36 @@ _EXTRA_PATH: dict[str, object] = { } CRITICAL_BUCKETS = frozenset( - {"unimplemented_501", "unsupported_message", "server_5xx", "exception"} + { + "unimplemented_501", + "unsupported_message", + "server_5xx", + "exception", + "empty_success_body", + "wrong_returns_envelope", + } +) + +# Path suffixes that real PVE usually runs via fork_worker → UPID string. +_WORKER_SUFFIXES = ( + "/status/start", + "/status/stop", + "/status/shutdown", + "/status/reboot", + "/status/reset", + "/status/suspend", + "/status/resume", + "/clone", + "/migrate", + "/remote_migrate", + "/snapshot", + "/rollback", + "/template", + "/resize", + "/move_disk", + "/move_volume", + "/vzdump", + "/apt/update", ) @@ -195,6 +240,64 @@ def classify(status: int, text: str) -> str: return f"other_{status}" +def returns_type(method: dict[str, Any]) -> str: + returns = method.get("returns") or {} + if isinstance(returns, dict): + return str(returns.get("type") or "") + return "" + + +def looks_like_worker(path_template: str, verb: str) -> bool: + if verb.upper() not in {"POST", "PUT"}: + return False + lowered = path_template.lower() + return any(lowered.endswith(suffix) for suffix in _WORKER_SUFFIXES) + + +def classify_envelope( + *, + status: int, + text: str, + method: dict[str, Any], + path_template: str, + verb: str, +) -> str | None: + """Return an extra critical bucket for 2xx envelope/returns mismatches, else None.""" + + if not (200 <= status < 300) or verb.upper() == "HEAD": + return None + returns = method.get("returns") or {} + rtype = returns_type(method) + try: + payload = json.loads(text) if text else None + except json.JSONDecodeError: + return "empty_success_body" + if not isinstance(payload, dict) or "data" not in payload: + return "empty_success_body" + data = payload.get("data") + if rtype == "null": + return None + if rtype == "string": + if data is None: + return "wrong_returns_envelope" + if looks_like_worker(path_template, verb): + if not isinstance(data, str) or not _UPID_RE.fullmatch(data): + return "wrong_returns_envelope" + return None + if isinstance(data, str): + return None + # Older PVE schemas sometimes mark list endpoints as opaque ``string`` + # (empty properties, no description). Accept non-null structured data. + if isinstance(returns, dict) and not ( + returns.get("description") or returns.get("properties") or returns.get("enum") + ): + return None + return "wrong_returns_envelope" + if rtype in {"array", "object"} and data is None: + return "wrong_returns_envelope" + return None + + def probe_major( client: httpx.Client, csrf: str, @@ -209,6 +312,8 @@ def probe_major( by_verb: dict[str, Counter[str]] = defaultdict(Counter) failures: list[dict[str, Any]] = [] method_results: list[dict[str, Any]] = [] + head_results: list[dict[str, Any]] = [] + get_paths: set[str] = set() methods: list[tuple[str, dict[str, Any]]] = [ (path["path"], method) @@ -225,6 +330,7 @@ def probe_major( params = body_for(method, path_template) try: if verb == "GET": + get_paths.add(path_template) response = client.get(url, headers=headers, params=params or None) elif verb == "PUT": response = client.put(url, data=params or {}, headers=headers) @@ -232,9 +338,7 @@ def probe_major( response = client.post(url, data=params or {}, headers=headers) elif verb == "DELETE": # Proxmox accepts delete identifiers as form or query params. - response = client.request( - "DELETE", url, data=params or {}, headers=headers - ) + response = client.request("DELETE", url, data=params or {}, headers=headers) else: by_verb[verb or "UNKNOWN"]["exception"] += 1 item = { @@ -262,6 +366,15 @@ def probe_major( text = response.text bucket = classify(response.status_code, text) + envelope = classify_envelope( + status=response.status_code, + text=text, + method=method, + path_template=path_template, + verb=verb, + ) + if envelope is not None: + bucket = envelope by_verb[verb][bucket] += 1 ok = bucket not in CRITICAL_BUCKETS item = { @@ -270,12 +383,40 @@ def probe_major( "status": response.status_code, "bucket": bucket, "ok": ok, + "returns": returns_type(method), } if not ok: item["body"] = text[:240] failures.append(item) method_results.append(item) + # Synthetic HEAD for every GET path (Starlette mirrors GET handlers). + for path_template in sorted(get_paths): + url = f"/api2/json{render_path(path_template)}" + try: + response = client.request("HEAD", url) + text = response.text + bucket = classify(response.status_code, text) + except Exception as exc: # noqa: BLE001 + bucket = "exception" + text = str(exc)[:200] + response = None + by_verb["HEAD"][bucket] += 1 + ok = bucket not in CRITICAL_BUCKETS + item = { + "verb": "HEAD", + "path": path_template, + "status": getattr(response, "status_code", None), + "bucket": bucket, + "ok": ok, + "synthetic": True, + } + if not ok: + item["body"] = text[:240] + item["error"] = text[:200] + failures.append(item) + head_results.append(item) + version, _ = MAJOR_REVISIONS[major] declared = int(snapshot.get("method_count") or len(method_results)) critical = len(failures) @@ -309,13 +450,23 @@ def probe_major( ) critical = len(failures) + verb_histogram = { + verb: { + "total": sum(counter.values()), + "buckets": dict(counter), + } + for verb, counter in sorted(by_verb.items()) + } + return { "major": major, "version": snapshot.get("source_version") or version, "apply": applied, "declared": declared, "probed": probed, + "head_probed": len(head_results), "by_verb": {verb: dict(counter) for verb, counter in by_verb.items()}, + "verb_histogram": verb_histogram, "success_2xx": success_2xx, "client_4xx": sum( c.get("client_4xx", 0) + c.get("auth_401_403", 0) for c in by_verb.values() @@ -325,6 +476,7 @@ def probe_major( "ok": critical == 0, "time": time.monotonic() - started, "methods": method_results, + "head_methods": head_results, } diff --git a/pulumi-tests/pulumi/report.py b/pulumi-tests/pulumi/report.py index 41cde5c..6c9cf2b 100644 --- a/pulumi-tests/pulumi/report.py +++ b/pulumi-tests/pulumi/report.py @@ -73,6 +73,33 @@ def _surface_error(major: dict[str, Any]) -> str: return f"{len(fails)} critical: " + "; ".join(parts) +def _verb_histogram_rows(surface: list[dict[str, Any]]) -> list[str]: + rows: list[str] = [] + for major in surface: + histogram = major.get("verb_histogram") or {} + if not histogram: + # Fall back to by_verb totals when slim payload lacks histogram. + by_verb = major.get("by_verb") or {} + histogram = { + verb: {"total": sum(buckets.values()), "buckets": buckets} + for verb, buckets in by_verb.items() + } + for verb, info in sorted(histogram.items()): + buckets = info.get("buckets") or {} + bucket_txt = ", ".join( + f"{name}={count}" for name, count in sorted(buckets.items()) if count + ) + rows.append( + "" + f"{html.escape(str(major.get('version') or major.get('major')))}" + f"{html.escape(str(verb))}" + f"{html.escape(str(info.get('total') or 0))}" + f"{html.escape(bucket_txt)}" + "" + ) + return rows + + def write_html(payload: dict[str, Any], path: Path) -> None: surface = payload.get("surface") or [] scenarios = payload.get("scenarios") or [] @@ -240,6 +267,20 @@ code {{ font-family: var(--mono); font-size: .85em; }} +

Verb histogram (incl. synthetic HEAD)

+

+ Contract verbs GET/PUT/POST/DELETE count toward coverage. + HEAD is probed on every GET path for the matrix but is not part of declared/probed. +

+ + + + + + {''.join(_verb_histogram_rows(surface)) or ''} + +
MajorVerbTotalBuckets
No histogram
+

Critical surface failures

diff --git a/pulumi-tests/pulumi/run_suite.py b/pulumi-tests/pulumi/run_suite.py index 503868b..e249370 100644 --- a/pulumi-tests/pulumi/run_suite.py +++ b/pulumi-tests/pulumi/run_suite.py @@ -22,9 +22,10 @@ sys.path.insert(0, str(ROOT)) def run_lifecycle(*, smoke: bool) -> dict[str, Any]: from pulumi import automation as auto + import uuid program_dir = PROGRAMS / "lifecycle" - stack_name = f"hxlife{os.getpid()}{int(time.time())}" + stack_name = f"hxlife-{uuid.uuid4().hex[:12]}" os.environ.setdefault("PULUMI_CONFIG_PASSPHRASE", "hx-test-passphrase") state = HERE / ".pulumi-state" state.mkdir(parents=True, exist_ok=True) @@ -43,6 +44,21 @@ def run_lifecycle(*, smoke: bool) -> dict[str, Any]: started = time.monotonic() stack = None + + def _cleanup() -> None: + if stack is None: + return + try: + stack.destroy(on_output=lambda _: None) + except Exception as exc: # noqa: BLE001 + message = str(exc).lower() + if "no stack named" not in message and "not found" not in message: + raise + try: + stack.workspace.remove_stack(stack_name) + except Exception: + pass + try: # Pass env (incl. PULUMI_BACKEND_URL) at workspace creation — assigning # workspace.env_vars after create_or_select_stack can lose stack selection. @@ -67,11 +83,7 @@ def run_lifecycle(*, smoke: bool) -> dict[str, Any]: item = outputs.get(key) if item is None or getattr(item, "value", None) in (None, "", {}, []): raise RuntimeError(f"lifecycle export {key!r} is empty") - stack.destroy(on_output=lambda _: None) - try: - stack.workspace.remove_stack(stack_name) - except Exception: - pass + _cleanup() elapsed = time.monotonic() - started if ids: return { @@ -89,15 +101,10 @@ def run_lifecycle(*, smoke: bool) -> dict[str, Any]: stderr = getattr(exc, "stderr", None) if stderr: detail = f"{detail}\n{stderr}" - if stack is not None: - try: - stack.destroy(on_output=lambda _: None) - except Exception: - pass - try: - stack.workspace.remove_stack(stack_name) - except Exception: - pass + try: + _cleanup() + except Exception as cleanup_exc: # noqa: BLE001 + detail = f"{detail}\ncleanup: {cleanup_exc}" return { "ok": False, "scenarios": [ @@ -166,9 +173,17 @@ def main() -> int: status = "ok" if item["ok"] else "FAIL" print( f" {status} PVE {item['version']}: declared={item['declared']} " - f"probed={item['probed']} critical={item['failure_count']} " + f"probed={item['probed']} head={item.get('head_probed', 0)} " + f"critical={item['failure_count']} " f"2xx={item['success_2xx']} 4xx={item['client_4xx']}" ) + hist = item.get("verb_histogram") or {} + if hist: + parts = [ + f"{verb}:{info.get('total', 0)}" + for verb, info in sorted(hist.items()) + ] + print(f" verbs: {', '.join(parts)}") if not item["ok"]: for fail in (item.get("failures") or [])[:10]: print( @@ -177,11 +192,12 @@ def main() -> int: file=sys.stderr, ) - # Slim JSON: drop full method lists (keep failures) + # Slim JSON: drop full method lists (keep failures + verb histogram) surface_slim = [] for item in surface: slim = dict(item) slim.pop("methods", None) + slim.pop("head_methods", None) surface_slim.append(slim) coverage = build_coverage(surface_slim, majors=list(majors)) diff --git a/scripts/up-local.sh b/scripts/up-local.sh new file mode 100755 index 0000000..a10cb5f --- /dev/null +++ b/scripts/up-local.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Generate a gitignored docker-compose.override.yml (busy host ports → free ones) +# and start the local stack. Invoked by `make up-local`. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +COMPOSE="${COMPOSE:-docker compose}" +OVERRIDE="${LOCAL_OVERRIDE:-docker-compose.override.yml}" + +PREFERRED_SIMULATOR="${SIMULATOR_PORT:-8006}" +PREFERRED_POSTGRES="${POSTGRES_HOST_PORT:-5432}" +PREFERRED_TLS="${TLS_GATEWAY_PORT:-8443}" + +port_in_use() { + local port="$1" + if command -v nc >/dev/null 2>&1; then + nc -z 127.0.0.1 "$port" >/dev/null 2>&1 + return $? + fi + python3 - "$port" <<'PY' +import socket, sys +port = int(sys.argv[1]) +s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +s.settimeout(0.25) +try: + sys.exit(0 if s.connect_ex(("127.0.0.1", port)) == 0 else 1) +finally: + s.close() +PY +} + +ours_on_port() { + local port="$1" + # Host publish lines look like: 0.0.0.0:8006->8006/tcp or 127.0.0.1:5432->5432/tcp + $COMPOSE ps --format '{{.Ports}}' 2>/dev/null \ + | grep -E "(^|[, ])(([0-9.]+|\[::\]):)?${port}->" >/dev/null 2>&1 +} + +pick_port() { + local preferred="$1" + local label="$2" + local max=$((preferred + 100)) + local p="$preferred" + while [ "$p" -le "$max" ]; do + if ! port_in_use "$p" || ours_on_port "$p"; then + if [ "$p" -ne "$preferred" ]; then + echo "Port ${preferred} busy for ${label} → using ${p}" >&2 + fi + echo "$p" + return 0 + fi + p=$((p + 1)) + done + echo "No free host port near ${preferred} for ${label}" >&2 + exit 1 +} + +SIMULATOR_HOST_PORT="$(pick_port "$PREFERRED_SIMULATOR" "simulator")" +POSTGRES_HOST_PORT="$(pick_port "$PREFERRED_POSTGRES" "postgres")" +TLS_HOST_PORT="$(pick_port "$PREFERRED_TLS" "tls-gateway")" + +cat >"$OVERRIDE" < None: connection = await asyncpg.connect(url) try: await migrate(connection) - await apply_seed(connection, small_profile()) + await apply_seed(connection, lab_profile()) finally: await connection.close() diff --git a/tests/compatibility/test_proxmoxer.py b/tests/compatibility/test_proxmoxer.py index 5436a91..7fb1d68 100644 --- a/tests/compatibility/test_proxmoxer.py +++ b/tests/compatibility/test_proxmoxer.py @@ -116,7 +116,8 @@ def test_proxmoxer_read_and_qemu_task_flow() -> None: assert proxmox.nodes("pve1").qemu("150").config.get()["name"] == "async-update" disk_api = proxmox.nodes("pve1").qemu("150") - assert disk_api.resize.put(disk="scsi0", size="+2G") is None + resize_upid = disk_api.resize.put(disk="scsi0", size="+2G") + assert wait_task(proxmox, resize_upid)["exitstatus"] == "OK" assert "size=10G" in disk_api.config.get()["scsi0"] move_upid = disk_api.move_disk.post(disk="scsi0", storage="shared") assert wait_task(proxmox, move_upid)["exitstatus"] == "OK" diff --git a/tests/integration/test_migrations.py b/tests/integration/test_migrations.py index e833d25..aa10213 100644 --- a/tests/integration/test_migrations.py +++ b/tests/integration/test_migrations.py @@ -11,7 +11,7 @@ from app.db.migrations import migrate from app.db.pool import AsyncpgDatabase from app.db.primitives import ConflictError from app.db.repositories.resources import ResourceRepository -from app.simulation.seed import apply_seed, small_profile +from app.simulation.seed import apply_seed, lab_profile pytestmark = [ pytest.mark.integration, @@ -45,8 +45,8 @@ async def test_small_seed_is_idempotent() -> None: connection = await asyncpg.connect(os.environ["TEST_DATABASE_URL"]) try: await migrate(connection) - await apply_seed(connection, small_profile()) - await apply_seed(connection, small_profile()) + await apply_seed(connection, lab_profile()) + await apply_seed(connection, lab_profile()) assert await connection.fetchval("SELECT count(*) FROM nodes WHERE name = 'pve01'") == 1 assert ( await connection.fetchval( @@ -58,8 +58,8 @@ async def test_small_seed_is_idempotent() -> None: assert await connection.fetchval("SELECT count(*) FROM tasks WHERE status = 'success'") == 2 assert await connection.fetchval("SELECT count(*) FROM virtual_machines") == 2 assert await connection.fetchval("SELECT count(*) FROM containers") == 1 - assert await connection.fetchval("SELECT count(*) FROM storages") == 2 - assert await connection.fetchval("SELECT count(*) FROM storage_contents") == 4 + assert await connection.fetchval("SELECT count(*) FROM storages") == 3 + assert await connection.fetchval("SELECT count(*) FROM storage_contents") == 7 assert await connection.fetchval("SELECT count(*) FROM identity_groups") == 1 assert await connection.fetchval("SELECT count(*) FROM identity_group_members") == 1 assert await connection.fetchval("SELECT count(*) FROM group_acl_entries") == 1 @@ -105,7 +105,7 @@ async def test_schema_readiness_and_optimistic_resource_repository() -> None: database = AsyncpgDatabase(Settings(database_url=url)) try: await migrate(connection) - await apply_seed(connection, small_profile()) + await apply_seed(connection, lab_profile()) await database.connect() assert await database.is_ready() diff --git a/tests/unit/test_backup_handlers.py b/tests/unit/test_backup_handlers.py new file mode 100644 index 0000000..c449943 --- /dev/null +++ b/tests/unit/test_backup_handlers.py @@ -0,0 +1,126 @@ +"""Cluster backup job handlers.""" + +from __future__ import annotations + +import json +from typing import Any, cast + +from fastapi import FastAPI, Request + +from app.api.registry import HandlerRegistry +from app.db.pool import AsyncpgDatabase +from app.handlers.backup import register_backup_handlers + + +class BackupPool: + def __init__(self) -> None: + self.metadata: dict[str, Any] = { + "backup_jobs": { + "backup-daily": { + "id": "backup-daily", + "schedule": "0 2 * * *", + "storage": "local", + "enabled": 1, + "vmid": "100", + } + } + } + self.guests = [ + { + "external_id": "100", + "kind": "qemu", + "state": '{"name":"demo"}', + "qemu_config": '{"scsi0":"local-lvm:vm-100-disk-0,size=32G"}', + "lxc_config": None, + }, + { + "external_id": "200", + "kind": "lxc", + "state": '{"name":"service"}', + "qemu_config": None, + "lxc_config": '{"rootfs":"local-lvm:vm-200-disk-0,size=8G"}', + }, + ] + + async def fetchrow(self, query: str, *_arguments: object) -> dict[str, Any] | None: + if "FROM clusters WHERE id" in query: + return {"metadata": json.dumps(self.metadata)} + raise AssertionError(query) + + async def fetch(self, query: str, *arguments: object) -> list[dict[str, Any]]: + if "FROM resources r" in query and "ANY($1" in query: + selected = set(str(item) for item in cast(list[Any], arguments[0])) + return [row for row in self.guests if row["external_id"] in selected] + if "FROM resources r" in query: + return list(self.guests) + raise AssertionError(query) + + async def execute(self, query: str, *arguments: object) -> str: + if "UPDATE clusters SET metadata" in query: + self.metadata = json.loads(str(arguments[1])) + return "UPDATE 1" + raise AssertionError(query) + + +def _request(pool: BackupPool) -> Request: + app = FastAPI() + app.state.database = cast(AsyncpgDatabase, type("DB", (), {"pool": pool})()) + return Request( + { + "type": "http", + "app": app, + "method": "POST", + "path": "/", + "headers": [], + "query_string": b"", + "server": ("test", 80), + "client": ("test", 123), + "scheme": "http", + } + ) + + +async def test_backup_jobs_crud_and_shapes() -> None: + registry = HandlerRegistry() + register_backup_handlers(registry) + pool = BackupPool() + http = _request(pool) + + listed = await registry.get("/cluster/backup", "GET")(http, {"values": {}}) + assert listed[0]["id"] == "backup-daily" + assert listed[0]["schedule"] == "0 2 * * *" + + await registry.get("/cluster/backup", "POST")( + http, + { + "values": { + "id": "backup-weekly", + "schedule": "@weekly", + "storage": "local", + "vmid": "200", + } + }, + ) + assert "backup-weekly" in pool.metadata["backup_jobs"] + + info = await registry.get("/cluster/backup-info", "GET")(http, {"values": {}}) + assert info == [{"subdir": "not-backed-up"}] + + missing = await registry.get("/cluster/backup-info/not-backed-up", "GET")(http, {"values": {}}) + assert missing == [] # 100 covered by daily; after weekly both covered? weekly adds 200 + # daily covers 100, weekly covers 200 → none missing + assert missing == [] + + included = await registry.get("/cluster/backup/{id}/included_volumes", "GET")( + http, {"values": {"id": "backup-daily"}} + ) + assert included["children"][0]["id"] == 100 + assert included["children"][0]["children"][0]["id"] == "scsi0" + + await registry.get("/cluster/backup/{id}", "DELETE")(http, {"values": {"id": "backup-weekly"}}) + assert "backup-weekly" not in pool.metadata["backup_jobs"] + missing_after = await registry.get("/cluster/backup-info/not-backed-up", "GET")( + http, {"values": {}} + ) + assert missing_after[0]["vmid"] == 200 + assert missing_after[0]["type"] == "lxc" diff --git a/tests/unit/test_ceph_handlers.py b/tests/unit/test_ceph_handlers.py index 5bd441f..5c2e2da 100644 --- a/tests/unit/test_ceph_handlers.py +++ b/tests/unit/test_ceph_handlers.py @@ -125,7 +125,7 @@ async def test_ceph_pool_and_osd_mutations_persist() -> None: await create_pool(http, {"values": {"node": "pve1", "name": "vms"}, "provided": frozenset()}) pools = await list_pool(http, {"values": {"node": "pve1"}, "provided": frozenset()}) - assert any(item["pool"] == "vms" for item in pools) + assert any(item.get("pool_name") == "vms" for item in pools) assert "vms" in pool.cluster_metadata["ceph"]["pools"] await create_osd(http, {"values": {"node": "pve1", "dev": "/dev/sdb"}, "provided": frozenset()}) diff --git a/tests/unit/test_cluster_meta_handlers.py b/tests/unit/test_cluster_meta_handlers.py index 92409e5..5935f0f 100644 --- a/tests/unit/test_cluster_meta_handlers.py +++ b/tests/unit/test_cluster_meta_handlers.py @@ -6,8 +6,10 @@ import json from typing import Any, cast from uuid import uuid4 +import pytest from fastapi import FastAPI, Request +from app.api.errors import ApiError from app.api.registry import HandlerRegistry from app.db.pool import AsyncpgDatabase from app.handlers.acme import register_acme_handlers @@ -34,6 +36,8 @@ class MetaPool: async def fetchval(self, query: str, *arguments: object) -> Any: if "EXISTS(SELECT 1 FROM nodes" in query: return str(arguments[0]) in self.nodes + if "SELECT name FROM nodes ORDER BY name LIMIT 1" in query: + return sorted(self.nodes)[0] if self.nodes else None raise AssertionError(query) async def execute(self, query: str, *arguments: object) -> str: @@ -52,6 +56,16 @@ class MetaPool: raise AssertionError(query) +class FakeTaskRepository: + def __init__(self, pool: MetaPool) -> None: + self.pool = pool + self.created: list[dict[str, Any]] = [] + + async def create(self, **kwargs: Any) -> Any: + self.created.append(kwargs) + return type("Task", (), {"upid": kwargs["upid"]})() + + async def call( registry: HandlerRegistry, path: str, verb: str, http: Request, inputs: dict[str, Any] ) -> Any: @@ -63,7 +77,7 @@ async def call( def request(pool: MetaPool) -> Request: app = FastAPI() app.state.database = cast(AsyncpgDatabase, type("DB", (), {"pool": pool})()) - return Request( + http = Request( { "type": "http", "app": app, @@ -76,15 +90,19 @@ def request(pool: MetaPool) -> Request: "scheme": "http", } ) + http.state.principal = "root@pam" + return http -async def test_mapping_acme_config_persist() -> None: +async def test_mapping_acme_config_persist(monkeypatch: pytest.MonkeyPatch) -> None: registry = HandlerRegistry() register_mapping_handlers(registry) register_acme_handlers(registry) register_cluster_config_handlers(registry) pool = MetaPool() http = request(pool) + repository = FakeTaskRepository(pool) + monkeypatch.setattr("app.handlers.cluster_config.TaskRepository", lambda _pool: repository) await call( registry, @@ -119,20 +137,214 @@ async def test_mapping_acme_config_persist() -> None: http, {"values": {"name": "default"}, "provided": frozenset()}, ) - assert account["name"] == "default" + assert account["account"]["name"] == "default" + assert account["directory"] assert "eab-hmac-key" not in account + assert "eab-hmac-key" not in account["account"] - await call( + upid = await call( registry, "/cluster/config", "POST", http, - {"values": {"clustername": "lab"}, "provided": frozenset()}, + {"values": {"clustername": "lab", "link0": "10.0.0.1"}, "provided": frozenset()}, ) + assert isinstance(upid, str) + assert upid.startswith("UPID:pve1:") + assert ":clustercreate:" in upid + assert repository.created[0]["task_type"] == "cluster-create" assert pool.metadata["cluster_config"]["clustername"] == "lab" assert pool.cluster_name == "lab" + assert pool.metadata["cluster_config"]["corosync_conf"] + assert pool.metadata["cluster_config"]["config_digest"] totem = await call( registry, "/cluster/config/totem", "GET", http, {"values": {}, "provided": frozenset()} ) assert totem["cluster_name"] == "lab" assert uuid4() + + +async def test_cluster_config_index_uses_name_links() -> None: + registry = HandlerRegistry() + register_cluster_config_handlers(registry) + pool = MetaPool() + index = await call( + registry, "/cluster/config", "GET", request(pool), {"values": {}, "provided": frozenset()} + ) + assert index == [ + {"name": "nodes"}, + {"name": "totem"}, + {"name": "join"}, + {"name": "qdevice"}, + {"name": "apiversion"}, + ] + + +async def test_cluster_join_info_shape_and_stable_digest( + monkeypatch: pytest.MonkeyPatch, +) -> None: + registry = HandlerRegistry() + register_cluster_config_handlers(registry) + pool = MetaPool() + http = request(pool) + monkeypatch.setattr( + "app.handlers.cluster_config.TaskRepository", + lambda _pool: FakeTaskRepository(pool), + ) + await call( + registry, + "/cluster/config", + "POST", + http, + {"values": {"clustername": "lab", "link0": "10.0.0.1"}, "provided": frozenset()}, + ) + first = await call( + registry, + "/cluster/config/join", + "GET", + http, + {"values": {}, "provided": frozenset()}, + ) + second = await call( + registry, + "/cluster/config/join", + "GET", + http, + {"values": {"node": "pve1"}, "provided": frozenset()}, + ) + assert set(first) == {"nodelist", "preferred_node", "totem", "config_digest"} + assert first["config_digest"] == second["config_digest"] + assert first["preferred_node"] == "pve1" + defaulted = await call( + registry, + "/cluster/config/join", + "GET", + http, + { + "values": {"node": "current connected node"}, + "provided": frozenset(), + }, + ) + assert defaulted["preferred_node"] == "pve1" + node = first["nodelist"][0] + assert node["name"] == "pve1" + assert node["pve_addr"] + assert node["ring0_addr"] + assert node["pve_fp"].count(":") == 31 + assert "password" not in json.dumps(first) + + +async def test_cluster_join_returns_upid(monkeypatch: pytest.MonkeyPatch) -> None: + registry = HandlerRegistry() + register_cluster_config_handlers(registry) + pool = MetaPool() + http = request(pool) + repository = FakeTaskRepository(pool) + monkeypatch.setattr("app.handlers.cluster_config.TaskRepository", lambda _pool: repository) + fingerprint = ":".join(["AB"] * 32) + upid = await call( + registry, + "/cluster/config/join", + "POST", + http, + { + "values": { + "hostname": "10.0.0.1", + "fingerprint": fingerprint, + "password": "secret", + }, + "provided": frozenset(), + }, + ) + assert upid.startswith("UPID:") + assert ":clusterjoin:" in upid + assert repository.created[0]["task_type"] == "cluster-join" + join = pool.metadata["cluster_config"]["join_info"]["10.0.0.1"] + assert join["password_set"] is True + assert "password" not in join + + +async def test_cluster_join_requires_contract_params() -> None: + registry = HandlerRegistry() + register_cluster_config_handlers(registry) + pool = MetaPool() + http = request(pool) + with pytest.raises(ApiError, match="hostname"): + await call( + registry, + "/cluster/config/join", + "POST", + http, + {"values": {"password": "x", "fingerprint": "y"}, "provided": frozenset()}, + ) + + +async def test_cluster_create_rejects_existing_corosync( + monkeypatch: pytest.MonkeyPatch, +) -> None: + registry = HandlerRegistry() + register_cluster_config_handlers(registry) + pool = MetaPool() + http = request(pool) + monkeypatch.setattr( + "app.handlers.cluster_config.TaskRepository", + lambda _pool: FakeTaskRepository(pool), + ) + await call( + registry, + "/cluster/config", + "POST", + http, + {"values": {"clustername": "lab", "link0": "10.0.0.1"}, "provided": frozenset()}, + ) + with pytest.raises(ApiError, match="cluster config already exists"): + await call( + registry, + "/cluster/config", + "POST", + http, + {"values": {"clustername": "other"}, "provided": frozenset()}, + ) + + +async def test_cluster_addnode_returns_corosync_object( + monkeypatch: pytest.MonkeyPatch, +) -> None: + registry = HandlerRegistry() + register_cluster_config_handlers(registry) + pool = MetaPool() + http = request(pool) + monkeypatch.setattr( + "app.handlers.cluster_config.TaskRepository", + lambda _pool: FakeTaskRepository(pool), + ) + await call( + registry, + "/cluster/config", + "POST", + http, + {"values": {"clustername": "lab", "link0": "10.0.0.1"}, "provided": frozenset()}, + ) + result = await call( + registry, + "/cluster/config/nodes/{node}", + "POST", + http, + { + "values": {"node": "pve2", "new_node_ip": "10.0.0.2", "votes": 1}, + "provided": frozenset(), + }, + ) + assert set(result) == {"corosync_authkey", "corosync_conf", "warnings"} + assert "nodelist" in result["corosync_conf"] + assert "pve2" in result["corosync_conf"] + assert result["warnings"] == [] + assert "pve2" in pool.nodes + nodes = await call( + registry, + "/cluster/config/nodes", + "GET", + http, + {"values": {}, "provided": frozenset()}, + ) + assert {item["node"] for item in nodes} == {"pve1", "pve2"} diff --git a/tests/unit/test_contract_catalog.py b/tests/unit/test_contract_catalog.py index e1f35f8..6eb632b 100644 --- a/tests/unit/test_contract_catalog.py +++ b/tests/unit/test_contract_catalog.py @@ -88,6 +88,74 @@ def test_method_payload_builds_examples() -> None: assert payload["resolved_path"] == "/nodes/pve01/qemu" assert payload["body_example"] == {"vmid": 100, "name": "example"} assert payload["implemented"] is True + indexed = cast(list[dict[str, Any]], payload["indexed_fields"]) + assert indexed[0]["name"] == "scsi0" + assert indexed[0]["template"] == "scsi[n]" + assert indexed[0]["optional"] is True + + +def test_method_payload_flattens_nested_body_example_into_params() -> None: + method = Method( + verb="POST", + name="create_mapping", + description="Create mapping.", + parameters=( + Parameter(name="id", definition=Schema(type="string")), + Parameter( + name="map", + definition=Schema( + type="array", + items=Schema(type="string"), + ), + ), + Parameter(name="comment", definition=Schema(type="string", optional=True)), + ), + returns=Schema(type="null"), + checksum="c" * 64, + ) + snapshot = Snapshot( + source_version="9.2.3", + retrieved_at=datetime(2026, 1, 1, tzinfo=UTC), + raw_sha256="d" * 64, + paths=(PathContract(path="/cluster/mapping/dir", methods=(method,)),), + path_count=1, + method_count=1, + ) + payload = method_payload( + snapshot, + major=9, + path="/cluster/mapping/dir", + verb="POST", + runtime_version="9.2.3", + implemented_methods=None, + ) + assert payload["body_example"] == {"id": "example", "map": ["example"]} + names = [str(item["name"]) for item in cast(list[dict[str, Any]], payload["body_fields"])] + assert "id" in names + assert "map.0" in names + assert "map" not in names + assert "comment" in names + + +@pytest.mark.asyncio +async def test_method_payload_property_string_examples_from_bundled_contract() -> None: + from app.web import contract_catalog + + contract_catalog._SNAPSHOT_CACHE.clear() + snapshot = await contract_catalog.load_snapshot(8, Path("contracts")) + payload = method_payload( + snapshot, + major=8, + path="/cluster/config", + verb="POST", + runtime_version="8.4.5", + implemented_methods=None, + ) + assert payload["body_example"] == {"clustername": "example"} + indexed = cast(list[dict[str, Any]], payload["indexed_fields"]) + link0 = next(item for item in indexed if item["name"] == "link0") + assert link0["example"] == "192.168.0.1" + assert link0["typetext"] == "[address=] [,priority=]" @pytest.mark.asyncio diff --git a/tests/unit/test_core_handlers.py b/tests/unit/test_core_handlers.py index b8ab445..49a11ed 100644 --- a/tests/unit/test_core_handlers.py +++ b/tests/unit/test_core_handlers.py @@ -39,20 +39,33 @@ class FakePool: async def fetch(self, sql: str, *args: object) -> list[dict[str, object]]: del args + if "FROM pool_members" in sql or "kind='ha'" in sql: + return [] if "FROM nodes" in sql: - return [{"node": "pve1", "status": "online"}] + return [{"node": "pve1", "name": "pve1", "status": "online"}] if "r.kind='qemu'" in sql: - return [{"vmid": 100, "state": '{"name":"demo","status":"stopped"}'}] - return [ - { - "type": "qemu", - "external_id": "100", - "state": '{"status":"stopped"}', - "node": "pve1", - } - ] + return [ + { + "vmid": 100, + "state": '{"name":"demo","status":"stopped"}', + "config": '{"name":"demo","memory":2048,"cores":2}', + } + ] + if "r.kind = ANY" in sql or "r.kind AS type" in sql: + return [ + { + "type": "qemu", + "external_id": "100", + "state": '{"status":"stopped","name":"demo"}', + "node": "pve1", + } + ] + return [] - async def fetchval(self, sql: str) -> int: + async def fetchval(self, sql: str, *args: object) -> object: + del args + if "SELECT name FROM clusters" in sql: + return "pve-simulator" return 100 if "pg_backend_pid" in sql else 1_700_000_000 @@ -194,10 +207,13 @@ async def test_core_login_and_read_endpoints( assert login.status_code == 200 assert login.json()["data"]["username"] == "root@pam" assert "ticket" in login.json()["data"] + assert login.json()["data"]["clustername"] == "pve-simulator" assert version.json()["data"]["version"] == "test" assert version.json()["data"]["release"] == "test" assert nodes.json()["data"][0]["node"] == "pve1" - assert status.json()["data"]["status"] == "online" + assert "uptime" in status.json()["data"] + assert "memory" in status.json()["data"] + assert "cpuinfo" in status.json()["data"] resource_types = {item["type"] for item in resources.json()["data"]} assert "qemu" in resource_types qemu_resources = [item for item in resources.json()["data"] if item["type"] == "qemu"] diff --git a/tests/unit/test_extended_handlers.py b/tests/unit/test_extended_handlers.py index 1e4f6a7..f74681a 100644 --- a/tests/unit/test_extended_handlers.py +++ b/tests/unit/test_extended_handlers.py @@ -42,6 +42,7 @@ class HandlerPool: { "external_id": "osd.0", "state": '{"osd_id":0,"status":"up","in":true,"weight":1.0}', + "node": "pve01", } ] if "FROM pools" in sql: @@ -54,7 +55,14 @@ class HandlerPool: } ] if "FROM pool_members" in sql: - return [{"external_id": "100"}] + return [ + { + "external_id": "100", + "kind": "qemu", + "state": '{"name":"demo","status":"stopped"}', + "node": "pve01", + } + ] if "FROM task_logs" in sql: return [{"message": "seeded task", "sequence": 1}] if "FROM tasks" in sql: @@ -101,6 +109,8 @@ class HandlerPool: return 150 if "count(*)::int FROM resources WHERE kind='ceph-osd'" in sql: return 300 + if "COUNT(*) FROM nodes" in sql or "count(*) FROM nodes" in sql: + return 1 if "SELECT resource_id FROM storages" in sql: return uuid.uuid4() return False @@ -139,7 +149,10 @@ async def test_cluster_status_and_nextid() -> None: registry = HandlerRegistry() register_cluster_handlers(registry) status = await _call(registry.get("/cluster/status", "GET"), {}) - assert status[0]["name"] == "pve01" + assert status[0]["type"] == "cluster" + assert status[0]["quorate"] is True + assert status[1]["name"] == "pve01" + assert status[1]["online"] is True nextid = await _call(registry.get("/cluster/nextid", "GET"), {}) assert nextid == 151 @@ -158,7 +171,8 @@ async def test_storage_and_ceph_handlers() -> None: registry.get("/nodes/{node}/ceph/osd", "GET"), {"node": "pve01"}, ) - assert osds[0]["status"] == "up" + assert osds["root"]["type"] == "root" + assert osds["root"]["children"][0]["children"][0]["status"] == "up" ceph_status = await _call(registry.get("/cluster/ceph/status", "GET"), {}) assert ceph_status["osdmap"]["num_osds"] == 1 @@ -169,7 +183,9 @@ async def test_pools_list() -> None: register_pool_handlers(registry) pools = await _call(registry.get("/pools", "GET"), {}) assert pools[0]["poolid"] == "production" - assert pools[0]["members"] == ["100"] + assert pools[0]["members"][0]["vmid"] == 100 + assert pools[0]["members"][0]["type"] == "qemu" + assert pools[0]["members"][0]["node"] == "pve01" @pytest.mark.asyncio diff --git a/tests/unit/test_gap_plan_handlers.py b/tests/unit/test_gap_plan_handlers.py index 3246356..85591d5 100644 --- a/tests/unit/test_gap_plan_handlers.py +++ b/tests/unit/test_gap_plan_handlers.py @@ -32,7 +32,6 @@ class GapPool: self.groups = {"operators": {"comment": "ops", "users": ["root@pam"]}} async def fetch(self, sql: str, *args: object) -> list[dict[str, object]]: - del args if "FROM nodes" in sql and "ORDER BY name" in sql: return [{"id": uuid.uuid4(), "name": "pve01", "status": "online"}] if "FROM tasks" in sql: @@ -64,6 +63,17 @@ class GapPool: } for group_id, data in self.groups.items() ] + if "FROM identity_group_members" in sql: + userid = str(args[0]) if args else "" + return [ + {"group_id": group_id} + for group_id, data in self.groups.items() + if userid in data["users"] + ] + if "FROM api_tokens" in sql: + return [] + if "FROM resources r JOIN nodes" in sql and "kind='ha'" in sql: + return [] raise AssertionError(sql) async def fetchrow(self, sql: str, *args: object) -> dict[str, object] | None: @@ -149,6 +159,8 @@ class GapPool: return json.dumps(self.node_metadata.get(str(args[0]), {})) if "SELECT name FROM nodes WHERE status" in sql: return "pve01" + if "SELECT name FROM nodes ORDER BY name LIMIT 1" in sql: + return "pve01" return False async def execute(self, sql: str, *args: object) -> str: @@ -217,7 +229,7 @@ async def test_cluster_index_and_replication_crud() -> None: ) assert created["id"] == "repl-100" jobs = await _call(registry.get("/cluster/replication", "GET"), {}, pool) - assert jobs[0]["guest"] == "100" + assert jobs[0]["guest"] == 100 fetched = await _call( registry.get("/cluster/replication/{id}", "GET"), {"id": "repl-100"}, pool ) @@ -257,10 +269,16 @@ async def test_access_user_and_group_detail() -> None: @pytest.mark.asyncio -async def test_storage_content_get_and_upload() -> None: +async def test_storage_content_get_and_upload(monkeypatch: pytest.MonkeyPatch) -> None: registry = HandlerRegistry() register_storage_handlers(registry) pool = GapPool() + + class FakeTaskRepository: + async def create(self, **kwargs: Any) -> Any: + return type("Task", (), {"upid": kwargs["upid"]})() + + monkeypatch.setattr("app.handlers.nodes.TaskRepository", lambda _pool: FakeTaskRepository()) item = await _call( registry.get("/nodes/{node}/storage/{storage}/content/{volume}", "GET"), { @@ -276,7 +294,8 @@ async def test_storage_content_get_and_upload() -> None: {"node": "pve01", "storage": "local-lvm", "filename": "image.iso"}, pool, ) - assert "uploadid" in upload + assert isinstance(upload, str) and upload.startswith("UPID:") + assert any("image.iso" in str(item["volume_id"]) for item in pool.storage_contents) @pytest.mark.asyncio diff --git a/tests/unit/test_gap_remaining_handlers.py b/tests/unit/test_gap_remaining_handlers.py index 23b8e20..ae076dd 100644 --- a/tests/unit/test_gap_remaining_handlers.py +++ b/tests/unit/test_gap_remaining_handlers.py @@ -28,8 +28,11 @@ class GapRemainingPool: raise AssertionError(query) async def fetchval(self, query: str, *arguments: object) -> Any: + del arguments if "EXISTS(SELECT 1 FROM nodes" in query: return True + if "SELECT name FROM nodes ORDER BY name LIMIT 1" in query: + return "pve01" raise AssertionError(query) async def execute(self, query: str, *arguments: object) -> str: @@ -68,10 +71,16 @@ def _request(pool: GapRemainingPool, *, method: str = "GET") -> Request: @pytest.mark.asyncio -async def test_disks_directory_create_persists() -> None: +async def test_disks_directory_create_persists(monkeypatch: pytest.MonkeyPatch) -> None: registry = HandlerRegistry() register_nodes_extra_handlers(registry) pool = GapRemainingPool() + + class FakeTaskRepository: + async def create(self, **kwargs: Any) -> Any: + return type("Task", (), {"upid": kwargs["upid"]})() + + monkeypatch.setattr("app.handlers.nodes.TaskRepository", lambda _pool: FakeTaskRepository()) create = registry.get("/nodes/{node}/disks/directory", "POST") assert create is not None created = await create( @@ -81,7 +90,7 @@ async def test_disks_directory_create_persists() -> None: "provided": frozenset(), }, ) - assert created["name"] == "tank" + assert isinstance(created, str) and created.startswith("UPID:") ops = pool.node_metadata["pve01"]["ops"] assert any(item["name"] == "tank" for item in ops["disks"]["directory"]) diff --git a/tests/unit/test_lxc_handlers.py b/tests/unit/test_lxc_handlers.py index 61492dd..4f38a25 100644 --- a/tests/unit/test_lxc_handlers.py +++ b/tests/unit/test_lxc_handlers.py @@ -37,7 +37,13 @@ class LxcPool: async def fetch(self, sql: str, *args: object) -> list[dict[str, object]]: del args if "FROM resources" in sql and "kind='lxc'" in sql: - return [{"vmid": 200, "state": '{"status":"stopped","name":"service"}'}] + return [ + { + "vmid": 200, + "state": '{"status":"stopped","name":"service"}', + "config": '{"hostname":"service","memory":512,"cores":1}', + } + ] assert "FROM snapshots" in sql return [ { @@ -118,7 +124,12 @@ async def test_lxc_list_returns_seeded_containers(registry: HandlerRegistry) -> handler = registry.get("/nodes/{node}/lxc", "GET") assert handler is not None result = await handler(_request(pool), {"values": {"node": "pve1"}}) - assert result == [{"vmid": 200, "status": "stopped", "name": "service"}] + assert len(result) == 1 + assert result[0]["vmid"] == 200 + assert result[0]["status"] == "stopped" + assert result[0]["name"] == "service" + assert result[0]["cpus"] == 1 + assert "maxmem" in result[0] async def test_lxc_create_rejects_duplicate_vmid(registry: HandlerRegistry) -> None: diff --git a/tests/unit/test_node_ops_handlers.py b/tests/unit/test_node_ops_handlers.py index c8dcc8e..88fa99e 100644 --- a/tests/unit/test_node_ops_handlers.py +++ b/tests/unit/test_node_ops_handlers.py @@ -5,6 +5,7 @@ from __future__ import annotations import json from typing import Any, cast +import pytest from fastapi import FastAPI, Request from app.api.registry import HandlerRegistry @@ -17,13 +18,17 @@ class NodePool: self.metadata: dict[str, Any] = {} async def fetchrow(self, query: str, *arguments: object) -> dict[str, Any] | None: + del arguments if "SELECT metadata FROM nodes" in query: return {"metadata": json.dumps(self.metadata)} raise AssertionError(query) async def fetchval(self, query: str, *arguments: object) -> Any: + del arguments if "EXISTS(SELECT 1 FROM nodes" in query: return True + if "SELECT name FROM nodes ORDER BY name LIMIT 1" in query: + return "pve01" raise AssertionError(query) async def execute(self, query: str, *arguments: object) -> str: @@ -33,6 +38,16 @@ class NodePool: raise AssertionError(query) +class FakeTaskRepository: + def __init__(self, pool: NodePool) -> None: + self.pool = pool + self.created: list[dict[str, Any]] = [] + + async def create(self, **kwargs: Any) -> Any: + self.created.append(kwargs) + return type("Task", (), {"upid": kwargs["upid"]})() + + class FakeDatabase: def __init__(self, pool: NodePool) -> None: self.pool = pool @@ -58,17 +73,29 @@ def request(pool: NodePool, *, method: str = "GET", path: str = "/") -> Request: return result -async def test_network_and_service_mutations_persist() -> None: +@pytest.fixture +def task_repo(monkeypatch: pytest.MonkeyPatch) -> FakeTaskRepository: + repository = FakeTaskRepository(NodePool()) + monkeypatch.setattr("app.handlers.nodes.TaskRepository", lambda _pool: repository) + return repository + + +async def test_network_and_service_mutations_persist( + monkeypatch: pytest.MonkeyPatch, +) -> None: registry = HandlerRegistry() register_node_ops_handlers(registry) pool = NodePool() + repository = FakeTaskRepository(pool) + monkeypatch.setattr("app.handlers.nodes.TaskRepository", lambda _pool: repository) create = registry.get("/nodes/{node}/network", "POST") listing = registry.get("/nodes/{node}/network", "GET") delete = registry.get("/nodes/{node}/network/{iface}", "DELETE") + reload_network = registry.get("/nodes/{node}/network", "PUT") stop = registry.get("/nodes/{node}/services/{service}/stop", "POST") state = registry.get("/nodes/{node}/services/{service}/state", "GET") - assert create and listing and delete and stop and state + assert create and listing and delete and reload_network and stop and state await create( request(pool, method="POST", path="/api2/json/nodes/pve01/network"), @@ -90,10 +117,17 @@ async def test_network_and_service_mutations_persist() -> None: ) assert all(item["iface"] != "vmbr9" for item in items) - await stop( + reload_upid = await reload_network( + request(pool, method="PUT", path="/api2/json/nodes/pve01/network"), + {"values": {"node": "pve01"}, "provided": frozenset()}, + ) + assert isinstance(reload_upid, str) and reload_upid.startswith("UPID:") + + stop_upid = await stop( request(pool, method="POST", path="/api2/json/nodes/pve01/services/pveproxy/stop"), {"values": {"node": "pve01", "service": "pveproxy"}, "provided": frozenset()}, ) + assert isinstance(stop_upid, str) and stop_upid.startswith("UPID:") service = await state( request(pool), {"values": {"node": "pve01", "service": "pveproxy"}, "provided": frozenset()}, @@ -102,23 +136,27 @@ async def test_network_and_service_mutations_persist() -> None: assert "ops" in pool.metadata -async def test_disk_init_and_wipe_persist() -> None: +async def test_disk_init_and_wipe_persist(monkeypatch: pytest.MonkeyPatch) -> None: registry = HandlerRegistry() register_node_ops_handlers(registry) pool = NodePool() + repository = FakeTaskRepository(pool) + monkeypatch.setattr("app.handlers.nodes.TaskRepository", lambda _pool: repository) initgpt = registry.get("/nodes/{node}/disks/initgpt", "POST") wipe = registry.get("/nodes/{node}/disks/wipedisk", "PUT") listing = registry.get("/nodes/{node}/disks/list", "GET") assert initgpt and wipe and listing - await initgpt( + init_upid = await initgpt( request(pool, method="POST"), {"values": {"node": "pve01", "disk": "/dev/sdb"}, "provided": frozenset()}, ) - await wipe( + wipe_upid = await wipe( request(pool, method="PUT"), {"values": {"node": "pve01", "disk": "/dev/sdb"}, "provided": frozenset()}, ) + assert init_upid.startswith("UPID:") + assert wipe_upid.startswith("UPID:") disks = await listing( request(pool), {"values": {"node": "pve01"}, "provided": frozenset()}, diff --git a/tests/unit/test_qemu_handlers.py b/tests/unit/test_qemu_handlers.py index 36b9710..c720be5 100644 --- a/tests/unit/test_qemu_handlers.py +++ b/tests/unit/test_qemu_handlers.py @@ -41,7 +41,13 @@ class QemuPool: async def fetch(self, sql: str, *args: object) -> list[dict[str, object]]: del args if "FROM resources" in sql: - return [{"vmid": 150, "state": '{"status":"stopped","name":"vm"}'}] + return [ + { + "vmid": 150, + "state": '{"status":"stopped","name":"vm"}', + "config": '{"name":"vm","memory":2048,"cores":2}', + } + ] assert "FROM snapshots" in sql return [ { @@ -325,9 +331,11 @@ async def test_qemu_clone_and_migrate_handlers(monkeypatch: pytest.MonkeyPatch) ) assert migrate_upid.startswith("UPID:pve1:") assert tasks[-1]["task_type"] == "qemu-migrate" - assert ( - await resize(http_request, inputs(node="pve1", vmid=150, disk="scsi0", size="+2G")) is None + resize_upid = await resize( + http_request, inputs(node="pve1", vmid=150, disk="scsi0", size="+2G") ) + assert resize_upid.startswith("UPID:pve1:") + assert tasks[-1]["task_type"] == "qemu-resize" move_upid = await move( http_request, inputs(node="pve1", vmid=150, disk="scsi0", storage="local") ) diff --git a/tests/unit/test_schema_examples.py b/tests/unit/test_schema_examples.py index e5f7c01..3fceb39 100644 --- a/tests/unit/test_schema_examples.py +++ b/tests/unit/test_schema_examples.py @@ -1,6 +1,11 @@ """Tests for contract example generation.""" -from app.contracts.examples import path_param_example, schema_example +from app.contracts.examples import ( + path_param_example, + property_string_example, + schema_example, + wire_param_name, +) from app.contracts.model import Schema @@ -9,6 +14,12 @@ def test_path_param_examples_use_known_placeholders() -> None: assert path_param_example("vmid") == 100 +def test_wire_param_name_expands_indexed() -> None: + assert wire_param_name("link[n]") == "link0" + assert wire_param_name("scsi[n]", index=1) == "scsi1" + assert wire_param_name("vmid") == "vmid" + + def test_schema_example_prefers_default_and_enum() -> None: assert schema_example(Schema(type="string", default="custom")) == "custom" assert schema_example(Schema(type="string", enum=("a", "b"))) == "a" @@ -23,3 +34,49 @@ def test_schema_example_builds_object_and_array() -> None: }, ) assert schema_example(schema) == {"count": 2} + + +def test_schema_example_uses_named_proxmox_formats() -> None: + assert schema_example(Schema(type="string", format="pve-node"), name="node") == "pve01" + assert schema_example(Schema(type="string", format="pve-storage-id")) == "local" + assert schema_example(Schema(type="string", format="ip")) == "192.168.0.1" + # Parameter name hints win over the format token when both apply. + assert schema_example(Schema(type="string", format="pve-node"), name="clustername") == "example" + + +def test_property_string_example_uses_bare_default_key() -> None: + fmt = { + "address": { + "default_key": 1, + "format": "address", + "type": "string", + }, + "priority": { + "default": 0, + "optional": 1, + "type": "integer", + }, + } + assert property_string_example(fmt) == "192.168.0.1" + assert schema_example(Schema(type="string", format=fmt), name="link[n]") == "192.168.0.1" + + +def test_property_string_example_includes_required_keys() -> None: + fmt = { + "enable": { + "default_key": 1, + "default": "1", + "type": "boolean", + }, + "burst": { + "default": 5, + "type": "integer", + "minimum": 0, + }, + "rate": { + "default": "1/second", + "optional": 1, + "type": "string", + }, + } + assert property_string_example(fmt) == "1,burst=5" diff --git a/tests/unit/test_sdn_handlers.py b/tests/unit/test_sdn_handlers.py index 3245a3d..0056df2 100644 --- a/tests/unit/test_sdn_handlers.py +++ b/tests/unit/test_sdn_handlers.py @@ -5,6 +5,7 @@ from __future__ import annotations import json from typing import Any, cast +import pytest from fastapi import FastAPI, Request from app.api.registry import HandlerRegistry @@ -17,6 +18,11 @@ class SdnPool: self.metadata: dict[str, Any] = {} self.nodes = {"pve1"} + async def fetch(self, query: str, *_arguments: object) -> list[dict[str, Any]]: + if "FROM nodes" in query: + return [{"name": name} for name in sorted(self.nodes)] + raise AssertionError(query) + async def fetchrow(self, query: str, *_arguments: object) -> dict[str, Any] | None: if "FROM clusters WHERE id" in query: return {"metadata": json.dumps(self.metadata)} @@ -25,6 +31,8 @@ class SdnPool: async def fetchval(self, query: str, *arguments: object) -> Any: if "EXISTS(SELECT 1 FROM nodes" in query: return str(arguments[0]) in self.nodes + if "SELECT name FROM nodes ORDER BY name LIMIT 1" in query: + return sorted(self.nodes)[0] raise AssertionError(query) async def execute(self, query: str, *arguments: object) -> str: @@ -34,6 +42,16 @@ class SdnPool: raise AssertionError(query) +class FakeTaskRepository: + def __init__(self, pool: SdnPool) -> None: + self.pool = pool + self.created: list[dict[str, Any]] = [] + + async def create(self, **kwargs: Any) -> Any: + self.created.append(kwargs) + return type("Task", (), {"upid": kwargs["upid"]})() + + async def call( registry: HandlerRegistry, path: str, verb: str, http: Request, inputs: dict[str, Any] ) -> Any: @@ -45,7 +63,7 @@ async def call( def request(pool: SdnPool) -> Request: app = FastAPI() app.state.database = cast(AsyncpgDatabase, type("DB", (), {"pool": pool})()) - return Request( + http = Request( { "type": "http", "app": app, @@ -58,13 +76,17 @@ def request(pool: SdnPool) -> Request: "scheme": "http", } ) + http.state.principal = "root@pam" + return http -async def test_sdn_zone_vnet_subnet_and_node_views() -> None: +async def test_sdn_zone_vnet_subnet_and_node_views(monkeypatch: pytest.MonkeyPatch) -> None: registry = HandlerRegistry() register_sdn_handlers(registry) pool = SdnPool() http = request(pool) + repository = FakeTaskRepository(pool) + monkeypatch.setattr("app.handlers.cluster_extra.TaskRepository", lambda _pool: repository) await call( registry, @@ -118,11 +140,20 @@ async def test_sdn_zone_vnet_subnet_and_node_views() -> None: ) assert node_zones[0]["zone"] == "localzone" assert pool.metadata["sdn"]["pending"] is True - await call( + upid = await call( registry, "/cluster/sdn", "PUT", http, {"values": {"release-lock": 1}, "provided": frozenset()}, ) + assert isinstance(upid, str) and upid.startswith("UPID:") assert pool.metadata["sdn"]["pending"] is False + lock_token = await call( + registry, + "/cluster/sdn/lock", + "POST", + http, + {"values": {}, "provided": frozenset()}, + ) + assert isinstance(lock_token, str) and len(lock_token) >= 8 diff --git a/tests/unit/test_seed.py b/tests/unit/test_seed.py index 549e98d..dd299ed 100644 --- a/tests/unit/test_seed.py +++ b/tests/unit/test_seed.py @@ -5,21 +5,23 @@ from typing import Any, cast import pytest from app.simulation.seed import ( + big_profile, build_profile, clear_simulation_state, cluster_domain_metadata, default_node_ops_for_seed, enrich_guest_state, enrich_storage_state, + lab_profile, large_profile, small_profile, stable_id, ) -def test_small_profile_matches_required_logical_shape() -> None: - first = small_profile() - second = small_profile() +def test_lab_profile_matches_required_logical_shape() -> None: + first = lab_profile() + second = lab_profile() assert first == second state = first.logical_state() @@ -29,17 +31,54 @@ def test_small_profile_matches_required_logical_shape() -> None: assert isinstance(resources, list) assert [resource["kind"] for resource in resources].count("qemu") == 2 assert [resource["kind"] for resource in resources].count("lxc") == 1 - assert [resource["kind"] for resource in resources].count("storage") == 2 + assert [resource["kind"] for resource in resources].count("storage") == 3 + assert [resource["kind"] for resource in resources].count("ceph-osd") == 3 + assert [resource["kind"] for resource in resources].count("ha") == 1 tasks = state["tasks"] assert isinstance(tasks, list) assert len(tasks) == 2 +def test_sized_cluster_profiles() -> None: + small = small_profile() + assert len(small.nodes) == 3 + assert sum(resource.kind in {"qemu", "lxc"} for resource in small.resources) == 50 + assert sum(resource.kind == "ceph-osd" for resource in small.resources) == 10 + assert sum(resource.kind == "ha" for resource in small.resources) == 3 + assert len(small.tasks) == 12 + + large = large_profile() + assert len(large.nodes) == 10 + assert sum(resource.kind in {"qemu", "lxc"} for resource in large.resources) == 1_000 + assert sum(resource.kind == "ceph-osd" for resource in large.resources) == 100 + assert sum(resource.kind == "pool" for resource in large.resources) == 2 + assert len(large.tasks) == 50 + + big = big_profile() + assert len(big.nodes) == 20 + assert sum(resource.kind in {"qemu", "lxc"} for resource in big.resources) == 2_000 + assert sum(resource.kind == "ceph-osd" for resource in big.resources) == 500 + assert sum(resource.kind == "pool" for resource in big.resources) == 3 + assert len(big.tasks) == 100 + assert build_profile("big") == big + + +def test_sized_profiles_spread_osds_across_nodes() -> None: + for profile_name, expected in (("small", 10), ("large", 100), ("big", 500)): + profile = build_profile(profile_name) + names = {node.id: node.name for node in profile.nodes} + counts: dict[str, int] = {name: 0 for name in names.values()} + for resource in profile.resources: + if resource.kind == "ceph-osd": + counts[names[resource.node_id]] += 1 + assert sum(counts.values()) == expected + assert max(counts.values()) - min(counts.values()) <= 1 + + def test_medium_and_fault_profiles_are_deterministic() -> None: medium = build_profile("medium") assert len(medium.nodes) == 3 - assert sum(resource.kind == "qemu" for resource in medium.resources) == 50 - assert sum(resource.kind == "lxc" for resource in medium.resources) == 20 + assert sum(resource.kind in {"qemu", "lxc"} for resource in medium.resources) == 50 assert build_profile("ha-demo") == build_profile("ha-demo") broken = build_profile("broken-storage") assert any(resource.state.get("status") == "offline" for resource in broken.resources) @@ -50,7 +89,10 @@ def test_large_profile_is_configurable_and_stable() -> None: second = large_profile(node_count=4, resource_count=1_000) assert first == second assert len(first.nodes) == 4 - assert len(first.resources) == 1_000 + guests = [r for r in first.resources if r.kind in {"qemu", "lxc"}] + assert len(guests) == 1_000 + assert any(r.kind == "storage" and r.external_id == "local" for r in first.resources) + assert any(r.kind == "ceph-osd" for r in first.resources) def test_profile_validation() -> None: @@ -108,7 +150,7 @@ def test_stable_ids_are_namespaced_and_repeatable() -> None: def test_cluster_domain_metadata_seeds_list_domains() -> None: - meta = cast(dict[str, Any], cluster_domain_metadata(small_profile())) + meta = cast(dict[str, Any], cluster_domain_metadata(lab_profile())) assert meta["firewall"]["scopes"]["cluster"]["rules"] assert meta["firewall"]["scopes"]["cluster"]["aliases"] assert meta["firewall"]["scopes"]["cluster"]["ipset"] @@ -127,23 +169,36 @@ def test_cluster_domain_metadata_seeds_list_domains() -> None: assert meta["notifications"]["matchers"] assert meta["notifications"]["matcher_fields"] assert meta["acme"]["accounts"] + assert meta["acme"]["accounts"]["letsencrypt"]["tos_url"] assert meta["acme"]["plugins"] assert meta["acme"]["directories"] assert meta["acme"]["challenge_schema"] assert meta["mapping"]["pci"] assert meta["mapping"]["usb"] assert meta["mapping"]["dir"] - assert meta["replication"] + # Single-node lab must not invent ghost peers. + assert meta["replication"] == [] + assert "pve02" not in str(meta["sdn"]["controllers"]) assert meta["metrics"]["servers"] assert meta["ha_groups"] assert meta["ceph"]["pools"] assert meta["ceph"]["flags"] + assert "noup" in meta["ceph"]["flags"] + assert meta["ceph"]["flags"]["nobackfill"] == 0 + assert meta["ceph"]["health"]["status"] == "HEALTH_OK" assert meta["ceph"]["version"] + assert len([r for r in lab_profile().resources if r.kind == "ceph-osd"]) == 3 + assert any(r.kind == "storage" and r.external_id == "ceph" for r in lab_profile().resources) + local = next(r for r in lab_profile().resources if r.external_id == "local") + assert int(cast(int, local.state["total_bytes"])) > 0 + assert lab_profile().tasks[0].payload["node"] == "pve01" assert meta["qemu_cpu_flags"] assert meta["metrics"]["export_data"] assert meta["jobs"]["schedule_analyze_results"] - assert meta["replication"][0]["log"] ops = cast(dict[str, Any], default_node_ops_for_seed("pve01")) + assert ops["vzdump"]["defaults"]["storage"] == "local" + assert ops["status"]["mem"] > 0 + assert ops["status"]["maxmem"] > 0 assert ops["capabilities"]["cpu"] assert ops["capabilities"]["machines"] assert ops["hosts"]["data"] @@ -154,11 +209,21 @@ def test_cluster_domain_metadata_seeds_list_domains() -> None: assert ops["status"]["uptime"] assert ops["ip"] assert meta["cluster_config"]["totem"] + assert meta["cluster_config"]["config_digest"] + assert meta["cluster_config"]["corosync_conf"] + assert meta["cluster_config"]["corosync_authkey"] + first_node = next(iter(meta["cluster_config"]["added_nodes"].values())) + assert first_node["pve_fp"].count(":") == 31 + assert first_node["pve_addr"] + assert meta["backup_jobs"]["backup-daily"]["storage"] == "local" assert meta["quorate"] == 1 guest = cast( dict[str, Any], enrich_guest_state({"name": "demo", "status": "stopped"}, kind="qemu", vmid="100"), ) + assert guest["scsi0"].startswith("local-lvm:") + assert guest["ostype"] == "l26" + assert str(guest["net0"]).startswith("virtio=") assert guest["agent"]["results"]["info"] assert guest["rrddata"] assert guest["migrate_preconditions"] @@ -170,6 +235,36 @@ def test_cluster_domain_metadata_seeds_list_domains() -> None: assert storage["file_restore"] assert storage["import_metadata"] + small_meta = cast(dict[str, Any], cluster_domain_metadata(small_profile())) + assert len(small_meta["replication"]) == 2 + assert small_meta["replication"][0]["guest"] == 101 + assert len(small_meta["backup_jobs"]) == 2 + assert len(small_meta["ha_groups"]) == 2 + assert small_meta["sdn"]["zones"]["public"]["digest"] + assert small_meta["sdn"]["zones"]["public"]["state"] == "available" + assert len(cast(dict[str, Any], cluster_domain_metadata(large_profile()))["replication"]) == 8 + assert len(cast(dict[str, Any], cluster_domain_metadata(large_profile()))["ha_groups"]) == 4 + assert len(cast(dict[str, Any], cluster_domain_metadata(big_profile()))["replication"]) == 16 + assert len(cast(dict[str, Any], cluster_domain_metadata(big_profile()))["ha_groups"]) == 6 + assert len(cast(dict[str, Any], cluster_domain_metadata(big_profile()))["ceph"]["pools"]) == 3 + + ops = cast(dict[str, Any], default_node_ops_for_seed("pve1", node_index=1, node_count=3)) + apt_repos = cast(dict[str, Any], cast(dict[str, Any], ops["apt"])["repositories"]) + assert "files" in apt_repos and "standard-repos" in apt_repos + cert = cast(list[dict[str, Any]], cast(dict[str, Any], ops["certificates"])["info"])[0] + assert cert["pem"] and cert["san"] and cert["public-key-bits"] == 4096 + + +def test_sized_profile_artifact_targets() -> None: + from app.simulation.seed import _CLUSTER_SIZE_SPECS + + assert _CLUSTER_SIZE_SPECS["small"]["backups"] == 10 + assert _CLUSTER_SIZE_SPECS["small"]["snapshots"] == 12 + assert _CLUSTER_SIZE_SPECS["large"]["backups"] == 80 + assert _CLUSTER_SIZE_SPECS["large"]["snapshots"] == 100 + assert _CLUSTER_SIZE_SPECS["big"]["backups"] == 160 + assert _CLUSTER_SIZE_SPECS["big"]["snapshots"] == 200 + @pytest.mark.asyncio async def test_clear_simulation_state_wipes_api_created_identity() -> None: