From 48df10b17e48ae9c52f0e75d30b4f8103449c3d5 Mon Sep 17 00:00:00 2001 From: Sergey Antropoff Date: Sat, 18 Jul 2026 04:18:05 +0300 Subject: [PATCH] =?UTF-8?q?Prepare=200.1.0=20for=20lab=20release:=20durabl?= =?UTF-8?q?e=20handlers,=20HTTP=20Compose,=20CI,=20and=20pulumi-tests.=20-?= =?UTF-8?q?=20Harden=20DB-backed=20handlers=20and=20seed=20profiles;=20ali?= =?UTF-8?q?gn=20client=20wire=20shapes=20for=20=20=20cluster=20resources,?= =?UTF-8?q?=20QEMU=20config,=20and=20node=20SSL=20fields=20-=20Serve=20pla?= =?UTF-8?q?in=20HTTP=20on=20Compose=20:8006;=20keep=20TLS=20optional=20(--?= =?UTF-8?q?profile=20tls)=20and=20=20=20terminate=20HTTPS=20at=20Kubernete?= =?UTF-8?q?s=20Ingress=20-=20Add=20pulumi-tests=20(full=20contract=20surfa?= =?UTF-8?q?ce=20majors=206=E2=80=939=20+=20BPG=20lifecycle)=20and=20=20=20?= =?UTF-8?q?make=20pulumi-tests=20-=20Ship=20bilingual=20docs,=20CHANGELOG,?= =?UTF-8?q?=20SECURITY,=20CONTRIBUTING,=20and=20GitHub=20Actions=20=20=20(?= =?UTF-8?q?make=20ci=20+=20Compose/Helm=20validation)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 4 +- .github/workflows/ci.yml | 60 + CHANGELOG.md | 30 + CONTRIBUTING.md | 66 + CONTRIBUTING.ru.md | 63 + Dockerfile | 4 +- Makefile | 31 +- README.md | 58 +- README.ru.md | 210 +++ SECURITY.md | 52 + app/api/registry.py | 68 +- app/contracts/runtime.py | 4 +- app/handlers/acme.py | 92 +- app/handlers/backup.py | 33 +- app/handlers/ceph.py | 155 +-- app/handlers/cluster.py | 33 +- app/handlers/cluster_config.py | 75 +- app/handlers/cluster_extra.py | 156 +-- app/handlers/core.py | 218 +++- app/handlers/firewall.py | 111 +- app/handlers/ha.py | 57 +- app/handlers/lxc.py | 10 +- app/handlers/lxc_extra.py | 24 +- app/handlers/mapping.py | 25 +- app/handlers/nodes.py | 135 +- app/handlers/nodes_extra.py | 304 ++--- app/handlers/notifications.py | 107 +- app/handlers/pools.py | 4 +- app/handlers/qemu.py | 108 +- app/handlers/qemu_extra.py | 72 +- app/handlers/sdn.py | 244 ++-- app/handlers/storage.py | 74 +- app/simulation/seed.py | 1147 ++++++++++++++++- app/surface_probe.py | 10 +- app/tasks/qemu.py | 4 +- app/web/index.html | 340 ++++- docker-compose.release.yml | 53 +- docker-compose.yml | 12 +- docker/tls/gateway.conf | 25 +- docs/README.md | 30 + docs/api-surface.md | 5 +- docs/api-versions.md | 2 + docs/architecture.md | 4 +- docs/authentication.md | 7 +- docs/clients.md | 45 +- docs/compatibility-0.1.0.md | 2 + docs/compatibility.md | 2 + docs/configuration.md | 35 +- docs/docker-hub-overview.md | 40 + docs/domains/README.md | 2 + docs/domains/access.md | 2 + docs/domains/ceph.md | 2 + docs/domains/cluster-extras.md | 2 + docs/domains/core-cluster.md | 2 + docs/domains/firewall.md | 2 + docs/domains/ha.md | 2 + docs/domains/lxc.md | 2 + docs/domains/pools.md | 2 + docs/domains/qemu.md | 2 + docs/domains/sdn.md | 2 + docs/domains/storage-backup.md | 2 + docs/domains/tasks.md | 2 + docs/examples/ansible.md | 2 + docs/examples/go.md | 2 + docs/examples/java.md | 2 + docs/examples/overview.md | 4 +- docs/examples/perl.md | 2 + docs/examples/pulumi.md | 2 + docs/examples/python-proxmoxer.md | 4 +- docs/examples/python-requests.md | 2 + docs/examples/terraform.md | 4 +- docs/examples/troubleshooting-clients.md | 4 +- docs/faq.md | 20 +- docs/getting-started.md | 34 +- docs/kubernetes.md | 42 +- docs/observability.md | 2 + docs/operations.md | 23 +- docs/ru/README.md | 30 + docs/ru/api-surface.md | 86 ++ docs/ru/api-versions.md | 79 ++ docs/ru/architecture.md | 240 ++++ docs/ru/authentication.md | 85 ++ docs/ru/clients.md | 42 + docs/ru/compatibility-0.1.0.md | 109 ++ docs/ru/compatibility.md | 78 ++ docs/ru/configuration.md | 107 ++ docs/ru/domains/README.md | 28 + docs/ru/domains/access.md | 21 + docs/ru/domains/ceph.md | 9 + docs/ru/domains/cluster-extras.md | 13 + docs/ru/domains/core-cluster.md | 25 + docs/ru/domains/firewall.md | 10 + docs/ru/domains/ha.md | 10 + docs/ru/domains/lxc.md | 15 + docs/ru/domains/pools.md | 6 + docs/ru/domains/qemu.md | 19 + docs/ru/domains/sdn.md | 10 + docs/ru/domains/storage-backup.md | 18 + docs/ru/domains/tasks.md | 25 + docs/ru/examples/ansible.md | 14 + docs/ru/examples/go.md | 13 + docs/ru/examples/java.md | 13 + docs/ru/examples/overview.md | 58 + docs/ru/examples/perl.md | 11 + docs/ru/examples/pulumi.md | 15 + docs/ru/examples/python-proxmoxer.md | 23 + docs/ru/examples/python-requests.md | 13 + docs/ru/examples/terraform.md | 20 + docs/ru/examples/troubleshooting-clients.md | 13 + docs/ru/faq.md | 58 + docs/ru/getting-started.md | 179 +++ docs/ru/kubernetes.md | 190 +++ docs/ru/observability.md | 43 + docs/ru/operations.md | 157 +++ docs/ru/security.md | 48 + docs/ru/seed-profiles.md | 67 + docs/ru/troubleshooting.md | 66 + docs/ru/web-ui.md | 64 + docs/security.md | 17 +- docs/seed-profiles.md | 23 +- docs/troubleshooting.md | 13 +- docs/web-ui.md | 4 + examples/README.md | 2 + examples/README.ru.md | 29 + examples/go/main.go | 2 +- examples/java/Cookbook.java | 2 +- examples/perl/cookbook.pl | 2 +- examples/pulumi/__main__.py | 2 +- examples/python/proxmoxer_cookbook.py | 2 +- examples/python/requests_cookbook.py | 2 +- examples/terraform/main.tf | 4 +- examples/terraform/terraform.tfvars.example | 2 +- helm/proxmox-api-simulator/Chart.yaml | 3 +- helm/proxmox-api-simulator/README.md | 10 +- helm/proxmox-api-simulator/README.ru.md | 47 + .../values-ingress-example.yaml | 8 +- helm/proxmox-api-simulator/values.yaml | 4 + pulumi-tests/.env.example | 14 + pulumi-tests/.gitignore | 11 + pulumi-tests/Makefile | 43 + pulumi-tests/README.md | 49 + pulumi-tests/README.ru.md | 49 + pulumi-tests/docker/Dockerfile.pulumi-runner | 22 + pulumi-tests/docker/docker-compose.yml | 188 +++ .../lifecycle/Pulumi.hxlife11784271995.yaml | 3 + .../lifecycle/Pulumi.hxlife11784272176.yaml | 3 + .../lifecycle/Pulumi.hxlife11784272263.yaml | 3 + .../lifecycle/Pulumi.hxlife11784272499.yaml | 3 + .../lifecycle/Pulumi.hxlife11784272593.yaml | 3 + .../lifecycle/Pulumi.hxlife11784272655.yaml | 3 + .../lifecycle/Pulumi.hxlife11784273435.yaml | 3 + .../lifecycle/Pulumi.hxlife11784273461.yaml | 3 + .../lifecycle/Pulumi.hxlife11784273781.yaml | 3 + .../lifecycle/Pulumi.hxlife11784273929.yaml | 3 + .../lifecycle/Pulumi.hxlife11784274056.yaml | 3 + .../lifecycle/Pulumi.hxlife11784284064.yaml | 3 + .../lifecycle/Pulumi.hxlife11784284141.yaml | 3 + .../pulumi/programs/lifecycle/Pulumi.yaml | 3 + .../pulumi/programs/lifecycle/__main__.py | 153 +++ pulumi-tests/pulumi/pvelib/__init__.py | 5 + pulumi-tests/pulumi/pvelib/api.py | 132 ++ pulumi-tests/pulumi/pvelib/surface.py | 353 +++++ pulumi-tests/pulumi/report.py | 265 ++++ pulumi-tests/pulumi/run_suite.py | 273 ++++ pulumi-tests/tools/cleanup_test_resources.py | 43 + pyproject.toml | 4 + tests/compatibility/test_proxmoxer.py | 28 +- tests/unit/test_core_handlers.py | 11 +- tests/unit/test_dynamic_routes.py | 26 +- tests/unit/test_extended_handlers.py | 8 +- tests/unit/test_qemu_handlers.py | 14 +- tests/unit/test_seed.py | 70 + 172 files changed, 7528 insertions(+), 1208 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md create mode 100644 CONTRIBUTING.ru.md create mode 100644 README.ru.md create mode 100644 SECURITY.md create mode 100644 docs/README.md create mode 100644 docs/docker-hub-overview.md create mode 100644 docs/ru/README.md create mode 100644 docs/ru/api-surface.md create mode 100644 docs/ru/api-versions.md create mode 100644 docs/ru/architecture.md create mode 100644 docs/ru/authentication.md create mode 100644 docs/ru/clients.md create mode 100644 docs/ru/compatibility-0.1.0.md create mode 100644 docs/ru/compatibility.md create mode 100644 docs/ru/configuration.md create mode 100644 docs/ru/domains/README.md create mode 100644 docs/ru/domains/access.md create mode 100644 docs/ru/domains/ceph.md create mode 100644 docs/ru/domains/cluster-extras.md create mode 100644 docs/ru/domains/core-cluster.md create mode 100644 docs/ru/domains/firewall.md create mode 100644 docs/ru/domains/ha.md create mode 100644 docs/ru/domains/lxc.md create mode 100644 docs/ru/domains/pools.md create mode 100644 docs/ru/domains/qemu.md create mode 100644 docs/ru/domains/sdn.md create mode 100644 docs/ru/domains/storage-backup.md create mode 100644 docs/ru/domains/tasks.md create mode 100644 docs/ru/examples/ansible.md create mode 100644 docs/ru/examples/go.md create mode 100644 docs/ru/examples/java.md create mode 100644 docs/ru/examples/overview.md create mode 100644 docs/ru/examples/perl.md create mode 100644 docs/ru/examples/pulumi.md create mode 100644 docs/ru/examples/python-proxmoxer.md create mode 100644 docs/ru/examples/python-requests.md create mode 100644 docs/ru/examples/terraform.md create mode 100644 docs/ru/examples/troubleshooting-clients.md create mode 100644 docs/ru/faq.md create mode 100644 docs/ru/getting-started.md create mode 100644 docs/ru/kubernetes.md create mode 100644 docs/ru/observability.md create mode 100644 docs/ru/operations.md create mode 100644 docs/ru/security.md create mode 100644 docs/ru/seed-profiles.md create mode 100644 docs/ru/troubleshooting.md create mode 100644 docs/ru/web-ui.md create mode 100644 examples/README.ru.md create mode 100644 helm/proxmox-api-simulator/README.ru.md create mode 100644 pulumi-tests/.env.example create mode 100644 pulumi-tests/.gitignore create mode 100644 pulumi-tests/Makefile create mode 100644 pulumi-tests/README.md create mode 100644 pulumi-tests/README.ru.md create mode 100644 pulumi-tests/docker/Dockerfile.pulumi-runner create mode 100644 pulumi-tests/docker/docker-compose.yml create mode 100644 pulumi-tests/pulumi/programs/lifecycle/Pulumi.hxlife11784271995.yaml create mode 100644 pulumi-tests/pulumi/programs/lifecycle/Pulumi.hxlife11784272176.yaml create mode 100644 pulumi-tests/pulumi/programs/lifecycle/Pulumi.hxlife11784272263.yaml create mode 100644 pulumi-tests/pulumi/programs/lifecycle/Pulumi.hxlife11784272499.yaml create mode 100644 pulumi-tests/pulumi/programs/lifecycle/Pulumi.hxlife11784272593.yaml create mode 100644 pulumi-tests/pulumi/programs/lifecycle/Pulumi.hxlife11784272655.yaml create mode 100644 pulumi-tests/pulumi/programs/lifecycle/Pulumi.hxlife11784273435.yaml create mode 100644 pulumi-tests/pulumi/programs/lifecycle/Pulumi.hxlife11784273461.yaml create mode 100644 pulumi-tests/pulumi/programs/lifecycle/Pulumi.hxlife11784273781.yaml create mode 100644 pulumi-tests/pulumi/programs/lifecycle/Pulumi.hxlife11784273929.yaml create mode 100644 pulumi-tests/pulumi/programs/lifecycle/Pulumi.hxlife11784274056.yaml create mode 100644 pulumi-tests/pulumi/programs/lifecycle/Pulumi.hxlife11784284064.yaml create mode 100644 pulumi-tests/pulumi/programs/lifecycle/Pulumi.hxlife11784284141.yaml create mode 100644 pulumi-tests/pulumi/programs/lifecycle/Pulumi.yaml create mode 100644 pulumi-tests/pulumi/programs/lifecycle/__main__.py create mode 100644 pulumi-tests/pulumi/pvelib/__init__.py create mode 100644 pulumi-tests/pulumi/pvelib/api.py create mode 100644 pulumi-tests/pulumi/pvelib/surface.py create mode 100644 pulumi-tests/pulumi/report.py create mode 100644 pulumi-tests/pulumi/run_suite.py create mode 100644 pulumi-tests/tools/cleanup_test_resources.py diff --git a/.env.example b/.env.example index 625b0b5..c3b0f74 100644 --- a/.env.example +++ b/.env.example @@ -23,5 +23,7 @@ SIMULATION_SEED=42 SIMULATION_TIME_SCALE=10 SIMULATOR_ADMIN_ENABLED=false SIMULATOR_ADMIN_TOKEN=replace-with-a-long-random-secret -PROXMOXER_HOST=tls-gateway +PROXMOXER_HOST= PROXMOXER_PORT=8443 +# For proxmoxer (HTTPS-only): docker compose --profile tls up -d +# then PROXMOXER_HOST=tls-gateway PROXMOXER_PORT=8443 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..2645711 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,60 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + validate: + name: Compose / Helm validate + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - name: Validate Compose files + run: | + docker compose -f docker-compose.yml config --quiet + docker compose -f docker-compose.release.yml config --quiet + + - uses: azure/setup-helm@v4 + with: + version: v3.16.4 + + - name: Lint and render Helm chart + run: | + make helm-lint + make helm-template >/dev/null + + quality: + name: make ci + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + + - name: Prepare lab env + run: test -f .env || cp .env.example .env + + - name: Build runtime and development images + run: make install + + - name: Ruff + mypy + offline pytest + API surface probe + run: make ci + + - name: Upload coverage XML + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-xml + path: coverage.xml + if-no-files-found: ignore diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..c1d12fe --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,30 @@ +# Changelog + +All notable changes to this project are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.1.0] - 2026-07-17 + +### Added + +- Stateful Proxmox VE API simulator backed by PostgreSQL with durable UPID task + workers. +- Imported official API contracts for PVE majors **6.4-15**, **7.4-16**, + **8.4.5**, and **9.2.3** with full handler registration for declared methods. +- Interactive Web UI (catalog, runtime contract Apply, task monitor). +- Development Compose stack with plain HTTP on host `:8006` (real PVE port; real + PVE uses HTTPS). Optional TLS gateway via `--profile tls` on `:8443`. +- Published runtime image workflow (`make release`) for + [`inecs/proxmox-api-simulator`](https://hub.docker.com/r/inecs/proxmox-api-simulator). +- `docker-compose.release.yml` + Helm chart with optional Ingress / cert-manager. +- Bilingual documentation (English + Russian) and client cookbooks. +- GitHub Actions CI (`make ci`, Compose/Helm validation). + +### Security + +- Explicit lab-only threat model: default secrets and open `/ui` / `/admin` + helpers are for local/CI use. See [SECURITY.md](SECURITY.md). + +[0.1.0]: https://github.com/sergeyantropoff/proxmox-api-simulator/releases/tag/v0.1.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..aae8224 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,66 @@ +**Language / Язык:** [English](CONTRIBUTING.md) | [Русский](CONTRIBUTING.ru.md) + +# Contributing + +Thanks for helping improve the Proxmox VE API laboratory simulator. + +## Prerequisites + +- Docker + Docker Compose +- `make` +- No local Python toolchain required for day-to-day work (tools run in Compose) + +## Local loop + +```bash +cp -n .env.example .env +make install +make up +make seed PROFILE=small +curl -sS http://localhost:8006/health/ready +``` + +Primary HTTP endpoint (same port as real PVE): `http://localhost:8006/` + +Optional HTTPS for proxmoxer: `docker compose --profile tls` → +`https://localhost:8443/` (self-signed — use `curl -sk` or accept the browser +warning). + +## Quality gates (must pass before a PR) + +```bash +make ci # ruff format/check + mypy + offline pytest + surface probe +make ci-all # also remaining integration + proxmoxer compatibility +make helm-lint # chart lint (+ ingress example values) +make pulumi-tests # Pulumi surface (majors 6–9) + pulumi-proxmoxve lifecycle +``` + +GitHub Actions runs `make ci` plus Compose/Helm validation on every push and PR +to `main`. Run `make ci-all`, `make helm-lint`, and `make pulumi-tests` locally +before larger API or client-facing changes. + +## Project rules worth remembering + +1. Mutations must **persist to PostgreSQL** (tables and/or jsonb metadata). +2. Do **not** add user-facing “not supported in the simulator” style errors — + see `.cursor/rules/durable-simulator.mdc`. +3. Prefer matching Proxmox request/response shapes from the contract snapshot. +4. Keep EN and RU docs in sync when you change operator-facing behaviour. + +## Docs + +- Index: [docs/README.md](docs/README.md) · [docs/ru/README.md](docs/ru/README.md) +- Security / lab threat model: [SECURITY.md](SECURITY.md) + +## Releases + +Maintainers publish the runtime image with: + +```bash +docker login +make release # pushes inecs/proxmox-api-simulator: (+ :latest) +``` + +After a public release, paste the overview from +[docs/docker-hub-overview.md](docs/docker-hub-overview.md) into the Docker Hub +repository description if it drifted. diff --git a/CONTRIBUTING.ru.md b/CONTRIBUTING.ru.md new file mode 100644 index 0000000..c8713e1 --- /dev/null +++ b/CONTRIBUTING.ru.md @@ -0,0 +1,63 @@ +**Language / Язык:** [English](CONTRIBUTING.md) | [Русский](CONTRIBUTING.ru.md) + +# Участие в разработке + +Спасибо за помощь с лабораторным симулятором Proxmox VE API. + +## Требования + +- Docker + Docker Compose +- `make` +- Локальный Python для повседневной работы не нужен (инструменты в Compose) + +## Локальный цикл + +```bash +cp -n .env.example .env +make install +make up +make seed PROFILE=small +curl -sS http://localhost:8006/health/ready +``` + +Основной HTTPS-эндпоинт (как у реального PVE): `http://localhost:8006/` +(self-signed — `curl -sk` или принять предупреждение в браузере). + +## Quality gates (обязательны перед PR) + +```bash +make ci # ruff format/check + mypy + offline pytest + surface probe +make ci-all # ещё integration + proxmoxer compatibility +make helm-lint # lint chart (+ ingress example values) +make pulumi-tests # Pulumi surface (majors 6–9) + pulumi-proxmoxve lifecycle +``` + +GitHub Actions на каждый push/PR в `main` запускает `make ci` и проверку +Compose/Helm. Перед крупными API/клиентскими изменениями локально прогоняйте +`make ci-all`, `make helm-lint` и `make pulumi-tests`. + +## Важные правила проекта + +1. Мутации должны **persist в PostgreSQL** (таблицы и/или jsonb metadata). +2. Не добавляйте user-facing сообщения вроде «not supported in the simulator» — + см. `.cursor/rules/durable-simulator.mdc`. +3. Сохраняйте формы запросов/ответов Proxmox из снимка контракта. +4. При изменении операторского поведения синхронизируйте EN и RU docs. + +## Документация + +- Индекс: [docs/README.md](docs/README.md) · [docs/ru/README.md](docs/ru/README.md) +- Безопасность / модель угроз лаборатории: [SECURITY.md](SECURITY.md) + +## Релизы + +Maintainers публикуют runtime-образ так: + +```bash +docker login +make release # пушит inecs/proxmox-api-simulator: (+ :latest) +``` + +После публичного релиза при необходимости вставьте overview из +[docs/docker-hub-overview.md](docs/docker-hub-overview.md) в описание репозитория +Docker Hub. diff --git a/Dockerfile b/Dockerfile index 9f7d23f..fc7838b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,9 @@ FROM python:3.13-slim AS runtime ARG APP_VERSION=0.1.0 LABEL org.opencontainers.image.title="proxmox-api-simulator" \ org.opencontainers.image.version="$APP_VERSION" \ - org.opencontainers.image.source="https://github.com/example/proxmox-api-simulator" + org.opencontainers.image.description="Stateful Proxmox VE API simulator for labs and CI" \ + org.opencontainers.image.source="https://github.com/sergeyantropoff/proxmox-api-simulator" \ + org.opencontainers.image.licenses="Apache-2.0" ENV PATH="/opt/venv/bin:$PATH" \ PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ diff --git a/Makefile b/Makefile index 0a3d50f..84a45a6 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ PUSH_LATEST ?= 1 COMPOSE_RELEASE ?= $(COMPOSE) -f docker-compose.release.yml HELM_CHART ?= ./helm/proxmox-api-simulator -.PHONY: help install format lint typecheck test test-unit test-integration test-contract test-compatibility test-surface evidence coverage run dev up down restart logs docker-build docker-up docker-down docker-logs docker-restart db-up db-down db-migrate db-reset api-import api-diff seed clean ci ci-all shell release release-build release-up release-down release-seed helm-deps helm-template +.PHONY: help install format lint typecheck test test-unit test-integration test-contract test-compatibility test-surface evidence coverage run dev up down restart logs docker-build docker-up docker-down docker-logs docker-restart db-up db-down db-migrate db-reset api-import api-diff seed clean ci ci-all shell release release-build release-up release-down release-seed helm-deps helm-template helm-lint pulumi-tests help: ## Show available commands @awk 'BEGIN {FS = ":.*## "}; /^[a-zA-Z0-9_-]+:.*## / {printf "%-22s %s\n", $$1, $$2}' $(MAKEFILE_LIST) @@ -45,11 +45,16 @@ test-integration: ## Run tests that require PostgreSQL test-contract: ## Run offline API contract tests $(COMPOSE) run --rm --no-deps $(SERVICE_DEV) pytest -m contract -test-compatibility: ## Run proxmoxer smoke flow against the Compose stack +test-compatibility: ## Run proxmoxer smoke flow (needs --profile tls; proxmoxer is HTTPS-only) @test -f .env || cp .env.example .env - $(COMPOSE) up -d --build --wait - $(COMPOSE) run --rm --entrypoint python $(SERVICE_SIM) -m app.simulation.seed_cli - $(COMPOSE) run --rm $(SERVICE_DEV) pytest -m compatibility + $(COMPOSE) --profile tls up -d --build --wait + # Medium profile provides pve1/pve2/pve3 required by the proxmoxer migration smoke. + $(COMPOSE) run --rm -e SEED_PROFILE=medium --entrypoint python $(SERVICE_SIM) \ + -m app.simulation.seed_cli + $(COMPOSE) run --rm \ + -e PROXMOXER_HOST=tls-gateway \ + -e PROXMOXER_PORT=8443 \ + $(SERVICE_DEV) pytest -m compatibility test-surface: ## Probe every declared method on majors 6-9 (0x501 / 0xexception) @test -f .env || cp .env.example .env @@ -66,7 +71,7 @@ run: ## Run the application in the foreground @test -f .env || cp .env.example .env $(COMPOSE) up --build -up: ## Start PostgreSQL, simulator, and TLS gateway +up: ## Start PostgreSQL and simulator (plain HTTP :8006) @test -f .env || cp .env.example .env $(COMPOSE) up -d --build --wait @@ -90,8 +95,8 @@ docker-build: ## Build runtime and development images docker-up: up ## Alias for up -docker-restart: ## Rebuild and recreate simulator and TLS gateway only - $(COMPOSE) up -d --build --force-recreate simulator tls-gateway +docker-restart: ## Rebuild and recreate the simulator + $(COMPOSE) up -d --build --force-recreate simulator docker-down: down ## Alias for down @@ -176,8 +181,18 @@ release-seed: ## Seed the published Hub stack (PROFILE=small by default) helm-deps: ## No-op placeholder (chart has no OCI dependencies) @echo "Chart $(HELM_CHART) vendors PostgreSQL templates; no helm dependency update required." +helm-lint: ## Lint the Helm chart (requires helm) + helm lint $(HELM_CHART) + helm lint $(HELM_CHART) -f $(HELM_CHART)/values-ingress-example.yaml \ + --set certManager.email=docs@example.com \ + --set secret.ticketSigningKey=docs-only-signing-key + helm-template: ## Render Helm manifests locally (requires helm) helm template pve-sim $(HELM_CHART) \ -f $(HELM_CHART)/values-ingress-example.yaml \ --set certManager.email=docs@example.com \ --set secret.ticketSigningKey=docs-only-signing-key + +pulumi-tests: ## Full Pulumi suite (surface majors 6–9 + lifecycle, HTML report) + $(MAKE) -C pulumi-tests up + $(MAKE) -C pulumi-tests test diff --git a/README.md b/README.md index 3ce67a2..22d4012 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,20 @@ +**Language / Язык:** [English](README.md) | [Русский](README.ru.md) + # proxmox-api-simulator +[![CI](https://github.com/sergeyantropoff/proxmox-api-simulator/actions/workflows/ci.yml/badge.svg)](https://github.com/sergeyantropoff/proxmox-api-simulator/actions/workflows/ci.yml) +[![Docker Image](https://img.shields.io/docker/v/inecs/proxmox-api-simulator?label=docker%20hub&sort=semver)](https://hub.docker.com/r/inecs/proxmox-api-simulator) +[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE) + Stateful asynchronous [Proxmox VE](https://www.proxmox.com/) API simulator for testing API clients and infrastructure tooling without a real hypervisor cluster. +> **Laboratory / CI only.** Default credentials, signing keys, and open UI/admin +> helpers are intentional lab defaults. Do **not** expose this stack to the +> public Internet without replacing secrets and adding your own controls. +> See [SECURITY.md](SECURITY.md) and [Security](docs/security.md). + The simulator is backed by PostgreSQL, driven by imported official API contracts, and exposes the same `/api2/json` and `/api2/extjs` surfaces as Proxmox VE. Semantic handlers persist mutations; long-running work returns @@ -37,13 +48,16 @@ Image: [`inecs/proxmox-api-simulator`](https://hub.docker.com/r/inecs/proxmox-ap ### Docker Compose +Needs a checkout that includes `docker-compose.release.yml`. Do not publish host +`:8006` beyond a trusted lab without rotating `TICKET_SIGNING_KEY` / DB password. + ```bash docker compose -f docker-compose.release.yml up -d docker compose -f docker-compose.release.yml run --rm --entrypoint python \ simulator -m app.simulation.seed_cli -curl http://localhost:8006/health/ready -curl http://localhost:8006/api2/json/version +curl -sS http://localhost:8006/health/ready +curl -sS http://localhost:8006/api2/json/version ``` Or: `make release-up && make release-seed PROFILE=small` @@ -57,10 +71,14 @@ helm upgrade --install pve-sim ./helm/proxmox-api-simulator \ --set certManager.email=you@example.com \ --set ingress.hosts[0].host=pve-sim.example.com \ --set ingress.tls[0].hosts[0]=pve-sim.example.com \ - --set secret.ticketSigningKey="$(openssl rand -hex 32)" + --set secret.ticketSigningKey="$(openssl rand -hex 32)" \ + --set postgresql.auth.password="$(openssl rand -hex 16)" ``` -Requires an Ingress controller and cert-manager. Details: +Requires an Ingress controller and cert-manager. The chart Service speaks +**HTTP** `:8006`; TLS terminates at Ingress. Compose also serves plain HTTP on +`:8006`; optional HTTPS for proxmoxer-style clients is +`docker compose --profile tls` on host `:8443`. Details: [Kubernetes / Helm](docs/kubernetes.md). - HTTP API and Web UI (Compose): [http://localhost:8006/](http://localhost:8006/) @@ -76,22 +94,24 @@ make install make up make seed PROFILE=small -curl http://localhost:8006/health/ready -curl http://localhost:8006/api2/json/version -curl -X POST -d 'username=root@pam&password=secret' \ +curl -sS http://localhost:8006/health/ready +curl -sS http://localhost:8006/api2/json/version +curl -sS -X POST -d 'username=root@pam&password=secret' \ http://localhost:8006/api2/json/access/ticket ``` - HTTP API and Web UI: [http://localhost:8006/](http://localhost:8006/) -- HTTPS gateway (self-signed, development only): `https://localhost:8007` - — checked-in `docker/tls/server.key` is a **lab-only** localhost cert; do not - reuse it outside local Compose. + (real PVE uses **HTTPS** on `:8006`; this lab uses plain HTTP on the same + port. Optional TLS for proxmoxer: `docker compose --profile tls` → + `https://localhost:8443/`) +- Port map and TLS notes: [Ports and TLS](docs/configuration.md#ports-and-tls). - FastAPI schema docs: [http://localhost:8006/docs](http://localhost:8006/docs) ### Web UI -Interactive console with light/dark themes, endpoint catalog for PVE 6–9, and -runtime contract hot-swap. More detail: [Web UI](docs/web-ui.md). +Interactive console with light/dark themes, endpoint catalog for PVE 6–9, +runtime contract hot-swap, and a UPID task monitor. More detail: +[Web UI](docs/web-ui.md). ![Web UI light theme](docs/images/web-ui-light.png) @@ -99,6 +119,10 @@ runtime contract hot-swap. More detail: [Web UI](docs/web-ui.md). ## Documentation +Documentation is bilingual. Use the **Language / Язык** switcher at the top of +each page, or open the Russian root [README.ru.md](README.ru.md). Index: +[docs/README.md](docs/README.md) · [docs/ru/README.md](docs/ru/README.md). + | Guide | Description | |---|---| | [Getting started](docs/getting-started.md) | First successful lab session | @@ -120,6 +144,8 @@ runtime contract hot-swap. More detail: [Web UI](docs/web-ui.md). | [Compatibility](docs/compatibility.md) | Evidence model and release matrix | Runnable cookbooks live under [`examples/`](examples/README.md). +Pulumi integration suite (contract surface majors 6–9 + lifecycle, HTML report): +[`pulumi-tests/`](pulumi-tests/README.md) (`make pulumi-tests`). ## proxmoxer (HTTPS gateway) @@ -128,7 +154,7 @@ from proxmoxer import ProxmoxAPI proxmox = ProxmoxAPI( "localhost", - port=8007, + port=8006, user="root@pam", password="secret", verify_ssl=False, # local self-signed development certificate only @@ -169,6 +195,12 @@ make release-build # build/tag only, no push make release-up && make release-seed # run the published stack locally ``` +## Contributing / security / changelog + +- [CONTRIBUTING.md](CONTRIBUTING.md) · [CONTRIBUTING.ru.md](CONTRIBUTING.ru.md) +- [SECURITY.md](SECURITY.md) +- [CHANGELOG.md](CHANGELOG.md) + ## What this is not - Not a hypervisor: no KVM/LXC execution on bare metal or nested hosts. diff --git a/README.ru.md b/README.ru.md new file mode 100644 index 0000000..c8d2b1b --- /dev/null +++ b/README.ru.md @@ -0,0 +1,210 @@ +**Language / Язык:** [English](README.md) | [Русский](README.ru.md) + +# proxmox-api-simulator + +[![CI](https://github.com/sergeyantropoff/proxmox-api-simulator/actions/workflows/ci.yml/badge.svg)](https://github.com/sergeyantropoff/proxmox-api-simulator/actions/workflows/ci.yml) +[![Docker Image](https://img.shields.io/docker/v/inecs/proxmox-api-simulator?label=docker%20hub&sort=semver)](https://hub.docker.com/r/inecs/proxmox-api-simulator) +[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE) + +Stateful-асинхронный симулятор API [Proxmox VE](https://www.proxmox.com/) для +тестирования API-клиентов и инфраструктурных инструментов без реального +гипервизорного кластера. + +> **Только лаборатория / CI.** Учётные данные, signing keys и открытые UI/admin +> helpers по умолчанию — намеренные лабораторные значения. **Не** выставляйте +> стек в публичный Интернет без замены секретов и своих сетевых ограничений. +> См. [SECURITY.md](SECURITY.md) и [Безопасность](docs/ru/security.md). + +Симулятор работает на PostgreSQL, опирается на импортированные официальные +контракты API и предоставляет те же поверхности `/api2/json` и `/api2/extjs`, +что и Proxmox VE. Семантические обработчики сохраняют мутации; длительные +операции возвращают устойчивые UPID, которые выполняют воркеры с арендой задач. + +## Проверенное покрытие API + +Реестр обработчиков и верифицированные ledger поверхности — **100%** для каждого +включённого major: + +| Контракт | Declared | Implemented | Verified | +|---|---:|---:|---:| +| PVE 6.4-15 | 504 | 504 | 504 | +| PVE 7.4-16 | 540 | 540 | 540 | +| PVE 8.4.5 | 605 | 605 | 605 | +| PVE 9.2.3 | 675 | 675 | 675 | + +Переключение активного контракта в runtime — из Web UI (**Apply as runtime**) или +`POST /ui/api/contract/apply?major=N` — каждый Apply загружает +`evidence/pve-{version}.json`, чтобы observed/verified следовали выбранному +major. После импорта нового контракта перегенерируйте ledger: `make evidence`. +Живые отчёты: `/admin/compatibility` (также `.md` / `.html`). См. +[Совместимость](docs/ru/compatibility.md) и [Версии API](docs/ru/api-versions.md). + +> Это измеримое покрытие контракта и обработчиков лабораторного симулятора — +> не утверждение, что каждый краевой случай Proxmox или удалённая интеграция +> ведёт себя идентично продакшен-железу. + +## Быстрый старт (опубликованный образ) + +Образ: [`inecs/proxmox-api-simulator`](https://hub.docker.com/r/inecs/proxmox-api-simulator) + +### Docker Compose + +Нужен checkout с `docker-compose.release.yml`. Не публикуйте хост `:8006` за +пределы доверенной лаборатории без ротации `TICKET_SIGNING_KEY` / пароля БД. + +```bash +docker compose -f docker-compose.release.yml up -d +docker compose -f docker-compose.release.yml run --rm --entrypoint python \ + simulator -m app.simulation.seed_cli + +curl -sS http://localhost:8006/health/ready +curl -sS http://localhost:8006/api2/json/version +``` + +Или: `make release-up && make release-seed PROFILE=small` + +### Helm (Kubernetes + Ingress + Let's Encrypt) + +```bash +helm upgrade --install pve-sim ./helm/proxmox-api-simulator \ + -n proxmox-sim --create-namespace \ + -f ./helm/proxmox-api-simulator/values-ingress-example.yaml \ + --set certManager.email=you@example.com \ + --set ingress.hosts[0].host=pve-sim.example.com \ + --set ingress.tls[0].hosts[0]=pve-sim.example.com \ + --set secret.ticketSigningKey="$(openssl rand -hex 32)" \ + --set postgresql.auth.password="$(openssl rand -hex 16)" +``` + +Нужны Ingress-контроллер и cert-manager. Service чарта говорит по **HTTP** +`:8006`; TLS — на Ingress. Compose тоже отдаёт plain HTTP на `:8006`; опциональный +HTTPS для proxmoxer — `docker compose --profile tls` на хосте `:8443`. +Подробности: [Kubernetes / Helm](docs/ru/kubernetes.md). + +- HTTP API и Web UI (Compose): [http://localhost:8006/](http://localhost:8006/) +- Схема FastAPI: [http://localhost:8006/docs](http://localhost:8006/docs) +- Админ по умолчанию после seed: `root@pam` / `secret` + +## Быстрый старт (разработка из репозитория) + +Сборка и запуск development-стека с bind-mount из этого репозитория: + +```bash +make install +make up +make seed PROFILE=small + +curl -sS http://localhost:8006/health/ready +curl -sS http://localhost:8006/api2/json/version +curl -sS -X POST -d 'username=root@pam&password=secret' \ + http://localhost:8006/api2/json/access/ticket +``` + +- HTTP API и Web UI: [http://localhost:8006/](http://localhost:8006/) + (реальный PVE — **HTTPS** на `:8006`; лаборатория — plain HTTP на том же + порту. Опциональный TLS для proxmoxer: `docker compose --profile tls` → + `https://localhost:8443/`) +- Карта портов и TLS: [Порты и TLS](docs/ru/configuration.md#порты-и-tls). +- Схема FastAPI: [http://localhost:8006/docs](http://localhost:8006/docs) + +### Web UI + +Интерактивная консоль со светлой/тёмной темой, каталогом эндпоинтов PVE 6–9, +горячей сменой runtime-контракта и монитором задач UPID. Подробнее: +[Web UI](docs/ru/web-ui.md). + +![Web UI, светлая тема](docs/images/web-ui-light.png) + +![Web UI, тёмная тема](docs/images/web-ui-dark.png) + +## Документация + +Документация двуязычная. Переключатель **Language / Язык** — в шапке каждой +страницы; английский корень — [README.md](README.md). Индекс: +[docs/README.md](docs/README.md) · [docs/ru/README.md](docs/ru/README.md). + +| Руководство | Описание | +|---|---| +| [Начало работы](docs/ru/getting-started.md) | Первая успешная лабораторная сессия | +| [Конфигурация](docs/ru/configuration.md) | Переменные окружения и Compose | +| [Аутентификация](docs/ru/authentication.md) | Тикеты, CSRF, API-токены, ACL | +| [Версии API](docs/ru/api-versions.md) | Контракты 6–9 и hot-swap | +| [Клиенты и примеры](docs/ru/clients.md) | Python, Go, Java, Perl, Ansible, Terraform, Pulumi | +| [Профили seed](docs/ru/seed-profiles.md) | Детерминированные фикстуры кластера | +| [Поверхность API](docs/ru/api-surface.md) | Маршрутизация, обработчики, fallback | +| [Домены](docs/ru/domains/README.md) | QEMU, LXC, storage, HA, SDN, … | +| [Web UI](docs/ru/web-ui.md) | Интерактивная консоль и каталоги | +| [Эксплуатация](docs/ru/operations.md) | Миграции, reseed, обновления | +| [Kubernetes / Helm](docs/ru/kubernetes.md) | Образ Hub + Ingress + Let's Encrypt | +| [Безопасность](docs/ru/security.md) | Модель угроз лаборатории и учётные данные | +| [Наблюдаемость](docs/ru/observability.md) | Health-эндпоинты и логирование | +| [Устранение неполадок](docs/ru/troubleshooting.md) | Типичные сбои | +| [FAQ](docs/ru/faq.md) | Краткие ответы | +| [Архитектура](docs/ru/architecture.md) | Границы компонентов | +| [Совместимость](docs/ru/compatibility.md) | Модель evidence и матрица релиза | + +Индекс гайдов: [`docs/ru/README.md`](docs/ru/README.md). +Запускаемые cookbook: [`examples/`](examples/README.ru.md). +Интеграционный набор Pulumi (surface majors 6–9 + lifecycle, HTML-отчёт): +[`pulumi-tests/`](pulumi-tests/README.ru.md) (`make pulumi-tests`). + +## proxmoxer (HTTPS-шлюз) + +```python +from proxmoxer import ProxmoxAPI + +proxmox = ProxmoxAPI( + "localhost", + port=8006, + user="root@pam", + password="secret", + verify_ssl=False, # только локальный self-signed сертификат разработки +) +print(proxmox.version.get()) +print(proxmox.nodes("pve01").qemu.get()) +``` + +Пример API-токена: пользователь `root@pam`, `token_name="automation"`, +`token_value="automation-secret"`. Запросам с токеном CSRF не нужен; мутациям +по тикету — нужен. + +## Частые цели Make + +```bash +make up / make down / make logs / make dev +make test # unit + contract (включая verified surface) +make test-integration # с PostgreSQL +make test-surface # все глаголы × majors 6-9 (0x501 / 0xexception) +make test-compatibility # proxmoxer против Compose +make evidence # перегенерация evidence/pve-*.json +make seed PROFILE=small +make db-migrate +make shell +make ci # ruff + mypy + offline pytest + surface probe +make release # сборка + push runtime-образа в Docker Hub +make release-up # pull/start docker-compose.release.yml +make release-seed PROFILE=small +``` + +Релиз в Docker Hub (нужен `docker login` владельца Hub; см. +[Эксплуатация](docs/ru/operations.md)): + +```bash +make release # inecs/proxmox-api-simulator:<версия pyproject> + :latest +make release VERSION=0.2.0 # переопределить тег +make release-build # только build/tag, без push +make release-up && make release-seed # запустить опубликованный стек локально +``` + +## Участие / безопасность / changelog + +- [CONTRIBUTING.md](CONTRIBUTING.md) · [CONTRIBUTING.ru.md](CONTRIBUTING.ru.md) +- [SECURITY.md](SECURITY.md) +- [CHANGELOG.md](CHANGELOG.md) + +## Чем это не является + +- Не гипервизор: нет выполнения KVM/LXC на железе или nested-хостах. +- Не drop-in замена мультиарендного продакшен-Proxmox. +- Удалённые IdP / LDAP / live Ceph / live ACME эндпоинты симулируются локально; + к реальным внешним системам они не обращаются. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..9dfcb81 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,52 @@ +**Language / Язык:** [English](SECURITY.md) | [Русский](docs/ru/security.md) + +# Security policy + +## Supported versions + +| Version | Supported | +|---|---| +| `0.1.x` (latest) | Yes — security reports accepted | +| older / untagged | Best-effort only | + +## Threat model (read this first) + +This repository is a **local / CI laboratory simulator**, not a hardened +multi-tenant public Proxmox deployment. + +Default Compose and Helm values intentionally ship convenient lab secrets, +seeded passwords, open Web UI helper routes (`/ui/api/*`), and compatibility +endpoints (`/admin/compatibility*`). Treat network reachability as the trust +boundary. + +**Do not** expose host port `8006` (or a public Ingress) to untrusted networks +without replacing every default secret and adding controls you own. + +Full lab notes: [docs/security.md](docs/security.md) · +[docs/ru/security.md](docs/ru/security.md). + +## Reporting a vulnerability + +Please **do not** open a public GitHub issue for sensitive reports. + +Email the maintainer privately (account that owns the GitHub repository / +Docker Hub `inecs` namespace), or use GitHub +[ privately reported vulnerabilities](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) +if enabled on the repository. + +Include: + +- Affected version / image tag (`inecs/proxmox-api-simulator:…`) +- Reproduction steps against a **local** lab (not third-party instances) +- Impact assessment (auth bypass, secret leak, RCE, etc.) + +You should receive an acknowledgement within a few business days. + +## Lab secrets that must be rotated outside toy labs + +| Secret | Where | +|---|---| +| `TICKET_SIGNING_KEY` | Compose / Helm | +| PostgreSQL password | Compose / Helm | +| Seeded `root@pam` / API tokens | After `seed` | +| `docker/tls/server.key` | Checked-in self-signed material — never reuse outside local Compose | diff --git a/app/api/registry.py b/app/api/registry.py index 862f0a6..89aeadf 100644 --- a/app/api/registry.py +++ b/app/api/registry.py @@ -17,7 +17,7 @@ from app.api.errors import ApiError, ContractValidationError from app.api.openapi import contract_openapi_tags from app.config import Settings from app.contracts.examples import schema_example -from app.contracts.model import Method, Schema, Snapshot +from app.contracts.model import Method, Parameter, Schema, Snapshot from app.db.pool import AsyncpgDatabase from app.security.acl import AclEntry, CapabilityRequirement, authorize, requirement_from_contract from app.security.auth import parse_api_token, verify_csrf, verify_secret, verify_ticket @@ -137,6 +137,64 @@ def register_legacy_handler_routes( return seen +_PATH_PARAM_RE = re.compile(r"\{([^{}]+)\}") + + +def _unbound_method(path: str, verb: str) -> Method: + """Minimal contract method so handler-only paths can be mounted.""" + + parameters = tuple( + Parameter(name=name, definition=Schema(type="string", optional=True)) + for name in _PATH_PARAM_RE.findall(path) + ) + return Method( + verb=verb.upper(), + name=f"unbound-{verb.lower()}-{path}", + description="Handler-backed route not present in the active contract snapshot", + parameters=parameters, + returns=Schema(), + permissions=None, + checksum="0" * 64, + ) + + +def register_unbound_handler_routes( + app: FastAPI, + handlers: HandlerRegistry, + fallback: FallbackMode = "error", + *, + existing: set[tuple[str, str, str]] | None = None, +) -> set[tuple[str, str, str]]: + """Mount semantic handlers that no cached contract currently declares. + + Keeps DB-backed handlers reachable (no FastAPI 404) when Proxmox schema + omits a path that the simulator still implements. + """ + + seen = existing if existing is not None else set() + for path, verb in sorted(handlers.keys()): + for renderer in ("json", "extjs"): + route = f"/api2/{renderer}{path}" + key = (route, verb, renderer) + if key in seen: + continue + method = _unbound_method(path, verb) + seen.add(key) + app.add_api_route( + route, + _endpoint(path, method, renderer, handlers, fallback), + methods=[verb], + name=f"unbound:{renderer}:{verb}:{path}", + tags=cast(list[str | Enum], contract_openapi_tags(path, renderer)), + openapi_extra={ + "x-proxmox-method-checksum": method.checksum, + "x-proxmox-implementation": "implemented", + "x-proxmox-unbound-handler": True, + }, + ) + return seen + + def _endpoint( semantic_path: str, method: Method, @@ -302,8 +360,14 @@ async def _parse_inputs(request: Request, method: Method) -> dict[str, Any]: continue if name not in supplied: if definition.optional: + # Proxmox schema ``default`` is often UI documentation text (e.g. + # bwlimit default = "restore limit from datacenter…"), not a value + # to materialize. Only inject defaults that coerce to the type. if definition.default is not None: - parsed[name] = definition.default + try: + parsed[name] = _coerce(definition.default, definition) + except (TypeError, ValueError): + pass continue errors[name] = "property is missing and it is not optional" continue diff --git a/app/contracts/runtime.py b/app/contracts/runtime.py index faba4e8..97e50d3 100644 --- a/app/contracts/runtime.py +++ b/app/contracts/runtime.py @@ -14,6 +14,7 @@ from app.api.registry import ( HandlerRegistry, register_contract_routes, register_legacy_handler_routes, + register_unbound_handler_routes, ) from app.compatibility import ( CompatibilityDimension, @@ -106,7 +107,7 @@ def apply_runtime_contract( clear_contract_routes(app) registered = register_contract_routes(app, snapshot, handlers, fallback) - register_legacy_handler_routes( + registered = register_legacy_handler_routes( app, handlers, store_root, @@ -114,6 +115,7 @@ def apply_runtime_contract( primary_version=snapshot.source_version, existing=registered, ) + register_unbound_handler_routes(app, handlers, fallback, existing=registered) report = build_compatibility_for_snapshot( snapshot, handlers, diff --git a/app/handlers/acme.py b/app/handlers/acme.py index 01d944d..4074390 100644 --- a/app/handlers/acme.py +++ b/app/handlers/acme.py @@ -10,41 +10,34 @@ from app.api.errors import ApiError from app.api.registry import HandlerRegistry from app.handlers.common import cluster_metadata, save_cluster_metadata, subdirs, values -_DEFAULT_DIRECTORIES = [ - { - "name": "Let's Encrypt V2", - "url": "https://acme-v02.api.letsencrypt.org/directory", - }, - { - "name": "Let's Encrypt V2 Staging", - "url": "https://acme-staging-v02.api.letsencrypt.org/directory", - }, -] - -_CHALLENGE_SCHEMA = [ - { - "id": "dns", - "name": "DNS plugin", - "type": "dns", - "fields": [{"name": "api", "type": "string"}], - } -] - def _acme(metadata: dict[str, Any]) -> dict[str, Any]: - current = metadata.setdefault( - "acme", - {"accounts": {}, "plugins": {}, "meta": {}}, - ) + current = metadata.get("acme") if not isinstance(current, dict): - current = {"accounts": {}, "plugins": {}, "meta": {}} + current = {} metadata["acme"] = current - current.setdefault("accounts", {}) - current.setdefault("plugins", {}) - current.setdefault("meta", {}) + if not isinstance(current.get("accounts"), dict): + current["accounts"] = {} + if not isinstance(current.get("plugins"), dict): + current["plugins"] = {} + if not isinstance(current.get("meta"), dict): + current["meta"] = {} + if not isinstance(current.get("directories"), list): + current["directories"] = [] + if not isinstance(current.get("challenge_schema"), list): + current["challenge_schema"] = [] return current +def _default_directory(acme: dict[str, Any]) -> str: + directories = acme.get("directories") + if isinstance(directories, list): + for item in directories: + if isinstance(item, dict) and item.get("url"): + return str(item["url"]) + return "" + + def register_acme_handlers(registry: HandlerRegistry) -> None: async def index(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, str]]: return subdirs( @@ -72,16 +65,17 @@ def register_acme_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) name = str(payload.get("name") or "default") metadata = await cluster_metadata(request) - accounts = _acme(metadata)["accounts"] + acme = _acme(metadata) + accounts = acme["accounts"] if name in accounts: raise ApiError(400, f"ACME account '{name}' already exists") + directory = str(payload.get("directory") or _default_directory(acme)) accounts[name] = { "name": name, "contact": payload.get("contact"), - "directory": payload.get("directory") or _DEFAULT_DIRECTORIES[0]["url"], + "directory": directory, "tos_url": payload.get("tos_url"), "eab-kid": payload.get("eab-kid"), - # eab-hmac-key stored but never returned "eab-hmac-key": payload.get("eab-hmac-key"), "location": f"https://acme.example.local/acct/{name}", } @@ -181,29 +175,33 @@ def register_acme_handlers(registry: HandlerRegistry) -> None: del plugins[plugin_id] await save_cluster_metadata(request, metadata) - async def directories(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: - return list(_DEFAULT_DIRECTORIES) + async def directories(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: + metadata = await cluster_metadata(request) + directories = _acme(metadata).get("directories") + if isinstance(directories, list): + return [dict(item) for item in directories if isinstance(item, dict)] + return [] - async def challenge_schema(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: - return list(_CHALLENGE_SCHEMA) + async def challenge_schema(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: + metadata = await cluster_metadata(request) + schema = _acme(metadata).get("challenge_schema") + if isinstance(schema, list): + return [dict(item) for item in schema if isinstance(item, dict)] + return [] async def meta(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: - directory = str(values(inputs).get("directory") or _DEFAULT_DIRECTORIES[0]["url"]) metadata = await cluster_metadata(request) - meta_store = _acme(metadata).setdefault("meta", {}) - payload = meta_store.setdefault( - directory, - { - "termsOfService": f"{directory.rstrip('/')}/tos", - "caaIdentities": ["letsencrypt.org"], - }, - ) - await save_cluster_metadata(request, metadata) + acme = _acme(metadata) + directory = str(values(inputs).get("directory") or _default_directory(acme)) + meta_raw = acme.get("meta") + meta_store: dict[str, Any] = dict(meta_raw) if isinstance(meta_raw, dict) else {} + payload = meta_store.get(directory) + if not isinstance(payload, dict): + return {} return dict(payload) async def tos(request: Request, inputs: dict[str, Any]) -> str: - directory = str(values(inputs).get("directory") or _DEFAULT_DIRECTORIES[0]["url"]) - result = await meta(request, {"values": {"directory": directory}, "provided": frozenset()}) + result = await meta(request, inputs) return str(result.get("termsOfService") or "") registry.register("/cluster/acme", "GET", index) diff --git a/app/handlers/backup.py b/app/handlers/backup.py index 46d3bbd..3d23ace 100644 --- a/app/handlers/backup.py +++ b/app/handlers/backup.py @@ -154,26 +154,31 @@ def register_backup_handlers(registry: HandlerRegistry) -> None: vmid = row["vmid"] return [f"qemu/{vmid}"] if vmid is not None else [str(row["volume_id"])] - async def vzdump_defaults(_request: Request, inputs: dict[str, Any]) -> dict[str, Any]: - await require_node(_request, str(values(inputs)["node"])) - return { - "all": 0, - "bwlimit": 0, - "compress": "zstd", - "dumpdir": "backup", - "mode": "snapshot", - "remove": 0, - "storage": "nfs-backup", - "mailto": "", - "notes-template": "{{guestname}}", - } + async def vzdump_defaults(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: + from app.handlers.nodes import load_node_ops + + node = str(values(inputs)["node"]) + await require_node(request, node) + ops = await load_node_ops(request, node) + vzdump = ops.get("vzdump") + defaults = vzdump.get("defaults") if isinstance(vzdump, dict) else None + return dict(defaults) if isinstance(defaults, dict) else {} async def vzdump_extractconfig(request: Request, inputs: dict[str, Any]) -> str: + from app.handlers.nodes import load_node_ops + payload = values(inputs) + node = str(payload["node"]) volid = str(payload.get("volume") or payload.get("volid") or "") if not volid: raise ApiError(400, "volume parameter required") - return f"# simulated vzdump config extracted from {volid}\name: demo\nmemory: 2048\n" + await require_node(request, node) + ops = await load_node_ops(request, node) + vzdump = ops.get("vzdump") + configs = vzdump.get("extractconfig") if isinstance(vzdump, dict) else None + if not isinstance(configs, dict): + return "" + return str(configs.get(volid) or "") async def vzdump_create(request: Request, inputs: dict[str, Any]) -> str: payload = values(inputs) diff --git a/app/handlers/ceph.py b/app/handlers/ceph.py index d35b7a5..1435c4d 100644 --- a/app/handlers/ceph.py +++ b/app/handlers/ceph.py @@ -4,7 +4,7 @@ from __future__ import annotations import json import secrets -from typing import Any, cast +from typing import Any from fastapi import Request @@ -21,37 +21,6 @@ from app.handlers.common import ( ) from app.simulation.seed import CLUSTER_ID -DEFAULT_CLUSTER_CEPH = { - "initialized": True, - "config": { - "network": "10.10.10.0/24", - "cluster-network": "10.10.10.0/24", - "size": 3, - "min_size": 2, - "pg_bits": 7, - }, - "cfg_db": [ - {"section": "global", "name": "auth_client_required", "value": "cephx"}, - {"section": "global", "name": "fsid", "value": "pve-simulator-fsid"}, - ], - "cfg_raw": "[global]\nfsid = pve-simulator-fsid\nauth_client_required = cephx\n", - "cfg_values": {}, - "pools": { - "rbd": { - "pool": "rbd", - "size": 3, - "min_size": 2, - "pg_num": 128, - "application": "rbd", - "crush_rule": "replicated_rule", - } - }, - "fs": {}, - "rules": [{"name": "replicated_rule", "id": 0}], - "crush": "device 0 osd.0 class hdd\n", - "running": True, -} - async def _load_cluster_ceph(request: Request) -> dict[str, Any]: row = await database(request).pool.fetchrow( @@ -60,14 +29,9 @@ async def _load_cluster_ceph(request: Request) -> dict[str, Any]: ) metadata = state(row["metadata"]) if row is not None else {} ceph = metadata.get("ceph") - if not isinstance(ceph, dict) or not ceph: - return dict(DEFAULT_CLUSTER_CEPH) - merged = dict(DEFAULT_CLUSTER_CEPH) - merged.update(ceph) - for key in ("config", "pools", "fs", "cfg_values"): - if not isinstance(merged.get(key), dict): - merged[key] = dict(cast(dict[str, Any], DEFAULT_CLUSTER_CEPH[key])) - return merged + if not isinstance(ceph, dict): + return {} + return dict(ceph) async def _save_cluster_ceph(request: Request, ceph: dict[str, Any]) -> None: @@ -82,34 +46,27 @@ async def _save_cluster_ceph(request: Request, ceph: dict[str, Any]) -> None: async def _load_node_ceph(request: Request, node: str) -> dict[str, Any]: metadata = await node_metadata(request, node) - ops = metadata.setdefault("ops", {}) - ceph = ops.setdefault( - "ceph", - { - "mds": {}, - "mgr": {}, - "mon": {f"{node}": {"name": node, "addr": f"{node}.local:6789", "rank": 0}}, - "log": [{"t": 1_700_000_000, "n": 0, "line": "ceph simulator ready"}], - }, - ) + ops = metadata.get("ops") + if not isinstance(ops, dict): + return {"mds": {}, "mgr": {}, "mon": {}, "log": []} + ceph = ops.get("ceph") if not isinstance(ceph, dict): - ceph = { - "mds": {}, - "mgr": {}, - "mon": {}, - "log": [], - } - ops["ceph"] = ceph - ceph.setdefault("mds", {}) - ceph.setdefault("mgr", {}) - ceph.setdefault("mon", {}) - ceph.setdefault("log", []) - return ceph + return {"mds": {}, "mgr": {}, "mon": {}, "log": []} + return { + "mds": ceph.get("mds") if isinstance(ceph.get("mds"), dict) else {}, + "mgr": ceph.get("mgr") if isinstance(ceph.get("mgr"), dict) else {}, + "mon": ceph.get("mon") if isinstance(ceph.get("mon"), dict) else {}, + "log": list(ceph.get("log") or []) if isinstance(ceph.get("log"), list) else [], + **{key: value for key, value in ceph.items() if key not in {"mds", "mgr", "mon", "log"}}, + } async def _save_node_ceph(request: Request, node: str, ceph: dict[str, Any]) -> None: metadata = await node_metadata(request, node) - ops = metadata.setdefault("ops", {}) + ops = metadata.get("ops") + if not isinstance(ops, dict): + ops = {} + metadata["ops"] = ops ops["ceph"] = ceph await save_node_metadata(request, node, metadata) @@ -175,10 +132,12 @@ def register_ceph_handlers(registry: HandlerRegistry) -> None: await require_node(request, str(payload["node"])) ceph = await _load_cluster_ceph(request) keys = [item.strip() for item in str(payload.get("config-keys") or "").split(",") if item] - stored = ceph.setdefault("cfg_values", {}) - result = {key: stored.get(key, "") for key in keys} if keys else dict(stored) - await _save_cluster_ceph(request, ceph) - return result + stored = ceph.get("cfg_values") + if not isinstance(stored, dict): + stored = {} + if keys: + return {key: stored.get(key, "") for key in keys} + return dict(stored) async def crush(request: Request, inputs: dict[str, Any]) -> str: await require_node(request, str(values(inputs)["node"])) @@ -202,12 +161,13 @@ def register_ceph_handlers(registry: HandlerRegistry) -> None: async def cmd_safety(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: payload = values(inputs) await require_node(request, str(payload["node"])) - return { - "safe": 1, - "action": payload.get("action"), - "service": payload.get("service"), - "id": payload.get("id"), - } + ceph = await _load_cluster_ceph(request) + safety = ceph.get("cmd_safety") + result = dict(safety) if isinstance(safety, dict) else {} + for key in ("action", "service", "id"): + if key in payload: + result[key] = payload[key] + return result async def init(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) @@ -368,13 +328,7 @@ def register_ceph_handlers(registry: HandlerRegistry) -> None: pool = (ceph.get("pools") or {}).get(name) if not isinstance(pool, dict): raise ApiError(404, "pool does not exist") - return { - **pool, - "pg_num": pool.get("pg_num", 128), - "bytes_used": 0, - "percent_used": 0.0, - "healthy": True, - } + return dict(pool) async def fs_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: await require_node(request, str(values(inputs)["node"])) @@ -682,13 +636,16 @@ def register_ceph_handlers(registry: HandlerRegistry) -> None: await require_node(request, node) row = await _osd_row(request, node, osdid) current = state(row["state"]) + metadata = current.get("metadata") + if isinstance(metadata, dict): + return dict(metadata) return { "osd": { "id": int(osdid) if osdid.isdigit() else osdid, - "uuid": current.get("uuid") or f"osd-uuid-{osdid}", - "device_class": current.get("device_class", "hdd"), + "uuid": current.get("uuid", ""), + "device_class": current.get("device_class", ""), }, - "devices": [{"dev": current.get("dev") or f"/dev/sd{osdid}"}], + "devices": [{"dev": current.get("dev", "")}] if current.get("dev") else [], } async def cluster_ceph_status(_request: Request, _inputs: dict[str, Any]) -> dict[str, Any]: @@ -698,17 +655,35 @@ def register_ceph_handlers(registry: HandlerRegistry) -> None: ) total = int(row["capacity_bytes"] or 0) if row is not None else 0 used = int(row["used_bytes"] or 0) if row is not None else 0 - osd_count = await database(_request).pool.fetchval( - "SELECT count(*)::int FROM resources WHERE kind='ceph-osd'" + osd_rows = await database(_request).pool.fetch( + "SELECT state FROM resources WHERE kind='ceph-osd'" ) + num_osds = len(osd_rows) + num_up = 0 + num_in = 0 + for osd_row in osd_rows: + osd_state = state(osd_row["state"]) + if str(osd_state.get("status") or "") == "up": + num_up += 1 + if osd_state.get("in") in (True, 1, "1"): + num_in += 1 ceph = await _load_cluster_ceph(_request) + version = ceph.get("version") + version_str = ( + str(version.get("str")) + if isinstance(version, dict) and version.get("str") is not None + else str(version or "") + ) + health = ceph.get("health") + if not isinstance(health, dict): + health = {"status": "HEALTH_OK" if ceph.get("running") else "HEALTH_WARN"} return { - "version": "17.2.7", - "health": {"status": "HEALTH_OK" if ceph.get("running", True) else "HEALTH_WARN"}, + "version": version_str, + "health": health, "osdmap": { - "num_osds": osd_count, - "num_up_osds": osd_count - 1, - "num_in_osds": osd_count - 1, + "num_osds": num_osds, + "num_up_osds": num_up, + "num_in_osds": num_in, }, "pgmap": {"bytes_used": used, "bytes_total": total}, "fsmap": {"filesystems": list((ceph.get("fs") or {}).keys())}, diff --git a/app/handlers/cluster.py b/app/handlers/cluster.py index 8128416..0bc4d1f 100644 --- a/app/handlers/cluster.py +++ b/app/handlers/cluster.py @@ -46,24 +46,31 @@ def register_cluster_handlers(registry: HandlerRegistry) -> None: "tasks", ) - async def cluster_status(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: - rows = await database(_request).pool.fetch( + async def cluster_status(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: + from app.handlers.nodes import load_node_ops + + rows = await database(request).pool.fetch( """SELECT id, name, status FROM nodes ORDER BY name""" ) + metadata = await cluster_metadata(request) + quorate = metadata.get("quorate") result: list[dict[str, Any]] = [] for index, row in enumerate(rows): online = str(row["status"]) == "online" + ops = await load_node_ops(request, str(row["name"])) + cluster_node = ops.get("cluster_status") + entry = dict(cluster_node) if isinstance(cluster_node, dict) else {} result.append( { "id": str(row["id"]), "name": str(row["name"]), - "nodeid": index, + "nodeid": entry.get("nodeid", index), "online": 1 if online else 0, - "local": 1 if index == 0 else 0, - "ip": f"10.32.{index // 254 + 1}.{index % 254 + 10}", - "level": "c", - "type": "node", - "quorate": 1, + "local": entry.get("local", 1 if index == 0 else 0), + "ip": entry.get("ip") or ops.get("ip") or "", + "level": entry.get("level", "c"), + "type": entry.get("type", "node"), + "quorate": entry.get("quorate", quorate if quorate is not None else 1), } ) return result @@ -95,14 +102,8 @@ def register_cluster_handlers(registry: HandlerRegistry) -> None: metadata = state(row["metadata"]) if row is not None else {} options = metadata.get("options", {}) if not isinstance(options, dict): - options = {} - return { - "keyboard": options.get("keyboard", "en-us"), - "email_from": options.get("email_from", "root@localhost"), - "http_proxy": options.get("http_proxy", ""), - "description": options.get("description", "Proxmox API emulator cluster"), - **options, - } + return {} + return dict(options) async def cluster_options_put(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: current = await cluster_options_get(request, inputs) diff --git a/app/handlers/cluster_config.py b/app/handlers/cluster_config.py index 6a31823..233ac6d 100644 --- a/app/handlers/cluster_config.py +++ b/app/handlers/cluster_config.py @@ -19,30 +19,8 @@ from app.handlers.common import ( def _config(metadata: dict[str, Any]) -> dict[str, Any]: - current = metadata.setdefault( - "cluster_config", - { - "clustername": "pve-simulator", - "votes": 1, - "links": {}, - "join_info": {}, - "totem": {"version": 2, "secauth": "on", "cluster_name": "pve-simulator"}, - "qdevice": {"status": "disabled"}, - "apiversion": 1, - }, - ) - if not isinstance(current, dict): - current = { - "clustername": "pve-simulator", - "votes": 1, - "links": {}, - "join_info": {}, - "totem": {"version": 2, "secauth": "on", "cluster_name": "pve-simulator"}, - "qdevice": {"status": "disabled"}, - "apiversion": 1, - } - metadata["cluster_config"] = current - return current + current = metadata.get("cluster_config") + return current if isinstance(current, dict) else {} def register_cluster_config_handlers(registry: HandlerRegistry) -> None: @@ -52,10 +30,12 @@ def register_cluster_config_handlers(registry: HandlerRegistry) -> None: async def create(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) metadata = await cluster_metadata(request) - config = _config(metadata) + config = dict(_config(metadata)) if payload.get("clustername"): config["clustername"] = str(payload["clustername"]) - config.setdefault("totem", {})["cluster_name"] = str(payload["clustername"]) + totem = dict(config.get("totem") or {}) + totem["cluster_name"] = str(payload["clustername"]) + config["totem"] = totem if "votes" in payload: config["votes"] = payload["votes"] if "nodeid" in payload: @@ -64,6 +44,7 @@ def register_cluster_config_handlers(registry: HandlerRegistry) -> None: if links: config["links"] = links config["token"] = secrets.token_hex(16) + metadata["cluster_config"] = config await save_cluster_metadata(request, metadata) await database(request).pool.execute( """UPDATE clusters @@ -97,11 +78,11 @@ def register_cluster_config_handlers(registry: HandlerRegistry) -> None: async def join_post(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) metadata = await cluster_metadata(request) - config = _config(metadata) + config = dict(_config(metadata)) hostname = str(payload.get("hostname") or payload.get("node") or "") if not hostname: raise ApiError(400, "parameter verification failed - 'hostname' missing") - joins = config.setdefault("join_info", {}) + joins = dict(config.get("join_info") or {}) joins[hostname] = { "hostname": hostname, "fingerprint": payload.get("fingerprint"), @@ -112,6 +93,8 @@ def register_cluster_config_handlers(registry: HandlerRegistry) -> None: # password accepted but not stored in clear form if payload.get("password"): joins[hostname]["password_set"] = True + config["join_info"] = joins + metadata["cluster_config"] = config exists = await database(request).pool.fetchval( "SELECT EXISTS(SELECT 1 FROM nodes WHERE name=$1)", hostname, @@ -128,14 +111,19 @@ def register_cluster_config_handlers(registry: HandlerRegistry) -> None: rows = await database(request).pool.fetch("SELECT name, status FROM nodes ORDER BY name") metadata = await cluster_metadata(request) config = _config(metadata) + added_raw = config.get("added_nodes") + added: dict[str, Any] = dict(added_raw) if isinstance(added_raw, dict) else {} result = [] for index, row in enumerate(rows, start=1): + name = str(row["name"]) + entry_raw = added.get(name) + entry: dict[str, Any] = dict(entry_raw) if isinstance(entry_raw, dict) else {} result.append( { - "node": str(row["name"]), - "nodeid": index, - "ring0_addr": f"{row['name']}.local", - "quorum_votes": config.get("votes", 1), + "node": name, + "nodeid": entry.get("nodeid", index), + "ring0_addr": entry.get("ring0_addr", ""), + "quorum_votes": entry.get("quorum_votes", config.get("votes", 0)), } ) return result @@ -144,8 +132,8 @@ def register_cluster_config_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) node = str(payload["node"]) metadata = await cluster_metadata(request) - config = _config(metadata) - added = config.setdefault("added_nodes", {}) + config = dict(_config(metadata)) + added = dict(config.get("added_nodes") or {}) added[node] = { "node": node, "nodeid": payload.get("nodeid"), @@ -153,10 +141,14 @@ def register_cluster_config_handlers(registry: HandlerRegistry) -> None: "votes": payload.get("votes", 1), "apiversion": payload.get("apiversion"), "force": payload.get("force"), + "ring0_addr": payload.get("ring0_addr") or f"{node}.local", + "quorum_votes": payload.get("quorum_votes", payload.get("votes", 1)), } links = {key: value for key, value in payload.items() if key.startswith("link")} if links: added[node]["links"] = links + config["added_nodes"] = added + metadata["cluster_config"] = config exists = await database(request).pool.fetchval( "SELECT EXISTS(SELECT 1 FROM nodes WHERE name=$1)", node, @@ -172,11 +164,14 @@ def register_cluster_config_handlers(registry: HandlerRegistry) -> None: async def nodes_delete(request: Request, inputs: dict[str, Any]) -> None: node = str(values(inputs)["node"]) metadata = await cluster_metadata(request) - config = _config(metadata) - added = config.setdefault("added_nodes", {}) + config = dict(_config(metadata)) + added = dict(config.get("added_nodes") or {}) added.pop(node, None) - joins = config.setdefault("join_info", {}) + joins = dict(config.get("join_info") or {}) joins.pop(node, None) + config["added_nodes"] = added + config["join_info"] = joins + metadata["cluster_config"] = config await save_cluster_metadata(request, metadata) # Keep node row; mark offline to avoid cascading guest deletes. await database(request).pool.execute( @@ -186,11 +181,13 @@ def register_cluster_config_handlers(registry: HandlerRegistry) -> None: async def qdevice(request: Request, _inputs: dict[str, Any]) -> dict[str, Any]: metadata = await cluster_metadata(request) - return dict(_config(metadata).get("qdevice") or {"status": "disabled"}) + qdevice = _config(metadata).get("qdevice") + return dict(qdevice) if isinstance(qdevice, dict) else {} async def totem(request: Request, _inputs: dict[str, Any]) -> dict[str, Any]: metadata = await cluster_metadata(request) - return dict(_config(metadata).get("totem") or {}) + totem = _config(metadata).get("totem") + return dict(totem) if isinstance(totem, dict) else {} registry.register("/cluster/config", "GET", index) registry.register("/cluster/config", "POST", create) diff --git a/app/handlers/cluster_extra.py b/app/handlers/cluster_extra.py index 34ec0e4..d5dfa46 100644 --- a/app/handlers/cluster_extra.py +++ b/app/handlers/cluster_extra.py @@ -2,7 +2,6 @@ from __future__ import annotations -import copy import secrets import time from typing import Any @@ -22,56 +21,20 @@ from app.handlers.common import ( from app.tasks.repository import TaskRepository from app.tasks.upid import Upid -DEFAULT_CEPH_FLAGS: dict[str, int] = { - "nobackfill": 0, - "nodeep-scrub": 0, - "nodown": 0, - "noin": 0, - "noout": 0, - "norebalance": 0, - "norecover": 0, - "noscrub": 0, - "notieragent": 0, - "pause": 0, -} - -DEFAULT_CPU_FLAGS: list[dict[str, Any]] = [ - {"name": "aes", "introduces": "Westmere"}, - {"name": "avx", "introduces": "SandyBridge"}, - {"name": "avx2", "introduces": "Haswell"}, -] - def _jobs(metadata: dict[str, Any]) -> dict[str, Any]: - jobs = metadata.setdefault("jobs", {}) - if not isinstance(jobs, dict): - jobs = {} - metadata["jobs"] = jobs - sync = jobs.setdefault("realm_sync", {}) - if not isinstance(sync, dict): - sync = {} - jobs["realm_sync"] = sync - return jobs + jobs = metadata.get("jobs") + return jobs if isinstance(jobs, dict) else {} def _metrics(metadata: dict[str, Any]) -> dict[str, Any]: - metrics = metadata.setdefault("metrics", {}) - if not isinstance(metrics, dict): - metrics = {} - metadata["metrics"] = metrics - servers = metrics.setdefault("servers", {}) - if not isinstance(servers, dict): - servers = {} - metrics["servers"] = servers - return metrics + metrics = metadata.get("metrics") + return metrics if isinstance(metrics, dict) else {} def _cpu_models(metadata: dict[str, Any]) -> dict[str, Any]: models = metadata.get("qemu_cpu_models") - if not isinstance(models, dict): - models = {} - metadata["qemu_cpu_models"] = models - return models + return models if isinstance(models, dict) else {} def _ha_rules_store(metadata: dict[str, Any]) -> list[dict[str, Any]]: @@ -84,12 +47,7 @@ def _ha_rules_store(metadata: dict[str, Any]) -> list[dict[str, Any]]: ] if isinstance(rules, list): return [dict(item) for item in rules if isinstance(item, dict)] - defaults = [ - {"rule": "node-fencing", "type": "node", "action": "restart"}, - {"rule": "service-ha", "type": "resource", "action": "failover"}, - ] - metadata["ha_rules"] = defaults - return list(defaults) + return [] def _save_ha_rules(metadata: dict[str, Any], rules: list[dict[str, Any]]) -> None: @@ -105,18 +63,7 @@ def _replication_jobs(metadata: dict[str, Any]) -> list[dict[str, Any]]: def _ceph(metadata: dict[str, Any]) -> dict[str, Any]: ceph = metadata.get("ceph") - if not isinstance(ceph, dict): - ceph = {} - flags = ceph.get("flags") - if not isinstance(flags, dict): - flags = copy.deepcopy(DEFAULT_CEPH_FLAGS) - else: - merged = copy.deepcopy(DEFAULT_CEPH_FLAGS) - merged.update({str(key): int(value) for key, value in flags.items()}) - flags = merged - ceph["flags"] = flags - metadata["ceph"] = ceph - return ceph + return ceph if isinstance(ceph, dict) else {} async def _cluster_task(request: Request, *, task_type: str, worker: str) -> str: @@ -174,8 +121,8 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) job_id = str(payload["id"]) metadata = await cluster_metadata(request) - jobs = _jobs(metadata) - sync = jobs.setdefault("realm_sync", {}) + jobs = dict(_jobs(metadata)) + sync = dict(jobs.get("realm_sync") or {}) if job_id in sync: raise ApiError(409, "realm-sync job already exists") entry = { @@ -185,6 +132,7 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: entry.setdefault("enabled", 1) entry.setdefault("realm", str(payload.get("realm") or "pam")) sync[job_id] = entry + jobs["realm_sync"] = sync metadata["jobs"] = jobs await save_cluster_metadata(request, metadata) return {"id": job_id, **entry} @@ -193,8 +141,8 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) job_id = str(payload["id"]) metadata = await cluster_metadata(request) - jobs = _jobs(metadata) - sync = jobs.setdefault("realm_sync", {}) + jobs = dict(_jobs(metadata)) + sync = dict(jobs.get("realm_sync") or {}) if job_id not in sync: raise ApiError(404, "realm-sync job does not exist") updated = { @@ -206,6 +154,7 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: }, } sync[job_id] = updated + jobs["realm_sync"] = sync metadata["jobs"] = jobs await save_cluster_metadata(request, metadata) return {"id": job_id, **updated} @@ -213,23 +162,26 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: async def realm_sync_delete(request: Request, inputs: dict[str, Any]) -> None: job_id = str(values(inputs)["id"]) metadata = await cluster_metadata(request) - jobs = _jobs(metadata) - sync = jobs.setdefault("realm_sync", {}) + jobs = dict(_jobs(metadata)) + sync = dict(jobs.get("realm_sync") or {}) if job_id not in sync: raise ApiError(404, "realm-sync job does not exist") del sync[job_id] + jobs["realm_sync"] = sync metadata["jobs"] = jobs await save_cluster_metadata(request, metadata) async def schedule_analyze(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: schedule = str(values(inputs).get("schedule") or "*/15") metadata = await cluster_metadata(request) - jobs = _jobs(metadata) + jobs = dict(_jobs(metadata)) + results = jobs.get("schedule_analyze_results") + if not isinstance(results, list): + results = [] jobs["last_schedule_analyze"] = {"schedule": schedule, "at": int(time.time())} metadata["jobs"] = jobs await save_cluster_metadata(request, metadata) - now = int(time.time()) - return [{"timestamp": now + offset * 900, "utc": True} for offset in range(4)] + return [dict(item) for item in results if isinstance(item, dict)] async def metrics_index(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, str]]: return subdirs("export", "server") @@ -238,8 +190,7 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: metadata = await cluster_metadata(request) metrics = _metrics(metadata) return { - "data": metrics.get("export_data") - or '# HELP pve_up Node is up\npve_up{node="pve01"} 1\n', + "data": str(metrics.get("export_data") or ""), "timestamp": int(time.time()), } @@ -248,9 +199,12 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: ) -> list[dict[str, Any]]: metadata = await cluster_metadata(request) metrics = _metrics(metadata) + servers = metrics.get("servers") + if not isinstance(servers, dict): + return [] return [ {"id": server_id, **dict(payload)} - for server_id, payload in sorted(metrics.get("servers", {}).items()) + for server_id, payload in sorted(servers.items()) if isinstance(payload, dict) ] @@ -267,8 +221,8 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) server_id = str(payload["id"]) metadata = await cluster_metadata(request) - metrics = _metrics(metadata) - servers = metrics.setdefault("servers", {}) + metrics = dict(_metrics(metadata)) + servers = dict(metrics.get("servers") or {}) if server_id in servers: raise ApiError(409, "metrics server already exists") entry = { @@ -279,6 +233,7 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: entry.setdefault("port", 8086) entry.setdefault("enable", 1) servers[server_id] = entry + metrics["servers"] = servers metadata["metrics"] = metrics await save_cluster_metadata(request, metadata) return {"id": server_id, **entry} @@ -287,8 +242,8 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) server_id = str(payload["id"]) metadata = await cluster_metadata(request) - metrics = _metrics(metadata) - servers = metrics.setdefault("servers", {}) + metrics = dict(_metrics(metadata)) + servers = dict(metrics.get("servers") or {}) if server_id not in servers: raise ApiError(404, "metrics server does not exist") updated = { @@ -300,6 +255,7 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: }, } servers[server_id] = updated + metrics["servers"] = servers metadata["metrics"] = metrics await save_cluster_metadata(request, metadata) return {"id": server_id, **updated} @@ -307,19 +263,24 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: async def metrics_server_delete(request: Request, inputs: dict[str, Any]) -> None: server_id = str(values(inputs)["id"]) metadata = await cluster_metadata(request) - metrics = _metrics(metadata) - servers = metrics.setdefault("servers", {}) + metrics = dict(_metrics(metadata)) + servers = dict(metrics.get("servers") or {}) if server_id not in servers: raise ApiError(404, "metrics server does not exist") del servers[server_id] + metrics["servers"] = servers metadata["metrics"] = metrics await save_cluster_metadata(request, metadata) async def qemu_index(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, str]]: return subdirs("cpu-flags", "custom-cpu-models") - async def qemu_cpu_flags(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: - return list(DEFAULT_CPU_FLAGS) + async def qemu_cpu_flags(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: + metadata = await cluster_metadata(request) + flags = metadata.get("qemu_cpu_flags") + if isinstance(flags, list): + return [dict(item) for item in flags if isinstance(item, dict)] + return [] async def cpu_models_list(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: metadata = await cluster_metadata(request) @@ -440,14 +401,16 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: async def ceph_flags_get(request: Request, _inputs: dict[str, Any]) -> dict[str, int]: metadata = await cluster_metadata(request) ceph = _ceph(metadata) - await save_cluster_metadata(request, metadata) - return {str(key): int(value) for key, value in ceph["flags"].items()} + flags = ceph.get("flags") + if not isinstance(flags, dict): + return {} + return {str(key): int(value) for key, value in flags.items()} async def ceph_flags_put(request: Request, inputs: dict[str, Any]) -> dict[str, int]: payload = values(inputs) metadata = await cluster_metadata(request) - ceph = _ceph(metadata) - flags = dict(ceph["flags"]) + ceph = dict(_ceph(metadata)) + flags = dict(ceph.get("flags") or {}) for key, value in payload.items(): if key in {"delete", "digest"}: continue @@ -461,8 +424,8 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: flag = str(values(inputs)["flag"]) metadata = await cluster_metadata(request) ceph = _ceph(metadata) - flags = ceph["flags"] - if flag not in flags: + flags = ceph.get("flags") + if not isinstance(flags, dict) or flag not in flags: raise ApiError(404, "ceph flag does not exist") return {flag: int(flags[flag])} @@ -470,8 +433,8 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) flag = str(payload["flag"]) metadata = await cluster_metadata(request) - ceph = _ceph(metadata) - flags = dict(ceph["flags"]) + ceph = dict(_ceph(metadata)) + flags = dict(ceph.get("flags") or {}) if "value" in payload: flags[flag] = int(payload["value"]) elif flag in payload: @@ -486,14 +449,15 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: async def ceph_metadata(request: Request, _inputs: dict[str, Any]) -> dict[str, Any]: metadata = await cluster_metadata(request) ceph = _ceph(metadata) - await save_cluster_metadata(request, metadata) + config_raw = ceph.get("config") + config: dict[str, Any] = dict(config_raw) if isinstance(config_raw, dict) else {} + version = ceph.get("version") + flags = ceph.get("flags") return { - "version": ceph.get("version") or {"str": "18.2.2", "parts": [18, 2, 2]}, - "fsid": ceph.get("config", {}).get("fsid") - if isinstance(ceph.get("config"), dict) - else "pve-simulator-fsid", - "initialized": int(bool(ceph.get("initialized", True))), - "flags": ceph.get("flags", {}), + "version": dict(version) if isinstance(version, dict) else {}, + "fsid": str(config.get("fsid") or ""), + "initialized": int(bool(ceph.get("initialized"))), + "flags": dict(flags) if isinstance(flags, dict) else {}, } async def ha_rule_create(request: Request, inputs: dict[str, Any]) -> None: @@ -588,7 +552,7 @@ def register_cluster_extra_handlers(registry: HandlerRegistry) -> None: log = job.get("log") if isinstance(log, list): return [dict(item) for item in log if isinstance(item, dict)] - return [{"t": int(time.time()), "n": 0, "msg": f"replication idle for {job.get('id')}"}] + return [] async def node_replication_status(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: job = await node_replication_get(request, inputs) diff --git a/app/handlers/core.py b/app/handlers/core.py index 2d431e1..8192c2c 100644 --- a/app/handlers/core.py +++ b/app/handlers/core.py @@ -68,44 +68,220 @@ def build_core_handlers(settings: Settings) -> HandlerRegistry: } async def nodes(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: + from app.handlers.nodes import load_node_ops + rows = await _database(request).pool.fetch( "SELECT name AS node, status FROM nodes ORDER BY name" ) - return [{"node": str(row["node"]), "status": str(row["status"])} for row in rows] + result: list[dict[str, Any]] = [] + for row in rows: + name = str(row["node"]) + ops = await load_node_ops(request, name) + status_payload = ops.get("status") + status_dict = dict(status_payload) if isinstance(status_payload, dict) else {} + fingerprint = status_dict.get("ssl_fingerprint") or status_dict.get("fingerprint") + if fingerprint in (None, "", 0) or isinstance(fingerprint, dict | list): + fingerprint = ":".join(["00"] * 32) + + def _as_float(value: object, default: float) -> float: + if isinstance(value, bool) or value is None or isinstance(value, dict | list): + return default + try: + return float(str(value)) + except (TypeError, ValueError): + return default + + def _as_int(value: object, default: int) -> int: + if isinstance(value, bool) or value is None or isinstance(value, dict | list): + return default + try: + return int(float(str(value))) + except (TypeError, ValueError): + return default + + item: dict[str, Any] = { + "node": name, + "status": str(row["status"]), + "type": "node", + "ssl_fingerprint": str(fingerprint), + "cpu": _as_float(status_dict.get("cpu"), 0.0), + "maxcpu": _as_int(status_dict.get("maxcpu"), 4), + "mem": _as_int(status_dict.get("mem"), _as_int(status_dict.get("memory"), 0)), + "maxmem": _as_int(status_dict.get("maxmem"), 8 * 1024**3), + "uptime": _as_int(status_dict.get("uptime"), 0), + "level": str(status_dict.get("level") or ""), + } + result.append(item) + return result async def node_status(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: + from app.handlers.nodes import load_node_ops + node = str(cast(dict[str, Any], inputs["values"])["node"]) row = await _database(request).pool.fetchrow( "SELECT name, status FROM nodes WHERE name=$1", node ) if row is None: raise ApiError(404, "node does not exist") + ops = await load_node_ops(request, node) + status = ops.get("status") + payload = dict(status) if isinstance(status, dict) else {} return { "status": str(row["status"]), "node": str(row["name"]), - "uptime": 0, - "cpu": 0.0, - "memory": {"used": 0, "total": 0}, + **payload, } - async def resources(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: - rows = await _database(request).pool.fetch( - """SELECT r.kind AS type, r.external_id, r.state, n.name AS node - FROM resources r JOIN nodes n ON n.id=r.node_id - ORDER BY r.kind, r.external_id""" - ) - result: list[dict[str, Any]] = [] - for row in rows: - raw_state = row["state"] - state = json.loads(raw_state) if isinstance(raw_state, str) else dict(raw_state) - result.append( - { - "type": str(row["type"]), - "id": f"{row['type']}/{row['external_id']}", - "node": str(row["node"]), - **state, - } + async def resources(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: + """Cluster-wide inventory in Proxmox ``/cluster/resources`` shape. + + Do not dump guest ``state`` / config blobs: QEMU ``cpu`` is a model + string (e.g. ``qemu64``), while this endpoint's ``cpu`` is utilization + (float). Bridged clients (bpg / pulumi-proxmoxve) decode strictly. + """ + + def _as_dict(raw: object) -> dict[str, Any]: + if isinstance(raw, str): + loaded = json.loads(raw) + return dict(loaded) if isinstance(loaded, dict) else {} + return dict(raw) if isinstance(raw, dict) else {} + + def _num(value: object, default: float | int) -> float | int: + if isinstance(value, bool) or value is None or isinstance(value, dict | list): + return default + try: + if isinstance(default, float): + return float(str(value)) + return int(float(str(value))) + except (TypeError, ValueError): + return default + + def _memory_bytes(state: dict[str, Any]) -> int: + raw = state.get("maxmem", state.get("memory")) + if raw in (None, ""): + return 0 + value = _num(raw, 0) + # QEMU config stores memory in MiB; cluster resources use bytes. + if isinstance(raw, str) or (isinstance(value, int) and 0 < value < 10_000_000): + return int(value) * 1024 * 1024 + return int(value) + + def _maxcpu(state: dict[str, Any]) -> int: + cores = int(_num(state.get("cores", state.get("cpus", 1)), 1)) + sockets = int(_num(state.get("sockets", 1), 1)) + return max(cores * sockets, 1) + + def _cpu_util(state: dict[str, Any], *, running: bool) -> float: + if not running: + return 0.0 + samples = state.get("rrddata") + if isinstance(samples, list) and samples: + last = samples[-1] + if isinstance(last, dict): + return float(_num(last.get("cpu"), 0.0)) + return ( + float(_num(state.get("cpu"), 0.0)) if not isinstance(state.get("cpu"), str) else 0.0 ) + + type_filter = cast(dict[str, Any], inputs.get("values") or {}).get("type") + result: list[dict[str, Any]] = [] + + if type_filter in (None, "node"): + node_rows = await _database(request).pool.fetch( + "SELECT name AS node, status FROM nodes ORDER BY name" + ) + from app.handlers.nodes import load_node_ops + + for row in node_rows: + name = str(row["node"]) + ops = await load_node_ops(request, name) + status_payload = ops.get("status") + status_dict = dict(status_payload) if isinstance(status_payload, dict) else {} + result.append( + { + "type": "node", + "id": f"node/{name}", + "node": name, + "status": str(row["status"]), + "cpu": float(_num(status_dict.get("cpu"), 0.0)), + "maxcpu": int(_num(status_dict.get("maxcpu"), 4)), + "mem": int( + _num(status_dict.get("mem"), _num(status_dict.get("memory"), 0)) + ), + "maxmem": int(_num(status_dict.get("maxmem"), 8 * 1024**3)), + "uptime": int(_num(status_dict.get("uptime"), 0)), + "level": str(status_dict.get("level") or ""), + } + ) + + kind_filter: tuple[str, ...] | None + if type_filter == "vm": + kind_filter = ("qemu", "lxc") + elif type_filter == "storage": + kind_filter = ("storage",) + elif type_filter in (None,): + kind_filter = ("qemu", "lxc", "storage") + elif type_filter in {"qemu", "lxc", "storage", "pool", "sdn"}: + kind_filter = (str(type_filter),) + else: + kind_filter = () + + if kind_filter: + rows = await _database(request).pool.fetch( + """SELECT r.kind AS type, r.external_id, r.state, n.name AS node + FROM resources r JOIN nodes n ON n.id=r.node_id + WHERE r.kind = ANY($1::text[]) + ORDER BY r.kind, r.external_id""", + list(kind_filter), + ) + for row in rows: + kind = str(row["type"]) + external_id = str(row["external_id"]) + node = str(row["node"]) + state = _as_dict(row["state"]) + if kind in {"qemu", "lxc"}: + status = str(state.get("status") or "stopped") + running = status in {"running", "paused"} + vmid = int(external_id) + item: dict[str, Any] = { + "type": kind, + "id": f"{kind}/{external_id}", + "node": node, + "vmid": vmid, + "name": str(state.get("name") or f"{kind}-{external_id}"), + "status": status, + "template": 1 if state.get("template") in {True, "1"} else 0, + "cpu": _cpu_util(state, running=running), + "maxcpu": _maxcpu(state), + "mem": int(_num(state.get("mem"), 0)) if running else 0, + "maxmem": _memory_bytes(state), + "disk": int(_num(state.get("disk"), 0)), + "maxdisk": int(_num(state.get("maxdisk"), 0)), + "uptime": int(_num(state.get("uptime"), 0)) if running else 0, + } + result.append(item) + elif kind == "storage": + content = state.get("content") + if isinstance(content, list): + content_text = ",".join(str(item) for item in content) + else: + content_text = str(content or "") + result.append( + { + "type": "storage", + "id": f"storage/{node}/{external_id}", + "node": node, + "storage": external_id, + "status": str(state.get("status") or "available"), + "content": content_text, + "disk": int(_num(state.get("disk"), 0)), + "maxdisk": int(_num(state.get("maxdisk"), 1 * 1024**3)), + "shared": int(_num(state.get("shared"), 0)), + "plugintype": str( + state.get("plugintype") or state.get("type") or "dir" + ), + } + ) return result async def node_index(request: Request, inputs: dict[str, Any]) -> list[dict[str, str]]: diff --git a/app/handlers/firewall.py b/app/handlers/firewall.py index 8eef5f3..69649d1 100644 --- a/app/handlers/firewall.py +++ b/app/handlers/firewall.py @@ -13,20 +13,6 @@ from app.api.registry import HandlerRegistry from app.handlers.common import database, require_node, state, subdirs, values from app.simulation.seed import CLUSTER_ID -DEFAULT_OPTIONS = { - "enable": 1, - "policy_in": "DROP", - "policy_out": "ACCEPT", - "log_level_in": "nolog", - "log_level_out": "nolog", -} - -DEFAULT_MACROS = [ - {"macro": "SSH", "descr": "Secure Shell"}, - {"macro": "HTTPS", "descr": "Secure web server"}, - {"macro": "HTTP", "descr": "Web server"}, -] - ScopeFn = Callable[[dict[str, Any]], str] @@ -50,19 +36,36 @@ async def _save_firewall(request: Request, firewall: dict[str, Any]) -> None: ) -def _scope_data(firewall: dict[str, Any], scope: str) -> dict[str, Any]: +def _empty_scope() -> dict[str, Any]: + return { + "options": {}, + "rules": [], + "aliases": {}, + "ipset": {}, + "groups": {}, + "log": [], + } + + +def _get_scope(firewall: dict[str, Any], scope: str) -> dict[str, Any] | None: + scopes = firewall.get("scopes") + if not isinstance(scopes, dict): + return None + section = scopes.get(scope) + return cast(dict[str, Any], section) if isinstance(section, dict) else None + + +def _ensure_scope(firewall: dict[str, Any], scope: str) -> dict[str, Any]: + """Create an empty durable scope on mutation; never injects catalog defaults.""" scopes = firewall.setdefault("scopes", {}) - if scope not in scopes or not isinstance(scopes[scope], dict): - scopes[scope] = { - "options": dict(DEFAULT_OPTIONS), - "rules": [], - "aliases": {}, - "ipset": {}, - "groups": {}, - "log": [], - } - section = scopes[scope] - section.setdefault("options", dict(DEFAULT_OPTIONS)) + if not isinstance(scopes, dict): + scopes = {} + firewall["scopes"] = scopes + section = scopes.get(scope) + if not isinstance(section, dict): + section = _empty_scope() + scopes[scope] = section + section.setdefault("options", {}) section.setdefault("rules", []) section.setdefault("aliases", {}) section.setdefault("ipset", {}) @@ -71,6 +74,11 @@ def _scope_data(firewall: dict[str, Any], scope: str) -> dict[str, Any]: return cast(dict[str, Any], section) +def _scope_data(firewall: dict[str, Any], scope: str) -> dict[str, Any]: + """Mutation helper — ensure scope exists without template defaults.""" + return _ensure_scope(firewall, scope) + + def register_firewall_handlers(registry: HandlerRegistry) -> None: def register_scope( base: str, @@ -98,13 +106,17 @@ def register_firewall_handlers(registry: HandlerRegistry) -> None: async def options_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: payload = await _ready(request, inputs) firewall = await _load_firewall(request) - return dict(_scope_data(firewall, scope_fn(payload)).get("options", DEFAULT_OPTIONS)) + section = _get_scope(firewall, scope_fn(payload)) + if section is None: + return {} + options = section.get("options", {}) + return dict(options) if isinstance(options, dict) else {} async def options_put(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: payload = await _ready(request, inputs) firewall = await _load_firewall(request) - section = _scope_data(firewall, scope_fn(payload)) - current = dict(section.get("options", DEFAULT_OPTIONS)) + section = _ensure_scope(firewall, scope_fn(payload)) + current = dict(section.get("options") or {}) for key, value in payload.items(): if key in {"node", "vmid", "delete", "digest"}: continue @@ -116,7 +128,10 @@ def register_firewall_handlers(registry: HandlerRegistry) -> None: async def rules_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: payload = await _ready(request, inputs) firewall = await _load_firewall(request) - rules = _scope_data(firewall, scope_fn(payload)).get("rules", []) + section = _get_scope(firewall, scope_fn(payload)) + if section is None: + return [] + rules = section.get("rules", []) return list(rules) if isinstance(rules, list) else [] async def rules_create(request: Request, inputs: dict[str, Any]) -> None: @@ -169,7 +184,8 @@ def register_firewall_handlers(registry: HandlerRegistry) -> None: async def aliases_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: payload = await _ready(request, inputs) firewall = await _load_firewall(request) - aliases = _scope_data(firewall, scope_fn(payload)).get("aliases", {}) + section = _get_scope(firewall, scope_fn(payload)) + aliases = section.get("aliases", {}) if section else {} if not isinstance(aliases, dict): return [] return [ @@ -236,7 +252,8 @@ def register_firewall_handlers(registry: HandlerRegistry) -> None: async def ipset_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: payload = await _ready(request, inputs) firewall = await _load_firewall(request) - ipsets = _scope_data(firewall, scope_fn(payload)).get("ipset", {}) + section = _get_scope(firewall, scope_fn(payload)) + ipsets = section.get("ipset", {}) if section else {} if not isinstance(ipsets, dict): return [] return [ @@ -352,27 +369,43 @@ def register_firewall_handlers(registry: HandlerRegistry) -> None: async def refs_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: payload = await _ready(request, inputs) firewall = await _load_firewall(request) - section = _scope_data(firewall, scope_fn(payload)) + section = _get_scope(firewall, scope_fn(payload)) + if section is None: + return [] refs: list[dict[str, Any]] = [] - for name in section.get("aliases", {}): + for name in section.get("aliases", {}) or {}: refs.append({"type": "alias", "name": name}) - for name in section.get("ipset", {}): + for name in section.get("ipset", {}) or {}: refs.append({"type": "ipset", "name": name}) return refs async def log_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: payload = await _ready(request, inputs) firewall = await _load_firewall(request) - log = _scope_data(firewall, scope_fn(payload)).get("log", []) + section = _get_scope(firewall, scope_fn(payload)) + if section is None: + return [] + log = section.get("log", []) return list(log) if isinstance(log, list) else [] - async def macros_list(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: - return list(DEFAULT_MACROS) + async def macros_list(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: + from app.handlers.common import cluster_metadata + + metadata = await cluster_metadata(request) + macros = metadata.get("firewall_macros") + if isinstance(macros, list): + return [dict(item) for item in macros if isinstance(item, dict)] + firewall = await _load_firewall(request) + nested = firewall.get("macros") + if isinstance(nested, list): + return [dict(item) for item in nested if isinstance(item, dict)] + return [] async def groups_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: payload = await _ready(request, inputs) firewall = await _load_firewall(request) - groups = _scope_data(firewall, scope_fn(payload)).get("groups", {}) + section = _get_scope(firewall, scope_fn(payload)) + groups = section.get("groups", {}) if section else {} if not isinstance(groups, dict): return [] return [ diff --git a/app/handlers/ha.py b/app/handlers/ha.py index 100f174..28ef7ce 100644 --- a/app/handlers/ha.py +++ b/app/handlers/ha.py @@ -142,7 +142,7 @@ def register_ha_handlers(registry: HandlerRegistry) -> None: async def ha_groups(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: metadata = await cluster_metadata(request) configured = _ha_groups(metadata) - result = [ + return [ { "group": group_id, "nodes": str(payload.get("nodes", "")), @@ -153,26 +153,6 @@ def register_ha_handlers(registry: HandlerRegistry) -> None: } for group_id, payload in sorted(configured.items()) ] - if result: - return result - rows = await database(request).pool.fetch( - """SELECT DISTINCT state->>'group' AS group_id - FROM resources WHERE kind='ha' AND state ? 'group' - ORDER BY 1""" - ) - node_names = await database(request).pool.fetch("SELECT name FROM nodes ORDER BY name") - nodes = ",".join(str(row["name"]) for row in node_names) or "pve01" - return [ - { - "group": str(row["group_id"]), - "nodes": nodes, - "nofailback": 0, - "restricted": 0, - "type": "group", - } - for row in rows - if row["group_id"] - ] async def ha_group_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: group = str(values(inputs)["group"]) @@ -238,38 +218,39 @@ def register_ha_handlers(registry: HandlerRegistry) -> None: "SELECT name FROM nodes WHERE status='online' ORDER BY name LIMIT 1" ) metadata = await cluster_metadata(request) - ha = metadata.get("ha", {}) if isinstance(metadata.get("ha"), dict) else {} + ha_raw = metadata.get("ha") + ha: dict[str, Any] = dict(ha_raw) if isinstance(ha_raw, dict) else {} + status_raw = ha.get("status_current") + status: dict[str, Any] = dict(status_raw) if isinstance(status_raw, dict) else {} + armed = bool(ha.get("armed")) if "armed" in ha else False return { - "quorate": 1, - "mode": "active" if ha.get("armed", True) else "disabled", - "master_node": str(master or "pve01"), + "quorate": status.get("quorate", metadata.get("quorate", 0)), + "mode": status.get("mode", "active" if armed else "disabled"), + "master_node": status.get("master_node", str(master or "")), "ha_started": int(row["started"] or 0), "ha_total": int(row["total"] or 0), - "armed": 1 if ha.get("armed", True) else 0, + "armed": 1 if armed else 0, } async def ha_manager_status(request: Request, _inputs: dict[str, Any]) -> dict[str, Any]: metadata = await cluster_metadata(request) - ha = metadata.get("ha", {}) if isinstance(metadata.get("ha"), dict) else {} - armed = bool(ha.get("armed", True)) + ha_raw = metadata.get("ha") + ha: dict[str, Any] = dict(ha_raw) if isinstance(ha_raw, dict) else {} + manager_raw = ha.get("manager_status") + manager: dict[str, Any] = dict(manager_raw) if isinstance(manager_raw, dict) else {} + armed = bool(ha.get("armed")) if "armed" in ha else False return { - "manager_status": "active" if armed else "disabled", - "quorum": "OK", + "manager_status": manager.get("manager_status", "active" if armed else "disabled"), + "quorum": manager.get("quorum", ""), "armed": 1 if armed else 0, } async def ha_rules(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: metadata = await cluster_metadata(request) rules = metadata.get("ha_rules") - if isinstance(rules, list) and rules: + if isinstance(rules, list): return [dict(item) for item in rules if isinstance(item, dict)] - defaults = [ - {"rule": "node-fencing", "type": "node", "action": "restart"}, - {"rule": "service-ha", "type": "resource", "action": "failover"}, - ] - metadata["ha_rules"] = defaults - await save_cluster_metadata(request, metadata) - return list(defaults) + return [] async def ha_relocate(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) diff --git a/app/handlers/lxc.py b/app/handlers/lxc.py index 3fbaeb0..79bf824 100644 --- a/app/handlers/lxc.py +++ b/app/handlers/lxc.py @@ -323,7 +323,7 @@ def register_lxc_handlers(registry: HandlerRegistry) -> None: async def migrate_preconditions(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: values = _values(inputs) - await _lxc_resource(request, str(values["node"]), str(values["vmid"])) + resource = await _lxc_resource(request, str(values["node"]), str(values["vmid"])) target = values.get("target") if target in {None, ""}: raise ApiError(400, "parameter 'target' is required") @@ -333,7 +333,13 @@ def register_lxc_handlers(registry: HandlerRegistry) -> None: ) if not exists: raise ApiError(404, "target node does not exist") - return {"local_disks": [], "local_resources": [], "running": False} + state = _state(resource["state"]) + pre = state.get("migrate_preconditions") + payload = dict(pre) if isinstance(pre, dict) else {} + payload["running"] = str(state.get("status") or "") == "running" + payload.setdefault("local_disks", []) + payload.setdefault("local_resources", []) + return payload async def migrate(request: Request, inputs: dict[str, Any]) -> str: values = _values(inputs) diff --git a/app/handlers/lxc_extra.py b/app/handlers/lxc_extra.py index d900451..ed66e84 100644 --- a/app/handlers/lxc_extra.py +++ b/app/handlers/lxc_extra.py @@ -39,12 +39,8 @@ def register_lxc_extra_handlers(registry: HandlerRegistry) -> None: values = _values(inputs) resource = await _lxc_resource(request, str(values["node"]), str(values["vmid"])) state = _state(resource["state"]) - ifaces = state.setdefault( - "interfaces", - [{"name": "eth0", "hwaddr": "02:00:00:00:00:11", "inet": "192.0.2.20/24"}], - ) - await _save_state(request, resource["id"], state) - return list(ifaces) if isinstance(ifaces, list) else [] + ifaces = state.get("interfaces") + return [dict(item) for item in ifaces] if isinstance(ifaces, list) else [] async def move_volume(request: Request, inputs: dict[str, Any]) -> str: values = _values(inputs) @@ -70,23 +66,15 @@ def register_lxc_extra_handlers(registry: HandlerRegistry) -> None: values = _values(inputs) resource = await _lxc_resource(request, str(values["node"]), str(values["vmid"])) state = _state(resource["state"]) - rrd_state = state.setdefault("rrd", {"filename": f"pve-ct-{values['vmid']}.rrd"}) - await _save_state(request, resource["id"], state) - return dict(rrd_state) + rrd_state = state.get("rrd") + return dict(rrd_state) if isinstance(rrd_state, dict) else {} async def rrddata(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: values = _values(inputs) resource = await _lxc_resource(request, str(values["node"]), str(values["vmid"])) state = _state(resource["state"]) - series = state.setdefault( - "rrddata", - [ - {"time": 1_700_000_000, "cpu": 0.02, "mem": 64 * 1024 * 1024}, - {"time": 1_700_000_060, "cpu": 0.03, "mem": 66 * 1024 * 1024}, - ], - ) - await _save_state(request, resource["id"], state) - return list(series) + series = state.get("rrddata") + return [dict(item) for item in series] if isinstance(series, list) else [] async def _console(request: Request, inputs: dict[str, Any], kind: str) -> dict[str, Any]: values = _values(inputs) diff --git a/app/handlers/mapping.py b/app/handlers/mapping.py index 8c39168..06df984 100644 --- a/app/handlers/mapping.py +++ b/app/handlers/mapping.py @@ -12,13 +12,14 @@ from app.handlers.common import cluster_metadata, save_cluster_metadata, subdirs def _mappings(metadata: dict[str, Any]) -> dict[str, Any]: - current = metadata.setdefault("mapping", {"dir": {}, "pci": {}, "usb": {}}) + current = metadata.get("mapping") if not isinstance(current, dict): - current = {"dir": {}, "pci": {}, "usb": {}} - metadata["mapping"] = current - for kind in ("dir", "pci", "usb"): - current.setdefault(kind, {}) - return current + return {"dir": {}, "pci": {}, "usb": {}} + return { + "dir": dict(current["dir"]) if isinstance(current.get("dir"), dict) else {}, + "pci": dict(current["pci"]) if isinstance(current.get("pci"), dict) else {}, + "usb": dict(current["usb"]) if isinstance(current.get("usb"), dict) else {}, + } def register_mapping_handlers(registry: HandlerRegistry) -> None: @@ -42,12 +43,14 @@ def register_mapping_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) item_id = str(payload["id"]) metadata = await cluster_metadata(request) - store = _mappings(metadata)[kind] + mapping = _mappings(metadata) + store = mapping[kind] if item_id in store: raise ApiError(400, f"{kind} mapping '{item_id}' already exists") store[item_id] = { key: value for key, value in payload.items() if key not in {"delete", "digest"} } + metadata["mapping"] = mapping await save_cluster_metadata(request, metadata) async def get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: @@ -62,7 +65,8 @@ def register_mapping_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) item_id = str(payload["id"]) metadata = await cluster_metadata(request) - store = _mappings(metadata)[kind] + mapping = _mappings(metadata) + store = mapping[kind] if item_id not in store: raise ApiError(404, f"{kind} mapping does not exist") current = dict(store[item_id]) @@ -76,15 +80,18 @@ def register_mapping_handlers(registry: HandlerRegistry) -> None: current[key] = value current["id"] = item_id store[item_id] = current + metadata["mapping"] = mapping await save_cluster_metadata(request, metadata) async def delete(request: Request, inputs: dict[str, Any]) -> None: item_id = str(values(inputs)["id"]) metadata = await cluster_metadata(request) - store = _mappings(metadata)[kind] + mapping = _mappings(metadata) + store = mapping[kind] if item_id not in store: raise ApiError(404, f"{kind} mapping does not exist") del store[item_id] + metadata["mapping"] = mapping await save_cluster_metadata(request, metadata) registry.register(base, "GET", list_items) diff --git a/app/handlers/nodes.py b/app/handlers/nodes.py index 7306637..f9d6cb1 100644 --- a/app/handlers/nodes.py +++ b/app/handlers/nodes.py @@ -2,7 +2,6 @@ from __future__ import annotations -import copy from typing import Any, cast from fastapi import Request @@ -20,92 +19,13 @@ from app.handlers.common import ( from app.tasks.repository import TaskRepository from app.tasks.upid import Upid -DEFAULT_NODE_OPS: dict[str, Any] = { - "network": [ - { - "iface": "vmbr0", - "type": "bridge", - "active": 1, - "method": "static", - "address": "10.0.0.10/24", - }, - { - "iface": "vmbr1", - "type": "bridge", - "active": 1, - "method": "static", - "address": "10.10.0.10/24", - }, - {"iface": "eno1", "type": "eth", "active": 1, "method": "manual"}, - ], - "disks": { - "list": [ - { - "devpath": "/dev/sda", - "size": 1_000_000_000_000, - "model": "SIM-DISK-01", - "serial": "SIM0001", - "gpt": 1, - }, - { - "devpath": "/dev/sdb", - "size": 2_000_000_000_000, - "model": "SIM-SSD-01", - "serial": "SIM0002", - "gpt": 0, - }, - ], - "directory": [], - "lvm": [], - "lvmthin": [], - "zfs": [], - "smart": {}, - }, - "services": { - "pveproxy": {"state": "running", "enabled": 1}, - "pvedaemon": {"state": "running", "enabled": 1}, - "pvestatd": {"state": "running", "enabled": 1}, - "corosync": {"state": "running", "enabled": 1}, - }, - "apt": { - "packages": [ - { - "Package": "pve-manager", - "Version": "9.2.3", - "OldVersion": "9.2.2", - "Status": "upgradable", - }, - {"Package": "libpve-common-perl", "Version": "9.0.3", "Status": "installed"}, - ], - "repositories": [ - { - "path": "/etc/apt/sources.list.d/pve-enterprise.list", - "enabled": 1, - "types": "deb", - "uri": "http://download.proxmox.com/debian/pve", - "suites": "bookworm", - "components": "pve-no-subscription", - } - ], - "update": {"status": "stopped", "exitstatus": "OK"}, - "changelogs": {}, - }, -} - - -def default_node_ops() -> dict[str, Any]: - return copy.deepcopy(DEFAULT_NODE_OPS) - async def load_node_ops(request: Request, node: str) -> dict[str, Any]: metadata = await node_metadata(request, node) ops = metadata.get("ops") - if isinstance(ops, dict) and ops: + if isinstance(ops, dict): return ops - ops = default_node_ops() - metadata["ops"] = ops - await save_node_metadata(request, node, metadata) - return ops + return {} async def save_node_ops(request: Request, node: str, ops: dict[str, Any]) -> None: @@ -135,19 +55,18 @@ def register_node_ops_handlers(registry: HandlerRegistry) -> None: node = str(values(inputs)["node"]) name = str(values(inputs).get("name") or "pve-manager") ops = await load_node_ops(request, node) - changelogs = ops.setdefault("apt", {}).setdefault("changelogs", {}) - if name not in changelogs: - changelogs[name] = f"simulated changelog for {name}\n\n * emulator build\n" - await save_node_ops(request, node, ops) - return str(changelogs[name]) + apt = ops.get("apt") + changelogs = apt.get("changelogs") if isinstance(apt, dict) else None + if not isinstance(changelogs, dict): + return "" + return str(changelogs.get(name) or "") async def apt_update_status(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: node = str(values(inputs)["node"]) ops = await load_node_ops(request, node) - update = ops.get("apt", {}).get("update", {"status": "stopped", "exitstatus": "OK"}) - if isinstance(update, dict): - return dict(update) - return {"status": "stopped", "exitstatus": "OK"} + apt = ops.get("apt") + update = apt.get("update") if isinstance(apt, dict) else None + return dict(update) if isinstance(update, dict) else {} async def apt_update_start(request: Request, inputs: dict[str, Any]) -> str: node = str(values(inputs)["node"]) @@ -230,11 +149,9 @@ def register_node_ops_handlers(registry: HandlerRegistry) -> None: async def _disks(request: Request, node: str) -> dict[str, Any]: ops = await load_node_ops(request, node) - disks = ops.setdefault("disks", default_node_ops()["disks"]) + disks = ops.get("disks") if not isinstance(disks, dict): - disks = default_node_ops()["disks"] - ops["disks"] = disks - await save_node_ops(request, node, ops) + return {} return cast(dict[str, Any], disks) async def disks_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: @@ -247,17 +164,9 @@ def register_node_ops_handlers(registry: HandlerRegistry) -> None: node = str(values(inputs)["node"]) disk = str(values(inputs).get("disk") or "/dev/sda") disks = await _disks(request, node) - smart = disks.setdefault("smart", {}) - if disk not in smart: - smart[disk] = { - "health": "PASSED", - "type": "scsi", - "model": "SIM-DISK", - "serial": disk.rsplit("/", 1)[-1], - } - ops = await load_node_ops(request, node) - ops["disks"] = disks - await save_node_ops(request, node, ops) + smart = disks.get("smart") + if not isinstance(smart, dict) or disk not in smart: + return {} return dict(smart[disk]) async def disks_collection( @@ -286,7 +195,10 @@ def register_node_ops_handlers(registry: HandlerRegistry) -> None: if not disk: raise ApiError(400, "parameter verification failed - 'disk' missing") ops = await load_node_ops(request, node) - disks = ops.setdefault("disks", default_node_ops()["disks"]) + disks = ops.get("disks") + if not isinstance(disks, dict): + disks = {} + ops["disks"] = disks items = list(disks.get("list") or []) found = False for item in items: @@ -314,7 +226,9 @@ def register_node_ops_handlers(registry: HandlerRegistry) -> None: if not disk: raise ApiError(400, "parameter verification failed - 'disk' missing") ops = await load_node_ops(request, node) - disks = ops.setdefault("disks", default_node_ops()["disks"]) + disks = ops.get("disks") + if not isinstance(disks, dict): + raise ApiError(404, "disk does not exist") items = list(disks.get("list") or []) for item in items: if item.get("devpath") == disk: @@ -324,8 +238,9 @@ def register_node_ops_handlers(registry: HandlerRegistry) -> None: else: raise ApiError(404, "disk does not exist") disks["list"] = items - smart = disks.setdefault("smart", {}) - smart.pop(disk, None) + smart = disks.get("smart") + if isinstance(smart, dict): + smart.pop(disk, None) ops["disks"] = disks await save_node_ops(request, node, ops) diff --git a/app/handlers/nodes_extra.py b/app/handlers/nodes_extra.py index 06462fa..1323d9e 100644 --- a/app/handlers/nodes_extra.py +++ b/app/handlers/nodes_extra.py @@ -2,7 +2,6 @@ from __future__ import annotations -import copy import json import secrets import time @@ -13,128 +12,49 @@ from fastapi import Request from app.api.errors import ApiError from app.api.registry import HandlerRegistry from app.handlers.common import database, require_node, subdirs, values -from app.handlers.nodes import default_node_ops, load_node_ops, save_node_ops +from app.handlers.nodes import load_node_ops, save_node_ops from app.tasks.repository import TaskRepository from app.tasks.upid import Upid -DEFAULT_HARDWARE: dict[str, Any] = { - "pci": [ - { - "id": "0000:00:1f.2", - "vendor_name": "Intel Corporation", - "device_name": "SATA Controller", - "iommugroup": 0, - }, - { - "id": "0000:01:00.0", - "vendor_name": "NVIDIA Corporation", - "device_name": "GP102 [GeForce GTX 1080 Ti]", - "iommugroup": 1, - "mdev": 1, - }, - ], - "usb": [ - {"busnum": 1, "devnum": 1, "level": 0, "port": "1", "prodid": "0002", "vendid": "1d6b"}, - {"busnum": 2, "devnum": 2, "level": 1, "port": "2", "prodid": "5591", "vendid": "0781"}, - ], - "mdev": { - "0000:01:00.0": [ - {"type": "nvidia-11", "available": 4, "description": "GRID profile"}, - ] - }, -} - -DEFAULT_SCAN: dict[str, list[dict[str, Any]]] = { - "cifs": [{"server": "files.local", "share": "backups"}], - "iscsi": [{"portal": "10.0.0.50:3260", "target": "iqn.2024-01.local:storage"}], - "lvm": [{"vg": "pve", "size": 500_000_000_000, "free": 100_000_000_000}], - "lvmthin": [{"lv": "data", "vg": "pve", "lv_size": 400_000_000_000}], - "nfs": [{"server": "nfs.local", "path": "/export/pve", "options": "vers=4"}], - "pbs": [{"server": "pbs.local", "datastore": "store1"}], - "zfs": [{"pool": "rpool", "name": "rpool/data", "size": 800_000_000_000}], -} - -DEFAULT_SUBSCRIPTION: dict[str, Any] = { - "status": "notfound", - "message": "There is no subscription key", - "serverid": "SIMULATOR", - "sockets": 1, - "productname": "Proxmox VE", - "url": "https://www.proxmox.com/en/proxmox-virtual-environment/pricing", -} - -DEFAULT_CONFIG: dict[str, Any] = { - "description": "Simulator node", - "startall-onboot-delay": 0, - "wakeonlan": "", -} - -DEFAULT_DNS: dict[str, Any] = { - "search": "local", - "dns1": "1.1.1.1", - "dns2": "8.8.8.8", - "dns3": "", -} - -DEFAULT_TIME: dict[str, Any] = { - "timezone": "UTC", - "time": 0, - "localtime": 0, -} - def _certificates(ops: dict[str, Any]) -> dict[str, Any]: - certs = ops.setdefault( - "certificates", - { - "custom": None, - "acme": {"account": "default", "domains": [], "certificate": None}, - "info": [], - }, - ) + certs = ops.get("certificates") if not isinstance(certs, dict): - certs = {"custom": None, "acme": {}, "info": []} - ops["certificates"] = certs - certs.setdefault("acme", {"account": "default", "domains": [], "certificate": None}) - certs.setdefault("info", []) + return {"custom": None, "acme": {}, "info": []} return certs def _hardware(ops: dict[str, Any]) -> dict[str, Any]: hardware = ops.get("hardware") - if not isinstance(hardware, dict) or not hardware: - hardware = copy.deepcopy(DEFAULT_HARDWARE) - ops["hardware"] = hardware - hardware.setdefault("pci", copy.deepcopy(DEFAULT_HARDWARE["pci"])) - hardware.setdefault("usb", copy.deepcopy(DEFAULT_HARDWARE["usb"])) - hardware.setdefault("mdev", copy.deepcopy(DEFAULT_HARDWARE["mdev"])) - return hardware + if not isinstance(hardware, dict): + return {"pci": [], "usb": [], "mdev": {}} + return { + "pci": list(hardware.get("pci") or []) if isinstance(hardware.get("pci"), list) else [], + "usb": list(hardware.get("usb") or []) if isinstance(hardware.get("usb"), list) else [], + "mdev": dict(hardware.get("mdev") or {}) if isinstance(hardware.get("mdev"), dict) else {}, + } def _scan_cache(ops: dict[str, Any]) -> dict[str, Any]: scan = ops.get("scan") - if not isinstance(scan, dict) or not scan: - scan = copy.deepcopy(DEFAULT_SCAN) - ops["scan"] = scan - for key, value in DEFAULT_SCAN.items(): - scan.setdefault(key, copy.deepcopy(value)) + if not isinstance(scan, dict): + return {} return scan def _subscription(ops: dict[str, Any]) -> dict[str, Any]: subscription = ops.get("subscription") - if not isinstance(subscription, dict) or not subscription: - subscription = copy.deepcopy(DEFAULT_SUBSCRIPTION) - ops["subscription"] = subscription + if not isinstance(subscription, dict): + return {} return subscription def _disk_items(ops: dict[str, Any], kind: str) -> list[dict[str, Any]]: - disks = ops.setdefault("disks", default_node_ops()["disks"]) + disks = ops.get("disks") if not isinstance(disks, dict): - disks = default_node_ops()["disks"] + disks = {} ops["disks"] = disks - items = disks.setdefault(kind, []) + items = disks.get(kind) if not isinstance(items, list): items = [] disks[kind] = items @@ -208,7 +128,11 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: } entry["name"] = name items.append(entry) - ops.setdefault("disks", default_node_ops()["disks"])[kind] = items + disks = ops.get("disks") + if not isinstance(disks, dict): + disks = {} + ops["disks"] = disks + disks[kind] = items await save_node_ops(request, node, ops) return entry @@ -360,35 +284,40 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: return subdirs("cpu", "cpu-flags", "machines", "migration") async def capabilities_cpu(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: - await require_node(request, str(values(inputs)["node"])) - return [ - {"name": "host", "vendor": "QEMU", "custom": 0}, - {"name": "x86-64-v2-AES", "vendor": "QEMU", "custom": 0}, - {"name": "kvm64", "vendor": "QEMU", "custom": 0}, - ] + node = str(values(inputs)["node"]) + await require_node(request, node) + ops = await load_node_ops(request, node) + caps = ops.get("capabilities") + items = caps.get("cpu") if isinstance(caps, dict) else None + return [dict(item) for item in items] if isinstance(items, list) else [] async def capabilities_cpu_flags( request: Request, inputs: dict[str, Any] ) -> list[dict[str, Any]]: - await require_node(request, str(values(inputs)["node"])) - return [ - {"name": "aes", "introduces": "Westmere"}, - {"name": "avx", "introduces": "SandyBridge"}, - {"name": "avx2", "introduces": "Haswell"}, - ] + node = str(values(inputs)["node"]) + await require_node(request, node) + ops = await load_node_ops(request, node) + caps = ops.get("capabilities") + items = caps.get("cpu_flags") if isinstance(caps, dict) else None + return [dict(item) for item in items] if isinstance(items, list) else [] async def capabilities_machines( request: Request, inputs: dict[str, Any] ) -> list[dict[str, Any]]: - await require_node(request, str(values(inputs)["node"])) - return [ - {"id": "pc-i440fx-9.0", "type": "i440fx", "version": "9.0"}, - {"id": "pc-q35-9.0", "type": "q35", "version": "9.0"}, - ] + node = str(values(inputs)["node"]) + await require_node(request, node) + ops = await load_node_ops(request, node) + caps = ops.get("capabilities") + items = caps.get("machines") if isinstance(caps, dict) else None + return [dict(item) for item in items] if isinstance(items, list) else [] async def capabilities_migration(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: - await require_node(request, str(values(inputs)["node"])) - return {"network": "", "type": "secure", "enabled": 1} + node = str(values(inputs)["node"]) + await require_node(request, node) + ops = await load_node_ops(request, node) + caps = ops.get("capabilities") + migration = caps.get("migration") if isinstance(caps, dict) else None + return dict(migration) if isinstance(migration, dict) else {} async def hardware_index(request: Request, inputs: dict[str, Any]) -> list[dict[str, str]]: await require_node(request, str(values(inputs)["node"])) @@ -399,8 +328,6 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: await require_node(request, node) ops = await load_node_ops(request, node) hardware = _hardware(ops) - ops["hardware"] = hardware - await save_node_ops(request, node, ops) return [dict(item) for item in hardware.get("pci", [])] async def hardware_pci_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: @@ -430,8 +357,6 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: await require_node(request, node) ops = await load_node_ops(request, node) hardware = _hardware(ops) - ops["hardware"] = hardware - await save_node_ops(request, node, ops) return [dict(item) for item in hardware.get("usb", [])] async def subscription_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: @@ -452,7 +377,10 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: current = _subscription(ops) method = request.method.upper() if method == "DELETE": - ops["subscription"] = copy.deepcopy(DEFAULT_SUBSCRIPTION) + ops["subscription"] = { + "status": "notfound", + "message": "There is no subscription key", + } await save_node_ops(request, node, ops) return None if method == "POST": @@ -481,17 +409,8 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: ops = await load_node_ops(request, node) items = ops.get("aplinfo") if not isinstance(items, list): - items = [ - { - "package": "alpine-3-standard", - "section": "system", - "type": "lxc", - "version": "3.20", - } - ] - ops["aplinfo"] = items - await save_node_ops(request, node, ops) - return [dict(item) for item in items] + return [] + return [dict(item) for item in items if isinstance(item, dict)] async def aplinfo_download(request: Request, inputs: dict[str, Any]) -> str: payload = values(inputs) @@ -514,7 +433,10 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: node = str(payload["node"]) await require_node(request, node) ops = await load_node_ops(request, node) - apt = ops.setdefault("apt", copy.deepcopy(default_node_ops()["apt"])) + apt = ops.get("apt") + if not isinstance(apt, dict): + apt = {} + ops["apt"] = apt repositories = list(apt.get("repositories") or []) method = request.method.upper() if method == "POST": @@ -561,18 +483,14 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: await require_node(request, node) ops = await load_node_ops(request, node) config = ops.get("config") - if not isinstance(config, dict): - config = copy.deepcopy(DEFAULT_CONFIG) - ops["config"] = config - await save_node_ops(request, node, ops) - return dict(config) + return dict(config) if isinstance(config, dict) else {} async def node_config_put(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: payload = values(inputs) node = str(payload["node"]) await require_node(request, node) ops = await load_node_ops(request, node) - config = dict(ops.get("config") or DEFAULT_CONFIG) + config = dict(ops.get("config") or {}) config.update( { key: value @@ -589,18 +507,14 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: await require_node(request, node) ops = await load_node_ops(request, node) dns = ops.get("dns") - if not isinstance(dns, dict): - dns = copy.deepcopy(DEFAULT_DNS) - ops["dns"] = dns - await save_node_ops(request, node, ops) - return dict(dns) + return dict(dns) if isinstance(dns, dict) else {} async def dns_put(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: payload = values(inputs) node = str(payload["node"]) await require_node(request, node) ops = await load_node_ops(request, node) - dns = dict(ops.get("dns") or DEFAULT_DNS) + dns = dict(ops.get("dns") or {}) dns.update( { key: value @@ -616,13 +530,10 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: node = str(values(inputs)["node"]) await require_node(request, node) ops = await load_node_ops(request, node) - current = dict(ops.get("time") or DEFAULT_TIME) + current = dict(ops.get("time") or {}) now = int(time.time()) current["time"] = now current["localtime"] = now - current.setdefault("timezone", "UTC") - ops["time"] = current - await save_node_ops(request, node, ops) return current async def time_put(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: @@ -630,7 +541,7 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: node = str(payload["node"]) await require_node(request, node) ops = await load_node_ops(request, node) - current = dict(ops.get("time") or DEFAULT_TIME) + current = dict(ops.get("time") or {}) if "timezone" in payload: current["timezone"] = str(payload["timezone"]) now = int(time.time()) @@ -668,12 +579,7 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: ops = await load_node_ops(request, node) hosts = ops.get("hosts") if not isinstance(hosts, dict): - hosts = { - "data": f"127.0.0.1 localhost\n10.0.0.10 {node}\n", - "digest": secrets.token_hex(8), - } - ops["hosts"] = hosts - await save_node_ops(request, node, ops) + return {"data": "", "digest": ""} return {"data": str(hosts.get("data", "")), "digest": str(hosts.get("digest", ""))} async def hosts_post(request: Request, inputs: dict[str, Any]) -> None: @@ -692,46 +598,50 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: await require_node(request, node) start = int(values(inputs).get("startcursor") or values(inputs).get("start") or 0) limit = int(values(inputs).get("limit") or 50) - lines = [ - f"{index}: {node} systemd[1]: Started simulated service {index}." - for index in range(start, start + limit) - ] - return lines + ops = await load_node_ops(request, node) + lines = ops.get("journal") + if not isinstance(lines, list): + return [] + sliced = lines[start : start + limit] + return [str(item) for item in sliced] async def syslog(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: node = str(values(inputs)["node"]) await require_node(request, node) limit = int(values(inputs).get("limit") or 50) - return [ - {"n": index, "t": f"{node} kernel: simulated syslog line {index}"} - for index in range(limit) - ] + ops = await load_node_ops(request, node) + lines = ops.get("syslog") + if not isinstance(lines, list): + return [] + return [dict(item) for item in lines[:limit] if isinstance(item, dict)] async def netstat(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: node = str(values(inputs)["node"]) await require_node(request, node) - return [ - {"in": 1_000_000, "out": 900_000, "vnet": "vmbr0", "hwaddr": "bc:24:11:00:00:01"}, - {"in": 500_000, "out": 450_000, "vnet": "vmbr1", "hwaddr": "bc:24:11:00:00:02"}, - ] + ops = await load_node_ops(request, node) + items = ops.get("netstat") + return [dict(item) for item in items] if isinstance(items, list) else [] async def report(request: Request, inputs: dict[str, Any]) -> str: node = str(values(inputs)["node"]) await require_node(request, node) - return f"==== Proxmox node report for {node} ====\nuptime: simulated\n" + ops = await load_node_ops(request, node) + report_text = ops.get("report") + return str(report_text) if report_text is not None else "" - async def rrd(_request: Request, inputs: dict[str, Any]) -> dict[str, Any]: - await require_node(_request, str(values(inputs)["node"])) - return {"filename": "/var/lib/rrdcached/db/pve-node.rrd"} + async def rrd(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: + node = str(values(inputs)["node"]) + await require_node(request, node) + ops = await load_node_ops(request, node) + rrd_state = ops.get("rrd") + return dict(rrd_state) if isinstance(rrd_state, dict) else {} - async def rrddata(_request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: - await require_node(_request, str(values(inputs)["node"])) - now = int(time.time()) - return [ - {"time": now - 120, "cpu": 0.05, "memused": 1_000_000_000}, - {"time": now - 60, "cpu": 0.07, "memused": 1_100_000_000}, - {"time": now, "cpu": 0.04, "memused": 1_050_000_000}, - ] + async def rrddata(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: + node = str(values(inputs)["node"]) + await require_node(request, node) + ops = await load_node_ops(request, node) + series = ops.get("rrddata") + return [dict(item) for item in series] if isinstance(series, list) else [] async def startall(request: Request, inputs: dict[str, Any]) -> str: node = str(values(inputs)["node"]) @@ -809,7 +719,7 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: await require_node(request, node) ops = await load_node_ops(request, node) if not isinstance(ops.get("network"), list): - ops["network"] = copy.deepcopy(default_node_ops()["network"]) + ops["network"] = [] ops["network_applied"] = False await save_node_ops(request, node, ops) @@ -818,28 +728,24 @@ def register_nodes_extra_handlers(registry: HandlerRegistry) -> None: await require_node(request, node) repo = str(values(inputs).get("repo") or "library/alpine") ops = await load_node_ops(request, node) - cache = ops.setdefault("oci_tags", {}) - if repo not in cache: - cache[repo] = [{"tag": "latest"}, {"tag": "3.20"}] - ops["oci_tags"] = cache - await save_node_ops(request, node, ops) - return [dict(item) for item in cache[repo]] + cache = ops.get("oci_tags") + if not isinstance(cache, dict): + return [] + items = cache.get(repo) + if not isinstance(items, list): + return [] + return [dict(item) for item in items if isinstance(item, dict)] async def query_url_metadata(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: node = str(values(inputs)["node"]) await require_node(request, node) url = str(values(inputs).get("url") or "") ops = await load_node_ops(request, node) - cache = ops.setdefault("url_metadata", {}) - if url not in cache: - cache[url] = { - "filename": url.rsplit("/", 1)[-1] or "download.bin", - "mimetype": "application/octet-stream", - "size": 1024, - } - ops["url_metadata"] = cache - await save_node_ops(request, node, ops) - return dict(cache[url]) + cache = ops.get("url_metadata") + if not isinstance(cache, dict): + return {} + payload = cache.get(url) + return dict(payload) if isinstance(payload, dict) else {} async def vncwebsocket(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: node = str(values(inputs)["node"]) diff --git a/app/handlers/notifications.py b/app/handlers/notifications.py index 52a5e67..37fbed7 100644 --- a/app/handlers/notifications.py +++ b/app/handlers/notifications.py @@ -13,53 +13,40 @@ from app.handlers.common import cluster_metadata, save_cluster_metadata, subdirs _SECRET_KEYS = frozenset({"token", "password", "secret"}) -DEFAULT_MATCHER_FIELDS = [ - {"name": "type", "type": "string"}, - {"name": "hostname", "type": "string"}, - {"name": "job-id", "type": "string"}, - {"name": "severity", "type": "string"}, -] - -DEFAULT_MATCHER_FIELD_VALUES = [ - {"field": "type", "value": "fencing"}, - {"field": "type", "value": "package-updates"}, - {"field": "type", "value": "replication"}, - {"field": "type", "value": "system-mail"}, -] - def _public(endpoint: dict[str, Any]) -> dict[str, Any]: return {key: value for key, value in endpoint.items() if key not in _SECRET_KEYS} def _notifications(metadata: dict[str, Any]) -> dict[str, Any]: - current = metadata.setdefault( - "notifications", - { - "endpoints": { - "gotify": {}, - "sendmail": {}, - "smtp": {}, - "webhook": {}, - }, - "matchers": {}, - "tests": [], - }, - ) + current = metadata.get("notifications") if not isinstance(current, dict): - current = { + return { "endpoints": {"gotify": {}, "sendmail": {}, "smtp": {}, "webhook": {}}, "matchers": {}, "tests": [], } - metadata["notifications"] = current - current.setdefault( - "endpoints", - {"gotify": {}, "sendmail": {}, "smtp": {}, "webhook": {}}, + endpoints = current.get("endpoints") + if not isinstance(endpoints, dict): + endpoints = {} + result = { + key: value + for key, value in current.items() + if key not in {"endpoints", "matchers", "tests"} + } + result["endpoints"] = { + "gotify": dict(endpoints["gotify"]) if isinstance(endpoints.get("gotify"), dict) else {}, + "sendmail": ( + dict(endpoints["sendmail"]) if isinstance(endpoints.get("sendmail"), dict) else {} + ), + "smtp": dict(endpoints["smtp"]) if isinstance(endpoints.get("smtp"), dict) else {}, + "webhook": dict(endpoints["webhook"]) if isinstance(endpoints.get("webhook"), dict) else {}, + } + result["matchers"] = ( + dict(current["matchers"]) if isinstance(current.get("matchers"), dict) else {} ) - current.setdefault("matchers", {}) - current.setdefault("tests", []) - return current + result["tests"] = list(current["tests"]) if isinstance(current.get("tests"), list) else [] + return result def register_notifications_handlers(registry: HandlerRegistry) -> None: @@ -80,26 +67,28 @@ def register_notifications_handlers(registry: HandlerRegistry) -> None: async def list_endpoints(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: metadata = await cluster_metadata(request) - store = _notifications(metadata)["endpoints"].setdefault(kind, {}) + store = _notifications(metadata)["endpoints"].get(kind) or {} return [_public({"name": name, **item}) for name, item in sorted(store.items())] async def create(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) name = str(payload["name"]) metadata = await cluster_metadata(request) - store = _notifications(metadata)["endpoints"].setdefault(kind, {}) + notifications = _notifications(metadata) + store = notifications["endpoints"].setdefault(kind, {}) if name in store: raise ApiError(400, f"{kind} endpoint '{name}' already exists") entry = {key: payload[key] for key in create_keys if key in payload} entry["name"] = name entry.setdefault("disable", 0) store[name] = entry + metadata["notifications"] = notifications await save_cluster_metadata(request, metadata) async def get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: name = str(values(inputs)["name"]) metadata = await cluster_metadata(request) - store = _notifications(metadata)["endpoints"].setdefault(kind, {}) + store = _notifications(metadata)["endpoints"].get(kind) or {} if name not in store: raise ApiError(404, f"{kind} endpoint does not exist") return _public({"name": name, **store[name]}) @@ -108,7 +97,8 @@ def register_notifications_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) name = str(payload["name"]) metadata = await cluster_metadata(request) - store = _notifications(metadata)["endpoints"].setdefault(kind, {}) + notifications = _notifications(metadata) + store = notifications["endpoints"].setdefault(kind, {}) if name not in store: raise ApiError(404, f"{kind} endpoint does not exist") current = dict(store[name]) @@ -123,15 +113,18 @@ def register_notifications_handlers(registry: HandlerRegistry) -> None: current[key] = value current["name"] = name store[name] = current + metadata["notifications"] = notifications await save_cluster_metadata(request, metadata) async def delete(request: Request, inputs: dict[str, Any]) -> None: name = str(values(inputs)["name"]) metadata = await cluster_metadata(request) - store = _notifications(metadata)["endpoints"].setdefault(kind, {}) + notifications = _notifications(metadata) + store = notifications["endpoints"].setdefault(kind, {}) if name not in store: raise ApiError(404, f"{kind} endpoint does not exist") del store[name] + metadata["notifications"] = notifications await save_cluster_metadata(request, metadata) registry.register(base, "GET", list_endpoints) @@ -149,12 +142,14 @@ def register_notifications_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) name = str(payload["name"]) metadata = await cluster_metadata(request) - store = _notifications(metadata)["matchers"] + notifications = _notifications(metadata) + store = notifications["matchers"] if name in store: raise ApiError(400, f"matcher '{name}' already exists") store[name] = { key: value for key, value in payload.items() if key not in {"delete", "digest"} } + metadata["notifications"] = notifications await save_cluster_metadata(request, metadata) async def matchers_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: @@ -169,7 +164,8 @@ def register_notifications_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) name = str(payload["name"]) metadata = await cluster_metadata(request) - store = _notifications(metadata)["matchers"] + notifications = _notifications(metadata) + store = notifications["matchers"] if name not in store: raise ApiError(404, "matcher does not exist") current = dict(store[name]) @@ -183,24 +179,35 @@ def register_notifications_handlers(registry: HandlerRegistry) -> None: current[key] = value current["name"] = name store[name] = current + metadata["notifications"] = notifications await save_cluster_metadata(request, metadata) async def matchers_delete(request: Request, inputs: dict[str, Any]) -> None: name = str(values(inputs)["name"]) metadata = await cluster_metadata(request) - store = _notifications(metadata)["matchers"] + notifications = _notifications(metadata) + store = notifications["matchers"] if name not in store: raise ApiError(404, "matcher does not exist") del store[name] + metadata["notifications"] = notifications await save_cluster_metadata(request, metadata) - async def matcher_fields(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: - return list(DEFAULT_MATCHER_FIELDS) + async def matcher_fields(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: + metadata = await cluster_metadata(request) + fields = _notifications(metadata).get("matcher_fields") + if isinstance(fields, list): + return [dict(item) for item in fields if isinstance(item, dict)] + return [] async def matcher_field_values( - _request: Request, _inputs: dict[str, Any] + request: Request, _inputs: dict[str, Any] ) -> list[dict[str, Any]]: - return list(DEFAULT_MATCHER_FIELD_VALUES) + metadata = await cluster_metadata(request) + values_list = _notifications(metadata).get("matcher_field_values") + if isinstance(values_list, list): + return [dict(item) for item in values_list if isinstance(item, dict)] + return [] async def targets(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: metadata = await cluster_metadata(request) @@ -231,10 +238,10 @@ def register_notifications_handlers(registry: HandlerRegistry) -> None: break if not found: raise ApiError(404, "notification target does not exist") - tests = notifications.setdefault("tests", []) - if not isinstance(tests, list): - tests = notifications["tests"] = [] + tests = list(notifications.get("tests") or []) tests.append({"name": name, "tested_at": int(time.time()), "ok": True}) + notifications["tests"] = tests + metadata["notifications"] = notifications await save_cluster_metadata(request, metadata) registry.register("/cluster/notifications", "GET", index) diff --git a/app/handlers/pools.py b/app/handlers/pools.py index 9d65005..4863f5d 100644 --- a/app/handlers/pools.py +++ b/app/handlers/pools.py @@ -10,7 +10,7 @@ from fastapi import Request from app.api.errors import ApiError from app.api.registry import HandlerRegistry -from app.handlers.common import database, state, values +from app.handlers.common import database, require_value, state, values from app.simulation.seed import CLUSTER_ID, stable_id @@ -117,7 +117,7 @@ def register_pool_handlers(registry: HandlerRegistry) -> None: ) async def pool_delete(request: Request, inputs: dict[str, Any]) -> None: - poolid = str(values(inputs)["poolid"]) + poolid = str(require_value(values(inputs), "poolid")) status = await database(request).pool.execute( "DELETE FROM pools WHERE pool_id=$1", poolid, diff --git a/app/handlers/qemu.py b/app/handlers/qemu.py index c16dd7b..8b39090 100644 --- a/app/handlers/qemu.py +++ b/app/handlers/qemu.py @@ -33,6 +33,51 @@ def _state(value: object) -> dict[str, Any]: return dict(cast(Mapping[str, Any], value)) +# Runtime-only keys stored in ``resources.state``; never echo on GET /config. +_QEMU_CONFIG_INTERNAL = frozenset( + { + "agent", + "cloudinit_dump", + "config", + "interfaces", + "migrate_preconditions", + "rrd", + "rrddata", + "sendkey", + "status", + } +) + + +def _agent_config_string(value: object) -> str: + """Proxmox QEMU config ``agent`` is a string (e.g. ``1`` / ``0``), not an object.""" + + if isinstance(value, dict): + return "1" if value.get("enabled", True) else "0" + if isinstance(value, bool): + return "1" if value else "0" + if value in (None, ""): + return "0" + text = str(value).strip() + return text or "0" + + +def _public_qemu_config(config: dict[str, Any], state: dict[str, Any]) -> dict[str, Any]: + """Wire-format QEMU config for clients (bpg / pulumi-proxmoxve).""" + + merged = {**state, **config} + payload: dict[str, Any] = {} + for key, value in merged.items(): + if key in _QEMU_CONFIG_INTERNAL: + continue + if isinstance(value, dict | list): + continue + payload[key] = value + agent = config.get("agent", state.get("agent", "0")) + payload["agent"] = _agent_config_string(agent) + return payload + + def register_qemu_handlers(registry: HandlerRegistry) -> None: async def qemu_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: node = str(_values(inputs)["node"]) @@ -117,7 +162,10 @@ def register_qemu_handlers(registry: HandlerRegistry) -> None: ) if row is None: raise ApiError(404, "virtual machine does not exist") - return {"vmid": int(vmid), **_state(row["config"]), **_state(row["state"])} + return { + "vmid": int(vmid), + **_public_qemu_config(_state(row["config"]), _state(row["state"])), + } async def mutate(operation: str, request: Request, inputs: dict[str, Any]) -> str: values = _values(inputs) @@ -406,7 +454,7 @@ def register_qemu_handlers(registry: HandlerRegistry) -> None: async def migrate_preconditions(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: values = _values(inputs) - await _qemu_resource(request, str(values["node"]), str(values["vmid"])) + resource = await _qemu_resource(request, str(values["node"]), str(values["vmid"])) target = values.get("target") if target in {None, ""}: raise ApiError(400, "parameter 'target' is required") @@ -416,7 +464,13 @@ def register_qemu_handlers(registry: HandlerRegistry) -> None: ) if not exists: raise ApiError(404, "target node does not exist") - return {"local_disks": [], "local_resources": [], "running": False} + state = _state(resource["state"]) + pre = state.get("migrate_preconditions") + payload = dict(pre) if isinstance(pre, dict) else {} + payload["running"] = str(state.get("status") or "") == "running" + payload.setdefault("local_disks", []) + payload.setdefault("local_resources", []) + return payload async def migrate(request: Request, inputs: dict[str, Any]) -> str: values = _values(inputs) @@ -544,39 +598,16 @@ def register_qemu_handlers(registry: HandlerRegistry) -> None: command: str, request: Request, inputs: dict[str, Any] ) -> dict[str, Any]: resource = await _agent_resource(request, _values(inputs)) - config = _state(resource["config"]) - vmid = str(_values(inputs)["vmid"]) - results: dict[str, Any] = { - "info": { - "version": "9.2.0-simulator", - "supported_commands": [ - {"name": name, "enabled": True, "success-response": True} - for name in ("guest-ping", "guest-info", "guest-get-osinfo") - ], - }, - "get-osinfo": { - "name": str(config.get("ostype", "linux")), - "pretty-name": "Proxmox Simulator Guest", - "version": "1.0", - "machine": "x86_64", - }, - "get-host-name": {"host-name": str(config.get("name", f"vm-{vmid}"))}, - "network-get-interfaces": [ - { - "name": "eth0", - "hardware-address": "02:00:00:00:00:01", - "ip-addresses": [ - {"ip-address": "192.0.2.10", "ip-address-type": "ipv4", "prefix": 24} - ], - } - ], - "ping": {}, - } if command == "get-time": seconds = int( await _database(request).pool.fetchval("SELECT extract(epoch from now())::bigint") ) return {"result": {"seconds": seconds, "nanoseconds": 0}} + state = _state(resource["state"]) + agent = state.get("agent") + results = agent.get("results") if isinstance(agent, dict) else None + if not isinstance(results, dict) or command not in results: + raise ApiError(404, f"agent command '{command}' has no stored result") return {"result": results[command]} async def agent_info(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: @@ -586,6 +617,12 @@ def register_qemu_handlers(registry: HandlerRegistry) -> None: return await agent_result("get-osinfo", request, inputs) async def agent_hostname(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: + resource = await _agent_resource(request, _values(inputs)) + state = _state(resource["state"]) + config = _state(resource["config"]) + name = state.get("name") or config.get("name") + if name: + return {"result": {"host-name": str(name)}} return await agent_result("get-host-name", request, inputs) async def agent_network(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: @@ -825,7 +862,14 @@ async def _agent_resource(request: Request, values: dict[str, Any]) -> Any: resource = await _qemu_resource(request, str(values["node"]), str(values["vmid"])) config = _state(resource["config"]) state = _state(resource["state"]) - if str(config.get("agent", "0")).split(",", 1)[0].lower() not in {"1", "true", "yes"}: + agent = state.get("agent") + if not isinstance(agent, dict): + agent = config.get("agent") + if isinstance(agent, dict): + enabled = bool(agent.get("enabled", True)) + else: + enabled = str(config.get("agent", "0")).split(",", 1)[0].lower() in {"1", "true", "yes"} + if not enabled: raise ApiError(409, "QEMU guest agent is not enabled") if state.get("status") != "running": raise ApiError(409, "QEMU guest agent is not running") diff --git a/app/handlers/qemu_extra.py b/app/handlers/qemu_extra.py index b622b88..0de74c6 100644 --- a/app/handlers/qemu_extra.py +++ b/app/handlers/qemu_extra.py @@ -79,20 +79,10 @@ def register_qemu_extra_handlers(registry: HandlerRegistry) -> None: async def _agent_blob(command: str, request: Request, inputs: dict[str, Any]) -> dict[str, Any]: resource = await _agent_resource(request, _values(inputs)) state = _state(resource["state"]) - agent = state.setdefault("agent", {}) - blobs = agent.setdefault("results", {}) - defaults: dict[str, Any] = { - "get-users": [{"user": "root", "login-time": 0}], - "get-fsinfo": [{"name": "/", "type": "ext4", "total-bytes": 32 * 1024**3}], - "get-memory-block-info": {"size": 1024**3}, - "get-memory-blocks": [{"start": 0, "size": 1024**3}], - "get-timezone": {"zone": "UTC", "offset": 0}, - "get-vcpus": [{"online": True, "can-offline": False}], - "fsfreeze-status": "thawed", - } - if command not in blobs: - blobs[command] = defaults.get(command, {}) - await _save_guest_state(request, resource["id"], state) + agent = state.get("agent") + blobs = agent.get("results") if isinstance(agent, dict) else None + if not isinstance(blobs, dict) or command not in blobs: + raise ApiError(404, f"agent command '{command}' has no stored result") return {"result": blobs[command]} async def agent_users(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: @@ -146,10 +136,10 @@ def register_qemu_extra_handlers(registry: HandlerRegistry) -> None: resource = await _agent_resource(request, payload) path = str(payload.get("file") or payload.get("path") or "/etc/hostname") state = _state(resource["state"]) - files = state.setdefault("agent", {}).setdefault("files", {}) - if path not in files: - files[path] = f"simulated:{path}\n" - await _save_guest_state(request, resource["id"], state) + agent = state.get("agent") + files = agent.get("files") if isinstance(agent, dict) else None + if not isinstance(files, dict) or path not in files: + raise ApiError(404, "file does not exist") content = str(files[path]) return {"result": {"content": content, "truncated": True}} @@ -188,9 +178,15 @@ def register_qemu_extra_handlers(registry: HandlerRegistry) -> None: async def agent_fstrim(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: resource = await _agent_resource(request, _values(inputs)) state = _state(resource["state"]) - state.setdefault("agent", {})["last_fstrim"] = True + agent = dict(state.get("agent") or {}) + agent["last_fstrim"] = True + state["agent"] = agent await _save_guest_state(request, resource["id"], state) - return {"result": {"paths": [{"path": "/", "trimmed": 0}]}} + results = agent.get("results") if isinstance(agent.get("results"), dict) else {} + fstrim = results.get("fstrim") if isinstance(results, dict) else None + if isinstance(fstrim, dict): + return {"result": fstrim} + return {"result": {}} async def agent_set_password(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: payload = _values(inputs) @@ -258,45 +254,23 @@ def register_qemu_extra_handlers(registry: HandlerRegistry) -> None: async def cloudinit_dump(request: Request, inputs: dict[str, Any]) -> str: values = _values(inputs) resource = await _qemu_resource(request, str(values["node"]), str(values["vmid"])) - config = _state(resource["config"]) - return ( - f"#cloud-config\nhostname: {config.get('name', values['vmid'])}\n" - f"manage_etc_hosts: true\n" - ) + state = _state(resource["state"]) + dump = state.get("cloudinit_dump") + return str(dump) if dump is not None else "" async def rrd(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: values = _values(inputs) resource = await _qemu_resource(request, str(values["node"]), str(values["vmid"])) state = _state(resource["state"]) - rrd_state = state.setdefault("rrd", {"filename": f"pve-vm-{values['vmid']}.rrd"}) - await _save_guest_state(request, resource["id"], state) - return dict(rrd_state) + rrd_state = state.get("rrd") + return dict(rrd_state) if isinstance(rrd_state, dict) else {} async def rrddata(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: values = _values(inputs) resource = await _qemu_resource(request, str(values["node"]), str(values["vmid"])) state = _state(resource["state"]) - series = state.setdefault( - "rrddata", - [ - { - "time": 1_700_000_000, - "cpu": 0.05, - "mem": 256 * 1024 * 1024, - "netin": 0, - "netout": 0, - }, - { - "time": 1_700_000_060, - "cpu": 0.08, - "mem": 260 * 1024 * 1024, - "netin": 100, - "netout": 80, - }, - ], - ) - await _save_guest_state(request, resource["id"], state) - return list(series) + series = state.get("rrddata") + return [dict(item) for item in series] if isinstance(series, list) else [] async def monitor(request: Request, inputs: dict[str, Any]) -> str: values = _values(inputs) diff --git a/app/handlers/sdn.py b/app/handlers/sdn.py index 844eecc..32f1fd6 100644 --- a/app/handlers/sdn.py +++ b/app/handlers/sdn.py @@ -21,52 +21,8 @@ _SECRET_KEYS = frozenset({"key", "token", "fingerprint"}) def _sdn(metadata: dict[str, Any]) -> dict[str, Any]: - current = metadata.setdefault( - "sdn", - { - "zones": {}, - "vnets": {}, - "controllers": {}, - "dns": {}, - "ipams": {}, - "fabrics": {}, - "fabric_nodes": {}, - "prefix_lists": {}, - "route_maps": {}, - "lock": None, - "pending": False, - "running_version": 1, - }, - ) - if not isinstance(current, dict): - current = { - "zones": {}, - "vnets": {}, - "controllers": {}, - "dns": {}, - "ipams": {}, - "fabrics": {}, - "fabric_nodes": {}, - "prefix_lists": {}, - "route_maps": {}, - "lock": None, - "pending": False, - "running_version": 1, - } - metadata["sdn"] = current - for key in ( - "zones", - "vnets", - "controllers", - "dns", - "ipams", - "fabrics", - "fabric_nodes", - "prefix_lists", - "route_maps", - ): - current.setdefault(key, {}) - return current + current = metadata.get("sdn") + return current if isinstance(current, dict) else {} def _public(item: dict[str, Any]) -> dict[str, Any]: @@ -77,9 +33,14 @@ def _store_list(store: dict[str, Any], *, id_key: str) -> list[dict[str, Any]]: return [_public({id_key: name, **item}) for name, item in sorted(store.items())] -async def _load(request: Request) -> tuple[dict[str, Any], dict[str, Any]]: +async def _load( + request: Request, *, for_write: bool = False +) -> tuple[dict[str, Any], dict[str, Any]]: metadata = await cluster_metadata(request) - return metadata, _sdn(metadata) + sdn = _sdn(metadata) + if for_write and metadata.get("sdn") is not sdn: + metadata["sdn"] = sdn + return metadata, sdn def register_sdn_handlers(registry: HandlerRegistry) -> None: @@ -100,7 +61,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def apply(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) lock = sdn.get("lock") token = payload.get("lock-token") if lock and token and lock.get("token") != token: @@ -112,7 +73,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: await save_cluster_metadata(request, metadata) async def lock_create(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) if sdn.get("lock") and not values(inputs).get("allow-pending"): raise ApiError(400, "SDN is already locked") token = secrets.token_hex(8) @@ -122,7 +83,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def lock_delete(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) lock = sdn.get("lock") if lock is None: return None @@ -133,14 +94,14 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def rollback(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) sdn["pending"] = False if payload.get("release-lock"): sdn["lock"] = None await save_cluster_metadata(request, metadata) async def dry_run(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) return [ {"type": "zone", "name": name, "action": "noop"} for name in sorted(sdn.get("zones") or {}) @@ -154,7 +115,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: create_required: str | None = None, ) -> None: async def list_items(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) items = _store_list(sdn.get(store_key) or {}, id_key=id_param) type_filter = values(inputs).get("type") if type_filter: @@ -164,7 +125,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def create(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) item_id = str(payload[create_required or id_param]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) store = sdn.setdefault(store_key, {}) if item_id in store: raise ApiError(400, f"{store_key} '{item_id}' already exists") @@ -179,7 +140,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: item_id = str(values(inputs)[id_param]) - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) item = (sdn.get(store_key) or {}).get(item_id) if not isinstance(item, dict): raise ApiError(404, f"{store_key} entry does not exist") @@ -188,7 +149,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def update(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) item_id = str(payload[id_param]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) store = sdn.setdefault(store_key, {}) if item_id not in store: raise ApiError(404, f"{store_key} entry does not exist") @@ -208,7 +169,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def delete(request: Request, inputs: dict[str, Any]) -> None: item_id = str(values(inputs)[id_param]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) store = sdn.setdefault(store_key, {}) if item_id not in store: raise ApiError(404, f"{store_key} entry does not exist") @@ -230,20 +191,20 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def ipam_status(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: ipam = str(values(inputs)["ipam"]) - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) if ipam not in (sdn.get("ipams") or {}): raise ApiError(404, "ipam does not exist") return {"status": "ok", "ipam": ipam} # vnets + nested async def vnets_list(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) return _store_list(sdn.get("vnets") or {}, id_key="vnet") async def vnets_create(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) vnet = str(payload["vnet"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) store = sdn.setdefault("vnets", {}) if vnet in store: raise ApiError(400, f"vnet '{vnet}' already exists") @@ -259,7 +220,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def vnet_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: vnet = str(values(inputs)["vnet"]) - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) item = (sdn.get("vnets") or {}).get(vnet) if not isinstance(item, dict): raise ApiError(404, "vnet does not exist") @@ -268,7 +229,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def vnet_update(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) vnet = str(payload["vnet"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) store = sdn.setdefault("vnets", {}) if vnet not in store: raise ApiError(404, "vnet does not exist") @@ -288,7 +249,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def vnet_delete(request: Request, inputs: dict[str, Any]) -> None: vnet = str(values(inputs)["vnet"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) store = sdn.setdefault("vnets", {}) if vnet not in store: raise ApiError(404, "vnet does not exist") @@ -307,7 +268,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def subnets_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: vnet = str(values(inputs)["vnet"]) - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) item = await _vnet(sdn, vnet) return _store_list(item.get("subnets") or {}, id_key="subnet") @@ -315,7 +276,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) vnet = str(payload["vnet"]) subnet = str(payload["subnet"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) item = await _vnet(sdn, vnet) subnets = item.setdefault("subnets", {}) if subnet in subnets: @@ -331,7 +292,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) vnet = str(payload["vnet"]) subnet = str(payload["subnet"]) - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) item = await _vnet(sdn, vnet) data = (item.get("subnets") or {}).get(subnet) if not isinstance(data, dict): @@ -342,7 +303,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) vnet = str(payload["vnet"]) subnet = str(payload["subnet"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) item = await _vnet(sdn, vnet) subnets = item.setdefault("subnets", {}) if subnet not in subnets: @@ -365,7 +326,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) vnet = str(payload["vnet"]) subnet = str(payload["subnet"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) item = await _vnet(sdn, vnet) subnets = item.setdefault("subnets", {}) if subnet not in subnets: @@ -377,7 +338,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def ips_create(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) vnet = str(payload["vnet"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) item = await _vnet(sdn, vnet) ips = item.setdefault("ips", []) if not isinstance(ips, list): @@ -398,7 +359,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def ips_delete(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) vnet = str(payload["vnet"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) item = await _vnet(sdn, vnet) ips = item.setdefault("ips", []) if not isinstance(ips, list): @@ -416,14 +377,14 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def fw_options_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: vnet = str(values(inputs)["vnet"]) - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) item = await _vnet(sdn, vnet) return dict(item.get("firewall", {}).get("options") or {"enable": 0}) async def fw_options_put(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) vnet = str(payload["vnet"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) item = await _vnet(sdn, vnet) options = dict(item.setdefault("firewall", {}).setdefault("options", {"enable": 0})) for key, value in payload.items(): @@ -435,7 +396,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def fw_rules_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: vnet = str(values(inputs)["vnet"]) - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) item = await _vnet(sdn, vnet) rules = item.get("firewall", {}).get("rules") or [] return list(rules) if isinstance(rules, list) else [] @@ -443,7 +404,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def fw_rules_create(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) vnet = str(payload["vnet"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) item = await _vnet(sdn, vnet) rules = item.setdefault("firewall", {}).setdefault("rules", []) if not isinstance(rules, list): @@ -464,7 +425,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) vnet = str(payload["vnet"]) pos = int(payload["pos"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) item = await _vnet(sdn, vnet) rules = item.setdefault("firewall", {}).setdefault("rules", []) if not isinstance(rules, list) or pos < 0 or pos >= len(rules): @@ -479,7 +440,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) vnet = str(payload["vnet"]) pos = int(payload["pos"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) item = await _vnet(sdn, vnet) rules = item.setdefault("firewall", {}).setdefault("rules", []) if not isinstance(rules, list) or pos < 0 or pos >= len(rules): @@ -492,7 +453,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: return subdirs("all", "fabric", "node") async def fabrics_all(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) return _store_list(sdn.get("fabrics") or {}, id_key="id") async def fabric_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: @@ -501,7 +462,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def fabric_create(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) fabric_id = str(payload["id"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) store = sdn.setdefault("fabrics", {}) if fabric_id in store: raise ApiError(400, f"fabric '{fabric_id}' already exists") @@ -514,7 +475,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def fabric_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: fabric_id = str(values(inputs)["id"]) - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) item = (sdn.get("fabrics") or {}).get(fabric_id) if not isinstance(item, dict): raise ApiError(404, "fabric does not exist") @@ -523,7 +484,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def fabric_update(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) fabric_id = str(payload["id"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) store = sdn.setdefault("fabrics", {}) if fabric_id not in store: raise ApiError(404, "fabric does not exist") @@ -543,7 +504,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def fabric_delete(request: Request, inputs: dict[str, Any]) -> None: fabric_id = str(values(inputs)["id"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) store = sdn.setdefault("fabrics", {}) if fabric_id not in store: raise ApiError(404, "fabric does not exist") @@ -554,7 +515,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: await save_cluster_metadata(request, metadata) async def fabric_nodes_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) fabric_id = values(inputs).get("fabric_id") nodes = sdn.get("fabric_nodes") or {} result: list[dict[str, Any]] = [] @@ -571,7 +532,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) fabric_id = str(payload["fabric_id"]) node_id = str(payload["node_id"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) if fabric_id not in (sdn.get("fabrics") or {}): raise ApiError(404, "fabric does not exist") store = sdn.setdefault("fabric_nodes", {}).setdefault(fabric_id, {}) @@ -588,7 +549,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) fabric_id = str(payload["fabric_id"]) node_id = str(payload["node_id"]) - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) item = ((sdn.get("fabric_nodes") or {}).get(fabric_id) or {}).get(node_id) if not isinstance(item, dict): raise ApiError(404, "fabric node does not exist") @@ -598,7 +559,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) fabric_id = str(payload["fabric_id"]) node_id = str(payload["node_id"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) store = sdn.setdefault("fabric_nodes", {}).setdefault(fabric_id, {}) if node_id not in store: raise ApiError(404, "fabric node does not exist") @@ -620,7 +581,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) fabric_id = str(payload["fabric_id"]) node_id = str(payload["node_id"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) store = sdn.setdefault("fabric_nodes", {}).setdefault(fabric_id, {}) if node_id not in store: raise ApiError(404, "fabric node does not exist") @@ -630,13 +591,13 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: # prefix lists async def prefix_list(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]: - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) return _store_list(sdn.get("prefix_lists") or {}, id_key="id") async def prefix_create(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) list_id = str(payload["id"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) store = sdn.setdefault("prefix_lists", {}) if list_id in store: raise ApiError(400, f"prefix-list '{list_id}' already exists") @@ -654,7 +615,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def prefix_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: list_id = str(values(inputs)["id"]) - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) item = (sdn.get("prefix_lists") or {}).get(list_id) if not isinstance(item, dict): raise ApiError(404, "prefix-list does not exist") @@ -663,7 +624,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def prefix_update(request: Request, inputs: dict[str, Any]) -> None: payload = values(inputs) list_id = str(payload["id"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) store = sdn.setdefault("prefix_lists", {}) if list_id not in store: raise ApiError(404, "prefix-list does not exist") @@ -676,7 +637,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def prefix_delete(request: Request, inputs: dict[str, Any]) -> None: list_id = str(values(inputs)["id"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) store = sdn.setdefault("prefix_lists", {}) if list_id not in store: raise ApiError(404, "prefix-list does not exist") @@ -686,7 +647,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def prefix_entries(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: list_id = str(values(inputs)["id"]) - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) item = (sdn.get("prefix_lists") or {}).get(list_id) if not isinstance(item, dict): raise ApiError(404, "prefix-list does not exist") @@ -699,7 +660,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) list_id = str(payload["id"]) seq = str(payload.get("seq") or secrets.randbelow(10000)) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) item = (sdn.get("prefix_lists") or {}).get(list_id) if not isinstance(item, dict): raise ApiError(404, "prefix-list does not exist") @@ -720,7 +681,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) list_id = str(payload["id"]) seq = str(payload["url_seq"]) - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) item = (sdn.get("prefix_lists") or {}).get(list_id) if not isinstance(item, dict): raise ApiError(404, "prefix-list does not exist") @@ -733,7 +694,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) list_id = str(payload["id"]) seq = str(payload["url_seq"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) item = (sdn.get("prefix_lists") or {}).get(list_id) if not isinstance(item, dict): raise ApiError(404, "prefix-list does not exist") @@ -752,7 +713,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) list_id = str(payload["id"]) seq = str(payload["url_seq"]) - metadata, sdn = await _load(request) + metadata, sdn = await _load(request, for_write=True) item = (sdn.get("prefix_lists") or {}).get(list_id) if not isinstance(item, dict): raise ApiError(404, "prefix-list does not exist") @@ -767,16 +728,44 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def route_maps_index(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, str]]: return subdirs("entries") + def _route_map_entries(item: object) -> dict[str, Any]: + if not isinstance(item, dict): + return {} + nested = item.get("entries") + if isinstance(nested, dict): + return { + str(key): value + for key, value in nested.items() + if isinstance(value, dict) and str(key).isdigit() + } + return { + str(key): value + for key, value in item.items() + if isinstance(value, dict) and str(key).isdigit() + } + + def _route_map_entries_mutable(sdn: dict[str, Any], map_id: str) -> dict[str, Any]: + bucket = sdn.setdefault("route_maps", {}).setdefault(map_id, {}) + if not isinstance(bucket, dict): + bucket = {} + sdn["route_maps"][map_id] = bucket + if "entries" in bucket or "name" in bucket: + entries = bucket.setdefault("entries", {}) + if not isinstance(entries, dict): + entries = {} + bucket["entries"] = entries + return entries + return bucket + async def route_entries_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) maps = sdn.get("route_maps") or {} route_map_id = values(inputs).get("route-map-id") result: list[dict[str, Any]] = [] - for map_id, entries in sorted(maps.items()): + for map_id, raw_entries in sorted(maps.items()): if route_map_id and map_id != route_map_id: continue - if not isinstance(entries, dict): - continue + entries = _route_map_entries(raw_entries) for order, entry in sorted(entries.items(), key=lambda pair: int(pair[0])): result.append({"route-map-id": map_id, "order": int(order), **entry}) return result @@ -785,8 +774,8 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) map_id = str(payload["route-map-id"]) order = str(payload.get("order") or 10) - metadata, sdn = await _load(request) - entries = sdn.setdefault("route_maps", {}).setdefault(map_id, {}) + metadata, sdn = await _load(request, for_write=True) + entries = _route_map_entries_mutable(sdn, map_id) if order in entries: raise ApiError(400, f"route-map entry '{order}' already exists") entries[order] = { @@ -809,8 +798,8 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) map_id = str(payload["route-map-id"]) order = str(payload["order"]) - _metadata, sdn = await _load(request) - entry = ((sdn.get("route_maps") or {}).get(map_id) or {}).get(order) + _metadata, sdn = await _load(request, for_write=True) + entry = _route_map_entries((sdn.get("route_maps") or {}).get(map_id)).get(order) if not isinstance(entry, dict): raise ApiError(404, "route-map entry does not exist") return {"route-map-id": map_id, "order": int(order), **entry} @@ -819,8 +808,8 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) map_id = str(payload["route-map-id"]) order = str(payload["order"]) - metadata, sdn = await _load(request) - entries = sdn.setdefault("route_maps", {}).setdefault(map_id, {}) + metadata, sdn = await _load(request, for_write=True) + entries = _route_map_entries_mutable(sdn, map_id) if order not in entries: raise ApiError(404, "route-map entry does not exist") current = dict(entries[order]) @@ -837,8 +826,8 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) map_id = str(payload["route-map-id"]) order = str(payload["order"]) - metadata, sdn = await _load(request) - entries = sdn.setdefault("route_maps", {}).setdefault(map_id, {}) + metadata, sdn = await _load(request, for_write=True) + entries = _route_map_entries_mutable(sdn, map_id) if order not in entries: raise ApiError(404, "route-map entry does not exist") del entries[order] @@ -852,13 +841,13 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def node_zones(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: await require_node(request, str(values(inputs)["node"])) - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) return _store_list(sdn.get("zones") or {}, id_key="zone") async def node_zone(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: await require_node(request, str(values(inputs)["node"])) zone = str(values(inputs)["zone"]) - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) item = (sdn.get("zones") or {}).get(zone) if not isinstance(item, dict): raise ApiError(404, "zone does not exist") @@ -866,13 +855,16 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def node_zone_bridges(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: zone = await node_zone(request, inputs) - bridge = zone.get("bridge") or f"vmbr-{zone.get('zone')}" - return [{"iface": bridge, "active": 1}] + bridges = zone.get("bridges") + if isinstance(bridges, list): + return [dict(item) for item in bridges if isinstance(item, dict)] + bridge = zone.get("bridge") + return [{"iface": bridge, "active": 1}] if bridge else [] async def node_zone_content(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: await require_node(request, str(values(inputs)["node"])) zone = str(values(inputs)["zone"]) - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) return [ {"vnet": name, **item} for name, item in sorted((sdn.get("vnets") or {}).items()) @@ -881,7 +873,11 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def node_zone_ip_vrf(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: zone = await node_zone(request, inputs) - return {"zone": zone.get("zone"), "vrf": f"vrf-{zone.get('zone')}", "table": 100} + return { + "zone": zone.get("zone"), + "vrf": zone.get("vrf") or "", + "table": zone.get("table") if zone.get("table") is not None else 0, + } async def node_vnet(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: await require_node(request, str(values(inputs)["node"])) @@ -889,7 +885,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: async def node_vnet_mac_vrf(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: vnet = await node_vnet(request, inputs) - return {"vnet": vnet.get("vnet"), "mac-vrf": f"macvrf-{vnet.get('vnet')}"} + return {"vnet": vnet.get("vnet"), "mac-vrf": vnet.get("mac-vrf") or ""} async def node_fabric(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: await require_node(request, str(values(inputs)["node"])) @@ -901,7 +897,7 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: ) -> list[dict[str, Any]]: await require_node(request, str(values(inputs)["node"])) fabric = str(values(inputs)["fabric"]) - _metadata, sdn = await _load(request) + _metadata, sdn = await _load(request, for_write=True) nodes = (sdn.get("fabric_nodes") or {}).get(fabric) or {} result = [] for node_id, item in nodes.items(): @@ -919,15 +915,23 @@ def register_sdn_handlers(registry: HandlerRegistry) -> None: fabric = str(values(inputs)["fabric"]) _metadata, sdn = await _load(request) nodes = (sdn.get("fabric_nodes") or {}).get(fabric) or {} - return [{"node": node_id, "state": "up"} for node_id in sorted(nodes)] + return [ + { + "node": node_id, + "state": item.get("state") if isinstance(item, dict) else "", + } + for node_id, item in sorted(nodes.items()) + ] async def node_fabric_routes(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: await require_node(request, str(values(inputs)["node"])) fabric = str(values(inputs)["fabric"]) _metadata, sdn = await _load(request) item = (sdn.get("fabrics") or {}).get(fabric) or {} - prefix = item.get("ip_prefix") or "10.0.0.0/24" - return [{"dst": prefix, "protocol": item.get("protocol") or "ospf"}] + routes = item.get("routes") if isinstance(item, dict) else None + if isinstance(routes, list): + return [dict(route) for route in routes if isinstance(route, dict)] + return [] # registrations registry.register("/cluster/sdn", "GET", index) diff --git a/app/handlers/storage.py b/app/handlers/storage.py index e1aa840..3acb266 100644 --- a/app/handlers/storage.py +++ b/app/handlers/storage.py @@ -9,7 +9,15 @@ from fastapi import Request from app.api.errors import ApiError from app.api.registry import HandlerRegistry -from app.handlers.common import database, require_node, state, storage_payload, subdirs, values +from app.handlers.common import ( + database, + require_node, + require_value, + state, + storage_payload, + subdirs, + values, +) from app.simulation.seed import CLUSTER_ID, stable_id @@ -485,18 +493,29 @@ def register_storage_handlers(registry: HandlerRegistry) -> None: async def file_restore_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: payload = values(inputs) await require_node(request, str(payload["node"])) - await _storage_row(request, None, str(payload["storage"])) + row = await _storage_row(request, None, str(payload["storage"])) filepath = str(payload.get("filepath") or "/") - return [{"filepath": filepath.rstrip("/") + "/etc", "type": "d", "text": "etc"}] + config = state(row["config"]) + restore = config.get("file_restore") + if not isinstance(restore, dict): + return [] + items = restore.get(filepath) or restore.get(filepath.rstrip("/") or "/") + return [dict(item) for item in items] if isinstance(items, list) else [] async def file_restore_download(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: payload = values(inputs) await require_node(request, str(payload["node"])) - await _storage_row(request, None, str(payload["storage"])) + row = await _storage_row(request, None, str(payload["storage"])) + config = state(row["config"]) + downloads = config.get("file_restore_downloads") + filepath = str(payload.get("filepath") or "/") + if isinstance(downloads, dict) and filepath in downloads: + entry = downloads[filepath] + return dict(entry) if isinstance(entry, dict) else {"filepath": filepath} return { "download-url": f"/api2/json/nodes/{payload['node']}/storage/" f"{payload['storage']}/file-restore/download", - "filepath": payload.get("filepath") or "/", + "filepath": filepath, "volume": payload.get("volume"), } @@ -504,40 +523,51 @@ def register_storage_handlers(registry: HandlerRegistry) -> None: payload = values(inputs) await require_node(request, str(payload["node"])) row = await _storage_row(request, None, str(payload["storage"])) + config = state(row["config"]) + identity = config.get("identity") + if isinstance(identity, dict): + return { + "storage": str(row["storage_id"]), + "type": str(identity.get("type") or row["storage_type"]), + "fingerprint": str(identity.get("fingerprint") or ""), + } return { "storage": str(row["storage_id"]), "type": str(row["storage_type"]), - "fingerprint": f"sim-{row['storage_id']}", + "fingerprint": "", } async def import_metadata(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: payload = values(inputs) await require_node(request, str(payload["node"])) storage_id = str(payload["storage"]) - volume = str(payload["volume"]) - await _storage_row(request, None, storage_id) - return { - "type": "qemu", - "source": volume, - "disks": {"scsi0": f"{storage_id}:0/vm-import.raw"}, - "net0": "virtio,bridge=vmbr0", - } + volume = str(require_value(payload, "volume")) + row = await _storage_row(request, None, storage_id) + config = state(row["config"]) + by_volume = config.get("import_metadata_by_volume") + if isinstance(by_volume, dict) and volume in by_volume: + meta = by_volume[volume] + return dict(meta) if isinstance(meta, dict) else {} + meta = config.get("import_metadata") + if isinstance(meta, dict): + return {"source": volume, **meta} + return {} async def storage_rrd(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: payload = values(inputs) await require_node(request, str(payload["node"])) - storage_id = str(payload["storage"]) - await _storage_row(request, None, storage_id) - return {"filename": f"pve-storage-{storage_id}.rrd"} + row = await _storage_row(request, None, str(payload["storage"])) + config = state(row["config"]) + rrd_state = config.get("rrd") + return dict(rrd_state) if isinstance(rrd_state, dict) else {} async def storage_rrddata(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: payload = values(inputs) await require_node(request, str(payload["node"])) - await _storage_row(request, None, str(payload["storage"])) - return [ - {"time": 1_700_000_000, "used": 10, "total": 100}, - {"time": 1_700_000_060, "used": 12, "total": 100}, - ] + row = await _storage_row(request, None, str(payload["storage"])) + config = state(row["config"]) + series = config.get("rrddata") + return [dict(item) for item in series] if isinstance(series, list) else [] registry.register("/storage", "GET", storage_ids) registry.register("/storage", "POST", storage_create) diff --git a/app/simulation/seed.py b/app/simulation/seed.py index 4b0b49d..038d928 100644 --- a/app/simulation/seed.py +++ b/app/simulation/seed.py @@ -16,21 +16,422 @@ CLUSTER_ID = uuid.UUID("dc760c47-d8d7-57e6-9404-f0c6f2395d8f") def default_node_ops_for_seed(node_name: str) -> dict[str, object]: - from app.handlers.nodes import default_node_ops + """Full durable node ops written to nodes.metadata at seed time.""" - ops = default_node_ops() - # Distinct but deterministic bridge addresses per node name. suffix = (stable_id(f"node-ip:{node_name}").int % 200) + 10 - network = ops.get("network") - if isinstance(network, list): - for item in network: - if not isinstance(item, dict): - continue - if item.get("iface") == "vmbr0": - item["address"] = f"10.0.0.{suffix}/24" - elif item.get("iface") == "vmbr1": - item["address"] = f"10.10.0.{suffix}/24" - return ops + return { + "network": [ + { + "iface": "vmbr0", + "type": "bridge", + "active": 1, + "method": "static", + "address": f"10.0.0.{suffix}/24", + }, + { + "iface": "vmbr1", + "type": "bridge", + "active": 1, + "method": "static", + "address": f"10.10.0.{suffix}/24", + }, + {"iface": "eno1", "type": "eth", "active": 1, "method": "manual"}, + ], + "disks": { + "list": [ + { + "devpath": "/dev/sda", + "size": 1_000_000_000_000, + "model": "SIM-DISK-01", + "serial": "SIM0001", + "gpt": 1, + }, + { + "devpath": "/dev/sdb", + "size": 2_000_000_000_000, + "model": "SIM-SSD-01", + "serial": "SIM0002", + "gpt": 0, + }, + ], + "directory": [], + "lvm": [], + "lvmthin": [], + "zfs": [], + "smart": { + "/dev/sda": { + "health": "PASSED", + "type": "scsi", + "model": "SIM-DISK-01", + "serial": "SIM0001", + }, + "/dev/sdb": { + "health": "PASSED", + "type": "scsi", + "model": "SIM-SSD-01", + "serial": "SIM0002", + }, + }, + }, + "services": { + "pveproxy": {"state": "running", "enabled": 1}, + "pvedaemon": {"state": "running", "enabled": 1}, + "pvestatd": {"state": "running", "enabled": 1}, + "corosync": {"state": "running", "enabled": 1}, + }, + "apt": { + "packages": [ + { + "Package": "pve-manager", + "Version": "9.2.3", + "OldVersion": "9.2.2", + "Status": "upgradable", + }, + {"Package": "libpve-common-perl", "Version": "9.0.3", "Status": "installed"}, + ], + "repositories": [ + { + "path": "/etc/apt/sources.list.d/pve-enterprise.list", + "enabled": 1, + "types": "deb", + "uri": "http://download.proxmox.com/debian/pve", + "suites": "bookworm", + "components": "pve-no-subscription", + } + ], + "update": {"status": "stopped", "exitstatus": "OK"}, + "changelogs": { + "pve-manager": "pve-manager (9.2.3) bookworm\n\n * simulator seed\n", + }, + }, + "hardware": { + "pci": [ + { + "id": "0000:00:1f.2", + "vendor_name": "Intel Corporation", + "device_name": "SATA Controller", + "iommugroup": 0, + }, + { + "id": "0000:01:00.0", + "vendor_name": "NVIDIA Corporation", + "device_name": "GP102 [GeForce GTX 1080 Ti]", + "iommugroup": 1, + "mdev": 1, + }, + ], + "usb": [ + { + "busnum": 1, + "devnum": 1, + "level": 0, + "port": "1", + "prodid": "0002", + "vendid": "1d6b", + }, + { + "busnum": 2, + "devnum": 2, + "level": 1, + "port": "2", + "prodid": "5591", + "vendid": "0781", + }, + ], + "mdev": { + "0000:01:00.0": [ + {"type": "nvidia-11", "available": 4, "description": "GRID profile"}, + ] + }, + }, + "scan": { + "cifs": [{"server": "files.local", "share": "backups"}], + "iscsi": [{"portal": "10.0.0.50:3260", "target": "iqn.2024-01.local:storage"}], + "lvm": [{"vg": "pve", "size": 500_000_000_000, "free": 100_000_000_000}], + "lvmthin": [{"lv": "data", "vg": "pve", "lv_size": 400_000_000_000}], + "nfs": [{"server": "nfs.local", "path": "/export/pve", "options": "vers=4"}], + "pbs": [{"server": "pbs.local", "datastore": "store1"}], + "zfs": [{"pool": "rpool", "name": "rpool/data", "size": 800_000_000_000}], + }, + "subscription": { + "status": "notfound", + "message": "There is no subscription key", + "serverid": "SIMULATOR", + "sockets": 1, + "productname": "Proxmox VE", + "url": "https://www.proxmox.com/en/proxmox-virtual-environment/pricing", + }, + "dns": {"search": "local", "dns1": "1.1.1.1", "dns2": "8.8.8.8", "dns3": ""}, + "time": {"timezone": "UTC", "time": 1_720_000_000, "localtime": 1_720_000_000}, + "config": { + "description": "Simulator node", + "startall-onboot-delay": 0, + "wakeonlan": "", + }, + "certificates": { + "custom": None, + "acme": {"account": "letsencrypt", "domains": [], "certificate": None}, + "info": [], + }, + "ceph": { + "mds": {}, + "mgr": { + node_name: { + "name": node_name, + "state": "active", + "addr": f"{node_name}.local:6800", + } + }, + "mon": { + node_name: { + "name": node_name, + "addr": f"{node_name}.local:6789", + "rank": 0, + } + }, + "log": [{"t": 1_700_000_000, "n": 0, "line": "ceph simulator ready"}], + }, + "aplinfo": [ + { + "package": "alpine-3-standard", + "section": "system", + "type": "lxc", + "version": "3.20", + } + ], + "oci_tags": {"library/alpine": [{"tag": "latest"}, {"tag": "3.20"}]}, + "capabilities": { + "cpu": [ + {"name": "host", "vendor": "QEMU", "custom": 0}, + {"name": "x86-64-v2-AES", "vendor": "QEMU", "custom": 0}, + {"name": "kvm64", "vendor": "QEMU", "custom": 0}, + ], + "cpu_flags": [ + {"name": "aes", "introduces": "Westmere"}, + {"name": "avx", "introduces": "SandyBridge"}, + {"name": "avx2", "introduces": "Haswell"}, + ], + "machines": [ + {"id": "pc-i440fx-9.0", "type": "i440fx", "version": "9.0"}, + {"id": "pc-q35-9.0", "type": "q35", "version": "9.0"}, + ], + "migration": {"network": "", "type": "secure", "enabled": 1}, + }, + "hosts": { + "data": f"127.0.0.1 localhost\n10.0.0.{suffix} {node_name}\n", + "digest": "seedhosts", + }, + "journal": [ + f"0: {node_name} systemd[1]: Started pveproxy.service.", + f"1: {node_name} systemd[1]: Started pvedaemon.service.", + f"2: {node_name} kernel: Booting simulator node {node_name}.", + ], + "syslog": [ + {"n": 0, "t": f"{node_name} kernel: simulator syslog ready"}, + {"n": 1, "t": f"{node_name} pvestatd[1]: status update ok"}, + ], + "netstat": [ + { + "in": 1_000_000, + "out": 900_000, + "vnet": "vmbr0", + "hwaddr": "bc:24:11:00:00:01", + }, + { + "in": 500_000, + "out": 450_000, + "vnet": "vmbr1", + "hwaddr": "bc:24:11:00:00:02", + }, + ], + "report": f"==== Proxmox node report for {node_name} ====\nuptime: seeded\n", + "rrd": {"filename": f"/var/lib/rrdcached/db/pve-node-{node_name}.rrd"}, + "rrddata": [ + {"time": 1_720_000_000 - 120, "cpu": 0.05, "memused": 1_000_000_000}, + {"time": 1_720_000_000 - 60, "cpu": 0.07, "memused": 1_100_000_000}, + {"time": 1_720_000_000, "cpu": 0.04, "memused": 1_050_000_000}, + ], + "vzdump": { + "defaults": { + "all": 0, + "bwlimit": 0, + "compress": "zstd", + "dumpdir": "backup", + "mode": "snapshot", + "remove": 0, + "storage": "nfs-backup", + "mailto": "", + "notes-template": "{{guestname}}", + }, + "extractconfig": { + "local:backup/vzdump-qemu-100.vma.zst": ( + "# vzdump config for local:backup/vzdump-qemu-100.vma.zst\n" + "name: demo\nmemory: 2048\n" + ), + }, + }, + "status": { + "uptime": 86_400 + suffix, + "cpu": 0.05 + (suffix % 10) * 0.01, + "memory": { + "used": 4_000_000_000 + suffix * 10_000_000, + "total": 32_000_000_000, + }, + "loadavg": ["0.10", "0.20", "0.30"], + "kversion": "6.8.12-1-pve", + "pveversion": "pve-manager/9.2.3", + }, + "ip": f"10.32.1.{suffix}", + "cluster_status": { + "ip": f"10.32.1.{suffix}", + "level": "c", + "type": "node", + "quorate": 1, + }, + } + + +def enrich_guest_state(state: dict[str, object], *, kind: str, vmid: str) -> dict[str, object]: + """Ensure qemu/lxc resources carry durable agent/rrd/migrate catalogs.""" + + enriched = dict(state) + name = str(enriched.get("name") or f"{kind}-{vmid}") + suffix = stable_id(f"guest-runtime:{kind}:{vmid}").int % 200 + agent_enabled = enriched.get("agent") + if isinstance(agent_enabled, bool) and not agent_enabled: + agent: dict[str, object] = {"enabled": False, "files": {}, "results": {}} + else: + existing_agent = enriched.get("agent") + agent = dict(existing_agent) if isinstance(existing_agent, dict) else {} + files = agent.get("files") + if not isinstance(files, dict): + files = {} + files.setdefault("/etc/hostname", f"{name}\n") + agent["files"] = files + results = agent.get("results") + if not isinstance(results, dict): + results = {} + results.setdefault( + "info", + { + "version": "9.2.0-simulator", + "supported_commands": [ + {"name": "guest-ping", "enabled": True, "success-response": True}, + {"name": "guest-info", "enabled": True, "success-response": True}, + {"name": "guest-get-osinfo", "enabled": True, "success-response": True}, + ], + }, + ) + results.setdefault( + "get-osinfo", + { + "name": str(enriched.get("ostype") or "l26"), + "pretty-name": f"Simulator guest {name}", + "version": "1.0", + "machine": "x86_64", + }, + ) + results.setdefault("get-host-name", {"host-name": name}) + results.setdefault( + "network-get-interfaces", + [ + { + "name": "eth0", + "hardware-address": f"02:00:00:00:{suffix:02x}:01", + "ip-addresses": [ + { + "ip-address": f"192.0.2.{(suffix % 200) + 10}", + "ip-address-type": "ipv4", + "prefix": 24, + } + ], + } + ], + ) + results.setdefault("ping", {}) + results.setdefault("get-users", [{"user": "root", "login-time": 0}]) + results.setdefault( + "get-fsinfo", + [{"name": "/", "type": "ext4", "total-bytes": 32 * 1024**3}], + ) + results.setdefault("get-memory-block-info", {"size": 1024**3}) + results.setdefault("get-memory-blocks", [{"start": 0, "size": 1024**3}]) + results.setdefault("get-timezone", {"zone": "UTC", "offset": 0}) + results.setdefault("get-vcpus", [{"online": True, "can-offline": False}]) + results.setdefault("fsfreeze-status", "thawed") + results.setdefault("fstrim", {"paths": [{"path": "/", "trimmed": 0}]}) + agent["results"] = results + agent["enabled"] = True + enriched["agent"] = agent + if "rrd" not in enriched or not isinstance(enriched.get("rrd"), dict): + prefix = "pve-vm" if kind == "qemu" else "pve-ct" + enriched["rrd"] = {"filename": f"{prefix}-{vmid}.rrd"} + if "rrddata" not in enriched or not isinstance(enriched.get("rrddata"), list): + memory = enriched.get("memory", 256) + mem_mb = int(memory) if isinstance(memory, int | float | str) else 256 + mem = mem_mb * 1024 * 1024 + enriched["rrddata"] = [ + {"time": 1_720_000_000, "cpu": 0.05, "mem": mem, "netin": 0, "netout": 0}, + { + "time": 1_720_000_060, + "cpu": 0.08, + "mem": mem + 4_000_000, + "netin": 100, + "netout": 80, + }, + ] + if "migrate_preconditions" not in enriched or not isinstance( + enriched.get("migrate_preconditions"), dict + ): + enriched["migrate_preconditions"] = {"local_disks": [], "local_resources": []} + if kind == "lxc" and ( + "interfaces" not in enriched or not isinstance(enriched.get("interfaces"), list) + ): + enriched["interfaces"] = [ + { + "name": "eth0", + "hwaddr": f"02:00:00:00:{suffix:02x}:11", + "inet": f"192.0.2.{(suffix % 200) + 20}/24", + } + ] + if "cloudinit_dump" not in enriched: + enriched["cloudinit_dump"] = f"#cloud-config\nhostname: {name}\nmanage_etc_hosts: true\n" + return enriched + + +def enrich_storage_state(state: dict[str, object], *, storage_id: str) -> dict[str, object]: + enriched = dict(state) + total_raw = enriched.get("total_bytes", 100) + total = int(total_raw) if isinstance(total_raw, int | float | str) else 100 + used_raw = enriched.get("used_bytes", max(total // 10, 1)) + used = int(used_raw) if isinstance(used_raw, int | float | str) else max(total // 10, 1) + if "rrd" not in enriched or not isinstance(enriched.get("rrd"), dict): + enriched["rrd"] = {"filename": f"pve-storage-{storage_id}.rrd"} + if "rrddata" not in enriched or not isinstance(enriched.get("rrddata"), list): + enriched["rrddata"] = [ + {"time": 1_720_000_000, "used": used, "total": total}, + {"time": 1_720_000_060, "used": min(used + total // 50, total), "total": total}, + ] + if "file_restore" not in enriched or not isinstance(enriched.get("file_restore"), dict): + enriched["file_restore"] = { + "/": [{"filepath": "/etc", "type": "d", "text": "etc"}], + "/etc": [ + {"filepath": "/etc/hostname", "type": "f", "text": "hostname"}, + {"filepath": "/etc/hosts", "type": "f", "text": "hosts"}, + ], + } + if "import_metadata" not in enriched or not isinstance(enriched.get("import_metadata"), dict): + enriched["import_metadata"] = { + "type": "qemu", + "disks": {"scsi0": f"{storage_id}:0/vm-import.raw"}, + "net0": "virtio,bridge=vmbr0", + } + if "identity" not in enriched or not isinstance(enriched.get("identity"), dict): + enriched["identity"] = { + "fingerprint": f"fp-{storage_id}", + "type": str(enriched.get("storage_type") or "dir"), + } + return enriched @dataclass(frozen=True, slots=True) @@ -87,6 +488,684 @@ def stable_id(name: str) -> uuid.UUID: return uuid.uuid5(NAMESPACE, name) +def _empty_firewall_scope(*, options: dict[str, object] | None = None) -> dict[str, object]: + return { + "options": dict( + options + or { + "enable": 1, + "policy_in": "DROP", + "policy_out": "ACCEPT", + "log_level_in": "nolog", + "log_level_out": "nolog", + } + ), + "rules": [], + "aliases": {}, + "ipset": {}, + "groups": {}, + "log": [], + } + + +def cluster_domain_metadata(profile: SeedProfile) -> dict[str, object]: + """Durable cluster metadata domains so list endpoints are non-empty after seed.""" + + node_names = [node.name for node in profile.nodes] + primary = node_names[0] if node_names else "pve01" + secondary = node_names[1] if len(node_names) > 1 else "pve02" + node_by_id = {node.id: node.name for node in profile.nodes} + + qemu_guests: list[tuple[str, str]] = [] + lxc_guests: list[tuple[str, str]] = [] + for resource in profile.resources: + node_name = node_by_id.get(resource.node_id) + if node_name is None: + continue + if resource.kind == "qemu": + qemu_guests.append((node_name, resource.external_id)) + elif resource.kind == "lxc": + lxc_guests.append((node_name, resource.external_id)) + + # Keep large profiles bounded; small/demo stay fully covered within the cap. + guest_cap = 128 if profile.name == "large" else 10_000 + qemu_guests = qemu_guests[:guest_cap] + lxc_guests = lxc_guests[:guest_cap] + replication_guest = qemu_guests[0][1] if qemu_guests else "100" + + cluster_fw = _empty_firewall_scope( + options={ + "enable": 1, + "policy_in": "DROP", + "policy_out": "ACCEPT", + "log_level_in": "info", + "log_level_out": "nolog", + } + ) + cluster_fw["aliases"] = { + "mgmt-net": { + "name": "mgmt-net", + "cidr": "10.0.0.0/24", + "comment": "Management network", + }, + "monitoring": { + "name": "monitoring", + "cidr": "10.20.0.0/24", + "comment": "Prometheus / Grafana", + }, + } + cluster_fw["ipset"] = { + "blocked-scanners": { + "name": "blocked-scanners", + "comment": "Known scanners", + "entries": { + "203.0.113.0/24": { + "cidr": "203.0.113.0/24", + "comment": "RFC5737 TEST-NET-3", + "nomatch": 0, + } + }, + } + } + cluster_fw["groups"] = { + "web-tier": { + "comment": "HTTP/HTTPS services", + "rules": [ + { + "type": "in", + "action": "ACCEPT", + "proto": "tcp", + "dport": "443", + "comment": "HTTPS", + "enable": 1, + "pos": 0, + } + ], + } + } + cluster_fw["rules"] = [ + { + "type": "in", + "action": "ACCEPT", + "source": "+mgmt-net", + "macro": "SSH", + "comment": "SSH from mgmt", + "enable": 1, + "pos": 0, + }, + { + "type": "in", + "action": "ACCEPT", + "source": "+monitoring", + "dport": "9100", + "proto": "tcp", + "comment": "Node exporter", + "enable": 1, + "pos": 1, + }, + { + "type": "in", + "action": "DROP", + "source": "+blocked-scanners", + "comment": "Drop scanners", + "enable": 1, + "pos": 2, + }, + ] + cluster_fw["log"] = [ + { + "n": 0, + "t": 1720000000, + "msg": "DROP IN 198.51.100.10 → 10.0.0.10:22 proto=tcp", + } + ] + + scopes: dict[str, object] = {"cluster": cluster_fw} + for node_name in node_names: + scope = _empty_firewall_scope() + scope["rules"] = [ + { + "type": "in", + "action": "ACCEPT", + "iface": "vmbr0", + "comment": f"Allow bridge traffic on {node_name}", + "enable": 1, + "pos": 0, + } + ] + scope["aliases"] = { + f"{node_name}-mgmt": { + "name": f"{node_name}-mgmt", + "cidr": "10.0.0.0/24", + "comment": f"{node_name} management", + } + } + scope["ipset"] = { + f"{node_name}-trusted": { + "name": f"{node_name}-trusted", + "comment": "Trusted clients", + "entries": { + "10.0.0.0/8": { + "cidr": "10.0.0.0/8", + "comment": "RFC1918", + "nomatch": 0, + } + }, + } + } + scope["log"] = [ + { + "n": 0, + "t": 1720000100, + "msg": f"ACCEPT IN vmbr0 on {node_name}", + } + ] + scopes[f"node:{node_name}"] = scope + + for node_name, vmid in qemu_guests: + scope = _empty_firewall_scope() + scope["rules"] = [ + { + "type": "in", + "action": "ACCEPT", + "proto": "tcp", + "dport": "443", + "comment": f"HTTPS on VM {vmid}", + "enable": 1, + "pos": 0, + } + ] + host = (stable_id(f"fw-vip:{vmid}").int % 250) + 1 + scope["aliases"] = { + f"vm{vmid}-vip": { + "name": f"vm{vmid}-vip", + "cidr": f"10.10.{host}.10/32", + "comment": f"VIP for VM {vmid}", + } + } + scopes[f"qemu:{node_name}:{vmid}"] = scope + + for node_name, vmid in lxc_guests: + scope = _empty_firewall_scope() + scope["rules"] = [ + { + "type": "in", + "action": "ACCEPT", + "proto": "tcp", + "dport": "22", + "comment": f"SSH on CT {vmid}", + "enable": 1, + "pos": 0, + } + ] + scopes[f"lxc:{node_name}:{vmid}"] = scope + + ha_nodes = ",".join(node_names) if node_names else primary + return { + "firewall": {"scopes": scopes}, + "options": { + "keyboard": "en-us", + "email_from": "pve@example.local", + "http_proxy": "", + "description": f"Proxmox API simulator ({profile.name})", + "mac_prefix": "BC:24:11", + }, + "quorate": 1, + "ha": { + "armed": True, + "status_current": {"quorate": 1, "mode": "active"}, + "manager_status": {"manager_status": "active", "quorum": "OK"}, + }, + "ha_groups": { + "primary": { + "nodes": ha_nodes, + "nofailback": 0, + "restricted": 0, + "comment": "Default HA group", + }, + "critical-services": { + "nodes": ha_nodes, + "nofailback": 1, + "restricted": 0, + "comment": "Critical services", + }, + }, + "ha_rules": [ + {"rule": "node-fencing", "type": "node", "action": "restart"}, + {"rule": "service-ha", "type": "resource", "action": "failover"}, + ], + "sdn": { + "zones": { + "public": { + "zone": "public", + "type": "simple", + "bridge": "vmbr0", + "bridges": [{"iface": "vmbr0", "active": 1}], + "vrf": "vrf-public", + "table": 100, + "mtu": 1500, + "comment": "Public zone", + }, + "internal": { + "zone": "internal", + "type": "vlan", + "bridge": "vmbr1", + "bridges": [{"iface": "vmbr1", "active": 1}], + "vrf": "vrf-internal", + "table": 101, + "vlan-protocol": "802.1q", + "comment": "Internal VLAN zone", + }, + }, + "vnets": { + "vnet0": { + "vnet": "vnet0", + "zone": "public", + "alias": "Public VNet", + "mac-vrf": "macvrf-vnet0", + "tag": 0, + "subnets": { + "10.0.0.0/24": { + "subnet": "10.0.0.0/24", + "gateway": "10.0.0.1", + "snat": 0, + } + }, + "ips": [], + "firewall": { + "options": {"enable": 1}, + "rules": [ + { + "type": "in", + "action": "ACCEPT", + "proto": "tcp", + "dport": "443", + "enable": 1, + "pos": 0, + } + ], + }, + }, + "vnet1": { + "vnet": "vnet1", + "zone": "internal", + "alias": "Internal VNet", + "mac-vrf": "macvrf-vnet1", + "tag": 100, + "subnets": { + "10.10.0.0/24": { + "subnet": "10.10.0.0/24", + "gateway": "10.10.0.1", + "snat": 1, + } + }, + "ips": [], + "firewall": {"options": {"enable": 0}, "rules": []}, + }, + }, + "controllers": { + "evpn1": { + "controller": "evpn1", + "type": "evpn", + "asn": 65000, + "peers": secondary, + "comment": "EVPN controller", + } + }, + "dns": { + "powerdns": { + "dns": "powerdns", + "type": "powerdns", + "url": "http://dns.example.local:8081", + "key": "seed-dns-key", + "ttl": 300, + } + }, + "ipams": { + "pve": {"ipam": "pve", "type": "pve", "comment": "Built-in IPAM"}, + "netbox": { + "ipam": "netbox", + "type": "netbox", + "url": "https://netbox.example.local", + "token": "seed-netbox-token", + }, + }, + "fabrics": { + "fabric0": { + "id": "fabric0", + "protocol": "ospf", + "ip-prefix": "10.99.0.0/24", + "comment": "Underlay fabric", + "routes": [{"dst": "10.99.0.0/24", "protocol": "ospf"}], + } + }, + "fabric_nodes": { + "fabric0": { + node_name: { + "node": node_name, + "interfaces": ["eno1"], + "state": "up", + } + for node_name in node_names[:3] or [primary] + } + }, + "prefix_lists": { + "pl-default": { + "name": "pl-default", + "entries": {"10": {"seq": "10", "action": "permit", "prefix": "10.0.0.0/8"}}, + } + }, + "route_maps": { + "rm-default": { + "name": "rm-default", + "entries": {"10": {"seq": "10", "action": "permit", "match": "pl-default"}}, + } + }, + "lock": None, + "pending": False, + "running_version": 1, + }, + "firewall_macros": [ + {"macro": "SSH", "descr": "Secure Shell"}, + {"macro": "HTTPS", "descr": "Secure web server"}, + {"macro": "HTTP", "descr": "Web server"}, + ], + "notifications": { + "matcher_fields": [ + {"name": "type", "type": "string"}, + {"name": "hostname", "type": "string"}, + {"name": "job-id", "type": "string"}, + {"name": "severity", "type": "string"}, + ], + "matcher_field_values": [ + {"field": "type", "value": "fencing"}, + {"field": "type", "value": "package-updates"}, + {"field": "type", "value": "replication"}, + {"field": "type", "value": "system-mail"}, + ], + "endpoints": { + "gotify": { + "ops-gotify": { + "name": "ops-gotify", + "server": "https://gotify.example.local", + "token": "seed-gotify-token", + "comment": "Ops Gotify", + "disable": 0, + } + }, + "sendmail": { + "local-mail": { + "name": "local-mail", + "mailto": "ops@example.local", + "from-address": "pve@example.local", + "comment": "Local sendmail", + "disable": 0, + } + }, + "smtp": { + "alerts": { + "name": "alerts", + "server": "smtp.example.local", + "port": 587, + "mode": "starttls", + "from-address": "pve@example.local", + "mailto": "ops@example.local", + "comment": "Ops SMTP", + "disable": 0, + } + }, + "webhook": { + "pager": { + "name": "pager", + "url": "https://hooks.example.local/pve", + "method": "post", + "comment": "Webhook pager", + "disable": 0, + } + }, + }, + "matchers": { + "backup-failures": { + "name": "backup-failures", + "match-field": ["type=package-updates"], + "target": ["alerts", "ops-gotify"], + "mode": "always", + "comment": "Route package/update noise", + "disable": 0, + } + }, + "tests": [], + }, + "acme": { + "accounts": { + "letsencrypt": { + "name": "letsencrypt", + "contact": ["mailto:admin@example.local"], + "directory": "https://acme-v02.api.letsencrypt.org/directory", + "location": "https://acme.example.local/acct/letsencrypt", + } + }, + "plugins": { + "cf-dns": { + "id": "cf-dns", + "type": "dns", + "api": "CF", + "data": {"CF_Token": "seed-cf-token"}, + "validation-delay": 30, + } + }, + "directories": [ + { + "name": "Let's Encrypt V2", + "url": "https://acme-v02.api.letsencrypt.org/directory", + }, + { + "name": "Let's Encrypt V2 Staging", + "url": "https://acme-staging-v02.api.letsencrypt.org/directory", + }, + ], + "challenge_schema": [ + { + "id": "dns", + "name": "DNS plugin", + "type": "dns", + "fields": [{"name": "api", "type": "string"}], + } + ], + "meta": { + "https://acme-v02.api.letsencrypt.org/directory": { + "termsOfService": "https://acme-v02.api.letsencrypt.org/directory/tos", + "caaIdentities": ["letsencrypt.org"], + }, + "https://acme-staging-v02.api.letsencrypt.org/directory": { + "termsOfService": ( + "https://acme-staging-v02.api.letsencrypt.org/directory/tos" + ), + "caaIdentities": ["letsencrypt.org"], + }, + }, + }, + "mapping": { + "pci": { + "gpu0": { + "id": "gpu0", + "map": [f"{primary};0000:01:00.0;"], + "description": "Passthrough GPU", + } + }, + "usb": { + "backup-key": { + "id": "backup-key", + "map": [f"{primary};2-2;"], + "description": "USB backup key", + } + }, + "dir": { + "shared-data": { + "id": "shared-data", + "map": [f"{primary};/mnt/shared;"], + "description": "Shared directory mapping", + } + }, + }, + "replication": [ + { + "id": f"{replication_guest}-0", + "guest": replication_guest, + "target": secondary, + "type": "local", + "schedule": "*/15", + "rate": 100, + "comment": f"Replicate VM/CT {replication_guest} to {secondary}", + "enabled": 1, + "source": primary, + "state": "OK", + "last_sync": 1720000200, + "duration": 12, + "fail_count": 0, + "error": "", + "log": [ + { + "t": 1720000200, + "n": 0, + "msg": f"replication ok for {replication_guest}-0", + } + ], + } + ], + "metrics": { + "export_data": '# HELP pve_up Node is up\npve_up{node="pve01"} 1\n', + "servers": { + "influx": { + "id": "influx", + "type": "influxdb", + "server": "influx.example.local", + "port": 8086, + "organization": "lab", + "bucket": "pve", + "enable": 1, + "comment": "InfluxDB metrics sink", + } + }, + }, + "jobs": { + "realm_sync": { + "ldap-sync": { + "id": "ldap-sync", + "realm": "pam", + "schedule": "0 3 * * *", + "enable": 1, + "comment": "Nightly realm sync", + } + }, + "schedule_analyze_results": [ + {"timestamp": 1_720_000_900, "utc": True}, + {"timestamp": 1_720_001_800, "utc": True}, + {"timestamp": 1_720_002_700, "utc": True}, + {"timestamp": 1_720_003_600, "utc": True}, + ], + }, + "qemu_cpu_models": { + "lab-cpu": { + "name": "lab-cpu", + "vendor": "GenuineIntel", + "flags": "+aes", + "comment": "Custom lab CPU model", + } + }, + "qemu_cpu_flags": [ + {"name": "aes", "introduces": "Westmere"}, + {"name": "avx", "introduces": "SandyBridge"}, + {"name": "avx2", "introduces": "Haswell"}, + ], + "ceph": { + "initialized": True, + "running": True, + "version": {"str": "18.2.2", "parts": [18, 2, 2]}, + "config": { + "network": "10.10.10.0/24", + "cluster-network": "10.10.10.0/24", + "size": 3, + "min_size": 2, + "pg_bits": 7, + "fsid": "pve-simulator-fsid", + }, + "cfg_db": [ + {"section": "global", "name": "auth_client_required", "value": "cephx"}, + {"section": "global", "name": "fsid", "value": "pve-simulator-fsid"}, + ], + "cfg_raw": "[global]\nfsid = pve-simulator-fsid\nauth_client_required = cephx\n", + "cfg_values": {}, + "pools": { + "rbd": { + "pool": "rbd", + "size": 3, + "min_size": 2, + "pg_num": 128, + "application": "rbd", + "crush_rule": "replicated_rule", + "bytes_used": 12_000_000_000, + "percent_used": 12.0, + "healthy": True, + } + }, + "fs": {}, + "rules": [{"name": "replicated_rule", "id": 0}], + "crush": "device 0 osd.0 class hdd\n", + "cmd_safety": {"safe": 1}, + "flags": { + "nobackfill": 0, + "nodeep-scrub": 0, + "nodown": 0, + "noin": 0, + "noout": 0, + "norebalance": 0, + "norecover": 0, + "noscrub": 0, + "notieragent": 0, + "pause": 0, + }, + }, + "cluster_config": { + "clustername": f"pve-{profile.name}", + "votes": 1, + "links": {}, + "join_info": {}, + "totem": { + "version": 2, + "secauth": "on", + "cluster_name": f"pve-{profile.name}", + }, + "qdevice": {"status": "disabled"}, + "apiversion": 1, + "added_nodes": { + node_name: { + "node": node_name, + "ring0_addr": f"{node_name}.local", + "quorum_votes": 1, + } + for node_name in node_names + }, + }, + } + + +def _seed_resource_state(resource: SeedResource) -> dict[str, object]: + if resource.kind in {"qemu", "lxc"}: + return enrich_guest_state( + dict(resource.state), kind=resource.kind, vmid=resource.external_id + ) + if resource.kind == "storage": + return enrich_storage_state(dict(resource.state), storage_id=resource.external_id) + if resource.kind == "ceph-osd": + state = dict(resource.state) + state.setdefault("uuid", f"osd-uuid-{resource.external_id}") + state.setdefault("dev", f"/dev/{resource.external_id.replace('.', '')}") + state.setdefault("device_class", "hdd") + return state + return dict(resource.state) + + def _string_list(state: dict[str, object], key: str) -> tuple[str, ...]: value = state.get(key, []) if not isinstance(value, list) or not all(isinstance(item, str) for item in value): @@ -117,9 +1196,36 @@ def _completed_task(index: int, task_type: str, resource_id: str) -> SeedTask: def small_profile() -> SeedProfile: node = _node("pve01") resources = ( - _resource(node, "qemu", "100", {"name": "demo", "status": "stopped"}), - _resource(node, "qemu", "101", {"name": "worker", "status": "stopped"}), - _resource(node, "lxc", "200", {"name": "service", "status": "stopped"}), + _resource( + node, + "qemu", + "100", + { + "name": "demo", + "status": "stopped", + "agent": {"files": {"/etc/hostname": "demo\n"}}, + }, + ), + _resource( + node, + "qemu", + "101", + { + "name": "worker", + "status": "stopped", + "agent": {"files": {"/etc/hostname": "worker\n"}}, + }, + ), + _resource( + node, + "lxc", + "200", + { + "name": "service", + "status": "stopped", + "agent": {"files": {"/etc/hostname": "service\n"}}, + }, + ), _resource(node, "storage", "local", {"content": ["iso", "backup"], "status": "available"}), _resource( node, "storage", "local-lvm", {"content": ["images", "rootdir"], "status": "available"} @@ -360,6 +1466,7 @@ async def apply_seed(connection: Connection, profile: SeedProfile) -> None: "profile": profile.name, "nodes": len(profile.nodes), "resources": len(profile.resources), + **cluster_domain_metadata(profile), }, sort_keys=True, ), @@ -385,7 +1492,7 @@ async def apply_seed(connection: Connection, profile: SeedProfile) -> None: resource.node_id, resource.kind, resource.external_id, - json.dumps(resource.state, sort_keys=True), + json.dumps(_seed_resource_state(resource), sort_keys=True), ) for resource in profile.resources ], @@ -399,7 +1506,7 @@ async def apply_seed(connection: Connection, profile: SeedProfile) -> None: ( resource.id, int(resource.external_id), - json.dumps(resource.state, sort_keys=True), + json.dumps(_seed_resource_state(resource), sort_keys=True), ) for resource in qemu ], @@ -413,7 +1520,7 @@ async def apply_seed(connection: Connection, profile: SeedProfile) -> None: ( resource.id, int(resource.external_id), - json.dumps(resource.state, sort_keys=True), + json.dumps(_seed_resource_state(resource), sort_keys=True), ) for resource in containers ], @@ -433,7 +1540,7 @@ async def apply_seed(connection: Connection, profile: SeedProfile) -> None: _storage_type(resource), bool(resource.state.get("shared", False)), *_storage_capacity(resource), - json.dumps(resource.state, sort_keys=True), + json.dumps(_seed_resource_state(resource), sort_keys=True), ) for resource in storages ], diff --git a/app/surface_probe.py b/app/surface_probe.py index fc479b5..43250f7 100644 --- a/app/surface_probe.py +++ b/app/surface_probe.py @@ -159,16 +159,16 @@ async def probe_major( verb = method.verb.upper() url = f"/api2/json{render_path(path_template)}" headers = {"CSRFPreventionToken": csrf} if verb != "GET" else {} - body = body_for(method, path_template) if verb in {"PUT", "POST"} else None + payload = body_for(method, path_template) try: if verb == "GET": - response = await client.get(url, headers=headers) + response = await client.get(url, headers=headers, params=payload or None) elif verb == "PUT": - response = await client.put(url, data=body or {}, headers=headers) + response = await client.put(url, data=payload or {}, headers=headers) elif verb == "POST": - response = await client.post(url, data=body or {}, headers=headers) + response = await client.post(url, data=payload or {}, headers=headers) elif verb == "DELETE": - response = await client.delete(url, headers=headers) + response = await client.request("DELETE", url, data=payload or {}, headers=headers) else: continue except Exception as exc: diff --git a/app/tasks/qemu.py b/app/tasks/qemu.py index 3d7b5db..4cf136f 100644 --- a/app/tasks/qemu.py +++ b/app/tasks/qemu.py @@ -62,10 +62,12 @@ def qemu_handler(repository: TaskRepository, clock: Clock) -> TaskHandler: async def _create(repository: TaskRepository, task: Task) -> dict[str, Any]: + from app.simulation.seed import enrich_guest_state + node, vmid = str(task.payload["node"]), int(task.payload["vmid"]) config = dict(task.payload.get("config", {})) resource_id = uuid.uuid4() - state = {"status": "stopped", **config} + state = enrich_guest_state({"status": "stopped", **config}, kind="qemu", vmid=str(vmid)) async with repository.pool.acquire() as connection: async with connection.transaction(): node_row = await connection.fetchrow( diff --git a/app/web/index.html b/app/web/index.html index af22e8f..c9a90a2 100644 --- a/app/web/index.html +++ b/app/web/index.html @@ -26,22 +26,22 @@ --muted: #5c6778; --accent: #ff8500; --accent-hover: #ff6a00; - --link: #1e88ff; + --link: #d97706; --ok: #22c55e; --ok-bg: #e6f8ed; --warn: #9a6700; --warn-bg: #fff8e6; --err: #b42318; --err-bg: #fdecea; - --info: #1e88ff; - --info-bg: #e8f2ff; + --info: #d97706; + --info-bg: #fff4e8; --debug: #5c6778; --debug-bg: #eef1f6; --surface-raised: #fafbfc; --surface-hover: #f3f5f8; --surface-toolbar: #f6f8fb; --surface-muted-hover: #eef1f6; - --surface-accent: #eef4ff; + --surface-accent: #fff4e8; --surface-danger: #fffafa; --border-strong: #c8d0db; --brand-ink: #0f1419; @@ -50,14 +50,14 @@ --brand-logo-bg: var(--overlay-head-bg); --brand-logo-border: var(--border-strong); --brand-subtle: #97a1b1; - --meta-desc-bg: #f4f7fb; - --meta-desc-border: #7eb6ff; - --meta-desc-text: #1e88ff; + --meta-desc-bg: #fff8f0; + --meta-desc-border: #ffb366; + --meta-desc-text: #c45a00; --overlay-head-bg: #eef1f6; --brand-accent: #ff8500; --brand-accent-hover: #ff6a00; - --panel-accent: #22c55e; - --panel-accent-hover: #16a34a; + --panel-accent: #ff8500; + --panel-accent-hover: #ff6a00; --mono: "SF Mono", "Cascadia Code", Consolas, monospace; --sans: "Segoe UI", system-ui, sans-serif; --header-h: 52px; @@ -74,22 +74,22 @@ --muted: #b0bac8; --accent: #ff8a1f; --accent-hover: #ffaa55; - --link: #93c5fd; + --link: #ffb366; --ok: #86efac; --ok-bg: #1a3d28; --warn: #f0c866; --warn-bg: #3d3218; --err: #fca5a5; --err-bg: #3d2222; - --info: #93c5fd; - --info-bg: #1a2d4a; + --info: #ffb366; + --info-bg: #3d2a18; --debug: #b0bac8; --debug-bg: #283240; --surface-raised: #242e3d; --surface-hover: #2f3a4b; --surface-toolbar: #151c27; --surface-muted-hover: #2f3a4b; - --surface-accent: #1e3150; + --surface-accent: #3d2a18; --surface-danger: #3a2626; --brand-ink: #f2f5f9; --brand-logo-core: #ffffff; @@ -97,14 +97,14 @@ --brand-logo-bg: var(--overlay-head-bg); --brand-logo-border: var(--border-strong); --brand-subtle: #9aa8ba; - --meta-desc-bg: #1e2836; - --meta-desc-border: #3d5a85; - --meta-desc-text: #c8d8ef; + --meta-desc-bg: #2a2218; + --meta-desc-border: #8a5a28; + --meta-desc-text: #ffc078; --overlay-head-bg: #242e3d; --brand-accent: #ff8a1f; --brand-accent-hover: #ffaa55; - --panel-accent: #4ade80; - --panel-accent-hover: #86efac; + --panel-accent: #ff8a1f; + --panel-accent-hover: #ffaa55; } html[data-theme="dark"] .header-tool-btn { @@ -537,7 +537,7 @@ } .header-tool-btn.data-badge-btn.loaded .data-badge-dot { - background: var(--ok); + background: var(--brand-accent); } .help-demo-panel { @@ -1124,7 +1124,7 @@ grid-row: 1; display: grid; position: relative; - font-size: 15px; + font-size: 16.5px; font-weight: 700; letter-spacing: 0.07em; text-transform: uppercase; @@ -1158,18 +1158,22 @@ .workspace-brand-sub { grid-row: 2; display: block; + box-sizing: border-box; width: 100%; + max-width: 100%; min-width: 0; - font-size: 9.75px; - font-weight: 600; - letter-spacing: 0.02em; - line-height: 1.2; + font-size: 7.75px; + font-weight: 500; + letter-spacing: 0.14em; + line-height: 1.15; text-transform: uppercase; - color: var(--muted); + text-align: left; + color: var(--brand-subtle); white-space: nowrap; - text-align: justify; - text-align-last: justify; - word-spacing: 0; + } + + html[data-theme="dark"] .workspace-brand-sub { + color: var(--border-strong); } .workspace-topbar-actions { @@ -1769,6 +1773,7 @@ .data-drawer, .url-drawer, .endpoints-drawer, + .upid-drawer, .help-drawer, .ui-modal { --accent: var(--panel-accent); @@ -1783,6 +1788,7 @@ .data-drawer-head, .url-drawer-head, .endpoints-drawer-head, + .upid-drawer-head, .help-drawer-head { display: flex; align-items: center; @@ -1803,6 +1809,7 @@ .data-drawer-head h2, .url-drawer-head h2, .endpoints-drawer-head h2, + .upid-drawer-head h2, .help-drawer-head h2 { margin: 0; font-size: 11px; @@ -1828,6 +1835,7 @@ .catalog-drawer-head .icon-btn, .environment-drawer-head .icon-btn, .data-drawer-head .icon-btn, + .upid-drawer-head .icon-btn, .url-drawer-head .icon-btn, .endpoints-drawer-head .icon-btn, .help-drawer-head .icon-btn { @@ -1842,6 +1850,7 @@ .catalog-drawer-head .icon-btn:hover, .environment-drawer-head .icon-btn:hover, .data-drawer-head .icon-btn:hover, + .upid-drawer-head .icon-btn:hover, .url-drawer-head .icon-btn:hover, .endpoints-drawer-head .icon-btn:hover, .help-drawer-head .icon-btn:hover { @@ -1869,6 +1878,7 @@ .catalog-backdrop, .environment-backdrop, .data-backdrop, + .upid-backdrop, .url-backdrop, .endpoints-backdrop, .help-backdrop { @@ -1887,6 +1897,7 @@ .catalog-backdrop.open, .environment-backdrop.open, .data-backdrop.open, + .upid-backdrop.open, .url-backdrop.open, .endpoints-backdrop.open, .help-backdrop.open { @@ -1900,6 +1911,7 @@ .catalog-drawer, .environment-drawer, .data-drawer, + .upid-drawer, .url-drawer, .endpoints-drawer, .help-drawer { @@ -1926,6 +1938,7 @@ .catalog-drawer.open, .environment-drawer.open, .data-drawer.open, + .upid-drawer.open, .url-drawer.open, .endpoints-drawer.open, .help-drawer.open { @@ -1938,6 +1951,7 @@ .auth-drawer-body, .environment-drawer-body, .data-drawer-body, + .upid-drawer-body, .url-drawer-body, .endpoints-drawer-body, .help-drawer-body { @@ -1959,6 +1973,7 @@ .catalog-drawer-body .btn, .environment-drawer-body .btn, .data-drawer-body .btn, + .upid-drawer-body .btn, .url-drawer-body .btn, .endpoints-drawer-body .btn, .help-drawer-body .btn, @@ -1973,6 +1988,80 @@ line-height: 1.2; } + .upid-panel { + display: flex; + flex-direction: column; + gap: 14px; + } + + .upid-hint { + margin: 0; + color: var(--muted); + font-size: 12px; + line-height: 1.45; + } + + .upid-hint code { + font-family: var(--mono); + font-size: 11px; + } + + .upid-form { + display: flex; + flex-direction: column; + gap: 12px; + } + + .upid-form .field label { + display: block; + margin-bottom: 6px; + font-size: 11px; + font-weight: 600; + letter-spacing: 0.04em; + text-transform: uppercase; + color: var(--muted); + } + + .upid-form .field input { + width: 100%; + box-sizing: border-box; + padding: 10px 12px; + border: 1px solid var(--border); + border-radius: 8px; + background: var(--surface-raised); + color: var(--text); + font-family: var(--mono); + font-size: 12px; + } + + .upid-form .field input:focus { + outline: none; + border-color: var(--brand-accent); + } + + .upid-actions { + display: flex; + flex-wrap: wrap; + gap: 8px; + } + + .upid-output { + margin: 0; + min-height: 220px; + max-height: min(55vh, 520px); + overflow: auto; + padding: 12px 14px; + border: 1px solid var(--border); + border-radius: 8px; + background: var(--surface-raised); + color: var(--text); + font-family: var(--mono); + font-size: 12px; + line-height: 1.45; + white-space: pre-wrap; + word-break: break-word; + } + .help-drawer-body { display: flex; flex-direction: column; @@ -3173,7 +3262,7 @@ PROXMOX - API Simulator + API Simulator
@@ -3188,6 +3277,7 @@ History +
+ + +