proxmox-api-simulator/app/handlers/cluster_extra.py

"""Additional cluster-level handlers with durable metadata persistence."""

from __future__ import annotations

import secrets
import time
from typing import Any

from fastapi import Request

from app.api.errors import ApiError
from app.api.registry import HandlerRegistry
from app.handlers.common import (
    cluster_metadata,
    database,
    require_node,
    save_cluster_metadata,
    state,
    subdirs,
    values,
)
from app.simulation.ceph_defaults import CEPH_FLAG_DESCRIPTIONS, default_ceph_flags
from app.tasks.repository import TaskRepository
from app.tasks.upid import Upid


def _jobs(metadata: dict[str, Any]) -> dict[str, Any]:
    jobs = metadata.get("jobs")
    return jobs if isinstance(jobs, dict) else {}


def _metrics(metadata: dict[str, Any]) -> dict[str, Any]:
    metrics = metadata.get("metrics")
    return metrics if isinstance(metrics, dict) else {}


def _cpu_models(metadata: dict[str, Any]) -> dict[str, Any]:
    models = metadata.get("qemu_cpu_models")
    return models if isinstance(models, dict) else {}


def _ha_rules_store(metadata: dict[str, Any]) -> list[dict[str, Any]]:
    rules = metadata.get("ha_rules")
    if isinstance(rules, dict):
        return [
            {"rule": str(name), **dict(value)}
            for name, value in rules.items()
            if isinstance(value, dict)
        ]
    if isinstance(rules, list):
        return [dict(item) for item in rules if isinstance(item, dict)]
    return []


def _save_ha_rules(metadata: dict[str, Any], rules: list[dict[str, Any]]) -> None:
    metadata["ha_rules"] = rules


def _replication_jobs(metadata: dict[str, Any]) -> list[dict[str, Any]]:
    from app.handlers.cluster import _replication_jobs as normalize

    return normalize(metadata)


def _ceph(metadata: dict[str, Any]) -> dict[str, Any]:
    ceph = metadata.get("ceph")
    return ceph if isinstance(ceph, dict) else {}


async def _cluster_task(request: Request, *, task_type: str, worker: str) -> str:
    from app.db.primitives import ConflictError

    pool = database(request).pool
    node = await pool.fetchval("SELECT name FROM nodes ORDER BY name LIMIT 1") or "localhost"
    upid = str(Upid.allocate(str(node), worker, "0", str(request.state.principal)))
    try:
        task = await TaskRepository(pool).create(
            upid=upid,
            task_type=task_type,
            payload={"cluster": True},
            resource_key=f"cluster:{task_type}:{secrets.token_hex(4)}",
        )
    except ConflictError as error:
        raise ApiError(409, str(error)) from error
    return task.upid


async def _bulk_guest_status(request: Request, status: str) -> None:
    await database(request).pool.execute(
        """UPDATE resources
        SET state = jsonb_set(COALESCE(state, '{}'::jsonb), '{status}', to_jsonb($1::text), true),
            updated_at=now()
        WHERE kind IN ('qemu', 'lxc')""",
        status,
    )


def register_cluster_extra_handlers(registry: HandlerRegistry) -> None:
    async def jobs_index(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, str]]:
        return subdirs("realm-sync", "schedule-analyze")

    async def realm_sync_list(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]:
        metadata = await cluster_metadata(request)
        jobs = _jobs(metadata)
        items = [
            {"id": job_id, **dict(payload)}
            for job_id, payload in sorted(jobs.get("realm_sync", {}).items())
            if isinstance(payload, dict)
        ]
        return items

    async def realm_sync_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]:
        job_id = str(values(inputs)["id"])
        metadata = await cluster_metadata(request)
        jobs = _jobs(metadata)
        payload = jobs.get("realm_sync", {}).get(job_id)
        if not isinstance(payload, dict):
            raise ApiError(404, "realm-sync job does not exist")
        return {"id": job_id, **payload}

    async def realm_sync_create(request: Request, inputs: dict[str, Any]) -> dict[str, Any]:
        payload = values(inputs)
        job_id = str(payload["id"])
        metadata = await cluster_metadata(request)
        jobs = dict(_jobs(metadata))
        sync = dict(jobs.get("realm_sync") or {})
        if job_id in sync:
            raise ApiError(409, "realm-sync job already exists")
        entry = {
            key: value for key, value in payload.items() if key not in {"id", "delete", "digest"}
        }
        entry.setdefault("schedule", "0 0 * * *")
        entry.setdefault("enabled", 1)
        entry.setdefault("realm", str(payload.get("realm") or "pam"))
        sync[job_id] = entry
        jobs["realm_sync"] = sync
        metadata["jobs"] = jobs
        await save_cluster_metadata(request, metadata)
        return {"id": job_id, **entry}

    async def realm_sync_update(request: Request, inputs: dict[str, Any]) -> dict[str, Any]:
        payload = values(inputs)
        job_id = str(payload["id"])
        metadata = await cluster_metadata(request)
        jobs = dict(_jobs(metadata))
        sync = dict(jobs.get("realm_sync") or {})
        if job_id not in sync:
            raise ApiError(404, "realm-sync job does not exist")
        updated = {
            **sync[job_id],
            **{
                key: value
                for key, value in payload.items()
                if key not in {"id", "delete", "digest"}
            },
        }
        sync[job_id] = updated
        jobs["realm_sync"] = sync
        metadata["jobs"] = jobs
        await save_cluster_metadata(request, metadata)
        return {"id": job_id, **updated}

    async def realm_sync_delete(request: Request, inputs: dict[str, Any]) -> None:
        job_id = str(values(inputs)["id"])
        metadata = await cluster_metadata(request)
        jobs = dict(_jobs(metadata))
        sync = dict(jobs.get("realm_sync") or {})
        if job_id not in sync:
            raise ApiError(404, "realm-sync job does not exist")
        del sync[job_id]
        jobs["realm_sync"] = sync
        metadata["jobs"] = jobs
        await save_cluster_metadata(request, metadata)

    async def schedule_analyze(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]:
        schedule = str(values(inputs).get("schedule") or "*/15")
        metadata = await cluster_metadata(request)
        jobs = dict(_jobs(metadata))
        results = jobs.get("schedule_analyze_results")
        if not isinstance(results, list):
            results = []
        jobs["last_schedule_analyze"] = {"schedule": schedule, "at": int(time.time())}
        metadata["jobs"] = jobs
        await save_cluster_metadata(request, metadata)
        return [dict(item) for item in results if isinstance(item, dict)]

    async def metrics_index(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, str]]:
        return subdirs("export", "server")

    async def metrics_export(request: Request, _inputs: dict[str, Any]) -> dict[str, Any]:
        metadata = await cluster_metadata(request)
        metrics = _metrics(metadata)
        now = int(time.time())
        seeded = metrics.get("export_series")
        if isinstance(seeded, list) and seeded:
            seeded_series = [dict(item) for item in seeded if isinstance(item, dict)]
            return {"data": seeded_series, "timestamp": now}
        # Build a PVE-shaped metric array from live inventory (scales with seed).
        rows = await database(request).pool.fetch("SELECT name FROM nodes ORDER BY name")
        series: list[dict[str, Any]] = []
        for row in rows:
            node = str(row["name"])
            series.append(
                {
                    "id": f"node/{node}",
                    "metric": "up",
                    "timestamp": now,
                    "type": "gauge",
                    "value": 1,
                }
            )
        guests = await database(request).pool.fetch(
            """SELECT r.kind, r.external_id, n.name AS node
            FROM resources r JOIN nodes n ON n.id=r.node_id
            WHERE r.kind IN ('qemu', 'lxc')
            ORDER BY r.external_id::integer"""
        )
        for guest in guests:
            series.append(
                {
                    "id": f"{guest['kind']}/{guest['external_id']}",
                    "metric": "up",
                    "timestamp": now,
                    "type": "gauge",
                    "value": 1,
                }
            )
        return {"data": series, "timestamp": now}

    async def metrics_server_list(
        request: Request, _inputs: dict[str, Any]
    ) -> list[dict[str, Any]]:
        metadata = await cluster_metadata(request)
        metrics = _metrics(metadata)
        servers = metrics.get("servers")
        if not isinstance(servers, dict):
            return []
        return [
            {"id": server_id, **dict(payload)}
            for server_id, payload in sorted(servers.items())
            if isinstance(payload, dict)
        ]

    async def metrics_server_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]:
        server_id = str(values(inputs)["id"])
        metadata = await cluster_metadata(request)
        metrics = _metrics(metadata)
        payload = metrics.get("servers", {}).get(server_id)
        if not isinstance(payload, dict):
            raise ApiError(404, "metrics server does not exist")
        return {"id": server_id, **payload}

    async def metrics_server_create(request: Request, inputs: dict[str, Any]) -> dict[str, Any]:
        payload = values(inputs)
        server_id = str(payload["id"])
        metadata = await cluster_metadata(request)
        metrics = dict(_metrics(metadata))
        servers = dict(metrics.get("servers") or {})
        if server_id in servers:
            raise ApiError(409, "metrics server already exists")
        entry = {
            key: value for key, value in payload.items() if key not in {"id", "delete", "digest"}
        }
        entry.setdefault("type", "influxdb")
        entry.setdefault("server", "127.0.0.1")
        entry.setdefault("port", 8086)
        entry.setdefault("enable", 1)
        servers[server_id] = entry
        metrics["servers"] = servers
        metadata["metrics"] = metrics
        await save_cluster_metadata(request, metadata)
        return {"id": server_id, **entry}

    async def metrics_server_update(request: Request, inputs: dict[str, Any]) -> dict[str, Any]:
        payload = values(inputs)
        server_id = str(payload["id"])
        metadata = await cluster_metadata(request)
        metrics = dict(_metrics(metadata))
        servers = dict(metrics.get("servers") or {})
        if server_id not in servers:
            raise ApiError(404, "metrics server does not exist")
        updated = {
            **servers[server_id],
            **{
                key: value
                for key, value in payload.items()
                if key not in {"id", "delete", "digest"}
            },
        }
        servers[server_id] = updated
        metrics["servers"] = servers
        metadata["metrics"] = metrics
        await save_cluster_metadata(request, metadata)
        return {"id": server_id, **updated}

    async def metrics_server_delete(request: Request, inputs: dict[str, Any]) -> None:
        server_id = str(values(inputs)["id"])
        metadata = await cluster_metadata(request)
        metrics = dict(_metrics(metadata))
        servers = dict(metrics.get("servers") or {})
        if server_id not in servers:
            raise ApiError(404, "metrics server does not exist")
        del servers[server_id]
        metrics["servers"] = servers
        metadata["metrics"] = metrics
        await save_cluster_metadata(request, metadata)

    async def qemu_index(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, str]]:
        return subdirs("cpu-flags", "custom-cpu-models")

    async def qemu_cpu_flags(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]:
        metadata = await cluster_metadata(request)
        flags = metadata.get("qemu_cpu_flags")
        if isinstance(flags, list):
            return [dict(item) for item in flags if isinstance(item, dict)]
        return []

    async def cpu_models_list(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]:
        metadata = await cluster_metadata(request)
        models = _cpu_models(metadata)
        return [
            {"name": name, **dict(payload)}
            for name, payload in sorted(models.items())
            if isinstance(payload, dict)
        ]

    async def cpu_models_create(request: Request, inputs: dict[str, Any]) -> dict[str, Any]:
        payload = values(inputs)
        name = str(payload.get("name") or payload.get("cputype") or "")
        if not name:
            raise ApiError(400, "parameter verification failed - 'name' missing")
        metadata = await cluster_metadata(request)
        models = _cpu_models(metadata)
        if name in models:
            raise ApiError(409, "custom cpu model already exists")
        entry = {
            key: value
            for key, value in payload.items()
            if key not in {"name", "cputype", "delete", "digest"}
        }
        entry.setdefault("vendor", "Custom")
        models[name] = entry
        metadata["qemu_cpu_models"] = models
        await save_cluster_metadata(request, metadata)
        return {"name": name, **entry}

    async def cpu_models_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]:
        name = str(values(inputs)["cputype"])
        metadata = await cluster_metadata(request)
        models = _cpu_models(metadata)
        payload = models.get(name)
        if not isinstance(payload, dict):
            raise ApiError(404, "custom cpu model does not exist")
        return {"name": name, **payload}

    async def cpu_models_update(request: Request, inputs: dict[str, Any]) -> dict[str, Any]:
        payload = values(inputs)
        name = str(payload["cputype"])
        metadata = await cluster_metadata(request)
        models = _cpu_models(metadata)
        if name not in models:
            raise ApiError(404, "custom cpu model does not exist")
        updated = {
            **models[name],
            **{
                key: value
                for key, value in payload.items()
                if key not in {"cputype", "delete", "digest"}
            },
        }
        models[name] = updated
        metadata["qemu_cpu_models"] = models
        await save_cluster_metadata(request, metadata)
        return {"name": name, **updated}

    async def cpu_models_delete(request: Request, inputs: dict[str, Any]) -> None:
        name = str(values(inputs)["cputype"])
        metadata = await cluster_metadata(request)
        models = _cpu_models(metadata)
        if name not in models:
            raise ApiError(404, "custom cpu model does not exist")
        del models[name]
        metadata["qemu_cpu_models"] = models
        await save_cluster_metadata(request, metadata)

    async def bulk_action_index(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, str]]:
        return subdirs("guest")

    async def bulk_guest_index(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, str]]:
        return subdirs("migrate", "shutdown", "start", "suspend")

    async def bulk_guest_action(request: Request, inputs: dict[str, Any], action: str) -> str:
        payload = values(inputs)
        metadata = await cluster_metadata(request)
        metadata["last_bulk_action"] = {
            "action": action,
            "payload": {
                key: value for key, value in payload.items() if key not in {"delete", "digest"}
            },
            "at": int(time.time()),
        }
        await save_cluster_metadata(request, metadata)
        if action == "start":
            await _bulk_guest_status(request, "running")
        elif action == "shutdown":
            await _bulk_guest_status(request, "stopped")
        elif action == "suspend":
            await _bulk_guest_status(request, "paused")
        elif action == "migrate":
            target = str(payload.get("target") or "")
            if target:
                target_row = await database(request).pool.fetchrow(
                    "SELECT id FROM nodes WHERE name=$1", target
                )
                if target_row is None:
                    raise ApiError(404, "target node does not exist")
                vms = payload.get("vms") or payload.get("guests") or ""
                if isinstance(vms, str) and vms:
                    ids = [part.strip() for part in vms.split(",") if part.strip()]
                    for vmid in ids:
                        await database(request).pool.execute(
                            """UPDATE resources SET node_id=$2, updated_at=now()
                            WHERE kind IN ('qemu', 'lxc') AND external_id=$1""",
                            vmid,
                            target_row["id"],
                        )
        return await _cluster_task(request, task_type=f"bulk-{action}", worker=f"bulk{action}")

    async def cluster_ceph_index(
        _request: Request, _inputs: dict[str, Any]
    ) -> list[dict[str, str]]:
        return subdirs("flags", "metadata", "status")

    async def ceph_flags_get(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]:
        metadata = await cluster_metadata(request)
        ceph = _ceph(metadata)
        stored = ceph.get("flags")
        flags = default_ceph_flags()
        if isinstance(stored, dict):
            for name, value in stored.items():
                if name in flags:
                    flags[name] = int(value)
        return [
            {
                "name": name,
                "value": bool(flags[name]),
                "description": description,
            }
            for name, description in sorted(CEPH_FLAG_DESCRIPTIONS.items())
        ]

    async def ceph_flags_put(request: Request, inputs: dict[str, Any]) -> str:
        payload = values(inputs)
        metadata = await cluster_metadata(request)
        ceph = dict(_ceph(metadata))
        flags = default_ceph_flags()
        stored = ceph.get("flags")
        if isinstance(stored, dict):
            for name, value in stored.items():
                if name in flags:
                    flags[name] = int(value)
        for key, value in payload.items():
            if key in {"delete", "digest"}:
                continue
            if key not in CEPH_FLAG_DESCRIPTIONS:
                raise ApiError(400, f"unknown ceph flag '{key}'")
            flags[str(key)] = int(value)
        ceph["flags"] = flags
        metadata["ceph"] = ceph
        await save_cluster_metadata(request, metadata)
        return await _cluster_task(request, task_type="ceph-flags", worker="cephflags")

    async def ceph_flag_get(request: Request, inputs: dict[str, Any]) -> bool:
        flag = str(values(inputs)["flag"])
        if flag not in CEPH_FLAG_DESCRIPTIONS:
            raise ApiError(404, "ceph flag does not exist")
        metadata = await cluster_metadata(request)
        ceph = _ceph(metadata)
        flags = ceph.get("flags")
        if isinstance(flags, dict) and flag in flags:
            return bool(int(flags[flag]))
        return False

    async def ceph_flag_put(request: Request, inputs: dict[str, Any]) -> None:
        payload = values(inputs)
        flag = str(payload["flag"])
        if flag not in CEPH_FLAG_DESCRIPTIONS:
            raise ApiError(404, "ceph flag does not exist")
        metadata = await cluster_metadata(request)
        ceph = dict(_ceph(metadata))
        flags = default_ceph_flags()
        stored = ceph.get("flags")
        if isinstance(stored, dict):
            for name, value in stored.items():
                if name in flags:
                    flags[name] = int(value)
        if "value" in payload:
            flags[flag] = int(payload["value"])
        elif flag in payload:
            flags[flag] = int(payload[flag])
        else:
            flags[flag] = 1
        ceph["flags"] = flags
        metadata["ceph"] = ceph
        await save_cluster_metadata(request, metadata)

    async def ceph_metadata(request: Request, _inputs: dict[str, Any]) -> dict[str, Any]:
        metadata = await cluster_metadata(request)
        ceph = _ceph(metadata)
        config_raw = ceph.get("config")
        config: dict[str, Any] = dict(config_raw) if isinstance(config_raw, dict) else {}
        version = ceph.get("version")
        nodes = await database(request).pool.fetch("SELECT name FROM nodes ORDER BY name")
        node_names = [str(row["name"]) for row in nodes]
        version_str = (
            str(version.get("str"))
            if isinstance(version, dict) and version.get("str")
            else "18.2.2"
        )
        version_short = ".".join(version_str.split(".")[:2]) if version_str else "18.2"
        osd_rows = await database(request).pool.fetch(
            """SELECT r.external_id, r.state, n.name AS node
            FROM resources r JOIN nodes n ON n.id=r.node_id
            WHERE r.kind='ceph-osd' ORDER BY r.external_id"""
        )
        osd_map: dict[str, Any] = {}
        for row in osd_rows:
            payload = state(row["state"])
            external = str(row["external_id"])
            osd_id = str(payload.get("osd_id", external.removeprefix("osd.")))
            hostname = str(row["node"])
            osd_map[osd_id] = {
                "id": int(osd_id) if osd_id.isdigit() else osd_id,
                "hostname": hostname,
                "name": f"osd.{osd_id}",
                "ceph_version": version_str,
                "ceph_version_short": version_short,
                "ceph_release": "reef",
                "device_class": payload.get("device_class", "hdd"),
                "dev": payload.get("dev", ""),
            }
        if not osd_map:
            fallback_host = node_names[0] if node_names else "pve01"
            osd_map["0"] = {
                "id": 0,
                "hostname": fallback_host,
                "name": "osd.0",
                "ceph_version": version_str,
                "ceph_version_short": version_short,
                "ceph_release": "reef",
            }

        def _service_entry(name: str, index: int) -> dict[str, Any]:
            return {
                "name": name,
                "hostname": name,
                "addr": f"10.0.0.{index}",
                "ceph_version": version_str,
                "ceph_version_short": version_short,
                "ceph_release": "reef",
            }

        return {
            "mds": {},
            "mgr": {
                f"mgr@{node}": _service_entry(node, index)
                for index, node in enumerate(node_names[:1] or ["pve01"], start=1)
            },
            "mon": {
                f"mon@{node}": _service_entry(node, index)
                for index, node in enumerate(node_names[:3] or ["pve01"], start=1)
            },
            "node": {
                node: {
                    "name": node,
                    "hostname": node,
                    "ceph_version": version_str,
                    "ceph_version_short": version_short,
                }
                for node in (node_names or ["pve01"])
            },
            "osd": list(osd_map.values()),
            "fsid": str(config.get("fsid") or ""),
        }

    async def ha_rule_create(request: Request, inputs: dict[str, Any]) -> None:
        payload = values(inputs)
        rule = str(payload.get("rule") or payload.get("name") or "")
        if not rule:
            raise ApiError(400, "parameter verification failed - 'rule' missing")
        metadata = await cluster_metadata(request)
        rules = _ha_rules_store(metadata)
        if any(str(item.get("rule")) == rule for item in rules):
            raise ApiError(409, "HA rule already exists")
        entry = {key: value for key, value in payload.items() if key not in {"delete", "digest"}}
        entry["rule"] = rule
        entry.setdefault("type", "resource")
        entry.setdefault("action", "migrate")
        rules.append(entry)
        _save_ha_rules(metadata, rules)
        await save_cluster_metadata(request, metadata)

    async def ha_rule_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]:
        rule = str(values(inputs)["rule"])
        metadata = await cluster_metadata(request)
        for item in _ha_rules_store(metadata):
            if str(item.get("rule")) == rule:
                return dict(item)
        raise ApiError(404, "HA rule does not exist")

    async def ha_rule_update(request: Request, inputs: dict[str, Any]) -> None:
        payload = values(inputs)
        rule = str(payload["rule"])
        metadata = await cluster_metadata(request)
        rules = _ha_rules_store(metadata)
        updated: list[dict[str, Any]] = []
        found = False
        for item in rules:
            if str(item.get("rule")) != rule:
                updated.append(item)
                continue
            found = True
            merged = {
                **item,
                **{
                    key: value
                    for key, value in payload.items()
                    if key not in {"rule", "delete", "digest"}
                },
            }
            merged["rule"] = rule
            updated.append(merged)
        if not found:
            raise ApiError(404, "HA rule does not exist")
        _save_ha_rules(metadata, updated)
        await save_cluster_metadata(request, metadata)

    async def ha_rule_delete(request: Request, inputs: dict[str, Any]) -> None:
        rule = str(values(inputs)["rule"])
        metadata = await cluster_metadata(request)
        rules = _ha_rules_store(metadata)
        remaining = [item for item in rules if str(item.get("rule")) != rule]
        if len(remaining) == len(rules):
            raise ApiError(404, "HA rule does not exist")
        _save_ha_rules(metadata, remaining)
        await save_cluster_metadata(request, metadata)

    async def node_replication_list(
        request: Request, inputs: dict[str, Any]
    ) -> list[dict[str, Any]]:
        node = str(values(inputs)["node"])
        await require_node(request, node)
        metadata = await cluster_metadata(request)
        jobs = _replication_jobs(metadata)
        return [
            job
            for job in jobs
            if str(job.get("source") or job.get("node") or node) == node
            or job.get("source") is None
        ]

    async def node_replication_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]:
        node = str(values(inputs)["node"])
        job_id = str(values(inputs)["id"])
        await require_node(request, node)
        for job in _replication_jobs(await cluster_metadata(request)):
            if str(job.get("id")) == job_id:
                return dict(job)
        raise ApiError(404, "replication job does not exist")

    async def node_replication_log(
        request: Request, inputs: dict[str, Any]
    ) -> list[dict[str, Any]]:
        job = await node_replication_get(request, inputs)
        log = job.get("log")
        if isinstance(log, list):
            return [dict(item) for item in log if isinstance(item, dict)]
        return []

    async def node_replication_status(request: Request, inputs: dict[str, Any]) -> dict[str, Any]:
        job = await node_replication_get(request, inputs)
        return {
            "id": job.get("id"),
            "last_sync": job.get("last_sync", 0),
            "duration": job.get("duration", 0),
            "fail_count": job.get("fail_count", 0),
            "error": job.get("error", ""),
            "state": job.get("state", "OK"),
        }

    async def node_replication_schedule_now(request: Request, inputs: dict[str, Any]) -> None:
        node = str(values(inputs)["node"])
        job_id = str(values(inputs)["id"])
        await require_node(request, node)
        metadata = await cluster_metadata(request)
        jobs = _replication_jobs(metadata)
        found = False
        for job in jobs:
            if str(job.get("id")) != job_id:
                continue
            found = True
            job["last_sync"] = int(time.time())
            job["state"] = "OK"
            job["schedule_now"] = 1
        if not found:
            raise ApiError(404, "replication job does not exist")
        metadata["replication"] = jobs
        await save_cluster_metadata(request, metadata)

    registry.register("/cluster/jobs", "GET", jobs_index)
    registry.register("/cluster/jobs/realm-sync", "GET", realm_sync_list)
    registry.register("/cluster/jobs/realm-sync/{id}", "GET", realm_sync_get)
    registry.register("/cluster/jobs/realm-sync/{id}", "POST", realm_sync_create)
    registry.register("/cluster/jobs/realm-sync/{id}", "PUT", realm_sync_update)
    registry.register("/cluster/jobs/realm-sync/{id}", "DELETE", realm_sync_delete)
    registry.register("/cluster/jobs/schedule-analyze", "GET", schedule_analyze)

    registry.register("/cluster/metrics", "GET", metrics_index)
    registry.register("/cluster/metrics/export", "GET", metrics_export)
    registry.register("/cluster/metrics/server", "GET", metrics_server_list)
    registry.register("/cluster/metrics/server/{id}", "GET", metrics_server_get)
    registry.register("/cluster/metrics/server/{id}", "POST", metrics_server_create)
    registry.register("/cluster/metrics/server/{id}", "PUT", metrics_server_update)
    registry.register("/cluster/metrics/server/{id}", "DELETE", metrics_server_delete)

    registry.register("/cluster/qemu", "GET", qemu_index)
    registry.register("/cluster/qemu/cpu-flags", "GET", qemu_cpu_flags)
    registry.register("/cluster/qemu/custom-cpu-models", "GET", cpu_models_list)
    registry.register("/cluster/qemu/custom-cpu-models", "POST", cpu_models_create)
    registry.register("/cluster/qemu/custom-cpu-models/{cputype}", "GET", cpu_models_get)
    registry.register("/cluster/qemu/custom-cpu-models/{cputype}", "PUT", cpu_models_update)
    registry.register("/cluster/qemu/custom-cpu-models/{cputype}", "DELETE", cpu_models_delete)

    registry.register("/cluster/bulk-action", "GET", bulk_action_index)
    registry.register("/cluster/bulk-action/guest", "GET", bulk_guest_index)
    registry.register(
        "/cluster/bulk-action/guest/migrate",
        "POST",
        lambda request, inputs: bulk_guest_action(request, inputs, "migrate"),
    )
    registry.register(
        "/cluster/bulk-action/guest/shutdown",
        "POST",
        lambda request, inputs: bulk_guest_action(request, inputs, "shutdown"),
    )
    registry.register(
        "/cluster/bulk-action/guest/start",
        "POST",
        lambda request, inputs: bulk_guest_action(request, inputs, "start"),
    )
    registry.register(
        "/cluster/bulk-action/guest/suspend",
        "POST",
        lambda request, inputs: bulk_guest_action(request, inputs, "suspend"),
    )

    registry.register("/cluster/ceph", "GET", cluster_ceph_index)
    registry.register("/cluster/ceph/flags", "GET", ceph_flags_get)
    registry.register("/cluster/ceph/flags", "PUT", ceph_flags_put)
    registry.register("/cluster/ceph/flags/{flag}", "GET", ceph_flag_get)
    registry.register("/cluster/ceph/flags/{flag}", "PUT", ceph_flag_put)
    registry.register("/cluster/ceph/metadata", "GET", ceph_metadata)

    registry.register("/cluster/ha/rules", "POST", ha_rule_create)
    registry.register("/cluster/ha/rules/{rule}", "GET", ha_rule_get)
    registry.register("/cluster/ha/rules/{rule}", "PUT", ha_rule_update)
    registry.register("/cluster/ha/rules/{rule}", "DELETE", ha_rule_delete)

    registry.register("/nodes/{node}/replication", "GET", node_replication_list)
    registry.register("/nodes/{node}/replication/{id}", "GET", node_replication_get)
    registry.register("/nodes/{node}/replication/{id}/log", "GET", node_replication_log)
    registry.register("/nodes/{node}/replication/{id}/status", "GET", node_replication_status)
    registry.register(
        "/nodes/{node}/replication/{id}/schedule_now", "POST", node_replication_schedule_now
    )