Sergey Antropoff
777926487b
initial QEMU slice, backed by imported contracts for majors 6–9. - Implement durable handlers for access/auth, cluster, LXC, storage, HA, firewall, Ceph, SDN, ACME, notifications, pools, mapping, and node ops - Serve an interactive Web UI with catalog browsing, demo seed controls, and OpenAPI/help surfaces - Bundle PVE 6.4-15, 7.4-16, and 8.4.5 contract revisions alongside 9.2.3 - Support in-memory runtime contract Apply (POST /ui/api/contract/apply) so /version and /api2 routes follow the selected major until restart - Expand seed profiles (including demo-cluster), migrations 007–008, TLS gateway config, Compose/Makefile tooling, and compatibility evidence - Tighten .gitignore for macOS, hidden directories (.*/), and local secrets
351 lines
14 KiB
Python
351 lines
14 KiB
Python
"""High availability semantic handlers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import Any
|
|
|
|
from fastapi import Request
|
|
|
|
from app.api.errors import ApiError
|
|
from app.api.registry import HandlerRegistry
|
|
from app.handlers.common import (
|
|
cluster_metadata,
|
|
database,
|
|
save_cluster_metadata,
|
|
state,
|
|
subdirs,
|
|
values,
|
|
)
|
|
from app.simulation.seed import CLUSTER_ID, stable_id
|
|
|
|
|
|
def _ha_groups(metadata: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
|
groups = metadata.get("ha_groups", {})
|
|
if not isinstance(groups, dict):
|
|
return {}
|
|
return {str(key): dict(value) for key, value in groups.items() if isinstance(value, dict)}
|
|
|
|
|
|
def register_ha_handlers(registry: HandlerRegistry) -> None:
|
|
async def ha_index(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, str]]:
|
|
return subdirs("groups", "resources", "rules", "status")
|
|
|
|
async def ha_resources(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]:
|
|
rows = await database(_request).pool.fetch(
|
|
"""SELECT r.external_id, r.state, n.name AS node
|
|
FROM resources r JOIN nodes n ON n.id=r.node_id
|
|
WHERE r.kind='ha' ORDER BY r.external_id"""
|
|
)
|
|
result: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
payload = state(row["state"])
|
|
sid = str(row["external_id"])
|
|
result.append(
|
|
{
|
|
"sid": sid,
|
|
"type": "vm" if sid.startswith("vm:") else "ct",
|
|
"state": payload.get("state", "started"),
|
|
"group": payload.get("group"),
|
|
"node": str(row["node"]),
|
|
"max_relocate": payload.get("max_relocate", 1),
|
|
"max_restart": payload.get("max_restart", 1),
|
|
}
|
|
)
|
|
return result
|
|
|
|
async def ha_resource_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]:
|
|
sid = str(values(inputs)["sid"])
|
|
items = await ha_resources(request, inputs)
|
|
for item in items:
|
|
if item["sid"] == sid:
|
|
return item
|
|
raise ApiError(404, "HA resource does not exist")
|
|
|
|
async def ha_resource_create(request: Request, inputs: dict[str, Any]) -> None:
|
|
payload = values(inputs)
|
|
sid = str(payload["sid"])
|
|
group = str(payload.get("group") or "")
|
|
exists = await database(request).pool.fetchval(
|
|
"""SELECT EXISTS(SELECT 1 FROM resources WHERE kind='ha' AND external_id=$1)""",
|
|
sid,
|
|
)
|
|
if exists:
|
|
raise ApiError(409, "HA resource already exists")
|
|
guest_kind, _, guest_id = sid.partition(":")
|
|
if guest_kind not in {"vm", "ct"} or not guest_id.isdigit():
|
|
raise ApiError(400, "invalid HA resource sid")
|
|
resource_kind = "qemu" if guest_kind == "vm" else "lxc"
|
|
guest = await database(request).pool.fetchrow(
|
|
"""SELECT r.id, n.name FROM resources r JOIN nodes n ON n.id=r.node_id
|
|
WHERE r.kind=$1 AND r.external_id=$2""",
|
|
resource_kind,
|
|
guest_id,
|
|
)
|
|
if guest is None:
|
|
raise ApiError(404, "guest does not exist")
|
|
node = await database(request).pool.fetchrow(
|
|
"SELECT id FROM nodes WHERE name=$1",
|
|
str(guest["name"]),
|
|
)
|
|
if node is None:
|
|
raise ApiError(404, "node does not exist")
|
|
ha_state = {
|
|
"state": str(payload.get("state") or "started"),
|
|
"group": group or None,
|
|
"max_relocate": int(payload.get("max_relocate") or 1),
|
|
"max_restart": int(payload.get("max_restart") or 1),
|
|
}
|
|
await database(request).pool.execute(
|
|
"""INSERT INTO resources(id, node_id, cluster_id, kind, external_id, state, metadata)
|
|
VALUES($1, $2, $3, 'ha', $4, $5::jsonb, '{}'::jsonb)""",
|
|
stable_id(f"ha:{sid}"),
|
|
node["id"],
|
|
CLUSTER_ID,
|
|
sid,
|
|
json.dumps(ha_state, sort_keys=True),
|
|
)
|
|
|
|
async def ha_resource_update(request: Request, inputs: dict[str, Any]) -> None:
|
|
sid = str(values(inputs)["sid"])
|
|
payload = values(inputs)
|
|
row = await database(request).pool.fetchrow(
|
|
"SELECT id, state FROM resources WHERE kind='ha' AND external_id=$1",
|
|
sid,
|
|
)
|
|
if row is None:
|
|
raise ApiError(404, "HA resource does not exist")
|
|
current = state(row["state"])
|
|
updated = {
|
|
**current,
|
|
**{
|
|
key: value
|
|
for key, value in payload.items()
|
|
if key not in {"sid", "delete", "digest"}
|
|
},
|
|
}
|
|
await database(request).pool.execute(
|
|
"UPDATE resources SET state=$2::jsonb, updated_at=now() WHERE id=$1",
|
|
row["id"],
|
|
json.dumps(updated, sort_keys=True),
|
|
)
|
|
|
|
async def ha_resource_delete(request: Request, inputs: dict[str, Any]) -> None:
|
|
sid = str(values(inputs)["sid"])
|
|
status = await database(request).pool.execute(
|
|
"DELETE FROM resources WHERE kind='ha' AND external_id=$1",
|
|
sid,
|
|
)
|
|
if status != "DELETE 1":
|
|
raise ApiError(404, "HA resource does not exist")
|
|
|
|
async def ha_groups(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]:
|
|
metadata = await cluster_metadata(request)
|
|
configured = _ha_groups(metadata)
|
|
result = [
|
|
{
|
|
"group": group_id,
|
|
"nodes": str(payload.get("nodes", "")),
|
|
"nofailback": int(payload.get("nofailback", 0)),
|
|
"restricted": int(payload.get("restricted", 0)),
|
|
"type": "group",
|
|
"comment": payload.get("comment", ""),
|
|
}
|
|
for group_id, payload in sorted(configured.items())
|
|
]
|
|
if result:
|
|
return result
|
|
rows = await database(request).pool.fetch(
|
|
"""SELECT DISTINCT state->>'group' AS group_id
|
|
FROM resources WHERE kind='ha' AND state ? 'group'
|
|
ORDER BY 1"""
|
|
)
|
|
node_names = await database(request).pool.fetch("SELECT name FROM nodes ORDER BY name")
|
|
nodes = ",".join(str(row["name"]) for row in node_names) or "pve01"
|
|
return [
|
|
{
|
|
"group": str(row["group_id"]),
|
|
"nodes": nodes,
|
|
"nofailback": 0,
|
|
"restricted": 0,
|
|
"type": "group",
|
|
}
|
|
for row in rows
|
|
if row["group_id"]
|
|
]
|
|
|
|
async def ha_group_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]:
|
|
group = str(values(inputs)["group"])
|
|
for item in await ha_groups(request, inputs):
|
|
if item["group"] == group:
|
|
return item
|
|
raise ApiError(404, "HA group does not exist")
|
|
|
|
async def ha_group_create(request: Request, inputs: dict[str, Any]) -> None:
|
|
payload = values(inputs)
|
|
group = str(payload["group"])
|
|
metadata = await cluster_metadata(request)
|
|
groups = _ha_groups(metadata)
|
|
if group in groups:
|
|
raise ApiError(409, "HA group already exists")
|
|
groups[group] = {
|
|
"nodes": str(payload.get("nodes") or ""),
|
|
"nofailback": int(payload.get("nofailback") or 0),
|
|
"restricted": int(payload.get("restricted") or 0),
|
|
"comment": str(payload.get("comment") or ""),
|
|
}
|
|
metadata["ha_groups"] = groups
|
|
await save_cluster_metadata(request, metadata)
|
|
|
|
async def ha_group_update(request: Request, inputs: dict[str, Any]) -> None:
|
|
group = str(values(inputs)["group"])
|
|
metadata = await cluster_metadata(request)
|
|
groups = _ha_groups(metadata)
|
|
if group not in groups:
|
|
raise ApiError(404, "HA group does not exist")
|
|
payload = values(inputs)
|
|
groups[group] = {
|
|
**groups[group],
|
|
**{
|
|
key: value
|
|
for key, value in payload.items()
|
|
if key not in {"group", "delete", "digest"}
|
|
},
|
|
}
|
|
metadata["ha_groups"] = groups
|
|
await save_cluster_metadata(request, metadata)
|
|
|
|
async def ha_group_delete(request: Request, inputs: dict[str, Any]) -> None:
|
|
group = str(values(inputs)["group"])
|
|
metadata = await cluster_metadata(request)
|
|
groups = _ha_groups(metadata)
|
|
if group not in groups:
|
|
raise ApiError(404, "HA group does not exist")
|
|
del groups[group]
|
|
metadata["ha_groups"] = groups
|
|
await save_cluster_metadata(request, metadata)
|
|
|
|
async def ha_status(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, str]]:
|
|
return subdirs("current", "manager_status")
|
|
|
|
async def ha_status_current(request: Request, _inputs: dict[str, Any]) -> dict[str, Any]:
|
|
row = await database(request).pool.fetchrow(
|
|
"""SELECT count(*) FILTER (WHERE state->>'state' = 'started') AS started,
|
|
count(*) AS total
|
|
FROM resources WHERE kind='ha'"""
|
|
)
|
|
master = await database(request).pool.fetchval(
|
|
"SELECT name FROM nodes WHERE status='online' ORDER BY name LIMIT 1"
|
|
)
|
|
metadata = await cluster_metadata(request)
|
|
ha = metadata.get("ha", {}) if isinstance(metadata.get("ha"), dict) else {}
|
|
return {
|
|
"quorate": 1,
|
|
"mode": "active" if ha.get("armed", True) else "disabled",
|
|
"master_node": str(master or "pve01"),
|
|
"ha_started": int(row["started"] or 0),
|
|
"ha_total": int(row["total"] or 0),
|
|
"armed": 1 if ha.get("armed", True) else 0,
|
|
}
|
|
|
|
async def ha_manager_status(request: Request, _inputs: dict[str, Any]) -> dict[str, Any]:
|
|
metadata = await cluster_metadata(request)
|
|
ha = metadata.get("ha", {}) if isinstance(metadata.get("ha"), dict) else {}
|
|
armed = bool(ha.get("armed", True))
|
|
return {
|
|
"manager_status": "active" if armed else "disabled",
|
|
"quorum": "OK",
|
|
"armed": 1 if armed else 0,
|
|
}
|
|
|
|
async def ha_rules(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]:
|
|
metadata = await cluster_metadata(request)
|
|
rules = metadata.get("ha_rules")
|
|
if isinstance(rules, list) and rules:
|
|
return [dict(item) for item in rules if isinstance(item, dict)]
|
|
defaults = [
|
|
{"rule": "node-fencing", "type": "node", "action": "restart"},
|
|
{"rule": "service-ha", "type": "resource", "action": "failover"},
|
|
]
|
|
metadata["ha_rules"] = defaults
|
|
await save_cluster_metadata(request, metadata)
|
|
return list(defaults)
|
|
|
|
async def ha_relocate(request: Request, inputs: dict[str, Any]) -> None:
|
|
payload = values(inputs)
|
|
sid = str(payload["sid"])
|
|
target = str(payload.get("node") or payload.get("target") or "")
|
|
if not target:
|
|
raise ApiError(400, "parameter verification failed - target node missing")
|
|
ha_row = await database(request).pool.fetchrow(
|
|
"SELECT id, state FROM resources WHERE kind='ha' AND external_id=$1",
|
|
sid,
|
|
)
|
|
if ha_row is None:
|
|
raise ApiError(404, "HA resource does not exist")
|
|
node = await database(request).pool.fetchrow(
|
|
"SELECT id, name FROM nodes WHERE name=$1",
|
|
target,
|
|
)
|
|
if node is None:
|
|
raise ApiError(404, "node does not exist")
|
|
guest_kind, _, guest_id = sid.partition(":")
|
|
resource_kind = "qemu" if guest_kind == "vm" else "lxc"
|
|
await database(request).pool.execute(
|
|
"UPDATE resources SET node_id=$2, updated_at=now() WHERE kind='ha' AND external_id=$1",
|
|
sid,
|
|
node["id"],
|
|
)
|
|
await database(request).pool.execute(
|
|
"""UPDATE resources SET node_id=$3, updated_at=now()
|
|
WHERE kind=$1 AND external_id=$2""",
|
|
resource_kind,
|
|
guest_id,
|
|
node["id"],
|
|
)
|
|
current = state(ha_row["state"])
|
|
current["node"] = target
|
|
current["state"] = current.get("state") or "started"
|
|
await database(request).pool.execute(
|
|
"UPDATE resources SET state=$2::jsonb, updated_at=now() WHERE id=$1",
|
|
ha_row["id"],
|
|
json.dumps(current, sort_keys=True),
|
|
)
|
|
|
|
async def ha_migrate(request: Request, inputs: dict[str, Any]) -> None:
|
|
await ha_relocate(request, inputs)
|
|
|
|
async def ha_arm(request: Request, _inputs: dict[str, Any]) -> None:
|
|
metadata = await cluster_metadata(request)
|
|
ha = dict(metadata.get("ha") or {})
|
|
ha["armed"] = True
|
|
metadata["ha"] = ha
|
|
await save_cluster_metadata(request, metadata)
|
|
|
|
async def ha_disarm(request: Request, _inputs: dict[str, Any]) -> None:
|
|
metadata = await cluster_metadata(request)
|
|
ha = dict(metadata.get("ha") or {})
|
|
ha["armed"] = False
|
|
metadata["ha"] = ha
|
|
await save_cluster_metadata(request, metadata)
|
|
|
|
registry.register("/cluster/ha", "GET", ha_index)
|
|
registry.register("/cluster/ha/groups", "GET", ha_groups)
|
|
registry.register("/cluster/ha/groups", "POST", ha_group_create)
|
|
registry.register("/cluster/ha/groups/{group}", "GET", ha_group_get)
|
|
registry.register("/cluster/ha/groups/{group}", "PUT", ha_group_update)
|
|
registry.register("/cluster/ha/groups/{group}", "DELETE", ha_group_delete)
|
|
registry.register("/cluster/ha/resources", "GET", ha_resources)
|
|
registry.register("/cluster/ha/resources", "POST", ha_resource_create)
|
|
registry.register("/cluster/ha/resources/{sid}", "GET", ha_resource_get)
|
|
registry.register("/cluster/ha/resources/{sid}", "PUT", ha_resource_update)
|
|
registry.register("/cluster/ha/resources/{sid}", "DELETE", ha_resource_delete)
|
|
registry.register("/cluster/ha/status", "GET", ha_status)
|
|
registry.register("/cluster/ha/status/current", "GET", ha_status_current)
|
|
registry.register("/cluster/ha/status/manager_status", "GET", ha_manager_status)
|
|
registry.register("/cluster/ha/rules", "GET", ha_rules)
|
|
registry.register("/cluster/ha/resources/{sid}/migrate", "POST", ha_migrate)
|
|
registry.register("/cluster/ha/resources/{sid}/relocate", "POST", ha_relocate)
|
|
registry.register("/cluster/ha/status/arm-ha", "POST", ha_arm)
|
|
registry.register("/cluster/ha/status/disarm-ha", "POST", ha_disarm)
|