Sergey Antropoff
0773f721ea
- Scale small/large/big seeds (3×50 / 10×1000 / 20×2000) with proportional backups, snapshots, HA, replication, Ceph capacity, and OSD totals (10 / 100 / 500) plus matching node disks and crush/pg metadata - Enrich handler responses for apt, certificates, qemu/lxc status, storage, SDN, metrics export, and related cluster/node dumps - Flatten nested body_example fields into PARAMS and sync the request body via dotted paths (oVirt-style) - Restyle DATA controls as size cards with full-width Reset to minimal / Refresh stats; unload reloads the minimal cluster
410 lines
17 KiB
Python
410 lines
17 KiB
Python
"""Cluster config / join / totem handlers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import secrets
|
|
from typing import Any
|
|
|
|
from fastapi import Request
|
|
|
|
from app.api.errors import ApiError
|
|
from app.api.registry import HandlerRegistry
|
|
from app.handlers.common import (
|
|
cluster_metadata,
|
|
database,
|
|
require_value,
|
|
save_cluster_metadata,
|
|
values,
|
|
)
|
|
from app.tasks.repository import TaskRepository
|
|
from app.tasks.upid import Upid
|
|
|
|
|
|
def _config(metadata: dict[str, Any]) -> dict[str, Any]:
|
|
current = metadata.get("cluster_config")
|
|
return current if isinstance(current, dict) else {}
|
|
|
|
|
|
def _principal(request: Request) -> str:
|
|
return str(getattr(request.state, "principal", None) or "root@pam")
|
|
|
|
|
|
def _fingerprint_sha256() -> str:
|
|
return ":".join(f"{byte:02X}" for byte in secrets.token_bytes(32))
|
|
|
|
|
|
def _stable_digest(material: str) -> str:
|
|
return hashlib.sha256(material.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _node_addr(node: str, index: int, entry: dict[str, Any]) -> str:
|
|
for key in ("pve_addr", "new_node_ip", "ring0_addr"):
|
|
value = entry.get(key)
|
|
if isinstance(value, str) and value and not value.endswith(".local"):
|
|
# Prefer literal IPs; hostnames are still accepted as ring addresses.
|
|
if key != "ring0_addr" or all(part.isdigit() for part in value.split(".")):
|
|
return value
|
|
return f"10.0.0.{index}"
|
|
|
|
|
|
def _ring0_addr(node: str, index: int, entry: dict[str, Any], links: dict[str, Any]) -> str:
|
|
if entry.get("ring0_addr"):
|
|
return str(entry["ring0_addr"])
|
|
link0 = links.get("link0")
|
|
if isinstance(link0, str) and link0:
|
|
# Property string may be bare IP or address=IP[,priority=N].
|
|
if link0.startswith("address="):
|
|
return link0.removeprefix("address=").split(",", 1)[0]
|
|
return link0.split(",", 1)[0]
|
|
if entry.get("new_node_ip"):
|
|
return str(entry["new_node_ip"])
|
|
return f"10.0.0.{index}"
|
|
|
|
|
|
def _render_corosync_conf(config: dict[str, Any], node_names: list[str]) -> str:
|
|
clustername = str(config.get("clustername") or "proxmox")
|
|
totem = dict(config.get("totem") or {})
|
|
added_raw = config.get("added_nodes")
|
|
added: dict[str, Any] = dict(added_raw) if isinstance(added_raw, dict) else {}
|
|
links = dict(config.get("links") or {})
|
|
lines = [
|
|
"totem {",
|
|
" version: 2",
|
|
f" cluster_name: {totem.get('cluster_name', clustername)}",
|
|
f" secauth: {totem.get('secauth', 'on')}",
|
|
"}",
|
|
"nodelist {",
|
|
]
|
|
for index, name in enumerate(node_names, start=1):
|
|
entry_raw = added.get(name)
|
|
entry: dict[str, Any] = dict(entry_raw) if isinstance(entry_raw, dict) else {}
|
|
nodeid = int(entry.get("nodeid") or index)
|
|
votes = int(entry.get("quorum_votes") or entry.get("votes") or config.get("votes") or 1)
|
|
ring0 = _ring0_addr(name, index, entry, links)
|
|
lines.extend(
|
|
[
|
|
" node {",
|
|
f" name: {name}",
|
|
f" nodeid: {nodeid}",
|
|
f" quorum_votes: {votes}",
|
|
f" ring0_addr: {ring0}",
|
|
" }",
|
|
]
|
|
)
|
|
lines.append("}")
|
|
lines.extend(["quorum {", " provider: corosync_votequorum", "}"])
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def _ensure_corosync_materials(
|
|
config: dict[str, Any], node_names: list[str]
|
|
) -> tuple[str, str, str]:
|
|
authkey = config.get("corosync_authkey")
|
|
if not isinstance(authkey, str) or not authkey:
|
|
authkey = secrets.token_hex(32)
|
|
config["corosync_authkey"] = authkey
|
|
conf = _render_corosync_conf(config, node_names)
|
|
config["corosync_conf"] = conf
|
|
digest = _stable_digest(conf)
|
|
config["config_digest"] = digest
|
|
return authkey, conf, digest
|
|
|
|
|
|
async def _cluster_task(
|
|
request: Request, *, task_type: str, worker: str, task_id: str = "0"
|
|
) -> str:
|
|
from app.db.primitives import ConflictError
|
|
|
|
pool = database(request).pool
|
|
node = await pool.fetchval("SELECT name FROM nodes ORDER BY name LIMIT 1") or "localhost"
|
|
upid = str(Upid.allocate(str(node), worker, task_id, _principal(request)))
|
|
try:
|
|
task = await TaskRepository(pool).create(
|
|
upid=upid,
|
|
task_type=task_type,
|
|
payload={"cluster": True, "worker": worker},
|
|
resource_key=f"cluster:{task_type}:{secrets.token_hex(4)}",
|
|
)
|
|
except ConflictError as error:
|
|
raise ApiError(409, str(error)) from error
|
|
return task.upid
|
|
|
|
|
|
def register_cluster_config_handlers(registry: HandlerRegistry) -> None:
|
|
async def index(_request: Request, _inputs: dict[str, Any]) -> list[dict[str, str]]:
|
|
# PVE uses {name}; contract child link is href="{name}".
|
|
return [{"name": name} for name in ("nodes", "totem", "join", "qdevice", "apiversion")]
|
|
|
|
async def create(request: Request, inputs: dict[str, Any]) -> str:
|
|
payload = values(inputs)
|
|
clustername = str(require_value(payload, "clustername"))
|
|
metadata = await cluster_metadata(request)
|
|
config = dict(_config(metadata))
|
|
# Match PVE: refuse when corosync materials already exist (seeded or prior create).
|
|
if config.get("corosync_conf") or config.get("config_digest"):
|
|
raise ApiError(400, "cluster config already exists")
|
|
config["clustername"] = clustername
|
|
totem = dict(config.get("totem") or {})
|
|
totem["version"] = int(totem.get("version") or 2)
|
|
totem["secauth"] = totem.get("secauth") or "on"
|
|
totem["cluster_name"] = clustername
|
|
# PVE defaults token-coefficient to 125 when omitted (majors that expose it).
|
|
totem["token_coefficient"] = payload.get("token-coefficient", 125)
|
|
config["totem"] = totem
|
|
if "votes" in payload:
|
|
config["votes"] = payload["votes"]
|
|
if "nodeid" in payload:
|
|
config["creator_nodeid"] = payload["nodeid"]
|
|
links = {key: value for key, value in payload.items() if key.startswith("link")}
|
|
if links:
|
|
config["links"] = links
|
|
config["token"] = secrets.token_hex(16)
|
|
|
|
rows = await database(request).pool.fetch("SELECT name FROM nodes ORDER BY name")
|
|
node_names = [str(row["name"]) for row in rows]
|
|
if not node_names:
|
|
node_names = ["localhost"]
|
|
creator = node_names[0]
|
|
added = dict(config.get("added_nodes") or {})
|
|
creator_entry = dict(added.get(creator) or {})
|
|
creator_entry.update(
|
|
{
|
|
"node": creator,
|
|
"nodeid": payload.get("nodeid") or creator_entry.get("nodeid") or 1,
|
|
"quorum_votes": payload.get("votes") or creator_entry.get("quorum_votes") or 1,
|
|
"ring0_addr": _ring0_addr(
|
|
creator, 1, creator_entry, dict(config.get("links") or {})
|
|
),
|
|
"pve_addr": creator_entry.get("pve_addr") or _node_addr(creator, 1, creator_entry),
|
|
"pve_fp": creator_entry.get("pve_fp") or _fingerprint_sha256(),
|
|
}
|
|
)
|
|
added[creator] = creator_entry
|
|
config["added_nodes"] = added
|
|
_ensure_corosync_materials(config, node_names)
|
|
|
|
metadata["cluster_config"] = config
|
|
await save_cluster_metadata(request, metadata)
|
|
await database(request).pool.execute(
|
|
"""UPDATE clusters
|
|
SET name=$1, updated_at=now()
|
|
WHERE id=(SELECT id FROM clusters LIMIT 1)""",
|
|
clustername,
|
|
)
|
|
return await _cluster_task(
|
|
request,
|
|
task_type="cluster-create",
|
|
worker="clustercreate",
|
|
task_id=clustername,
|
|
)
|
|
|
|
async def apiversion(_request: Request, _inputs: dict[str, Any]) -> int:
|
|
metadata = await cluster_metadata(_request)
|
|
return int(_config(metadata).get("apiversion") or 1)
|
|
|
|
async def join_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]:
|
|
metadata = await cluster_metadata(request)
|
|
config = _config(metadata)
|
|
preferred_raw = values(inputs).get("node")
|
|
# Contract documents the omitted-node default as this phrase; PVE uses local nodename.
|
|
if preferred_raw in {None, "", "current connected node"}:
|
|
preferred = None
|
|
else:
|
|
preferred = str(preferred_raw)
|
|
rows = await database(request).pool.fetch("SELECT name, status FROM nodes ORDER BY name")
|
|
added_raw = config.get("added_nodes")
|
|
added: dict[str, Any] = dict(added_raw) if isinstance(added_raw, dict) else {}
|
|
links = dict(config.get("links") or {})
|
|
nodelist: list[dict[str, Any]] = []
|
|
for index, row in enumerate(rows, start=1):
|
|
name = str(row["name"])
|
|
entry_raw = added.get(name)
|
|
entry: dict[str, Any] = dict(entry_raw) if isinstance(entry_raw, dict) else {}
|
|
if "pve_fp" not in entry:
|
|
entry["pve_fp"] = _fingerprint_sha256()
|
|
added[name] = entry
|
|
if "pve_addr" not in entry:
|
|
entry["pve_addr"] = _node_addr(name, index, entry)
|
|
added[name] = entry
|
|
nodelist.append(
|
|
{
|
|
"name": name,
|
|
"nodeid": int(entry.get("nodeid") or index),
|
|
"quorum_votes": int(
|
|
entry.get("quorum_votes") or entry.get("votes") or config.get("votes") or 1
|
|
),
|
|
"ring0_addr": _ring0_addr(name, index, entry, links),
|
|
"pve_addr": str(entry["pve_addr"]),
|
|
"pve_fp": str(entry["pve_fp"]),
|
|
}
|
|
)
|
|
if added != dict(added_raw or {}):
|
|
updated = dict(config)
|
|
updated["added_nodes"] = added
|
|
if not updated.get("config_digest"):
|
|
_ensure_corosync_materials(updated, [item["name"] for item in nodelist])
|
|
metadata["cluster_config"] = updated
|
|
await save_cluster_metadata(request, metadata)
|
|
config = updated
|
|
preferred_node = preferred or (nodelist[0]["name"] if nodelist else None)
|
|
digest = config.get("config_digest")
|
|
if not isinstance(digest, str) or not digest:
|
|
updated = dict(config)
|
|
_, _, digest = _ensure_corosync_materials(updated, [item["name"] for item in nodelist])
|
|
metadata["cluster_config"] = updated
|
|
await save_cluster_metadata(request, metadata)
|
|
config = updated
|
|
return {
|
|
"nodelist": nodelist,
|
|
"preferred_node": preferred_node,
|
|
"totem": dict(config.get("totem") or {}),
|
|
"config_digest": digest,
|
|
}
|
|
|
|
async def join_post(request: Request, inputs: dict[str, Any]) -> str:
|
|
payload = values(inputs)
|
|
hostname = str(require_value(payload, "hostname"))
|
|
require_value(payload, "fingerprint")
|
|
require_value(payload, "password")
|
|
metadata = await cluster_metadata(request)
|
|
config = dict(_config(metadata))
|
|
joins = dict(config.get("join_info") or {})
|
|
joins[hostname] = {
|
|
"hostname": hostname,
|
|
"fingerprint": payload.get("fingerprint"),
|
|
"nodeid": payload.get("nodeid"),
|
|
"votes": payload.get("votes", 1),
|
|
"force": payload.get("force"),
|
|
"password_set": True,
|
|
}
|
|
links = {key: value for key, value in payload.items() if key.startswith("link")}
|
|
if links:
|
|
joins[hostname]["links"] = links
|
|
config["join_info"] = joins
|
|
rows = await database(request).pool.fetch("SELECT name FROM nodes ORDER BY name")
|
|
node_names = [str(row["name"]) for row in rows]
|
|
_ensure_corosync_materials(config, node_names or ["localhost"])
|
|
metadata["cluster_config"] = config
|
|
await save_cluster_metadata(request, metadata)
|
|
return await _cluster_task(
|
|
request,
|
|
task_type="cluster-join",
|
|
worker="clusterjoin",
|
|
)
|
|
|
|
async def nodes_list(request: Request, _inputs: dict[str, Any]) -> list[dict[str, Any]]:
|
|
rows = await database(request).pool.fetch("SELECT name, status FROM nodes ORDER BY name")
|
|
metadata = await cluster_metadata(request)
|
|
config = _config(metadata)
|
|
added_raw = config.get("added_nodes")
|
|
added: dict[str, Any] = dict(added_raw) if isinstance(added_raw, dict) else {}
|
|
links = dict(config.get("links") or {})
|
|
result = []
|
|
for index, row in enumerate(rows, start=1):
|
|
name = str(row["name"])
|
|
entry_raw = added.get(name)
|
|
entry: dict[str, Any] = dict(entry_raw) if isinstance(entry_raw, dict) else {}
|
|
result.append(
|
|
{
|
|
"node": name,
|
|
"nodeid": entry.get("nodeid", index),
|
|
"ring0_addr": _ring0_addr(name, index, entry, links),
|
|
"quorum_votes": entry.get("quorum_votes", config.get("votes", 0)),
|
|
}
|
|
)
|
|
return result
|
|
|
|
async def nodes_add(request: Request, inputs: dict[str, Any]) -> dict[str, Any]:
|
|
payload = values(inputs)
|
|
node = str(require_value(payload, "node"))
|
|
metadata = await cluster_metadata(request)
|
|
config = dict(_config(metadata))
|
|
added = dict(config.get("added_nodes") or {})
|
|
if node in added and not payload.get("force"):
|
|
raise ApiError(400, f"can't add existing node '{node}'")
|
|
index = len(added) + 1
|
|
links = {key: value for key, value in payload.items() if key.startswith("link")}
|
|
entry = {
|
|
"node": node,
|
|
"nodeid": payload.get("nodeid") or index,
|
|
"new_node_ip": payload.get("new_node_ip"),
|
|
"votes": payload.get("votes", 1),
|
|
"apiversion": payload.get("apiversion"),
|
|
"force": payload.get("force"),
|
|
"ring0_addr": payload.get("ring0_addr")
|
|
or payload.get("new_node_ip")
|
|
or _ring0_addr(node, index, {}, links or dict(config.get("links") or {})),
|
|
"quorum_votes": payload.get("quorum_votes", payload.get("votes", 1)),
|
|
"pve_addr": payload.get("new_node_ip") or _node_addr(node, index, {}),
|
|
"pve_fp": _fingerprint_sha256(),
|
|
}
|
|
if links:
|
|
entry["links"] = links
|
|
added[node] = entry
|
|
config["added_nodes"] = added
|
|
exists = await database(request).pool.fetchval(
|
|
"SELECT EXISTS(SELECT 1 FROM nodes WHERE name=$1)",
|
|
node,
|
|
)
|
|
if not exists:
|
|
await database(request).pool.execute(
|
|
"""INSERT INTO nodes(id, name, status, metadata)
|
|
VALUES(gen_random_uuid(), $1, 'online', '{}'::jsonb)""",
|
|
node,
|
|
)
|
|
rows = await database(request).pool.fetch("SELECT name FROM nodes ORDER BY name")
|
|
node_names = [str(row["name"]) for row in rows]
|
|
if node not in node_names:
|
|
node_names.append(node)
|
|
authkey, conf, _digest = _ensure_corosync_materials(config, node_names)
|
|
metadata["cluster_config"] = config
|
|
await save_cluster_metadata(request, metadata)
|
|
return {
|
|
"corosync_authkey": authkey,
|
|
"corosync_conf": conf,
|
|
"warnings": [],
|
|
}
|
|
|
|
async def nodes_delete(request: Request, inputs: dict[str, Any]) -> None:
|
|
node = str(values(inputs)["node"])
|
|
metadata = await cluster_metadata(request)
|
|
config = dict(_config(metadata))
|
|
added = dict(config.get("added_nodes") or {})
|
|
added.pop(node, None)
|
|
joins = dict(config.get("join_info") or {})
|
|
joins.pop(node, None)
|
|
config["added_nodes"] = added
|
|
config["join_info"] = joins
|
|
rows = await database(request).pool.fetch("SELECT name FROM nodes ORDER BY name")
|
|
node_names = [str(row["name"]) for row in rows if str(row["name"]) != node]
|
|
_ensure_corosync_materials(config, node_names or ["localhost"])
|
|
metadata["cluster_config"] = config
|
|
await save_cluster_metadata(request, metadata)
|
|
# Keep node row; mark offline to avoid cascading guest deletes.
|
|
await database(request).pool.execute(
|
|
"UPDATE nodes SET status='offline', updated_at=now() WHERE name=$1",
|
|
node,
|
|
)
|
|
|
|
async def qdevice(request: Request, _inputs: dict[str, Any]) -> dict[str, Any]:
|
|
metadata = await cluster_metadata(request)
|
|
qdevice = _config(metadata).get("qdevice")
|
|
return dict(qdevice) if isinstance(qdevice, dict) else {}
|
|
|
|
async def totem(request: Request, _inputs: dict[str, Any]) -> dict[str, Any]:
|
|
metadata = await cluster_metadata(request)
|
|
totem = _config(metadata).get("totem")
|
|
return dict(totem) if isinstance(totem, dict) else {}
|
|
|
|
registry.register("/cluster/config", "GET", index)
|
|
registry.register("/cluster/config", "POST", create)
|
|
registry.register("/cluster/config/apiversion", "GET", apiversion)
|
|
registry.register("/cluster/config/join", "GET", join_get)
|
|
registry.register("/cluster/config/join", "POST", join_post)
|
|
registry.register("/cluster/config/nodes", "GET", nodes_list)
|
|
registry.register("/cluster/config/nodes/{node}", "POST", nodes_add)
|
|
registry.register("/cluster/config/nodes/{node}", "DELETE", nodes_delete)
|
|
registry.register("/cluster/config/qdevice", "GET", qdevice)
|
|
registry.register("/cluster/config/totem", "GET", totem)
|