Expand lab seed tiers, OpenAPI PARAMS, and console DATA/Params UX.
This commit is contained in:
@@ -0,0 +1,515 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a compact vSphere REST param index from official Automation OpenAPI.
|
||||
|
||||
Source (default):
|
||||
https://raw.githubusercontent.com/vmware/vcf-api-specs/main/specifications/vsphere/openapi/automation/vcenter.yaml
|
||||
|
||||
Output:
|
||||
app/vsphere/rest/param_index.json
|
||||
|
||||
Keys are ``VERB /api/...`` matching the simulator catalog (OpenAPI servers already
|
||||
use ``/api`` as the base). Paths that OpenAPI encodes as
|
||||
``/vcenter/vm/{vm}/power?action=start`` are collapsed onto the template path with
|
||||
an ``action`` query field (enum of all observed actions).
|
||||
|
||||
Requires PyYAML at generation time only (not a runtime dependency).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app.vsphere.rest.param_fields import body_fields_from_example # noqa: E402
|
||||
|
||||
DEFAULT_URL = (
|
||||
"https://raw.githubusercontent.com/vmware/vcf-api-specs/main/"
|
||||
"specifications/vsphere/openapi/automation/vcenter.yaml"
|
||||
)
|
||||
OUT = ROOT / "app" / "vsphere" / "rest" / "param_index.json"
|
||||
|
||||
_PATH_QUERY = re.compile(r"^(?P<path>[^?]+)(?:\?(?P<query>.*))?$")
|
||||
_ENUM_CAP = 24
|
||||
|
||||
# Prefer lab seed identifiers when property names match.
|
||||
# Filter examples must AND-match on every profile (small / large / demo-cluster):
|
||||
# web-01 (vm-101) is always POWERED_ON on host-11.
|
||||
_LAB_EXAMPLES: dict[str, Any] = {
|
||||
"vm": "vm-101",
|
||||
"vms": ["vm-101"],
|
||||
"name": "web-01",
|
||||
"names": ["web-01"],
|
||||
"host": "host-11",
|
||||
"hosts": ["host-11"],
|
||||
"folder": "group-v23",
|
||||
"folders": ["group-v23"],
|
||||
"datastore": "datastore-31",
|
||||
"datastores": ["datastore-31"],
|
||||
"datacenter": "datacenter-21",
|
||||
"datacenters": ["datacenter-21"],
|
||||
"cluster": "domain-c21",
|
||||
"clusters": ["domain-c21"],
|
||||
"resource_pool": "resgroup-22",
|
||||
"resource_pools": ["resgroup-22"],
|
||||
"network": "network-41",
|
||||
"networks": ["network-41"],
|
||||
"power_states": ["POWERED_ON"],
|
||||
"guest_os": "OTHER_GUEST_64",
|
||||
"guest_OS": "OTHER_GUEST_64",
|
||||
"action": "start",
|
||||
"category_id": "urn:vmomi:InventoryServiceCategory:demo:GLOBAL",
|
||||
"tag_id": "urn:vmomi:InventoryServiceTag:demo:GLOBAL",
|
||||
"item_id": "item-demo",
|
||||
"library_id": "library-demo",
|
||||
"task": "task-1",
|
||||
"snapshot": "snapshot-1",
|
||||
"count": 2,
|
||||
"size_mib": 2048,
|
||||
"size_MiB": 2048,
|
||||
"memory_size_MiB": 2048,
|
||||
"cpu_count": 2,
|
||||
}
|
||||
|
||||
|
||||
def _load_yaml(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
import yaml
|
||||
except ImportError as exc: # pragma: no cover
|
||||
raise SystemExit("PyYAML is required to generate the param index") from exc
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(data, dict):
|
||||
raise SystemExit("OpenAPI root must be a mapping")
|
||||
return data
|
||||
|
||||
|
||||
def _fetch(url: str, dest: Path) -> None:
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
print(f"Fetching {url}", file=sys.stderr)
|
||||
with urllib.request.urlopen(url, timeout=120) as resp: # noqa: S310
|
||||
dest.write_bytes(resp.read())
|
||||
print(f"Wrote {dest} ({dest.stat().st_size} bytes)", file=sys.stderr)
|
||||
|
||||
|
||||
def _resolve_ref(spec: dict[str, Any], ref: str) -> dict[str, Any]:
|
||||
if not ref.startswith("#/"):
|
||||
return {}
|
||||
node: Any = spec
|
||||
for part in ref[2:].split("/"):
|
||||
if not isinstance(node, dict):
|
||||
return {}
|
||||
node = node.get(part)
|
||||
return node if isinstance(node, dict) else {}
|
||||
|
||||
|
||||
def _deref(spec: dict[str, Any], schema: dict[str, Any] | None) -> dict[str, Any]:
|
||||
if not schema:
|
||||
return {}
|
||||
if "$ref" in schema:
|
||||
resolved = _resolve_ref(spec, str(schema["$ref"]))
|
||||
merged = dict(resolved)
|
||||
for key, value in schema.items():
|
||||
if key != "$ref":
|
||||
merged[key] = value
|
||||
return _deref(spec, merged) if ("$ref" in merged or "allOf" in merged) else merged
|
||||
if "allOf" in schema and isinstance(schema["allOf"], list):
|
||||
merged: dict[str, Any] = {}
|
||||
props: dict[str, Any] = {}
|
||||
required: list[str] = []
|
||||
for part in schema["allOf"]:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
resolved = _deref(spec, part)
|
||||
for key, value in resolved.items():
|
||||
if key == "properties" and isinstance(value, dict):
|
||||
props.update(value)
|
||||
elif key == "required" and isinstance(value, list):
|
||||
required.extend(str(item) for item in value)
|
||||
elif key not in {"properties", "required"}:
|
||||
merged[key] = value
|
||||
for key, value in schema.items():
|
||||
if key == "allOf":
|
||||
continue
|
||||
if key == "properties" and isinstance(value, dict):
|
||||
props.update(value)
|
||||
elif key == "required" and isinstance(value, list):
|
||||
required.extend(str(item) for item in value)
|
||||
else:
|
||||
merged[key] = value
|
||||
if props:
|
||||
merged["properties"] = props
|
||||
if required:
|
||||
merged["required"] = list(dict.fromkeys(required))
|
||||
if "type" not in merged and props:
|
||||
merged["type"] = "object"
|
||||
return merged
|
||||
return schema
|
||||
|
||||
|
||||
def _short_desc(text: Any) -> str | None:
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
return None
|
||||
line = text.strip().split("\n", 1)[0].strip()
|
||||
if len(line) > 160:
|
||||
return line[:157] + "..."
|
||||
return line
|
||||
|
||||
|
||||
def _example_for(
|
||||
spec: dict[str, Any],
|
||||
name: str,
|
||||
schema: dict[str, Any],
|
||||
*,
|
||||
depth: int = 0,
|
||||
) -> Any:
|
||||
schema = _deref(spec, schema)
|
||||
if name in _LAB_EXAMPLES:
|
||||
return _LAB_EXAMPLES[name]
|
||||
if "example" in schema:
|
||||
return schema["example"]
|
||||
if "default" in schema:
|
||||
return schema["default"]
|
||||
enum = schema.get("enum")
|
||||
if isinstance(enum, list) and enum:
|
||||
return enum[0]
|
||||
typ = schema.get("type")
|
||||
if typ == "array":
|
||||
items = schema.get("items") if isinstance(schema.get("items"), dict) else {}
|
||||
item_ex = _example_for(spec, name.rstrip("s") or name, items, depth=depth + 1)
|
||||
return [item_ex] if item_ex is not None else []
|
||||
if typ == "object" or "properties" in schema:
|
||||
if depth >= 3:
|
||||
return {}
|
||||
props = schema.get("properties") if isinstance(schema.get("properties"), dict) else {}
|
||||
req = set(schema.get("required") or [])
|
||||
# Prefer required props; include a few well-known optional lab fields.
|
||||
keys = list(req)
|
||||
for extra in ("name", "placement", "cpu", "memory", "description", "spec", "create_spec"):
|
||||
if extra in props and extra not in keys:
|
||||
keys.append(extra)
|
||||
if not keys:
|
||||
keys = list(props.keys())[:6]
|
||||
out: dict[str, Any] = {}
|
||||
for key in keys:
|
||||
prop = props.get(key)
|
||||
if not isinstance(prop, dict):
|
||||
continue
|
||||
out[key] = _example_for(spec, key, prop, depth=depth + 1)
|
||||
return out
|
||||
if typ == "boolean":
|
||||
return False
|
||||
if typ == "integer":
|
||||
return int(schema["minimum"]) if isinstance(schema.get("minimum"), (int, float)) else 1
|
||||
if typ == "number":
|
||||
return float(schema["minimum"]) if isinstance(schema.get("minimum"), (int, float)) else 1.0
|
||||
if typ == "string" or typ is None:
|
||||
return "example"
|
||||
return None
|
||||
|
||||
|
||||
def _field_from_schema(
|
||||
spec: dict[str, Any],
|
||||
name: str,
|
||||
schema: dict[str, Any],
|
||||
*,
|
||||
optional: bool,
|
||||
description: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
schema = _deref(spec, schema)
|
||||
typ = schema.get("type")
|
||||
if typ is None and "properties" in schema:
|
||||
typ = "object"
|
||||
if typ is None and "items" in schema:
|
||||
typ = "array"
|
||||
enum = schema.get("enum") if isinstance(schema.get("enum"), list) else []
|
||||
if len(enum) > _ENUM_CAP:
|
||||
enum = enum[:_ENUM_CAP]
|
||||
example = _example_for(spec, name, schema)
|
||||
return {
|
||||
"name": name,
|
||||
"type": typ or "string",
|
||||
"optional": optional,
|
||||
"description": _short_desc(description or schema.get("description")),
|
||||
"enum": enum,
|
||||
"example": example,
|
||||
}
|
||||
|
||||
|
||||
def _collect_params(
|
||||
spec: dict[str, Any],
|
||||
operation: dict[str, Any],
|
||||
path_item_params: list[dict[str, Any]],
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
query: list[dict[str, Any]] = []
|
||||
path: list[dict[str, Any]] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for raw in list(path_item_params) + list(operation.get("parameters") or []):
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
param = _deref(spec, raw) if "$ref" in raw else raw
|
||||
name = str(param.get("name") or "")
|
||||
location = str(param.get("in") or "")
|
||||
if not name or location not in {"query", "path"}:
|
||||
continue
|
||||
key = (location, name)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
schema = param.get("schema") if isinstance(param.get("schema"), dict) else {}
|
||||
field = _field_from_schema(
|
||||
spec,
|
||||
name,
|
||||
schema,
|
||||
optional=not bool(param.get("required")),
|
||||
description=param.get("description") if isinstance(param.get("description"), str) else None,
|
||||
)
|
||||
if location == "query":
|
||||
query.append(field)
|
||||
else:
|
||||
path.append(field)
|
||||
return query, path
|
||||
|
||||
|
||||
def _body_from_operation(
|
||||
spec: dict[str, Any],
|
||||
operation: dict[str, Any],
|
||||
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
||||
request_body = operation.get("requestBody")
|
||||
if not isinstance(request_body, dict):
|
||||
return [], {}
|
||||
request_body = _deref(spec, request_body) if "$ref" in request_body else request_body
|
||||
content = request_body.get("content") if isinstance(request_body.get("content"), dict) else {}
|
||||
media = content.get("application/json") or next(iter(content.values()), None)
|
||||
if not isinstance(media, dict):
|
||||
return [], {}
|
||||
schema = _deref(spec, media.get("schema") if isinstance(media.get("schema"), dict) else {})
|
||||
if not schema:
|
||||
return [], {}
|
||||
props = schema.get("properties") if isinstance(schema.get("properties"), dict) else {}
|
||||
if props:
|
||||
example = _example_for(spec, "", schema)
|
||||
example_dict = example if isinstance(example, dict) else {}
|
||||
# PARAM drawer leaves come from nested body_example (placement.host, …).
|
||||
return body_fields_from_example(example_dict), example_dict
|
||||
# Body is a naked $ref / non-object — still emit an example blob.
|
||||
example = _example_for(spec, "body", schema)
|
||||
example_dict = example if isinstance(example, dict) else {}
|
||||
return body_fields_from_example(example_dict), example_dict
|
||||
|
||||
|
||||
def _normalize_path_key(raw_path: str) -> tuple[str, dict[str, str]]:
|
||||
"""Return (/api/... template, query extras from path key like action=start)."""
|
||||
match = _PATH_QUERY.match(raw_path)
|
||||
if not match:
|
||||
return raw_path, {}
|
||||
path = match.group("path")
|
||||
query_raw = match.group("query") or ""
|
||||
extras: dict[str, str] = {}
|
||||
if query_raw:
|
||||
for part in query_raw.split("&"):
|
||||
if "=" in part:
|
||||
key, value = part.split("=", 1)
|
||||
extras[key] = value
|
||||
elif part:
|
||||
extras[part] = ""
|
||||
if not path.startswith("/api/"):
|
||||
path = "/api" + path if path.startswith("/") else "/api/" + path
|
||||
return path, extras
|
||||
|
||||
|
||||
def _merge_action_field(query_fields: list[dict[str, Any]], actions: set[str]) -> None:
|
||||
if not actions:
|
||||
return
|
||||
existing = next((field for field in query_fields if field["name"] == "action"), None)
|
||||
ordered = sorted(actions)
|
||||
if existing is None:
|
||||
query_fields.insert(
|
||||
0,
|
||||
{
|
||||
"name": "action",
|
||||
"type": "string",
|
||||
"optional": False,
|
||||
"description": "Operation action query parameter",
|
||||
"enum": ordered,
|
||||
"example": ordered[0],
|
||||
},
|
||||
)
|
||||
return
|
||||
enum = list(dict.fromkeys([*list(existing.get("enum") or []), *ordered]))
|
||||
existing["enum"] = enum
|
||||
existing["optional"] = False
|
||||
if not existing.get("example"):
|
||||
existing["example"] = enum[0]
|
||||
|
||||
|
||||
def build_index(spec: dict[str, Any]) -> dict[str, Any]:
|
||||
paths = spec.get("paths") if isinstance(spec.get("paths"), dict) else {}
|
||||
# Accumulate by VERB + template path so ?action= variants collapse.
|
||||
buckets: dict[str, dict[str, Any]] = {}
|
||||
|
||||
def ensure_bucket(verb: str, template: str) -> dict[str, Any]:
|
||||
key = f"{verb} {template}"
|
||||
return buckets.setdefault(
|
||||
key,
|
||||
{
|
||||
"verb": verb,
|
||||
"path": template,
|
||||
"query_fields": [],
|
||||
"path_fields": [],
|
||||
"body_fields": [],
|
||||
"body_example": {},
|
||||
"operation_ids": [],
|
||||
"actions": set(),
|
||||
},
|
||||
)
|
||||
|
||||
def merge_operation(
|
||||
*,
|
||||
verb: str,
|
||||
template: str,
|
||||
operation: dict[str, Any],
|
||||
path_params: list[dict[str, Any]],
|
||||
action: str | None = None,
|
||||
) -> None:
|
||||
key = f"{verb} {template}"
|
||||
if action:
|
||||
existing = buckets.get(key)
|
||||
# Do not pollute CreateSpec-style POSTs with clone/register actions.
|
||||
if existing is not None and (existing["body_fields"] or existing["body_example"]):
|
||||
return
|
||||
bucket = ensure_bucket(verb, template)
|
||||
bucket["actions"].add(action)
|
||||
_query, path_fields = _collect_params(spec, operation, path_params)
|
||||
for field in path_fields:
|
||||
names = {item["name"] for item in bucket["path_fields"]}
|
||||
if field["name"] not in names:
|
||||
bucket["path_fields"].append(field)
|
||||
op_id = operation.get("operationId")
|
||||
if isinstance(op_id, str) and op_id not in bucket["operation_ids"]:
|
||||
bucket["operation_ids"].append(op_id)
|
||||
return
|
||||
|
||||
bucket = ensure_bucket(verb, template)
|
||||
query_fields, path_fields = _collect_params(spec, operation, path_params)
|
||||
body_fields, body_example = _body_from_operation(spec, operation)
|
||||
for field in query_fields:
|
||||
names = {item["name"] for item in bucket["query_fields"]}
|
||||
if field["name"] not in names:
|
||||
bucket["query_fields"].append(field)
|
||||
for field in path_fields:
|
||||
names = {item["name"] for item in bucket["path_fields"]}
|
||||
if field["name"] not in names:
|
||||
bucket["path_fields"].append(field)
|
||||
if body_fields and not bucket["body_fields"]:
|
||||
bucket["body_fields"] = body_fields
|
||||
if body_example and not bucket["body_example"]:
|
||||
bucket["body_example"] = body_example
|
||||
op_id = operation.get("operationId")
|
||||
if isinstance(op_id, str) and op_id not in bucket["operation_ids"]:
|
||||
bucket["operation_ids"].append(op_id)
|
||||
|
||||
# Pass 1: concrete paths without ?query — establish CreateSpec etc.
|
||||
# Pass 2: ?action= variants — attach only onto action-style endpoints.
|
||||
ordered_paths = sorted(paths.items(), key=lambda item: ("?" in str(item[0]), str(item[0])))
|
||||
for raw_path, path_item in ordered_paths:
|
||||
if not isinstance(path_item, dict):
|
||||
continue
|
||||
template, extras = _normalize_path_key(str(raw_path))
|
||||
path_params = [p for p in (path_item.get("parameters") or []) if isinstance(p, dict)]
|
||||
for verb, operation in path_item.items():
|
||||
if verb.startswith("x-") or verb == "parameters" or not isinstance(operation, dict):
|
||||
continue
|
||||
upper = verb.upper()
|
||||
if upper not in {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"}:
|
||||
continue
|
||||
merge_operation(
|
||||
verb=upper,
|
||||
template=template,
|
||||
operation=operation,
|
||||
path_params=path_params,
|
||||
action=extras.get("action") or None,
|
||||
)
|
||||
|
||||
methods: dict[str, Any] = {}
|
||||
for key, bucket in sorted(buckets.items()):
|
||||
actions: set[str] = bucket.pop("actions")
|
||||
_merge_action_field(bucket["query_fields"], actions)
|
||||
entry = {
|
||||
"verb": bucket["verb"],
|
||||
"path": bucket["path"],
|
||||
"query_fields": bucket["query_fields"],
|
||||
"path_fields": bucket["path_fields"],
|
||||
"body_fields": bucket["body_fields"],
|
||||
"body_example": bucket["body_example"],
|
||||
"operation_ids": bucket["operation_ids"],
|
||||
}
|
||||
methods[key] = entry
|
||||
|
||||
info = spec.get("info") if isinstance(spec.get("info"), dict) else {}
|
||||
return {
|
||||
"source": "vmware/vcf-api-specs specifications/vsphere/openapi/automation/vcenter.yaml",
|
||||
"openapi": spec.get("openapi"),
|
||||
"title": info.get("title"),
|
||||
"version": info.get("version"),
|
||||
"method_count": len(methods),
|
||||
"methods": methods,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--url", default=DEFAULT_URL, help="OpenAPI YAML URL")
|
||||
parser.add_argument(
|
||||
"--input",
|
||||
type=Path,
|
||||
help="Local OpenAPI YAML (skips download)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cache",
|
||||
type=Path,
|
||||
default=ROOT / ".cache" / "vcenter.openapi.yaml",
|
||||
help="Download cache path",
|
||||
)
|
||||
parser.add_argument("--output", type=Path, default=OUT)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.input is not None:
|
||||
source = args.input
|
||||
else:
|
||||
if not args.cache.exists():
|
||||
_fetch(args.url, args.cache)
|
||||
source = args.cache
|
||||
|
||||
spec = _load_yaml(source)
|
||||
index = build_index(spec)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(index, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
print(
|
||||
f"Wrote {args.output} methods={index['method_count']} version={index.get('version')}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
# Spot-check critical routes.
|
||||
for probe in ("GET /api/vcenter/vm", "POST /api/vcenter/vm", "POST /api/vcenter/vm/{vm}/power"):
|
||||
entry = index["methods"].get(probe)
|
||||
if not entry:
|
||||
print(f"WARN missing {probe}", file=sys.stderr)
|
||||
continue
|
||||
print(
|
||||
f"OK {probe} query={len(entry['query_fields'])} "
|
||||
f"body_fields={len(entry['body_fields'])} "
|
||||
f"body_example_keys={list(entry['body_example'])[:6]}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+65
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env bash
|
||||
# Generate a gitignored docker-compose.override.yml on free host ports, then
|
||||
# start the stack. Invoked by `make up-local`.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
COMPOSE="${COMPOSE:-docker compose}"
|
||||
COMPOSE_OVERRIDE="${COMPOSE_OVERRIDE:-docker-compose.override.yml}"
|
||||
LOCAL_HTTP_PORT="${LOCAL_HTTP_PORT:-18080}"
|
||||
LOCAL_HTTPS_PORT="${LOCAL_HTTPS_PORT:-18443}"
|
||||
LOCAL_POSTGRES_PORT="${LOCAL_POSTGRES_PORT:-15434}"
|
||||
|
||||
port_ok() {
|
||||
local p="$1"
|
||||
if ! lsof -nP -iTCP:"$p" -sTCP:LISTEN >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
# Reuse ports already published by this compose project.
|
||||
$COMPOSE port api-gateway 80 2>/dev/null | grep -q ":${p}$" && return 0
|
||||
$COMPOSE port api-gateway 443 2>/dev/null | grep -q ":${p}$" && return 0
|
||||
$COMPOSE port postgres 5432 2>/dev/null | grep -q ":${p}$" && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
pick() {
|
||||
local start="$1" name="$2" p
|
||||
for p in $(seq "$start" $((start + 40))); do
|
||||
if port_ok "$p"; then
|
||||
echo "$p"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
echo "No free host port near ${start} for ${name}" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
test -f .env || cp .env.example .env
|
||||
|
||||
http="$(pick "$LOCAL_HTTP_PORT" HTTP)"
|
||||
https="$(pick "$LOCAL_HTTPS_PORT" HTTPS)"
|
||||
pg="$(pick "$LOCAL_POSTGRES_PORT" Postgres)"
|
||||
|
||||
cat >"$COMPOSE_OVERRIDE" <<EOF
|
||||
# Generated by make up-local — gitignored, do not commit.
|
||||
services:
|
||||
postgres:
|
||||
ports:
|
||||
- "127.0.0.1:${pg}:5432"
|
||||
api-gateway:
|
||||
ports:
|
||||
- "${http}:80"
|
||||
- "${https}:443"
|
||||
EOF
|
||||
|
||||
echo "Wrote ${COMPOSE_OVERRIDE}: HTTP=${http} HTTPS=${https} Postgres=127.0.0.1:${pg}"
|
||||
|
||||
$COMPOSE up -d --build --wait
|
||||
|
||||
echo ""
|
||||
echo "Local stack is up (override ports, not committed):"
|
||||
echo " HTTPS https://localhost:${https}"
|
||||
echo " HTTP http://localhost:${http}"
|
||||
echo " DB 127.0.0.1:${pg}"
|
||||
@@ -0,0 +1,325 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Audit seeded inventory against live Automation API responses.
|
||||
|
||||
Compares the declarative profile (small / large / big) to live
|
||||
GET /api/vcenter/* dumps: counts, MOID/name/power for every VM and host,
|
||||
per-host placement via filter, and a canonical AND-filter that must hit on
|
||||
all profiles (web-01 @ host-11, POWERED_ON).
|
||||
|
||||
Run against a freshly seeded lab (``make seed``) before matrix probes mutate
|
||||
inventory. Exit 0 only on a 100% dump match.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from base64 import b64encode
|
||||
from collections import defaultdict
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from app.vsphere.profiles import build_vsphere_profile
|
||||
|
||||
BASE = os.getenv("VSPHERE_BASE", "https://localhost")
|
||||
USER = os.getenv("VSPHERE_USER", "administrator@vsphere.local")
|
||||
PASSWORD = os.getenv("VSPHERE_PASSWORD", "VMware1!")
|
||||
|
||||
# Works on small (3h), large (10h), and big (20h).
|
||||
CANONICAL_FILTER = {
|
||||
"names": "web-01",
|
||||
"hosts": "host-11",
|
||||
"power_states": "POWERED_ON",
|
||||
}
|
||||
|
||||
|
||||
def _ctx() -> ssl.SSLContext | None:
|
||||
if not BASE.startswith("https://"):
|
||||
return None
|
||||
return ssl._create_unverified_context() # noqa: S323
|
||||
|
||||
|
||||
def _request(method: str, path: str, *, headers: dict[str, str]) -> tuple[int, Any]:
|
||||
req = urllib.request.Request(f"{BASE}{path}", method=method, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, context=_ctx(), timeout=120) as resp: # noqa: S310
|
||||
raw = resp.read().decode("utf-8", errors="replace")
|
||||
return int(resp.status), json.loads(raw) if raw.strip() else None
|
||||
except urllib.error.HTTPError as error:
|
||||
raw = error.read().decode("utf-8", errors="replace")
|
||||
try:
|
||||
body = json.loads(raw) if raw.strip() else raw
|
||||
except json.JSONDecodeError:
|
||||
body = raw
|
||||
return int(error.code), body
|
||||
|
||||
|
||||
def _session() -> str:
|
||||
basic = b64encode(f"{USER}:{PASSWORD}".encode()).decode()
|
||||
code, body = _request("POST", "/api/session", headers={"Authorization": f"Basic {basic}"})
|
||||
if code not in {200, 201} or not isinstance(body, str):
|
||||
raise SystemExit(json.dumps({"error": "session failed", "status": code, "body": body}))
|
||||
return body
|
||||
|
||||
|
||||
def _by_type(profile_objects: tuple[Any, ...]) -> dict[str, list[Any]]:
|
||||
out: dict[str, list[Any]] = {}
|
||||
for obj in profile_objects:
|
||||
out.setdefault(obj.type, []).append(obj)
|
||||
return out
|
||||
|
||||
|
||||
def audit_profile(profile_name: str, *, hosts: int | None, vms: int | None) -> dict[str, Any]:
|
||||
profile = build_vsphere_profile(profile_name, large_hosts=hosts, large_vms=vms)
|
||||
expected = _by_type(profile.objects)
|
||||
session = _session()
|
||||
headers = {"vmware-api-session-id": session, "Accept": "application/json"}
|
||||
|
||||
failures: list[dict[str, Any]] = []
|
||||
|
||||
live_vms_code, live_vms = _request("GET", "/api/vcenter/vm", headers=headers)
|
||||
live_hosts_code, live_hosts = _request("GET", "/api/vcenter/host", headers=headers)
|
||||
live_ds_code, live_ds = _request("GET", "/api/vcenter/datastore", headers=headers)
|
||||
live_net_code, live_net = _request("GET", "/api/vcenter/network", headers=headers)
|
||||
live_cl_code, live_cl = _request("GET", "/api/vcenter/cluster", headers=headers)
|
||||
live_dc_code, live_dc = _request("GET", "/api/vcenter/datacenter", headers=headers)
|
||||
live_folder_code, live_folder = _request("GET", "/api/vcenter/folder", headers=headers)
|
||||
live_rp_code, live_rp = _request("GET", "/api/vcenter/resource-pool", headers=headers)
|
||||
|
||||
for label, code, payload in (
|
||||
("vm", live_vms_code, live_vms),
|
||||
("host", live_hosts_code, live_hosts),
|
||||
("datastore", live_ds_code, live_ds),
|
||||
("network", live_net_code, live_net),
|
||||
("cluster", live_cl_code, live_cl),
|
||||
("datacenter", live_dc_code, live_dc),
|
||||
("folder", live_folder_code, live_folder),
|
||||
("resource-pool", live_rp_code, live_rp),
|
||||
):
|
||||
if code != 200 or not isinstance(payload, list):
|
||||
failures.append({"check": f"list/{label}", "status": code, "body": str(payload)[:160]})
|
||||
|
||||
exp_vms = expected.get("VirtualMachine", [])
|
||||
exp_hosts = expected.get("HostSystem", [])
|
||||
exp_ds = expected.get("Datastore", [])
|
||||
exp_nets = [
|
||||
*expected.get("Network", []),
|
||||
*expected.get("DistributedVirtualPortgroup", []),
|
||||
]
|
||||
exp_clusters = expected.get("ClusterComputeResource", [])
|
||||
exp_dcs = expected.get("Datacenter", [])
|
||||
exp_folders = expected.get("Folder", [])
|
||||
exp_rps = expected.get("ResourcePool", [])
|
||||
|
||||
def _count(label: str, got: Any, want: int) -> None:
|
||||
if not isinstance(got, list):
|
||||
return
|
||||
if len(got) != want:
|
||||
failures.append({"check": f"count/{label}", "expected": want, "actual": len(got)})
|
||||
|
||||
_count("vm", live_vms, len(exp_vms))
|
||||
_count("host", live_hosts, len(exp_hosts))
|
||||
_count("datastore", live_ds, len(exp_ds))
|
||||
_count("network", live_net, len(exp_nets))
|
||||
_count("cluster", live_cl, len(exp_clusters))
|
||||
_count("datacenter", live_dc, len(exp_dcs))
|
||||
_count("folder", live_folder, len(exp_folders))
|
||||
_count("resource-pool", live_rp, len(exp_rps))
|
||||
|
||||
if isinstance(live_hosts, list):
|
||||
live_host_map = {row.get("host"): row for row in live_hosts if isinstance(row, dict)}
|
||||
for obj in exp_hosts:
|
||||
row = live_host_map.get(obj.moid)
|
||||
if row is None:
|
||||
failures.append({"check": "host/missing", "host": obj.moid})
|
||||
continue
|
||||
if row.get("name") != obj.name:
|
||||
failures.append(
|
||||
{
|
||||
"check": "host/name",
|
||||
"host": obj.moid,
|
||||
"expected": obj.name,
|
||||
"actual": row.get("name"),
|
||||
}
|
||||
)
|
||||
|
||||
if isinstance(live_vms, list):
|
||||
live_vm_map = {row.get("vm"): row for row in live_vms if isinstance(row, dict)}
|
||||
for obj in exp_vms:
|
||||
row = live_vm_map.get(obj.moid)
|
||||
if row is None:
|
||||
failures.append({"check": "vm/missing", "vm": obj.moid, "name": obj.name})
|
||||
continue
|
||||
want = {
|
||||
"name": obj.name,
|
||||
"power_state": obj.props.get("power_state"),
|
||||
"cpu_count": obj.props.get("cpu_count"),
|
||||
"memory_size_MiB": obj.props.get("memory_size_mib"),
|
||||
}
|
||||
for key, expected_value in want.items():
|
||||
if row.get(key) != expected_value:
|
||||
failures.append(
|
||||
{
|
||||
"check": f"vm/{key}",
|
||||
"vm": obj.moid,
|
||||
"expected": expected_value,
|
||||
"actual": row.get(key),
|
||||
}
|
||||
)
|
||||
|
||||
# Per-host placement dump via filter (O(hosts), not O(vms)).
|
||||
by_host: dict[str, set[str]] = defaultdict(set)
|
||||
for obj in exp_vms:
|
||||
by_host[str(obj.props.get("host"))].add(obj.moid)
|
||||
for host_moid, want_ids in sorted(by_host.items()):
|
||||
qs = urlencode({"hosts": host_moid})
|
||||
code, filtered = _request("GET", f"/api/vcenter/vm?{qs}", headers=headers)
|
||||
if code != 200 or not isinstance(filtered, list):
|
||||
failures.append({"check": "host-filter", "host": host_moid, "status": code})
|
||||
continue
|
||||
got_ids = {row.get("vm") for row in filtered if isinstance(row, dict)}
|
||||
missing = sorted(want_ids - got_ids)
|
||||
extra = sorted(got_ids - want_ids)
|
||||
if missing or extra:
|
||||
failures.append(
|
||||
{
|
||||
"check": "host-filter/mismatch",
|
||||
"host": host_moid,
|
||||
"missing": missing[:20],
|
||||
"extra": extra[:20],
|
||||
"expected": len(want_ids),
|
||||
"actual": len(got_ids),
|
||||
}
|
||||
)
|
||||
|
||||
qs = urlencode(CANONICAL_FILTER)
|
||||
code, filtered = _request("GET", f"/api/vcenter/vm?{qs}", headers=headers)
|
||||
if code != 200 or not isinstance(filtered, list) or len(filtered) != 1:
|
||||
failures.append(
|
||||
{
|
||||
"check": "canonical-filter",
|
||||
"query": CANONICAL_FILTER,
|
||||
"status": code,
|
||||
"hits": filtered if not isinstance(filtered, list) else len(filtered),
|
||||
"body": filtered[:3] if isinstance(filtered, list) else filtered,
|
||||
}
|
||||
)
|
||||
elif filtered[0].get("name") != "web-01" or filtered[0].get("vm") != "vm-101":
|
||||
failures.append(
|
||||
{
|
||||
"check": "canonical-filter/identity",
|
||||
"expected": {"vm": "vm-101", "name": "web-01"},
|
||||
"actual": filtered[0],
|
||||
}
|
||||
)
|
||||
|
||||
detail_code, detail = _request("GET", "/api/vcenter/vm/vm-101", headers=headers)
|
||||
if detail_code != 200 or not isinstance(detail, dict) or detail.get("name") != "web-01":
|
||||
failures.append({"check": "vm/detail", "status": detail_code, "body": str(detail)[:200]})
|
||||
|
||||
# --- proportional platform extras ---
|
||||
extras_scale = int(getattr(profile, "extras_scale", 1) or 1)
|
||||
want_libraries = 2 + max(0, extras_scale - 1) # local+published + scaled locals
|
||||
want_categories = 3 + max(0, extras_scale - 1) # Environment/Owner/Lab + scaled
|
||||
want_folders = 8 + max(0, (extras_scale - 1) * 2)
|
||||
|
||||
lib_code, libraries = _request("GET", "/api/content/library", headers=headers)
|
||||
if lib_code != 200 or not isinstance(libraries, list):
|
||||
failures.append({"check": "extras/libraries", "status": lib_code})
|
||||
elif len(libraries) < want_libraries:
|
||||
failures.append(
|
||||
{
|
||||
"check": "extras/libraries/count",
|
||||
"expected_min": want_libraries,
|
||||
"actual": len(libraries),
|
||||
"extras_scale": extras_scale,
|
||||
}
|
||||
)
|
||||
|
||||
cat_code, categories = _request("GET", "/api/cis/tagging/category", headers=headers)
|
||||
if cat_code != 200 or not isinstance(categories, list):
|
||||
failures.append({"check": "extras/categories", "status": cat_code})
|
||||
elif len(categories) < want_categories:
|
||||
failures.append(
|
||||
{
|
||||
"check": "extras/categories/count",
|
||||
"expected_min": want_categories,
|
||||
"actual": len(categories),
|
||||
"extras_scale": extras_scale,
|
||||
}
|
||||
)
|
||||
|
||||
if isinstance(live_folder, list) and len(live_folder) != want_folders:
|
||||
failures.append(
|
||||
{
|
||||
"check": "count/folder-scaled",
|
||||
"expected": want_folders,
|
||||
"actual": len(live_folder),
|
||||
"extras_scale": extras_scale,
|
||||
}
|
||||
)
|
||||
|
||||
# Datastore references on VMs must exist in inventory.
|
||||
ds_ids = {obj.moid for obj in exp_ds}
|
||||
for obj in exp_vms:
|
||||
ds = obj.props.get("datastore")
|
||||
if ds and ds not in ds_ids:
|
||||
failures.append({"check": "vm/datastore-missing", "vm": obj.moid, "datastore": ds})
|
||||
break
|
||||
|
||||
return {
|
||||
"base": BASE,
|
||||
"profile": profile.name,
|
||||
"extras_scale": extras_scale,
|
||||
"expected": {
|
||||
"hosts": len(exp_hosts),
|
||||
"vms": len(exp_vms),
|
||||
"datastores": len(exp_ds),
|
||||
"networks": len(exp_nets),
|
||||
"clusters": len(exp_clusters),
|
||||
"datacenters": len(exp_dcs),
|
||||
"folders": want_folders,
|
||||
"resource_pools": len(exp_rps),
|
||||
"libraries_min": want_libraries,
|
||||
"categories_min": want_categories,
|
||||
},
|
||||
"live": {
|
||||
"hosts": len(live_hosts) if isinstance(live_hosts, list) else None,
|
||||
"vms": len(live_vms) if isinstance(live_vms, list) else None,
|
||||
"datastores": len(live_ds) if isinstance(live_ds, list) else None,
|
||||
"networks": len(live_net) if isinstance(live_net, list) else None,
|
||||
"clusters": len(live_cl) if isinstance(live_cl, list) else None,
|
||||
"datacenters": len(live_dc) if isinstance(live_dc, list) else None,
|
||||
"folders": len(live_folder) if isinstance(live_folder, list) else None,
|
||||
"resource_pools": len(live_rp) if isinstance(live_rp, list) else None,
|
||||
"libraries": len(libraries) if isinstance(libraries, list) else None,
|
||||
"categories": len(categories) if isinstance(categories, list) else None,
|
||||
},
|
||||
"failure_count": len(failures),
|
||||
"failures": failures[:100],
|
||||
"ok": len(failures) == 0,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--profile",
|
||||
default=os.getenv("SEED_VSPHERE_PROFILE", "large"),
|
||||
help="small | large | big (demo-cluster aliases big)",
|
||||
)
|
||||
parser.add_argument("--hosts", type=int, default=None)
|
||||
parser.add_argument("--vms", type=int, default=None)
|
||||
args = parser.parse_args()
|
||||
report = audit_profile(args.profile, hosts=args.hosts, vms=args.vms)
|
||||
print(json.dumps(report, indent=2, ensure_ascii=False))
|
||||
return 0 if report["ok"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -40,7 +40,8 @@ def _concrete(path: str) -> str:
|
||||
"{folder}": "group-v23",
|
||||
"{datacenter}": "datacenter-21",
|
||||
"{cluster}": "domain-c21",
|
||||
"{resource_pool}": "resgroup-22",
|
||||
# Disposable id — seed resgroup-22 is protected from DELETE.
|
||||
"{resource_pool}": "resgroup-missing",
|
||||
"{permission_id}": "1",
|
||||
"{policy}": "policy-default",
|
||||
"{disk}": "2000",
|
||||
|
||||
Reference in New Issue
Block a user