DevOpsTools/vmware-api-simulator
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Expand lab seed tiers, OpenAPI PARAMS, and console DATA/Params UX.

Browse Source
This commit is contained in:
Sergey Antropoff
2026-07-18 08:46:39 +03:00
parent f8d3cbdd59
commit 63cc409424
71 changed files with 38380 additions and 796 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pulumi-tests/lib/rest_matrix.py
+168 -131
View File
@@ -1,7 +1,12 @@
"""Full REST verb×path×majors matrix for pulumi-tests (hybrid suite).
"""Full REST verb×path×majors matrix for pulumi-tests (Layer A HTTP contract).
Reuses path substitution / session patterns from scripts/vsphere_full_matrix_probe.py.
Pass rules match that probe: no 5xx/501; inventory GETs nonempty + non-stub on major 9.
Probes ``methods_for_major(M)`` for majors 6–9 with session auth.
Verbs: whatever is in IMPLEMENTED (GET/PUT/PATCH/POST/DELETE) plus a synthetic
HEAD for every GET path. Pass requires ``critical == 0`` and
``probed == declared`` per major and in aggregate.
100% coverage here means the HTTP contract matrix — not pulumi-vsphere resource
count (Layer B).
"""
from __future__ import annotations
@@ -18,7 +23,7 @@ from collections import Counter
from typing import Any
from urllib.parse import urlencode
from app.vsphere.contracts.matrix import VERSIONS, catalog_entries_for_major, methods_for_major
from app.vsphere.contracts.matrix import VERSIONS, methods_for_major
from app.vsphere.rest.coverage import CORE_IMPLEMENTED, IMPLEMENTED
_PATH_SUBS = {
@@ -30,6 +35,7 @@ _PATH_SUBS = {
"{category_id}": "cat-lab-1",
"{tag_id}": "tag-lab-1",
"{item_id}": "item-ubuntu",
"{library_item_id}": "item-ubuntu",
"{library_id}": "lib-local-1",
"{folder}": "group-v23",
"{datacenter}": "datacenter-21",
@@ -75,20 +81,47 @@ _PATH_SUBS = {
_ACCEPT_CLIENT = {400, 401, 403, 404, 405, 409, 412, 422}
_SKIP_DELETE = frozenset(
{
("DELETE", "/api/session"),
("DELETE", "/rest/com/vmware/cis/session"),
}
)
# Collection / entity GETs that must be non-empty after small seed (major 9).
_INVENTORY_CRITICAL = {
"/api/vcenter/vm",
"/api/vcenter/host",
"/api/vcenter/datastore",
"/api/vcenter/network",
"/api/vcenter/cluster",
"/api/vcenter/datacenter",
"/api/vcenter/folder",
"/api/vcenter/resource-pool",
"/api/cis/tagging/category",
"/api/cis/tagging/tag",
"/api/content/library",
"/api/content/local-library",
"/api/vcenter/network/dvs",
"/api/vcenter/storage/policies",
"/api/vcenter/privilege",
"/api/vcenter/authorization/roles",
"/api/vcenter/authorization/permissions",
"/api/cis/tasks",
"/api/esx/settings/clusters/{cluster}/software",
"/api/vcenter/namespace-management/supervisors/{supervisor}/summary",
"/api/appliance/access/ssh",
"/api/appliance/services",
}
# GET paths that may legally return empty/null bodies.
_EMPTY_OK_GET = frozenset(
{
"/api/session",
"/rest/com/vmware/cis/session",
}
)
def _base() -> str:
explicit = os.environ.get("VSPHERE_BASE")
@@ -127,7 +160,6 @@ def request(
headers: dict[str, str],
data: bytes | None = None,
) -> tuple[int, str]:
# Paths may already include query strings from _payload_for.
if "?" in path:
base_path, query = path.split("?", 1)
url = f"{_base()}{concrete_path(base_path)}?{query}"
@@ -152,6 +184,16 @@ def login() -> str:
return json.loads(body)
def declared_routes(major: int) -> list[tuple[str, str]]:
"""Registry routes for a major, plus synthetic HEAD for each GET."""
methods = methods_for_major(major)
base = [(verb, path) for (verb, path) in sorted(methods) if (verb, path) not in _SKIP_DELETE]
heads = [("HEAD", path) for verb, path in base if verb == "GET"]
# Keep HEAD adjacent after its GET in verb order during probe via sort key.
return base + heads
def _payload_for(verb: str, path: str) -> tuple[str, bytes | None]:
if verb not in {"POST", "PUT", "PATCH"}:
return path, None
@@ -165,7 +207,14 @@ def _payload_for(verb: str, path: str) -> tuple[str, bytes | None]:
return f"{path}?action=enter", b"{}"
if path.endswith("/folder/{folder}") and verb == "POST":
return f"{path}?action=rename", json.dumps({"name": "folder-renamed-probe"}).encode()
# Do not rename seed folder MOIDs (breaks /Datacenter/vm/... inventory paths).
return (
"/api/vcenter/folder/folder-missing-matrix?action=rename",
json.dumps({"name": "folder-renamed-probe"}).encode(),
)
if path == "/api/cis/tasks" and verb == "POST":
return f"{path}?action=list", json.dumps({"filter_spec": {}}).encode()
suffix = secrets.token_hex(4)
bodies: dict[str, dict[str, Any]] = {
@@ -175,10 +224,15 @@ def _payload_for(verb: str, path: str) -> tuple[str, bytes | None]:
"cpu_count": 1,
"memory_size_MiB": 512,
},
"/api/vcenter/datacenter": {"name": f"probe-dc-{suffix}"},
"/api/vcenter/cluster": {"name": f"probe-cluster-{suffix}"},
# Missing parent → 404 (client_4xx). Avoid creating extra Datacenter/Cluster
# trees that leave a second ResourcePool named "Resources" for govmomi.
"/api/vcenter/datacenter": {"name": f"probe-dc-{suffix}", "folder": "folder-missing-matrix"},
"/api/vcenter/cluster": {"name": f"probe-cluster-{suffix}", "folder": "folder-missing-matrix"},
"/api/vcenter/folder": {"name": f"probe-folder-{suffix}", "parent": "group-v23"},
"/api/vcenter/resource-pool": {"name": f"probe-rp-{suffix}", "parent": "resgroup-22"},
"/api/vcenter/resource-pool": {
"name": f"probe-rp-{suffix}",
"parent": "resgroup-missing-matrix",
},
"/api/vcenter/network/dvs": {"name": f"probe-dvs-{suffix}"},
"/api/vcenter/network/dvpg": {
"name": f"probe-dvpg-{suffix}",
@@ -263,14 +317,25 @@ def apply_major(major: int, headers: dict[str, str]) -> dict[str, Any]:
def _classify(verb: str, path: str) -> str:
status = CORE_IMPLEMENTED.get((verb, path)) or IMPLEMENTED.get((verb, path))
if (verb, path) in CORE_IMPLEMENTED:
lookup = verb if verb != "HEAD" else "GET"
if (lookup, path) in CORE_IMPLEMENTED:
return "deep"
status = IMPLEMENTED.get((lookup, path))
if status == "stub":
return "stub"
return "deep" if status == "implemented" else "unknown"
def _is_empty_payload(body: str) -> bool:
if not body or not body.strip():
return True
try:
parsed = json.loads(body)
except json.JSONDecodeError:
return False
return parsed in ([], {}, None, "")
def _record_result(
*,
major: int,
@@ -284,82 +349,55 @@ def _record_result(
) -> None:
kind = _classify(verb, path)
deep_stub[kind] += 1
def _fail(expected: str, *, bucket: str) -> None:
buckets[bucket] += 1
failures.append(
{
"major": major,
"verb": verb,
"path": path,
"kind": kind,
"status": code,
"critical": True,
"body": body[:200],
"expected": expected,
}
)
if 200 <= code < 300:
buckets["success_2xx"] += 1
if kind == "stub":
buckets["stub_ok"] += 1
if major == 9 and verb == "GET" and body:
# HEAD bodies are always empty by design.
if verb == "HEAD":
return
if major == 9 and verb == "GET" and path not in _EMPTY_OK_GET:
if '"stub": true' in body or '"stub":true' in body:
if path in _INVENTORY_CRITICAL or kind == "deep":
buckets["stub_marker"] += 1
failures.append(
{
"major": major,
"verb": verb,
"path": path,
"kind": kind,
"status": code,
"body": body[:200],
"expected": "non-stub JSON",
}
)
elif path in _INVENTORY_CRITICAL:
try:
parsed = json.loads(body)
except json.JSONDecodeError:
parsed = None
empty = parsed in ([], {}, None, "")
if empty:
buckets["empty_inventory"] += 1
failures.append(
{
"major": major,
"verb": verb,
"path": path,
"kind": kind,
"status": code,
"body": body[:200],
"expected": "non-empty seeded data",
}
)
elif code in _ACCEPT_CLIENT:
_fail("non-stub JSON", bucket="stub_marker")
elif path in _INVENTORY_CRITICAL and _is_empty_payload(body):
_fail("non-empty seeded data", bucket="empty_inventory")
return
# Synthetic HEAD must be served (middleware); 405 is critical for HEAD only.
if verb == "HEAD" and code == 405:
_fail("HEAD supported via GET route", bucket="head_405")
return
if code in _ACCEPT_CLIENT:
buckets["client_4xx"] += 1
elif code == 501:
buckets["unexpected_501"] += 1
failures.append(
{
"major": major,
"verb": verb,
"path": path,
"kind": kind,
"status": code,
"body": body[:200],
}
)
elif code >= 500:
buckets["server_5xx"] += 1
failures.append(
{
"major": major,
"verb": verb,
"path": path,
"kind": kind,
"status": code,
"body": body[:200],
}
)
else:
buckets[f"other_{code}"] += 1
failures.append(
{
"major": major,
"verb": verb,
"path": path,
"kind": kind,
"status": code,
"body": body[:200],
}
)
return
if code == 501:
_fail("no 501 on registered route", bucket="unexpected_501")
return
if code >= 500:
_fail("no 5xx", bucket="server_5xx")
return
_fail(f"unexpected status {code}", bucket=f"other_{code}")
def probe_major(major: int, session: str) -> dict[str, Any]:
@@ -369,12 +407,11 @@ def probe_major(major: int, session: str) -> dict[str, Any]:
"Accept": "application/json",
}
applied = apply_major(major, headers)
active = methods_for_major(major)
verb_order = {"GET": 0, "PUT": 1, "PATCH": 2, "POST": 3, "DELETE": 4}
entries = sorted(
catalog_entries_for_major(major),
key=lambda e: (verb_order.get(e["verb"], 9), e["path"]),
)
routes = declared_routes(major)
declared = len(routes)
verb_order = {"GET": 0, "HEAD": 1, "PUT": 2, "PATCH": 3, "POST": 4, "DELETE": 5}
routes = sorted(routes, key=lambda item: (verb_order.get(item[0], 9), item[1]))
buckets: Counter[str] = Counter()
deep_stub: Counter[str] = Counter()
@@ -382,30 +419,35 @@ def probe_major(major: int, session: str) -> dict[str, Any]:
probed = 0
by_verb: Counter[str] = Counter()
for entry in entries:
verb = entry["verb"]
path = entry["path"]
for verb, path in routes:
by_verb[verb] += 1
if verb == "DELETE" and path in {"/api/session", "/rest/com/vmware/cis/session"}:
continue
if verb == "DELETE" and path in {
"/api/vcenter/datacenter/{datacenter}",
"/api/vcenter/cluster/{cluster}",
"/api/vcenter/folder/{folder}",
"/api/vcenter/resource-pool/{resource_pool}",
"/api/vcenter/vm/{vm}",
"/api/content/local-library/{library_id}",
"/api/content/library/item/{library_item_id}",
}:
if path.endswith("{vm}"):
url_path = path.replace("{vm}", "vm-missing-matrix")
elif path.endswith("{datacenter}"):
url_path = path.replace("{datacenter}", "dc-missing")
elif path.endswith("{cluster}"):
url_path = path.replace("{cluster}", "cluster-missing")
elif path.endswith("{folder}"):
url_path = path.replace("{folder}", "folder-missing")
else:
url_path = path.replace("{resource_pool}", "rp-missing")
# Avoid destroying seed MOIDs — probe missing ids (expect 404).
url_path = path
for token, missing in (
("{vm}", "vm-missing-matrix"),
("{datacenter}", "dc-missing"),
("{cluster}", "cluster-missing"),
("{folder}", "folder-missing"),
("{resource_pool}", "rp-missing"),
("{library_id}", "lib-missing-matrix"),
("{library_item_id}", "item-missing-matrix"),
):
url_path = url_path.replace(token, missing)
code, body = request(verb, url_path, headers=headers)
elif verb == "HEAD":
url_path = path
if path == "/api/content/library/item":
url_path = f"{path}?library_id=lib-local-1"
code, body = request("HEAD", url_path, headers=headers)
else:
url_path, data = _payload_for(verb, path)
if verb == "GET" and path == "/api/content/library/item":
@@ -424,44 +466,28 @@ def probe_major(major: int, session: str) -> dict[str, Any]:
deep_stub=deep_stub,
)
above_floor = 0
for (verb, path), _status in sorted(IMPLEMENTED.items()):
if (verb, path) in active:
continue
if verb == "DELETE" and path in {"/api/session", "/rest/com/vmware/cis/session"}:
continue
url_path, data = _payload_for(verb, path)
code, body = request(verb, url_path, headers=headers, data=data)
above_floor += 1
probed += 1
by_verb[verb] += 1
_record_result(
major=major,
verb=verb,
path=path,
code=code,
body=body,
buckets=buckets,
failures=failures,
deep_stub=deep_stub,
)
critical = sum(1 for f in failures if f.get("critical"))
coverage_ok = probed == declared and critical == 0
return {
"major": major,
"version": applied.get("runtime_version"),
"method_count": len(entries),
"declared": declared,
"method_count": len(methods_for_major(major)),
"by_verb": dict(by_verb),
"probed": probed,
"above_floor_checked": above_floor,
"probed_eq_declared": probed == declared,
"buckets": dict(buckets),
"deep_vs_stub": dict(deep_stub),
"failures": failures,
"failed": len(failures),
"critical": critical,
"coverage_line": f"{probed - critical}/{declared}",
"ok": coverage_ok,
}
def run_rest_matrix(*, majors: list[int] | None = None) -> dict[str, Any]:
"""Probe IMPLEMENTED × majors. Returns summary suitable for suite JSON/HTML."""
"""Probe IMPLEMENTED × majors (+ HEAD). Returns summary for suite JSON/HTML."""
if majors is None:
majors = [6, 7, 8, 9]
@@ -473,25 +499,36 @@ def run_rest_matrix(*, majors: list[int] | None = None) -> dict[str, Any]:
reports: list[dict[str, Any]] = []
all_failures: list[dict[str, Any]] = []
verb_totals: Counter[str] = Counter()
declared_total = 0
probed_total = 0
critical_total = 0
for major in majors:
report = probe_major(major, session)
reports.append(report)
all_failures.extend(report["failures"])
declared_total += int(report["declared"])
probed_total += int(report["probed"])
critical_total += int(report["critical"])
for verb, count in report["by_verb"].items():
verb_totals[verb] += count
session = login()
apply_major(9, {"vmware-api-session-id": session, "Content-Type": "application/json"})
total = sum(r["probed"] for r in reports)
failed = len(all_failures)
coverage_line = f"{probed_total - critical_total}/{declared_total}"
ok = critical_total == 0 and probed_total == declared_total and all(r["ok"] for r in reports)
return {
"base": _base(),
"majors": reports,
"by_verb": dict(verb_totals),
"total": total,
"failed": failed,
"declared": declared_total,
"probed": probed_total,
"total": probed_total,
"critical": critical_total,
"failed": critical_total,
"failures": all_failures[:120],
"ok": failed == 0,
"coverage_line": coverage_line,
"probed_eq_declared": probed_total == declared_total,
"ok": ok,
}