411 lines
14 KiB
Python
411 lines
14 KiB
Python
#!/usr/bin/env python3
|
||
"""Hybrid pulumi-tests suite: pulumi-vsphere + full REST matrix + CRUD + SOAP WSDL ops."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import traceback
|
||
import xml.etree.ElementTree as ET
|
||
from datetime import UTC, datetime
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parent
|
||
sys.path.insert(0, str(ROOT / "lib"))
|
||
|
||
from assert_nonempty import assert_nonempty # noqa: E402
|
||
from report_html import write_report # noqa: E402
|
||
|
||
SMOKE = os.environ.get("TEST_SMOKE", "").lower() in {"1", "true", "yes"}
|
||
REPORT_DIR = Path(os.environ.get("REPORT_DIR", "/reports"))
|
||
JUNIT_PATH = Path(os.environ.get("REPORT_PATH", str(REPORT_DIR / "pulumi-junit.xml")))
|
||
HTML_PATH = Path(os.environ.get("HTML_REPORT_PATH", str(REPORT_DIR / "pulumi-report.html")))
|
||
JSON_PATH = Path(os.environ.get("JSON_REPORT_PATH", str(REPORT_DIR / "pulumi-summary.json")))
|
||
PROGRAMS = ROOT / "programs"
|
||
|
||
# Ensure VSPHERE_BASE for HTTP probes when only VSPHERE_SERVER is set (compose).
|
||
if not os.environ.get("VSPHERE_BASE") and os.environ.get("VSPHERE_SERVER"):
|
||
server = os.environ["VSPHERE_SERVER"]
|
||
if not server.startswith("http://") and not server.startswith("https://"):
|
||
os.environ["VSPHERE_BASE"] = f"https://{server}"
|
||
|
||
PULUMI_CASES = [
|
||
{
|
||
"id": "PU-INV",
|
||
"title": "Inventory data sources (dc/cluster/ds/net/vm/host/folder/pool)",
|
||
"dir": "inventory",
|
||
"smoke": True,
|
||
"required": [
|
||
"datacenter_id",
|
||
"datacenter_name",
|
||
"datastore_id",
|
||
"datastore_name",
|
||
"cluster_id",
|
||
"cluster_name",
|
||
"resource_pool_id",
|
||
"resource_pool_name",
|
||
"cluster_resource_pool_id",
|
||
"network_id",
|
||
"network_name",
|
||
"vm_id",
|
||
"vm_name",
|
||
"host_id",
|
||
"host_name",
|
||
"folder_id",
|
||
"folder_path",
|
||
],
|
||
},
|
||
{
|
||
"id": "PU-FOLDER",
|
||
"title": "Folder create/destroy",
|
||
"dir": "folders",
|
||
"smoke": False,
|
||
"required": ["folder_id", "folder_path", "datacenter_id"],
|
||
},
|
||
{
|
||
"id": "PU-VM",
|
||
"title": "VirtualMachine create/destroy",
|
||
"dir": "vm_lifecycle",
|
||
"smoke": False,
|
||
"required": [
|
||
"lab_vm_id",
|
||
"lab_vm_name",
|
||
"resource_pool_id",
|
||
"datastore_id",
|
||
"network_id",
|
||
],
|
||
},
|
||
{
|
||
"id": "PU-TAG",
|
||
"title": "TagCategory + Tag create/destroy",
|
||
"dir": "tags",
|
||
"smoke": False,
|
||
"required": ["category_id", "category_name", "tag_id", "tag_name"],
|
||
},
|
||
]
|
||
|
||
|
||
def _ensure_program_deps() -> None:
|
||
"""Install per-program requirements once (image usually already has them)."""
|
||
|
||
seen: set[str] = set()
|
||
for case in PULUMI_CASES:
|
||
req = PROGRAMS / case["dir"] / "requirements.txt"
|
||
key = str(req.resolve()) if req.exists() else ""
|
||
if not key or key in seen:
|
||
continue
|
||
seen.add(key)
|
||
subprocess.run(
|
||
[sys.executable, "-m", "pip", "install", "-q", "-r", str(req)],
|
||
check=False,
|
||
capture_output=True,
|
||
)
|
||
|
||
|
||
def _run_pulumi_case(case: dict) -> dict:
|
||
work_dir = PROGRAMS / case["dir"]
|
||
if not (work_dir / "__main__.py").is_file():
|
||
return {
|
||
"id": case["id"],
|
||
"title": case["title"],
|
||
"status": "failed",
|
||
"error": f"missing program {work_dir}",
|
||
"outputs": {},
|
||
}
|
||
|
||
try:
|
||
from pulumi import automation as auto
|
||
except Exception as exc: # noqa: BLE001
|
||
return {
|
||
"id": case["id"],
|
||
"title": case["title"],
|
||
"status": "failed",
|
||
"error": f"pulumi automation unavailable: {exc}",
|
||
"outputs": {},
|
||
}
|
||
|
||
os.environ.setdefault("PULUMI_CONFIG_PASSPHRASE", "lab")
|
||
os.environ.setdefault("PULUMI_BACKEND_URL", "file:///tmp/pulumi-state")
|
||
Path("/tmp/pulumi-state").mkdir(parents=True, exist_ok=True)
|
||
|
||
stack_name = f"{case['id'].lower()}-{os.environ.get('TEST_RUN_ID', 'lab')[:8]}"
|
||
try:
|
||
stack = auto.create_or_select_stack(stack_name=stack_name, work_dir=str(work_dir))
|
||
try:
|
||
result = stack.up(on_output=lambda *_: None)
|
||
outputs = {k: v.value for k, v in (result.outputs or {}).items()}
|
||
errors = assert_nonempty(outputs, required=case["required"])
|
||
if errors:
|
||
return {
|
||
"id": case["id"],
|
||
"title": case["title"],
|
||
"status": "failed",
|
||
"error": "; ".join(errors),
|
||
"outputs": outputs,
|
||
}
|
||
return {
|
||
"id": case["id"],
|
||
"title": case["title"],
|
||
"status": "passed",
|
||
"error": "",
|
||
"outputs": outputs,
|
||
}
|
||
finally:
|
||
try:
|
||
stack.destroy(on_output=lambda *_: None)
|
||
except Exception:
|
||
pass
|
||
try:
|
||
stack.workspace.remove_stack(stack_name)
|
||
except Exception:
|
||
pass
|
||
except Exception as exc: # noqa: BLE001
|
||
return {
|
||
"id": case["id"],
|
||
"title": case["title"],
|
||
"status": "failed",
|
||
"error": str(exc)[-3000:],
|
||
"outputs": {},
|
||
}
|
||
|
||
|
||
def _run_rest_matrix() -> dict:
|
||
from rest_matrix import run_rest_matrix
|
||
|
||
majors = [9] if SMOKE else [6, 7, 8, 9]
|
||
try:
|
||
summary = run_rest_matrix(majors=majors)
|
||
except Exception as exc: # noqa: BLE001
|
||
return {
|
||
"id": "PU-REST",
|
||
"title": f"REST full matrix majors={majors}",
|
||
"status": "failed",
|
||
"error": f"{exc}\n{traceback.format_exc()[-1500:]}",
|
||
"outputs": {},
|
||
"rest": {"total": 0, "failed": 1, "failures": [{"error": str(exc)}]},
|
||
}
|
||
status = "passed" if summary.get("ok") else "failed"
|
||
err = ""
|
||
coverage = summary.get("coverage_line") or "?"
|
||
if not summary.get("ok"):
|
||
sample = summary.get("failures") or []
|
||
err = (
|
||
f"REST matrix coverage={coverage} critical={summary.get('critical')} "
|
||
f"probed={summary.get('probed')} declared={summary.get('declared')}; "
|
||
)
|
||
err += "; ".join(f"{f.get('verb')} {f.get('path')} → {f.get('status')}" for f in sample[:8])
|
||
return {
|
||
"id": "PU-REST",
|
||
"title": f"REST HTTP matrix majors={majors} (IMPLEMENTED + HEAD; Layer A)",
|
||
"status": status,
|
||
"error": err,
|
||
"outputs": {
|
||
"coverage": coverage,
|
||
"critical": summary.get("critical"),
|
||
"probed": summary.get("probed"),
|
||
"declared": summary.get("declared"),
|
||
"by_verb": summary.get("by_verb"),
|
||
"majors": [m.get("major") for m in summary.get("majors") or []],
|
||
},
|
||
"rest": summary,
|
||
}
|
||
|
||
|
||
def _run_rest_crud() -> dict:
|
||
from rest_crud import run_rest_crud
|
||
|
||
try:
|
||
summary = run_rest_crud()
|
||
except Exception as exc: # noqa: BLE001
|
||
return {
|
||
"id": "PU-CRUD",
|
||
"title": "Deep REST CRUD (session/folder/tag/library/vm)",
|
||
"status": "failed",
|
||
"error": f"{exc}\n{traceback.format_exc()[-1500:]}",
|
||
"outputs": {},
|
||
"crud": {"total": 0, "failed": 1, "failures": [{"error": str(exc)}]},
|
||
}
|
||
status = "passed" if summary.get("ok") else "failed"
|
||
err = ""
|
||
if not summary.get("ok"):
|
||
err = "; ".join(
|
||
f"{f.get('flow')}/{f.get('step')}: {f.get('error')}"
|
||
for f in (summary.get("failures") or [])
|
||
)
|
||
return {
|
||
"id": "PU-CRUD",
|
||
"title": "Deep REST CRUD (session/folder/tag/library/vm)",
|
||
"status": status,
|
||
"error": err,
|
||
"outputs": {
|
||
"total": summary.get("total"),
|
||
"failed": summary.get("failed"),
|
||
"flows": [f.get("flow") for f in summary.get("flows") or []],
|
||
},
|
||
"crud": summary,
|
||
}
|
||
|
||
|
||
def _run_soap_ops() -> dict:
|
||
from soap_ops import run_soap_ops
|
||
|
||
try:
|
||
summary = run_soap_ops()
|
||
except Exception as exc: # noqa: BLE001
|
||
return {
|
||
"id": "PU-SOAP",
|
||
"title": "SOAP WSDL ops (all /sdk operations)",
|
||
"status": "failed",
|
||
"error": f"{exc}\n{traceback.format_exc()[-1500:]}",
|
||
"outputs": {},
|
||
"soap": {"total": 0, "failed": 1, "failures": [{"error": str(exc)}]},
|
||
}
|
||
status = "passed" if summary.get("ok") else "failed"
|
||
err = ""
|
||
if not summary.get("ok"):
|
||
err = "; ".join(
|
||
f"{f.get('op')}: {f.get('error')}" for f in (summary.get("failures") or [])[:12]
|
||
)
|
||
return {
|
||
"id": "PU-SOAP",
|
||
"title": f"SOAP WSDL ops ({summary.get('wsdl_ops', '?')} operations)",
|
||
"status": status,
|
||
"error": err,
|
||
"outputs": {
|
||
"total": summary.get("total"),
|
||
"failed": summary.get("failed"),
|
||
"wsdl_ops": summary.get("wsdl_ops"),
|
||
},
|
||
"soap": summary,
|
||
}
|
||
|
||
|
||
def main() -> int:
|
||
REPORT_DIR.mkdir(parents=True, exist_ok=True)
|
||
_ensure_program_deps()
|
||
pulumi_cases = [c for c in PULUMI_CASES if c["smoke"]] if SMOKE else PULUMI_CASES
|
||
print(
|
||
f"pulumi hybrid suite pulumi_cases={len(pulumi_cases)} smoke={SMOKE} "
|
||
f"server={os.environ.get('VSPHERE_SERVER', 'api-gateway')} "
|
||
f"base={os.environ.get('VSPHERE_BASE', '')}",
|
||
flush=True,
|
||
)
|
||
|
||
results: list[dict] = []
|
||
failed = 0
|
||
|
||
for case in pulumi_cases:
|
||
print(f"== {case['id']}: {case['title']} ==", flush=True)
|
||
result = _run_pulumi_case(case)
|
||
results.append(result)
|
||
print(f"{case['id']}: {result['status']}", flush=True)
|
||
if result["status"] == "failed":
|
||
failed += 1
|
||
print(result["error"][:800], flush=True)
|
||
|
||
# REST matrix (smoke = major 9 only; full = 6–9)
|
||
print("== PU-REST: REST matrix ==", flush=True)
|
||
rest_result = _run_rest_matrix()
|
||
results.append(rest_result)
|
||
print(f"PU-REST: {rest_result['status']}", flush=True)
|
||
if rest_result["status"] == "failed":
|
||
failed += 1
|
||
print(rest_result["error"][:800], flush=True)
|
||
|
||
rest_summary = rest_result.get("rest") or {}
|
||
crud_summary: dict = {}
|
||
soap_summary: dict = {}
|
||
|
||
if not SMOKE:
|
||
print("== PU-CRUD: deep REST CRUD ==", flush=True)
|
||
crud_result = _run_rest_crud()
|
||
results.append(crud_result)
|
||
crud_summary = crud_result.get("crud") or {}
|
||
print(f"PU-CRUD: {crud_result['status']}", flush=True)
|
||
if crud_result["status"] == "failed":
|
||
failed += 1
|
||
print(crud_result["error"][:800], flush=True)
|
||
|
||
print("== PU-SOAP: WSDL ops ==", flush=True)
|
||
soap_result = _run_soap_ops()
|
||
results.append(soap_result)
|
||
soap_summary = soap_result.get("soap") or {}
|
||
print(f"PU-SOAP: {soap_result['status']}", flush=True)
|
||
if soap_result["status"] == "failed":
|
||
failed += 1
|
||
print(soap_result["error"][:800], flush=True)
|
||
|
||
summary = {
|
||
"generated_at": datetime.now(UTC).isoformat(),
|
||
"vsphere_server": os.environ.get("VSPHERE_SERVER", "api-gateway"),
|
||
"vsphere_base": os.environ.get("VSPHERE_BASE", ""),
|
||
"smoke": SMOKE,
|
||
"provider": "pulumi-vsphere+rest+soap",
|
||
"cases": results,
|
||
"total_failed": failed,
|
||
"rest": {
|
||
"total": rest_summary.get("total", 0),
|
||
"probed": rest_summary.get("probed", rest_summary.get("total", 0)),
|
||
"declared": rest_summary.get("declared", 0),
|
||
"critical": rest_summary.get("critical", rest_summary.get("failed", 0)),
|
||
"failed": rest_summary.get("failed", 0),
|
||
"coverage_line": rest_summary.get("coverage_line"),
|
||
"probed_eq_declared": rest_summary.get("probed_eq_declared"),
|
||
"by_verb": rest_summary.get("by_verb"),
|
||
"majors": rest_summary.get("majors"),
|
||
"failures": rest_summary.get("failures"),
|
||
},
|
||
"crud": {
|
||
"total": crud_summary.get("total", 0),
|
||
"failed": crud_summary.get("failed", 0),
|
||
"flows": crud_summary.get("flows"),
|
||
"failures": crud_summary.get("failures"),
|
||
},
|
||
"soap": {
|
||
"total": soap_summary.get("total", 0),
|
||
"failed": soap_summary.get("failed", 0),
|
||
"ops": soap_summary.get("ops"),
|
||
"failures": soap_summary.get("failures"),
|
||
},
|
||
}
|
||
JSON_PATH.write_text(json.dumps(summary, indent=2, default=str), encoding="utf-8")
|
||
write_report(summary, HTML_PATH)
|
||
|
||
suite = ET.Element(
|
||
"testsuite",
|
||
name="pulumi-hybrid",
|
||
tests=str(len(results)),
|
||
failures=str(failed),
|
||
)
|
||
for result in results:
|
||
node = ET.SubElement(
|
||
suite,
|
||
"testcase",
|
||
classname="pulumi-hybrid",
|
||
name=f"{result['id']} {result['title']}",
|
||
)
|
||
if result["status"] == "failed":
|
||
ET.SubElement(node, "failure", message=result["error"][:500])
|
||
elif result["status"] == "skipped":
|
||
ET.SubElement(node, "skipped", message=result.get("error") or "skipped")
|
||
ET.ElementTree(suite).write(JUNIT_PATH, encoding="utf-8", xml_declaration=True)
|
||
|
||
print(f"Wrote {HTML_PATH}", flush=True)
|
||
print(f"Wrote {JSON_PATH}", flush=True)
|
||
print(f"Wrote {JUNIT_PATH}", flush=True)
|
||
print(
|
||
f"SUMMARY failed={failed} total={len(results)} "
|
||
f"rest.coverage={summary['rest'].get('coverage_line')} "
|
||
f"rest.critical={summary['rest']['critical']} "
|
||
f"crud.failed={summary['crud']['failed']} "
|
||
f"soap.failed={summary['soap']['failed']}",
|
||
flush=True,
|
||
)
|
||
return 1 if failed else 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|