Initial release of the oVirt/RHV Engine API simulator.
Stateful FastAPI lab with contract packs, Compose/Helm, Docker Hub release targets, and Pulumi coverage across all Engine series (GET/POST/PUT/DELETE/HEAD).
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
.pulumi-state/
|
||||
venv/
|
||||
*.pyc
|
||||
__pycache__/
|
||||
@@ -0,0 +1 @@
|
||||
encryptionsalt: v1:uKSY1aWoXM0=:v1:AdhzDtb0B+jQK+zV:ENNlOSH7hBoBda9thNn/ZdOIVEMDWA==
|
||||
@@ -0,0 +1,4 @@
|
||||
name: ovirt-contract-coverage
|
||||
runtime:
|
||||
name: python
|
||||
description: Cover every oVirt Engine contract operation across all series packs via Pulumi Automation API
|
||||
@@ -0,0 +1,7 @@
|
||||
"""Pulumi project marker. Coverage is driven by run_suite.py (Automation API)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pulumi
|
||||
|
||||
pulumi.export("hint", "Use python run_suite.py / make pulumi-tests")
|
||||
@@ -0,0 +1 @@
|
||||
"""Pulumi coverage package."""
|
||||
@@ -0,0 +1,410 @@
|
||||
"""Execute every contract operation for a series against the Engine API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from collections import Counter
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from shared.config import SuiteConfig
|
||||
from shared.http_client import OVirtClient
|
||||
|
||||
_PATH_PARAM = re.compile(r"\{([^{}]+)\}")
|
||||
_SERIES_API_MAJOR = {
|
||||
"3.0": "3",
|
||||
"3.1": "3",
|
||||
"3.2": "3",
|
||||
"3.3": "3",
|
||||
"3.4": "3",
|
||||
"3.5": "3",
|
||||
"3.6": "3",
|
||||
"4.3": "4",
|
||||
"4.4": "4",
|
||||
"4.5": "4",
|
||||
"master": "4",
|
||||
}
|
||||
|
||||
# Reachable Engine responses count as covered (including 501 Not Implemented).
|
||||
# 401 is not a pass: the suite always authenticates, so auth failures are real gaps.
|
||||
_PASS_STATUSES = frozenset({200, 201, 202, 204, 400, 403, 404, 405, 409, 415, 422, 501})
|
||||
# Success statuses that must return a non-empty payload (204/DELETE/errors exempt).
|
||||
_BODY_REQUIRED_STATUSES = frozenset({200, 201, 202})
|
||||
# Methods expected on a full (non-smoke, unfiltered) coverage run.
|
||||
_FULL_RUN_METHODS = frozenset({"GET", "POST", "PUT", "DELETE", "HEAD"})
|
||||
|
||||
|
||||
def _value_empty(value: Any) -> bool:
|
||||
return value is None or value == "" or value == [] or value == {}
|
||||
|
||||
|
||||
def _payload_nonempty(response: Any, *, method: str) -> tuple[bool, str]:
|
||||
"""Require non-empty response data for successful body-bearing statuses."""
|
||||
# HEAD has no body; DELETE often returns 200 with an empty body (Engine-style).
|
||||
if method in {"HEAD", "DELETE"}:
|
||||
return True, ""
|
||||
status = getattr(response, "status_code", None)
|
||||
if status not in _BODY_REQUIRED_STATUSES:
|
||||
return True, ""
|
||||
text = (getattr(response, "text", None) or "").strip()
|
||||
if not text:
|
||||
return False, "empty response body"
|
||||
try:
|
||||
data = response.json()
|
||||
except Exception:
|
||||
return True, ""
|
||||
if data is None:
|
||||
return False, "null JSON body"
|
||||
if isinstance(data, (list, dict)) and len(data) == 0:
|
||||
return False, "empty JSON body"
|
||||
if isinstance(data, dict) and all(_value_empty(v) for v in data.values()):
|
||||
# Allow Engine empty collections: {"vms": []} has structure but no rows.
|
||||
if len(data) == 1 and isinstance(next(iter(data.values())), list):
|
||||
return True, ""
|
||||
return False, "JSON body has only empty fields"
|
||||
return True, ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class OpResult:
|
||||
series: str
|
||||
operation_id: str
|
||||
method: str
|
||||
path_template: str
|
||||
path_resolved: str
|
||||
kind: str
|
||||
status: str # passed | failed | skipped
|
||||
http_status: int | None
|
||||
expected_hint: int
|
||||
duration_ms: float
|
||||
detail: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SeriesSummary:
|
||||
series: str
|
||||
api_version: str
|
||||
total: int = 0
|
||||
passed: int = 0
|
||||
failed: int = 0
|
||||
skipped: int = 0
|
||||
duration_ms: float = 0.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class CoverageReport:
|
||||
generated_at: str
|
||||
engine_url: str
|
||||
series: list[SeriesSummary] = field(default_factory=list)
|
||||
results: list[OpResult] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def totals(self) -> dict[str, int]:
|
||||
return {
|
||||
"total": sum(s.total for s in self.series),
|
||||
"passed": sum(s.passed for s in self.series),
|
||||
"failed": sum(s.failed for s in self.series),
|
||||
"skipped": sum(s.skipped for s in self.series),
|
||||
}
|
||||
|
||||
@property
|
||||
def methods(self) -> dict[str, int]:
|
||||
counts = Counter(r.method for r in self.results)
|
||||
return {method: counts[method] for method in sorted(counts)}
|
||||
|
||||
|
||||
def list_series(contracts_root: Path) -> list[str]:
|
||||
return sorted(
|
||||
p.name
|
||||
for p in contracts_root.iterdir()
|
||||
if p.is_dir() and (p / "api.json").is_file()
|
||||
)
|
||||
|
||||
|
||||
def load_operations(contracts_root: Path, series: str) -> list[dict[str, Any]]:
|
||||
data = json.loads((contracts_root / series / "api.json").read_text())
|
||||
return list(data.get("operations") or [])
|
||||
|
||||
|
||||
def synthesize_head_ops(ops: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Add a HEAD twin for every GET op (contracts omit HEAD; Engine accepts it)."""
|
||||
heads: list[dict[str, Any]] = []
|
||||
for op in ops:
|
||||
if str(op.get("method", "")).upper() != "GET":
|
||||
continue
|
||||
path = str(op["path"])
|
||||
original_id = str(op.get("operation_id") or f"GET:{path}")
|
||||
head = dict(op)
|
||||
head["method"] = "HEAD"
|
||||
head["operation_id"] = f"head.{original_id}"
|
||||
heads.append(head)
|
||||
return heads
|
||||
|
||||
|
||||
class Inventory:
|
||||
"""Cache of collection → first entity id for path placeholder expansion."""
|
||||
|
||||
def __init__(self, client: OVirtClient, version: str) -> None:
|
||||
self.client = client
|
||||
self.version = version
|
||||
self._ids: dict[str, str] = {}
|
||||
self._listed: set[str] = set()
|
||||
|
||||
def id_for(self, collection: str) -> str | None:
|
||||
collection = collection.strip("/")
|
||||
if collection in self._ids:
|
||||
return self._ids[collection]
|
||||
if collection in self._listed:
|
||||
return None
|
||||
self._listed.add(collection)
|
||||
path = f"/ovirt-engine/api/{collection}"
|
||||
try:
|
||||
r = self.client.request("GET", path, headers=self.client.headers(version=self.version))
|
||||
except Exception:
|
||||
return None
|
||||
if r.status_code != 200:
|
||||
return None
|
||||
try:
|
||||
body = r.json()
|
||||
except Exception:
|
||||
return None
|
||||
# Engine collections are usually { "<singular_or_plural>": [ {...}, ... ] }
|
||||
for value in body.values() if isinstance(body, dict) else []:
|
||||
if isinstance(value, list) and value:
|
||||
first = value[0]
|
||||
if isinstance(first, dict) and first.get("id"):
|
||||
self._ids[collection] = str(first["id"])
|
||||
return self._ids[collection]
|
||||
if isinstance(value, dict) and value.get("id"):
|
||||
self._ids[collection] = str(value["id"])
|
||||
return self._ids[collection]
|
||||
return None
|
||||
|
||||
|
||||
def _collection_before_param(parts: list[str], index: int) -> str | None:
|
||||
# /ovirt-engine/api/vms/{id}/nics/{id} → for first {id} use vms, for second use nics
|
||||
if index <= 0:
|
||||
return None
|
||||
prev = parts[index - 1]
|
||||
if prev.startswith("{") or prev in {"ovirt-engine", "api", "v3", "v4"}:
|
||||
return None
|
||||
return prev
|
||||
|
||||
|
||||
def resolve_path(template: str, inventory: Inventory) -> tuple[str, bool]:
|
||||
"""Return resolved path and whether every placeholder was satisfied from inventory."""
|
||||
|
||||
parts = template.strip("/").split("/")
|
||||
resolved: list[str] = []
|
||||
complete = True
|
||||
for i, part in enumerate(parts):
|
||||
match = _PATH_PARAM.fullmatch(part)
|
||||
if not match:
|
||||
resolved.append(part)
|
||||
continue
|
||||
collection = _collection_before_param(parts, i)
|
||||
entity_id = inventory.id_for(collection) if collection else None
|
||||
if entity_id:
|
||||
resolved.append(entity_id)
|
||||
else:
|
||||
resolved.append(str(uuid4()))
|
||||
complete = False
|
||||
return "/" + "/".join(resolved), complete
|
||||
|
||||
|
||||
def _minimal_body(op: dict[str, Any]) -> dict[str, Any] | None:
|
||||
method = op["method"].upper()
|
||||
if method in {"GET", "DELETE", "HEAD"}:
|
||||
return None
|
||||
element = str(op.get("element") or op.get("resource_type") or "object")
|
||||
kind = str(op.get("kind") or "")
|
||||
if kind == "action":
|
||||
return {}
|
||||
# Generic create/update wrapper used by Engine JSON
|
||||
return {element: {"name": f"pulumi-{uuid4().hex[:8]}", "description": "pulumi coverage"}}
|
||||
|
||||
|
||||
def execute_operation(
|
||||
client: OVirtClient,
|
||||
*,
|
||||
series: str,
|
||||
version: str,
|
||||
op: dict[str, Any],
|
||||
inventory: Inventory,
|
||||
) -> OpResult:
|
||||
method = str(op["method"]).upper()
|
||||
template = str(op["path"])
|
||||
kind = str(op.get("kind") or "")
|
||||
expected = int(op.get("create_status") or op.get("status_code") or 200) if method == "POST" else int(
|
||||
op.get("status_code") or 200
|
||||
)
|
||||
path, _complete = resolve_path(template, inventory)
|
||||
body = _minimal_body(op)
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
kwargs: dict[str, Any] = {"headers": client.headers(version=version)}
|
||||
if body is not None:
|
||||
kwargs["json"] = body
|
||||
response = client.request(method, path, **kwargs)
|
||||
duration = (time.perf_counter() - started) * 1000
|
||||
ok = response.status_code in _PASS_STATUSES
|
||||
detail = ""
|
||||
if ok:
|
||||
body_ok, body_detail = _payload_nonempty(response, method=method)
|
||||
if not body_ok:
|
||||
ok = False
|
||||
detail = body_detail
|
||||
else:
|
||||
detail = response.text[:240]
|
||||
return OpResult(
|
||||
series=series,
|
||||
operation_id=str(op.get("operation_id") or f"{method}:{template}"),
|
||||
method=method,
|
||||
path_template=template,
|
||||
path_resolved=path,
|
||||
kind=kind,
|
||||
status="passed" if ok else "failed",
|
||||
http_status=response.status_code,
|
||||
expected_hint=expected,
|
||||
duration_ms=round(duration, 2),
|
||||
detail=detail,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — surface every transport failure
|
||||
duration = (time.perf_counter() - started) * 1000
|
||||
return OpResult(
|
||||
series=series,
|
||||
operation_id=str(op.get("operation_id") or f"{method}:{template}"),
|
||||
method=method,
|
||||
path_template=template,
|
||||
path_resolved=path,
|
||||
kind=kind,
|
||||
status="failed",
|
||||
http_status=None,
|
||||
expected_hint=expected,
|
||||
duration_ms=round(duration, 2),
|
||||
detail=str(exc)[:240],
|
||||
)
|
||||
|
||||
|
||||
def run_coverage(cfg: SuiteConfig) -> CoverageReport:
|
||||
from datetime import UTC, datetime
|
||||
|
||||
contracts_root = Path(cfg.contracts_root)
|
||||
if not contracts_root.is_dir():
|
||||
# Dev checkout fallback
|
||||
alt = Path(__file__).resolve().parents[3] / "contracts" / "ovirt"
|
||||
if alt.is_dir():
|
||||
contracts_root = alt
|
||||
else:
|
||||
raise FileNotFoundError(f"contracts root not found: {cfg.contracts_root}")
|
||||
|
||||
series_list = list_series(contracts_root)
|
||||
if cfg.series_filter:
|
||||
wanted = {s.strip() for s in cfg.series_filter.split(",") if s.strip()}
|
||||
series_list = [s for s in series_list if s in wanted]
|
||||
if cfg.smoke_only:
|
||||
# Fast path: one modern + one legacy series
|
||||
preferred = [s for s in ("4.5", "3.6") if s in series_list]
|
||||
series_list = preferred or series_list[:1]
|
||||
|
||||
methods_filter = {m.strip() for m in cfg.methods_filter.split(",") if m.strip()} if cfg.methods_filter else set()
|
||||
|
||||
report = CoverageReport(
|
||||
generated_at=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
engine_url=cfg.api_url,
|
||||
)
|
||||
|
||||
with OVirtClient(cfg) as client:
|
||||
client.basic_probe()
|
||||
for series in series_list:
|
||||
version = _SERIES_API_MAJOR.get(series, "4")
|
||||
client.api_version = version
|
||||
act = client.activate_series(series)
|
||||
if act.status_code != 200:
|
||||
summary = SeriesSummary(series=series, api_version=version, total=1, failed=1)
|
||||
report.series.append(summary)
|
||||
report.results.append(
|
||||
OpResult(
|
||||
series=series,
|
||||
operation_id="contracts.activate",
|
||||
method="POST",
|
||||
path_template="/ui/api/ovirt/contracts/activate",
|
||||
path_resolved="/ui/api/ovirt/contracts/activate",
|
||||
kind="meta",
|
||||
status="failed",
|
||||
http_status=act.status_code,
|
||||
expected_hint=200,
|
||||
duration_ms=0,
|
||||
detail=act.text[:240],
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
# Reset to minimal inventory so mutation side-effects from earlier
|
||||
# series do not cascade into later packs. Unload truncates tokens.
|
||||
try:
|
||||
client.request(
|
||||
"POST",
|
||||
"/ui/api/demo/unload",
|
||||
headers={"Accept": "application/json", "Content-Type": "application/json"},
|
||||
json={},
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fresh token after activate + unload (TRUNCATE clears ov_tokens).
|
||||
client.login()
|
||||
|
||||
ops = load_operations(contracts_root, series)
|
||||
if methods_filter:
|
||||
ops = [o for o in ops if str(o.get("method", "")).upper() in methods_filter]
|
||||
if cfg.smoke_only:
|
||||
# Keep root + a handful of collection GETs for smoke
|
||||
kept: list[dict[str, Any]] = []
|
||||
for o in ops:
|
||||
if o.get("kind") == "root" or (
|
||||
o.get("kind") == "collection" and o.get("method") == "GET" and len(kept) < 25
|
||||
):
|
||||
kept.append(o)
|
||||
ops = kept
|
||||
|
||||
# Contracts omit HEAD; Engine accepts HEAD (fallback / HEAD→GET).
|
||||
if not methods_filter or "HEAD" in methods_filter:
|
||||
ops = list(ops) + synthesize_head_ops(ops)
|
||||
|
||||
inventory = Inventory(client, version)
|
||||
summary = SeriesSummary(series=series, api_version=version)
|
||||
series_started = time.perf_counter()
|
||||
for op in ops:
|
||||
result = execute_operation(client, series=series, version=version, op=op, inventory=inventory)
|
||||
report.results.append(result)
|
||||
summary.total += 1
|
||||
if result.status == "passed":
|
||||
summary.passed += 1
|
||||
elif result.status == "skipped":
|
||||
summary.skipped += 1
|
||||
else:
|
||||
summary.failed += 1
|
||||
summary.duration_ms = round((time.perf_counter() - series_started) * 1000, 2)
|
||||
if summary.total != len(ops):
|
||||
raise AssertionError(
|
||||
f"series {series}: executed {summary.total} ops, expected {len(ops)}"
|
||||
)
|
||||
report.series.append(summary)
|
||||
|
||||
return report
|
||||
|
||||
|
||||
def report_to_dict(report: CoverageReport) -> dict[str, Any]:
|
||||
return {
|
||||
"generated_at": report.generated_at,
|
||||
"engine_url": report.engine_url,
|
||||
"totals": report.totals,
|
||||
"methods": report.methods,
|
||||
"series": [asdict(s) for s in report.series],
|
||||
"results": [asdict(r) for r in report.results],
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
"""Write JSON + self-contained HTML coverage reports."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def write_reports(payload: dict[str, Any], report_dir: Path) -> tuple[Path, Path]:
|
||||
report_dir.mkdir(parents=True, exist_ok=True)
|
||||
json_path = report_dir / "pulumi-contract-coverage.json"
|
||||
html_path = report_dir / "pulumi-contract-coverage.html"
|
||||
json_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
|
||||
html_path.write_text(render_html(payload), encoding="utf-8")
|
||||
return json_path, html_path
|
||||
|
||||
|
||||
def render_html(payload: dict[str, Any]) -> str:
|
||||
totals = payload.get("totals") or {}
|
||||
methods = payload.get("methods") or {}
|
||||
series_rows = []
|
||||
for s in payload.get("series") or []:
|
||||
series_rows.append(
|
||||
"<tr>"
|
||||
f"<td>{html.escape(str(s.get('series')))}</td>"
|
||||
f"<td>{html.escape(str(s.get('api_version')))}</td>"
|
||||
f"<td>{s.get('total', 0)}</td>"
|
||||
f"<td class='ok'>{s.get('passed', 0)}</td>"
|
||||
f"<td class='bad'>{s.get('failed', 0)}</td>"
|
||||
f"<td>{s.get('skipped', 0)}</td>"
|
||||
f"<td>{s.get('duration_ms', 0):.0f} ms</td>"
|
||||
"</tr>"
|
||||
)
|
||||
|
||||
method_rows = []
|
||||
for method, count in sorted(methods.items()):
|
||||
method_rows.append(
|
||||
"<tr>"
|
||||
f"<td>{html.escape(str(method))}</td>"
|
||||
f"<td>{count}</td>"
|
||||
"</tr>"
|
||||
)
|
||||
if not method_rows:
|
||||
method_rows.append("<tr><td colspan='2'>No methods recorded.</td></tr>")
|
||||
|
||||
failed = [r for r in (payload.get("results") or []) if r.get("status") == "failed"]
|
||||
fail_rows = []
|
||||
for r in failed[:500]:
|
||||
fail_rows.append(
|
||||
"<tr>"
|
||||
f"<td>{html.escape(str(r.get('series')))}</td>"
|
||||
f"<td><code>{html.escape(str(r.get('operation_id')))}</code></td>"
|
||||
f"<td>{html.escape(str(r.get('method')))}</td>"
|
||||
f"<td><code>{html.escape(str(r.get('path_template')))}</code></td>"
|
||||
f"<td>{html.escape(str(r.get('http_status')))}</td>"
|
||||
f"<td><code>{html.escape(str(r.get('detail') or '')[:180])}</code></td>"
|
||||
"</tr>"
|
||||
)
|
||||
if not fail_rows:
|
||||
fail_rows.append("<tr><td colspan='6'>No failures.</td></tr>")
|
||||
|
||||
# Compact sample of passed ops (first 100) for confidence
|
||||
passed = [r for r in (payload.get("results") or []) if r.get("status") == "passed"]
|
||||
pass_sample = []
|
||||
for r in passed[:100]:
|
||||
pass_sample.append(
|
||||
"<tr>"
|
||||
f"<td>{html.escape(str(r.get('series')))}</td>"
|
||||
f"<td><code>{html.escape(str(r.get('operation_id')))}</code></td>"
|
||||
f"<td>{html.escape(str(r.get('method')))}</td>"
|
||||
f"<td>{html.escape(str(r.get('http_status')))}</td>"
|
||||
f"<td>{r.get('duration_ms', 0)}</td>"
|
||||
"</tr>"
|
||||
)
|
||||
|
||||
return f"""<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<title>oVirt Pulumi contract coverage</title>
|
||||
<style>
|
||||
:root {{
|
||||
--bg: #0f1419;
|
||||
--panel: #1d2226;
|
||||
--text: #f0f3f5;
|
||||
--muted: #9aa3a8;
|
||||
--ok: #3f9c35;
|
||||
--bad: #c9190b;
|
||||
--accent: #0076b6;
|
||||
--line: #2d363c;
|
||||
}}
|
||||
body {{
|
||||
margin: 0; padding: 32px;
|
||||
font-family: "IBM Plex Sans", "Segoe UI", sans-serif;
|
||||
background: linear-gradient(160deg, #0f1419, #16202a 45%, #1a2733);
|
||||
color: var(--text);
|
||||
}}
|
||||
h1 {{ margin: 0 0 8px; font-size: 28px; }}
|
||||
.sub {{ color: var(--muted); margin-bottom: 24px; }}
|
||||
.cards {{ display: grid; grid-template-columns: repeat(4, minmax(120px, 1fr)); gap: 12px; margin-bottom: 28px; }}
|
||||
.card {{ background: var(--panel); border: 1px solid var(--line); padding: 16px; border-radius: 8px; }}
|
||||
.card .label {{ color: var(--muted); font-size: 12px; text-transform: uppercase; letter-spacing: .04em; }}
|
||||
.card .value {{ font-size: 28px; margin-top: 6px; font-weight: 600; }}
|
||||
.card.ok .value {{ color: var(--ok); }}
|
||||
.card.bad .value {{ color: var(--bad); }}
|
||||
table {{ width: 100%; border-collapse: collapse; background: var(--panel); border: 1px solid var(--line); margin-bottom: 28px; }}
|
||||
th, td {{ text-align: left; padding: 10px 12px; border-bottom: 1px solid var(--line); vertical-align: top; font-size: 13px; }}
|
||||
th {{ color: var(--muted); font-weight: 600; background: #161b1f; }}
|
||||
code {{ font-family: "IBM Plex Mono", ui-monospace, monospace; font-size: 12px; }}
|
||||
.ok {{ color: var(--ok); }}
|
||||
.bad {{ color: var(--bad); }}
|
||||
h2 {{ font-size: 18px; margin: 0 0 12px; color: var(--accent); }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>oVirt Pulumi contract coverage</h1>
|
||||
<div class="sub">
|
||||
Generated {html.escape(str(payload.get('generated_at')))}
|
||||
· Engine {html.escape(str(payload.get('engine_url')))}
|
||||
</div>
|
||||
<div class="cards">
|
||||
<div class="card"><div class="label">Total</div><div class="value">{totals.get('total', 0)}</div></div>
|
||||
<div class="card ok"><div class="label">Passed</div><div class="value">{totals.get('passed', 0)}</div></div>
|
||||
<div class="card bad"><div class="label">Failed</div><div class="value">{totals.get('failed', 0)}</div></div>
|
||||
<div class="card"><div class="label">Skipped</div><div class="value">{totals.get('skipped', 0)}</div></div>
|
||||
</div>
|
||||
|
||||
<h2>By HTTP method</h2>
|
||||
<table>
|
||||
<thead><tr><th>Method</th><th>Count</th></tr></thead>
|
||||
<tbody>
|
||||
{''.join(method_rows)}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<h2>By series</h2>
|
||||
<table>
|
||||
<thead><tr><th>Series</th><th>API</th><th>Total</th><th>Passed</th><th>Failed</th><th>Skipped</th><th>Duration</th></tr></thead>
|
||||
<tbody>
|
||||
{''.join(series_rows)}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<h2>Failures (up to 500)</h2>
|
||||
<table>
|
||||
<thead><tr><th>Series</th><th>Operation</th><th>Method</th><th>Path</th><th>HTTP</th><th>Detail</th></tr></thead>
|
||||
<tbody>
|
||||
{''.join(fail_rows)}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<h2>Passed sample (first 100)</h2>
|
||||
<table>
|
||||
<thead><tr><th>Series</th><th>Operation</th><th>Method</th><th>HTTP</th><th>ms</th></tr></thead>
|
||||
<tbody>
|
||||
{''.join(pass_sample) if pass_sample else '<tr><td colspan="5">No passed operations.</td></tr>'}
|
||||
</tbody>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
@@ -0,0 +1,2 @@
|
||||
pulumi>=3.140,<4
|
||||
httpx>=0.28,<0.29
|
||||
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run Engine contract coverage via Pulumi Automation API and write HTML report."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from pulumi import automation as auto
|
||||
|
||||
WORK_DIR = Path(__file__).resolve().parent
|
||||
ROOT = WORK_DIR.parent
|
||||
|
||||
|
||||
def main() -> int:
|
||||
os.environ.setdefault("PULUMI_CONFIG_PASSPHRASE", "ovirt-lab")
|
||||
state_dir = WORK_DIR / ".pulumi-state"
|
||||
state_dir.mkdir(parents=True, exist_ok=True)
|
||||
backend = os.environ.setdefault("PULUMI_BACKEND_URL", f"file://{state_dir}")
|
||||
report_dir = Path(os.environ.setdefault("REPORT_DIR", str(ROOT / "reports")))
|
||||
report_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
pythonpath = os.environ.get("PYTHONPATH", "")
|
||||
parts = [str(ROOT), str(WORK_DIR)]
|
||||
os.environ["PYTHONPATH"] = os.pathsep.join(parts + ([pythonpath] if pythonpath else []))
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
if str(WORK_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(WORK_DIR))
|
||||
|
||||
from coverage.executor import _FULL_RUN_METHODS, report_to_dict, run_coverage
|
||||
from coverage.report import write_reports
|
||||
from shared.config import SuiteConfig
|
||||
|
||||
print("Running Engine contract coverage…", flush=True)
|
||||
cfg = SuiteConfig.from_env()
|
||||
report = run_coverage(cfg)
|
||||
payload = report_to_dict(report)
|
||||
json_path, html_path = write_reports(payload, report_dir)
|
||||
totals = payload["totals"]
|
||||
methods = payload.get("methods") or {}
|
||||
series_names = [s["series"] for s in payload["series"]]
|
||||
|
||||
summary_path = report_dir / "pulumi-stack-summary.json"
|
||||
summary_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"total": totals["total"],
|
||||
"passed": totals["passed"],
|
||||
"failed": totals["failed"],
|
||||
"skipped": totals["skipped"],
|
||||
"methods": methods,
|
||||
"series": series_names,
|
||||
"report_json": str(json_path),
|
||||
"report_html": str(html_path),
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
subprocess.check_call(["pulumi", "login", backend], cwd=str(WORK_DIR))
|
||||
|
||||
def pulumi_program() -> None:
|
||||
import pulumi
|
||||
|
||||
data = json.loads(summary_path.read_text(encoding="utf-8"))
|
||||
for key, value in data.items():
|
||||
pulumi.export(key, value)
|
||||
|
||||
stack_name = os.environ.get("PULUMI_STACK", "dev")
|
||||
stack = auto.create_or_select_stack(
|
||||
stack_name=stack_name,
|
||||
project_name="ovirt-contract-coverage",
|
||||
program=pulumi_program,
|
||||
)
|
||||
print(f"Publishing results to Pulumi stack {stack_name}…", flush=True)
|
||||
result = stack.up(on_output=print)
|
||||
outputs = {k: v.value for k, v in result.outputs.items()}
|
||||
print(json.dumps({"outputs": outputs}, indent=2), flush=True)
|
||||
|
||||
failed = int(outputs.get("failed") or totals["failed"])
|
||||
total = int(outputs.get("total") or totals["total"])
|
||||
passed = int(outputs.get("passed") or totals["passed"])
|
||||
print(f"SUMMARY total={total} passed={passed} failed={failed}", flush=True)
|
||||
print(f"METHODS {json.dumps(methods, sort_keys=True)}", flush=True)
|
||||
print(f"HTML report: {html_path}", flush=True)
|
||||
|
||||
full_run = not cfg.smoke_only and not cfg.methods_filter
|
||||
missing_methods = sorted(_FULL_RUN_METHODS - set(methods)) if full_run else []
|
||||
if missing_methods:
|
||||
print(f"MISSING METHODS on full run: {', '.join(missing_methods)}", flush=True)
|
||||
if failed or missing_methods:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user