Initial commit: stateful OpenStack API laboratory simulator.
Ship Keystone auth, multi-service handlers (Yoga→Dalmatian), Compose/Helm packaging, API contract packs, and pytest/Pulumi coverage labs.
This commit is contained in:
@@ -0,0 +1 @@
|
||||
# Pulumi OpenStack coverage helpers.
|
||||
@@ -0,0 +1,208 @@
|
||||
"""HTTP pack coverage with completeness and non-empty body checks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from _lib.validate import payload_nonempty
|
||||
|
||||
# Methods that normally return a JSON body on success (DELETE / 204 may be empty).
|
||||
_BODY_METHODS = frozenset({"GET", "POST", "PUT", "PATCH"})
|
||||
|
||||
|
||||
def _expected_ops(packs: dict[str, Any], *, collections_only: bool) -> int:
|
||||
if not collections_only:
|
||||
return sum(len(p.operations) for p in packs.values())
|
||||
total = 0
|
||||
for pack in packs.values():
|
||||
for op in pack.operations:
|
||||
if op.method == "GET" and "{" not in op.path:
|
||||
total += 1
|
||||
return total
|
||||
|
||||
|
||||
def _methods_breakdown(results: list[Any]) -> dict[str, int]:
|
||||
counts: Counter[str] = Counter()
|
||||
for r in results:
|
||||
method = getattr(r, "method", None) or (r.get("method") if isinstance(r, dict) else None)
|
||||
if method:
|
||||
counts[str(method).upper()] += 1
|
||||
return {m: counts.get(m, 0) for m in ("GET", "POST", "PUT", "PATCH", "DELETE")}
|
||||
|
||||
|
||||
def _nonempty_from_lifecycle(report: Any) -> list[dict[str, Any]]:
|
||||
"""Check succeeded lifecycle bodies (skip DELETE / 204 / 202 / no-body)."""
|
||||
failures: list[dict[str, Any]] = []
|
||||
for r in report.results:
|
||||
if not r.succeeded:
|
||||
continue
|
||||
if r.method == "DELETE" or r.status in {202, 204}:
|
||||
continue
|
||||
if r.method not in _BODY_METHODS:
|
||||
continue
|
||||
# OpenStack often returns 200/201 with an empty body (Swift PUT, tag put).
|
||||
if r.payload is None:
|
||||
continue
|
||||
if not payload_nonempty(r.payload, collection_key=r.collection_key, method=r.method):
|
||||
failures.append(
|
||||
{
|
||||
"service": r.service,
|
||||
"operation_id": r.operation_id,
|
||||
"method": r.method,
|
||||
"path": r.path,
|
||||
"status": r.status,
|
||||
"detail": "empty response body",
|
||||
"ok": False,
|
||||
"nonempty": False,
|
||||
}
|
||||
)
|
||||
return failures
|
||||
|
||||
|
||||
def _nonempty_smoke_collections(
|
||||
series: str,
|
||||
*,
|
||||
host: str,
|
||||
packs: dict[str, Any],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Re-check collection GET bodies for smoke mode (stable seed data)."""
|
||||
from app.openstack.surface_probe import (
|
||||
SUCCESS,
|
||||
fill_path,
|
||||
http_request,
|
||||
issue_token,
|
||||
_seed_context,
|
||||
)
|
||||
|
||||
token, auth_body = issue_token(host)
|
||||
project_id = str(((auth_body.get("token") or {}).get("project") or {}).get("id") or "")
|
||||
ctx = _seed_context(host, token, project_id)
|
||||
failures: list[dict[str, Any]] = []
|
||||
for name in sorted(packs):
|
||||
pack = packs[name]
|
||||
for op in pack.operations:
|
||||
if "{" in op.path or op.method != "GET":
|
||||
continue
|
||||
path = fill_path(
|
||||
op.path,
|
||||
{
|
||||
**ctx,
|
||||
"project_id": project_id,
|
||||
"project": project_id,
|
||||
"tenant_id": project_id,
|
||||
"account": project_id,
|
||||
},
|
||||
)
|
||||
url = f"{host.rstrip('/')}{path}"
|
||||
status, payload = http_request(op.method, url, token=token, service=pack.name)
|
||||
if status not in SUCCESS or status == 204:
|
||||
continue
|
||||
if not payload_nonempty(payload, collection_key=op.collection_key, method=op.method):
|
||||
failures.append(
|
||||
{
|
||||
"service": pack.name,
|
||||
"operation_id": op.operation_id,
|
||||
"method": op.method,
|
||||
"path": op.path,
|
||||
"status": status,
|
||||
"detail": "empty response body",
|
||||
"ok": False,
|
||||
"nonempty": False,
|
||||
}
|
||||
)
|
||||
return failures
|
||||
|
||||
|
||||
def probe_pack_operations(
|
||||
series: str,
|
||||
*,
|
||||
host: str,
|
||||
collections_only: bool = False,
|
||||
require_nonempty: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
from app.openstack.contract_loader import load_series_pack
|
||||
from app.openstack.surface_probe import probe_series
|
||||
|
||||
report = probe_series(
|
||||
series,
|
||||
host=host,
|
||||
collections_only=collections_only,
|
||||
lifecycle=not collections_only,
|
||||
)
|
||||
|
||||
packs = load_series_pack(series)
|
||||
expected_ops = _expected_ops(packs, collections_only=collections_only)
|
||||
methods = _methods_breakdown(report.results)
|
||||
coverage_incomplete = len(report.results) != expected_ops
|
||||
|
||||
nonempty_failures: list[dict[str, Any]] = []
|
||||
if require_nonempty:
|
||||
if collections_only:
|
||||
nonempty_failures = _nonempty_smoke_collections(series, host=host, packs=packs)
|
||||
else:
|
||||
nonempty_failures = _nonempty_from_lifecycle(report)
|
||||
|
||||
probe_failures = [
|
||||
{
|
||||
"service": r.service,
|
||||
"operation_id": r.operation_id,
|
||||
"method": r.method,
|
||||
"path": r.path,
|
||||
"status": r.status,
|
||||
"detail": r.detail,
|
||||
"ok": False,
|
||||
"nonempty": True,
|
||||
}
|
||||
for r in report.failures
|
||||
]
|
||||
|
||||
coverage_failures: list[dict[str, Any]] = []
|
||||
if coverage_incomplete:
|
||||
coverage_failures.append(
|
||||
{
|
||||
"service": "_coverage",
|
||||
"operation_id": "coverage_incomplete",
|
||||
"method": "*",
|
||||
"path": "*",
|
||||
"status": 0,
|
||||
"detail": f"coverage_incomplete: total={len(report.results)} expected_ops={expected_ops}",
|
||||
"ok": False,
|
||||
"nonempty": True,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"series": series,
|
||||
"host": host,
|
||||
"mode": report.mode,
|
||||
"total": len(report.results),
|
||||
"expected_ops": expected_ops,
|
||||
"coverage_incomplete": coverage_incomplete,
|
||||
"methods": methods,
|
||||
"ok_count": len(report.results) - len(report.failures),
|
||||
"fail_count": len(report.failures) + (1 if coverage_incomplete else 0),
|
||||
"nonempty_fail_count": len(nonempty_failures),
|
||||
"results": [
|
||||
{
|
||||
"service": r.service,
|
||||
"method": r.method,
|
||||
"path": r.path,
|
||||
"operation_id": r.operation_id,
|
||||
"status": r.status,
|
||||
"detail": r.detail,
|
||||
"mode": r.mode,
|
||||
"ok": r.ok,
|
||||
"succeeded": r.succeeded,
|
||||
}
|
||||
for r in report.results
|
||||
],
|
||||
"failures": coverage_failures + probe_failures + nonempty_failures,
|
||||
}
|
||||
|
||||
|
||||
def write_probe_json(payload: dict[str, Any], path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Render HTML report from Pulumi + HTTP coverage results."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
SERIES_ORDER = ("yoga", "antelope", "caracal", "dalmatian")
|
||||
|
||||
|
||||
def _methods_line(methods: dict[str, Any]) -> str:
|
||||
return " ".join(f"{m}={methods.get(m, 0)}" for m in ("GET", "POST", "PUT", "PATCH", "DELETE"))
|
||||
|
||||
|
||||
def render_html(summary: dict[str, Any], series_reports: list[dict[str, Any]]) -> str:
|
||||
generated = datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
cards = []
|
||||
for rep in series_reports:
|
||||
series = html.escape(str(rep.get("series", "?")))
|
||||
pu = rep.get("pulumi", {})
|
||||
http = rep.get("http", {})
|
||||
cards.append(
|
||||
f"""
|
||||
<div class="card">
|
||||
<h3>{series}</h3>
|
||||
<p class="muted">pulumi_openstack + HTTP pack probe</p>
|
||||
<div class="stats">
|
||||
<span class="ok">pulumi exports ok={len(pu.get("outputs", {})) - len(pu.get("empty_exports", []))}</span>
|
||||
<span class="fail">empty exports={len(pu.get("empty_exports", []))}</span>
|
||||
</div>
|
||||
<div class="stats">
|
||||
<span class="ok">http ok={http.get("ok_count", 0)}</span>
|
||||
<span class="fail">http fail={http.get("fail_count", 0)} nonempty_fail={http.get("nonempty_fail_count", 0)}</span>
|
||||
<span>http total={http.get("total", 0)}/{http.get("expected_ops", "?")}</span>
|
||||
</div>
|
||||
<div class="stats muted">
|
||||
methods: {html.escape(_methods_line(http.get("methods") or {}))}
|
||||
{" · <span class='fail'>coverage incomplete</span>" if http.get("coverage_incomplete") else ""}
|
||||
</div>
|
||||
</div>"""
|
||||
)
|
||||
|
||||
detail_rows = []
|
||||
for rep in series_reports:
|
||||
series = rep.get("series", "?")
|
||||
for item in rep.get("http", {}).get("failures", [])[:500]:
|
||||
detail_rows.append(
|
||||
'<tr class="fail">'
|
||||
f"<td>{html.escape(str(series))}</td>"
|
||||
f"<td>{html.escape(str(item.get('service', '')))}</td>"
|
||||
f"<td>{html.escape(str(item.get('operation_id', '')))}</td>"
|
||||
f"<td>{html.escape(str(item.get('method', '')))}</td>"
|
||||
f"<td><code>{html.escape(str(item.get('path', '')))}</code></td>"
|
||||
f"<td>{html.escape(str(item.get('status', '')))}</td>"
|
||||
f"<td>{html.escape(str(item.get('detail', ''))[:240])}</td>"
|
||||
"</tr>"
|
||||
)
|
||||
for empty in rep.get("pulumi", {}).get("empty_exports", []):
|
||||
detail_rows.append(
|
||||
'<tr class="fail">'
|
||||
f"<td>{html.escape(str(series))}</td>"
|
||||
f"<td>pulumi_openstack</td>"
|
||||
f"<td>export_nonempty</td>"
|
||||
f"<td>—</td><td><code>export</code></td><td>—</td>"
|
||||
f"<td>{html.escape(str(empty))}</td>"
|
||||
"</tr>"
|
||||
)
|
||||
|
||||
return f"""<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<title>Pulumi OpenStack coverage</title>
|
||||
<style>
|
||||
:root {{ --bg:#0f1419; --panel:#1a222c; --text:#e7ecf3; --muted:#9aa7b8; --ok:#3dd68c; --fail:#ff6b6b; --border:#2a3544; }}
|
||||
body {{ margin:0; font-family:"IBM Plex Sans",sans-serif; background:radial-gradient(1000px 500px at 0% 0%,#1b2a3d,var(--bg)); color:var(--text); }}
|
||||
header, main {{ max-width:1200px; margin:0 auto; padding:1.5rem; }}
|
||||
.summary, .cards {{ display:grid; gap:.75rem; grid-template-columns:repeat(auto-fit,minmax(160px,1fr)); margin:1rem 0; }}
|
||||
.summary div, .card {{ background:var(--panel); border:1px solid var(--border); border-radius:10px; padding:1rem; }}
|
||||
.ok {{ color:var(--ok); font-weight:600; }} .fail {{ color:var(--fail); font-weight:600; }}
|
||||
.muted {{ color:var(--muted); }}
|
||||
table {{ width:100%; border-collapse:collapse; background:var(--panel); border:1px solid var(--border); border-radius:10px; overflow:hidden; font-size:.9rem; }}
|
||||
th, td {{ padding:.45rem .6rem; border-bottom:1px solid var(--border); text-align:left; vertical-align:top; }}
|
||||
th {{ color:var(--muted); background:#121820; }}
|
||||
tr.fail {{ background:rgba(255,107,107,.08); }}
|
||||
code {{ font-family:ui-monospace,monospace; font-size:.82rem; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>Pulumi OpenStack API coverage</h1>
|
||||
<p class="muted">Generated {html.escape(generated)} · pulumi_openstack primary + HTTP pack probe with non-empty checks</p>
|
||||
</header>
|
||||
<main>
|
||||
<div class="summary">
|
||||
<div><strong>{summary.get("series_count", 0)}</strong><span class="muted"> series</span></div>
|
||||
<div><strong class="ok">{summary.get("pulumi_ok", 0)}</strong><span class="muted"> pulumi stacks ok</span></div>
|
||||
<div><strong class="fail">{summary.get("pulumi_fail", 0)}</strong><span class="muted"> pulumi failures</span></div>
|
||||
<div><strong class="ok">{summary.get("http_ok", 0)}</strong><span class="muted"> http ops ok</span></div>
|
||||
<div><strong class="fail">{summary.get("http_fail", 0)}</strong><span class="muted"> http / nonempty fails</span></div>
|
||||
</div>
|
||||
<h2>Series</h2>
|
||||
<div class="cards">{"".join(cards)}</div>
|
||||
<h2>Failures</h2>
|
||||
<table>
|
||||
<thead><tr><th>Series</th><th>Service</th><th>Operation</th><th>Method</th><th>Path</th><th>HTTP</th><th>Detail</th></tr></thead>
|
||||
<tbody>{"".join(detail_rows) if detail_rows else '<tr><td colspan="7">No failures</td></tr>'}</tbody>
|
||||
</table>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
def write_html(
|
||||
report_dir: Path, summary: dict[str, Any], series_reports: list[dict[str, Any]]
|
||||
) -> Path:
|
||||
path = report_dir / "pulumi-report.html"
|
||||
path.write_text(render_html(summary, series_reports), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def load_series_files(report_dir: Path) -> list[dict[str, Any]]:
|
||||
out = []
|
||||
for series in SERIES_ORDER:
|
||||
path = report_dir / f"series-{series}.json"
|
||||
if path.exists():
|
||||
out.append(json.loads(path.read_text(encoding="utf-8")))
|
||||
return out
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Helpers for series activation and non-empty validation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
|
||||
def activate_series(host: str, series: str) -> None:
|
||||
body = json.dumps({"series": series}).encode()
|
||||
req = urllib.request.Request(
|
||||
f"{host.rstrip('/')}/ui/api/openstack/contracts/activate",
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json", "Accept": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as res:
|
||||
if res.status >= 400:
|
||||
raise RuntimeError(f"activate {series}: HTTP {res.status}")
|
||||
except urllib.error.HTTPError as exc:
|
||||
raw = exc.read().decode()
|
||||
raise RuntimeError(f"activate {series}: HTTP {exc.code} {raw[:300]}") from exc
|
||||
|
||||
|
||||
def is_nonempty(value: Any) -> bool:
|
||||
if value is None:
|
||||
return False
|
||||
if isinstance(value, str) and not value.strip():
|
||||
return False
|
||||
if isinstance(value, (list, tuple, set, dict)) and len(value) == 0:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def assert_outputs_nonempty(outputs: dict[str, Any], *, min_count: int = 15) -> list[str]:
|
||||
"""Return list of failing export names (empty if all good)."""
|
||||
failures: list[str] = []
|
||||
if len(outputs) < min_count:
|
||||
failures.append(f"__export_count__={len(outputs)}<{min_count}")
|
||||
for key, value in sorted(outputs.items()):
|
||||
if not is_nonempty(value):
|
||||
failures.append(f"{key}={value!r}")
|
||||
return failures
|
||||
|
||||
|
||||
def payload_nonempty(
|
||||
payload: Any, *, collection_key: str | None = None, method: str = "GET"
|
||||
) -> bool:
|
||||
"""True when a successful response body has meaningful content.
|
||||
|
||||
For GET list/show and POST create we require real data — empty ``[]`` /
|
||||
``{}`` / blank strings fail. DELETE/204-style empties are not checked here.
|
||||
"""
|
||||
if payload is None:
|
||||
return False
|
||||
if isinstance(payload, str):
|
||||
return bool(payload.strip())
|
||||
if isinstance(payload, list):
|
||||
return len(payload) > 0
|
||||
if not isinstance(payload, dict):
|
||||
return True
|
||||
if collection_key and collection_key in payload:
|
||||
value = payload[collection_key]
|
||||
if isinstance(value, list):
|
||||
return len(value) > 0
|
||||
return is_nonempty(value)
|
||||
# Common OpenStack envelopes
|
||||
for key, value in payload.items():
|
||||
if key in {"versions", "version", "id", "token", "links", "status", "name"}:
|
||||
if is_nonempty(value):
|
||||
return True
|
||||
if isinstance(value, list) and value:
|
||||
return True
|
||||
if isinstance(value, dict) and (value.get("id") or value.get("name") or value.get("uuid")):
|
||||
return True
|
||||
if isinstance(value, str) and value.strip():
|
||||
return True
|
||||
if isinstance(value, (int, float, bool)):
|
||||
return True
|
||||
# Non-empty dict with any nested content
|
||||
return any(is_nonempty(v) for v in payload.values())
|
||||
@@ -0,0 +1 @@
|
||||
encryptionsalt: v1:sHIQhiAs888=:v1:YU5x9MYzw3X3DaH0:SmpMQVn+3UVBkcz4HBf42IVasHoToQ==
|
||||
@@ -0,0 +1 @@
|
||||
encryptionsalt: v1:jXbSF2V0mog=:v1:ZhW0i106mP+RAph4:sS0KukNKHIsJ/z0Ozeoaa2Zf3Cj8nw==
|
||||
@@ -0,0 +1 @@
|
||||
encryptionsalt: v1:SXzsEq8DgEw=:v1:RQUIpYKUzggs7nqf:yUC20qV384YMu+fnGhjFKH7aSD5Dkg==
|
||||
@@ -0,0 +1 @@
|
||||
encryptionsalt: v1:B2EuV1+nh3k=:v1:9khdcorRMBI/WL1d:e08aEeItHe3flfvAWfDkSO9LUxEUrQ==
|
||||
@@ -0,0 +1,3 @@
|
||||
name: os-coverage
|
||||
runtime: python
|
||||
description: pulumi_openstack coverage stack for openstack-api-simulator
|
||||
@@ -0,0 +1,157 @@
|
||||
"""Maximize pulumi_openstack coverage against the OpenStack API simulator.
|
||||
|
||||
Creates / looks up resources via the official provider, then exports IDs so
|
||||
the suite can assert every stack output is non-empty.
|
||||
|
||||
Notes vs simulator limits:
|
||||
- flavor_id is seeded ``1`` (get_flavor fails on extra_specs shape)
|
||||
- skip Heat/Designate/Swift/Octavia creates (status codes or catalog URL
|
||||
shapes the lab does not yet match)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import uuid
|
||||
|
||||
import pulumi
|
||||
from pulumi_openstack import blockstorage, compute, identity, images, networking
|
||||
|
||||
series = os.environ.get("OPENSTACK_SERIES", "dalmatian")
|
||||
tag = f"pu-{series}-{uuid.uuid4().hex[:6]}"
|
||||
|
||||
# --- Data sources (reads) ---
|
||||
auth = identity.get_auth_scope(name="lab-token")
|
||||
image = images.get_image(name="cirros", most_recent=True)
|
||||
demo_net = networking.get_network(name="demo-net")
|
||||
flavor_id = "1" # seeded m1.tiny
|
||||
|
||||
# --- Identity (writes) ---
|
||||
project = identity.Project(
|
||||
f"{tag}-project",
|
||||
name=f"{tag}-project",
|
||||
description=f"pulumi coverage {series}",
|
||||
enabled=True,
|
||||
)
|
||||
user = identity.User(
|
||||
f"{tag}-user",
|
||||
name=f"{tag}-user",
|
||||
password="pulumi-lab-secret",
|
||||
description=f"pulumi coverage {series}",
|
||||
enabled=True,
|
||||
)
|
||||
|
||||
# --- Networking ---
|
||||
app_net = networking.Network(f"{tag}-net", name=f"{tag}-net", admin_state_up=True)
|
||||
app_subnet = networking.Subnet(
|
||||
f"{tag}-subnet",
|
||||
name=f"{tag}-subnet",
|
||||
network_id=app_net.id,
|
||||
cidr="10.88.0.0/24",
|
||||
ip_version=4,
|
||||
enable_dhcp=True,
|
||||
)
|
||||
router = networking.Router(f"{tag}-router", name=f"{tag}-router", admin_state_up=True)
|
||||
router_iface = networking.RouterInterface(
|
||||
f"{tag}-rtr-if",
|
||||
router_id=router.id,
|
||||
subnet_id=app_subnet.id,
|
||||
)
|
||||
sg = networking.SecGroup(
|
||||
f"{tag}-sg",
|
||||
name=f"{tag}-sg",
|
||||
description=f"pulumi {series}",
|
||||
delete_default_rules=True,
|
||||
)
|
||||
sg_rule = networking.SecGroupRule(
|
||||
f"{tag}-sg-ssh",
|
||||
direction="ingress",
|
||||
ethertype="IPv4",
|
||||
protocol="tcp",
|
||||
port_range_min=22,
|
||||
port_range_max=22,
|
||||
remote_ip_prefix="0.0.0.0/0",
|
||||
security_group_id=sg.id,
|
||||
)
|
||||
port = networking.Port(
|
||||
f"{tag}-port",
|
||||
name=f"{tag}-port",
|
||||
network_id=app_net.id,
|
||||
admin_state_up=True,
|
||||
security_group_ids=[sg.id],
|
||||
fixed_ips=[networking.PortFixedIpArgs(subnet_id=app_subnet.id)],
|
||||
)
|
||||
|
||||
# --- Compute ---
|
||||
keypair = compute.Keypair(f"{tag}-kp", name=f"{tag}-kp")
|
||||
server_group = compute.ServerGroup(
|
||||
f"{tag}-sgroup",
|
||||
name=f"{tag}-sgroup",
|
||||
policies="anti-affinity",
|
||||
)
|
||||
server = compute.Instance(
|
||||
f"{tag}-vm",
|
||||
name=f"{tag}-vm",
|
||||
flavor_id=flavor_id,
|
||||
image_id=image.id,
|
||||
key_pair=keypair.name,
|
||||
security_groups=[sg.name],
|
||||
networks=[compute.InstanceNetworkArgs(uuid=demo_net.id)],
|
||||
scheduler_hints=[compute.InstanceSchedulerHintArgs(group=server_group.id)],
|
||||
metadata={"managed_by": "pulumi", "series": series, "tag": tag},
|
||||
)
|
||||
iface = compute.InterfaceAttach(
|
||||
f"{tag}-iface",
|
||||
instance_id=server.id,
|
||||
port_id=port.id,
|
||||
)
|
||||
|
||||
# --- Block storage ---
|
||||
volume = blockstorage.Volume(
|
||||
f"{tag}-vol",
|
||||
name=f"{tag}-vol",
|
||||
size=1,
|
||||
description=f"pulumi coverage {series}",
|
||||
)
|
||||
vol_attach = compute.VolumeAttach(
|
||||
f"{tag}-attach",
|
||||
instance_id=server.id,
|
||||
volume_id=volume.id,
|
||||
)
|
||||
volume2 = blockstorage.Volume(
|
||||
f"{tag}-vol2",
|
||||
name=f"{tag}-vol2",
|
||||
size=1,
|
||||
description=f"pulumi second volume {series}",
|
||||
)
|
||||
|
||||
# Octavia / Designate / Swift / Heat omitted: catalog paths or status codes
|
||||
# in the lab gateway do not yet match what pulumi_openstack expects.
|
||||
|
||||
# --- Exports (all must be non-empty; suite requires >= 25) ---
|
||||
pulumi.export("series", series)
|
||||
pulumi.export("tag", tag)
|
||||
pulumi.export("auth_user_id", auth.user_id)
|
||||
pulumi.export("auth_project_id", auth.project_id)
|
||||
pulumi.export("project_name", auth.project_name)
|
||||
pulumi.export("flavor_id", flavor_id)
|
||||
pulumi.export("image_id", image.id)
|
||||
pulumi.export("image_name", image.name)
|
||||
pulumi.export("demo_net_id", demo_net.id)
|
||||
pulumi.export("created_project_id", project.id)
|
||||
pulumi.export("created_user_id", user.id)
|
||||
pulumi.export("network_id", app_net.id)
|
||||
pulumi.export("subnet_id", app_subnet.id)
|
||||
pulumi.export("router_id", router.id)
|
||||
pulumi.export("router_iface_id", router_iface.id)
|
||||
pulumi.export("secgroup_id", sg.id)
|
||||
pulumi.export("secgroup_rule_id", sg_rule.id)
|
||||
pulumi.export("port_id", port.id)
|
||||
pulumi.export("keypair_name", keypair.name)
|
||||
pulumi.export("server_group_id", server_group.id)
|
||||
pulumi.export("server_id", server.id)
|
||||
pulumi.export("server_name", server.name)
|
||||
pulumi.export("interface_id", iface.id)
|
||||
pulumi.export("volume_id", volume.id)
|
||||
pulumi.export("volume2_id", volume2.id)
|
||||
pulumi.export("volume_attach_id", vol_attach.id)
|
||||
@@ -0,0 +1,2 @@
|
||||
pulumi>=3.140,<4
|
||||
pulumi-openstack>=5.0,<6
|
||||
@@ -0,0 +1,294 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Pulumi OpenStack coverage runner.
|
||||
|
||||
For each series (yoga → dalmatian):
|
||||
1. Activate the OpenStack pack
|
||||
2. ``pulumi up`` a pulumi_openstack program (maximises provider coverage)
|
||||
3. Assert every stack export is non-empty
|
||||
4. HTTP-probe remaining pack operations; require non-empty GET/POST bodies
|
||||
5. ``pulumi destroy``
|
||||
6. Emit JUnit + HTML report
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
SERIES_ORDER = ("yoga", "antelope", "caracal", "dalmatian")
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
REPO = Path(os.environ.get("WORKSPACE", "/workspace"))
|
||||
PROGRAM = ROOT / "pulumi" / "programs" / "os_coverage"
|
||||
REPORT_DIR = Path(os.environ.get("REPORT_DIR", "/reports"))
|
||||
|
||||
sys.path.insert(0, str(REPO))
|
||||
sys.path.insert(0, str(ROOT / "pulumi"))
|
||||
|
||||
|
||||
def _smoke() -> bool:
|
||||
return os.environ.get("TEST_SMOKE", "").strip().lower() in {"1", "true", "yes"}
|
||||
|
||||
|
||||
def _series_list() -> list[str]:
|
||||
raw = os.environ.get("OPENSTACK_SERIES_LIST", "").strip()
|
||||
if raw:
|
||||
return [s.strip() for s in raw.split(",") if s.strip()]
|
||||
return list(SERIES_ORDER)
|
||||
|
||||
|
||||
def _host() -> str:
|
||||
return os.environ.get("OS_GATEWAY_URL") or os.environ.get(
|
||||
"OS_AUTH_URL", "http://api-gateway:5000/v3"
|
||||
).removesuffix("/v3").rstrip("/")
|
||||
|
||||
|
||||
def _env_vars(series: str) -> dict[str, str]:
|
||||
backend = os.environ.get("PULUMI_BACKEND_URL", "file:///tmp/pulumi-state")
|
||||
if backend.startswith("file://"):
|
||||
Path(backend.removeprefix("file://").split("?", 1)[0]).mkdir(parents=True, exist_ok=True)
|
||||
return {
|
||||
"OS_AUTH_URL": os.environ.get("OS_AUTH_URL", f"{_host()}/v3"),
|
||||
"OS_USERNAME": os.environ.get("OS_USERNAME", "admin"),
|
||||
"OS_PASSWORD": os.environ.get("OS_PASSWORD", "secret"),
|
||||
"OS_PROJECT_NAME": os.environ.get("OS_PROJECT_NAME", "demo"),
|
||||
"OS_USER_DOMAIN_NAME": os.environ.get("OS_USER_DOMAIN_NAME", "Default"),
|
||||
"OS_PROJECT_DOMAIN_NAME": os.environ.get("OS_PROJECT_DOMAIN_NAME", "Default"),
|
||||
"OS_REGION_NAME": os.environ.get("OS_REGION_NAME", "RegionOne"),
|
||||
"OS_INTERFACE": "public",
|
||||
"OS_IDENTITY_API_VERSION": "3",
|
||||
"OPENSTACK_SERIES": series,
|
||||
"PULUMI_CONFIG_PASSPHRASE": os.environ.get("PULUMI_CONFIG_PASSPHRASE", "lab"),
|
||||
"PULUMI_BACKEND_URL": backend,
|
||||
"PYTHONPATH": f"{REPO}:{ROOT / 'pulumi'}:{os.environ.get('PYTHONPATH', '')}",
|
||||
"WORKSPACE": str(REPO),
|
||||
}
|
||||
|
||||
|
||||
def run_pulumi_stack(series: str) -> dict[str, Any]:
|
||||
from pulumi import automation as auto
|
||||
|
||||
from _lib.validate import activate_series, assert_outputs_nonempty
|
||||
|
||||
host = _host()
|
||||
activate_series(host, series)
|
||||
env_vars = _env_vars(series)
|
||||
stack_name = f"os-coverage-{series}"
|
||||
t0 = time.time()
|
||||
|
||||
stack = auto.create_or_select_stack(
|
||||
stack_name=stack_name,
|
||||
work_dir=str(PROGRAM),
|
||||
opts=auto.LocalWorkspaceOptions(
|
||||
env_vars=env_vars,
|
||||
# Install program requirements into workspace on first run.
|
||||
),
|
||||
)
|
||||
try:
|
||||
stack.workspace.install_plugin("openstack", "v5.3.1")
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
try:
|
||||
# Ensure python deps for the program
|
||||
stack.workspace.run_cmd(["python3", "-m", "pip", "install", "-q", "-r", "requirements.txt"])
|
||||
except Exception: # noqa: BLE001
|
||||
# LocalWorkspace may not expose run_cmd on all versions — pip in image instead.
|
||||
pass
|
||||
|
||||
empty: list[str] = []
|
||||
outputs: dict[str, Any] = {}
|
||||
error: str | None = None
|
||||
try:
|
||||
up = stack.up(on_output=lambda _: None)
|
||||
outputs = {
|
||||
k: (v.value if hasattr(v, "value") else v) for k, v in (up.outputs or {}).items()
|
||||
}
|
||||
empty = assert_outputs_nonempty(outputs, min_count=25)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
error = str(exc)[:4000]
|
||||
finally:
|
||||
try:
|
||||
stack.destroy(on_output=lambda _: None)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
return {
|
||||
"series": series,
|
||||
"elapsed_s": round(time.time() - t0, 2),
|
||||
"error": error,
|
||||
"outputs": {k: outputs[k] for k in sorted(outputs)},
|
||||
"empty_exports": empty,
|
||||
"ok": error is None and not empty,
|
||||
}
|
||||
|
||||
|
||||
def run_http_coverage(series: str, *, collections_only: bool) -> dict[str, Any]:
|
||||
from _lib.http_coverage import probe_pack_operations
|
||||
|
||||
return probe_pack_operations(
|
||||
series,
|
||||
host=_host(),
|
||||
collections_only=collections_only,
|
||||
require_nonempty=True,
|
||||
)
|
||||
|
||||
|
||||
def write_junit(series_reports: list[dict[str, Any]], path: Path) -> None:
|
||||
cases: list[tuple[str, bool, str]] = []
|
||||
for rep in series_reports:
|
||||
series = rep["series"]
|
||||
pu = rep["pulumi"]
|
||||
cases.append(
|
||||
(
|
||||
f"{series}.pulumi_openstack.stack",
|
||||
not pu.get("ok", False),
|
||||
pu.get("error") or (", ".join(pu.get("empty_exports") or []) or "ok"),
|
||||
)
|
||||
)
|
||||
for empty in pu.get("empty_exports") or []:
|
||||
cases.append((f"{series}.pulumi_openstack.nonempty.{empty.split('=')[0]}", True, empty))
|
||||
http = rep.get("http") or {}
|
||||
for item in http.get("results") or []:
|
||||
name = f"{series}.http.{item.get('service')}.{item.get('operation_id')}"
|
||||
failed = (
|
||||
http.get("mode") == "lifecycle"
|
||||
and item.get("mode") == "lifecycle"
|
||||
and not item.get("succeeded")
|
||||
) or (not item.get("ok"))
|
||||
# nonempty failures are separate entries in failures list
|
||||
cases.append(
|
||||
(
|
||||
name,
|
||||
bool(failed),
|
||||
f"{item.get('method')} {item.get('path')} → {item.get('status')}",
|
||||
)
|
||||
)
|
||||
for fail in http.get("failures") or []:
|
||||
detail = str(fail.get("detail") or "")
|
||||
if detail == "empty response body":
|
||||
cases.append(
|
||||
(
|
||||
f"{series}.http.nonempty.{fail.get('service')}.{fail.get('operation_id')}",
|
||||
True,
|
||||
"empty response body",
|
||||
)
|
||||
)
|
||||
elif fail.get("operation_id") == "coverage_incomplete" or detail.startswith(
|
||||
"coverage_incomplete"
|
||||
):
|
||||
cases.append(
|
||||
(
|
||||
f"{series}.http.coverage_incomplete",
|
||||
True,
|
||||
detail,
|
||||
)
|
||||
)
|
||||
|
||||
suite = ET.Element(
|
||||
"testsuite",
|
||||
name="pulumi-openstack-coverage",
|
||||
tests=str(len(cases)),
|
||||
failures=str(sum(1 for _, failed, _ in cases if failed)),
|
||||
)
|
||||
for name, failed, detail in cases:
|
||||
case = ET.SubElement(suite, "testcase", classname="pulumi", name=name)
|
||||
if failed:
|
||||
node = ET.SubElement(case, "failure", message=detail[:300])
|
||||
node.text = detail
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
ET.ElementTree(suite).write(path, encoding="utf-8", xml_declaration=True)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
REPORT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
collections_only = _smoke() or os.environ.get("COLLECTIONS_ONLY", "").lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
}
|
||||
skip_http = os.environ.get("SKIP_HTTP_COVERAGE", "").lower() in {"1", "true", "yes"}
|
||||
series_list = _series_list()
|
||||
print(
|
||||
f"Pulumi coverage: series={','.join(series_list)} "
|
||||
f"collections_only={collections_only} host={_host()}"
|
||||
)
|
||||
|
||||
series_reports: list[dict[str, Any]] = []
|
||||
for series in series_list:
|
||||
print(f"==> series {series}: pulumi_openstack")
|
||||
pu = run_pulumi_stack(series)
|
||||
print(
|
||||
f"{series}: pulumi ok={pu.get('ok')} empty_exports={len(pu.get('empty_exports') or [])} "
|
||||
f"error={'yes' if pu.get('error') else 'no'}"
|
||||
)
|
||||
http: dict[str, Any]
|
||||
if skip_http:
|
||||
http = {
|
||||
"series": series,
|
||||
"total": 0,
|
||||
"expected_ops": 0,
|
||||
"coverage_incomplete": False,
|
||||
"methods": {"GET": 0, "POST": 0, "PUT": 0, "PATCH": 0, "DELETE": 0},
|
||||
"ok_count": 0,
|
||||
"fail_count": 0,
|
||||
"nonempty_fail_count": 0,
|
||||
"results": [],
|
||||
"failures": [],
|
||||
}
|
||||
else:
|
||||
print(f"==> series {series}: HTTP pack probe (nonempty)")
|
||||
http = run_http_coverage(series, collections_only=collections_only)
|
||||
methods = http.get("methods") or {}
|
||||
methods_s = " ".join(
|
||||
f"{m}={methods.get(m, 0)}" for m in ("GET", "POST", "PUT", "PATCH", "DELETE")
|
||||
)
|
||||
print(
|
||||
f"{series}: http ok={http.get('ok_count')} fail={http.get('fail_count')} "
|
||||
f"nonempty_fail={http.get('nonempty_fail_count')} "
|
||||
f"total={http.get('total')}/{http.get('expected_ops')} "
|
||||
f"coverage_incomplete={http.get('coverage_incomplete')} "
|
||||
f"methods[{methods_s}]"
|
||||
)
|
||||
rep = {"series": series, "pulumi": pu, "http": http}
|
||||
series_reports.append(rep)
|
||||
(REPORT_DIR / f"series-{series}.json").write_text(
|
||||
json.dumps(rep, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
write_junit(series_reports, REPORT_DIR / "pulumi-junit.xml")
|
||||
|
||||
from _lib.report_html import write_html
|
||||
|
||||
summary = {
|
||||
"series_count": len(series_reports),
|
||||
"pulumi_ok": sum(1 for r in series_reports if r["pulumi"].get("ok")),
|
||||
"pulumi_fail": sum(1 for r in series_reports if not r["pulumi"].get("ok")),
|
||||
"http_ok": sum(int(r["http"].get("ok_count", 0)) for r in series_reports),
|
||||
"http_fail": sum(
|
||||
int(r["http"].get("fail_count", 0)) + int(r["http"].get("nonempty_fail_count", 0))
|
||||
for r in series_reports
|
||||
),
|
||||
"coverage_incomplete": sum(
|
||||
1 for r in series_reports if r["http"].get("coverage_incomplete")
|
||||
),
|
||||
"collections_only": collections_only,
|
||||
}
|
||||
html_path = write_html(REPORT_DIR, summary, series_reports)
|
||||
(REPORT_DIR / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
||||
print(f"JUnit: {REPORT_DIR / 'pulumi-junit.xml'}")
|
||||
print(f"HTML: {html_path}")
|
||||
print(json.dumps(summary, indent=2))
|
||||
|
||||
failed = (
|
||||
summary["pulumi_fail"] > 0 or summary["http_fail"] > 0 or summary["coverage_incomplete"] > 0
|
||||
)
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user