DevOpsTools/openstack-api-simulator
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial commit: stateful OpenStack API laboratory simulator.

Browse Source 
Ship Keystone auth, multi-service handlers (Yoga→Dalmatian), Compose/Helm
packaging, API contract packs, and pytest/Pulumi coverage labs.
This commit is contained in:
Sergey Antropoff
2026-07-18 04:26:48 +03:00
commit 6033967e6a
509 changed files with 464404 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/os_api_inventory/__init__.py
View File
tools/os_api_inventory/catalog.py
+1067
View File
File diff suppressed because it is too large Load Diff
tools/os_api_inventory/coverage_report.py
+83
View File
@@ -0,0 +1,83 @@
#!/usr/bin/env python3
"""Write docs/api_coverage.md from OpenStack contract packs."""
from __future__ import annotations
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
def main() -> int:
series = sys.argv[1] if len(sys.argv) > 1 else "dalmatian"
pack_root = ROOT / "contracts" / "openstack"
man_path = pack_root / series / "manifest.json"
if not man_path.is_file():
print(f"missing {man_path}", file=sys.stderr)
return 1
man = json.loads(man_path.read_text())
series_rows: list[str] = []
for path in sorted(pack_root.glob("*/manifest.json")):
other = json.loads(path.read_text())
series_rows.append(
f"| {str(other['series']).title()} | {other['major']} | {other['operation_count']} |"
)
lines = [
f"# OpenStack API coverage — {man['series']}",
"",
f"Generated from `contracts/openstack/{series}/manifest.json`.",
"",
f"- **Services:** {man['service_count']}",
f"- **Operations:** {man['operation_count']}",
f"- **Checksum:** `{man['checksum']}`",
f"- **Generated at:** {man.get('generated_at', '')}",
"",
"## Series deltas",
"",
"| Series | Major | Operations |",
"|---|---:|---:|",
*series_rows,
"",
"Older series omit paths introduced later (`tools/os_api_inventory/series_deltas.py`)",
"and use lower microversion ceilings. Apply a pack in the Environment drawer to hot-swap.",
"",
"Surface-complete means every operation in the pack is mounted by the schema engine",
"(specialized routers still win on overlapping stateful paths).",
"",
"| Service | Type | Port | Operations | Microversions |",
"|---|---|---:|---:|---|",
]
for svc in sorted(man["services"], key=lambda s: s["name"]):
mv = ""
if svc.get("default_microversion"):
mv = f"{svc['default_microversion']}–{svc.get('max_microversion') or '?'}"
lines.append(
f"| {svc['name']} | {svc['type']} | {svc['port']} | {svc['operation_count']} | {mv or '—'} |"
)
lines.extend(
[
"",
"## Core minimums",
"",
"| Service | Required | Actual |",
"|---|---:|---:|",
]
)
by_name = {s["name"]: s for s in man["services"]}
for svc, required in (man.get("min_core_operations") or {}).items():
actual = by_name.get(svc, {}).get("operation_count", 0)
status = "OK" if actual >= required else "GAP"
lines.append(f"| {svc} | {required} | {actual} ({status}) |")
lines.append("")
out = ROOT / "docs" / "api_coverage.md"
out.write_text("\n".join(lines))
print(f"wrote {out} ({man['operation_count']} ops)")
return 0
if __name__ == "__main__":
raise SystemExit(main())
tools/os_api_inventory/generate_packs.py
+151
View File
@@ -0,0 +1,151 @@
#!/usr/bin/env python3
"""Generate contracts/openstack/<series> API packs from the inventory catalog."""
from __future__ import annotations
import argparse
import hashlib
import json
import sys
from datetime import datetime, timezone
from pathlib import Path
# Allow running as script from repo root or tools dir.
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "tools"))
from os_api_inventory.catalog import SERIES, SERVICES_META, build_all_operations # noqa: E402
from os_api_inventory.series_deltas import ( # noqa: E402
filter_ops_for_series,
microversions_for,
)
def _dedupe(ops: list[dict]) -> list[dict]:
seen: set[tuple[str, str]] = set()
out: list[dict] = []
for op in ops:
key = (op["method"], op["path"])
if key in seen and op.get("kind") == "action" and op.get("action_name") not in {None, "*"}:
continue
if key in seen and op.get("kind") != "action":
continue
if key in seen:
continue
seen.add(key)
out.append(op)
return out
def _write_service(
series_dir: Path,
name: str,
typ: str,
port: int,
version_path: str,
default_mv: str | None,
max_mv: str | None,
ops: list[dict],
) -> dict:
ops = _dedupe(ops)
for op in ops:
op.setdefault("requires_auth", True)
op.setdefault("requires_project", True)
op.setdefault("service", name)
if default_mv:
op.setdefault("microversion_min", "2.1" if name == "nova" else default_mv)
op.setdefault("microversion_max", max_mv)
payload = {
"service": name,
"type": typ,
"port": port,
"version_path": version_path,
"default_microversion": default_mv,
"max_microversion": max_mv,
"operations": ops,
}
svc_dir = series_dir / name
svc_dir.mkdir(parents=True, exist_ok=True)
api_path = svc_dir / "api.json"
raw = json.dumps(payload, indent=2, sort_keys=True) + "\n"
api_path.write_text(raw)
checksum = hashlib.sha256(raw.encode()).hexdigest()
return {
"name": name,
"type": typ,
"port": port,
"version_path": version_path,
"default_microversion": default_mv,
"max_microversion": max_mv,
"operation_count": len(ops),
"checksum": checksum,
}
def generate(series: str, major: int, out_root: Path) -> Path:
series_dir = out_root / series
series_dir.mkdir(parents=True, exist_ok=True)
all_ops = build_all_operations()
services_info: list[dict] = []
total = 0
for name, typ, port, version_path, default_mv, max_mv in SERVICES_META:
ops = filter_ops_for_series(all_ops.get(name, []), series)
mv_min, mv_max = microversions_for(series, name, default_mv, max_mv)
info = _write_service(series_dir, name, typ, port, version_path, mv_min, mv_max, ops)
services_info.append(info)
total += info["operation_count"]
# Soften min gates for older trimmed series while keeping core identity/compute/network.
min_core = {"keystone": 40, "nova": 70, "neutron": 70}
if series == "yoga":
min_core = {"keystone": 40, "nova": 70, "neutron": 60}
manifest = {
"series": series,
"major": major,
"generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"service_count": len(services_info),
"operation_count": total,
"services": services_info,
"min_core_operations": min_core,
}
root = Path(__file__).resolve().parents[0]
# Annotate series differentiation for operators.
joined = "|".join(s["checksum"] for s in services_info)
manifest["checksum"] = hashlib.sha256(joined.encode()).hexdigest()
(series_dir / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n")
by_name = {s["name"]: s for s in services_info}
for svc, minimum in manifest["min_core_operations"].items():
if by_name[svc]["operation_count"] < minimum:
raise SystemExit(
f"{series}/{svc}: {by_name[svc]['operation_count']} ops < required {minimum}"
)
_ = root
return series_dir
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--out",
type=Path,
default=ROOT / "contracts" / "openstack",
help="Output root for series packs",
)
parser.add_argument("--series", action="append", help="Limit to series (repeatable)")
args = parser.parse_args()
selected = {s.lower() for s in args.series} if args.series else None
for series, major in SERIES:
if selected and series not in selected:
continue
path = generate(series, major, args.out)
man = json.loads((path / "manifest.json").read_text())
print(
f"{series}: {man['operation_count']} operations across {man['service_count']} services"
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
tools/os_api_inventory/series_deltas.py
+162
View File
@@ -0,0 +1,162 @@
"""Per-series OpenStack surface deltas (Yoga → Dalmatian).
Dalmatian keeps the full inventory. Older series drop paths introduced later
and use lower microversion ceilings.
"""
from __future__ import annotations
from typing import Any
SERIES_ORDER = ("yoga", "antelope", "caracal", "dalmatian")
# Approximate public API microversion ceilings per coordinated release.
SERIES_MICROVERSIONS: dict[str, dict[str, tuple[str, str]]] = {
"yoga": {
"nova": ("2.1", "2.90"),
"cinder": ("3.0", "3.68"),
"placement": ("1.0", "1.36"),
"ironic": ("1.1", "1.82"),
"manila": ("2.0", "2.70"),
},
"antelope": {
"nova": ("2.1", "2.93"),
"cinder": ("3.0", "3.69"),
"placement": ("1.0", "1.37"),
"ironic": ("1.1", "1.84"),
"manila": ("2.0", "2.74"),
},
"caracal": {
"nova": ("2.1", "2.95"),
"cinder": ("3.0", "3.70"),
"placement": ("1.0", "1.38"),
"ironic": ("1.1", "1.88"),
"manila": ("2.0", "2.79"),
},
"dalmatian": {
"nova": ("2.1", "2.96"),
"cinder": ("3.0", "3.70"),
"placement": ("1.0", "1.39"),
"ironic": ("1.1", "1.90"),
"manila": ("2.0", "2.82"),
},
}
# Path prefixes first available in a given series (inclusive).
# Anything not matched is available from Yoga.
PATH_INTRODUCED: list[tuple[str, str]] = [
# Antelope
("/v2.1/servers/{server_id}/diagnostics", "antelope"),
("/v2.1/servers/{server_id}/remote-consoles", "antelope"),
("/v2.1/os-simple-tenant-usage", "antelope"),
("/v2.1/flavors/{id}/os-extra_specs", "antelope"),
("/v2.1/servers/{server_id}/os-instance-actions/{request_id}", "antelope"),
("/v2.0/local_ips", "antelope"),
("/v2.0/ndp_proxies", "antelope"),
("/v2.0/log/", "antelope"),
("/v2/lbaas/flavorprofiles", "antelope"),
("/v2/octavia/amphorae", "antelope"),
("/v2/share-groups", "antelope"),
("/v2/shares/{id}/action", "antelope"),
# Caracal
("/v2.1/os-hosts", "caracal"),
("/v2.1/os-assisted-volume-snapshots", "caracal"),
("/v2.1/os-server-external-events", "caracal"),
("/v2.1/os-instance_usage_audit_log", "caracal"),
("/v2.0/routers/{router_id}/conntrack_helpers", "caracal"),
("/v2.0/bgpvpn/", "caracal"),
("/v2.0/vpn/", "caracal"),
("/v2/lbaas/providers", "caracal"),
("/v2/lbaas/l7policies", "caracal"),
("/v2/zones/{zone_id}/recordsets", "caracal"), # keep zones themselves in yoga
("/v2/tlds", "caracal"),
("/v2/blacklists", "caracal"),
("/vnfpkgm/", "caracal"),
("/vnflcm/", "caracal"),
# Dalmatian
("/v2.0/network-ip-availabilities", "dalmatian"),
("/v2.0/auto-allocated-topology", "dalmatian"),
("/v2.0/qos/rule-types", "dalmatian"),
("/v2.0/fwaas/", "dalmatian"),
("/v2.0/address-groups", "dalmatian"),
("/v2.0/bgp-speakers", "dalmatian"),
("/v2.0/bgp-peers", "dalmatian"),
("/v2.0/segments", "dalmatian"),
("/v2.0/network_segment_ranges", "dalmatian"),
("/v2.0/default-security-group-rules", "dalmatian"),
("/v2.0/vpn/ikepolicies", "dalmatian"),
("/v2.0/vpn/ipsecpolicies", "dalmatian"),
("/v2.0/vpn/endpoint-groups", "dalmatian"),
("/v2.1/extensions", "dalmatian"),
("/v2.1/os-agents", "dalmatian"),
("/v2.1/servers/{server_id}/migrations", "dalmatian"),
("/v2.1/servers/{server_id}/consoles", "dalmatian"),
("/v2.1/servers/{server_id}/topology", "dalmatian"),
("/v2.1/os-console-auth-tokens", "dalmatian"),
("/v2/info/import", "dalmatian"),
("/v2/info/stores", "dalmatian"),
("/v1/capsules", "dalmatian"),
("/v2/share-replicas", "dalmatian"),
("/v1/audit_templates", "dalmatian"),
("/v1/audits", "dalmatian"),
("/v1/action_plans", "dalmatian"),
("/v1/scoring_engines", "dalmatian"),
("/v2/queues", "dalmatian"),
("/v2/health", "dalmatian"),
("/v2/ping", "dalmatian"),
]
def series_index(series: str) -> int:
try:
return SERIES_ORDER.index(series)
except ValueError as exc:
raise ValueError(f"unknown series: {series}") from exc
def _path_introduced(path: str) -> str:
best = "yoga"
best_idx = 0
for prefix, series in PATH_INTRODUCED:
if path == prefix or path.startswith(prefix):
idx = series_index(series)
if idx >= best_idx:
best = series
best_idx = idx
return best
def apply_introduced_tags(ops: list[dict[str, Any]]) -> list[dict[str, Any]]:
out: list[dict[str, Any]] = []
for op in ops:
item = dict(op)
if "introduced_in" not in item:
item["introduced_in"] = _path_introduced(str(item.get("path") or ""))
out.append(item)
return out
def filter_ops_for_series(ops: list[dict[str, Any]], series: str) -> list[dict[str, Any]]:
target = series_index(series)
kept: list[dict[str, Any]] = []
for op in apply_introduced_tags(ops):
since = str(op.get("introduced_in") or "yoga")
if series_index(since) <= target:
# Drop series-private metadata from emitted contracts (keep path surface clean).
emitted = {k: v for k, v in op.items() if k != "introduced_in"}
# Still keep introduced_in for UI / debugging — useful for operators.
emitted["introduced_in"] = since
kept.append(emitted)
return kept
def microversions_for(
series: str,
service: str,
default_min: str | None,
default_max: str | None,
) -> tuple[str | None, str | None]:
table = SERIES_MICROVERSIONS.get(series) or {}
if service in table:
return table[service]
return default_min, default_max
tools/scan_empty_collections.py
+101
View File
@@ -0,0 +1,101 @@
#!/usr/bin/env python3
"""Scan GET collection endpoints for empty list payloads across all series."""
from __future__ import annotations
import argparse
import json
import sys
from collections import defaultdict
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from app.openstack.contract_loader import list_series, load_series_pack # noqa: E402
from app.openstack.surface_probe import ( # noqa: E402
activate_series,
fill_path,
http_request,
issue_token,
)
def _is_empty_list_payload(body: object) -> tuple[bool, str | None]:
if not isinstance(body, dict):
return False, None
if body.get("data") == []:
return True, "data"
for key, value in body.items():
if key in {"links", "metadata", "versions", "version", "id", "status"}:
continue
if isinstance(value, list) and len(value) == 0:
return True, key
return False, None
def _is_top_level_collection(op) -> bool: # noqa: ANN001
if op.method != "GET":
return False
if op.kind in {"collection", "detail"}:
return "{" not in op.path or op.path.rstrip("/").endswith("/detail")
if op.kind == "custom" and "{" not in op.path:
return True
return False
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--host", default="http://api-gateway:5000")
parser.add_argument("--series", action="append", default=[])
args = parser.parse_args()
host = args.host.rstrip("/")
if args.series:
series_list = args.series
else:
series_list = [str(item["series"]) for item in list_series()]
token, _ = issue_token(host, user="admin", project="admin")
empties: list[tuple[str, str, str, str, str | None, int, str | None]] = []
checked = 0
for series in series_list:
activate_series(host, series)
packs = load_series_pack(series)
for name, pack in sorted(packs.items()):
for op in pack.operations:
if not _is_top_level_collection(op):
continue
path = fill_path(op.path)
status, body = http_request("GET", f"{host}{path}", token=token, service=name)
checked += 1
empty, key = _is_empty_list_payload(body)
if empty:
empties.append(
(
series,
name,
op.path,
op.resource_type,
op.collection_key,
status,
key,
)
)
by_path: dict[tuple[str, str, str, str | None], list[str]] = defaultdict(list)
for series, svc, path, rtype, ckey, _status, _key in empties:
by_path[(svc, rtype, path, ckey)].append(series)
print(json.dumps({"checked": checked, "empty": len(empties), "unique": len(by_path)}, indent=2))
print("\n=== EMPTY COLLECTIONS ===")
for (svc, rtype, path, ckey), serieses in sorted(by_path.items()):
print(
f"{svc:12} {rtype:28} key={str(ckey):24} {path} "
f"series={','.join(sorted(set(serieses)))}"
)
return 0 if not by_path else 1
if __name__ == "__main__":
raise SystemExit(main())