diff --git a/README.md b/README.md index 4e4a9a5..c8f9b02 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,11 @@ tests verify root access, inherited `Sys.Audit`/`VM.Audit`, operator power management, token privilege intersection, denial, and identical denial for an existing and a nonexistent VM when the principal lacks `VM.Audit`. +QEMU snapshots are durable PostgreSQL records. Create, delete, and rollback use +UPID tasks and the same per-VM lock as other mutations; rollback restores both +the captured VM configuration and runtime state. Snapshot listing, inspection, +and description updates are available through the native Proxmox API paths. + Token lifecycle is available at `/access/users/{userid}/token[/{tokenid}]`. A generated secret is returned only by create or explicit regenerate; only its scrypt hash is stored. List/read never diff --git a/app/handlers/qemu.py b/app/handlers/qemu.py index 3aa316a..b961897 100644 --- a/app/handlers/qemu.py +++ b/app/handlers/qemu.py @@ -236,6 +236,92 @@ def register_qemu_handlers(registry: HandlerRegistry) -> None: async def resume(request: Request, inputs: dict[str, Any]) -> str: return await mutate("resume", request, inputs) + async def snapshot_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: + values = _values(inputs) + resource = await _qemu_resource(request, str(values["node"]), str(values["vmid"])) + rows = await _database(request).pool.fetch( + """SELECT name, parent_name, description, created_at FROM snapshots + WHERE resource_id=$1 ORDER BY created_at, name""", + resource["id"], + ) + return [ + { + "name": row["name"], + "parent": row["parent_name"], + "description": row["description"] or "", + "snaptime": int(row["created_at"].timestamp()), + } + for row in rows + ] + + async def snapshot_get(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: + values = _values(inputs) + row = await _snapshot(request, values) + state = _state(row["state"]) + return { + "name": row["name"], + "parent": row["parent_name"], + "description": row["description"] or "", + "snaptime": int(row["created_at"].timestamp()), + **state, + } + + async def snapshot_config(request: Request, inputs: dict[str, Any]) -> dict[str, Any]: + row = await _snapshot(request, _values(inputs)) + return {"description": row["description"] or "", **_state(row["state"])} + + async def snapshot_update(request: Request, inputs: dict[str, Any]) -> None: + values = _values(inputs) + row = await _snapshot(request, values) + await _database(request).pool.execute( + "UPDATE snapshots SET description=$2 WHERE id=$1", + row["id"], + str(values.get("description", "")), + ) + + async def snapshot_task(operation: str, request: Request, inputs: dict[str, Any]) -> str: + values = _values(inputs) + node, vmid, snapname = ( + str(values["node"]), + str(values["vmid"]), + str(values["snapname"]), + ) + resource = await _qemu_resource(request, node, vmid) + if operation == "snapshot-create": + exists = await _database(request).pool.fetchval( + "SELECT EXISTS(SELECT 1 FROM snapshots WHERE resource_id=$1 AND name=$2)", + resource["id"], + snapname, + ) + if exists: + raise ApiError(409, "snapshot already exists") + else: + await _snapshot(request, values) + return await _create_task( + request, + node=node, + vmid=vmid, + task_type=f"qemu-{operation}", + payload={ + "node": node, + "vmid": vmid, + "resource_id": str(resource["id"]), + "snapname": snapname, + "description": str(values.get("description", "")), + "vmstate": bool(values.get("vmstate", False)), + "start": bool(values.get("start", False)), + }, + ) + + async def snapshot_create(request: Request, inputs: dict[str, Any]) -> str: + return await snapshot_task("snapshot-create", request, inputs) + + async def snapshot_delete(request: Request, inputs: dict[str, Any]) -> str: + return await snapshot_task("snapshot-delete", request, inputs) + + async def snapshot_rollback(request: Request, inputs: dict[str, Any]) -> str: + return await snapshot_task("snapshot-rollback", request, inputs) + async def task_list(request: Request, inputs: dict[str, Any]) -> list[dict[str, Any]]: tasks = await TaskRepository(_database(request).pool).list_for_node( str(_values(inputs)["node"]) @@ -283,6 +369,19 @@ def register_qemu_handlers(registry: HandlerRegistry) -> None: registry.register("/nodes/{node}/qemu/{vmid}/status/reset", "POST", reset) registry.register("/nodes/{node}/qemu/{vmid}/status/suspend", "POST", suspend) registry.register("/nodes/{node}/qemu/{vmid}/status/resume", "POST", resume) + registry.register("/nodes/{node}/qemu/{vmid}/snapshot", "GET", snapshot_list) + registry.register("/nodes/{node}/qemu/{vmid}/snapshot", "POST", snapshot_create) + registry.register("/nodes/{node}/qemu/{vmid}/snapshot/{snapname}", "GET", snapshot_get) + registry.register("/nodes/{node}/qemu/{vmid}/snapshot/{snapname}", "DELETE", snapshot_delete) + registry.register( + "/nodes/{node}/qemu/{vmid}/snapshot/{snapname}/config", "GET", snapshot_config + ) + registry.register( + "/nodes/{node}/qemu/{vmid}/snapshot/{snapname}/config", "PUT", snapshot_update + ) + registry.register( + "/nodes/{node}/qemu/{vmid}/snapshot/{snapname}/rollback", "POST", snapshot_rollback + ) registry.register("/nodes/{node}/tasks", "GET", task_list) registry.register("/nodes/{node}/tasks/{upid}/status", "GET", task_status) registry.register("/nodes/{node}/tasks/{upid}/log", "GET", task_log) @@ -303,6 +402,9 @@ async def _create_task( "qemu-create": "qmcreate", "qemu-delete": "qmdestroy", "qemu-update": "qmconfig", + "qemu-snapshot-create": "qmsnapshot", + "qemu-snapshot-delete": "qmdelsnapshot", + "qemu-snapshot-rollback": "qmrollback", }[task_type] upid = str(Upid(node, pid, pid, timestamp, worker_type, vmid, str(request.state.principal))) try: @@ -316,3 +418,31 @@ async def _create_task( except ConflictError as error: raise ApiError(409, str(error)) from error return task.upid + + +async def _qemu_resource(request: Request, node: str, vmid: str) -> Any: + row = await _database(request).pool.fetchrow( + """SELECT r.id, r.state, v.config FROM resources r + JOIN nodes n ON n.id=r.node_id + JOIN virtual_machines v ON v.resource_id=r.id + WHERE n.name=$1 AND r.kind='qemu' AND r.external_id=$2""", + node, + vmid, + ) + if row is None: + raise ApiError(404, "virtual machine does not exist") + return row + + +async def _snapshot(request: Request, values: dict[str, Any]) -> Any: + row = await _database(request).pool.fetchrow( + """SELECT s.* FROM snapshots s + JOIN resources r ON r.id=s.resource_id JOIN nodes n ON n.id=r.node_id + WHERE n.name=$1 AND r.kind='qemu' AND r.external_id=$2 AND s.name=$3""", + str(values["node"]), + str(values["vmid"]), + str(values["snapname"]), + ) + if row is None: + raise ApiError(404, "snapshot does not exist") + return row diff --git a/app/main.py b/app/main.py index 8619549..6fabca6 100644 --- a/app/main.py +++ b/app/main.py @@ -50,6 +50,9 @@ def create_app( "qemu-reset": handler, "qemu-resume": handler, "qemu-shutdown": handler, + "qemu-snapshot-create": handler, + "qemu-snapshot-delete": handler, + "qemu-snapshot-rollback": handler, "qemu-start": handler, "qemu-stop": handler, "qemu-suspend": handler, diff --git a/app/tasks/qemu.py b/app/tasks/qemu.py index dac90ff..13b1cb6 100644 --- a/app/tasks/qemu.py +++ b/app/tasks/qemu.py @@ -23,6 +23,10 @@ def qemu_handler(repository: TaskRepository, clock: Clock) -> TaskHandler: return await _update(repository, task, resource_id) if operation == "delete": return await _delete(repository, task, resource_id) + if operation.startswith("snapshot-"): + return await _snapshot( + repository, task, resource_id, operation.removeprefix("snapshot-") + ) async with repository.pool.acquire() as connection: row = await connection.fetchrow("SELECT state FROM resources WHERE id=$1", resource_id) if row is None: @@ -129,5 +133,73 @@ async def _delete(repository: TaskRepository, task: Task, resource_id: uuid.UUID return {"deleted": True} +async def _snapshot( + repository: TaskRepository, + task: Task, + resource_id: uuid.UUID, + operation: str, +) -> dict[str, Any]: + name = str(task.payload["snapname"]) + async with repository.pool.acquire() as connection: + async with connection.transaction(): + if operation == "create": + row = await connection.fetchrow( + """SELECT r.state, v.config FROM resources r + JOIN virtual_machines v ON v.resource_id=r.id WHERE r.id=$1""", + resource_id, + ) + if row is None: + raise ValueError("resource disappeared") + captured = { + "resource_state": _object(row["state"]), + "config": _object(row["config"]), + "vmstate": bool(task.payload.get("vmstate", False)), + } + await connection.execute( + """INSERT INTO snapshots(id, resource_id, name, description, state) + VALUES($1, $2, $3, $4, $5::jsonb)""", + uuid.uuid4(), + resource_id, + name, + str(task.payload.get("description", "")), + json.dumps(captured, sort_keys=True), + ) + elif operation == "delete": + status = await connection.execute( + "DELETE FROM snapshots WHERE resource_id=$1 AND name=$2", + resource_id, + name, + ) + if status != "DELETE 1": + raise ValueError("snapshot disappeared") + elif operation == "rollback": + row = await connection.fetchrow( + "SELECT state FROM snapshots WHERE resource_id=$1 AND name=$2", + resource_id, + name, + ) + if row is None: + raise ValueError("snapshot disappeared") + captured = _object(row["state"]) + state = dict(cast(Mapping[str, Any], captured["resource_state"])) + if bool(task.payload.get("start", False)): + state["status"] = "running" + await connection.execute( + """UPDATE resources SET state=$2::jsonb, version=version+1, + updated_at=now() WHERE id=$1""", + resource_id, + json.dumps(state, sort_keys=True), + ) + await connection.execute( + "UPDATE virtual_machines SET config=$2::jsonb WHERE resource_id=$1", + resource_id, + json.dumps(captured["config"], sort_keys=True), + ) + else: + raise ValueError(f"unsupported snapshot operation: {operation}") + await repository.append_log(task.id, f"snapshot {name} {operation} completed") + return {"snapshot": name, "operation": operation} + + def _object(value: object) -> dict[str, Any]: return json.loads(value) if isinstance(value, str) else dict(cast(Mapping[str, Any], value)) diff --git a/docs/compatibility-0.1.0.md b/docs/compatibility-0.1.0.md index a93124f..78179e3 100644 --- a/docs/compatibility-0.1.0.md +++ b/docs/compatibility-0.1.0.md @@ -9,8 +9,8 @@ Proxmox compatibility. | Level | Methods | Contract share | Evidence | |---|---:|---:|---| | Declared and dynamically routed | 675 | 100% | Imported immutable API Viewer artifact | -| Stateful semantics implemented on current main | 27 | 4.00% | Handler registry and unit/integration tests | -| Schema-only or explicitly unsupported | 648 | 96.00% | Default 501 fallback | +| Stateful semantics implemented on current main | 34 | 5.04% | Handler registry and unit/integration tests | +| Schema-only or explicitly unsupported | 641 | 94.96% | Default 501 fallback | | proxmoxer smoke exercised | 9 | 1.33% | Unmodified proxmoxer 2.3 compatibility test | The smoke set is `POST /access/ticket`, `GET /version`, `GET /nodes`, diff --git a/docs/original-prompt-gap-plan.md b/docs/original-prompt-gap-plan.md index 262bac8..5e5bbd9 100644 --- a/docs/original-prompt-gap-plan.md +++ b/docs/original-prompt-gap-plan.md @@ -57,8 +57,9 @@ plaintext password, ticket, CSRF token, or token secret reaches storage/logs. - [x] Create, synchronous/asynchronous update, and delete. - [x] Shutdown, reboot, reset, suspend and resume. -- [ ] Snapshots and rollback, clone, local/remote migration, resize and move - disk, selected agent endpoints, pending/status data. +- [x] Snapshots and rollback. +- [ ] Clone, local/remote migration, resize and move disk, selected agent + endpoints, pending/status data. - [ ] Persist normalized CPU/memory/common fields plus unknown PVE parameters in JSONB; simulate usage, uptime, PID, IO/network, lock, template, QMP, HA and guest-agent availability. diff --git a/evidence/pve-9.2.3-0.1.0.json b/evidence/pve-9.2.3-0.1.0.json index 978c472..96823ef 100644 --- a/evidence/pve-9.2.3-0.1.0.json +++ b/evidence/pve-9.2.3-0.1.0.json @@ -104,6 +104,48 @@ "verb": "POST", "dimensions": ["input_parameters", "parameter_requiredness", "types_constraints", "http_status", "json_structure", "state_semantics", "long_task_behavior", "errors_prohibitions", "permissions"], "sources": ["tests/compatibility/test_proxmoxer.py", "tests/unit/test_transitions.py", "tests/unit/test_qemu_task.py"] + }, + { + "path": "/nodes/{node}/qemu/{vmid}/snapshot", + "verb": "GET", + "dimensions": ["input_parameters", "parameter_requiredness", "types_constraints", "http_status", "json_structure", "state_semantics", "errors_prohibitions", "permissions"], + "sources": ["tests/compatibility/test_proxmoxer.py", "tests/unit/test_qemu_task.py"] + }, + { + "path": "/nodes/{node}/qemu/{vmid}/snapshot", + "verb": "POST", + "dimensions": ["input_parameters", "parameter_requiredness", "types_constraints", "http_status", "json_structure", "state_semantics", "long_task_behavior", "errors_prohibitions", "permissions"], + "sources": ["tests/compatibility/test_proxmoxer.py", "tests/unit/test_qemu_task.py"] + }, + { + "path": "/nodes/{node}/qemu/{vmid}/snapshot/{snapname}", + "verb": "GET", + "dimensions": ["input_parameters", "parameter_requiredness", "types_constraints", "http_status", "json_structure", "state_semantics", "errors_prohibitions", "permissions"], + "sources": ["tests/compatibility/test_proxmoxer.py"] + }, + { + "path": "/nodes/{node}/qemu/{vmid}/snapshot/{snapname}", + "verb": "DELETE", + "dimensions": ["input_parameters", "parameter_requiredness", "types_constraints", "http_status", "json_structure", "state_semantics", "long_task_behavior", "errors_prohibitions", "permissions"], + "sources": ["tests/compatibility/test_proxmoxer.py", "tests/unit/test_qemu_task.py"] + }, + { + "path": "/nodes/{node}/qemu/{vmid}/snapshot/{snapname}/config", + "verb": "GET", + "dimensions": ["input_parameters", "parameter_requiredness", "types_constraints", "http_status", "json_structure", "state_semantics", "errors_prohibitions", "permissions"], + "sources": ["tests/compatibility/test_proxmoxer.py"] + }, + { + "path": "/nodes/{node}/qemu/{vmid}/snapshot/{snapname}/config", + "verb": "PUT", + "dimensions": ["input_parameters", "parameter_requiredness", "types_constraints", "http_status", "json_structure", "state_semantics", "errors_prohibitions", "permissions"], + "sources": ["tests/compatibility/test_proxmoxer.py"] + }, + { + "path": "/nodes/{node}/qemu/{vmid}/snapshot/{snapname}/rollback", + "verb": "POST", + "dimensions": ["input_parameters", "parameter_requiredness", "types_constraints", "http_status", "json_structure", "state_semantics", "long_task_behavior", "errors_prohibitions", "permissions"], + "sources": ["tests/compatibility/test_proxmoxer.py", "tests/unit/test_qemu_task.py"] } ] } diff --git a/tests/compatibility/test_proxmoxer.py b/tests/compatibility/test_proxmoxer.py index 72f7779..38f0fbc 100644 --- a/tests/compatibility/test_proxmoxer.py +++ b/tests/compatibility/test_proxmoxer.py @@ -110,6 +110,22 @@ def test_proxmoxer_read_and_qemu_task_flow() -> None: assert wait_task(proxmox, update_upid)["exitstatus"] == "OK" assert proxmox.nodes("pve1").qemu("150").config.get()["name"] == "async-update" + snapshots = proxmox.nodes("pve1").qemu("150").snapshot + snapshot_upid = snapshots.post(snapname="baseline", description="before change") + assert wait_task(proxmox, snapshot_upid)["exitstatus"] == "OK" + assert any(item["name"] == "baseline" for item in snapshots.get()) + baseline = snapshots("baseline") + assert baseline.get()["description"] == "before change" + assert baseline.config.put(description="stable baseline") is None + assert baseline.config.get()["description"] == "stable baseline" + assert proxmox.nodes("pve1").qemu("150").config.put(name="after-snapshot") is None + rollback_upid = baseline.rollback.post() + assert wait_task(proxmox, rollback_upid)["exitstatus"] == "OK" + assert proxmox.nodes("pve1").qemu("150").config.get()["name"] == "async-update" + snapshot_delete_upid = baseline.delete() + assert wait_task(proxmox, snapshot_delete_upid)["exitstatus"] == "OK" + assert not snapshots.get() + delete_upid = proxmox.nodes("pve1").qemu("150").delete() assert wait_task(proxmox, delete_upid)["exitstatus"] == "OK" with pytest.raises(ResourceException) as deleted_vm: diff --git a/tests/unit/test_qemu_handlers.py b/tests/unit/test_qemu_handlers.py index a288009..9410db5 100644 --- a/tests/unit/test_qemu_handlers.py +++ b/tests/unit/test_qemu_handlers.py @@ -1,6 +1,7 @@ """Persistent QEMU CRUD semantic handler tests.""" import uuid +from datetime import UTC, datetime from typing import Any, cast import pytest @@ -32,8 +33,24 @@ class QemuPool: return True if "FROM resources" in sql: return self.resource_exists + if "FROM snapshots" in sql: + return False raise AssertionError(sql) + async def fetch(self, sql: str, *args: object) -> list[dict[str, object]]: + del args + if "FROM resources" in sql: + return [{"vmid": 150, "state": '{"status":"stopped","name":"vm"}'}] + assert "FROM snapshots" in sql + return [ + { + "name": "baseline", + "parent_name": None, + "description": "stable", + "created_at": datetime(2026, 1, 1, tzinfo=UTC), + } + ] + async def fetchrow(self, sql: str, *args: object) -> dict[str, object] | None: del args if self.missing: @@ -45,9 +62,22 @@ class QemuPool: "state": '{"name":"old","status":"stopped"}', "config": '{"name":"old"}', } + if "SELECT r.state, v.config" in sql: + return {"state": '{"status":"stopped"}', "config": '{"name":"vm"}'} if "SELECT r.id, r.state" in sql: status = "running" if self.running else "stopped" return {"id": self.resource_id, "state": f'{{"status":"{status}"}}'} + if "SELECT r.id, r.state, v.config" in sql: + return {"id": self.resource_id, "state": '{"status":"stopped"}', "config": "{}"} + if "SELECT s.* FROM snapshots" in sql: + return { + "id": uuid.uuid4(), + "name": "baseline", + "parent_name": None, + "description": "stable", + "state": '{"config":{"name":"old"}}', + "created_at": datetime(2026, 1, 1, tzinfo=UTC), + } raise AssertionError(sql) async def execute(self, sql: str, *args: object) -> str: @@ -113,10 +143,17 @@ async def test_qemu_create_sync_async_update_and_delete( pool = QemuPool() http_request = request(pool) create = registry.get("/nodes/{node}/qemu", "POST") + listing = registry.get("/nodes/{node}/qemu", "GET") + config = registry.get("/nodes/{node}/qemu/{vmid}/config", "GET") + current = registry.get("/nodes/{node}/qemu/{vmid}/status/current", "GET") update_sync = registry.get("/nodes/{node}/qemu/{vmid}/config", "PUT") update_async = registry.get("/nodes/{node}/qemu/{vmid}/config", "POST") delete = registry.get("/nodes/{node}/qemu/{vmid}", "DELETE") - assert create and update_sync and update_async and delete + assert create and listing and config and current and update_sync and update_async and delete + + assert (await listing(http_request, inputs(node="pve1")))[0]["name"] == "vm" + assert (await config(http_request, inputs(node="pve1", vmid=150)))["name"] == "vm" + assert (await current(http_request, inputs(node="pve1", vmid=150)))["status"] == "stopped" create_upid = await create( http_request, @@ -185,3 +222,43 @@ async def test_qemu_crud_conflicts_and_missing_resources( with pytest.raises(ApiError) as running: await delete(http_request, inputs(node="pve1", vmid=150)) assert running.value.status_code == 409 + + +async def test_qemu_snapshot_handlers(monkeypatch: pytest.MonkeyPatch) -> None: + tasks: list[str] = [] + + class FakeTaskRepository: + def __init__(self, pool: object) -> None: + del pool + + async def create(self, **kwargs: Any) -> Task: + tasks.append(str(kwargs["task_type"])) + return Task(uuid.uuid4(), str(kwargs["upid"]), tasks[-1], "queued", {}, 0, False, 0) + + monkeypatch.setattr("app.handlers.qemu.TaskRepository", FakeTaskRepository) + registry = HandlerRegistry() + register_qemu_handlers(registry) + pool = QemuPool() + http_request = request(pool) + base = "/nodes/{node}/qemu/{vmid}/snapshot" + + listing = registry.get(base, "GET") + create = registry.get(base, "POST") + get = registry.get(f"{base}/{{snapname}}", "GET") + delete = registry.get(f"{base}/{{snapname}}", "DELETE") + config_get = registry.get(f"{base}/{{snapname}}/config", "GET") + config_put = registry.get(f"{base}/{{snapname}}/config", "PUT") + rollback = registry.get(f"{base}/{{snapname}}/rollback", "POST") + assert listing and create and get and delete and config_get and config_put and rollback + + common = inputs(node="pve1", vmid=150, snapname="baseline") + assert (await listing(http_request, inputs(node="pve1", vmid=150)))[0]["name"] == "baseline" + assert (await get(http_request, common))["description"] == "stable" + assert (await config_get(http_request, common))["config"] == {"name": "old"} + assert await config_put(http_request, inputs(**common["values"], description="updated")) is None + assert ( + await create(http_request, inputs(**common["values"], description="stable")) + ).startswith("UPID:pve1:") + assert (await rollback(http_request, common)).startswith("UPID:pve1:") + assert (await delete(http_request, common)).startswith("UPID:pve1:") + assert tasks == ["qemu-snapshot-create", "qemu-snapshot-rollback", "qemu-snapshot-delete"] diff --git a/tests/unit/test_qemu_task.py b/tests/unit/test_qemu_task.py index 202f452..8de6527 100644 --- a/tests/unit/test_qemu_task.py +++ b/tests/unit/test_qemu_task.py @@ -82,6 +82,12 @@ class CrudConnection: return {"id": uuid.uuid4(), "cluster_id": uuid.uuid4()} if "JOIN virtual_machines" in sql: return {"state": '{"status":"stopped","name":"old"}', "config": '{"name":"old"}'} + if "FROM snapshots" in sql: + return { + "state": ( + '{"resource_state":{"status":"stopped","name":"old"},"config":{"name":"old"}}' + ) + } return None async def execute(self, sql: str, *args: object) -> str: @@ -175,3 +181,39 @@ async def test_qemu_worker_create_update_and_delete_are_persistent() -> None: assert any("INSERT INTO resources" in command for command in repository.connection.commands) assert any("UPDATE virtual_machines" in command for command in repository.connection.commands) assert any("DELETE FROM resources" in command for command in repository.connection.commands) + + +async def test_qemu_worker_snapshot_create_rollback_and_delete_are_persistent() -> None: + repository = CrudRepository() + handler = qemu_handler(cast(TaskRepository, repository), cast(Clock, ImmediateClock())) + resource_id = uuid.uuid4() + + async def run(operation: str, **payload: object) -> dict[str, object]: + result = await handler( + Task( + uuid.uuid4(), + f"UPID:{operation}", + f"qemu-snapshot-{operation}", + "running", + {"resource_id": str(resource_id), "snapname": "baseline", **payload}, + 0, + False, + 1, + ) + ) + assert result is not None + return cast(dict[str, object], result) + + assert await run("create", description="stable") == { + "snapshot": "baseline", + "operation": "create", + } + assert await run("rollback", start=True) == { + "snapshot": "baseline", + "operation": "rollback", + } + assert await run("delete") == {"snapshot": "baseline", "operation": "delete"} + commands = repository.connection.commands + assert any("INSERT INTO snapshots" in command for command in commands) + assert any("UPDATE virtual_machines" in command for command in commands) + assert any("DELETE FROM snapshots" in command for command in commands)