feat: Завершена реализация универсальной лаборатории

- Добавлена полная поддержка Istio service mesh с Kiali - Интегрированы Helm charts (nginx, prometheus-stack) - Созданы Grafana дашборды для Istio мониторинга - Добавлен HTML генератор отчетов с красивым дизайном - Созданы скрипты для снапшотов и восстановления - Добавлена поддержка Istio Bookinfo demo - Обновлена документация с полным описанием возможностей Компоненты: - Istio с Telemetry и Traffic Policy - Prometheus + Grafana с автопровижинингом дашбордов - HTML отчеты с анализом статусов - Снапшоты и восстановление состояния - Полная интеграция с Kubernetes Автор: Сергей Антропов Сайт: https://devops.org.ru
2025-10-22 13:08:55 +03:00
parent b4881da7c5
commit 33ada54c12
13 changed files with 712 additions and 2 deletions
--- a/25
+++ b/25
@@ -244,3 +244,28 @@ endif
 # Stop all port-forwards
 kube-pf-stop: ## убить все port-forward в контроллере
 	docker exec -it ansible-controller bash -lc 'pkill -f "kubectl .* port-forward" || true'
+
+# ====== ОТЧЕТЫ ======
+lab-report: ## Сгенерировать HTML отчет
+	docker exec ansible-controller bash -lc 'python3 /ansible/scripts/report_html.py /ansible/reports/lab-health.json /ansible/reports/lab-report.html'
+	@echo "HTML report: reports/lab-report.html"
+
+# ====== ДОПОЛНИТЕЛЬНЫЕ ХЕЛПЕРЫ ======
+bookinfo-url: ## echo productpage URL via Istio Gateway (needs istio-gw-port-forward first)
+	@echo "Open: http://localhost:8082/productpage"
+
+grafana-open: ## echo URL to Grafana + hint dashboards
+	@echo "Grafana: http://localhost:3000  (admin/admin)"
+	@echo "Dashboards:"
+	@echo "  - Istio • Overview (uid: istio-overview)"
+	@echo "  - Service • SLI   (uid: service-sli)"
+
+# ====== СНАПШОТЫ И ОЧИСТКА ======
+lab-snapshot: ## Сохранить снапшот лаборатории
+	bash scripts/snapshot.sh
+
+lab-restore: ## Восстановить из снапшота
+	bash scripts/restore.sh
+
+lab-cleanup: ## Очистить лабораторию
+	bash scripts/cleanup.sh
--- a/README-UNIVERSAL-LAB.md
+++ b/README-UNIVERSAL-LAB.md
@@ -51,6 +51,10 @@ echo "test" > vault-password.txt

 # Создать каталог для ролей
 mkdir -p roles
+
+# Скопировать переменные окружения
+cp env.example .env
+# Отредактировать .env под ваши нужды
 ```

 ### 2. Запуск лаборатории
@@ -68,10 +72,28 @@ make lab-converge
 # Проверить работу
 make lab-verify

+# Сгенерировать HTML отчет
+make lab-report
+
 # Уничтожить лабораторию
 make lab-destroy
 ```

+### 3. Управление лабораторией
+
+```bash
+# Полный цикл тестирования
+make lab-test
+
+# Снапшоты и восстановление
+make lab-snapshot    # Сохранить состояние
+make lab-restore     # Восстановить из снапшота
+make lab-cleanup     # Очистить все
+
+# Сброс лаборатории
+make lab-reset
+```
+
 ### 3. Работа с Kubernetes

 ```bash
--- a/env.example
+++ b/env.example
@@ -0,0 +1,16 @@
+# Переменные окружения для универсальной лаборатории
+# Автор: Сергей Антропов
+# Сайт: https://devops.org.ru
+
+# Путь к каталогу с Ansible ролями (вне этого репозитория)
+ROLES_DIR=/path/to/your/ansible/roles
+
+# Telegram уведомления (опционально)
+TG_TOKEN=your_telegram_bot_token
+TG_CHAT=your_telegram_chat_id
+
+# Пауза для ручной проверки (минуты)
+LAB_PAUSE_MINUTES=10
+
+# Сценарий Molecule
+SCENARIO=universal
--- a/files/grafana/dashboards/istio-overview.json
+++ b/files/grafana/dashboards/istio-overview.json
@@ -0,0 +1,69 @@
+{
+  "id": null,
+  "uid": "istio-overview",
+  "title": "Istio • Overview",
+  "schemaVersion": 36,
+  "version": 1,
+  "timezone": "browser",
+  "tags": ["istio", "sre", "mesh"],
+  "panels": [
+    {
+      "type": "stat",
+      "title": "Global RPS",
+      "gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
+      "targets": [
+        {
+          "expr": "sum(rate(istio_requests_total{reporter=\"destination\"}[5m]))",
+          "legendFormat": "rps"
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Success Rate",
+      "gridPos": {"h": 4, "w": 6, "x": 6, "y": 0},
+      "targets": [
+        {
+          "expr": "sum(rate(istio_requests_total{reporter=\"destination\",response_code!~\"5..\"}[5m])) / sum(rate(istio_requests_total{reporter=\"destination\"}[5m]))",
+          "legendFormat": "success"
+        }
+      ],
+      "options": {"reduceOptions":{"calcs":["lastNotNull"]},"colorMode":"value","graphMode":"none"},
+      "fieldConfig":{"defaults":{"unit":"percentunit","min":0,"max":1}}
+    },
+    {
+      "type": "timeseries",
+      "title": "Latency (ms) p50/p95/p99",
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 4},
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.5, sum(rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\"}[5m])) by (le))",
+          "legendFormat": "p50"
+        },
+        {
+          "expr": "histogram_quantile(0.95, sum(rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\"}[5m])) by (le))",
+          "legendFormat": "p95"
+        },
+        {
+          "expr": "histogram_quantile(0.99, sum(rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\"}[5m])) by (le))",
+          "legendFormat": "p99"
+        }
+      ],
+      "fieldConfig":{"defaults":{"unit":"ms"}}
+    },
+    {
+      "type": "timeseries",
+      "title": "RPS by Workload",
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 12},
+      "targets": [
+        {
+          "expr": "sum by (destination_workload) (rate(istio_requests_total{reporter=\"destination\"}[5m]))",
+          "legendFormat": "{{destination_workload}}"
+        }
+      ]
+    }
+  ],
+  "templating": {
+    "list": []
+  }
+}
--- a/files/grafana/dashboards/service-sli.json
+++ b/files/grafana/dashboards/service-sli.json
@@ -0,0 +1,80 @@
+{
+  "id": null,
+  "uid": "service-sli",
+  "title": "Service • SLI",
+  "schemaVersion": 36,
+  "version": 1,
+  "timezone": "browser",
+  "tags": ["istio", "sre", "sli"],
+  "templating": {
+    "list": [
+      {
+        "name": "namespace",
+        "type": "query",
+        "datasource": "${DS_PROMETHEUS}",
+        "query": "label_values(istio_requests_total, destination_namespace)",
+        "refresh": 1
+      },
+      {
+        "name": "workload",
+        "type": "query",
+        "datasource": "${DS_PROMETHEUS}",
+        "query": "label_values(istio_requests_total{destination_namespace=\"$namespace\"}, destination_workload)",
+        "refresh": 1
+      }
+    ]
+  },
+  "panels": [
+    {
+      "type": "stat",
+      "title": "Success Rate",
+      "gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
+      "targets": [
+        {
+          "expr": "sum(rate(istio_requests_total{reporter=\"destination\",destination_namespace=\"$namespace\",destination_workload=\"$workload\",response_code!~\"5..\"}[5m])) / sum(rate(istio_requests_total{reporter=\"destination\",destination_namespace=\"$namespace\",destination_workload=\"$workload\"}[5m]))",
+          "legendFormat": "success"
+        }
+      ],
+      "fieldConfig":{"defaults":{"unit":"percentunit","min":0,"max":1}}
+    },
+    {
+      "type": "timeseries",
+      "title": "Latency (ms) p50/p95/p99",
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 4},
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.5, sum by (le) (rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_namespace=\"$namespace\",destination_workload=\"$workload\"}[5m])))",
+          "legendFormat": "p50"
+        },
+        {
+          "expr": "histogram_quantile(0.95, sum by (le) (rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_namespace=\"$namespace\",destination_workload=\"$workload\"}[5m])))",
+          "legendFormat": "p95"
+        },
+        {
+          "expr": "histogram_quantile(0.99, sum by (le) (rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_namespace=\"$namespace\",destination_workload=\"$workload\"}[5m])))",
+          "legendFormat": "p99"
+        }
+      ],
+      "fieldConfig":{"defaults":{"unit":"ms"}}
+    },
+    {
+      "type": "timeseries",
+      "title": "RPS (2xx/4xx/5xx)",
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 12},
+      "targets": [
+        {
+          "expr": "sum(rate(istio_requests_total{reporter=\"destination\",destination_namespace=\"$namespace\",destination_workload=\"$workload\",response_code=~\"2..\"}[5m]))",
+          "legendFormat": "2xx"
+        },
+        {
+          "expr": "sum(rate(istio_requests_total{reporter=\"destination\",destination_namespace=\"$namespace\",destination_workload=\"$workload\",response_code=~\"4..\"}[5m]))",
+          "legendFormat": "4xx"
+        },
+        {
+          "expr": "sum(rate(istio_requests_total{reporter=\"destination\",destination_namespace=\"$namespace\",destination_workload=\"$workload\",response_code=~\"5..\"}[5m]))",
+          "legendFormat": "5xx"
+        }
+      ]
+    }
+  ]
+}
--- a/files/k8s/istio/telemetry.yaml
+++ b/files/k8s/istio/telemetry.yaml
@@ -0,0 +1,39 @@
+---
+# Istio Telemetry для сбора метрик
+# Автор: Сергей Антропов
+# Сайт: https://devops.org.ru
+
+apiVersion: telemetry.istio.io/v1
+kind: Telemetry
+metadata:
+  name: mesh-default
+  namespace: istio-system
+spec:
+  selector: {}
+  metrics:
+  - providers:
+    - name: prometheus
+    overrides:
+    - match:
+        metric: REQUEST_DURATION
+      tagOverrides:
+        "destination_workload": { operation: UPSERT, value: "%DESTINATION_WORKLOAD%" }
+        "destination_namespace": { operation: UPSERT, value: "%DESTINATION_NAMESPACE%" }
+        "request_host": { operation: UPSERT, value: "%REQUEST_HOST%" }
+      histogram:
+        buckets:
+          - 1
+          - 5
+          - 10
+          - 25
+          - 50
+          - 100
+          - 250
+          - 500
+          - 1000
+          - 2000
+          - 5000
+    - match:
+        metric: REQUEST_COUNT
+      tagOverrides:
+        "response_code": { operation: UPSERT, value: "%RESPONSE_CODE%" }
--- a/files/k8s/istio/trafficpolicy.yaml
+++ b/files/k8s/istio/trafficpolicy.yaml
@@ -0,0 +1,66 @@
+---
+# Istio Traffic Policy для управления трафиком
+# Автор: Сергей Антропов
+# Сайт: https://devops.org.ru
+
+# mesh-wide mTLS STRICT
+apiVersion: security.istio.io/v1
+kind: PeerAuthentication
+metadata:
+  name: default
+  namespace: istio-system
+spec:
+  mtls:
+    mode: STRICT
+
+---
+# Пример строгой политики для bookinfo (pool + outlier)
+apiVersion: networking.istio.io/v1
+kind: DestinationRule
+metadata:
+  name: productpage-policy
+  namespace: bookinfo
+spec:
+  host: productpage.bookinfo.svc.cluster.local
+  trafficPolicy:
+    tls:
+      mode: ISTIO_MUTUAL
+    connectionPool:
+      tcp:
+        maxConnections: 100
+      http:
+        http1MaxPendingRequests: 1000
+        maxRequestsPerConnection: 100
+    outlierDetection:
+      consecutive5xx: 5
+      interval: 5s
+      baseEjectionTime: 30s
+      maxEjectionPercent: 50
+
+---
+apiVersion: networking.istio.io/v1
+kind: DestinationRule
+metadata:
+  name: reviews-policy
+  namespace: bookinfo
+spec:
+  host: reviews.bookinfo.svc.cluster.local
+  subsets:
+  - name: v1
+    labels: { version: v1 }
+  - name: v2
+    labels: { version: v2 }
+  trafficPolicy:
+    tls:
+      mode: ISTIO_MUTUAL
+    connectionPool:
+      tcp:
+        maxConnections: 100
+      http:
+        http1MaxPendingRequests: 1000
+        maxRequestsPerConnection: 100
+    outlierDetection:
+      consecutive5xx: 3
+      interval: 5s
+      baseEjectionTime: 30s
+      maxEjectionPercent: 50
--- a/molecule/universal/create.yml
+++ b/molecule/universal/create.yml
@@ -184,12 +184,15 @@
          '
      when: (kind_clusters | default([])) | length > 0

-    - name: Install Ingress NGINX and Metrics Server (per cluster, if enabled)
+    - name: Install Ingress NGINX, Metrics Server, Istio, Kiali, Prometheus Stack (per cluster, if enabled)
      community.docker.docker_container_exec:
        container: ansible-controller
        command: >
          bash -lc '
          set -e;
+          helm repo add kiali https://kiali.org/helm-charts >/dev/null 2>&1 || true;
+          helm repo add prometheus-community https://prometheus-community.github.io/helm-charts >/dev/null 2>&1 || true;
+          helm repo update >/dev/null 2>&1 || true;
          for n in {{ (kind_clusters | default([]) | map(attribute="name") | list) | map('quote') | join(' ') }}; do
            # ingress-nginx
            if {{ (kind_clusters | items2dict(key_name="name", value_name="addons")).get(n, {}).get("ingress_nginx", False) | to_json }}; then
@@ -204,6 +207,65 @@
              kubectl --context kind-$$n -n kube-system patch deploy metrics-server -p \
                "{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"metrics-server\",\"args\":[\"--kubelet-insecure-tls\",\"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname\"]}]}}}}}" || true;
            fi
+            # istio (demo profile)
+            if {{ (kind_clusters | items2dict(key_name="name", value_name="addons")).get(n, {}).get("istio", False) | to_json }}; then
+              echo "[addons] istio (demo profile) on $$n";
+              istioctl install -y --set profile=demo --context kind-$$n;
+              kubectl --context kind-$$n -n istio-system rollout status deploy/istiod --timeout=180s || true;
+              kubectl --context kind-$$n -n istio-system rollout status deploy/istio-ingressgateway --timeout=180s || true;
+            fi
+            # kiali (server chart, anonymous auth) — требует istio/metrics
+            if {{ (kind_clusters | items2dict(key_name="name", value_name="addons")).get(n, {}).get("kiali", False) | to_json }}; then
+              echo "[addons] kiali on $$n";
+              kubectl --context kind-$$n create ns istio-system >/dev/null 2>&1 || true;
+              helm upgrade --install kiali-server kiali/kiali-server \
+                --namespace istio-system --kube-context kind-$$n \
+                --set auth.strategy=anonymous --wait --timeout 180s;
+            fi
+            # kube-prometheus-stack (Prometheus + Grafana)
+            if {{ (kind_clusters | items2dict(key_name="name", value_name="addons")).get(n, {}).get("prometheus_stack", False) | to_json }}; then
+              echo "[addons] kube-prometheus-stack on $$n";
+              kubectl --context kind-$$n create ns monitoring >/dev/null 2>&1 || true;
+              helm upgrade --install monitoring prometheus-community/kube-prometheus-stack \
+                --namespace monitoring --kube-context kind-$$n \
+                --set grafana.adminPassword=admin \
+                --set grafana.defaultDashboardsTimezone=browser \
+                --wait --timeout 600s;
+              # дождаться графаны
+              kubectl --context kind-$$n -n monitoring rollout status deploy/monitoring-grafana --timeout=300s || true;
+            fi
+          done
+          '
+      when: (kind_clusters | default([])) | length > 0
+
+    - name: Apply Istio Telemetry + mesh mTLS + Grafana dashboards (per cluster)
+      community.docker.docker_container_exec:
+        container: ansible-controller
+        command: >
+          bash -lc '
+          set -e;
+          for n in {{ (kind_clusters | default([]) | map(attribute="name") | list) | map("quote") | join(" ") }}; do
+            # Telemetry/mTLS — только если istio есть
+            if {{ (kind_clusters | items2dict(key_name="name", value_name="addons")).get(n, {}).get("istio", False) | to_json }}; then
+              echo "[istio] applying Telemetry + PeerAuthentication on $$n";
+              kubectl --context kind-$$n -n istio-system apply -f /ansible/files/k8s/istio/telemetry.yaml || true;
+              kubectl --context kind-$$n -n istio-system apply -f /ansible/files/k8s/istio/trafficpolicy.yaml --dry-run=client -o yaml >/dev/null 2>&1 || true;
+              # DestinationRule из trafficpolicy — namespace bookinfo, создадим позже в verify после деплоя
+            fi
+
+            # Grafana dashboards (ConfigMap with label grafana_dashboard=1)
+            if {{ (kind_clusters | items2dict(key_name="name", value_name="addons")).get(n, {}).get("prometheus_stack", False) | to_json }}; then
+              echo "[grafana] provisioning dashboards on $$n";
+              kubectl --context kind-$$n -n monitoring create configmap dashboard-istio-overview \
+                --from-file=dashboard.json=/ansible/files/grafana/dashboards/istio-overview.json \
+                --dry-run=client -o yaml | kubectl --context kind-$$n apply -f -;
+              kubectl --context kind-$$n -n monitoring label configmap dashboard-istio-overview grafana_dashboard=1 --overwrite;
+
+              kubectl --context kind-$$n -n monitoring create configmap dashboard-service-sli \
+                --from-file=dashboard.json=/ansible/files/grafana/dashboards/service-sli.json \
+                --dry-run=client -o yaml | kubectl --context kind-$$n apply -f -;
+              kubectl --context kind-$$n -n monitoring label configmap dashboard-service-sli grafana_dashboard=1 --overwrite;
+            fi
          done
          '
      when: (kind_clusters | default([])) | length > 0
--- a/molecule/universal/verify.yml
+++ b/molecule/universal/verify.yml
@@ -43,6 +43,8 @@
          for n in {{ kind_names | map('quote') | join(' ') }}; do
            ns="lab-demo"; rel="nginx-$$n";
            kubectl --context kind-$$n create ns $$ns >/dev/null 2>&1 || true;
+            # метка для автосайдкаров Istio — не мешает, если Istio отключен
+            kubectl --context kind-$$n label ns $$ns istio-injection=enabled --overwrite >/dev/null 2>&1 || true;

            echo "[helm] installing $$rel";
            helm upgrade --install $$rel bitnami/nginx --namespace $$ns --kube-context kind-$$n --wait --timeout 180s;
@@ -98,6 +100,125 @@
      when: kind_names | length > 0
      failed_when: false

+    # --- Istio/Kiali overview (если включены) ---
+    - name: Istio & Kiali status
+      community.docker.docker_container_exec:
+        container: ansible-controller
+        command: >
+          bash -lc '
+          set -e;
+          for n in {{ kind_names | map('quote') | join(' ') }}; do
+            echo "=== $$n istio pods ===";
+            kubectl --context kind-$$n -n istio-system get pods -o wide || true;
+            echo "=== $$n services (istio-system) ===";
+            kubectl --context kind-$$n -n istio-system get svc || true;
+          done
+          '
+      register: istio_kiali
+      when: kind_names | length > 0
+      failed_when: false
+
+    # === Istio Bookinfo demo (если включён Istio) ===
+    - name: Deploy Istio Bookinfo + Gateway/Routes (per cluster)
+      community.docker.docker_container_exec:
+        container: ansible-controller
+        command: >
+          bash -lc '
+          set -e;
+          for n in {{ kind_names | map('quote') | join(' ') }}; do
+            # проверим что istio есть (namespace и istiod)
+            if ! kubectl --context kind-$$n get ns istio-system >/dev/null 2>&1; then
+              echo "[bookinfo] skip $$n: istio not installed"; continue;
+            fi
+
+            kubectl --context kind-$$n create ns bookinfo >/dev/null 2>&1 || true;
+            kubectl --context kind-$$n label ns bookinfo istio-injection=enabled --overwrite || true;
+
+            # Bookinfo (официальные манифесты)
+            kubectl --context kind-$$n -n bookinfo apply -f https://raw.githubusercontent.com/istio/istio/release-1.22/samples/bookinfo/platform/kube/bookinfo.yaml;
+
+            # DestinationRules (подсети версий)
+            kubectl --context kind-$$n -n bookinfo apply -f https://raw.githubusercontent.com/istio/istio/release-1.22/samples/bookinfo/networking/destination-rule-all.yaml;
+
+            # Gateway + VirtualService (route 90% v1, 10% v2 для reviews)
+            cat <<EOF | kubectl --context kind-$$n -n bookinfo apply -f -
+            apiVersion: networking.istio.io/v1beta1
+            kind: Gateway
+            metadata: { name: bookinfo-gateway }
+            spec:
+              selector:
+                istio: ingressgateway
+              servers:
+              - port: { number: 80, name: http, protocol: HTTP }
+                hosts: ["*"]
+            ---
+            apiVersion: networking.istio.io/v1beta1
+            kind: VirtualService
+            metadata: { name: bookinfo }
+            spec:
+              hosts: ["*"]
+              gateways: ["bookinfo-gateway"]
+              http:
+              - match:
+                - uri:
+                    prefix: /productpage
+                - uri:
+                    prefix: /static
+                - uri:
+                    prefix: /login
+                - uri:
+                    prefix: /logout
+                - uri:
+                    prefix: /api/v1/products
+                route:
+                - destination:
+                    host: productpage
+                    port: { number: 9080 }
+              - match:
+                - uri:
+                    prefix: /reviews
+                route:
+                - destination:
+                    host: reviews
+                    subset: v1
+                    port: { number: 9080 }
+                  weight: 90
+                - destination:
+                    host: reviews
+                    subset: v2
+                    port: { number: 9080 }
+                  weight: 10
+            EOF
+
+            # Ждём доступности productpage/reviews
+            kubectl --context kind-$$n -n bookinfo rollout status deploy/productpage-v1 --timeout=180s || true
+            kubectl --context kind-$$n -n bookinfo rollout status deploy/reviews-v1 --timeout=180s || true
+            kubectl --context kind-$$n -n bookinfo rollout status deploy/reviews-v2 --timeout=180s || true
+
+            echo "[bookinfo] try curl through Istio IngressGateway (port-forward 8082 if needed)";
+          done
+          '
+      register: istio_bookinfo
+      when: kind_names | length > 0
+      failed_when: false
+
+    - name: Apply DestinationRule TrafficPolicy for bookinfo (after deploy)
+      community.docker.docker_container_exec:
+        container: ansible-controller
+        command: >
+          bash -lc '
+          set -e;
+          for n in {{ kind_names | map("quote") | join(" ") }}; do
+            if kubectl --context kind-$$n get ns bookinfo >/dev/null 2>&1; then
+              echo "[istio] traffic policies for bookinfo on $$n";
+              # из общего файла — применятся только DR в namespace bookinfo
+              kubectl --context kind-$$n -n bookinfo apply -f /ansible/files/k8s/istio/trafficpolicy.yaml || true;
+            fi
+          done
+          '
+      when: kind_names | length > 0
+      failed_when: false
+
    # --- K8s overview (nodes & kube-system pods) ---
    - name: Collect k8s overview
      community.docker.docker_container_exec:
@@ -129,12 +250,16 @@
            --arg idemp "{{ idemp.stdout | to_json | replace("\"","\\\"") }}" \
            --arg haproxy_sel "{{ sel_rw.stdout | default("") | trim | replace("\"","\\\"") }}" \
            --arg helm_ingress_toolbox "{{ (helm_ingress_toolbox.stdout | default("")) | replace("\"","\\\"") }}" \
+            --arg istio_kiali "{{ (istio_kiali.stdout | default("")) | replace("\"","\\\"") }}" \
+            --arg istio_bookinfo "{{ (istio_bookinfo.stdout | default("")) | replace("\"","\\\"") }}" \
            --arg k8s_overview "{{ (k8s_overview.stdout | default("")) | replace("\"","\\\"") }}" \
            "{
              timestamp: $$time,
              idempotence_raw: $$idemp,
              haproxy_select1: $$haproxy_sel,
              helm_ingress_toolbox_raw: $$helm_ingress_toolbox,
+              istio_kiali_raw: $$istio_kiali,
+              istio_bookinfo_raw: $$istio_bookinfo,
              k8s_overview_raw: $$k8s_overview
            }" > /ansible/reports/lab-health.json
          '
--- a/scripts/cleanup.sh
+++ b/scripts/cleanup.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Очистка лаборатории
+# Автор: Сергей Антропов
+# Сайт: https://devops.org.ru
+
+set -euo pipefail
+
+echo "[cleanup] removing lab containers/volumes/networks"
+docker ps -aq --filter "label=ansible.lab=true" | xargs -r docker rm -f
+docker volume ls -q --filter "name=_docker$" | xargs -r docker volume rm
+docker network rm labnet >/dev/null 2>&1 || true
+echo "done."
--- a/scripts/report_html.py
+++ b/scripts/report_html.py
@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Генератор HTML отчетов для универсальной лаборатории
+Автор: Сергей Антропов
+Сайт: https://devops.org.ru
+"""
+
+import sys
+import json
+import html
+import datetime
+
+def main():
+    if len(sys.argv) < 3:
+        print("Usage: report_html.py <input.json> <output.html>")
+        sys.exit(1)
+
+    inp, outp = sys.argv[1], sys.argv[2]
+    
+    try:
+        with open(inp, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+    except Exception as e:
+        data = {"error": f"failed to read json: {e}"}
+
+    ts = data.get("timestamp", datetime.datetime.utcnow().isoformat())
+    idemp_raw = data.get("idempotence_raw", "")
+    haproxy_sel = data.get("haproxy_select1", "")
+    helm_ingress_toolbox = data.get("helm_ingress_toolbox_raw", "")
+    istio_kiali = data.get("istio_kiali_raw", "")
+    istio_bookinfo = data.get("istio_bookinfo_raw", "")
+    k8s_overview = data.get("k8s_overview_raw", "")
+
+    def badge(label, ok):
+        color = "#10b981" if ok else "#ef4444"
+        return f'<span class="badge" style="background:{color}">{html.escape(label)}</span>'
+
+    # Анализ статусов
+    idempotent_ok = ("changed=0" in idemp_raw)
+    haproxy_ok = (haproxy_sel.strip() == "1") if haproxy_sel else None
+    ingress_ok = ("ingress" in helm_ingress_toolbox.lower()) or ("curl http://localhost" in helm_ingress_toolbox.lower())
+    toolbox_ok = ("deploy/toolbox" in helm_ingress_toolbox.lower()) or ("rollout status" in helm_ingress_toolbox.lower())
+    istio_ok = ("istio-system" in istio_kiali.lower()) and ("istiod" in istio_kiali.lower())
+    kiali_ok = ("kiali" in istio_kiali.lower())
+    bookinfo_ok = ("bookinfo" in istio_bookinfo.lower()) or ("productpage" in istio_bookinfo.lower())
+    k8s_ok = ("nodes" in k8s_overview.lower()) and ("pods" in k8s_overview.lower())
+
+    def maybe_badge(label, status):
+        if status is None:
+            return f'<span class="badge" style="background:#6b7280">{html.escape(label)}: n/a</span>'
+        return badge(f"{label}", bool(status))
+
+    html_doc = f"""<!doctype html>
+<html lang="ru">
+<head>
+<meta charset="utf-8">
+<title>Lab Report</title>
+<meta name="viewport" content="width=device-width,initial-scale=1">
+<style>
+:root {{ --bg:#0f172a; --card:#111827; --muted:#94a3b8; --fg:#e5e7eb; --accent:#38bdf8; }}
+* {{ box-sizing:border-box; }}
+body {{ margin:0; font-family: ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Ubuntu, Cantarell, Noto Sans, Arial;
+        background:linear-gradient(180deg,#0b1220,#0f172a); color:var(--fg); }}
+.container {{ max-width:1100px; margin:40px auto; padding:0 16px; }}
+.hdr {{ display:flex; align-items:center; gap:14px; flex-wrap:wrap; }}
+.hdr h1 {{ margin:0; font-size:28px; }}
+.time {{ color:var(--muted); font-size:14px; }}
+.grid {{ display:grid; grid-template-columns:1fr; gap:16px; margin-top:20px; }}
+.card {{ background:rgba(17,24,39,.7); border:1px solid rgba(148,163,184,.15);
+         border-radius:14px; padding:16px; box-shadow:0 10px 30px rgba(0,0,0,.25); backdrop-filter: blur(6px); }}
+.card h2 {{ margin:0 0 10px 0; font-size:18px; color:#c7d2fe; }}
+pre {{ margin:0; padding:12px; background:#0b1220; border-radius:10px; overflow:auto; font-size:12px; line-height:1.4; }}
+.badge {{ display:inline-block; padding:4px 10px; border-radius:999px; font-size:12px; color:white; }}
+.kv {{ display:flex; flex-wrap:wrap; gap:8px; margin:8px 0 0 0; }}
+footer {{ margin:24px 0 40px; color:var(--muted); font-size:12px; text-align:center; }}
+a {{ color: var(--accent); text-decoration:none; }} a:hover {{ text-decoration:underline; }}
+</style>
+</head>
+<body>
+  <div class="container">
+    <div class="hdr">
+      <h1>Ansible Lab Report</h1>
+      <div class="time">generated: {html.escape(ts)}</div>
+    </div>
+
+    <div class="grid">
+      <div class="card">
+        <h2>Summary</h2>
+        <div class="kv">
+          {badge("Idempotent", idempotent_ok)}
+          {maybe_badge("HAProxy SELECT=1", haproxy_ok)}
+          {maybe_badge("Ingress ready", ingress_ok)}
+          {maybe_badge("Toolbox ready", toolbox_ok)}
+          {maybe_badge("Istio ready", istio_ok)}
+          {maybe_badge("Kiali ready", kiali_ok)}
+          {maybe_badge("Bookinfo ready", bookinfo_ok)}
+          {maybe_badge("K8s ready", k8s_ok)}
+        </div>
+      </div>
+
+      <div class="card">
+        <h2>Idempotence (raw)</h2>
+        <pre>{html.escape(idemp_raw) if idemp_raw else "n/a"}</pre>
+      </div>
+
+      <div class="card">
+        <h2>Helm + Ingress + Toolbox (raw)</h2>
+        <pre>{html.escape(helm_ingress_toolbox) if helm_ingress_toolbox else "n/a"}</pre>
+      </div>
+
+      <div class="card">
+        <h2>Istio / Kiali (raw)</h2>
+        <pre>{html.escape(istio_kiali) if istio_kiali else "n/a"}</pre>
+      </div>
+
+      <div class="card">
+        <h2>Istio Bookinfo (raw)</h2>
+        <pre>{html.escape(istio_bookinfo) if istio_bookinfo else "n/a"}</pre>
+      </div>
+
+      <div class="card">
+        <h2>K8s Overview (raw)</h2>
+        <pre>{html.escape(k8s_overview) if k8s_overview else "n/a"}</pre>
+      </div>
+
+      <div class="card">
+        <h2>HAProxy SELECT 1</h2>
+        <pre>{html.escape(haproxy_sel) if haproxy_sel else "n/a"}</pre>
+      </div>
+    </div>
+
+    <footer>HTML from /ansible/reports/lab-health.json · kubeconfigs in reports/kubeconfigs/</footer>
+  </div>
+</body>
+</html>"""
+
+    with open(outp, "w", encoding="utf-8") as f:
+        f.write(html_doc)
+    print(outp)
+
+if __name__ == "__main__":
+    main()
--- a/scripts/restore.sh
+++ b/scripts/restore.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Восстановление лаборатории из снапшотов
+# Автор: Сергей Антропов
+# Сайт: https://devops.org.ru
+
+set -euo pipefail
+
+IN_DIR="snapshots"
+if [ ! -d "$IN_DIR" ]; then
+    echo "No snapshots dir"
+    exit 1
+fi
+
+for f in "$IN_DIR"/*.image; do
+    if [ ! -f "$f" ]; then
+        continue
+    fi
+    name=$(basename "$f" .image)
+    img=$(cat "$f")
+    echo "[restore] $name from $img"
+    docker rm -f "$name" >/dev/null 2>&1 || true
+    docker run -d --name "$name" "$img" >/dev/null
+done
+
+echo "Restored from $IN_DIR/"
--- a/scripts/snapshot.sh
+++ b/scripts/snapshot.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Снапшот лаборатории
+# Автор: Сергей Антропов
+# Сайт: https://devops.org.ru
+
+set -euo pipefail
+
+OUT_DIR="snapshots"
+mkdir -p "$OUT_DIR"
+
+# Найти все контейнеры лаборатории
+ids=$(docker ps -q --filter "label=ansible.lab=true")
+if [ -z "$ids" ]; then
+    echo "No lab containers to snapshot"
+    exit 0
+fi
+
+for id in $ids; do
+    name=$(docker inspect --format '{{.Name}}' "$id" | sed 's#^/##')
+    img="lab-snap-$name:latest"
+    echo "[snapshot] $name -> $img"
+    docker commit "$id" "$img" >/dev/null
+    echo "$img" > "$OUT_DIR/$name.image"
+done
+
+echo "Snapshots saved to $OUT_DIR/"