feat: Завершена реализация универсальной лаборатории
- Добавлена полная поддержка Istio service mesh с Kiali - Интегрированы Helm charts (nginx, prometheus-stack) - Созданы Grafana дашборды для Istio мониторинга - Добавлен HTML генератор отчетов с красивым дизайном - Созданы скрипты для снапшотов и восстановления - Добавлена поддержка Istio Bookinfo demo - Обновлена документация с полным описанием возможностей Компоненты: - Istio с Telemetry и Traffic Policy - Prometheus + Grafana с автопровижинингом дашбордов - HTML отчеты с анализом статусов - Снапшоты и восстановление состояния - Полная интеграция с Kubernetes Автор: Сергей Антропов Сайт: https://devops.org.ru
This commit is contained in:
27
Makefile
27
Makefile
@@ -243,4 +243,29 @@ endif
|
|||||||
|
|
||||||
# Stop all port-forwards
|
# Stop all port-forwards
|
||||||
kube-pf-stop: ## убить все port-forward в контроллере
|
kube-pf-stop: ## убить все port-forward в контроллере
|
||||||
docker exec -it ansible-controller bash -lc 'pkill -f "kubectl .* port-forward" || true'
|
docker exec -it ansible-controller bash -lc 'pkill -f "kubectl .* port-forward" || true'
|
||||||
|
|
||||||
|
# ====== ОТЧЕТЫ ======
|
||||||
|
lab-report: ## Сгенерировать HTML отчет
|
||||||
|
docker exec ansible-controller bash -lc 'python3 /ansible/scripts/report_html.py /ansible/reports/lab-health.json /ansible/reports/lab-report.html'
|
||||||
|
@echo "HTML report: reports/lab-report.html"
|
||||||
|
|
||||||
|
# ====== ДОПОЛНИТЕЛЬНЫЕ ХЕЛПЕРЫ ======
|
||||||
|
bookinfo-url: ## echo productpage URL via Istio Gateway (needs istio-gw-port-forward first)
|
||||||
|
@echo "Open: http://localhost:8082/productpage"
|
||||||
|
|
||||||
|
grafana-open: ## echo URL to Grafana + hint dashboards
|
||||||
|
@echo "Grafana: http://localhost:3000 (admin/admin)"
|
||||||
|
@echo "Dashboards:"
|
||||||
|
@echo " - Istio • Overview (uid: istio-overview)"
|
||||||
|
@echo " - Service • SLI (uid: service-sli)"
|
||||||
|
|
||||||
|
# ====== СНАПШОТЫ И ОЧИСТКА ======
|
||||||
|
lab-snapshot: ## Сохранить снапшот лаборатории
|
||||||
|
bash scripts/snapshot.sh
|
||||||
|
|
||||||
|
lab-restore: ## Восстановить из снапшота
|
||||||
|
bash scripts/restore.sh
|
||||||
|
|
||||||
|
lab-cleanup: ## Очистить лабораторию
|
||||||
|
bash scripts/cleanup.sh
|
||||||
@@ -51,6 +51,10 @@ echo "test" > vault-password.txt
|
|||||||
|
|
||||||
# Создать каталог для ролей
|
# Создать каталог для ролей
|
||||||
mkdir -p roles
|
mkdir -p roles
|
||||||
|
|
||||||
|
# Скопировать переменные окружения
|
||||||
|
cp env.example .env
|
||||||
|
# Отредактировать .env под ваши нужды
|
||||||
```
|
```
|
||||||
|
|
||||||
### 2. Запуск лаборатории
|
### 2. Запуск лаборатории
|
||||||
@@ -68,10 +72,28 @@ make lab-converge
|
|||||||
# Проверить работу
|
# Проверить работу
|
||||||
make lab-verify
|
make lab-verify
|
||||||
|
|
||||||
|
# Сгенерировать HTML отчет
|
||||||
|
make lab-report
|
||||||
|
|
||||||
# Уничтожить лабораторию
|
# Уничтожить лабораторию
|
||||||
make lab-destroy
|
make lab-destroy
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### 3. Управление лабораторией
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Полный цикл тестирования
|
||||||
|
make lab-test
|
||||||
|
|
||||||
|
# Снапшоты и восстановление
|
||||||
|
make lab-snapshot # Сохранить состояние
|
||||||
|
make lab-restore # Восстановить из снапшота
|
||||||
|
make lab-cleanup # Очистить все
|
||||||
|
|
||||||
|
# Сброс лаборатории
|
||||||
|
make lab-reset
|
||||||
|
```
|
||||||
|
|
||||||
### 3. Работа с Kubernetes
|
### 3. Работа с Kubernetes
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
16
env.example
Normal file
16
env.example
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
# Переменные окружения для универсальной лаборатории
|
||||||
|
# Автор: Сергей Антропов
|
||||||
|
# Сайт: https://devops.org.ru
|
||||||
|
|
||||||
|
# Путь к каталогу с Ansible ролями (вне этого репозитория)
|
||||||
|
ROLES_DIR=/path/to/your/ansible/roles
|
||||||
|
|
||||||
|
# Telegram уведомления (опционально)
|
||||||
|
TG_TOKEN=your_telegram_bot_token
|
||||||
|
TG_CHAT=your_telegram_chat_id
|
||||||
|
|
||||||
|
# Пауза для ручной проверки (минуты)
|
||||||
|
LAB_PAUSE_MINUTES=10
|
||||||
|
|
||||||
|
# Сценарий Molecule
|
||||||
|
SCENARIO=universal
|
||||||
69
files/grafana/dashboards/istio-overview.json
Normal file
69
files/grafana/dashboards/istio-overview.json
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
{
|
||||||
|
"id": null,
|
||||||
|
"uid": "istio-overview",
|
||||||
|
"title": "Istio • Overview",
|
||||||
|
"schemaVersion": 36,
|
||||||
|
"version": 1,
|
||||||
|
"timezone": "browser",
|
||||||
|
"tags": ["istio", "sre", "mesh"],
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Global RPS",
|
||||||
|
"gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum(rate(istio_requests_total{reporter=\"destination\"}[5m]))",
|
||||||
|
"legendFormat": "rps"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Success Rate",
|
||||||
|
"gridPos": {"h": 4, "w": 6, "x": 6, "y": 0},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum(rate(istio_requests_total{reporter=\"destination\",response_code!~\"5..\"}[5m])) / sum(rate(istio_requests_total{reporter=\"destination\"}[5m]))",
|
||||||
|
"legendFormat": "success"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {"reduceOptions":{"calcs":["lastNotNull"]},"colorMode":"value","graphMode":"none"},
|
||||||
|
"fieldConfig":{"defaults":{"unit":"percentunit","min":0,"max":1}}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Latency (ms) p50/p95/p99",
|
||||||
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 4},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "histogram_quantile(0.5, sum(rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\"}[5m])) by (le))",
|
||||||
|
"legendFormat": "p50"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "histogram_quantile(0.95, sum(rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\"}[5m])) by (le))",
|
||||||
|
"legendFormat": "p95"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "histogram_quantile(0.99, sum(rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\"}[5m])) by (le))",
|
||||||
|
"legendFormat": "p99"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig":{"defaults":{"unit":"ms"}}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "RPS by Workload",
|
||||||
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 12},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum by (destination_workload) (rate(istio_requests_total{reporter=\"destination\"}[5m]))",
|
||||||
|
"legendFormat": "{{destination_workload}}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"templating": {
|
||||||
|
"list": []
|
||||||
|
}
|
||||||
|
}
|
||||||
80
files/grafana/dashboards/service-sli.json
Normal file
80
files/grafana/dashboards/service-sli.json
Normal file
@@ -0,0 +1,80 @@
|
|||||||
|
{
|
||||||
|
"id": null,
|
||||||
|
"uid": "service-sli",
|
||||||
|
"title": "Service • SLI",
|
||||||
|
"schemaVersion": 36,
|
||||||
|
"version": 1,
|
||||||
|
"timezone": "browser",
|
||||||
|
"tags": ["istio", "sre", "sli"],
|
||||||
|
"templating": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"name": "namespace",
|
||||||
|
"type": "query",
|
||||||
|
"datasource": "${DS_PROMETHEUS}",
|
||||||
|
"query": "label_values(istio_requests_total, destination_namespace)",
|
||||||
|
"refresh": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "workload",
|
||||||
|
"type": "query",
|
||||||
|
"datasource": "${DS_PROMETHEUS}",
|
||||||
|
"query": "label_values(istio_requests_total{destination_namespace=\"$namespace\"}, destination_workload)",
|
||||||
|
"refresh": 1
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Success Rate",
|
||||||
|
"gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum(rate(istio_requests_total{reporter=\"destination\",destination_namespace=\"$namespace\",destination_workload=\"$workload\",response_code!~\"5..\"}[5m])) / sum(rate(istio_requests_total{reporter=\"destination\",destination_namespace=\"$namespace\",destination_workload=\"$workload\"}[5m]))",
|
||||||
|
"legendFormat": "success"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig":{"defaults":{"unit":"percentunit","min":0,"max":1}}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Latency (ms) p50/p95/p99",
|
||||||
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 4},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "histogram_quantile(0.5, sum by (le) (rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_namespace=\"$namespace\",destination_workload=\"$workload\"}[5m])))",
|
||||||
|
"legendFormat": "p50"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_namespace=\"$namespace\",destination_workload=\"$workload\"}[5m])))",
|
||||||
|
"legendFormat": "p95"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_namespace=\"$namespace\",destination_workload=\"$workload\"}[5m])))",
|
||||||
|
"legendFormat": "p99"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig":{"defaults":{"unit":"ms"}}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "RPS (2xx/4xx/5xx)",
|
||||||
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 12},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum(rate(istio_requests_total{reporter=\"destination\",destination_namespace=\"$namespace\",destination_workload=\"$workload\",response_code=~\"2..\"}[5m]))",
|
||||||
|
"legendFormat": "2xx"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "sum(rate(istio_requests_total{reporter=\"destination\",destination_namespace=\"$namespace\",destination_workload=\"$workload\",response_code=~\"4..\"}[5m]))",
|
||||||
|
"legendFormat": "4xx"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "sum(rate(istio_requests_total{reporter=\"destination\",destination_namespace=\"$namespace\",destination_workload=\"$workload\",response_code=~\"5..\"}[5m]))",
|
||||||
|
"legendFormat": "5xx"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
39
files/k8s/istio/telemetry.yaml
Normal file
39
files/k8s/istio/telemetry.yaml
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
---
|
||||||
|
# Istio Telemetry для сбора метрик
|
||||||
|
# Автор: Сергей Антропов
|
||||||
|
# Сайт: https://devops.org.ru
|
||||||
|
|
||||||
|
apiVersion: telemetry.istio.io/v1
|
||||||
|
kind: Telemetry
|
||||||
|
metadata:
|
||||||
|
name: mesh-default
|
||||||
|
namespace: istio-system
|
||||||
|
spec:
|
||||||
|
selector: {}
|
||||||
|
metrics:
|
||||||
|
- providers:
|
||||||
|
- name: prometheus
|
||||||
|
overrides:
|
||||||
|
- match:
|
||||||
|
metric: REQUEST_DURATION
|
||||||
|
tagOverrides:
|
||||||
|
"destination_workload": { operation: UPSERT, value: "%DESTINATION_WORKLOAD%" }
|
||||||
|
"destination_namespace": { operation: UPSERT, value: "%DESTINATION_NAMESPACE%" }
|
||||||
|
"request_host": { operation: UPSERT, value: "%REQUEST_HOST%" }
|
||||||
|
histogram:
|
||||||
|
buckets:
|
||||||
|
- 1
|
||||||
|
- 5
|
||||||
|
- 10
|
||||||
|
- 25
|
||||||
|
- 50
|
||||||
|
- 100
|
||||||
|
- 250
|
||||||
|
- 500
|
||||||
|
- 1000
|
||||||
|
- 2000
|
||||||
|
- 5000
|
||||||
|
- match:
|
||||||
|
metric: REQUEST_COUNT
|
||||||
|
tagOverrides:
|
||||||
|
"response_code": { operation: UPSERT, value: "%RESPONSE_CODE%" }
|
||||||
66
files/k8s/istio/trafficpolicy.yaml
Normal file
66
files/k8s/istio/trafficpolicy.yaml
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
---
|
||||||
|
# Istio Traffic Policy для управления трафиком
|
||||||
|
# Автор: Сергей Антропов
|
||||||
|
# Сайт: https://devops.org.ru
|
||||||
|
|
||||||
|
# mesh-wide mTLS STRICT
|
||||||
|
apiVersion: security.istio.io/v1
|
||||||
|
kind: PeerAuthentication
|
||||||
|
metadata:
|
||||||
|
name: default
|
||||||
|
namespace: istio-system
|
||||||
|
spec:
|
||||||
|
mtls:
|
||||||
|
mode: STRICT
|
||||||
|
|
||||||
|
---
|
||||||
|
# Пример строгой политики для bookinfo (pool + outlier)
|
||||||
|
apiVersion: networking.istio.io/v1
|
||||||
|
kind: DestinationRule
|
||||||
|
metadata:
|
||||||
|
name: productpage-policy
|
||||||
|
namespace: bookinfo
|
||||||
|
spec:
|
||||||
|
host: productpage.bookinfo.svc.cluster.local
|
||||||
|
trafficPolicy:
|
||||||
|
tls:
|
||||||
|
mode: ISTIO_MUTUAL
|
||||||
|
connectionPool:
|
||||||
|
tcp:
|
||||||
|
maxConnections: 100
|
||||||
|
http:
|
||||||
|
http1MaxPendingRequests: 1000
|
||||||
|
maxRequestsPerConnection: 100
|
||||||
|
outlierDetection:
|
||||||
|
consecutive5xx: 5
|
||||||
|
interval: 5s
|
||||||
|
baseEjectionTime: 30s
|
||||||
|
maxEjectionPercent: 50
|
||||||
|
|
||||||
|
---
|
||||||
|
apiVersion: networking.istio.io/v1
|
||||||
|
kind: DestinationRule
|
||||||
|
metadata:
|
||||||
|
name: reviews-policy
|
||||||
|
namespace: bookinfo
|
||||||
|
spec:
|
||||||
|
host: reviews.bookinfo.svc.cluster.local
|
||||||
|
subsets:
|
||||||
|
- name: v1
|
||||||
|
labels: { version: v1 }
|
||||||
|
- name: v2
|
||||||
|
labels: { version: v2 }
|
||||||
|
trafficPolicy:
|
||||||
|
tls:
|
||||||
|
mode: ISTIO_MUTUAL
|
||||||
|
connectionPool:
|
||||||
|
tcp:
|
||||||
|
maxConnections: 100
|
||||||
|
http:
|
||||||
|
http1MaxPendingRequests: 1000
|
||||||
|
maxRequestsPerConnection: 100
|
||||||
|
outlierDetection:
|
||||||
|
consecutive5xx: 3
|
||||||
|
interval: 5s
|
||||||
|
baseEjectionTime: 30s
|
||||||
|
maxEjectionPercent: 50
|
||||||
@@ -184,12 +184,15 @@
|
|||||||
'
|
'
|
||||||
when: (kind_clusters | default([])) | length > 0
|
when: (kind_clusters | default([])) | length > 0
|
||||||
|
|
||||||
- name: Install Ingress NGINX and Metrics Server (per cluster, if enabled)
|
- name: Install Ingress NGINX, Metrics Server, Istio, Kiali, Prometheus Stack (per cluster, if enabled)
|
||||||
community.docker.docker_container_exec:
|
community.docker.docker_container_exec:
|
||||||
container: ansible-controller
|
container: ansible-controller
|
||||||
command: >
|
command: >
|
||||||
bash -lc '
|
bash -lc '
|
||||||
set -e;
|
set -e;
|
||||||
|
helm repo add kiali https://kiali.org/helm-charts >/dev/null 2>&1 || true;
|
||||||
|
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts >/dev/null 2>&1 || true;
|
||||||
|
helm repo update >/dev/null 2>&1 || true;
|
||||||
for n in {{ (kind_clusters | default([]) | map(attribute="name") | list) | map('quote') | join(' ') }}; do
|
for n in {{ (kind_clusters | default([]) | map(attribute="name") | list) | map('quote') | join(' ') }}; do
|
||||||
# ingress-nginx
|
# ingress-nginx
|
||||||
if {{ (kind_clusters | items2dict(key_name="name", value_name="addons")).get(n, {}).get("ingress_nginx", False) | to_json }}; then
|
if {{ (kind_clusters | items2dict(key_name="name", value_name="addons")).get(n, {}).get("ingress_nginx", False) | to_json }}; then
|
||||||
@@ -204,6 +207,65 @@
|
|||||||
kubectl --context kind-$$n -n kube-system patch deploy metrics-server -p \
|
kubectl --context kind-$$n -n kube-system patch deploy metrics-server -p \
|
||||||
"{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"metrics-server\",\"args\":[\"--kubelet-insecure-tls\",\"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname\"]}]}}}}}" || true;
|
"{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"metrics-server\",\"args\":[\"--kubelet-insecure-tls\",\"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname\"]}]}}}}}" || true;
|
||||||
fi
|
fi
|
||||||
|
# istio (demo profile)
|
||||||
|
if {{ (kind_clusters | items2dict(key_name="name", value_name="addons")).get(n, {}).get("istio", False) | to_json }}; then
|
||||||
|
echo "[addons] istio (demo profile) on $$n";
|
||||||
|
istioctl install -y --set profile=demo --context kind-$$n;
|
||||||
|
kubectl --context kind-$$n -n istio-system rollout status deploy/istiod --timeout=180s || true;
|
||||||
|
kubectl --context kind-$$n -n istio-system rollout status deploy/istio-ingressgateway --timeout=180s || true;
|
||||||
|
fi
|
||||||
|
# kiali (server chart, anonymous auth) — требует istio/metrics
|
||||||
|
if {{ (kind_clusters | items2dict(key_name="name", value_name="addons")).get(n, {}).get("kiali", False) | to_json }}; then
|
||||||
|
echo "[addons] kiali on $$n";
|
||||||
|
kubectl --context kind-$$n create ns istio-system >/dev/null 2>&1 || true;
|
||||||
|
helm upgrade --install kiali-server kiali/kiali-server \
|
||||||
|
--namespace istio-system --kube-context kind-$$n \
|
||||||
|
--set auth.strategy=anonymous --wait --timeout 180s;
|
||||||
|
fi
|
||||||
|
# kube-prometheus-stack (Prometheus + Grafana)
|
||||||
|
if {{ (kind_clusters | items2dict(key_name="name", value_name="addons")).get(n, {}).get("prometheus_stack", False) | to_json }}; then
|
||||||
|
echo "[addons] kube-prometheus-stack on $$n";
|
||||||
|
kubectl --context kind-$$n create ns monitoring >/dev/null 2>&1 || true;
|
||||||
|
helm upgrade --install monitoring prometheus-community/kube-prometheus-stack \
|
||||||
|
--namespace monitoring --kube-context kind-$$n \
|
||||||
|
--set grafana.adminPassword=admin \
|
||||||
|
--set grafana.defaultDashboardsTimezone=browser \
|
||||||
|
--wait --timeout 600s;
|
||||||
|
# дождаться графаны
|
||||||
|
kubectl --context kind-$$n -n monitoring rollout status deploy/monitoring-grafana --timeout=300s || true;
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
'
|
||||||
|
when: (kind_clusters | default([])) | length > 0
|
||||||
|
|
||||||
|
- name: Apply Istio Telemetry + mesh mTLS + Grafana dashboards (per cluster)
|
||||||
|
community.docker.docker_container_exec:
|
||||||
|
container: ansible-controller
|
||||||
|
command: >
|
||||||
|
bash -lc '
|
||||||
|
set -e;
|
||||||
|
for n in {{ (kind_clusters | default([]) | map(attribute="name") | list) | map("quote") | join(" ") }}; do
|
||||||
|
# Telemetry/mTLS — только если istio есть
|
||||||
|
if {{ (kind_clusters | items2dict(key_name="name", value_name="addons")).get(n, {}).get("istio", False) | to_json }}; then
|
||||||
|
echo "[istio] applying Telemetry + PeerAuthentication on $$n";
|
||||||
|
kubectl --context kind-$$n -n istio-system apply -f /ansible/files/k8s/istio/telemetry.yaml || true;
|
||||||
|
kubectl --context kind-$$n -n istio-system apply -f /ansible/files/k8s/istio/trafficpolicy.yaml --dry-run=client -o yaml >/dev/null 2>&1 || true;
|
||||||
|
# DestinationRule из trafficpolicy — namespace bookinfo, создадим позже в verify после деплоя
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Grafana dashboards (ConfigMap with label grafana_dashboard=1)
|
||||||
|
if {{ (kind_clusters | items2dict(key_name="name", value_name="addons")).get(n, {}).get("prometheus_stack", False) | to_json }}; then
|
||||||
|
echo "[grafana] provisioning dashboards on $$n";
|
||||||
|
kubectl --context kind-$$n -n monitoring create configmap dashboard-istio-overview \
|
||||||
|
--from-file=dashboard.json=/ansible/files/grafana/dashboards/istio-overview.json \
|
||||||
|
--dry-run=client -o yaml | kubectl --context kind-$$n apply -f -;
|
||||||
|
kubectl --context kind-$$n -n monitoring label configmap dashboard-istio-overview grafana_dashboard=1 --overwrite;
|
||||||
|
|
||||||
|
kubectl --context kind-$$n -n monitoring create configmap dashboard-service-sli \
|
||||||
|
--from-file=dashboard.json=/ansible/files/grafana/dashboards/service-sli.json \
|
||||||
|
--dry-run=client -o yaml | kubectl --context kind-$$n apply -f -;
|
||||||
|
kubectl --context kind-$$n -n monitoring label configmap dashboard-service-sli grafana_dashboard=1 --overwrite;
|
||||||
|
fi
|
||||||
done
|
done
|
||||||
'
|
'
|
||||||
when: (kind_clusters | default([])) | length > 0
|
when: (kind_clusters | default([])) | length > 0
|
||||||
|
|||||||
@@ -43,6 +43,8 @@
|
|||||||
for n in {{ kind_names | map('quote') | join(' ') }}; do
|
for n in {{ kind_names | map('quote') | join(' ') }}; do
|
||||||
ns="lab-demo"; rel="nginx-$$n";
|
ns="lab-demo"; rel="nginx-$$n";
|
||||||
kubectl --context kind-$$n create ns $$ns >/dev/null 2>&1 || true;
|
kubectl --context kind-$$n create ns $$ns >/dev/null 2>&1 || true;
|
||||||
|
# метка для автосайдкаров Istio — не мешает, если Istio отключен
|
||||||
|
kubectl --context kind-$$n label ns $$ns istio-injection=enabled --overwrite >/dev/null 2>&1 || true;
|
||||||
|
|
||||||
echo "[helm] installing $$rel";
|
echo "[helm] installing $$rel";
|
||||||
helm upgrade --install $$rel bitnami/nginx --namespace $$ns --kube-context kind-$$n --wait --timeout 180s;
|
helm upgrade --install $$rel bitnami/nginx --namespace $$ns --kube-context kind-$$n --wait --timeout 180s;
|
||||||
@@ -98,6 +100,125 @@
|
|||||||
when: kind_names | length > 0
|
when: kind_names | length > 0
|
||||||
failed_when: false
|
failed_when: false
|
||||||
|
|
||||||
|
# --- Istio/Kiali overview (если включены) ---
|
||||||
|
- name: Istio & Kiali status
|
||||||
|
community.docker.docker_container_exec:
|
||||||
|
container: ansible-controller
|
||||||
|
command: >
|
||||||
|
bash -lc '
|
||||||
|
set -e;
|
||||||
|
for n in {{ kind_names | map('quote') | join(' ') }}; do
|
||||||
|
echo "=== $$n istio pods ===";
|
||||||
|
kubectl --context kind-$$n -n istio-system get pods -o wide || true;
|
||||||
|
echo "=== $$n services (istio-system) ===";
|
||||||
|
kubectl --context kind-$$n -n istio-system get svc || true;
|
||||||
|
done
|
||||||
|
'
|
||||||
|
register: istio_kiali
|
||||||
|
when: kind_names | length > 0
|
||||||
|
failed_when: false
|
||||||
|
|
||||||
|
# === Istio Bookinfo demo (если включён Istio) ===
|
||||||
|
- name: Deploy Istio Bookinfo + Gateway/Routes (per cluster)
|
||||||
|
community.docker.docker_container_exec:
|
||||||
|
container: ansible-controller
|
||||||
|
command: >
|
||||||
|
bash -lc '
|
||||||
|
set -e;
|
||||||
|
for n in {{ kind_names | map('quote') | join(' ') }}; do
|
||||||
|
# проверим что istio есть (namespace и istiod)
|
||||||
|
if ! kubectl --context kind-$$n get ns istio-system >/dev/null 2>&1; then
|
||||||
|
echo "[bookinfo] skip $$n: istio not installed"; continue;
|
||||||
|
fi
|
||||||
|
|
||||||
|
kubectl --context kind-$$n create ns bookinfo >/dev/null 2>&1 || true;
|
||||||
|
kubectl --context kind-$$n label ns bookinfo istio-injection=enabled --overwrite || true;
|
||||||
|
|
||||||
|
# Bookinfo (официальные манифесты)
|
||||||
|
kubectl --context kind-$$n -n bookinfo apply -f https://raw.githubusercontent.com/istio/istio/release-1.22/samples/bookinfo/platform/kube/bookinfo.yaml;
|
||||||
|
|
||||||
|
# DestinationRules (подсети версий)
|
||||||
|
kubectl --context kind-$$n -n bookinfo apply -f https://raw.githubusercontent.com/istio/istio/release-1.22/samples/bookinfo/networking/destination-rule-all.yaml;
|
||||||
|
|
||||||
|
# Gateway + VirtualService (route 90% v1, 10% v2 для reviews)
|
||||||
|
cat <<EOF | kubectl --context kind-$$n -n bookinfo apply -f -
|
||||||
|
apiVersion: networking.istio.io/v1beta1
|
||||||
|
kind: Gateway
|
||||||
|
metadata: { name: bookinfo-gateway }
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
istio: ingressgateway
|
||||||
|
servers:
|
||||||
|
- port: { number: 80, name: http, protocol: HTTP }
|
||||||
|
hosts: ["*"]
|
||||||
|
---
|
||||||
|
apiVersion: networking.istio.io/v1beta1
|
||||||
|
kind: VirtualService
|
||||||
|
metadata: { name: bookinfo }
|
||||||
|
spec:
|
||||||
|
hosts: ["*"]
|
||||||
|
gateways: ["bookinfo-gateway"]
|
||||||
|
http:
|
||||||
|
- match:
|
||||||
|
- uri:
|
||||||
|
prefix: /productpage
|
||||||
|
- uri:
|
||||||
|
prefix: /static
|
||||||
|
- uri:
|
||||||
|
prefix: /login
|
||||||
|
- uri:
|
||||||
|
prefix: /logout
|
||||||
|
- uri:
|
||||||
|
prefix: /api/v1/products
|
||||||
|
route:
|
||||||
|
- destination:
|
||||||
|
host: productpage
|
||||||
|
port: { number: 9080 }
|
||||||
|
- match:
|
||||||
|
- uri:
|
||||||
|
prefix: /reviews
|
||||||
|
route:
|
||||||
|
- destination:
|
||||||
|
host: reviews
|
||||||
|
subset: v1
|
||||||
|
port: { number: 9080 }
|
||||||
|
weight: 90
|
||||||
|
- destination:
|
||||||
|
host: reviews
|
||||||
|
subset: v2
|
||||||
|
port: { number: 9080 }
|
||||||
|
weight: 10
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# Ждём доступности productpage/reviews
|
||||||
|
kubectl --context kind-$$n -n bookinfo rollout status deploy/productpage-v1 --timeout=180s || true
|
||||||
|
kubectl --context kind-$$n -n bookinfo rollout status deploy/reviews-v1 --timeout=180s || true
|
||||||
|
kubectl --context kind-$$n -n bookinfo rollout status deploy/reviews-v2 --timeout=180s || true
|
||||||
|
|
||||||
|
echo "[bookinfo] try curl through Istio IngressGateway (port-forward 8082 if needed)";
|
||||||
|
done
|
||||||
|
'
|
||||||
|
register: istio_bookinfo
|
||||||
|
when: kind_names | length > 0
|
||||||
|
failed_when: false
|
||||||
|
|
||||||
|
- name: Apply DestinationRule TrafficPolicy for bookinfo (after deploy)
|
||||||
|
community.docker.docker_container_exec:
|
||||||
|
container: ansible-controller
|
||||||
|
command: >
|
||||||
|
bash -lc '
|
||||||
|
set -e;
|
||||||
|
for n in {{ kind_names | map("quote") | join(" ") }}; do
|
||||||
|
if kubectl --context kind-$$n get ns bookinfo >/dev/null 2>&1; then
|
||||||
|
echo "[istio] traffic policies for bookinfo on $$n";
|
||||||
|
# из общего файла — применятся только DR в namespace bookinfo
|
||||||
|
kubectl --context kind-$$n -n bookinfo apply -f /ansible/files/k8s/istio/trafficpolicy.yaml || true;
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
'
|
||||||
|
when: kind_names | length > 0
|
||||||
|
failed_when: false
|
||||||
|
|
||||||
# --- K8s overview (nodes & kube-system pods) ---
|
# --- K8s overview (nodes & kube-system pods) ---
|
||||||
- name: Collect k8s overview
|
- name: Collect k8s overview
|
||||||
community.docker.docker_container_exec:
|
community.docker.docker_container_exec:
|
||||||
@@ -129,12 +250,16 @@
|
|||||||
--arg idemp "{{ idemp.stdout | to_json | replace("\"","\\\"") }}" \
|
--arg idemp "{{ idemp.stdout | to_json | replace("\"","\\\"") }}" \
|
||||||
--arg haproxy_sel "{{ sel_rw.stdout | default("") | trim | replace("\"","\\\"") }}" \
|
--arg haproxy_sel "{{ sel_rw.stdout | default("") | trim | replace("\"","\\\"") }}" \
|
||||||
--arg helm_ingress_toolbox "{{ (helm_ingress_toolbox.stdout | default("")) | replace("\"","\\\"") }}" \
|
--arg helm_ingress_toolbox "{{ (helm_ingress_toolbox.stdout | default("")) | replace("\"","\\\"") }}" \
|
||||||
|
--arg istio_kiali "{{ (istio_kiali.stdout | default("")) | replace("\"","\\\"") }}" \
|
||||||
|
--arg istio_bookinfo "{{ (istio_bookinfo.stdout | default("")) | replace("\"","\\\"") }}" \
|
||||||
--arg k8s_overview "{{ (k8s_overview.stdout | default("")) | replace("\"","\\\"") }}" \
|
--arg k8s_overview "{{ (k8s_overview.stdout | default("")) | replace("\"","\\\"") }}" \
|
||||||
"{
|
"{
|
||||||
timestamp: $$time,
|
timestamp: $$time,
|
||||||
idempotence_raw: $$idemp,
|
idempotence_raw: $$idemp,
|
||||||
haproxy_select1: $$haproxy_sel,
|
haproxy_select1: $$haproxy_sel,
|
||||||
helm_ingress_toolbox_raw: $$helm_ingress_toolbox,
|
helm_ingress_toolbox_raw: $$helm_ingress_toolbox,
|
||||||
|
istio_kiali_raw: $$istio_kiali,
|
||||||
|
istio_bookinfo_raw: $$istio_bookinfo,
|
||||||
k8s_overview_raw: $$k8s_overview
|
k8s_overview_raw: $$k8s_overview
|
||||||
}" > /ansible/reports/lab-health.json
|
}" > /ansible/reports/lab-health.json
|
||||||
'
|
'
|
||||||
|
|||||||
12
scripts/cleanup.sh
Executable file
12
scripts/cleanup.sh
Executable file
@@ -0,0 +1,12 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Очистка лаборатории
|
||||||
|
# Автор: Сергей Антропов
|
||||||
|
# Сайт: https://devops.org.ru
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
echo "[cleanup] removing lab containers/volumes/networks"
|
||||||
|
docker ps -aq --filter "label=ansible.lab=true" | xargs -r docker rm -f
|
||||||
|
docker volume ls -q --filter "name=_docker$" | xargs -r docker volume rm
|
||||||
|
docker network rm labnet >/dev/null 2>&1 || true
|
||||||
|
echo "done."
|
||||||
143
scripts/report_html.py
Executable file
143
scripts/report_html.py
Executable file
@@ -0,0 +1,143 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
Генератор HTML отчетов для универсальной лаборатории
|
||||||
|
Автор: Сергей Антропов
|
||||||
|
Сайт: https://devops.org.ru
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import html
|
||||||
|
import datetime
|
||||||
|
|
||||||
|
def main():
|
||||||
|
if len(sys.argv) < 3:
|
||||||
|
print("Usage: report_html.py <input.json> <output.html>")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
inp, outp = sys.argv[1], sys.argv[2]
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(inp, 'r', encoding='utf-8') as f:
|
||||||
|
data = json.load(f)
|
||||||
|
except Exception as e:
|
||||||
|
data = {"error": f"failed to read json: {e}"}
|
||||||
|
|
||||||
|
ts = data.get("timestamp", datetime.datetime.utcnow().isoformat())
|
||||||
|
idemp_raw = data.get("idempotence_raw", "")
|
||||||
|
haproxy_sel = data.get("haproxy_select1", "")
|
||||||
|
helm_ingress_toolbox = data.get("helm_ingress_toolbox_raw", "")
|
||||||
|
istio_kiali = data.get("istio_kiali_raw", "")
|
||||||
|
istio_bookinfo = data.get("istio_bookinfo_raw", "")
|
||||||
|
k8s_overview = data.get("k8s_overview_raw", "")
|
||||||
|
|
||||||
|
def badge(label, ok):
|
||||||
|
color = "#10b981" if ok else "#ef4444"
|
||||||
|
return f'<span class="badge" style="background:{color}">{html.escape(label)}</span>'
|
||||||
|
|
||||||
|
# Анализ статусов
|
||||||
|
idempotent_ok = ("changed=0" in idemp_raw)
|
||||||
|
haproxy_ok = (haproxy_sel.strip() == "1") if haproxy_sel else None
|
||||||
|
ingress_ok = ("ingress" in helm_ingress_toolbox.lower()) or ("curl http://localhost" in helm_ingress_toolbox.lower())
|
||||||
|
toolbox_ok = ("deploy/toolbox" in helm_ingress_toolbox.lower()) or ("rollout status" in helm_ingress_toolbox.lower())
|
||||||
|
istio_ok = ("istio-system" in istio_kiali.lower()) and ("istiod" in istio_kiali.lower())
|
||||||
|
kiali_ok = ("kiali" in istio_kiali.lower())
|
||||||
|
bookinfo_ok = ("bookinfo" in istio_bookinfo.lower()) or ("productpage" in istio_bookinfo.lower())
|
||||||
|
k8s_ok = ("nodes" in k8s_overview.lower()) and ("pods" in k8s_overview.lower())
|
||||||
|
|
||||||
|
def maybe_badge(label, status):
|
||||||
|
if status is None:
|
||||||
|
return f'<span class="badge" style="background:#6b7280">{html.escape(label)}: n/a</span>'
|
||||||
|
return badge(f"{label}", bool(status))
|
||||||
|
|
||||||
|
html_doc = f"""<!doctype html>
|
||||||
|
<html lang="ru">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<title>Lab Report</title>
|
||||||
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||||
|
<style>
|
||||||
|
:root {{ --bg:#0f172a; --card:#111827; --muted:#94a3b8; --fg:#e5e7eb; --accent:#38bdf8; }}
|
||||||
|
* {{ box-sizing:border-box; }}
|
||||||
|
body {{ margin:0; font-family: ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Ubuntu, Cantarell, Noto Sans, Arial;
|
||||||
|
background:linear-gradient(180deg,#0b1220,#0f172a); color:var(--fg); }}
|
||||||
|
.container {{ max-width:1100px; margin:40px auto; padding:0 16px; }}
|
||||||
|
.hdr {{ display:flex; align-items:center; gap:14px; flex-wrap:wrap; }}
|
||||||
|
.hdr h1 {{ margin:0; font-size:28px; }}
|
||||||
|
.time {{ color:var(--muted); font-size:14px; }}
|
||||||
|
.grid {{ display:grid; grid-template-columns:1fr; gap:16px; margin-top:20px; }}
|
||||||
|
.card {{ background:rgba(17,24,39,.7); border:1px solid rgba(148,163,184,.15);
|
||||||
|
border-radius:14px; padding:16px; box-shadow:0 10px 30px rgba(0,0,0,.25); backdrop-filter: blur(6px); }}
|
||||||
|
.card h2 {{ margin:0 0 10px 0; font-size:18px; color:#c7d2fe; }}
|
||||||
|
pre {{ margin:0; padding:12px; background:#0b1220; border-radius:10px; overflow:auto; font-size:12px; line-height:1.4; }}
|
||||||
|
.badge {{ display:inline-block; padding:4px 10px; border-radius:999px; font-size:12px; color:white; }}
|
||||||
|
.kv {{ display:flex; flex-wrap:wrap; gap:8px; margin:8px 0 0 0; }}
|
||||||
|
footer {{ margin:24px 0 40px; color:var(--muted); font-size:12px; text-align:center; }}
|
||||||
|
a {{ color: var(--accent); text-decoration:none; }} a:hover {{ text-decoration:underline; }}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="container">
|
||||||
|
<div class="hdr">
|
||||||
|
<h1>Ansible Lab Report</h1>
|
||||||
|
<div class="time">generated: {html.escape(ts)}</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="grid">
|
||||||
|
<div class="card">
|
||||||
|
<h2>Summary</h2>
|
||||||
|
<div class="kv">
|
||||||
|
{badge("Idempotent", idempotent_ok)}
|
||||||
|
{maybe_badge("HAProxy SELECT=1", haproxy_ok)}
|
||||||
|
{maybe_badge("Ingress ready", ingress_ok)}
|
||||||
|
{maybe_badge("Toolbox ready", toolbox_ok)}
|
||||||
|
{maybe_badge("Istio ready", istio_ok)}
|
||||||
|
{maybe_badge("Kiali ready", kiali_ok)}
|
||||||
|
{maybe_badge("Bookinfo ready", bookinfo_ok)}
|
||||||
|
{maybe_badge("K8s ready", k8s_ok)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card">
|
||||||
|
<h2>Idempotence (raw)</h2>
|
||||||
|
<pre>{html.escape(idemp_raw) if idemp_raw else "n/a"}</pre>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card">
|
||||||
|
<h2>Helm + Ingress + Toolbox (raw)</h2>
|
||||||
|
<pre>{html.escape(helm_ingress_toolbox) if helm_ingress_toolbox else "n/a"}</pre>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card">
|
||||||
|
<h2>Istio / Kiali (raw)</h2>
|
||||||
|
<pre>{html.escape(istio_kiali) if istio_kiali else "n/a"}</pre>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card">
|
||||||
|
<h2>Istio Bookinfo (raw)</h2>
|
||||||
|
<pre>{html.escape(istio_bookinfo) if istio_bookinfo else "n/a"}</pre>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card">
|
||||||
|
<h2>K8s Overview (raw)</h2>
|
||||||
|
<pre>{html.escape(k8s_overview) if k8s_overview else "n/a"}</pre>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card">
|
||||||
|
<h2>HAProxy SELECT 1</h2>
|
||||||
|
<pre>{html.escape(haproxy_sel) if haproxy_sel else "n/a"}</pre>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<footer>HTML from /ansible/reports/lab-health.json · kubeconfigs in reports/kubeconfigs/</footer>
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>"""
|
||||||
|
|
||||||
|
with open(outp, "w", encoding="utf-8") as f:
|
||||||
|
f.write(html_doc)
|
||||||
|
print(outp)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
25
scripts/restore.sh
Executable file
25
scripts/restore.sh
Executable file
@@ -0,0 +1,25 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Восстановление лаборатории из снапшотов
|
||||||
|
# Автор: Сергей Антропов
|
||||||
|
# Сайт: https://devops.org.ru
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
IN_DIR="snapshots"
|
||||||
|
if [ ! -d "$IN_DIR" ]; then
|
||||||
|
echo "No snapshots dir"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
for f in "$IN_DIR"/*.image; do
|
||||||
|
if [ ! -f "$f" ]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
name=$(basename "$f" .image)
|
||||||
|
img=$(cat "$f")
|
||||||
|
echo "[restore] $name from $img"
|
||||||
|
docker rm -f "$name" >/dev/null 2>&1 || true
|
||||||
|
docker run -d --name "$name" "$img" >/dev/null
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "Restored from $IN_DIR/"
|
||||||
26
scripts/snapshot.sh
Executable file
26
scripts/snapshot.sh
Executable file
@@ -0,0 +1,26 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Снапшот лаборатории
|
||||||
|
# Автор: Сергей Антропов
|
||||||
|
# Сайт: https://devops.org.ru
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
OUT_DIR="snapshots"
|
||||||
|
mkdir -p "$OUT_DIR"
|
||||||
|
|
||||||
|
# Найти все контейнеры лаборатории
|
||||||
|
ids=$(docker ps -q --filter "label=ansible.lab=true")
|
||||||
|
if [ -z "$ids" ]; then
|
||||||
|
echo "No lab containers to snapshot"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
for id in $ids; do
|
||||||
|
name=$(docker inspect --format '{{.Name}}' "$id" | sed 's#^/##')
|
||||||
|
img="lab-snap-$name:latest"
|
||||||
|
echo "[snapshot] $name -> $img"
|
||||||
|
docker commit "$id" "$img" >/dev/null
|
||||||
|
echo "$img" > "$OUT_DIR/$name.image"
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "Snapshots saved to $OUT_DIR/"
|
||||||
Reference in New Issue
Block a user