SensusInfra/nginx/clickhouse-lb.conf

# Автор: Сергей Антропов, сайт: https://devops.org.ru
# Назначение: Конфигурация Nginx для балансировки нагрузки ClickHouse
# Балансировка между двумя репликами ClickHouse

user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log notice;
pid /var/run/nginx.pid;

events {
    worker_connections 1024;
    use epoll;
    multi_accept on;
}

# HTTP балансировка для ClickHouse HTTP интерфейса
http {
    include /etc/nginx/mime.types;
    default_type application/octet-stream;

    # Логирование
    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
                    '$status $body_bytes_sent "$http_referer" '
                    '"$http_user_agent" "$http_x_forwarded_for"';

    access_log /var/log/nginx/access.log main;

    # Основные настройки
    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 65;
    types_hash_max_size 2048;
    client_max_body_size 100M;

    # Gzip сжатие
    gzip on;
    gzip_vary on;
    gzip_min_length 1024;
    gzip_proxied any;
    gzip_comp_level 6;
    gzip_types
        text/plain
        text/css
        text/xml
        text/javascript
        application/json
        application/javascript
        application/xml+rss
        application/atom+xml
        image/svg+xml;

    # Upstream для ClickHouse HTTP интерфейса
    upstream clickhouse_http {
        # Балансировка по round-robin
        least_conn;
        
        # ClickHouse реплика 1
        server clickhouse-1:8123 max_fails=3 fail_timeout=30s weight=1;
        
        # ClickHouse реплика 2
        server clickhouse-2:8123 max_fails=3 fail_timeout=30s weight=1;
        
        # Резервные серверы (если основные недоступны)
        keepalive 32;
    }

    # Upstream для ClickHouse Native протокола (TCP)
    upstream clickhouse_native {
        # ClickHouse реплика 1
        server clickhouse-1:9000 max_fails=3 fail_timeout=30s weight=1;
        
        # ClickHouse реплика 2
        server clickhouse-2:9000 max_fails=3 fail_timeout=30s weight=1;
        
        keepalive 32;
    }

    # Основной сервер для HTTP интерфейса ClickHouse
    server {
        listen 80;
        server_name _;

        # Health check endpoint
        location /health {
            access_log off;
            return 200 "healthy\n";
            add_header Content-Type text/plain;
        }

        # Статус балансировщика
        location /status {
            access_log off;
            return 200 "ClickHouse Load Balancer is running\n";
            add_header Content-Type text/plain;
        }

        # Проксирование всех запросов к ClickHouse HTTP интерфейсу
        location / {
            # Проксирование к ClickHouse
            proxy_pass http://clickhouse_http;
            
            # Заголовки для ClickHouse
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            
            # Настройки таймаутов
            proxy_connect_timeout 5s;
            proxy_send_timeout 60s;
            proxy_read_timeout 60s;
            
            # Буферизация
            proxy_buffering on;
            proxy_buffer_size 4k;
            proxy_buffers 8 4k;
            proxy_busy_buffers_size 8k;
            
            # Обработка ошибок
            proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
            proxy_next_upstream_tries 2;
            proxy_next_upstream_timeout 10s;
            
            # Поддержка WebSocket (если потребуется)
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "upgrade";
        }

        # Специальный endpoint для ping
        location /ping {
            proxy_pass http://clickhouse_http/ping;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
        }

        # Endpoint для получения информации о сервере
        location /info {
            proxy_pass http://clickhouse_http/info;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
        }
    }
}

# TCP балансировка для ClickHouse Native протокола
stream {
    # Логирование для stream
    error_log /var/log/nginx/stream_error.log;

    # Upstream для ClickHouse Native протокола
    upstream clickhouse_native_backend {
        # ClickHouse реплика 1
        server clickhouse-1:9000 max_fails=3 fail_timeout=30s weight=1;
        
        # ClickHouse реплика 2
        server clickhouse-2:9000 max_fails=3 fail_timeout=30s weight=1;
    }

    # TCP сервер для ClickHouse Native протокола
    server {
        listen 9000;
        proxy_pass clickhouse_native_backend;
        proxy_timeout 1s;
        proxy_responses 1;
        proxy_bind $remote_addr transparent;
    }
}