ELK + Redis Docker 企业级部署落地方案

原创

已于 2025-12-15 11:55:56 修改 · 1.1k 阅读

标签

#jenkins #运维 #大数据 #elasticsearch #docker

于 2025-12-15 11:54:01 首次发布

ELK + Redis Docker 企业级部署落地方案

📋 方案概述

架构设计

🚀 一、准备工作

1.1 环境要求

# 最低硬件配置
- 内存: 16GB RAM (推荐32GB+)
- 存储: 200GB SSD (推荐NVMe)
- CPU: 8核 (推荐16核+)
- 网络: 千兆网卡

# 软件要求
- Docker 20.10+
- Docker Compose 2.0+
- 操作系统: Ubuntu 20.04/22.04, CentOS 8+

# 端口规划
┌─────────────┬─────────┬───────────────┐
│ 服务        │ 端口    │ 用途          │
├─────────────┼─────────┼───────────────┤
│ Elasticsearch│ 9200   │ HTTP API      │
│ Elasticsearch│ 9300   │ 节点通信      │
│ Kibana      │ 5601    │ Web界面       │
│ Logstash    │ 5044    │ Beats输入     │
│ Logstash    │ 5000    │ TCP输入       │
│ Redis       │ 6379    │ 缓存队列      │
│ Cerebro     │ 9000    │ ES集群管理    │
│ Prometheus  │ 9090    │ 监控          │
│ Grafana     │ 3000    │ 可视化        │
└─────────────┴─────────┴───────────────┘

1.2 目录结构

elk-docker-enterprise/
├── .env                    # 环境变量配置
├── docker-compose.yml     # 主编排文件
├── README.md              # 部署文档
├── LICENSE
├── scripts/               # 维护脚本
│   ├── init.sh
│   ├── backup.sh
│   ├── restore.sh
│   └── health-check.sh
├── configs/               # 配置文件
│   ├── elasticsearch/
│   │   ├── elasticsearch.yml
│   │   ├── jvm.options
│   │   └── log4j2.properties
│   ├── kibana/
│   │   └── kibana.yml
│   ├── logstash/
│   │   ├── logstash.yml
│   │   ├── pipelines.yml
│   │   ├── patterns/      # Grok模式
│   │   └── conf.d/        # 管道配置
│   ├── filebeat/
│   │   └── filebeat.yml
│   └── redis/
│       └── redis.conf
├── data/                  # 持久化数据
│   ├── elasticsearch/
│   ├── redis/
│   ├── logstash/
│   └── prometheus/
├── logs/                  # 容器日志
├── certificates/          # TLS证书
│   ├── ca.crt
│   ├── elasticsearch.crt
│   └── elasticsearch.key
└── grafana/
    ├── dashboards/
    └── provisioning/

1.3 创建部署目录

# 创建项目目录结构
mkdir -p elk-docker-enterprise/{configs,data,logs,scripts,certificates,grafana}
cd elk-docker-enterprise

# 创建子目录
mkdir -p configs/{elasticsearch,kibana,logstash/{patterns,conf.d},filebeat,redis}
mkdir -p data/{elasticsearch,redis,logstash,prometheus}
mkdir -p grafana/{dashboards,provisioning}
mkdir -p certificates
mkdir -p logs/{elasticsearch,kibana,logstash,filebeat,redis}

🐳 二、Docker Compose 编排文件

2.1 环境变量文件 (`.env`)

# ELK 版本配置
ELK_VERSION=8.11.0
REDIS_VERSION=7.2

# 网络配置
NETWORK_NAME=elk-network
SUBNET=172.20.0.0/16

# Elasticsearch 配置
ES_CLUSTER_NAME=elk-enterprise-cluster
ES_NODE_NAME=node-1
ES_DISCOVERY_TYPE=single-node
ES_HEAP_SIZE=4g
ES_JAVA_OPTS=-Xms4g -Xmx4g

# 安全配置
ELASTIC_PASSWORD=YourSecurePassword123!
KIBANA_PASSWORD=KibanaSecurePass456!
LOGSTASH_SYSTEM_PASSWORD=LogstashSystem789!
BEATS_SYSTEM_PASSWORD=BeatsSystemPass321!

# Redis 配置
REDIS_PASSWORD=RedisSecurePass654!
REDIS_MAXMEMORY=2gb

# 存储路径
ELK_DATA_PATH=/data/elk
ELK_LOGS_PATH=/var/log/elk

# 资源限制
ES_CPU_LIMIT=4
ES_MEMORY_LIMIT=8g
LOGSTASH_CPU_LIMIT=2
LOGSTASH_MEMORY_LIMIT=4g
KIBANA_CPU_LIMIT=1
KIBANA_MEMORY_LIMIT=2g

# 节点数量
ES_NODES_COUNT=3
LOGSTASH_NODES_COUNT=2
KIBANA_NODES_COUNT=2

2.2 Docker Compose 主文件 (`docker-compose.yml`)

version: '3.8'

networks:
  elk-network:
    driver: bridge
    ipam:
      config:
        - subnet: ${SUBNET:-172.20.0.0/16}
    attachable: true

volumes:
  elasticsearch-data:
    driver: local
    driver_opts:
      type: none
      o: bind
      device: ${ELK_DATA_PATH:-./data}/elasticsearch
  redis-data:
    driver: local
    driver_opts:
      type: none
      o: bind
      device: ${ELK_DATA_PATH:-./data}/redis
  prometheus-data:
    driver: local

services:
  # ============== Redis 集群（缓冲队列）==============
  redis-master:
    image: redis:${REDIS_VERSION:-7.2}-alpine
    container_name: redis-master
    hostname: redis-master
    restart: unless-stopped
    command: redis-server /usr/local/etc/redis/redis.conf --requirepass ${REDIS_PASSWORD}
    environment:
      - REDIS_REPLICATION_MODE=master
    volumes:
      - ./configs/redis/redis.conf:/usr/local/etc/redis/redis.conf
      - redis-data:/data
      - ./logs/redis:/var/log/redis
    ports:
      - "6379:6379"
    networks:
      elk-network:
        ipv4_address: 172.20.0.10
    healthcheck:
      test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"]
      interval: 10s
      timeout: 5s
      retries: 3
    deploy:
      resources:
        limits:
          cpus: '1'
          memory: 2G
        reservations:
          cpus: '0.5'
          memory: 1G

  redis-replica:
    image: redis:${REDIS_VERSION:-7.2}-alpine
    container_name: redis-replica
    hostname: redis-replica
    restart: unless-stopped
    command: redis-server /usr/local/etc/redis/redis.conf --requirepass ${REDIS_PASSWORD} --slaveof redis-master 6379
    environment:
      - REDIS_REPLICATION_MODE=slave
      - REDIS_MASTER_HOST=redis-master
      - REDIS_MASTER_PORT_NUMBER=6379
      - REDIS_MASTER_PASSWORD=${REDIS_PASSWORD}
    volumes:
      - ./configs/redis/redis.conf:/usr/local/etc/redis/redis.conf
    depends_on:
      - redis-master
    networks:
      elk-network:
        ipv4_address: 172.20.0.11
    healthcheck:
      test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"]
      interval: 10s
      timeout: 5s
      retries: 3

  # ============== Elasticsearch 集群 ==============
  elasticsearch-node1:
    image: docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
    container_name: elasticsearch-node1
    hostname: elasticsearch-node1
    restart: unless-stopped
    environment:
      - node.name=elasticsearch-node1
      - cluster.name=${ES_CLUSTER_NAME:-elk-enterprise-cluster}
      - cluster.initial_master_nodes=elasticsearch-node1,elasticsearch-node2,elasticsearch-node3
      - discovery.seed_hosts=elasticsearch-node2,elasticsearch-node3
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=${ES_JAVA_OPTS:--Xms4g -Xmx4g}"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=false
      - xpack.license.self_generated.type=basic
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
      - LOGSTASH_SYSTEM_PASSWORD=${LOGSTASH_SYSTEM_PASSWORD}
      - KIBANA_SYSTEM_PASSWORD=${KIBANA_PASSWORD}
      - BEATS_SYSTEM_PASSWORD=${BEATS_SYSTEM_PASSWORD}
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 65536
        hard: 65536
    volumes:
      - elasticsearch-data:/usr/share/elasticsearch/data
      - ./configs/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - ./configs/elasticsearch/jvm.options:/usr/share/elasticsearch/config/jvm.options
      - ./logs/elasticsearch:/usr/share/elasticsearch/logs
    ports:
      - "9200:9200"
      - "9300:9300"
    networks:
      elk-network:
        ipv4_address: 172.20.0.20
    healthcheck:
      test: ["CMD-SHELL", "curl -s -u elastic:${ELASTIC_PASSWORD} http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"'"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 60s
    deploy:
      resources:
        limits:
          cpus: '${ES_CPU_LIMIT:-4}'
          memory: ${ES_MEMORY_LIMIT:-8g}
        reservations:
          cpus: '${ES_CPU_LIMIT:-2}'
          memory: ${ES_MEMORY_LIMIT:-4g}

  elasticsearch-node2:
    image: docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
    container_name: elasticsearch-node2
    hostname: elasticsearch-node2
    restart: unless-stopped
    environment:
      - node.name=elasticsearch-node2
      - cluster.name=${ES_CLUSTER_NAME:-elk-enterprise-cluster}
      - cluster.initial_master_nodes=elasticsearch-node1,elasticsearch-node2,elasticsearch-node3
      - discovery.seed_hosts=elasticsearch-node1,elasticsearch-node3
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=${ES_JAVA_OPTS:--Xms4g -Xmx4g}"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=false
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - elasticsearch-data-node2:/usr/share/elasticsearch/data
      - ./configs/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - ./logs/elasticsearch-node2:/usr/share/elasticsearch/logs
    networks:
      elk-network:
        ipv4_address: 172.20.0.21
    depends_on:
      - elasticsearch-node1
    healthcheck:
      test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"'"]
      interval: 30s
      timeout: 10s
      retries: 3

  elasticsearch-node3:
    image: docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
    container_name: elasticsearch-node3
    hostname: elasticsearch-node3
    restart: unless-stopped
    environment:
      - node.name=elasticsearch-node3
      - cluster.name=${ES_CLUSTER_NAME:-elk-enterprise-cluster}
      - cluster.initial_master_nodes=elasticsearch-node1,elasticsearch-node2,elasticsearch-node3
      - discovery.seed_hosts=elasticsearch-node1,elasticsearch-node2
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=${ES_JAVA_OPTS:--Xms4g -Xmx4g}"
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=false
      - xpack.security.transport.ssl.enabled=false
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - elasticsearch-data-node3:/usr/share/elasticsearch/data
      - ./configs/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - ./logs/elasticsearch-node3:/usr/share/elasticsearch/logs
    networks:
      elk-network:
        ipv4_address: 172.20.0.22
    depends_on:
      - elasticsearch-node1
    healthcheck:
      test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"'"]
      interval: 30s
      timeout: 10s
      retries: 3

  # ============== Logstash 集群 ==============
  logstash-node1:
    image: docker.elastic.co/logstash/logstash:${ELK_VERSION}
    container_name: logstash-node1
    hostname: logstash-node1
    restart: unless-stopped
    environment:
      - LS_JAVA_OPTS=-Xmx2g -Xms2g
      - XPACK_MONITORING_ENABLED=true
      - XPACK_MONITORING_ELASTICSEARCH_URL=http://elasticsearch-node1:9200
      - XPACK_MONITORING_ELASTICSEARCH_USERNAME=logstash_system
      - XPACK_MONITORING_ELASTICSEARCH_PASSWORD=${LOGSTASH_SYSTEM_PASSWORD}
      - ELASTICSEARCH_HOSTS=http://elasticsearch-node1:9200
    volumes:
      - ./configs/logstash/logstash.yml:/usr/share/logstash/config/logstash.yml
      - ./configs/logstash/pipelines.yml:/usr/share/logstash/config/pipelines.yml
      - ./configs/logstash/conf.d:/usr/share/logstash/pipeline
      - ./configs/logstash/patterns:/usr/share/logstash/patterns
      - ./logs/logstash:/usr/share/logstash/logs
    ports:
      - "5044:5044"   # Beats input
      - "5000:5000"   # TCP input
      - "5001:5001"   # UDP input
      - "9600:9600"   # Logstash monitoring API
    networks:
      elk-network:
        ipv4_address: 172.20.0.30
    depends_on:
      - elasticsearch-node1
      - redis-master
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9600"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 60s
    deploy:
      resources:
        limits:
          cpus: '${LOGSTASH_CPU_LIMIT:-2}'
          memory: ${LOGSTASH_MEMORY_LIMIT:-4g}
        reservations:
          cpus: '1'
          memory: 2g

  logstash-node2:
    image: docker.elastic.co/logstash/logstash:${ELK_VERSION}
    container_name: logstash-node2
    hostname: logstash-node2
    restart: unless-stopped
    environment:
      - LS_JAVA_OPTS=-Xmx2g -Xms2g
      - XPACK_MONITORING_ENABLED=true
      - XPACK_MONITORING_ELASTICSEARCH_URL=http://