ELK不香了?企业级日志平台后起之秀 Graylog
架构之美
共 6079字,需浏览 13分钟
·
2021-12-12 23:14
- 前言 -
当我们公司内部部署很多服务以及测试、正式环境的时候,查看日志就变成了一个非常刚需的需求了。
- Filebeat 工具介绍 -
- Filebeat 配置文件 -
# 配置输入来源的日志信息
# 我们合理将其配置到了 inputs.d 目录下的所有 yml 文件
filebeat.config.inputs:
enabled: true
path: ${path.config}/inputs.d/*.yml
# 若收取日志格式为 json 的 log 请开启此配置
# json.keys_under_root: true
# 配置 Filebeat 需要加载的模块
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 1
# 配置将日志信息发送那个地址上面
output.logstash:
hosts: ["11.22.33.44:5500"]
# output.file:
# enable: true
processors:
- add_host_metadata: ~
- rename:
fields:
- from: "log"
to: "message"
- add_fields:
target: ""
fields:
# 加 Token 是为了防止无认证的服务上 Graylog 服务发送数据
token: "0uxxxxaM-1111-2222-3333-VQZJxxxxxwgX "
# 收集的数据类型
- type: log
enabled: true
# 日志文件的路径地址
paths:
- /var/log/supervisor/app_escape_worker-stderr.log
- /var/log/supervisor/app_escape_prod-stderr.log
symlinks: true
# 包含的关键字信息
include_lines: ["WARNING", "ERROR"]
# 打上数据标签
tags: ["app", "escape", "test"]
# 防止程序堆栈信息被分行识别
multiline.pattern: '^\[?[0-9]...{3}'
multiline.negate: true
multiline.match: after
# 需要配置多个日志时可加多个 type 字段
- type: log
enabled: true
......
# iptables
- module: iptables
log:
enabled: true
var.paths: ["/var/log/iptables.log"]
var.input: "file"
# postgres
- module: postgresql
log:
enabled: true
var.paths: ["/path/to/log/postgres/*.log*"]
# nginx
- module: nginx
access:
enabled: true
var.paths: ["/path/to/log/nginx/access.log*"]
error:
enabled: true
var.paths: ["/path/to/log/nginx/error.log*"]
- Graylog 服务介绍 -
- Graylog 组件功能 -
rule "discard debug messages"
when
to_long($message.level) > 6
then
drop_message();
end
- 服务安装和部署 -
# Ubuntu(deb)
$ curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.8.1-amd64.deb
$ sudo dpkg -i filebeat-7.8.1-amd64.deb
$ sudo systemctl enable filebeat
$ sudo service filebeat start
# 使用 Docker 启动
docker run -d --name=filebeat --user=root \
--volume="./filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro" \
--volume="/var/lib/docker/containers:/var/lib/docker/containers:ro" \
--volume="/var/run/docker.sock:/var/run/docker.sock:ro" \
docker.elastic.co/beats/filebeat:7.8.1 filebeat -e -strict.perms=false \
-E output.elasticsearch.hosts=["elasticsearch:9200"]
# 生成 password_secret 密码(最少 16 位)
$ sudo apt install -y pwgen
$ pwgen -N 1 -s 16
zscMb65...FxR9ag
# 生成后续 Web 登录时所需要使用的密码
$ echo -n "Enter Password: " && head -1 '\n' | sha256sum | cut -d" " -f1
Enter Password: zscMb65...FxR9ag
77e29e0f...557515f
version: "3"
services:
mongo:
restart: on-failure
container_name: graylog_mongo
image: "mongo:3"
volumes:
- "./mongodb:/data/db"
networks:
- graylog_network
elasticsearch:
restart: on-failure
container_name: graylog_es
image: "elasticsearch:6.8.5"
volumes:
- "./es_data:/usr/share/elasticsearch/data"
environment:
- http.host=0.0.0.0
- transport.host=localhost
- network.host=0.0.0.0
- "ES_JAVA_OPTS=-Xms512m -Xmx5120m"
ulimits:
memlock:
soft: -1
hard: -1
deploy:
resources:
limits:
memory: 12g
networks:
- graylog_network
graylog:
restart: on-failure
container_name: graylog_web
image: "graylog/graylog:3.3"
ports:
- 9000:9000 # Web 服务提供的访问端口
- 5044:5044 # Filebeat 工具提供端口
- 12201:12201 # GELF TCP
- 12201:12201/udp # GELF UDP
- 1514:1514 # Syslog TCP
- 1514:1514/udp # Syslog UDP
volumes:
- "./graylog_journal:/usr/share/graylog/data/journal"
environment:
- GRAYLOG_PASSWORD_SECRET=zscMb65...FxR9ag
- GRAYLOG_ROOT_PASSWORD_SHA2=77e29e0f...557515f
- GRAYLOG_HTTP_EXTERNAL_URI=http://11.22.33.44:9000/
- GRAYLOG_TIMEZONE=Asia/Shanghai
- GRAYLOG_ROOT_TIMEZONE=Asia/Shanghai
networks:
- graylog
depends_on:
- mongo
- elasticsearch
networks:
graylog_network:
driver: bridge
# [docker] 启动容器指定地址和 driver
docker run --rm=true \
--log-driver=gelf \
--log-opt gelf-address=udp://11.22.33.44:12201 \
--log-opt tag=myapp \
myapp:0.0.1
# [docker-compose] 启动使用方式
version: "3"
services:
redis:
restart: always
image: redis
container_name: "redis"
logging:
driver: gelf
options:
gelf-address: udp://11.22.33.44:12201
tag: "redis"
......
- Graylog 界面功能 -
作者:Escape
来源:escapelife.site/posts/38c81b25.html
评论