一、使用filebeat采集应用日志

1.1 使用filebeat采集nginx日志

1.1.1 搭建nginx环境

1、在elk123节点上添加yum源

[root@elk123 ~]# cat > /etc/yum.repos.d/nginx.repo <<'EOF'
[nginx-stable]
name=nginx stable repo
baseurl=http://nginx.org/packages/centos/$releasever/$basearch/
gpgcheck=1
enabled=1
gpgkey=https://nginx.org/keys/nginx_signing.key
module_hotfixes=true
[nginx-mainline]
name=nginx mainline repo
baseurl=http://nginx.org/packages/mainline/centos/$releasever/$basearch/
gpgcheck=1
enabled=0
gpgkey=https://nginx.org/keys/nginx_signing.key
module_hotfixes=true
EOF

2、在elk123节点上安装nginx

[root@elk123 ~]# yum -y install nginx
[root@elk123 ~]# systemctl start nginx

3、在浏览器输入http://192.168.1.123进行访问测试

1.1.2 使用filebeat采集nginx日志

1、在elk123节点上修改filebeat配置文件

[root@elk123 ~]# vim /es/softwares/filebeat-7.17.5-linux-x86_64/config/06-log_nginx-to-console.yaml
filebeat.inputs:
- type: log
paths:
- /var/log/nginx/access.log*
output.console:
# 表示输出的内容以漂亮的格式显示
pretty: true

2、在elk123节点上启动filebeat的实例

[root@elk123 ~]# filebeat -e -c /es/softwares/filebeat-7.17.5-linux-x86_64/config/06-log_nginx-to-console.yaml

相关参数说明：

  • -e: 这个选项表示在前台运行，而不是作为后台服务。使用 -e 选项，Filebeat 将输出日志信息到终端，便于调试和查看实时日志。

  • -c：这个选项后面通常跟着指定配置文件的路径。

1.1.3 使用filebeat采集nginx的json格式日志

1、在elk123节点上修改nginx配置文件

注释第18行-第22行内容

18 #log_format main '$remote_addr - $remote_user [$time_local] "$request" '
19 # '$status $body_bytes_sent "$http_referer" '
20 # '"$http_user_agent" "$http_x_forwarded_for"';
21
22 #access_log /var/log/nginx/access.log main;

在第23行新填如下内容

log_format nginx_json '{"@timestamp":"$time_iso8601",'
'"host":"$server_addr",'
'"clientip":"$remote_addr",'
'"SendBytes":$body_bytes_sent,'
'"responsetime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"upstreamhost":"$upstream_addr",'
'"http_host":"$host",'
'"uri":"$uri",'
'"domain":"$host",'
'"xff":"$http_x_forwarded_for",'
'"referer":"$http_referer",'
'"tcp_xff":"$proxy_protocol_addr",'
'"http_user_agent":"$http_user_agent",'
'"status":"$status"}';
access_log /var/log/nginx/access.log nginx_json;

修改完成后，内容如下：

[root@elk123 ~]# cat /etc/nginx/nginx.conf
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log notice;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
#log_format main '$remote_addr - $remote_user [$time_local] "$request" '
# '$status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for"';
#access_log /var/log/nginx/access.log main;
log_format nginx_json '{"@timestamp":"$time_iso8601",'
'"host":"$server_addr",'
'"clientip":"$remote_addr",'
'"SendBytes":$body_bytes_sent,'
'"responsetime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"upstreamhost":"$upstream_addr",'
'"http_host":"$host",'
'"uri":"$uri",'
'"domain":"$host",'
'"xff":"$http_x_forwarded_for",'
'"referer":"$http_referer",'
'"tcp_xff":"$proxy_protocol_addr",'
'"http_user_agent":"$http_user_agent",'
'"status":"$status"}';
access_log /var/log/nginx/access.log nginx_json;
sendfile on;
#tcp_nopush on;
keepalive_timeout 65;
#gzip on;
include /etc/nginx/conf.d/*.conf;
}

2、在elk123节点上热加载nginx

[root@elk123 ~]# nginx -t
[root@elk123 ~]# systemctl reload nginx

3、在elk123节点上清除之前访问的日志

[root@elk123 ~]# > /var/log/nginx/access.log

4、在elk123节点上测试访问nginx

[root@elk123 ~]# curl http://192.168.1.123/

查看访问日志，观察到已转换为json格式

[root@elk123 ~]# cat /var/log/nginx/access.log
{"@timestamp":"2024-01-12T20:30:39+08:00","host":"192.168.1.123","clientip":"192.168.1.123","SendBytes":615,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"192.168.1.123","uri":"/index.html","domain":"192.168.1.123","xff":"-","referer":"-","tcp_xff":"-","http_user_agent":"curl/7.29.0","status":"200"}

5、在elk123节点上修改filebeat配置文件

[root@elk123 ~]# vim /es/softwares/filebeat-7.17.5-linux-x86_64/config/07-log_nginx_json-to-console.yaml
filebeat.inputs:
- type: log
paths:
- /var/log/nginx/access.log*
#将message字段的json数据格式进行解析，并将解析的结果放在顶级字段中
json.keys_under_root: true
#如果解析json格式失败，则会将错误信息添加为一个"error"字段输出
json.add_error_key: true
output.console:
# 表示输出的内容以漂亮的格式显示
pretty: true

6、在elk123节点上启动filebeat实例

[root@elk123 ~]# filebeat -e -c /es/softwares/filebeat-7.17.5-linux-x86_64/config/07-log_nginx_json-to-console.yaml

1.2 使用filebeat采集tomcat日志

1.2.1 搭建tomcat环境

说明：此时环境是已经完成前面ES部署，本身存在JDK环境

1、下载tomcat软件包

下载链接：https://archive.apache.org/dist/tomcat/tomcat-9/v9.0.73/bin/

2、在elk123节点上解压软件包

[root@elk123 ~]# tar xf apache-tomcat-9.0.73.tar.gz -C /es/softwares/

3、在elk123节点上修改tomcat的配置文件

修改第151行到第167行内容

[root@elk123 ~]# cd /es/softwares/apache-tomcat-9.0.73/conf
[root@elk123 conf]# vim +151 server.xml
<Host name="www.tomcat.com" appBase="webapps"
unpackWARs="true" autoDeploy="true">
<Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
prefix="www.tomcat.com_access_log" suffix=".txt"
pattern="{&quot;clientip&quot;:&quot;%h&quot;,&quot;ClientUser&quot;:&quot;%l&quot;,&quot;authenticated&quot;:&quot;%u&quot;,&quot;AccessTime&quot;:&quot;%t&quot;,&quot;request&quot;:&quot;%r&quot;,&quot;status&quot;:&quot;%s&quot;,&quot;SendBytes&quot;:&quot;%b&quot;,&quot;Query?string&quot;:&quot;%q&quot;,&quot;partner&quot;:&quot;%{Referer}i&quot;,&quot;http_user_agent&quot;:&quot;%{User-Agent}i&quot;}"/>
</Host>

4、在elk123节点上配置环境变量并启动tomcat服务

[root@elk123 ~]# cd /es/softwares/apache-tomcat-9.0.73/conf
[root@elk123 conf]# vim /etc/profile.d/tomcat.sh
#!/bin/bash
export TOMCAT_HOME=/es/softwares/apache-tomcat-9.0.73
export PATH=$PATH:$TOMCAT_HOME/bin

配置文件生效

[root@elk123 conf]# source /etc/profile.d/tomcat.sh

启动tomcat服务

[root@elk123 ~]# cd /es/softwares/apache-tomcat-9.0.73/bin
[root@elk123 bin]# catalina.sh start

5、在elk123节点上查看tomcat访问日志文件

测试访问

[root@elk123 logs]# curl -H "Host: www.tomcat.com" http://192.168.1.123:8080

查看访问日志

[root@elk123 ~]# cd /es/softwares/apache-tomcat-9.0.73/logs
[root@elk123 logs]# cat www.tomcat.com_access_log.2024-01-12.txt
{"clientip":"192.168.1.123","ClientUser":"-","authenticated":"-","AccessTime":"[12/Jan/2024:22:15:48 +0800]","request":"GET / HTTP/1.1","status":"200","SendBytes":"11230","Query?string":"","partner":"-","http_user_agent":"curl/7.29.0"}

1.2.2 使用filebeat采集tomcat访问日志

1、在elk123节点上修改filebeat配置文件

[root@elk123 ~]# cd /es/softwares/filebeat-7.17.5-linux-x86_64/
[root@elk123 filebeat-7.17.5-linux-x86_64]# vim config/08-log_tomcat-to-console.yaml
filebeat.inputs:
- type: log
paths:
- /es/softwares/apache-tomcat-9.0.73/logs/www.tomcat.com_access_log*.txt
json.keys_under_root: true
json.add_error_key: true
output.console:
# 表示输出的内容以漂亮的格式显示
pretty: true

2、在elk123节点上启动filebeat实例

[root@elk123 filebeat-7.17.5-linux-x86_64]# rm -rf data/
[root@elk123 filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/08-log_tomcat-to-console.yaml

1.2.3 使用filebeat采集tomcat的错误日志多行匹配案例

1、在elk123节点上准备错误日志

故意改错tomcat的配置文件

[root@elk123 ~]# cd /es/softwares/apache-tomcat-9.0.73/conf/
[root@elk123 conf]# catalina.sh stop
[root@elk123 conf]# vim +168 server.xml
…
…
</Host1111>
….
…

修改后启动tomcat

[root@elk123 ~]# cd /es/softwares/apache-tomcat-9.0.73/bin
[root@elk123 bin]# catalina.sh start

查看启动错误日志

[root@elk123 conf]# cd /es/softwares/apache-tomcat-9.0.73/logs
[root@elk123 logs]# tail -f 20 catalina.out
tail: 无法打开"20" 读取数据: 没有那个文件或目录
==> catalina.out <==
at org.apache.catalina.startup.Catalina.parseServerXml(Catalina.java:617)
at org.apache.catalina.startup.Catalina.load(Catalina.java:709)
at org.apache.catalina.startup.Catalina.load(Catalina.java:746)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.catalina.startup.Bootstrap.load(Bootstrap.java:307)
at org.apache.catalina.startup.Bootstrap.main(Bootstrap.java:477)
12-Jan-2024 22:31:15.381 严重 [main] org.apache.catalina.startup.Catalina.start 无法启动服务器，服务器实例未配置

还原正确配置，正常启动tomcat

[root@elk123 ~]# cd /es/softwares/apache-tomcat-9.0.73/bin
[root@elk123 bin]# catalina.sh start

2、修改filebeat配置文件

[root@elk123 config]# cd /es/softwares/filebeat-7.17.5-linux-x86_64
[root@elk123 filebeat-7.17.5-linux-x86_64]# vim config/09-log-tomcat_error-to-es.yaml
filebeat.inputs:
- type: log
paths:
- /es/softwares/apache-tomcat-9.0.73/logs/catalina*
multiline.type: pattern
multiline.pattern: '^\d{2}'
multiline.negate: true
multiline.match: after
# 指定输出端为ES集群
output.elasticsearch:
hosts: ["http://192.168.1.121:9200","http://192.168.1.122:9200","http://192.168.1.123:9200"]

3、在elk123节点上启动filebeat实例

[root@elk123 filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/09-log-tomcat_error-to-es.yaml

4、打开ES Head扩展程序查看索引名为filebeat-7.17.5-2024.01.12-000001

5、在Postman工具上创建GET请求，输入192.168.1.121:9200/filebeat-7.17.5-2024.01.12-000001/_search，进行查询，观察到at错误合并成一行

{
    "query":{
        "match":{
            "message": "at"
        }
    }
}

1.3 使用filebeat采集docker日志(7.2版本后弃用)

1.3.1 搭建docker环境

1、找一台可以联网的linux，下载docker的RPM依赖包而不进行安装

(1)添加软件源信息

yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

(2)更新 yum 缓存

yum makecache fast

(3)下载所有安装依赖到指定路径下

mkdir -p /home/dockerrpm/
yum install docker-ce-23.0.1 --downloadonly --downloaddir=/home/dockerrpm/

(4)将所有依赖的rpm环境打包

tar zcvf docker-ce-23_0_1.tar.gz /home/dockerrpm/*

2、将docker-ce-23_0_1.tar.gz上传到elk123主机上进行安装

[root@elk123 ~]# tar xf docker-ce-23_0_1.tar.gz
[root@elk123 ~]# yum -y localinstall /home/dockerrpm/*.rpm

3、在elk123节点上配置docker的镜像加速

[root@elk123 ~]# vim /etc/docker/daemon.json
{
"data-root": "/var/lib/docker",
"registry-mirrors": ["https://y0araofw.mirror.aliyuncs.com","https://hub-mirror.c.1com/","https://docker.mirrors.ustc.edu.cn","https://reg-mirror.qiniu.com"]
}

4、在elk123节点上启动docker

[root@elk123 ~]# systemctl enable --now docker

1.3.2 使用filebeat采集docker容器日志

官方链接：https://elastic.co/guide/en/beats/filebeat/7.17/filebeat-input-docker.html

1、在elk123节点上分别启动nginx容器和tomcat容器

[root@elk123 ~]# docker run -dp 88:80 --name mynginx --restart always nginx:1.22.1-alpine
[root@elk123 ~]# docker run -dp 89:8080 --name mytomcat --restart always tomcat:jre8-alpine

验证

[root@elk123 filebeat-7.17.5-linux-x86_64]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
6021e06223cb tomcat:jre8-alpine "catalina.sh run" 5 minutes ago Up 5 minutes 0.0.0.0:89->8080/tcp, :::89->8080/tcp mytomcat
6dbcb54af989 nginx:1.22.1-alpine "/docker-entrypoint.…" 10 minutes ago Up 10 minutes 0.0.0.0:88->80/tcp, :::88->80/tcp mynginx

2、在elk123节点上使用filebeat采集容器日志

[root@elk123 ~]# cd /es/softwares/filebeat-7.17.5-linux-x86_64
[root@elk123 filebeat-7.17.5-linux-x86_64]# vim config/11-docker-to-console.yaml
filebeat.inputs:
# 指定输入类型为docker类型
- type: docker
#默认监控容器路径为/var/lib/docker/containers/
# 指定容器的ID，这里指监控所有容器
containers.ids:
- '*'
output.console:
pretty: true

3、在elk123节点上测试访问nginx和tomcat

[root@elk123 ~]# curl 192.168.1.123:88
[root@elk123 ~]# curl 192.168.1.123:89

4、在elk123节点上启动filebeat实例

[root@elk123 filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/11-docker-to-console.yaml

1.3.3 使用filebeat采集container日志

官方文档：https://www.elastic.co/guide/en/beats/filebeat/7.17/filebeat-input-container.html

1、在elk123节点上分别启动nginx容器和tomcat容器

[root@elk123 ~]# docker run -dp 88:80 --name mynginx --restart always nginx:1.22.1-alpine
[root@elk123 ~]# docker run -dp 89:8080 --name mytomcat --restart always tomcat:jre8-alpine

验证

[root@elk123 filebeat-7.17.5-linux-x86_64]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
6021e06223cb tomcat:jre8-alpine "catalina.sh run" 5 minutes ago Up 5 minutes 0.0.0.0:89->8080/tcp, :::89->8080/tcp mytomcat
6dbcb54af989 nginx:1.22.1-alpine "/docker-entrypoint.…" 10 minutes ago Up 10 minutes 0.0.0.0:88->80/tcp, :::88->80/tcp mynginx

2、在elk123节点上使用filebeat采集container日志

[root@elk123 ~]# cd /es/softwares/filebeat-7.17.5-linux-x86_64
[root@elk123 filebeat-7.17.5-linux-x86_64]# vim config/12-container-to-console.yaml
filebeat.inputs:
- type: container
paths:
- '/var/lib/docker/containers/*/*.log'
#屏幕输出
output.console:
pretty: true
#ES输出
#output.elasticsearch:
# hosts: ["http://192.168.1.121:9200","http://192.168.1.122:9200","http://192.168.1.123:9200"]

3、在elk123节点上测试访问nginx和tomcat

[root@elk123 ~]# curl 192.168.1.123:88
[root@elk123 ~]# curl 192.168.1.123:89

4、在elk123节点上启动filebeat实例

[root@elk123 filebeat-7.17.5-linux-x86_64]# rm -rf data/
[root@elk123 filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/12-container-to-console.yaml

二、课堂练习

2.1 多行匹配(每7行进行匹配)并导入ES

1、上传shopping.json文件到elk123主机的/tmp/es目录下

2、修改shopping.json文件

[root@elk123 es]# sed -i s/},/}/g shopping.json

3、在elk123节点上创建工作目录

[root@elk123 ~]# cd /es/softwares/filebeat-7.17.5-linux-x86_64
[root@elk123 filebeat-7.17.5-linux-x86_64]# mkdir config

4、在elk123节点上编写配置文件

[root@elk123 filebeat-7.17.5-linux-x86_64]# vim config/14-ketanglianxi.yaml
filebeat.inputs:
- type: filestream
enabled: true
paths:
- /tmp/oldboyedu-linux85/shopping.json
parsers:
- multiline:
type: count
count_lines: 7
- ndjson:
add_error_key: true
overwrite_keys: true
output.elasticsearch:
hosts: ["http://192.168.1.121:9200","http://192.168.1.122:9200","http://192.168.1.123:9200"]
[root@elk123 filebeat-7.17.5-linux-x86_64]# vim config/14-ketanglianxi.yaml
[root@elk123 filebeat-7.17.5-linux-x86_64]# cat config/14-ketanglianxi.yaml
filebeat.inputs:
- type: filestream
enabled: true
paths:
- /tmp/es/shopping.json
parsers:
- multiline:
type: count
count_lines: 7
- ndjson:
add_error_key: true
overwrite_keys: true
output.elasticsearch:
hosts: ["http://192.168.1.121:9200","http://192.168.1.122:9200","http://192.168.1.123:9200"]

5、在elk123节点上启动filebeat的实例

[root@elk123 filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/14-ketanglianxi.yaml

相关参数说明：

  • -e: 这个选项表示在前台运行，而不是作为后台服务。使用 -e 选项，Filebeat 将输出日志信息到终端，便于调试和查看实时日志。

  • -c：这个选项后面通常跟着指定配置文件的路径。

6、打开浏览器输入http://192.168.1.123:5601/登录kibana界面后，建立索引模式

7、点击【菜单栏】-【Discover】即可详细查看数据