一、安装redis

sed -i 's#^bind 127.0.0.1#bind 127.0.0.1 10.0.0.12#' /etc/redis.conf

[root@k8s-node1 ~]# redis-cli -h 10.0.0.12
10.0.0.12:6379> 

 二、修改fileabeat配置文件

[root@k8s-node2 filebeat]# cat filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["access"]

- type: log
  enabled: true
  paths:
    - /var/log/nginx/error.log
  tags: ["error"]

- type: log
  enabled: true
  paths:
    - /tomcat/apache-tomcat-8.0.38/logs/localhost_access_log.*.txt
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["tomcat"]

output.redis:
  hosts: ["10.0.0.12"]
  keys:
    - key: "nginx_access"
      when.contains:
        tags: "access"
    - key: "nginx_error"
      when.contains:
        tags: "error"
    - key: "tomcat_access"
      when.contains:
        tags: "error"

setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.enabled: false
setup.template.overwrite: true

setup.template.name: "tomcat"
setup.template.pattern: "tomcat_*"
setup.template.enabled: false
setup.template.overwrite: true
 

 三、配置nginx日志为json格式

四、清除nginx旧日志

五、查看redis数据

[root@k8s-node1 ~]# redis-cli -h 10.0.0.12
10.0.0.12:6379> keys *
1) "nginx_access"
2) "tomcat_access"
3) "nginx_error"
 

[root@k8s-node1 ~]# redis-cli LRANGE nginx_access 0 -1 
1) "{\"@timestamp\":\"2020-11-08T10:01:43.297Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"doc\",\"version\":\"6.6.0\"},\"remote_addr\":\"10.0.0.13\",\"host\":{\"name\":\"k8s-node2\"},\"request\":\"GET / HTTP/1.1\",\"request_time\":\"0.000\",\"x_forwarded\":\"-\",\"status\":200,\"up_addr\":\"-\",\"agent\":\"curl/7.29.0\",\"source\":\"/var/log/nginx/access.log\",\"beat\":{\"name\":

[root@k8s-node2 ~]# redis-cli LLEN nginx_access
(integer) 45
 

这说明我们redis是好了，

六、安装logstash

cd /opt/

rpm -ivh logstash-6.6.0

修改配置文件

[root@k8s-node1 opt]# cat /etc/logstash/conf.d/redis.conf 
input {
  redis {
    host => "127.0.0.1"
    port => "6379"
    db => "0"
    key => "nginx_access"
    data_type => "list"
  }
  redis {
    host => "127.0.0.1"
    port => "6379"
    db => "0"
    key => "nginx_error"
    data_type => "list"
  }
  redis {
    host => "127.0.0.1"
    port => "6379"
    db => "0"
    key => "tomcat_access"
    data_type => "list"

}

filter {
  mutate {
    convert => ["upstream_time", "float"]
    convert => ["request_time", "float"]
  }
}

output {
   stdout {}
   if "access" in [tags] {
      elasticsearch {
        hosts => "http://10.0.0.11:9200"
        manage_template => false
        index => "nginx_access-%{+yyyy.MM}"
      }
    }
    if "error" in [tags] {
      elasticsearch {
        hosts => "http://10.0.0.11:9200"
        manage_template => false
        index => "nginx_error-%{+yyyy.MM}"
      }
    }
    if "tomcat" in [tags] {
      elasticseatch {
        hosts => "http://10.0.0.11:9200"
    manage_template => false
    index => "tomcat_access-%{+yyyy.MM}"
      }
    }
}
 

前台启动拍错用

/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/redis.conf 

 后台启动

systemctl start logstash