环境介绍

架构图示

共4台虚拟机

配置过程

logstash105配置读取本地的系统日志和nginx日志写入kafka

1. 准备系统日志和nginx日志

   [root@logstash-105 conf.d]# systemctl start nginx
                      
   [root@logstash-105 conf.d]# ll /var/log/messages
   -rwxr-xr-x 1 root root 305922 Dec 14 16:09 /var/log/messages
   [root@logstash-105 conf.d]# ll /var/log/nginx/access.log 
   -rw-r--r-- 1 root root 1680 Dec 13 19:22 /var/log/nginx/access.log
      
   给2个文件加上所有用户读权限
   

2. 配置logstash读取nginx和系统日志，输入到kafka

   [root@logstash-105 conf.d]# cat log-kafka.conf 
   input {
    file {
        path => "/var/log/messages"
        type => "kakfa-syslog-105"
        start_position => "beginning"
        stat_interval => "2"
    }
    file {
        path => "/var/log/nginx/access.log"
        type => "kakfa-ngxlog-105"
        start_position => "beginning"
        stat_interval => "2"
        codec => "json"
      		
    }
   }
      
      
   output {
      
    if [type] == "kafka-syslog-105" {
        kafka {
            topic_id => "kafka-syslog-105"
            bootstrap_servers => "192.168.80.107:9092"
            batch_size =>5
            codec => "json"
        }
        file {
            path => "/tmp/syslog.log"
        }
    }
    if [type] == "kafka-ngxlog-105" {
        kafka {
            topic_id => "kafka-ngxlog-105"
            bootstrap_servers => "192.168.80.107:9092"
            batch_size =>5
            codec => "json"
        }
        file {
            path => "/tmp/ngxlog.log"
        }
    }
      
      	
   }
   其中输出到/tmp目录是为了测试
   

3. kafka查看新增的topic

   [root@es2 bin]# ./kafka-topics.sh --zookeeper 192.168.80.107:2181 --list
   hello
      
   没有新增topic，
   /tmp目录也没有测试输出
   排错时，logstash没有报错，配置文件语法也对，测试输出到控制台也没有，难道是版本bug？
   

logstash108从kafka读取数据

kibana界面创建索引并查看数据