linux安装logstash6.6.2

  • Z时代
  • 2024-01-10
  • 分类：综合

编程

./bin/logstash -f ./config/logstash.conf

 

后台启动

./bin/logstash -f ./config/logstash.conf > /dev/null 2>&1 &

 

命令输出模式启动

./bin/logstash -e "input{stdin{}}output{stdout{codec=>rubydebug}}"

 

问题

Exiting: 1 error: Error reading fileset mysql/error: Error reading manifest file: config file ("/opt/filebeat-6.2.2-linux-x86_64/module/mysql/error/manifest.yml") must be owned by the beat user (uid=0) or root

原因：这些检查的目的是防止未经授权的用户提供或修改Beat所运行的配置。配置文件的所有者必须root 是执行Beat进程的用户，或者是该用户。

解决方法：

官方说明：https://www.elastic.co/guide/en/beats/libbeat/5.3/config-file-permissions.html#config-file-permissions 

To correct this problem you can use either chown root {beatname}.yml or chown 501 {beatname}.yml to change the owner of the configuration file.
Exiting: error loading config file: config file ("{beatname}.yml") can only be
writable by the owner but the permissions are "-rw-rw-r--" (to fix the
permissions use: "chmod go-w /etc/{beatname}/{beatname}.yml")
To correct this problem, use chmod go-w /etc/{beatname}/{beatname}.yml to remove write privileges from anyone other than the owner.

通过chown root {beatname}.yml，将不同的yml文件授权给root用户，比如：filebeat-6.2.2-linux-x86_64/module/mysql下的*.yml,执行：chown root manifest.yml后重新./filebeat setup即可

 

参考文档

https://doc.yonyoucloud.com/doc/logstash-best-practice-cn/input/stdin.html

https://github.com/logstash-plugins/logstash-patterns-core/tree/master/patterns

 

添加过滤配置 

input {
  beats {
    port => 5044
  }
}
#过滤
filter {
    #nginx 日志过滤
    if "nginx-accesslog" in [tags] {
        grok {
                match => { "message" => "%{HTTPDATE:timestamp}|%{IP:remote_addr}|%{IPORHOST:http_host}|(?:%{DATA:http_x_forwarded_for}|-)|%{DATA:request_method}|%{DATA:request_uri}|%{DATA:server_protocol}|%{NUMBER:status}|(?:%{NUMBER:body_bytes_sent}|-)|(?:%{DATA:http_referer}|-)|%{DATA:http_user_agent}|(?:%{DATA:request_time}|-)|"}
        }
        mutate {
                convert => ["status","integer"]
                convert => ["body_bytes_sent","integer"]
                convert => ["request_time","float"]
        }
        geoip {
                source=>"remote_addr"
        }
        date {
                match => [ "timestamp","dd/MMM/YYYY:HH:mm:ss Z"]
        }
        useragent {
                source=>"http_user_agent"
        }
    }
    #logback日志过滤
    if "logback"  in [tags] {
        grok {         
#支持多个正则匹配，只到最佳 
   	      match => {"message" => "[%{LOGLEVEL:level}]s[%{DATA:thread}]s[%{DATA:time}]s%{DATA:class}s[%{INT:line}]s-s(?<msg>.+)$"}
              #match => {"message" => "[%{LOGLEVEL:level}]s[%{DATA:thread}]s[%{HTTPDATE:logdate}]s%{DATA:class}s[%{INT:line}]s-s(?<msg>.+)$"}            
              #match => {"message" => "[%{LOGLEVEL:level}]s[%{DATA:thread}]s[(?<logdate>%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME})]s%{DATA:class}s[%{INT:line}]s-s(?<msg>.+)$"} 
   		match => {"message" => "-s(?<msg>.+)$"}
                # 删除字段，防止重复存储相同内容
                remove_field => "message
                # 添加字段
                add_field => {
                     "device" => "logback"
                 }
        }
        date {  
                # 时间格式化
                match =>  [ "time", "YYYY-MM-DD  HH:mm:ss" ]
        }
    } 
    #log4j2过滤
    if "log4j2"  in [tags] {
        grok {
                match => {"message" => "%{DATA:timestamp} [%{DATA:log_pid}] %{LOGLEVEL:level}- %{DATA:log_message}" }
        }
        date {
                match =>  [ "timestamp", "YYYY-MM-DD  HH:mm:ss" ]
        }
    }
}
output {
  elasticsearch {
    hosts => ["http://localhost:9200"]
    index => "java-logs-%{[@metadata][version]}-%{+yyyy.MM.dd}"
    #user => "elastic"
    #password => "changeme"
  }
}

输入日志文件 

input {
     file {
        path => "/usr/local/bin/contract/logs/*.log "
        start_position => "beginning"
        tags=> "logback"
        codec => json {
            charset => "UTF-8"
        }        
     }
     file {
        path => "/var/log/mysqld.log"
        start_position => "beginning"
        tags=> "sql"
        codec => json {
            charset => "UTF-8"
        }        
     }
}

输出到多个索引 

output {
    # 通过自定义tags进行判断
    if "logback" in [tags] {
        elasticsearch {
            hosts => ["http://localhost:9200"]
            index => "java-logs-%{[@metadata][version]}-%{+yyyy.MM.dd}"
        }    
    }else {
        elasticsearch {
            hosts => ["http://localhost:9200"]
            index => "filebeat-%{[@metadata][version]}-%{+yyyy.MM.dd}"
        }          
    }       
}

过滤器配置 

if [type] == "nginx" {
	grok {
	  match => { "message" => ["(?<RemoteIP>(d*.d*.d*.d*)) - %{DATA:[nginx][access][user_name]} [%{HTTPDATE:[nginx][access][time]}] "%{WORD:[nginx][access][method]} %{DATA:[nginx][access][url]} HTTP/%{NUMBER:[nginx][access][http_version]}" %{NUMBER:[nginx][access][response_code]} %{NUMBER:[nginx][access][body_sent][bytes]} "%{DATA:[nginx][access][referrer]}" "%{DATA:[nginx][access][agent]}""] }
		add_field => {
		  "Device" => "Charles Desktop"
		}
		#删除字段
		remove_field => "message"
		remove_field => "beat.version"
		remove_field => "beat.name"
	}
}

 

以上是 linux安装logstash6.6.2 的全部内容， 来源链接： utcz.com/z/516152.html

回到顶部