使用docker-elk搭建并实践ELK日志分析框架

    - type: log
      # Change to true to enable this prospector configuration.
      enabled: true
      # Paths that should be crawled and fetched. Glob based paths.
      paths:
    - /applog/*.log

    #output.logstash:
      # The Logstash hosts
      hosts: ["localhost:5044"]
      # Optional SSL. By default is off.
      # List of root certificates for HTTPS server verifications
      ssl.certificate_authorities: ["/home/hik/docker/logstash-beats.crt"]

echo "start filebeat now ..."
nohup /opt/filebeat-6.2.2-linux-x86_64/filebeat -e -c /opt/filebeat-6.2.2-linux-x86_64/filebeat.yml >/dev/null 2>&1 &
echo "start springboot jar now ..."
java -jar $1

#基础镜像
FROM java:8u111-jdk

#定义日志文件存放目录，这个要和web应用的日志配置一致
ENV APP_LOG_PATH /applog

#定义证书文件目录，这个要和filebeat.yml的配置一致
ENV FILE_BEAT_CRT_PATH /etc/pki/tls/certs

#定义filebeat文件夹名称
ENV FILE_BEAT_PACKAGE_NAME filebeat-6.2.2-linux-x86_64

#定义证书文件名称
ENV FILE_BEAT_CRT_NAME logstash-beats.crt

#定义启动文件名称
ENV ENTRYPOINT_FILE_NAME filebeat-springboot-entrypoint.sh

#定义时区参数
ENV TZ=Asia/Shanghai

#设置时区
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone

#使之生效
RUN . /etc/profile

#创建日志目录文件夹
RUN mkdir $APP_LOG_PATH

#存放证书的文件夹
RUN mkdir -p $FILE_BEAT_CRT_PATH

#从当前目录將证书文件复制到镜像中
COPY ./$FILE_BEAT_CRT_NAME $FILE_BEAT_CRT_PATH/

#从当前目录將filebeat文件复制到镜像中
COPY ./$FILE_BEAT_PACKAGE_NAME /opt/$FILE_BEAT_PACKAGE_NAME

#复制启动文件
COPY ./$ENTRYPOINT_FILE_NAME /$ENTRYPOINT_FILE_NAME

#赋写权限
RUN chmod a+x /$ENTRYPOINT_FILE_NAME

docker build -t springboot-app-filebeat:0.0.3 .

"FILE"
          class="ch.qos.logback.core.rolling.RollingFileAppender">
    "ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
        /applog/output.%d{yyyy-MM-dd}.log
        7
    
    "ch.qos.logback.classic.encoder.PatternLayoutEncoder">
        ${ENCODER_PATTERN}
    

   com.spotify
   docker-maven-plugin
   0.4.12
   
   
      
      bolingcavalry/${project.artifactId}
      
      
         ${project.version}
      
      
      springboot-app-filebeat:0.0.3
      
      ["sh","/filebeat-springboot-entrypoint.sh","/${project.build.finalName}.jar"]
      
      http://10.66.35.173:2375
      
         
            /
            ${project.build.directory}
            ${project.build.finalName}.jar
         
      
   

 @RequestMapping(value="/hello/{username}",method= RequestMethod.GET)
public String hello(@PathVariable String username) {
    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    String time = sdf.format(new Date());
    logger.info("execute hello from {} - {}", username, time);
    return "hello " + username + ", " + time;
}

vi docker-compose.yml

version: '2'
services:
  elk:
    image: elk:6.4.2
    ports:
      - "5601:5601"
      - "9200:9200"
    restart: always
  webapp:
    image: elkdemo:1.0
    depends_on:
      - elk
    links:
      - elk:elkhost
    ports:
      - "18080:8080"
restart: always

#-------------------------- Elasticsearch output ------------------------------
output.elasticsearch:
  # Array of hosts to connect to.
  # hosts: ["localhost:9200"]   #注释掉这句，filebeat不再输出到elasticsearch，改为向logstash输出

  # Optional protocol and basic auth credentials.
  #protocol: "https"
  #username: "elastic"
  #password: "changeme"

#----------------------------- Logstash output --------------------------------
#output.logstash:
  # The Logstash hosts
  hosts: ["localhost:5044"]

  # Optional SSL. By default is off.
  # List of root certificates for HTTPS server verifications
  ssl.certificate_authorities: ["/home/hik/docker/logstash-beats.crt"]

hik@root:~/docker/filebeat-6.2.2-linux-x86_64$ sudo docker ps
CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                                                                NAMES
c02f0f0f6047        elkdemo:1.0         "sh /filebeat-spring…"   8 weeks ago         Up About an hour    0.0.0.0:18080->8080/tcp                                              docker_webapp_1
308de13c21e0        elk:6.4.2           "/usr/local/bin/star…"   8 weeks ago         Up About an hour    0.0.0.0:5601->5601/tcp, 5044/tcp, 9300/tcp, 0.0.0.0:9200->9200/tcp   docker_elk_1

hik@root:~/docker/filebeat-6.2.2-linux-x86_64$ sudo docker exec -it 308de13c21e0 /bin/bash
root@308de13c21e0:/# ls
bd_build  bin  boot  dev  etc  home  lib  lib64  media  mnt  opt  proc  root  run  sbin  srv  sys  tmp  usr  var

#本操作使用json插件，处理结果删除掉message字段。
input {
  beats {
    port => 5044
  }
}
filter {
    json {
        source => "message"
        remove_field => ["message"]
    }
}
output {
  elasticsearch {
    hosts => ["http://localhost:9200"]
    index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
    #user => "elastic"
    #password => "changeme"
  }
}

bin/logstash -f logstash-sample.conf

 {
  "_index": "filebeat-2019.05.22",
  "_type": "doc",
  "_id": "CnCs3WoBtWWwz0jz3Y4z",
  "_version": 1,
  "_score": null,
  "_source": {
    "beat": {
      "name": "be58d50b9f23",
      "hostname": "be58d50b9f23",
      "version": "6.2.2"
    },
    "tags": [
      "beats_input_codec_plain_applied"
    ],
    "@version": "1",
    "source": "/applog/output.2019-05-22.log",
    "@timestamp": "2019-05-22T03:53:09.901Z",
    "offset": 2722,
    "prospector": {
      "type": "log"
    },
    "host": "be58d50b9f23"
  },
  "fields": {
    "@timestamp": [
      "2019-05-22T03:53:09.901Z"
    ]
  },
  "sort": [
    1558497189901
  ]
}