CICD之logstash服务的Dockerfile使用Gitlab Runner打docker包
gitlab提交代码后，经gitlab Runner打docker包，推送到docker仓库，然后kubernetes选择版本更新
Dockerfile
FROM openjdk:8-jre-alpine
# ensure logstash user exists
RUN addgroup -S logstash && adduser -S -G logstash logstash
# install plugin dependencies
RUN apk add --no-cache \
# env: can't execute 'bash': No such file or directory
bash \
libc6-compat \
libzmq
# grab su-exec for easy step-down from root
RUN apk add --no-cache 'su-exec>=0.2'
# https://www.elastic.co/guide/en/logstash/5.0/installing-logstash.html#_apt
# https://artifacts.elastic.co/GPG-KEY-elasticsearch
ENV LOGSTASH_PATH /usr/share/logstash/bin
ENV PATH $LOGSTASH_PATH:$PATH
# LOGSTASH_TARBALL="https://artifacts.elastic.co/downloads/logstash/logstash-5.5.0.tar.gz"
COPY logstash-5.5.0.tar.gz /logstash.tar.gz
RUN set -ex; \
apk add --no-cache --virtual .fetch-deps \
ca-certificates \
gnupg \
openssl \
tar ; \
dir="$(dirname "$LOGSTASH_PATH")"; \
mkdir -p "$dir"; \
tar -xf /logstash.tar.gz --strip-components=1 -C "$dir"; \
rm logstash.tar.gz; \
apk del .fetch-deps; \
export LS_SETTINGS_DIR="$dir/config"; \
# if the "log4j2.properties" file exists (logstash 5.x), let's empty it out so we get the default: "logging only errors to the console"
if [ -f "$LS_SETTINGS_DIR/log4j2.properties" ]; then \
cp "$LS_SETTINGS_DIR/log4j2.properties" "$LS_SETTINGS_DIR/log4j2.properties.dist"; \
truncate -s 0 "$LS_SETTINGS_DIR/log4j2.properties"; \
fi; \
# set up some file permissions
for userDir in \
"$dir/config" \
"$dir/data" \
; do \
if [ -d "$userDir" ]; then \
chown -R logstash:logstash "$userDir"; \
fi; \
done; \
logstash --version
COPY docker-entrypoint.sh /
RUN chmod +x /docker-entrypoint.sh
COPY logstash-shipper.conf /
RUN mkdir -p /data/logs/sincedb
RUN chown logstash.logstash -R /data/logs/sincedb
WORKDIR /
ENTRYPOINT ["/docker-entrypoint.sh"]
CMD ["-f", "/logstash-shipper.conf"]
docker-entrypoint.sh
#!/bin/bash
set -e
mkdir -p /data/logs/sincedb
chown logstash.logstash -R /data/logs/sincedb
# first arg is `-f` or `--some-option`
if [ "${1#-}" != "$1" ]; then
set -- logstash "$@"
fi
# Run as user "logstash" if the command is "logstash"
# allow the container to be started with `--user`
if [ "$1" = 'logstash' -a "$(id -u)" = '0' ]; then
set -- su-exec logstash "$@"
fi
exec "$@"
logstash-5.5.0.tar.gz 从官方下载 https://www.elastic.co/cn/downloads/logstash
logstash-shipper.conf样例
input {
file {
path => [ "/data/logs/service/*/*.log"]
type => "service"
sincedb_path => "/data/logs/sincedb/service"
codec => multiline {
pattern => "^\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d.\d\d\d .+"
negate => true
what => "previous"
max_lines => 30
}
}
file {
path => [ "/data/logs/web/*/access_log*.log"]
codec => plain { format => "%{message}" }
type => "web"
sincedb_path => "/data/logs/sincedb/web"
}
}
output {
if [type] == 'service' {
kafka {
codec => plain { format => "%{message}" }
bootstrap_servers => "139.219.*.*:9092"
topic_id => "service"
}
}
if [type] == 'web' {
kafka {
codec => plain { format => "%{message}" }
bootstrap_servers => "139.219.*.*:9092"
topic_id => "web"
}
}
}
service的日志开头是2017-12-01 12:01:01，所以pattern匹配时间，根据时间判断日志的起始点；web日志原封不动传过去，output到kafka集群，logstash-indexer从kafka获取日志后归入elasticsearch
logstash-indexer.conf示例
input {
kafka {
bootstrap_servers => "139.219.*.*:9092"
topics => "service"
type => "service"
}
kafka {
bootstrap_servers =>"139.219.*.*:9092"
topics => "web"
type => "web"
}
}
filter {
if [type] != ['web'] {
if "_grokparsefailure" in [tags] {
drop { }
}
grok {
match => {
"message" => "%{TIMESTAMP_ISO8601:timestamp} %{GREEDYDATA}"
}
}
date {
match => ["timestamp","yyyy-MM-dd HH:mm:ss.SSS"]
locale => "cn"
}
}
if [type] == 'web' {
if "_grokparsefailure" in [tags] {
drop { }
}
grok {
match => {
"message" => '%{IP} - - \[%{HTTPDATE:time}\] "%{WORD:methord} %{URIPATHPARAM:request} HTTP/%
{NUMBER:httpversion}" %{NUMBER:response} %{GREEDYDATA}'
}
}
date {
match => ["time","dd/MMM/yyyy:HH:mm:ss +\d+"]
locale => "cn"
}
}
}
output {
if [type] == 'service' {
elasticsearch {
hosts => "172.16.1.1:9200"
index => "bbotte-service-%{+YYYY.MM.dd}"
}
}
if [type] == 'web' {
elasticsearch {
hosts => "172.16.1.1:9200"
index => "bbotte-web-%{+YYYY.MM.dd}"
}
}
}
最后就是gitlabci配置示例
# cat .gitlab-ci.yml
image: docker:latest
stages:
- LogstashPubTest
- LogstashPubProd
image-build-test:
stage: LogstashPubTest
script:
- "current_date=`TZ='UTC-8' date +'%m%d%H%M'`"
- "commit_sha=$CI_COMMIT_SHA"
- "docker build -t bbotte.com:5000/logstash:$CI_COMMIT_REF_NAME-$current_date-${commit_sha:0:8} ."
- "docker login -u admin -p 123456 bbotte.com:5000"
- "docker push bbotte.com:5000/logstash:$CI_COMMIT_REF_NAME-$current_date-${commit_sha:0:8}"
only:
- test
image-build-master:
stage: LogstashPubProd
script:
- "current_date=`TZ='UTC-8' date +'%m%d%H%M'`"
- "commit_sha=$CI_COMMIT_SHA"
- "docker build -t bbotte.com:5000/logstash:$CI_COMMIT_REF_NAME-$current_date-${commit_sha:0:8} ."
- "docker login -u admin -p 123456 bbotte.com:5000"
- "docker push bbotte.com:5000/logstash:$CI_COMMIT_REF_NAME-$current_date-${commit_sha:0:8}"
only:
- master
目录结构如下：
logstash$ ls -a
. docker-entrypoint.sh .git logstash-5.5.0.tar.gz
.. Dockerfile .gitlab-ci.yml logstash-shipper.conf
2018年01月08日 于 linux工匠 发表