chore(*): cherrypick master and bump filebeat to 6.7.2 (Starbucks OEM) (

#39) * higher resource limit (#32) (cherry picked from commit d17d4bb) * feat(metrics): export Filebeat metrics with beat-exporter (#35) * added beat-exporter * revert imagePullPolicy (cherry picked from commit d2dbc2f) * remove 2.9.0 content * bump filebeat version to 6.7.2 * [release-1.2] Update Filebeat version to 6.5.4 for upstream bug fix (#37) * improve Makefile * always drop log field * add missing CM name for OUTPUT_CONFIG env
caicloud · Sep 5, 2019 · 931a1de · 931a1de
1 parent 8510a93
commit 931a1de
Show file tree

Hide file tree

Showing 4 changed files with 31 additions and 17 deletions.
diff --git a/Makefile b/Makefile
@@ -5,8 +5,10 @@ COMMIT := $(if $(COMMIT),$(COMMIT),"Unknown")
 # This repo's root import path (under GOPATH).
 ROOT := github.com/caicloud/log-pilot
 
+UPSTREAM_REGISTRY := cargo.caicloudprivatetest.com/caicloud
+
 # Container registries. You can use multiple registries for a single project.
-REGISTRIES ?= cargo.caicloudprivatetest.com/caicloud
+REGISTRIES ?= cargo.caicloudprivatetest.com/release
 
 # Project output directory.
 OUTPUT_DIR := ./bin
@@ -21,11 +23,11 @@ CMD_DIR := ./cmd
 VERSION ?= $(COMMIT)
 
 # Target binaries. You can build multiple binaries for a single project.
-TARGETS := log-pilot filebeat-keeper
+TARGETS ?= log-pilot filebeat-keeper
 
 # Docker image name.
-LOG_PILOT_IMAGE := cargo.caicloudprivatetest.com/caicloud/log-pilot:$(VERSION)
-FILEBEAT_IMAGE := cargo.caicloudprivatetest.com/caicloud/filebeat:$(VERSION)
+LOG_PILOT_IMAGE := $(REGISTRIES)/log-pilot:$(VERSION)
+FILEBEAT_IMAGE := $(REGISTRIES)/filebeat:$(VERSION)
 
 .PHONY: default container push
 
@@ -41,7 +43,7 @@ build-linux:
 	      -e GOARCH=amd64                                                              \
 	      -e CGO_ENABLED=0                                                             \
 	      -e GOPATH=/go                                                                \
-	        $${registry}/golang:1.10-alpine3.6                                         \
+	        $(UPSTREAM_REGISTRY)/golang:1.10-alpine3.6                                 \
 	          go build -i -v -o $(OUTPUT_DIR)/$${target}                               \
 	            -ldflags "-s -w -X $(ROOT)/pkg/version.Version=$(VERSION)              \
 	            -X $(ROOT)/pkg/version.Commit=$(COMMIT)                                \

diff --git a/build/filebeat/Dockerfile b/build/filebeat/Dockerfile
@@ -1,4 +1,4 @@
-FROM cargo.caicloudprivatetest.com/library/filebeat:6.5.3
+FROM cargo.caicloudprivatetest.com/library/filebeat:6.7.2
 
 USER root
 RUN yum install -y gettext && yum clean all

diff --git a/build/filebeat/filebeat.yml.tpl b/build/filebeat/filebeat.yml.tpl
@@ -59,17 +59,13 @@ filebeat.inputs:
 # TODO: etcd, apiserver and more..
 
 processors:
-{{ if .multilinePattern -}}
 - drop_fields:
-    fields: ["log"]
-{{- end }}
+    fields: ["beat", "host.name", "input.type", "prospector.type", "offset", "source", "log"]
 - rename:
     fields:
     - from: message
       to: log
     ignore_missing: true
-- drop_fields:
-    fields: ["beat", "host.name", "input.type", "prospector.type", "offset", "source", ]
 
 {{- if eq .type "elasticsearch" }}
 setup.template.enabled: true

diff --git a/release/logging-filebeat.yaml b/release/logging-filebeat.yaml
@@ -17,17 +17,17 @@ _config:
           key: name
       resources:
         limits:
-          memory: 200Mi
+          memory: 500Mi
           cpu: "2"
         requests:
-          memory: 50Mi
-          cpu: 100m
+          memory: 100Mi
+          cpu: 200m
       image: '[[ registry_release ]]/filebeat:[[ imageTagFromGitTag ]]'
       imagePullPolicy: Always
       command:
       - /opt/filebeat-keeper/filebeat-keeper
       args:
-      - "--path.config=/etc/filebeat" 
+      - "--path.config=/etc/filebeat"
       - "--path.home=/opt/filebeat"
       - "-e"
       mounts:
@@ -57,10 +57,11 @@ _config:
       - --path.filebeat-home=/opt/filebeat
       - --logLevel=debug
       - -e
-      env: 
+      env:
       - name: OUTPUT_CONFIG
-        from: 
+        from:
           type: Config
+          name: filebeat-output
           key: filebeat-output.yml
       resources:
         limits:
@@ -83,6 +84,21 @@ _config:
       - name: docker-sock
         path: /var/run/docker.sock
         readonly: true
+    - image: '[[ registry_release ]]/beat-exporter:v0.1.2'
+      imagePullPolicy: Always
+      resources:
+        limits:
+          cpu: 100m
+          memory: 200Mi
+        requests:
+          cpu: 10m
+          memory: 20Mi
+    pod:
+      annotations:
+      - key: prometheus.io/port
+        value: '9479'
+      - key: prometheus.io/scrape
+        value: 'true'
     type: DaemonSet
     volumes:
     - name: varlog