Phase V

7 years ago · a261ed7acd
10 changed files with 16642 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1 @@
+.vscode/*
--- a/7
+++ b/7
@ -0,0 +1,7 @@
+FROM confluentinc/cp-kafka
+
+WORKDIR /jmx
+
+COPY jmx_prometheus_javaagent-0.11.0.jar /jmx
+
+# RUN java -javaagent:/jmx/jmx_prometheus_javaagent-0.11.0.jar=8080:config.yaml -jar yourJar.jar
--- a/burrow.toml
+++ b/burrow.toml
@ -0,0 +1,29 @@
+[zookeeper]
+servers=[ "localhost:22181", "localhost:32181", "localhost:42181" ]
+timeout=6
+root-path="/burrow"
+
+[cluster.local]
+class-name="kafka"
+servers=[ "localhost:19092", "localhost:29092", "localhost:39092" ]
+topic-refresh=10
+offset-refresh=10
+
+[consumer.local]
+class-name="kafka"
+cluster="local"
+servers=[ "localhost:19092", "localhost:29092", "localhost:39092" ]
+group-blacklist="^(console-consumer-|python-kafka-consumer-).*$"
+group-whitelist=""
+
+[consumer.local_zk]
+class-name="kafka_zk"
+cluster="local"
+servers=[ "localhost:22181", "localhost:32181", "localhost:42181" ]
+zookeeper-path="/local"
+zookeeper-timeout=30
+group-blacklist="^(console-consumer-|python-kafka-consumer-).*$"
+group-whitelist=""
+
+[httpserver.default]
+address=":8000"
--- a/config-2.yml
+++ b/config-2.yml
@ -0,0 +1,86 @@
+lowercaseOutputName: true
+
+rules:
+# Special cases and very specific rules
+- pattern : kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value
+  name: kafka_server_$1_$2
+  type: GAUGE
+  labels:
+    clientId: "$3"
+    topic: "$4"
+    partition: "$5"
+- pattern : kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value
+  name: kafka_server_$1_$2
+  type: GAUGE
+  labels:
+    clientId: "$3"
+    broker: "$4:$5"
+
+# Generic per-second counters with 0-2 key/value pairs
+- pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+), (.+)=(.+)><>Count
+  name: kafka_$1_$2_$3_total
+  type: COUNTER
+  labels:
+    "$4": "$5"
+    "$6": "$7"
+- pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+)><>Count
+  name: kafka_$1_$2_$3_total
+  type: COUNTER
+  labels:
+    "$4": "$5"
+  type: COUNTER
+- pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*><>Count
+  name: kafka_$1_$2_$3_total
+  type: COUNTER
+
+# Generic gauges with 0-2 key/value pairs
+- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Value
+  name: kafka_$1_$2_$3
+  type: GAUGE
+  labels:
+    "$4": "$5"
+    "$6": "$7"
+- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Value
+  name: kafka_$1_$2_$3
+  type: GAUGE
+  labels:
+    "$4": "$5"
+- pattern: kafka.(\w+)<type=(.+), name=(.+)><>Value
+  name: kafka_$1_$2_$3
+  type: GAUGE
+
+# Emulate Prometheus 'Summary' metrics for the exported 'Histogram's.
+#
+# Note that these are missing the '_sum' metric!
+- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Count
+  name: kafka_$1_$2_$3_count
+  type: COUNTER
+  labels:
+    "$4": "$5"
+    "$6": "$7"
+- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)><>(\d+)thPercentile
+  name: kafka_$1_$2_$3
+  type: GAUGE
+  labels:
+    "$4": "$5"
+    "$6": "$7"
+    quantile: "0.$8"
+- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Count
+  name: kafka_$1_$2_$3_count
+  type: COUNTER
+  labels:
+    "$4": "$5"
+- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*)><>(\d+)thPercentile
+  name: kafka_$1_$2_$3
+  type: GAUGE
+  labels:
+    "$4": "$5"
+    quantile: "0.$6"
+- pattern: kafka.(\w+)<type=(.+), name=(.+)><>Count
+  name: kafka_$1_$2_$3_count
+  type: COUNTER
+- pattern: kafka.(\w+)<type=(.+), name=(.+)><>(\d+)thPercentile
+  name: kafka_$1_$2_$3
+  type: GAUGE
+  labels:
+    quantile: "0.$4"
--- a/config.yml
+++ b/config.yml
@ -0,0 +1,106 @@
+# This has been tested on Kafka client 1.1.1, commidId: 98b6346a977495f6
+# Metrics ending with "-total" are counters.
+# Everything else are gauges
+
+# Explaination of regular expression: (((?!(-total|-rate|\.)).)*)
+# this exclude metrics with names "-total", "-rate" and "."
+# "-total": already handled by the (-total) rule
+# "-rate": calculated by the client.  Not needed since we can do the same with the rate()
+# ".": metrics w/ a "." in the name have be deprecated.
+#
+# eg:
+#  "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=client-1:topic-1-0.records-lag-max"
+# has been repplace by
+#  "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=client-1,topic=topic-1,partition=0:records-lag-max"
+
+lowercaseOutputName: true
+
+rules:
+
+#-----------------------------------------------------------------------------
+# Start w/ the special metrics, then most keys to least keys
+#-----------------------------------------------------------------------------
+
+# Special metric: We just want to track the client version/commid-id number
+# kafka.consumer:type=app-info,client-id=consumer-1:version
+# kafka.consumer:type=app-info,client-id=consumer-1:commid-id
+# kafka.producer:type=app-info,client-id=producer-1:version
+# kafka.producer:type=app-info,client-id=producer-1:commid-id
+- pattern: kafka.(\w+)<type=(app-info), client-id=(.+)><>(.+):\s?(.+)
+  name: kafka_$1_$2
+  value: 0
+  labels:
+    "client-id": "$3"
+    "$4": "$5"
+
+#-----------------------------------------------------------------------------
+# metrics w/ 3 keys
+#-----------------------------------------------------------------------------
+
+# dont think there are COUNTER metrics w/ 3 keys. Leaving here in case new client version has them
+- pattern: kafka.(\w+)<type=(.+), (.+)=(.+), (.+)=(.+), (.+)=(.+)><>(.+-total)
+  name: kafka_$1_$2_$9
+  type: COUNTER
+  labels:
+    "$3": "$4"
+    "$5": "$6"
+    "$7": "$8"
+
+# kafka.consumer:type=consumer-fetch-manager-metrics,client-id=consumer-1,topic=barTopic,partition=0:records-lag
+- pattern: kafka.(\w+)<type=(.+), (.+)=(.+), (.+)=(.+), (.+)=(.+)><>(((?!(-total|-rate|\.)).)*)(:)
+  name: kafka_$1_$2_$9
+  type: GAUGE
+  labels:
+    "$3": "$4"
+    "$5": "$6"
+    "$7": "$8"
+
+#-----------------------------------------------------------------------------
+# metrics w/ 2 keys
+#-----------------------------------------------------------------------------
+
+# kafka.consumer:type=consumer-fetch-manager-metrics,client-id=consumer-1:bytes-consumed-total
+# kafka.consumer:type=consumer-node-metrics,client-id=consumer-1,node-id=node-111:incoming-byte-total
+# kafka.producer:type=producer-node-metrics,client-id=producer-1,node-id=node-121:outgoing-byte-total
+# kafka.producer:type=producer-topic-metrics,client-id=producer-1,topic=barTopic:record-send-total
+- pattern: kafka.(\w+)<type=(.+), (.+)=(.+), (.+)=(.+)><>(.+-total)
+  name: kafka_$1_$2_$7
+  type: COUNTER
+  labels:
+    "$3": "$4"
+    "$5": "$6"
+
+# kafka.consumer:type=consumer-fetch-manager-metrics,client-id=consumer-1:fetch-latency-avg
+# kafka.consumer:type=consumer-node-metrics,client-id=consumer-1,node-id=node-111:request-latency-max
+# kafka.producer:type=producer-node-metrics,client-id=producer-1,node-id=node-121:request-size-avg
+# kafka.producer:type=producer-topic-metrics,client-id=producer-1,topic=barTopic:compression-rate
+- pattern: kafka.(\w+)<type=(.+), (.+)=(.+), (.+)=(.+)><>(((?!(-total|-rate|\.)).)*)(:)
+  name: kafka_$1_$2_$7
+  type: GAUGE
+  labels:
+    "$3": "$4"
+    "$5": "$6"
+
+#-----------------------------------------------------------------------------
+# metrics w/ 1 key
+#-----------------------------------------------------------------------------
+
+# kafka.consumer:type=consumer-coordinator-metrics,client-id=consumer-1:heartbeat-total
+# kafka.consumer:type=consumer-fetch-manager-metrics,client-id=consumer-1:bytes-consumed-total
+# kafka.consumer:type=consumer-metrics,client-id=consumer-1:incoming-byte-total
+# kafka.producer:type=producer-metrics,client-id=producer-1:request-total
+- pattern: kafka.(\w+)<type=(.+), (.+)=(.+)><>(.+-total)
+  name: kafka_$1_$2_$5
+  type: COUNTER
+  labels:
+    "$3": "$4"
+
+# kafka.consumer:type=consumer-coordinator-metrics,client-id=consumer-1:assigned-partitions
+# kafka.consumer:type=consumer-fetch-manager-metrics,client-id=consumer-1:fetch-latency-avg
+# kafka.consumer:type=consumer-metrics,client-id=consumer-1:request-size-avg
+# kafka.producer:type=producer-metrics,client-id=producer-1:compression-rate-avg
+- pattern: kafka.(\w+)<type=(.+), (.+)=(.+)><>(((?!(-total|-rate|\.)).)*)(:)
+  name: kafka_$1_$2_$5
+  type: GAUGE
+  labels:
+    "$3": "$4"
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -0,0 +1,155 @@
+---
+version: '2'
+services:
+  zookeeper-1:
+    image: confluentinc/cp-zookeeper:latest
+    environment:
+      ZOOKEEPER_SERVER_ID: 1
+      ZOOKEEPER_CLIENT_PORT: 22181
+      ZOOKEEPER_TICK_TIME: 2000
+      ZOOKEEPER_INIT_LIMIT: 5
+      ZOOKEEPER_SYNC_LIMIT: 2
+      ZOOKEEPER_SERVERS: localhost:22888:23888;localhost:32888:33888;localhost:42888:43888
+    network_mode: host
+    extra_hosts:
+      - "moby:127.0.0.1"
+
+  zookeeper-2:
+    image: confluentinc/cp-zookeeper:latest
+    environment:
+      ZOOKEEPER_SERVER_ID: 2
+      ZOOKEEPER_CLIENT_PORT: 32181
+      ZOOKEEPER_TICK_TIME: 2000
+      ZOOKEEPER_INIT_LIMIT: 5
+      ZOOKEEPER_SYNC_LIMIT: 2
+      ZOOKEEPER_SERVERS: localhost:22888:23888;localhost:32888:33888;localhost:42888:43888
+    network_mode: host
+    extra_hosts:
+      - "moby:127.0.0.1"
+
+  zookeeper-3:
+    image: confluentinc/cp-zookeeper:latest
+    environment:
+      ZOOKEEPER_SERVER_ID: 3
+      ZOOKEEPER_CLIENT_PORT: 42181
+      ZOOKEEPER_TICK_TIME: 2000
+      ZOOKEEPER_INIT_LIMIT: 5
+      ZOOKEEPER_SYNC_LIMIT: 2
+      ZOOKEEPER_SERVERS: localhost:22888:23888;localhost:32888:33888;localhost:42888:43888
+    network_mode: host
+    extra_hosts:
+      - "moby:127.0.0.1"
+
+  kafka-1:
+    image: confluentinc/cp-kafka:latest
+    network_mode: host
+    depends_on:
+      - zookeeper-1
+      - zookeeper-2
+      - zookeeper-3
+    volumes:
+      - ./jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar:/usr/src/jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar
+      - ./config.yml:/usr/src/config.yml
+      - ./config.yml:/usr/src/config-2.yml
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: localhost:22181,localhost:32181,localhost:42181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:19092
+      # KAFKA_OPTS: -javaagent:/usr/src/jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar=7071:/usr/src/config.yml
+      KAFKA_OPTS: -javaagent:/usr/src/jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar=7071:/usr/src/config-2.yml
+    extra_hosts:
+      - "moby:127.0.0.1"
+
+  kafka-2:
+    image: confluentinc/cp-kafka:latest
+    network_mode: host
+    depends_on:
+      - zookeeper-1
+      - zookeeper-2
+      - zookeeper-3
+    volumes:
+      - ./jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar:/usr/src/jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar
+      - ./config.yml:/usr/src/config.yml
+      - ./config.yml:/usr/src/config-2.yml
+    environment:
+      KAFKA_BROKER_ID: 2
+      KAFKA_ZOOKEEPER_CONNECT: localhost:22181,localhost:32181,localhost:42181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:29092
+      # KAFKA_OPTS: -javaagent:/usr/src/jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar=7072:/usr/src/config.yml
+      KAFKA_OPTS: -javaagent:/usr/src/jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar=7072:/usr/src/config-2.yml
+    extra_hosts:
+      - "moby:127.0.0.1"
+
+  kafka-3:
+    image: confluentinc/cp-kafka:latest
+    network_mode: host
+    depends_on:
+      - zookeeper-1
+      - zookeeper-2
+      - zookeeper-3
+    volumes:
+      - ./jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar:/usr/src/jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar
+      - ./config.yml:/usr/src/config.yml
+      - ./config.yml:/usr/src/config-2.yml
+    environment:
+      KAFKA_BROKER_ID: 3
+      KAFKA_ZOOKEEPER_CONNECT: localhost:22181,localhost:32181,localhost:42181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:39092
+      # KAFKA_OPTS: -javaagent:/usr/src/jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar=7073:/usr/src/config.yml
+      KAFKA_OPTS: -javaagent:/usr/src/jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar=7073:/usr/src/config-2.yml
+    extra_hosts:
+      - "moby:127.0.0.1"
+
+  prometheus:
+    image: prom/prometheus
+    network_mode: host
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    depends_on:
+      - kafka-1
+      - kafka-2
+      - kafka-3
+
+  burrow:
+    image: artemz/burrow
+    network_mode: host
+    volumes:
+      - ./burrow.toml:/etc/burrow/burrow.toml
+      - burrow_data:/var/tmp/burrow
+    ports:
+      - "8000:8000"
+    depends_on:
+      - zookeeper-1
+      - zookeeper-2
+      - zookeeper-3
+      - kafka-1
+      - kafka-2
+      - kafka-3
+    restart: always
+
+  dashboard:
+    image: joway/burrow-dashboard
+    network_mode: host
+    ports:
+      - "80:80"
+    environment:
+      BURROW_BACKEND: http://localhost:8000
+    depends_on:
+      - burrow
+    restart: always
+
+  grafana:
+    image: grafana/grafana
+    network_mode: host
+    ports:
+      - "3000:3000"
+    volumes:
+      - grafana_data:/var/lib/grafana
+    depends_on:
+      - prometheus
+
+volumes:
+  grafana_data:
+  burrow_data:
--- a/jmx_prometheus_javaagent-0.11.0.jar
+++ b/jmx_prometheus_javaagent-0.11.0.jar
--- a/jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar
+++ b/jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar
--- a/16238
+++ b/16238
--- a/prometheus.yml
+++ b/prometheus.yml
@ -0,0 +1,20 @@
+global:
+  scrape_interval:     10s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+scrape_configs:
+  - job_name: 'kafka-1' # Job name
+    static_configs:
+      - targets: ['localhost:7071'] # mysqld_exporter address and port
+        labels:
+            alias: 'kafka-1' # alias name given to this instance
+
+  - job_name: 'kafka-2' # Job name
+    static_configs:
+      - targets: ['localhost:7072'] # mysqld_exporter address and port
+        labels:
+            alias: 'kafka-2' # alias name given to this instance
+
+  - job_name: 'kafka-3' # Job name
+    static_configs:
+      - targets: ['localhost:7073'] # mysqld_exporter address and port
+        labels:
+            alias: 'kafka-3' # alias name given to this instance