yutsuo
/
kafka
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

Phase V

master
Yutsuo 7 years ago
commit
a261ed7acd
10 changed files with 16642 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      .gitignore
  2. 7
      Dockerfile
  3. 29
      burrow.toml
  4. 86
      config-2.yml
  5. 106
      config.yml
  6. 155
      docker-compose.yml
  7. BIN
      jmx_prometheus_javaagent-0.11.0.jar
  8. BIN
      jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar
  9. 16238
      metrics
  10. 20
      prometheus.yml

1
.gitignore vendored
Unescape View File

@ -0,0 +1 @@
.vscode/*

7
Dockerfile
Unescape View File

@ -0,0 +1,7 @@
FROM confluentinc/cp-kafka
WORKDIR /jmx
COPY jmx_prometheus_javaagent-0.11.0.jar /jmx
# RUN java -javaagent:/jmx/jmx_prometheus_javaagent-0.11.0.jar=8080:config.yaml -jar yourJar.jar

29
burrow.toml
Unescape View File

@ -0,0 +1,29 @@
[zookeeper]
servers=[ "localhost:22181", "localhost:32181", "localhost:42181" ]
timeout=6
root-path="/burrow"
[cluster.local]
class-name="kafka"
servers=[ "localhost:19092", "localhost:29092", "localhost:39092" ]
topic-refresh=10
offset-refresh=10
[consumer.local]
class-name="kafka"
cluster="local"
servers=[ "localhost:19092", "localhost:29092", "localhost:39092" ]
group-blacklist="^(console-consumer-|python-kafka-consumer-).*$"
group-whitelist=""
[consumer.local_zk]
class-name="kafka_zk"
cluster="local"
servers=[ "localhost:22181", "localhost:32181", "localhost:42181" ]
zookeeper-path="/local"
zookeeper-timeout=30
group-blacklist="^(console-consumer-|python-kafka-consumer-).*$"
group-whitelist=""
[httpserver.default]
address=":8000"

86
config-2.yml
Unescape View File

@ -0,0 +1,86 @@
lowercaseOutputName: true
rules:
# Special cases and very specific rules
- pattern : kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value
name: kafka_server_$1_$2
type: GAUGE
labels:
clientId: "$3"
topic: "$4"
partition: "$5"
- pattern : kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value
name: kafka_server_$1_$2
type: GAUGE
labels:
clientId: "$3"
broker: "$4:$5"
# Generic per-second counters with 0-2 key/value pairs
- pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+), (.+)=(.+)><>Count
name: kafka_$1_$2_$3_total
type: COUNTER
labels:
"$4": "$5"
"$6": "$7"
- pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, (.+)=(.+)><>Count
name: kafka_$1_$2_$3_total
type: COUNTER
labels:
"$4": "$5"
type: COUNTER
- pattern: kafka.(\w+)<type=(.+), name=(.+)PerSec\w*><>Count
name: kafka_$1_$2_$3_total
type: COUNTER
# Generic gauges with 0-2 key/value pairs
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Value
name: kafka_$1_$2_$3
type: GAUGE
labels:
"$4": "$5"
"$6": "$7"
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Value
name: kafka_$1_$2_$3
type: GAUGE
labels:
"$4": "$5"
- pattern: kafka.(\w+)<type=(.+), name=(.+)><>Value
name: kafka_$1_$2_$3
type: GAUGE
# Emulate Prometheus 'Summary' metrics for the exported 'Histogram's.
#
# Note that these are missing the '_sum' metric!
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+), (.+)=(.+)><>Count
name: kafka_$1_$2_$3_count
type: COUNTER
labels:
"$4": "$5"
"$6": "$7"
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*), (.+)=(.+)><>(\d+)thPercentile
name: kafka_$1_$2_$3
type: GAUGE
labels:
"$4": "$5"
"$6": "$7"
quantile: "0.$8"
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.+)><>Count
name: kafka_$1_$2_$3_count
type: COUNTER
labels:
"$4": "$5"
- pattern: kafka.(\w+)<type=(.+), name=(.+), (.+)=(.*)><>(\d+)thPercentile
name: kafka_$1_$2_$3
type: GAUGE
labels:
"$4": "$5"
quantile: "0.$6"
- pattern: kafka.(\w+)<type=(.+), name=(.+)><>Count
name: kafka_$1_$2_$3_count
type: COUNTER
- pattern: kafka.(\w+)<type=(.+), name=(.+)><>(\d+)thPercentile
name: kafka_$1_$2_$3
type: GAUGE
labels:
quantile: "0.$4"

106
config.yml
Unescape View File

@ -0,0 +1,106 @@
# This has been tested on Kafka client 1.1.1, commidId: 98b6346a977495f6
# Metrics ending with "-total" are counters.
# Everything else are gauges
# Explaination of regular expression: (((?!(-total|-rate|\.)).)*)
# this exclude metrics with names "-total", "-rate" and "."
# "-total": already handled by the (-total) rule
# "-rate": calculated by the client. Not needed since we can do the same with the rate()
# ".": metrics w/ a "." in the name have be deprecated.
#
# eg:
# "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=client-1:topic-1-0.records-lag-max"
# has been repplace by
# "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=client-1,topic=topic-1,partition=0:records-lag-max"
lowercaseOutputName: true
rules:
#-----------------------------------------------------------------------------
# Start w/ the special metrics, then most keys to least keys
#-----------------------------------------------------------------------------
# Special metric: We just want to track the client version/commid-id number
# kafka.consumer:type=app-info,client-id=consumer-1:version
# kafka.consumer:type=app-info,client-id=consumer-1:commid-id
# kafka.producer:type=app-info,client-id=producer-1:version
# kafka.producer:type=app-info,client-id=producer-1:commid-id
- pattern: kafka.(\w+)<type=(app-info), client-id=(.+)><>(.+):\s?(.+)
name: kafka_$1_$2
value: 0
labels:
"client-id": "$3"
"$4": "$5"
#-----------------------------------------------------------------------------
# metrics w/ 3 keys
#-----------------------------------------------------------------------------
# dont think there are COUNTER metrics w/ 3 keys. Leaving here in case new client version has them
- pattern: kafka.(\w+)<type=(.+), (.+)=(.+), (.+)=(.+), (.+)=(.+)><>(.+-total)
name: kafka_$1_$2_$9
type: COUNTER
labels:
"$3": "$4"
"$5": "$6"
"$7": "$8"
# kafka.consumer:type=consumer-fetch-manager-metrics,client-id=consumer-1,topic=barTopic,partition=0:records-lag
- pattern: kafka.(\w+)<type=(.+), (.+)=(.+), (.+)=(.+), (.+)=(.+)><>(((?!(-total|-rate|\.)).)*)(:)
name: kafka_$1_$2_$9
type: GAUGE
labels:
"$3": "$4"
"$5": "$6"
"$7": "$8"
#-----------------------------------------------------------------------------
# metrics w/ 2 keys
#-----------------------------------------------------------------------------
# kafka.consumer:type=consumer-fetch-manager-metrics,client-id=consumer-1:bytes-consumed-total
# kafka.consumer:type=consumer-node-metrics,client-id=consumer-1,node-id=node-111:incoming-byte-total
# kafka.producer:type=producer-node-metrics,client-id=producer-1,node-id=node-121:outgoing-byte-total
# kafka.producer:type=producer-topic-metrics,client-id=producer-1,topic=barTopic:record-send-total
- pattern: kafka.(\w+)<type=(.+), (.+)=(.+), (.+)=(.+)><>(.+-total)
name: kafka_$1_$2_$7
type: COUNTER
labels:
"$3": "$4"
"$5": "$6"
# kafka.consumer:type=consumer-fetch-manager-metrics,client-id=consumer-1:fetch-latency-avg
# kafka.consumer:type=consumer-node-metrics,client-id=consumer-1,node-id=node-111:request-latency-max
# kafka.producer:type=producer-node-metrics,client-id=producer-1,node-id=node-121:request-size-avg
# kafka.producer:type=producer-topic-metrics,client-id=producer-1,topic=barTopic:compression-rate
- pattern: kafka.(\w+)<type=(.+), (.+)=(.+), (.+)=(.+)><>(((?!(-total|-rate|\.)).)*)(:)
name: kafka_$1_$2_$7
type: GAUGE
labels:
"$3": "$4"
"$5": "$6"
#-----------------------------------------------------------------------------
# metrics w/ 1 key
#-----------------------------------------------------------------------------
# kafka.consumer:type=consumer-coordinator-metrics,client-id=consumer-1:heartbeat-total
# kafka.consumer:type=consumer-fetch-manager-metrics,client-id=consumer-1:bytes-consumed-total
# kafka.consumer:type=consumer-metrics,client-id=consumer-1:incoming-byte-total
# kafka.producer:type=producer-metrics,client-id=producer-1:request-total
- pattern: kafka.(\w+)<type=(.+), (.+)=(.+)><>(.+-total)
name: kafka_$1_$2_$5
type: COUNTER
labels:
"$3": "$4"
# kafka.consumer:type=consumer-coordinator-metrics,client-id=consumer-1:assigned-partitions
# kafka.consumer:type=consumer-fetch-manager-metrics,client-id=consumer-1:fetch-latency-avg
# kafka.consumer:type=consumer-metrics,client-id=consumer-1:request-size-avg
# kafka.producer:type=producer-metrics,client-id=producer-1:compression-rate-avg
- pattern: kafka.(\w+)<type=(.+), (.+)=(.+)><>(((?!(-total|-rate|\.)).)*)(:)
name: kafka_$1_$2_$5
type: GAUGE
labels:
"$3": "$4"

155
docker-compose.yml
Unescape View File

@ -0,0 +1,155 @@
---
version: '2'
services:
zookeeper-1:
image: confluentinc/cp-zookeeper:latest
environment:
ZOOKEEPER_SERVER_ID: 1
ZOOKEEPER_CLIENT_PORT: 22181
ZOOKEEPER_TICK_TIME: 2000
ZOOKEEPER_INIT_LIMIT: 5
ZOOKEEPER_SYNC_LIMIT: 2
ZOOKEEPER_SERVERS: localhost:22888:23888;localhost:32888:33888;localhost:42888:43888
network_mode: host
extra_hosts:
- "moby:127.0.0.1"
zookeeper-2:
image: confluentinc/cp-zookeeper:latest
environment:
ZOOKEEPER_SERVER_ID: 2
ZOOKEEPER_CLIENT_PORT: 32181
ZOOKEEPER_TICK_TIME: 2000
ZOOKEEPER_INIT_LIMIT: 5
ZOOKEEPER_SYNC_LIMIT: 2
ZOOKEEPER_SERVERS: localhost:22888:23888;localhost:32888:33888;localhost:42888:43888
network_mode: host
extra_hosts:
- "moby:127.0.0.1"
zookeeper-3:
image: confluentinc/cp-zookeeper:latest
environment:
ZOOKEEPER_SERVER_ID: 3
ZOOKEEPER_CLIENT_PORT: 42181
ZOOKEEPER_TICK_TIME: 2000
ZOOKEEPER_INIT_LIMIT: 5
ZOOKEEPER_SYNC_LIMIT: 2
ZOOKEEPER_SERVERS: localhost:22888:23888;localhost:32888:33888;localhost:42888:43888
network_mode: host
extra_hosts:
- "moby:127.0.0.1"
kafka-1:
image: confluentinc/cp-kafka:latest
network_mode: host
depends_on:
- zookeeper-1
- zookeeper-2
- zookeeper-3
volumes:
- ./jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar:/usr/src/jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar
- ./config.yml:/usr/src/config.yml
- ./config.yml:/usr/src/config-2.yml
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: localhost:22181,localhost:32181,localhost:42181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:19092
# KAFKA_OPTS: -javaagent:/usr/src/jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar=7071:/usr/src/config.yml
KAFKA_OPTS: -javaagent:/usr/src/jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar=7071:/usr/src/config-2.yml
extra_hosts:
- "moby:127.0.0.1"
kafka-2:
image: confluentinc/cp-kafka:latest
network_mode: host
depends_on:
- zookeeper-1
- zookeeper-2
- zookeeper-3
volumes:
- ./jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar:/usr/src/jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar
- ./config.yml:/usr/src/config.yml
- ./config.yml:/usr/src/config-2.yml
environment:
KAFKA_BROKER_ID: 2
KAFKA_ZOOKEEPER_CONNECT: localhost:22181,localhost:32181,localhost:42181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:29092
# KAFKA_OPTS: -javaagent:/usr/src/jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar=7072:/usr/src/config.yml
KAFKA_OPTS: -javaagent:/usr/src/jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar=7072:/usr/src/config-2.yml
extra_hosts:
- "moby:127.0.0.1"
kafka-3:
image: confluentinc/cp-kafka:latest
network_mode: host
depends_on:
- zookeeper-1
- zookeeper-2
- zookeeper-3
volumes:
- ./jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar:/usr/src/jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar
- ./config.yml:/usr/src/config.yml
- ./config.yml:/usr/src/config-2.yml
environment:
KAFKA_BROKER_ID: 3
KAFKA_ZOOKEEPER_CONNECT: localhost:22181,localhost:32181,localhost:42181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:39092
# KAFKA_OPTS: -javaagent:/usr/src/jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar=7073:/usr/src/config.yml
KAFKA_OPTS: -javaagent:/usr/src/jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar=7073:/usr/src/config-2.yml
extra_hosts:
- "moby:127.0.0.1"
prometheus:
image: prom/prometheus
network_mode: host
ports:
- "9090:9090"
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml
depends_on:
- kafka-1
- kafka-2
- kafka-3
burrow:
image: artemz/burrow
network_mode: host
volumes:
- ./burrow.toml:/etc/burrow/burrow.toml
- burrow_data:/var/tmp/burrow
ports:
- "8000:8000"
depends_on:
- zookeeper-1
- zookeeper-2
- zookeeper-3
- kafka-1
- kafka-2
- kafka-3
restart: always
dashboard:
image: joway/burrow-dashboard
network_mode: host
ports:
- "80:80"
environment:
BURROW_BACKEND: http://localhost:8000
depends_on:
- burrow
restart: always
grafana:
image: grafana/grafana
network_mode: host
ports:
- "3000:3000"
volumes:
- grafana_data:/var/lib/grafana
depends_on:
- prometheus
volumes:
grafana_data:
burrow_data:

BIN
jmx_prometheus_javaagent-0.11.0.jar
View File

Binary file not shown.

BIN
jmx_prometheus_javaagent-0.11.1-SNAPSHOT.jar
View File

Binary file not shown.

16238
metrics
View File

File diff suppressed because it is too large Load Diff

20
prometheus.yml
Unescape View File

@ -0,0 +1,20 @@
global:
scrape_interval: 10s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
scrape_configs:
- job_name: 'kafka-1' # Job name
static_configs:
- targets: ['localhost:7071'] # mysqld_exporter address and port
labels:
alias: 'kafka-1' # alias name given to this instance
- job_name: 'kafka-2' # Job name
static_configs:
- targets: ['localhost:7072'] # mysqld_exporter address and port
labels:
alias: 'kafka-2' # alias name given to this instance
- job_name: 'kafka-3' # Job name
static_configs:
- targets: ['localhost:7073'] # mysqld_exporter address and port
labels:
alias: 'kafka-3' # alias name given to this instance
Write Preview
Loading…
Cancel
Save