아래는 ZooKeeper 기반 구성 기록이다. Kafka 4.0 부터 ZooKeeper 가 제거돼
zookeeper.connect는 더 이상 쓰지 않는다.
현행 버전 구성은 Kafka 4.3.1 설치 가이드 (KRaft) 를 볼 것.
3.x 이하를 운영 중이라면 이 문서가 그대로 유효하다.
#!/bin/bash
# Kafka server list (Kafka 서버 목록, 공백으로 구분)
KAFKA_SERVER_LIST="***01 ***02 ***03"
# Kafka Owner (Kafka 서버를 실행할 Linux user)
KAFKA_OWNER="manager"
# Kafka home directory (Kafka 설치 경로)
KAFKA_HOME="/home/manager/kafka"
#################################################################################################
NO_COLOR='\033[0m'
RED_COLOR='\033[0;31m'
GREEN_COLOR='\033[0;32m'
SIGNAL=${SIGNAL:-TERM}
# PID getter
pid_getter(){
PIDS=$(ssh -o LogLevel=error $KAFKA_OWNER@$i "ps ax | grep $KAFKA_HOME/bin | grep java | grep -v grep | awk '{print \$1}'")
echo "${PIDS}"
}
if [ $# -ne 1 ];
then
echo "USAGE: $0 [ start | stop | status ]"
exit 1
fi
if [ $1 = "start" ];
then
for i in $KAFKA_SERVER_LIST
do
PIDS=$(pid_getter $i)
if [ -z "$PIDS" ]; then
echo
echo "[$i] Kafka server is starting."
START_RESULT=$(ssh -o LogLevel=error $KAFKA_OWNER@$i "$KAFKA_HOME/bin/kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties")
sleep 1s
PRINT_PIDS=$(pid_getter $i)
if [ -z "$PRINT_PIDS" ]; then
echo -e "[$i] Kafka server startup ${RED_COLOR}failed.${NO_COLOR}"
else
echo -e "[$i] Kafka server started ${GREEN_COLOR}successful.${NO_COLOR}"
echo "[$i] Kafka server pid is $PRINT_PIDS"
fi
else
echo "[$i] Kafka server is already running."
fi
done
# Kafka Stop
elif [ $1 = "stop" ];
then
echo -e "[${RED_COLOR}Danger${NO_COLOR}] All servers will be shut down."
for i in $KAFKA_SERVER_LIST
do
PIDS=$(pid_getter $i)
echo
if [ -z "$PIDS" ]; then
echo "[$i] kafka server is not running."
else
ssh -o LogLevel=error $KAFKA_OWNER@$i "kill -s $SIGNAL $PIDS"
echo "[$i] Kafka server has been shut down."
fi
done
elif [ $1 = "status" ];
then
for i in $KAFKA_SERVER_LIST
do
PIDS=$(pid_getter $i)
echo
if [ -z "$PIDS" ]; then
echo -e "[$i] Kafka server is ${RED_COLOR}stopped.${NO_COLOR}"
else
echo -e "[$i] Kafka server is ${GREEN_COLOR}running.${NO_COLOR}"
echo -e "[$i] pid is $PIDS"
fi
done
else
echo "USAGE: $0 [ start | stop | status ]"
exit 1
fi
broker.id=1
listeners=PLAINTEXT://***.***.***.114:9092
api.version.request=false
auto.commit.enable=false
delete.topic.enable=true
auto.create.topics.enable=false
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/home/manager/kafka/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=72
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=***.***.***.114:12181,***.***.***.115:12181,***.***.***.***:12181
zookeeper.connection.timeout.ms=18000
message.max.bytes=157286400
max.message.bytes=157286400
group.initial.rebalance.delay.ms=0
#!/usr/bin/env bash
# use POSIX interface, symlink is followed automatically
#ZOOKEEPER_HOME=$HOME/kafka-zookeeper
ZOOKEEPER_HOME=$HOME/kafka-zk
ZOOBIN=$ZOOKEEPER_HOME/bin
ZOOBINDIR="$(cd "${ZOOBIN}"; pwd)"
SERVER1=***01
SERVER2=***02
SERVER3=***03
case $1 in
start)
ssh manager@$SERVER1 "$ZOOKEEPER_HOME/bin/zkServer.sh start"
echo -n "Starting zookeeper ... server 2"
ssh manager@$SERVER2 "$ZOOKEEPER_HOME/bin/zkServer.sh start"
echo -n "Starting zookeeper ... server 3"
ssh manager@$SERVER3 "$ZOOKEEPER_HOME/bin/zkServer.sh start"
exit 0
;;
stop)
echo -n "Trying to stop zookeeper ... server 1"
echo .
ssh manager@$SERVER1 "$ZOOKEEPER_HOME/bin/zkServer.sh stop"
echo -n "Trying to stop zookeeper ... server 2"
echo .
ssh manager@$SERVER2 "$ZOOKEEPER_HOME/bin/zkServer.sh stop"
echo -n "Trying to stop zookeeper ... server 3"
echo .
ssh manager@$SERVER3 "$ZOOKEEPER_HOME/bin/zkServer.sh stop"
exit 0
;;
restart)
echo -n "Trying to restart zookeeper ... server 1"
echo .
ssh manager@$SERVER1 "$ZOOKEEPER_HOME/bin/zkServer.sh restart"
echo -n "Trying to restart zookeeper ... server 2"
echo .
ssh manager@$SERVER2 "$ZOOKEEPER_HOME/bin/zkServer.sh restart"
echo -n "Trying to restart zookeeper ... server 3"
echo .
ssh manager@$SERVER3 "$ZOOKEEPER_HOME/bin/zkServer.sh restart"
exit 0
;;
*)
echo "Usage: $0 {start|stop||restart}" >&2
esac
# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial
# synchronization phase can take
initLimit=10
# The number of ticks that can pass between
# sending a request and getting an acknowledgement
syncLimit=5
# the directory where the snapshot is stored.
# do not use /tmp for storage, /tmp here is just
# example sakes.
#dataDir=/tmp/zookeeper
dataDir=/home/manager/kafka-zk/zookeeper-data
# the port at which the clients will connect
clientPort=12181
# the maximum number of client connections.
# increase this if you need to handle more clients
#maxClientCnxns=60
#
# Be sure to read the maintenance section of the
# administrator guide before turning on autopurge.
#
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
# The number of snapshots to retain in dataDir
#autopurge.snapRetainCount=3
# Purge task interval in hours
# Set to "0" to disable auto purge feature
#autopurge.purgeInterval=1
## Metrics Providers
#
# https://prometheus.io Metrics Exporter
#metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider
#metricsProvider.httpPort=7000
#metricsProvider.exportJvmInfo=true
#server.1=bramtr01:12888:13888
server.1=***.***.***.***:12888:13888
server.2=***.***.***.***:12888:13888
server.3=***.***.***.***:12888:13888
_
#!/bin/bash
export JAVA_HOME=$HOME/java/jdk-11.0.2
nohup $HOME/cmak/bin/cmak -Dconfig.file=$HOME/cmak/conf/application.conf -Dhttp.port=58080 2>&1 &
play.crypto.secret="${REDACTED}"
play.crypto.secret=${?APPLICATION_SECRET}
play.http.session.maxAge="1h"
# The application languages
# ~~~~~
play.i18n.langs=["en"]
play.http.requestHandler = "play.http.DefaultHttpRequestHandler"
play.http.context = "/"
play.application.loader=loader.KafkaManagerLoader
# Settings prefixed with 'kafka-manager.' will be deprecated, use 'cmak.' instead.
# https://github.com/yahoo/CMAK/issues/713
#kafka-manager.zkhosts="kafka-manager-zookeeper:2181"
kafka-manager.zkhosts="***.***.***.114:12181,***.***.***.115:12181,***.***.***.***:12181"
kafka-manager.zkhosts=${?ZK_HOSTS}
#cmak.zkhosts="kafka-manager-zookeeper:2181"
cmak.zkhosts="***.***.***.114:12181,***.***.***.115:12181,***.***.***.***:12181"
cmak.zkhosts=${?ZK_HOSTS}
pinned-dispatcher.type="PinnedDispatcher"
pinned-dispatcher.executor="thread-pool-executor"
application.features=["KMClusterManagerFeature","KMTopicManagerFeature","KMPreferredReplicaElectionFeature","KMReassignPartitionsFeature", "KMScheduleLeaderElectionFeature"]
akka {
loggers = ["akka.event.slf4j.Slf4jLogger"]
loglevel = "INFO"
}
akka.logger-startup-timeout = 60s
basicAuthentication.enabled=false
basicAuthentication.enabled=${?KAFKA_MANAGER_AUTH_ENABLED}
basicAuthentication.ldap.enabled=false
basicAuthentication.ldap.enabled=${?KAFKA_MANAGER_LDAP_ENABLED}
basicAuthentication.ldap.server=""
basicAuthentication.ldap.server=${?KAFKA_MANAGER_LDAP_SERVER}
basicAuthentication.ldap.port=389
basicAuthentication.ldap.port=${?KAFKA_MANAGER_LDAP_PORT}
basicAuthentication.ldap.username=""
basicAuthentication.ldap.username=${?KAFKA_MANAGER_LDAP_USERNAME}
basicAuthentication.ldap.password=""
basicAuthentication.ldap.password=${?KAFKA_MANAGER_LDAP_PASSWORD}
basicAuthentication.ldap.search-base-dn=""
basicAuthentication.ldap.search-base-dn=${?KAFKA_MANAGER_LDAP_SEARCH_BASE_DN}
basicAuthentication.ldap.search-filter="(uid=$capturedLogin$)"
basicAuthentication.ldap.search-filter=${?KAFKA_MANAGER_LDAP_SEARCH_FILTER}
basicAuthentication.ldap.group-filter=""
basicAuthentication.ldap.group-filter=${?KAFKA_MANAGER_LDAP_GROUP_FILTER}
basicAuthentication.ldap.connection-pool-size=10
basicAuthentication.ldap.connection-pool-size=${?KAFKA_MANAGER_LDAP_CONNECTION_POOL_SIZE}
basicAuthentication.ldap.ssl=false
basicAuthentication.ldap.ssl=${?KAFKA_MANAGER_LDAP_SSL}
basicAuthentication.username="admin"
basicAuthentication.username=${?KAFKA_MANAGER_USERNAME}
basicAuthentication.password="password"
basicAuthentication.password=${?KAFKA_MANAGER_PASSWORD}
basicAuthentication.realm="Kafka-Manager"
basicAuthentication.excluded=["/api/health"] # ping the health of your instance without authentification
kafka-manager.consumer.properties.file=${?CONSUMER_PROPERTIES_FILE}
_