hadoop生态搭建（3节点）-08.kafka配置

如果之前没有安装jdk和zookeeper，安装了的请直接跳过

# https://www.oracle.com/technetwork/java/javase/downloads/java-archive-javase8-2177648.html

# ==================================================================安装 jdk

mkdir -p /usr/java

tar -zxvf ~/jdk-8u111-linux-x64.tar.gz -C /usr/java

rm -r ~/jdk-8u111-linux-x64.tar.gz

# http://archive.apache.org/dist/zookeeper/

# ==================================================================安装 zookeeper
# zookeeper集群搭建要至少3台服务器，服务器都要部署zookeeper

tar -zxvf ~/zookeeper-3.4.12.tar.gz -C /usr/local

rm -r ~/zookeeper-3.4.12.tar.gz

# https://www.scala-lang.org/download/2.12.4.html
# ==================================================================安装 scala

tar -zxvf ~/scala-2.12.4.tgz -C /usr/local

rm –r ~/scala-2.12.4.tgz

# http://archive.apache.org/dist/kafka/0.10.2.1/

# ==================================================================安装 kafka

tar -zxvf ~/kafka_2.12-0.10.2.1.tgz -C /usr/local

rm –r ~/kafka_2.12-0.10.2.1.tgz

# 环境变量
# ==================================================================node1 node2 node3

vi /etc/profile

# 在export PATH USER LOGNAME MAIL HOSTNAME HISTSIZE HISTCONTROL下添加

export JAVA_HOME=/usr/java/jdk1.8.0_111

export ZOOKEEPER_HOME=/usr/local/zookeeper-3.4.12

export HADOOP_HOME=/usr/local/hadoop/hadoop-2.7.6

export MYSQL_HOME=/usr/local/mysql

export HBASE_HOME=/usr/local/hbase-1.2.4

export HIVE_HOME=/usr/local/hive-2.1.1

export SCALA_HOME=/usr/local/scala-2.12.4

export KAFKA_HOME=/usr/local/kafka_2.12-0.10.2.1

export PATH=$PATH:$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$ZOOKEEPER_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$MYSQL_HOME/bin:$HBASE_HOME/bin:$HIVE_HOME/bin:$SCALA_HOME/bin:$KAFKA_HOME/bin

export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

export HADOOP_INSTALL=$HADOOP_HOME

export HADOOP_MAPRED_HOME=$HADOOP_HOME

export HADOOP_COMMON_HOME=$HADOOP_HOME

export HADOOP_HDFS_HOME=$HADOOP_HOME

export YARN_HOME=$HADOOP_HOME

export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native

# ==================================================================node1

# 使环境变量生效

source /etc/profile

# 查看配置结果

echo $KAFKA_HOME

# ==================================================================node2 node3

mkdir -p /usr/java

# ==================================================================node1

# jdk

# zookeeper

# https://www.cnblogs.com/zcf5522/p/9753539.html

# scala

scp -r $SCALA_HOME node2:/usr/local/

scp -r $SCALA_HOME node3:/usr/local/

# ==================================================================配置 kafka

mkdir $KAFKA_HOME/logs

# 配置 kafka server.properties

vi $KAFKA_HOME/config/server.properties

broker.id=1

delete.topic.enable=true

auto.create.topics.enable=true

listeners=PLAINTEXT://node1:9092

#listeners=SASL_PLAINTEXT://node1:9092

log.dirs=/usr/local/kafka_2.12-0.10.2.1/logs

zookeeper.connect=node1:2181,node2:2181,node3:2181/kafka

# 配置ACL入口类

#authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer

# 本例使用SASL_PLAINTEXT

#security.inter.broker.protocol=SASL_PLAINTEXT

#sasl.mechanism.inter.broker.protocol=PLAIN

#sasl.enabled.mechanisms=PLAIN

# 设置本例中admin为超级用户

#super.users=User:admin;User:kafka

scp -r $KAFKA_HOME node2:/usr/local/

scp -r $KAFKA_HOME node3:/usr/local/

# ==================================================================node2 node3

# 使环境变量生效

source /etc/profile

# 查看配置结果

echo $KAFKA_HOME

# ==================================================================node2

vi $KAFKA_HOME/config/server.properties

broker.id=2

listeners=PLAINTEXT://node2:9092

#listeners=SASL_PLAINTEXT://node2:9092

# ==================================================================node3

vi $KAFKA_HOME/config/server.properties

broker.id=3

listeners=PLAINTEXT://node3:9092

#listeners=SASL_PLAINTEXT://node3:9092

# kafka集群启动

# ==================================================================node1 node2 node3

# 启动zookeeper

zkServer.sh start

zkServer.sh status

# zkServer.sh stop

# ==================================================================node1

# 需要手动在ZooKeeper中创建路径/kafka，使用如下命令连接到任意一台ZooKeeper服务器

zkCli.sh

create /kafka ''

# ==================================================================node1 node2 node3

# 启动 kafka

kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties

# kafka-server-start.sh $KAFKA_HOME/config/server.properties &

# 验证启动进程

jps

# ==================================================================kafka集群测试1

# kafka-topics.sh --delete --zookeeper node1:2181,node2:2181,node3:2181/kafka --topic clustertopic1

# 创建topic

kafka-topics.sh --create --zookeeper node1:2181,node2:2181,node3:2181/kafka --replication-factor 3 --partitions 1 --topic clustertopic1

# 查看topic列表

kafka-topics.sh --list --zookeeper node1:2181,node2:2181,node3:2181/kafka

# 查询topic状态

kafka-topics.sh --describe --zookeeper node1:2181,node2:2181,node3:2181/kafka --topic clustertopic1

# 生产消息

kafka-console-producer.sh --broker-list node1:9092,node2:9092,node3:9092 --topic clustertopic1

> This is a message

> This is another message

# 消费消息

kafka-console-consumer.sh --bootstrap-server node1:9092,node2:9092,node3:9092 --from-beginning --topic clustertopic1

# 停止

kafka-server-stop.sh

zkServer.sh stop

# ==================================================================kafka集群测试2

# kafka-topics.sh --delete --zookeeper node1:2181,node2:2181,node3:2181/kafka --topic clustertopic2

# 创建topic

kafka-topics.sh --create --zookeeper node1:2181,node2:2181,node3:2181/kafka --replication-factor 3 --partitions 3 --topic clustertopic2

# 查看topic列表

kafka-topics.sh --list --zookeeper node1:2181/kafka

# 查询topic状态

kafka-topics.sh --describe --zookeeper node1:2181,node2:2181,node3:2181/kafka --topic clustertopic2

# 生产消息 在一台服务器上创建一个生产者

kafka-console-producer.sh --broker-list node1:9092,node2:9092,node3:9092 --topic clustertopic2

> This is a message

> This is another message

# 消费消息，在另外一个节点启动消费进程

kafka-console-consumer.sh --bootstrap-server node1:9092,node2:9092,node3:9092 --from-beginning --topic clustertopic2

# 停止

kafka-server-stop.sh

zkServer.sh stop

shutdown -h now

# 快照 kafka

# 以下配置可以暂时不用配置

# ==================================================================使用 SASL/PLAIN 认证

# ==================================================================node1

# 配置kafka server端（每个broker）

vi $KAFKA_HOME/config/server.properties

broker.id=1

delete.topic.enable=true

auto.create.topics.enable=true

#listeners=PLAINTEXT://node1:9092

listeners=SASL_PLAINTEXT://node1:9092

log.dirs=/usr/local/kafka_2.12-0.10.2.1/logs

zookeeper.connect=node1:2181,node2:2181,node3:2181/kafka

# SASL机制

security.inter.broker.protocol=SASL_PLAINTEXT

sasl.mechanism.inter.broker.protocol=PLAIN

sasl.enabled.mechanisms=PLAIN

# 完成身份验证的类

authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer

# 如果没有找到ACL（访问控制列表）配置，则允许任何操作。

allow.everyone.if.no.acl.found=true

# 设置admin为超级用户

super.users=User:admin;User:kafka

# ==================================================================node2

vi $KAFKA_HOME/config/server.properties

broker.id=2

delete.topic.enable=true

auto.create.topics.enable=true

#listeners=PLAINTEXT://node2:9092

listeners=SASL_PLAINTEXT://node2:9092

log.dirs=/usr/local/kafka_2.12-0.10.2.1/logs

zookeeper.connect=node1:2181,node2:2181,node3:2181/kafka

# SASL机制

security.inter.broker.protocol=SASL_PLAINTEXT

sasl.mechanism.inter.broker.protocol=PLAIN

sasl.enabled.mechanisms=PLAIN

# 完成身份验证的类

authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer

# 如果没有找到ACL（访问控制列表）配置，则允许任何操作。

allow.everyone.if.no.acl.found=true

# 设置admin为超级用户

super.users=User:admin;User:kafka

# ==================================================================node3

vi $KAFKA_HOME/config/server.properties

broker.id=3

delete.topic.enable=true

auto.create.topics.enable=true

#listeners=PLAINTEXT://node3:9092

listeners=SASL_PLAINTEXT://node3:9092

log.dirs=/usr/local/kafka_2.12-0.10.2.1/logs

zookeeper.connect=node1:2181,node2:2181,node3:2181/kafka

# SASL机制

security.inter.broker.protocol=SASL_PLAINTEXT

sasl.mechanism.inter.broker.protocol=PLAIN

sasl.enabled.mechanisms=PLAIN

# 完成身份验证的类

authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer

# 如果没有找到ACL（访问控制列表）配置，则允许任何操作。

allow.everyone.if.no.acl.found=true

# 设置admin为超级用户

super.users=User:admin;User:kafka

# ==================================================================node1

# user_userName必须配置admin用户

vi $KAFKA_HOME/config/kafka_server_jaas.conf

KafkaServer {

 org.apache.kafka.common.security.plain.PlainLoginModule required

 username="admin"

 password="Admin*123456"

 user_admin="Admin*123456"

 user_kafka="Kafka*123456"

 user_producer="Producer*123456"

 user_consumer="Consumer*123456";

};

# 这个配置定义了3个用户（kafka、producer和consumer）。在KafkaServer部分，username和password是broker用于初始化连接到其他的broker，

# 在这个例子中，kafka用户为broker间的通讯，useruserName定义了所有连接到broker和broker验证的所有的客户端连接包括其他broker的用户密码。

# useruserName必须配置admin用户，否则报错(useruserName必须配置kafka用户未验证)

# JAAS文件作为每个broker的jvm参数，在kafka-server-start.sh脚本中增加如下配置

vi $KAFKA_HOME/bin/kafka-server-start.sh

if [ "x$KAFKA_OPTS" = "x" ]; then

    export KAFKA_OPTS="-Djava.security.auth.login.config=/usr/local/kafka_2.12-0.10.2.1/config/kafka_server_jaas.conf"

fi

# 配置kafka client端 PLAIN机制下kafka生产者/消费者如何生产/消费

# kafka_client_jaas 可以分别建立不同如 producer_jaas.conf 和 consumer_jaas.conf

vi $KAFKA_HOME/config/kafka_client_jaas.conf

KafkaClient {

 org.apache.kafka.common.security.plain.PlainLoginModule required

 username="kafka"

 password="Kafka*123456";

};

# vi $KAFKA_HOME/config/producer_jaas.conf

#KafkaClient {

# org.apache.kafka.common.security.plain.PlainLoginModule required

# username="producer"

# password="Producer*123456";

#};

#vi $KAFKA_HOME/config/consumer_jaas.conf

#KafkaClient {

# org.apache.kafka.common.security.plain.PlainLoginModule required

# username="consumer"

# password="Consumer*123456";

#};

# KafkaClient部分，username和password是客户端用来配置客户端连接broker的用户，在这个例子中，客户端使用kafka 用户连接到broker。

# 修改 producer 和 consumer 的配置文件（配置注意空格不然报错）

vi $KAFKA_HOME/config/producer.properties

security.protocol=SASL_PLAINTEXT

sasl.mechanism=PLAIN

vi $KAFKA_HOME/config/consumer.properties

security.protocol=SASL_PLAINTEXT

sasl.mechanism=PLAIN

# 在启动 producer 和 consumer 时，分别添加jvm参数

vi $KAFKA_HOME/bin/kafka-console-consumer.sh

if [ "x$KAFKA_OPTS" = "x" ]; then

    export KAFKA_OPTS="-Djava.security.auth.login.config=/usr/local/kafka_2.12-0.10.2.1/config/kafka_client_jaas.conf"

fi

vi $KAFKA_HOME/bin/kafka-console-producer.sh

if [ "x$KAFKA_OPTS" = "x" ]; then

    export KAFKA_OPTS="-Djava.security.auth.login.config=/usr/local/kafka_2.12-0.10.2.1/config/kafka_client_jaas.conf"

fi

# 在启动 producer 和 consumer 时，分别添加jvm参数

#vi $KAFKA_HOME/bin/kafka-console-producer.sh

#if [ "x$KAFKA_OPTS" = "x" ]; then

#    export KAFKA_OPTS="-Djava.security.auth.login.config=/usr/local/kafka_2.12-0.10.2.1/config/producer_jaas.conf"

#fi

#vi $KAFKA_HOME/bin/kafka-console-consumer.sh

#if [ "x$KAFKA_OPTS" = "x" ]; then

#    export KAFKA_OPTS="-Djava.security.auth.login.config=/usr/local/kafka_2.12-0.10.2.1/config/consumer_jaas.conf"

#fi

# ==================================================================node1

scp -r $KAFKA_HOME/config/kafka_server_jaas.conf node2:$KAFKA_HOME/config/

scp -r $KAFKA_HOME/config/kafka_client_jaas.conf node2:$KAFKA_HOME/config/

scp -r $KAFKA_HOME/config/producer.properties node2:$KAFKA_HOME/config/

scp -r $KAFKA_HOME/config/consumer.properties node2:$KAFKA_HOME/config/

scp -r $KAFKA_HOME/bin/kafka-server-start.sh node2:$KAFKA_HOME/bin/

scp -r $KAFKA_HOME/bin/kafka-console-consumer.sh node2:$KAFKA_HOME/bin/

scp -r $KAFKA_HOME/bin/kafka-console-producer.sh node2:$KAFKA_HOME/bin/

scp -r $KAFKA_HOME/config/kafka_server_jaas.conf node3:$KAFKA_HOME/config/

scp -r $KAFKA_HOME/config/kafka_client_jaas.conf node3:$KAFKA_HOME/config/

scp -r $KAFKA_HOME/config/producer.properties node3:$KAFKA_HOME/config/

scp -r $KAFKA_HOME/config/consumer.properties node3:$KAFKA_HOME/config/

scp -r $KAFKA_HOME/bin/kafka-server-start.sh node3:$KAFKA_HOME/bin/

scp -r $KAFKA_HOME/bin/kafka-console-consumer.sh node3:$KAFKA_HOME/bin/

scp -r $KAFKA_HOME/bin/kafka-console-producer.sh node3:$KAFKA_HOME/bin/

# 启动

# ==================================================================node1 node2 node3

zkServer.sh start

zkServer.sh status

kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties

# ==================================================================node1

# kafka-topics.sh --delete --zookeeper node1:2181/kafka --topic sasltopic1

# 创建topic

kafka-topics.sh --create --zookeeper node1:2181/kafka --replication-factor 1 --partitions 1 --topic sasltopic1

# 查看topic列表

kafka-topics.sh --list --zookeeper node1:2181/kafka

# 查询topic状态

kafka-topics.sh --describe --zookeeper node1:2181/kafka --topic sasltopic1

# 生产者授权

kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=node1:2181/kafka --add --allow-principal User:kafka --operation Write --topic sasltopic1

# 生产者

kafka-console-producer.sh --broker-list node1:9092 --topic sasltopic1 --producer.config $KAFKA_HOME/config/producer.properties

> This is a message

> This is another message

# 消费者 主题Read授权

kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=node1:2181/kafka --add --allow-principal User:kafka --operation Read --topic sasltopic1

# 消费者 组Read授权

kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=node1:2181/kafka --add --allow-principal User:kafka --operation Read --group sasltopic1group1

# 消费者

kafka-console-consumer.sh --bootstrap-server node2:9092 --topic sasltopic1 --from-beginning --consumer.config $KAFKA_HOME/config/consumer.properties

# 停止

kafka-server-stop.sh

zkServer.sh stop

# 启动

# ==================================================================node1 node2 node3

zkServer.sh start

zkServer.sh status

kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties

# ==================================================================node1

# kafka-topics.sh --delete --zookeeper node1:2181/kafka --topic sasltopic2

# 创建topic

kafka-topics.sh --create --zookeeper node1:2181,node2:2181,node3:2181/kafka --replication-factor 3 --partitions 3 --topic sasltopic2

# 查看topic列表

kafka-topics.sh --list --zookeeper node1:2181/kafka

# 查询topic状态

kafka-topics.sh --describe --zookeeper node1:2181,node2:2181,node3:2181/kafka --topic sasltopic2

# 生产者授权

kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=node1:2181,node2:2181,node3:2181/kafka --add --allow-principal User:kafka --operation Write --topic sasltopic2

# 生产者

kafka-console-producer.sh --broker-list node1:9092,node2:9092,node3:9092 --topic sasltopic2 --producer.config $KAFKA_HOME/config/producer.properties

> This is a message

> This is another message

# 消费者 主题Read授权

kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=node1:2181,node2:2181,node3:2181/kafka --add --allow-principal User:kafka --operation Read --topic sasltopic2

# 消费者 组Read授权

kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=node1:2181,node2:2181,node3:2181/kafka --add --allow-principal User:kafka --operation Read --group sasltopic2group2

# 消费者

kafka-console-consumer.sh --bootstrap-server node1:9092,node2:9092,node3:9092 --topic sasltopic2 --from-beginning --consumer.config $KAFKA_HOME/config/consumer.properties

# 停止
kafka-server-stop.sh
zkServer.sh stop

shutdown -h now
# 快照 kafka_saslplain

# ==================================================================多节点zookeeper下认证 SASL认证配置

# ==================================================================node1

vi $ZOOKEEPER_HOME/conf/zoo.cfg

authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider

requireClientAuthScheme=sasl

jaasLoginRenew=3600000

# ==================================================================node2

vi $ZOOKEEPER_HOME/conf/zoo.cfg

authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider

requireClientAuthScheme=sasl

jaasLoginRenew=3600000

# ==================================================================node3

vi $ZOOKEEPER_HOME/conf/zoo.cfg

authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider

requireClientAuthScheme=sasl

jaasLoginRenew=3600000

# ==================================================================node1

vi $KAFKA_HOME/config/kafka_zoo_jaas.conf

ZKServer {

 org.apache.kafka.common.security.plain.PlainLoginModule required

 username="kafka"

 password="Kafka*123456"

 user_kafka="Kafka*123456";

};

vi $KAFKA_HOME/bin/zookeeper-server-start.sh

if [ "x$KAFKA_OPTS" = "x" ]; then

  export KAFKA_OPTS="-Djava.security.auth.login.config=/usr/local/kafka_2.12-0.10.2.1/config/kafka_zoo_jaas.conf -Dzookeeper.sasl.serverconfig=ZKServer"

fi

scp -r $KAFKA_HOME/config/kafka_zoo_jaas.conf node2:$KAFKA_HOME/config/

scp -r $KAFKA_HOME/config/kafka_zoo_jaas.conf node3:$KAFKA_HOME/config/

scp -r $KAFKA_HOME/bin/zookeeper-server-start.sh node2:$KAFKA_HOME/bin/

scp -r $KAFKA_HOME/bin/zookeeper-server-start.sh node3:$KAFKA_HOME/bin/

# 启动

# ==================================================================node1 node2 node3

zkServer.sh start

zkServer.sh status

kafka-server-start.sh -daemon $KAFKA_HOME/config/server.properties

# ==================================================================node1

# kafka-topics.sh --delete --zookeeper node1:2181/kafka --topic sasltopic3

# 创建topic

kafka-topics.sh --create --zookeeper node1:2181,node2:2181,node3:2181/kafka --replication-factor 3 --partitions 3 --topic sasltopic3

# 查看topic列表

kafka-topics.sh --list --zookeeper node1:2181/kafka

# 查询topic状态

kafka-topics.sh --describe --zookeeper node1:2181,node2:2181,node3:2181/kafka --topic sasltopic3

# 生产者授权

kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=node1:2181,node2:2181,node3:2181/kafka --add --allow-principal User:kafka --operation Write --topic sasltopic3

# 生产者

kafka-console-producer.sh --broker-list node1:9092,node2:9092,node3:9092 --topic sasltopic3 --producer.config $KAFKA_HOME/config/producer.properties

> This is a message

> This is another message

# 消费者 主题Read授权

kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=node1:2181,node2:2181,node3:2181/kafka --add --allow-principal User:kafka --operation Read --topic sasltopic3

# 消费者 组Read授权

kafka-acls.sh --authorizer kafka.security.auth.SimpleAclAuthorizer --authorizer-properties zookeeper.connect=node1:2181,node2:2181,node3:2181/kafka --add --allow-principal User:kafka --operation Read --group sasltopic3group3

# 消费者

kafka-console-consumer.sh --bootstrap-server node1:9092,node2:9092,node3:9092 --topic sasltopic3 --from-beginning --consumer.config $KAFKA_HOME/config/consumer.properties

# 停止

kafka-server-stop.sh

zkServer.sh stop

shutdown -h now

# 快照 kafka_morezookeeper_saslplain