https://docs.aws.amazon.com/ja_jp/msk/latest/developerguide/getting-started.html
フルマネージド型で高可用性の安全な Apache Kafka サービス
-- 1. コマンド等のインストール
-- 1.1 aws cli version 2 インストール
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip awscliv2.zip
sudo ./aws/install
aws --version
-- 1.2 jqインストール
sudo yum -y install jq
{
"BrokerNodeGroupInfo": {
"BrokerAZDistribution": "DEFAULT",
"ClientSubnets": [
"subnet-11111111111111111",
"subnet-22222222222222222"
],
"InstanceType": "kafka.t3.small",
"SecurityGroups": [
"sg-33333333333333333"
],
"StorageInfo": {
"EbsStorageInfo": {
"ProvisionedThroughput": {
"Enabled": false
},
"VolumeSize": 1
}
},
"ConnectivityInfo": {
"PublicAccess": {
"Type": "DISABLED"
}
}
},
"ClientAuthentication": {
"Sasl": {
"Scram": {
"Enabled": false
},
"Iam": {
"Enabled": false
}
},
"Tls": {
"CertificateAuthorityArnList": [],
"Enabled": false
},
"Unauthenticated": {
"Enabled": true
}
},
"EncryptionInfo": {
"EncryptionAtRest": {
"DataVolumeKMSKeyId": "arn:aws:kms:ap-northeast-1:999999999999:key/44444444-4444-4444-4444-444444444444"
},
"EncryptionInTransit": {
"ClientBroker": "PLAINTEXT",
"InCluster": false
}
},
"EnhancedMonitoring": "DEFAULT",
"OpenMonitoring": {
"Prometheus": {
"JmxExporter": {
"EnabledInBroker": false
},
"NodeExporter": {
"EnabledInBroker": false
}
}
},
"KafkaVersion": "2.6.2",
"LoggingInfo": {
"BrokerLogs": {
"CloudWatchLogs": {
"Enabled": false
},
"Firehose": {
"Enabled": false
},
"S3": {
"Enabled": false
}
}
},
"NumberOfBrokerNodes": 2
}
aws kafka create-cluster-v2 \
--cluster-name cluster01 \
--provisioned file://a.json \
aws kafka list-clusters-v2
aws kafka describe-cluster-v2 \
--cluster-arn arn:aws:kafka:ap-northeast-1:999999999999:cluster/cluster01/55555555-5555-5555-5555-555555555555-5
aws kafka get-bootstrap-brokers \
--cluster-arn arn:aws:kafka:ap-northeast-1:999999999999:cluster/cluster01/55555555-5555-5555-5555-555555555555-5
-- 3. トピックの作成
sudo yum -y install java-1.8.0
wget https://archive.apache.org/dist/kafka/2.6.2/kafka_2.12-2.6.2.tgz
tar -xzf kafka_2.12-2.6.2.tgz
cd kafka_2.12-2.6.2
./bin/kafka-topics.sh \
--create \
--bootstrap-server b-2.cluster01.666666.c4.kafka.ap-northeast-1.amazonaws.com:9092,b-1.cluster01.666666.c4.kafka.ap-northeast-1.amazonaws.com:9092 \
--replication-factor 2 \
--partitions 1 \
--topic topic01
-- 4. データの生成と消費
vim client.properties
security.protocol=PLAINTEXT
./bin/kafka-console-producer.sh \
--broker-list b-2.cluster01.666666.c4.kafka.ap-northeast-1.amazonaws.com:9092,b-1.cluster01.666666.c4.kafka.ap-northeast-1.amazonaws.com:9092 \
--producer.config client.properties \
--topic topic01
別端末を開く
cd kafka_2.12-2.6.2
./bin/kafka-console-consumer.sh \
--bootstrap-server b-2.cluster01.666666.c4.kafka.ap-northeast-1.amazonaws.com:9092,b-1.cluster01.666666.c4.kafka.ap-northeast-1.amazonaws.com:9092 \
--consumer.config client.properties \
--topic topic01 \
--from-beginning
-- 5. クリーンアップ
-- クラスターの削除
aws kafka list-clusters-v2
aws kafka describe-cluster-v2 \
--cluster-arn arn:aws:kafka:ap-northeast-1:999999999999:cluster/cluster01/55555555-5555-5555-5555-555555555555-5
aws kafka delete-cluster \
--cluster-arn arn:aws:kafka:ap-northeast-1:999999999999:cluster/cluster01/55555555-5555-5555-5555-555555555555-5