https://docs.aws.amazon.com/ja_jp/sns/latest/dg/sns-firehose-as-subscriber.html
SNS --> Kinesis Data Firehose --> S3
-- 1. コマンド等のインストール
-- 1.1 aws cli version 2 インストール
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip awscliv2.zip
sudo ./aws/install
aws --version
-- 1.2 jqインストール
sudo yum -y install jq
-- 2. S3 バケットを作成する
aws s3 mb s3://bucket123
aws s3 ls
-- 3. CloudWatchロググループの作成
aws logs create-log-group --log-group-name /aws/kinesisfirehose/hose01
aws logs describe-log-groups --log-group-name-prefix /aws/kinesisfirehose/hose01
aws logs put-retention-policy \
--log-group-name /aws/kinesisfirehose/hose01 \
--retention-in-days 1
-- 4. CloudWatchログストリームの作成
aws logs create-log-stream \
--log-group-name /aws/kinesisfirehose/hose01 \
--log-stream-name stream01
aws logs describe-log-streams \
--log-group-name /aws/kinesisfirehose/hose01
-- 5. IAMポリシー作成(firehose用)
vim policy01.json
{
"Version": "2012-10-17",
"Statement":
[
{
"Effect": "Allow",
"Action": [
"s3:AbortMultipartUpload",
"s3:GetBucketLocation",
"s3:GetObject",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:PutObject"
],
"Resource": [
"arn:aws:s3:::bucket123",
"arn:aws:s3:::bucket123/*"
]
},
{
"Effect": "Allow",
"Action": [
"logs:PutLogEvents"
],
"Resource": [
"arn:aws:logs:ap-northeast-1:999999999999:log-group:/aws/kinesisfirehose/hose01:log-stream:stream01"
]
}
]
}
aws iam create-policy \
--policy-name policy01 \
--policy-document file://policy01.json
-- 6. IAMロール作成(firehose用)
vim role01.json
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": "firehose.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
aws iam create-role \
--role-name role01 \
--assume-role-policy-document file://role01.json
-- 7. ポリシーをロールにアタッチ
aws iam attach-role-policy \
--policy-arn arn:aws:iam::999999999999:policy/policy01 \
--role-name role01
-- 8. Amazon Kinesis Data Firehose 配信ストリームの作成
{
"RoleARN": "arn:aws:iam::999999999999:role/role01",
"BucketARN": "arn:aws:s3:::bucket123",
"Prefix": "",
"ErrorOutputPrefix": "",
"BufferingHints": {
"SizeInMBs": 1,
"IntervalInSeconds": 60
},
"CompressionFormat": "UNCOMPRESSED",
"EncryptionConfiguration": {
"NoEncryptionConfig": "NoEncryption"
},
"CloudWatchLoggingOptions": {
"Enabled": true,
"LogGroupName": "/aws/kinesisfirehose/hose01",
"LogStreamName": "stream01"
},
"ProcessingConfiguration": {
"Enabled": false,
"Processors": []
},
"S3BackupMode": "Disabled",
"DataFormatConversionConfiguration": {
"Enabled": false
}
}
aws firehose create-delivery-stream \
--delivery-stream-name hose01 \
--delivery-stream-type DirectPut \
--extended-s3-destination-configuration file://a.json
aws firehose describe-delivery-stream \
--delivery-stream-name hose01
-- 9. IAMポリシー作成(sns用)
vim policy02.json
{
"Version": "2012-10-17",
"Statement": [
{
"Action": [
"firehose:DescribeDeliveryStream",
"firehose:ListDeliveryStreams",
"firehose:ListTagsForDeliveryStream",
"firehose:PutRecord",
"firehose:PutRecordBatch"
],
"Resource": [
"arn:aws:firehose:ap-northeast-1:999999999999:deliverystream/hose01"
],
"Effect": "Allow"
}
]
}
aws iam create-policy \
--policy-name policy02 \
--policy-document file://policy02.json
-- 10. IAMロール作成(sns用)
vim role02.json
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": "sns.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
aws iam create-role \
--role-name role02 \
--assume-role-policy-document file://role02.json
-- 11. ポリシーをロールにアタッチ
aws iam attach-role-policy \
--policy-arn arn:aws:iam::999999999999:policy/policy02 \
--role-name role02
-- 12. SNSトピック作成
aws sns list-topics
aws sns list-subscriptions
aws sns create-topic --name topic01
aws sns subscribe \
--topic-arn arn:aws:sns:ap-northeast-1:999999999999:topic01 \
--protocol firehose \
--attributes RawMessageDelivery=true,SubscriptionRoleArn=arn:aws:iam::999999999999:role/role02 \
--notification-endpoint arn:aws:firehose:ap-northeast-1:999999999999:deliverystream/hose01
-- 13. 動作確認
aws sns publish \
--topic-arn arn:aws:sns:ap-northeast-1:999999999999:topic01 \
--message '{"BookingDate":"2020-12-15","BookingTime":"2020-12-15 04:15:05","Destination":"Miami","FlyingFrom":"Vancouver","TicketNumber":"abcd1234"}' \
--subject "subject01"
aws s3 ls s3://bucket123 --recursive
aws s3 cp s3://bucket123/2022/03/15/16/hose01-1-2022-03-15-16-51-05-3ed551e2-c472-43b8-be11-e3b6281534b9 -
-- 14. クリーンアップ
-- SNSトピック削除
aws sns unsubscribe --subscription-arn arn:aws:sns:ap-northeast-1:999999999999:topic01:11111111-2222-3333-4444-555555555555
aws sns delete-topic --topic-arn arn:aws:sns:ap-northeast-1:999999999999:topic01
aws sns list-topics
aws sns list-subscriptions
-- Amazon Kinesis Data Firehose 配信ストリームの削除
aws firehose describe-delivery-stream \
--delivery-stream-name hose01
aws firehose delete-delivery-stream \
--delivery-stream-name hose01 \
--allow-force-delete
-- IAMロールの削除
aws iam list-roles | grep role01
aws iam list-roles | grep role02
aws iam detach-role-policy \
--role-name role01 \
--policy-arn arn:aws:iam::999999999999:policy/policy01
aws iam detach-role-policy \
--role-name role02 \
--policy-arn arn:aws:iam::999999999999:policy/policy02
aws iam delete-role --role-name role01
aws iam delete-role --role-name role02
-- IAMポリシーの削除
aws iam list-policies | grep policy01
aws iam list-policies | grep policy02
aws iam delete-policy \
--policy-arn arn:aws:iam::999999999999:policy/policy01
aws iam delete-policy \
--policy-arn arn:aws:iam::999999999999:policy/policy02
-- CloudWatchロググループの削除
aws logs describe-log-groups --log-group-name-prefix /aws/kinesisfirehose/hose01
aws logs delete-log-group --log-group-name /aws/kinesisfirehose/hose01
-- バケットの削除
aws s3 ls
aws s3 rb s3://bucket123 --force
