{S3}イベント通知の有効化 - HTN20190109の日記

https://www.ooooouchi.info/entry/2020/05/25/080000
https://dev.classmethod.jp/articles/lim-s3-event-notifications/
https://blog.denet.co.jp/convert-charactercode-in-lambda-using-s3event-notification/
https://sayjoyblog.com/aws-cli-sqs-handson/#2

Amazon S3 は次のイベントの通知を発行できます
・新しいオブジェクトの作成イベント
・オブジェクト削除イベント
・オブジェクト復元イベント
・低冗長化ストレージ (RRS) オブジェクト消失イベント
・レプリケーションイベント

イベントの発行先
・SNS
・SQS
・Lambda

(1)イベント発行先がSNSの場合

-- 1. コマンド等のインストール

-- 1.1 aws cli version 2 インストール

curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip awscliv2.zip
sudo ./aws/install
aws --version

-- 2. S3 バケットを作成する

aws s3 mb s3://bucket123

aws s3 ls

-- 3. SNSトピック作成

aws sns list-topics
aws sns list-subscriptions

aws sns create-topic --name topic01

aws sns subscribe \
--topic-arn arn:aws:sns:ap-northeast-1:999999999999:topic01 \
--protocol email \
--notification-endpoint test@example.com

-- 4. SNSアクセスポリシー設定

aws sns get-topic-attributes \
--topic-arn arn:aws:sns:ap-northeast-1:999999999999:topic01

vim a.json

{
"Version": "2012-10-17",
"Id": "example-ID",
"Statement": [
{
"Sid": "example-statement-ID",
"Effect": "Allow",
"Principal": {
"Service": "s3.amazonaws.com"
},
"Action": [
"SNS:Publish"
],
"Resource": "arn:aws:sns:ap-northeast-1:999999999999:topic01",
"Condition": {
"ArnLike": { "aws:SourceArn": "arn:aws:s3:*:*:bucket123" },
"StringEquals": { "aws:SourceAccount": "999999999999" }
}
}
]
}

aws sns set-topic-attributes \
--topic-arn arn:aws:sns:ap-northeast-1:999999999999:topic01 \
--attribute-name Policy \
--attribute-value file://a.json

-- 5. イベント通知を作成

aws s3api get-bucket-notification-configuration \
--bucket bucket123

vim b.json

{
"TopicConfigurations": [
{
"Id": "en01",
"TopicArn": "arn:aws:sns:ap-northeast-1:999999999999:topic01",
"Events": [
"s3:ObjectCreated:Put"
],
"Filter": {
"Key": {
"FilterRules": [
{
"Name": "Prefix",
"Value": ""
},
{
"Name": "Suffix",
"Value": ""
}
]
}
}
}
]
}

aws s3api put-bucket-notification-configuration \
--bucket bucket123 \
--notification-configuration file://b.json

-- 6. 動作確認

echo test01 > test01.txt

aws s3api put-object --bucket bucket123 --key test01.txt --body test01.txt

aws s3 ls s3://bucket123 --recursive

ファイルをアップロードするとメールが届く

-- 7. クリーンアップ

-- SNSトピック削除

aws sns unsubscribe --subscription-arn arn:aws:sns:ap-northeast-1:999999999999:topic01:11111111-2222-3333-4444-555555555555
aws sns delete-topic --topic-arn arn:aws:sns:ap-northeast-1:999999999999:topic01

aws sns list-topics
aws sns list-subscriptions

-- バケットの削除
aws s3 ls
aws s3 rb s3://bucket123 --force

(2)イベント発行先がSQSの場合

-- 1. コマンド等のインストール

-- 1.1 aws cli version 2 インストール

curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip awscliv2.zip
sudo ./aws/install
aws --version

-- 2. S3 バケットを作成する

aws s3 mb s3://bucket123

aws s3 ls

-- 3. SQSキュー作成

aws sqs create-queue \
--queue-name qu01

aws sqs list-queues

-- 4. SQSアクセスポリシー設定

aws sqs get-queue-attributes \
--queue-url https://sqs.ap-northeast-1.amazonaws.com/999999999999/qu01 \
--attribute-names Policy

aws sqs set-queue-attributes \
--queue-url https://sqs.ap-northeast-1.amazonaws.com/999999999999/qu01 \
--attributes '{"Policy":"{\"Version\":\"2012-10-17\",\"Id\":\"example-ID\",\"Statement\":[{\"Sid\":\"example-statement-ID\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"s3.amazonaws.com\"},\"Action\":[\"SQS:SendMessage\"],\"Resource\":\"arn:aws:sqs:ap-northeast-1:999999999999:qu01\",\"Condition\":{\"ArnLike\":{\"aws:SourceArn\":\"arn:aws:s3:*:*:bucket123\"},\"StringEquals\":{\"aws:SourceAccount\":\"999999999999\"}}}]}"}'

-- 5. イベント通知を作成

aws s3api get-bucket-notification-configuration \
--bucket bucket123

vim b.json

{
"QueueConfigurations": [
{
"Id": "en02",
"QueueArn": "arn:aws:sqs:ap-northeast-1:999999999999:qu01",
"Events": [
"s3:ObjectCreated:Put"
],
"Filter": {
"Key": {
"FilterRules": [
{
"Name": "Prefix",
"Value": "02/"
},
{
"Name": "Suffix",
"Value": ""
}
]
}
}
}
]
}

aws s3api put-bucket-notification-configuration \
--bucket bucket123 \
--notification-configuration file://b.json

-- 6. テストファイルアップロード

echo test01 > test01.txt

aws s3api put-object --bucket bucket123 --key 02/test01.txt --body test01.txt
aws s3api put-object --bucket bucket123 --key 02/test02.txt --body test01.txt
aws s3api put-object --bucket bucket123 --key 02/test03.txt --body test01.txt
aws s3api put-object --bucket bucket123 --key 01/test04.txt --body test01.txt

aws s3 ls s3://bucket123 --recursive

-- 7. SQSキュー確認

メッセージ数確認

aws sqs get-queue-attributes \
--queue-url https://sqs.ap-northeast-1.amazonaws.com/999999999999/qu01 \
--attribute-names ApproximateNumberOfMessages

メッセージ受信

aws sqs receive-message \
--queue-url https://sqs.ap-northeast-1.amazonaws.com/999999999999/qu01

メッセージ削除
aws sqs delete-message \
--queue-url https://sqs.ap-northeast-1.amazonaws.com/999999999999/qu01 \
--receipt-handle "AQEB7CMEcxBk1X2rTIHpMTS/LFtEzpbduLa5E+9vnEyZ8pGxlxZRuk7TyqdSy3pWvL0alBY7WsYkap5j00lsxAucNvyIiNYBfazoXqzerE81ZaFCgF3suxitXAR9ozxUv9/F3qz1f0moxvEMJ9Eb6eLTE9C6Hxt9ycIsePHD48pA0A9pcfsrNDPDV3FllRVbsZCIXUzAXS1p9o07dZSdx2iNTbmR/ovGNMQJpP5AU8uOIo0MF3ZRi1xgLbd7bf3VjJ+Kbs72qg+bi2NqZDIiCyiNacH1EDiKUCW60+PveZpVWrbkCzL/qTrWRlkfJlS4fpHJGfXhfYwz7xWe1I27S+eB6+TvFZPl7azT1/qEWNhbGfsIlc0AgUGvMGsQnU26ppSHfAU2854Zl1YzZSswKXSJXQ=="

全メッセージ削除
aws sqs purge-queue \
--queue-url https://sqs.ap-northeast-1.amazonaws.com/999999999999/qu01

-- 8. クリーンアップ

-- SQSキュー削除
aws sqs list-queues

aws sqs delete-queue \
--queue-url https://sqs.ap-northeast-1.amazonaws.com/999999999999/qu01

※削除してもしばらく消えない

-- バケットの削除
aws s3 ls
aws s3 rb s3://bucket123 --force

(3)イベント発行先がLambdaの場合

-- 1. コマンド等のインストール

-- 1.1 aws cli version 2 インストール

curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip awscliv2.zip
sudo ./aws/install
aws --version

-- 1.2 jq インストール

sudo yum -y install jq

-- 2. S3 バケットを作成する

aws s3 mb s3://bucket123

aws s3 ls

-- 2. Lambda用IAMロール作成
vim role01.json

{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": "lambda.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}

aws iam create-role \
--role-name role01 \
--assume-role-policy-document file://role01.json

-- 3. AWS管理ポリシーをロールにアタッチ

aws iam attach-role-policy \
--policy-arn arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole \
--role-name role01

-- 4. Lambda関数作成

vim test.py

#!/usr/bin/python
def lambda_handler(event, context):
print(event)
return 'OK'

chmod 755 test.py
zip -r test.zip test.py

aws lambda create-function \
--region ap-northeast-1 \
--function-name test \
--zip-file fileb://test.zip \
--role arn:aws:iam::999999999999:role/role01 \
--handler test.lambda_handler \
--runtime python3.8 \
--timeout 60

aws lambda list-functions
aws lambda list-functions | jq -c '.Functions | [ .FunctionName ]'

aws lambda get-function --function-name test

-- 5. Lambdaアクセス権限の設定

aws lambda add-permission \
--function-name test \
--action lambda:InvokeFunction \
--statement-id s3 \
--principal s3.amazonaws.com \
--source-arn arn:aws:s3:::bucket123 \
--source-account 999999999999

aws lambda get-policy \
--function-name test

-- 6. イベント通知を作成

aws s3api get-bucket-notification-configuration \
--bucket bucket123

vim b.json

{
"LambdaFunctionConfigurations": [
{
"Id": "en03",
"LambdaFunctionArn": "arn:aws:lambda:ap-northeast-1:999999999999:function:test",
"Events": [
"s3:ObjectCreated:Put"
],
"Filter": {
"Key": {
"FilterRules": [
{
"Name": "Prefix",
"Value": ""
},
{
"Name": "Suffix",
"Value": ".log"
}
]
}
}
}
]
}

aws s3api put-bucket-notification-configuration \
--bucket bucket123 \
--notification-configuration file://b.json

-- 7. 動作確認

echo test01 > test01.txt

aws s3api put-object --bucket bucket123 --key test01.log --body test01.txt
aws s3api put-object --bucket bucket123 --key test02.txt --body test01.txt
aws s3api put-object --bucket bucket123 --key test03.log --body test01.txt

aws s3 ls s3://bucket123 --recursive

CloudWatchログでLambda実行を確認

-- 8. クリーンアップ
-- lambda関数の一覧
aws lambda list-functions | jq -c '.Functions | [ .FunctionName ]'

-- lambda関数の削除
aws lambda delete-function --function-name test

-- ロールの一覧
aws iam list-roles | grep role01

-- ロールの削除

aws iam detach-role-policy \
--role-name role01 \
--policy-arn arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole

aws iam delete-role --role-name role01

-- バケットの削除
aws s3 ls
aws s3 rb s3://bucket123 --force