{IoT Analytics}Getting started with AWS IoT Analytics (console)

https://catalog.us-east-1.prod.workshops.aws/workshops/03a4f79f-6971-441f-bc12-e8b755392d2c/ja-JP
https://docs.aws.amazon.com/iotanalytics/latest/userguide/quickstart.html#quickstart-create-channel

-- 1. コマンド等のインストール

-- 1.1 aws cli version 2 インストール

curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip awscliv2.zip
sudo ./aws/install
aws --version

-- 1.2 jqインストール
sudo yum -y install jq

-- 2. S3バケットの作成

-- 2.1 S3バケットの作成(チャネル用)

aws s3 mb s3://bucket123ch

aws s3 ls

-- 2.2 S3バケットの作成(データストア用)

aws s3 mb s3://bucket123ds

aws s3 ls

-- 2.3 バケットポリシー設定(チャネル用)

vim b1.json

{
"Version": "2012-10-17",
"Id": "MyPolicyID",
"Statement": [
{
"Sid": "MyStatementSid",
"Effect": "Allow",
"Principal": {
"Service": "iotanalytics.amazonaws.com"
},
"Action": [
"s3:GetBucketLocation",
"s3:GetObject",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:ListMultipartUploadParts",
"s3:AbortMultipartUpload",
"s3:PutObject",
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::bucket123ch",
"arn:aws:s3:::bucket123ch/*"
]
}
]
}

aws s3api put-bucket-policy \
--bucket bucket123ch \
--policy file://b1.json

aws s3api get-bucket-policy \
--bucket bucket123ch

-- 2.4 バケットポリシー設定(データストア用)

vim b2.json

aws s3api put-bucket-policy \
--bucket bucket123ds \
--policy file://b2.json

aws s3api get-bucket-policy \
--bucket bucket123ds

-- 3. ロールの作成(チャネル用)

-- 3.1 ポリシーの作成

vim policy01.json

{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:PutObject",
"s3:GetObject",
"s3:ListBucket",
"s3:GetBucketLocation"
],
"Resource": [
"arn:aws:s3:::bucket123ch",
"arn:aws:s3:::bucket123ch/*"
]
}
]
}

aws iam create-policy \
--policy-name policy01 \
--policy-document file://policy01.json

-- 3.2 ロールの作成

vim role01.json

{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": "iotanalytics.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}

aws iam create-role \
--role-name role01 \
--assume-role-policy-document file://role01.json

-- 3.3 ポリシーをロールにアタッチ

aws iam attach-role-policy --policy-arn arn:aws:iam::999999999999:policy/policy01 --role-name role01

-- 4. ロールの作成(データストア用)

-- 4.1 ポリシーの作成

vim policy02.json

{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:PutObject",
"s3:DeleteObject",
"s3:GetBucketLocation"
],
"Resource": [
"arn:aws:s3:::bucket123ds",
"arn:aws:s3:::bucket123ds/*"
]
}
]
}

aws iam create-policy \
--policy-name policy02 \
--policy-document file://policy02.json

-- 4.2 ロールの作成

vim role02.json

{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": "iotanalytics.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}

aws iam create-role \
--role-name role02 \
--assume-role-policy-document file://role02.json

-- 4.3 ポリシーをロールにアタッチ

aws iam attach-role-policy --policy-arn arn:aws:iam::999999999999:policy/policy02 --role-name role02

-- 5. ロールの作成(ログ用)

-- 5.1 ポリシーの作成

vim policy03.json

{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream"
],
"Resource": [
"arn:aws:logs:*:*:*"
]
}
]
}

aws iam create-policy \
--policy-name policy03 \
--policy-document file://policy03.json

-- 5.2 ロールの作成

vim role03.json

{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": "iotanalytics.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}

aws iam create-role \
--role-name role03 \
--assume-role-policy-document file://role03.json

-- 5.3 ポリシーをロールにアタッチ

aws iam attach-role-policy --policy-arn arn:aws:iam::999999999999:policy/policy03 --role-name role03

-- 6. AWS IoT Analyticsの作成

-- 6.1 チャネルの作成

aws iotanalytics create-channel \
--channel-name channel01 \
--channel-storage '{
"customerManagedS3": {
"bucket": "bucket123ch",
"roleArn": "arn:aws:iam::999999999999:role/role01"
}
}'

aws iotanalytics list-channels

aws iotanalytics describe-channel \
--channel-name channel01

-- 6.2 データストアの作成

aws iotanalytics create-datastore \
--datastore-name store01 \
--datastore-storage '{
"customerManagedS3": {
"bucket": "bucket123ds",
"roleArn": "arn:aws:iam::999999999999:role/role02"
}
}'

aws iotanalytics list-datastores

aws iotanalytics describe-datastore \
--datastore-name store01

-- 6.3 パイプラインの作成

aws iotanalytics create-pipeline \
--pipeline-name pipe01 \
--pipeline-activities ' [
{
"channel": {
"name": "10",
"channelName": "channel01",
"next": "20"
}
},
{
"datastore": {
"name": "20",
"datastoreName": "store01"
}
}
]'

aws iotanalytics list-pipelines

aws iotanalytics describe-pipeline \
--pipeline-name pipe01

-- 7. データセットの作成

-- 7.1 データ送信

vim main.py

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
import boto3
import copy
from datetime import datetime, date, timedelta
import random
import uuid
import json

iota = boto3.client('iotanalytics')

# AWS IoT Analyticsに送信するデータの構造
DATA_TEMPLATE = {
"timestamp": "",
"device_name": "",
"temperature": 0,
"humidity": 0
}

# AWS IoT Analyticsのチャンネル名(自分の環境に合わせて修正してください)
CHANNEL_NAME="channel01"

# 過去30日分のダミーデータを作成する。もっと長い期間で試したい場合は、days=の数字を変えてください
now = datetime.now()
n_days_ago = now - timedelta(days=30)
start_time = int(n_days_ago.timestamp())
end_time = int(now.timestamp())

cnt = 0
total = 0
messages =

# 現在まで、1分ごとのデータを作成します
while end_time > start_time:
dt = datetime.fromtimestamp(start_time)
item = copy.copy(DATA_TEMPLATE)
item["timestamp"] = dt.strftime("%Y-%m-%d %H:%M:%S")
item["device_name"] = "test_device"
item["temperature"] = random.randrange(10, 39)
item["humidity"] = random.randrange(40, 99)

messages.append({'messageId':str(uuid.uuid4()),'payload': json.dumps(item)})
cnt = cnt + 1

# batch putで送れるデータは、デフォルトだと100件までとなっています
# https://docs.aws.amazon.com/iotanalytics/latest/userguide/limits.html
if cnt >= 100:
total = total + cnt
print("batch put:", total)
ret = iota.batch_put_message(channelName=CHANNEL_NAME, messages=messages)
cnt = 0
messages =

start_time = start_time + 60

if cnt > 0:
total = total + cnt
print("batch put:", total)
ret = iota.batch_put_message(channelName=CHANNEL_NAME, messages=messages)

pip3 install boto3
python3 main.py

-- 7.2 データセットの作成

aws iotanalytics create-dataset \
--dataset-name ds01 \
--actions "[
{
\"actionName\": \"action01\",
\"queryAction\": {
\"sqlQuery\": \"SELECT date_trunc('hour', date_parse(timestamp, '%Y-%m-%d %H:%i:%S')) as dt_hour,\ndevice_name,\nround(avg(temperature), 1) as temp_avg,\nround(min(temperature), 1) as temp_min,\nround(max(temperature), 1) as temp_max,\nround(avg(humidity), 1) as hud_avg,\nround(min(humidity), 1) as hud_min,\nround(max(humidity), 1) as hud_max\nFROM store01\nGROUP BY 1, 2\",
\"filters\": []
}
}
]" \
--triggers ' [
{
"schedule": {
"expression": "cron(0/30 * * * ? *)"
}
}
]' \
--retention-period '{
"unlimited": false,
"numberOfDays": 3
}'

aws iotanalytics list-datasets

aws iotanalytics describe-dataset \
--dataset-name ds01

-- 7.3 データセットの取得

aws iotanalytics create-dataset-content \
--dataset-name ds01

aws iotanalytics get-dataset-content \
--dataset-name ds01

-- 8. IoT Analyticsのログ記録有効化

aws iotanalytics describe-logging-options

aws iotanalytics put-logging-options \
--logging-options '{
"roleArn": "arn:aws:iam::999999999999:role/role03",
"level": "ERROR",
"enabled": true
}'

-- 9. クリーンアップ

-- IoT Analyticsのログ記録無効化

aws iotanalytics describe-logging-options

aws iotanalytics put-logging-options \
--logging-options '{
"roleArn": "arn:aws:iam::999999999999:role/role03",
"level": "ERROR",
"enabled": false
}'

-- データセットの削除
aws iotanalytics list-datasets
aws iotanalytics delete-dataset \
--dataset-name ds01

-- パイプラインの削除
aws iotanalytics list-pipelines
aws iotanalytics delete-pipeline \
--pipeline-name pipe01

-- データストアの削除

aws iotanalytics list-datastores
aws iotanalytics delete-datastore \
--datastore-name store01

-- チャネルの削除

aws iotanalytics list-channels
aws iotanalytics delete-channel \
--channel-name channel01

-- ロールの削除
aws iam list-roles | grep role01
aws iam list-roles | grep role02
aws iam list-roles | grep role03

aws iam detach-role-policy \
--role-name role01 \
--policy-arn arn:aws:iam::999999999999:policy/policy01

aws iam detach-role-policy \
--role-name role02 \
--policy-arn arn:aws:iam::999999999999:policy/policy02

aws iam detach-role-policy \
--role-name role03 \
--policy-arn arn:aws:iam::999999999999:policy/policy03

aws iam delete-role --role-name role01
aws iam delete-role --role-name role02
aws iam delete-role --role-name role03

-- ポリシーの削除
aws iam list-policies | grep policy01
aws iam list-policies | grep policy02
aws iam list-policies | grep policy03

aws iam delete-policy \
--policy-arn arn:aws:iam::999999999999:policy/policy01

aws iam delete-policy \
--policy-arn arn:aws:iam::999999999999:policy/policy02

aws iam delete-policy \
--policy-arn arn:aws:iam::999999999999:policy/policy03

-- S3バケット削除

aws s3 ls

aws s3 rb s3://bucket123ch --force
aws s3 rb s3://bucket123ds --force