https://docs.aws.amazon.com/ja_jp/appflow/latest/userguide/flow-tutorial.html
S3 -> AppFlow -> S3
-- 1. コマンド等のインストール
-- 1.1 aws cli version 2 インストール
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip awscliv2.zip
sudo ./aws/install
aws --version
-- 1.2 jqインストール
sudo yum -y install jq
-- 2. S3 バケットを作成する
aws s3 ls
aws s3 mb s3://bucket123src
aws s3 mb s3://bucket123dst
-- 3. Upload data to Amazon S3
wget https://docs.aws.amazon.com/ja_jp/appflow/latest/userguide/samples/tutorial-account-data.zip
unzip tutorial-account-data.zip
aws s3api put-object --bucket bucket123src --key source/tutorial-account-data.csv --body tutorial-account-data.csv --content-type text/csv
aws s3 ls s3://bucket123src --recursive
aws s3 ls s3://bucket123dst --recursive
-- 4. フローの作成
aws appflow list-connectors | jq -r .connectors[].connectorDescription
aws appflow create-flow \
--flow-name flow01 \
--trigger-config '{
"triggerType": "OnDemand"
}' \
--source-flow-config '{
"connectorType": "S3",
"sourceConnectorProperties": {
"S3": {
"bucketName": "bucket123src",
"bucketPrefix": "source",
"s3InputFormatConfig": {
"s3InputFileType": "CSV"
}
}
}
}' \
--destination-flow-config-list ' [
{
"connectorType": "S3",
"destinationConnectorProperties": {
"S3": {
"bucketName": "bucket123dst",
"s3OutputFormatConfig": {
"fileType": "CSV",
"prefixConfig": {},
"aggregationConfig": {
"aggregationType": "None"
}
}
}
}
}
]' \
--tasks '[
{
"sourceFields": [
"Account Name",
"Account Type",
"Billing State/Province",
"Account Rating",
"Industry"
],
"connectorOperator": {
"S3": "PROJECTION"
},
"taskType": "Filter",
"taskProperties": {}
},
{
"sourceFields": [
"Account Name"
],
"connectorOperator": {
"S3": "NO_OP"
},
"destinationField": "Account Name",
"taskType": "Map",
"taskProperties": {
"DESTINATION_DATA_TYPE": "string",
"SOURCE_DATA_TYPE": "string"
}
},
{
"sourceFields": [
"Account Type"
],
"connectorOperator": {
"S3": "NO_OP"
},
"destinationField": "Account Type",
"taskType": "Map",
"taskProperties": {
"DESTINATION_DATA_TYPE": "string",
"SOURCE_DATA_TYPE": "string"
}
},
{
"sourceFields": [
"Billing State/Province"
],
"connectorOperator": {
"S3": "NO_OP"
},
"destinationField": "Billing State/Province",
"taskType": "Map",
"taskProperties": {
"DESTINATION_DATA_TYPE": "string",
"SOURCE_DATA_TYPE": "string"
}
},
{
"sourceFields": [
"Account Rating"
],
"connectorOperator": {
"S3": "NO_OP"
},
"destinationField": "Account Rating",
"taskType": "Map",
"taskProperties": {
"DESTINATION_DATA_TYPE": "string",
"SOURCE_DATA_TYPE": "string"
}
},
{
"sourceFields": [
"Industry"
],
"connectorOperator": {
"S3": "NO_OP"
},
"destinationField": "Industry",
"taskType": "Map",
"taskProperties": {
"DESTINATION_DATA_TYPE": "string",
"SOURCE_DATA_TYPE": "string"
}
}
]'
aws appflow list-flows
aws appflow describe-flow \
--flow-name flow01
-- 5. フローの実行
aws appflow start-flow \
--flow-name flow01
-- 6. 結果確認
aws appflow describe-flow-execution-records \
--flow-name flow01
aws s3 ls s3://bucket123src --recursive
aws s3 ls s3://bucket123dst --recursive
aws s3 cp s3://bucket123dst/flow01/11111111-1111-1111-1111-111111111111/-111111111-2022-08-26T10:39:10 -
-- 7. クリーンアップ
-- フローの削除
aws appflow delete-flow \
--flow-name flow01
aws appflow list-flows
aws appflow describe-flow \
--flow-name flow01
aws s3 rb s3://bucket123src --force
aws s3 rb s3://bucket123dst --force
