{DataSync}AWS DataSync での AWS Command Line Interface の使用

https://docs.aws.amazon.com/ja_jp/datasync/latest/userguide/using-cli.html

AWS DataSyncは、ストレージシステム間、サービス間でのデータの移動を簡素化、自動化、および高速化するオンラインデータ転送サービスです。

東京リージョンS3 ---> 大阪リージョンS3

-- 1. コマンド等のインストール

-- 1.1 aws cli version 2 インストール

curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip awscliv2.zip
sudo ./aws/install
aws --version

-- 1.2 jqインストール
sudo yum -y install jq

-- 2. 送信元S3 バケットを作成する

aws s3 mb s3://bucket123src

aws s3 ls

-- 3. 送信先S3 バケットを作成する
export AWS_DEFAULT_REGION=ap-northeast-3

aws s3 mb s3://bucket123dst
aws s3 ls

export AWS_DEFAULT_REGION=ap-northeast-1

-- 4. ロールの作成(送信元S3用)

-- 4.1 ポリシーの作成

vim policy01.json

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Action": [
                "s3:GetBucketLocation",
                "s3:ListBucket",
                "s3:ListBucketMultipartUploads"
            ],
            "Effect": "Allow",
            "Resource": "arn:aws:s3:::bucket123src"
        },
        {
            "Action": [
                "s3:AbortMultipartUpload",
                "s3:DeleteObject",
                "s3:GetObject",
                "s3:ListMultipartUploadParts",
                "s3:PutObjectTagging",
                "s3:GetObjectTagging",
                "s3:PutObject"
            ],
            "Effect": "Allow",
            "Resource": "arn:aws:s3:::bucket123src/*"
        }
    ]
}

aws iam create-policy \
--policy-name policy01 \
--policy-document file://policy01.json

-- 4.2 ロールの作成

vim role01.json

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Principal": {
        "Service": "datasync.amazonaws.com"
      },
      "Action": "sts:AssumeRole"
    }
  ]
}

aws iam create-role \
--role-name role01 \
--assume-role-policy-document file://role01.json

-- 4.3 ポリシーをロールにアタッチ

aws iam attach-role-policy --policy-arn arn:aws:iam::999999999999:policy/policy01 --role-name role01

-- 5. ロールの作成(送信先S3用)

-- 5.1 ポリシーの作成

vim policy02.json

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Action": [
                "s3:GetBucketLocation",
                "s3:ListBucket",
                "s3:ListBucketMultipartUploads"
            ],
            "Effect": "Allow",
            "Resource": "arn:aws:s3:::bucket123dst"
        },
        {
            "Action": [
                "s3:AbortMultipartUpload",
                "s3:DeleteObject",
                "s3:GetObject",
                "s3:ListMultipartUploadParts",
                "s3:PutObjectTagging",
                "s3:GetObjectTagging",
                "s3:PutObject"
            ],
            "Effect": "Allow",
            "Resource": "arn:aws:s3:::bucket123dst/*"
        }
    ]
}

aws iam create-policy \
--policy-name policy02 \
--policy-document file://policy02.json

-- 5.2 ロールの作成

vim role02.json

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Principal": {
        "Service": "datasync.amazonaws.com"
      },
      "Action": "sts:AssumeRole"
    }
  ]
}

aws iam create-role \
--role-name role02 \
--assume-role-policy-document file://role02.json

-- 5.3 ポリシーをロールにアタッチ

aws iam attach-role-policy --policy-arn arn:aws:iam::999999999999:policy/policy02 --role-name role02

-- 6. 送信元S3ロケーションを作成する

aws datasync create-location-s3 \
--s3-bucket-arn 'arn:aws:s3:::bucket123src' \
--s3-storage-class 'STANDARD' \
--s3-config 'BucketAccessRoleArn=arn:aws:iam::999999999999:role/role01'

aws datasync list-locations

aws datasync describe-location-s3 \
--location-arn arn:aws:datasync:ap-northeast-1:999999999999:location/loc-11111111111111111

-- 7. 送信先S3ロケーションを作成する
export AWS_DEFAULT_REGION=ap-northeast-3

aws datasync create-location-s3 \
--s3-bucket-arn 'arn:aws:s3:::bucket123dst' \
--s3-storage-class 'STANDARD' \
--s3-config 'BucketAccessRoleArn=arn:aws:iam::999999999999:role/role02'

aws datasync list-locations

aws datasync describe-location-s3 \
--location-arn arn:aws:datasync:ap-northeast-3:999999999999:location/loc-22222222222222222

export AWS_DEFAULT_REGION=ap-northeast-1

-- 8. タスクの作成

-- 8.1 ロググループの作成

aws logs create-log-group --log-group-name lg01

aws logs describe-log-groups --log-group-name-prefix lg01

-- 8.2 ロググループリソースポリシーの設定

vim policy03.json

{
    "Statement": [
        {
            "Sid": "DataSyncLogsToCloudWatchLogs",
            "Effect": "Allow",
            "Action": [
                "logs:PutLogEvents",
                "logs:CreateLogStream"
            ],
            "Principal": {
                "Service": "datasync.amazonaws.com"
            },
            "Condition": {
                "ArnLike": {
                    "aws:SourceArn": [
                        "arn:aws:datasync:ap-northeast-1:999999999999:task/*"
                    ]
                },
                "StringEquals": {
                    "aws:SourceAccount": "999999999999"
                }
            },
            "Resource": "arn:aws:logs:ap-northeast-1:999999999999:log-group:*:*"
        }
    ],
    "Version": "2012-10-17"
}

aws logs put-resource-policy \
--policy-name policy03 \
--policy-document file://policy03.json

aws logs describe-resource-policies

-- 8.3 タスクの作成

aws datasync create-task \
--source-location-arn 'arn:aws:datasync:ap-northeast-1:999999999999:location/loc-11111111111111111' \
--destination-location-arn 'arn:aws:datasync:ap-northeast-3:999999999999:location/loc-22222222222222222' \
--cloud-watch-log-group-arn 'arn:aws:logs:ap-northeast-1:999999999999:log-group:lg01:*' \
--name task01 \
--options '{
  "VerifyMode": "NONE",
  "OverwriteMode": "ALWAYS",
  "Atime": "BEST_EFFORT",
  "Mtime": "PRESERVE",
  "PreserveDeletedFiles": "REMOVE",
  "PreserveDevices": "NONE",
  "TaskQueueing": "ENABLED",
  "LogLevel": "TRANSFER",
  "TransferMode": "CHANGED",
  "ObjectTags": "PRESERVE"
}'

aws datasync list-tasks

aws datasync describe-task \
--task-arn arn:aws:datasync:ap-northeast-1:999999999999:task/task-33333333333333333

-- 9. テスト用ファイルの作成

dd if=/dev/urandom of=20M.dmp bs=1M count=20

aws s3api put-object --bucket bucket123src --key test01.dmp --body 20M.dmp
aws s3api put-object --bucket bucket123src --key test02.dmp --body 20M.dmp
aws s3api put-object --bucket bucket123src --key test03.dmp --body 20M.dmp

aws s3 ls s3://bucket123src --recursive
aws s3 ls s3://bucket123dst --recursive

-- 10. タスクを開始する

aws datasync start-task-execution \
--task-arn arn:aws:datasync:ap-northeast-1:999999999999:task/task-33333333333333333

aws datasync list-task-executions

-- 11. タスクのモニタリング

aws datasync describe-task-execution \
--task-execution-arn arn:aws:datasync:ap-northeast-1:999999999999:task/task-33333333333333333/execution/exec-44444444444444444

aws s3 ls s3://bucket123src --recursive
aws s3 ls s3://bucket123dst --recursive

-- 12. 送信元ファイル変更時の挙動確認

dd if=/dev/urandom of=1M.dmp bs=1M count=1

aws s3api delete-object --bucket bucket123src --key test01.dmp
aws s3api put-object --bucket bucket123src --key test04.dmp --body 20M.dmp
aws s3api put-object --bucket bucket123src --key test03.dmp --body 1M.dmp

aws s3 ls s3://bucket123src --recursive
aws s3 ls s3://bucket123dst --recursive

aws datasync start-task-execution \
--task-arn arn:aws:datasync:ap-northeast-1:999999999999:task/task-33333333333333333

aws datasync list-task-executions

aws datasync describe-task-execution \
--task-execution-arn arn:aws:datasync:ap-northeast-1:999999999999:task/task-33333333333333333/execution/exec-55555555555555555

aws s3 ls s3://bucket123src --recursive
aws s3 ls s3://bucket123dst --recursive

-- 13. クリーンアップ

-- タスクの削除

aws datasync list-tasks

aws datasync delete-task \
--task-arn arn:aws:datasync:ap-northeast-1:999999999999:task/task-33333333333333333

-- ロググループリソースポリシーの削除
aws logs describe-resource-policies

aws logs delete-resource-policy \
--policy-name policy03

-- ロググループの削除
aws logs describe-log-groups --log-group-name-prefix lg01
aws logs delete-log-group --log-group-name lg01

-- 送信先S3ロケーションの削除
export AWS_DEFAULT_REGION=ap-northeast-3

aws datasync list-locations

aws datasync delete-location \
--location-arn arn:aws:datasync:ap-northeast-3:999999999999:location/loc-22222222222222222

export AWS_DEFAULT_REGION=ap-northeast-1

-- 送信元S3ロケーションの削除

aws datasync list-locations

aws datasync delete-location \
--location-arn arn:aws:datasync:ap-northeast-1:999999999999:location/loc-11111111111111111

-- ロールの削除(送信先S3用)
aws iam list-roles | grep role02

aws iam detach-role-policy \
--role-name role02 \
--policy-arn arn:aws:iam::999999999999:policy/policy02

aws iam delete-role --role-name role02

-- ポリシーの削除(送信先S3用)
aws iam list-policies | grep policy02

aws iam delete-policy \
--policy-arn arn:aws:iam::999999999999:policy/policy02

-- ロールの削除(送信元S3用)
aws iam list-roles | grep role01

aws iam detach-role-policy \
--role-name role01 \
--policy-arn arn:aws:iam::999999999999:policy/policy01

aws iam delete-role --role-name role01

-- ポリシーの削除(送信元S3用)
aws iam list-policies | grep policy01

aws iam delete-policy \
--policy-arn arn:aws:iam::999999999999:policy/policy01

-- 送信先S3バケットの削除
export AWS_DEFAULT_REGION=ap-northeast-3

aws s3 ls

aws s3 rb s3://bucket123dst --force

export AWS_DEFAULT_REGION=ap-northeast-1

-- 送信元S3バケットの削除
aws s3 ls

aws s3 rb s3://bucket123src --force