{DynamoDB}Amazon DynamoDB の変更データキャプチャ

DynamoDB 用 Kinesis Data Streams
・データ保持期間 1日～365日
・レコードの順序異なる場合あり
・重複レコードあり

DynamoDB Streams
・データ保持期間 1日
・レコードの順序同じ
・重複レコードなし

前提：aws-cli/2.2.31
※aws cli version 1の場合、エラー発生

(1) DynamoDB 用 Kinesis Data Streams

-- 1. テストテーブル作成
aws dynamodb create-table \
--table-name tab1 \
--attribute-definitions \
AttributeName=col1,AttributeType=S \
AttributeName=col2,AttributeType=S \
--key-schema \
AttributeName=col1,KeyType=HASH \
AttributeName=col2,KeyType=RANGE \
--billing-mode=PAY_PER_REQUEST

aws dynamodb describe-table --table-name tab1

-- 2. Kinesis ストリーミングの作成
aws kinesis create-stream --stream-name stream01 --shard-count 1

aws kinesis describe-stream --stream-name stream01

-- 3. Kinesis ストリーミングの有効化

aws dynamodb enable-kinesis-streaming-destination \
--table-name tab1 \
--stream-arn arn:aws:kinesis:ap-northeast-1:999999999999:stream/stream01

aws dynamodb describe-kinesis-streaming-destination --table-name tab1

-- 4. 動作確認

aws kinesis get-shard-iterator \
--stream-name stream01 \
--shard-id shardId-000000000000 \
--shard-iterator-type LATEST

aws dynamodb put-item \
--table-name tab1 \
--item \
'{"col1": {"S": "val11"}, "col2": {"S": "val12"}, "col3": {"S": "val13"}, "col4": {"N": "1"}}'

aws dynamodb put-item \
--table-name tab1 \
--item \
'{"col1": {"S": "val21"}, "col2": {"S": "val22"}, "col3": {"S": "val23"}, "col4": {"N": "2"} }'

aws kinesis get-records --shard-iterator "AAAAAAAAAAEZR7+oKR1P/43ab+UKER4Hq/7CPkpxdsrd66dpjpQtgqmTpK9MZrGDjFxmAyEQJkJvcDZGE15kPciByLeUglnB/Zu+fVoFHOQXDt+js/DohpwZ9OS3zCYzZsbE7JSgcy7r7Do6y8gck3MBoDAflfp5JS1gndJJXQAC/eZDyCqNceRZpPkP+ghqgMneIWv7RMEgeQ6kp85+guSPn6SjN2J5lKIUJb1InXABS1FtzPJKZg=="

aws kinesis get-records --shard-iterator "AAAAAAAAAAHO8weVyAj2T9PR0d+bqlBy5FD4ZJGsVoDmOEAak92nLZ2jd/5VZ6xdrR2jh8R7d6hGIzs1WrXasrdqUUtts+CA0bW+ECCUqtAP8/VHhaNZMgt731fhev2bfwAQM1Y2BiUxJVShXvVm8D6iiTOcbEKyXkhEgijj9XJUi2V6Qi0dpIi2y+eidZQXFvrLmaOqxLINBxPbdBqust/9Q9NBCQaMDG3aAt8W9pPYzSd04ezebg=="

-- 5. クリーンアップ

-- テーブル一覧
aws dynamodb list-tables

-- テーブル削除
aws dynamodb delete-table --table-name tab1

-- Kinesis ストリーミングの一覧
aws kinesis list-streams

-- Kinesis ストリーミングの削除

aws kinesis delete-stream --stream-name stream01

(2) DynamoDB Streams

aws dynamodb describe-table --table-name tab1

-- 2. ストリームの有効化

aws dynamodb update-table \
--table-name tab1 \
--stream-specification StreamEnabled=true,StreamViewType=NEW_AND_OLD_IMAGES

aws dynamodb describe-table --table-name tab1

-- 3. 動作確認

aws dynamodbstreams list-streams --table-name tab1
aws dynamodbstreams describe-stream --stream-arn arn:aws:dynamodb:ap-northeast-1:999999999999:table/tab1/stream/2021-08-21T01:16:45.273

aws dynamodbstreams get-shard-iterator \
--stream-arn arn:aws:dynamodb:ap-northeast-1:999999999999:table/tab1/stream/2021-08-21T01:16:45.273 \
--shard-id shardId-00000001629668606139-5afda5ce \
--shard-iterator-type TRIM_HORIZON

aws dynamodbstreams get-shard-iterator \
--stream-arn arn:aws:dynamodb:ap-northeast-1:999999999999:table/tab1/stream/2021-08-21T01:16:45.273 \
--shard-id shardId-00000001629508776223-591e3ac9 \
--shard-iterator-type TRIM_HORIZON

aws dynamodbstreams get-shard-iterator \
--stream-arn arn:aws:dynamodb:ap-northeast-1:999999999999:table/tab1/stream/2021-08-21T01:16:45.273 \
--shard-id shardId-00000001629508603427-571ac434 \
--shard-iterator-type TRIM_HORIZON

aws dynamodbstreams get-shard-iterator \
--stream-arn arn:aws:dynamodb:ap-northeast-1:999999999999:table/tab1/stream/2021-08-21T01:16:45.273 \
--shard-id shardId-00000001622208606432-823a54b2 \
--shard-iterator-type TRIM_HORIZON

aws dynamodb put-item \
--table-name tab1 \
--item \
'{"col1": {"S": "val11"}, "col2": {"S": "val12"}, "col3": {"S": "val13"}, "col4": {"N": "1"}}'

aws dynamodb put-item \
--table-name tab1 \
--item \
'{"col1": {"S": "val21"}, "col2": {"S": "val22"}, "col3": {"S": "val23"}, "col4": {"N": "2"} }'

aws dynamodbstreams get-records \
--shard-iterator "arn:aws:dynamodb:ap-northeast-1:999999999999:table/tab1/stream/2021-08-21T01:16:45.273|1|AAAAAAAAAAFAvaufuJlzfI3/k66KwgzU19pu+nPMa82X7qqKdlGjD7CDP22dt3Purz42q/7ojXD5Fjvei7IO3TCLfDbf+AfyE3ep5f21fIb1DVCR8HLlaYRsE4aF2K8BX8TYuSSR34MXnjyistFeMg+XStr0NLmAvSBdKWLRMfEWXKP+qQtnCue6Juy2gHzG2iqNrZKS/E6CfW5GEtfmEXtfzzc5f80aY1MCO5W5wZdlT9/SjXRvi02qFw2+zVTXXYyzr3vtREaGHaA6OQXWc3YjmSqEAnr1vsz9VQdUNUezkd5xTsTUfeGtS3bSN1PzB4VdxgclgCVqUYl2fUG4srySdlD1fpwCt3wsKzq4r0lqU5OYuwYwSsjtczK/1xLxInK1aYr0oc1rMuLA1X+DtqQruM1/fMFGPfgW1NQhUE/jQfETrUQpw5d6oGLqvM38k3B5c6fgWFUoYY+CxUUduREwWx6jcheeaSQnbIzHGQ4y9uGqKL4DGMkpJsg7glzGf0Je6/I0gDF2xW0bdiUy0zV1Q4ddkliHyVlbteJeyD+x0E7aZnRKyOfqdPwo68mCMbgeQs9uDVObcq8hSPDj1GQF/1crE6qFU+NGtTb7yMjoGpIycWvClKb3wghzKDsenuiaAb47H55jkRHwMQx0c3mAf31dqpUEOs8IhOcvGgk4LkN5tmw65QBJ3wiTE34UuSfoDO6s2/LtJlCHMgdKd7vqHmOqUOAqz7eAmw65QBJ3wiTE34UWuy=="

aws dynamodbstreams get-records \
--shard-iterator "arn:aws:dynamodb:ap-northeast-1:999999999999:table/tab1/stream/2021-08-21T01:16:45.273|1|AAAAAAAAAA6wICA7zZ8NOEJVX7rPZZRqoOf4PrJ3w9XuviJgCQzWrdQlERl7p86sWV4Qqe/wWnFAC6RuNIpqcvpEKi7M4gtE1R9h5IwiV9LFGbYYy26Pm2xgrJbZiX46PCEzwHZO3yQvEJmMYgeHs8guj7L3xApIYbcKxasixv3ttANh6zZmD6gByDLIAiO4eXMH+LIr+eEKV0GOes+60Hy27thlCkAx87+xJTlhiy/ibGmsWL+Er1vH4gc7q3FcG2gWlg9pziIpJjCsPlH+gwwR1JIc5KPPEaVYEH2KEwpsKLkrvW/+qThcCmqhvAUfwl9nEHIbEIQQi/zujhB6G4arMablIMemxuS/BO1a1OVNGbpAHMudZnYrTYFrDRm2XnW7u7yu/qp8lji49WN21sBKJZ2voirp/q/d2F06iseMTzE04SBtQxsvjLTIWPXJET2A8y1gDl6MgZgneF2XNGKL76A/unF1MeJeQjSW3ZNUwkAJWdEFYo34bYqRQmM1fdsgCYf9eY9dv4bMDMp3VG8SVHOp0FreDjuz5Cz5H6B9usj+xUnnZLYgZbqlrhz62eFA+Wi22KAUVKozLmAy5moiCTV1fFmCuyVB3aYV68mSFi90Qc2sA8HZ/CoN/fC19giS78RmEej8HozSv1fu5HCGvQ6hwBrOpKyW4o9kSmcQzlIhQdmAzFmCxulVUoKhNslOIXC0o1ZljaR5vvYIUBx9jCVAW91/VajQeAmw65QBJ3wiTE34UW=="

aws dynamodbstreams get-records \
--shard-iterator "arn:aws:dynamodb:ap-northeast-1:999999999999:table/tab1/stream/2021-08-21T01:16:45.273|1|AAAAAAAAAAkp96Gryzel3zApSqY2dZfM2myqA/eTTQ6koFcLKaK7CryFYtEECPLLyEhQDyS77isKexoUfMcTXlsKVWG5QrUtWDxSBGI47Ask8Ur14kuhP3Lv8pfcLFZi4vlApsTrqanF06zXdCdLuLgDs54ofP+aClbRP5CqsnRzOTt+aSKifdzrm9vlDTrJTQ/gQl6WBq35+isZr+W283Fe3ne6aV2kH2MGlJS5laJnPJAyHnPvTBWhEFX1rLVoQcy7BjW45KMLJdss6mec7Mes3uRR5WO+BNgK4IBKZXblJi90XBq7J+DC3gFNogYm9jgZOldzS/+ZHSOqQ3PteIBFB9LSMgDpN5XchD2mvP0YqEstcNsbv/0Nn0t7smB6JASSx068LdKQldesa9WJfBAnNgfYcVRKgJ1sXN19gi6MQtdJDSfNtkR55cZO5RhokHPrCFn64PwDXIytek7vHy+sTkoS65hWJfiN7nx1LQddBN8QxgowjX3zcHZ2yoUnSWnjK35SfRuYpMN4PWq6DD4Ifp3SWmqZlYmOvftfK3Le8uSJ7Eiz2asCeR+rBLz8Ix2B96ZM69dYLccgwwyAQ73tFfaX6rfOyc7Ea1WPQ5s/q6FBqnEGNWuyqjZN5VjwAyRFudNgI1kj9GPmB5u0sfsqdbdzbfBvOT969+DeDiyFdx7yEzLUyT5QUEdCRtjjqGlptJpPAWiIIKWq1z8zegCu5eSvEVDrBHcrMeWhTwkXQSeAmw65QB=="

aws dynamodbstreams get-records \
--shard-iterator "arn:aws:dynamodb:ap-northeast-1:999999999999:table/tab1/stream/2021-08-21T01:16:45.273|1|AAAAAAAAAAGhkI9uUPcZPLJFipwU2RRN56WgH2qXbyWmmFNLjjdnqB2TyiN3TD8Hps5CrBh+fAF4kQSf5QNFNN70Wf/hWw+npUKh07rKr4xre5UTZ1EedNPExg5a/AD5BiEqusmc1/cfvseq6bvGDkr7ahbaplcIaFFsTqK7FwBocgsuMIUcCY7VtNy3XlMNAemdtWUQtOGMYMfsJofwac/yjQqoUBJ5zpuFmBbJQLBMMgJNRR5iLsi1mMW/OHJaWDfx9ZLmu87fJ4jYSVn9WjTtV43x4VenQoznHfBD6YdMAU4cKMizJHf7M1ApLxhzPkZi19HUUTZY8/XPjtfxNianm64e7THD6bfnOHqKxG5XOPCEijBNf9gA20lrbEU6tzUsD8LskQi53KYe3dhdqnnm4t9Y71dqnuIByoAr/qzrIBaI928si8DHwK/RAJRmHiWx8p0IMwaqsNyOHKyc9HJIHx4Xc+hI/Cl/1lYqT9tUjbEtNxtA3hMesufqx9rsCnnDUzdwQ/D2vs/Chxylghxb1RjWl/dszjFnOsAJ5X+xqVc0DU7UiYqFXlusHm3l+2q9f/8QlELZ0YsFK39FOgy7ITkpHL2O/H3pvOvFn8I7TDlCPGs8h9jl5M88mlF5DzyXufwPpaLtbBHGwoXHFbPeObruukEQF499jE2PJ9xBseOLHqD/l2lUq6HnxJmCP5GcdM6IHsOlk0b1AyIL6NBOTpmBU70vaE7DeAmw65QBJ3wiTE34UW=="

-- 4. クリーンアップ

-- テーブル一覧
aws dynamodb list-tables

-- テーブル削除
aws dynamodb delete-table --table-name tab1