Slide 13
Slide 13 text
DynamoDB
S3
Step Functionsで実装
• S3にデータファイルを格納
• ファイル格納をトリガーにして、DynamoDBに格納
• データは一意、更新はなし
Step
Functions
イベント通知 TransactWriteItems
1.S3イベント通知でEventBridge経由し
Step Functionsを起動
2.Step FunctionsがS3からデータ取得
3.データを100件づつに分割
4.TransactWriteItemsで100件づつ書き込み
EventBridge
{
"QueryLanguage": "JSONata",
"StartAt": "GetObject",
"States": {
"GetObject": {
"Type": "Task",
"Resource": "arn:aws:states:::aws-sdk:s3:getObject",
"Arguments": {
"Bucket": "h-test-stepfunctions-bucket",
"Key": "{% $states.input.detail.object.key %}"
},
"Next": "ProcessLargeData"
},
"ProcessLargeData": {
"Type": "Pass",
"Assign": {
"records": "{% $parse($states.input.Body) %}",
"totalRecords": "{% $count($parse($states.input.Body)) %}",
"batches": "{% $partition($parse($states.input.Body), 100) %}"
},
"Next": "ProcessBatches"
},
"ProcessBatches": {
"Type": "Map",
"Items": "{% $batches %}",
"MaxConcurrency": 5,
"ItemProcessor": {
"StartAt": "WriteBatch",
"States": {
"WriteBatch": {
"Type": "Task",
"Resource": "arn:aws:states:::aws-sdk:dynamodb:transactWriteItems",
"Arguments": {
"TransactItems": "{% $map($states.input, function($item) { { 'Put': { 'TableName': 'h-test-
stepfunctions-table', 'Item': { 'UserId': {'S': $item.userId}, 'Score': {'N': $string($item.score)},
'Name': {'S': $item.name}, 'Timestamp': {'S': $item.timestamp}, 'ProcessedAt': {'S':
$now()} } } } }) %}"
},
"End": true
}
}
},
"End": true
}
}
}
43Line