📝
Fargate での RunTask API で ThrottlingException を発生させてみた
AWS Fargate スロットリングのクォータ - Amazon Elastic Container Service
Amazon ECS RunTask API への呼び出しのレートクォータは毎秒 20 コール (バーストおよび持続的) です。
詳解: Amazon Elastic Container Service と AWS Fargate のタスク起動レートの向上 | Amazon Web Services ブログ
RunTask API にはいくつかの制限があります。まず、API コールごとに最大 10 個のタスクを起動できます。さらに、RunTask API を呼び出す速度を制御するトークンバケットのレート制限があります。このレート制限により、API へのコールを最大 100 回までバーストできます。トークンバケットが更新されると、1 秒あたり 40 回の持続的な RunTask API 呼び出しのレートが許可されます
実際にどの程度の呼び出しで ThrottlingException が発生するのかを試してみました。
結論
Step Functions から 400 回並列で呼び出した場合に発生しました。
Step Functions ステートマシン
以下の定義を使用しました。
ステートマシン定義
{
"Comment": "RunTask API parallel execution with throttling detection via Lambda",
"StartAt": "GenerateParallelTasks",
"States": {
"GenerateParallelTasks": {
"Type": "Pass",
"Result": {
"tasks": [
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{},
{}
]
},
"ResultPath": "$.tasksData",
"Next": "ParallelRunTask"
},
"ParallelRunTask": {
"Type": "Map",
"ItemsPath": "$.tasksData.tasks",
"MaxConcurrency": 400,
"Iterator": {
"StartAt": "Run ECS Task",
"States": {
"Run ECS Task": {
"Type": "Task",
"Resource": "arn:aws:states:::ecs:runTask",
"Parameters": {
"LaunchType": "FARGATE",
"Cluster": "test",
"TaskDefinition": "arn:aws:ecs:ap-northeast-1:012345678901:task-definition/test:1",
"NetworkConfiguration": {
"AwsvpcConfiguration": {
"Subnets": [
"subnet-xxx"
],
"SecurityGroups": [
"sg-xxx"
],
"AssignPublicIp": "ENABLED"
}
}
},
"Catch": [
{
"ErrorEquals": [
"States.ALL"
],
"ResultPath": "$.error",
"Next": "HandleErrorInLambda"
}
],
"End": true
},
"HandleErrorInLambda": {
"Type": "Task",
"Resource": "arn:aws:lambda:ap-northeast-1:012345678901:function:test",
"InputPath": "$.error",
"ResultPath": "$.lambdaResult",
"Next": "CheckLambdaResult"
},
"CheckLambdaResult": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.lambdaResult.isThrottling",
"BooleanEquals": true,
"Next": "RaiseThrottlingError"
}
],
"Default": "RaiseGenericError"
},
"RaiseThrottlingError": {
"Type": "Fail",
"Error": "RunTaskThrottled",
"Cause": "ThrottlingException detected by Lambda"
},
"RaiseGenericError": {
"Type": "Fail",
"Error": "RunTaskFailed",
"Cause": "Other error detected"
}
}
},
"Catch": [
{
"ErrorEquals": [
"RunTaskThrottled"
],
"ResultPath": "$.error",
"Next": "ThrottlingDetected"
},
{
"ErrorEquals": [
"RunTaskFailed"
],
"ResultPath": "$.error",
"Next": "GenericFailure"
},
{
"ErrorEquals": [
"States.ALL"
],
"ResultPath": "$.error",
"Next": "GenericFailure"
}
],
"Next": "AllTasksSucceeded"
},
"ThrottlingDetected": {
"Type": "Fail",
"Error": "ThrottlingDetected",
"Cause": "Throttling occurred during RunTask execution"
},
"GenericFailure": {
"Type": "Fail",
"Error": "RunTaskError",
"Cause": "An error occurred during RunTask execution"
},
"AllTasksSucceeded": {
"Type": "Succeed"
}
}
}
処理概要は以下の通りです。
- Map ステートで RunTask API を並列で 400 回呼び出す
- RunTask API で何らかのエラーが発生したら Lambda 関数を呼び出す
- Lambda 関数ではエラーに ThrottlingException が含まれているかどうかをチェックする
- ThrottlingException が含まれていた場合、ThrottlingException である旨を出力する
Lambda 関数
コードは以下の通りです。
index.mjs
export const handler = async (event) => {
const cause = event.Cause || "";
const isThrottling = cause.includes("ThrottlingException");
return { isThrottling };
};
実行結果
ステートマシンの実行では以下の通り ThrottlingException の分岐に進むことを確認できました。
CloudTrail にも RunTask で ThrottlingException が発生した記録が残っていました。
なお、当初は Map ステートの MaxConcurrency を 200 で実行していたのですが、ThrottlingException は発生しませんでした。
具体的な閾値は非公開だと思われますが、エラーメッセージに ThrottlingException が含まれていたのでリトライにはつなげられると思います。
まとめ
今回は Fargate での RunTask API で ThrottlingException を発生させてみました。
どなたかの参考になれば幸いです。
Discussion