ECS Task で動かすEmbulkでAssume Roleを利用し他アカウントのS3にアクセス

当然Embulkにそんな機能はないので、涙ぐましいことをする羽目になる、大変。

見ればわかるけど、 aws sts assume-role で assume role を利用したアクセスが可能な状態にAWSのrole/policyを設定しておく必要はある。

role/policy の terraform 実装イメージ

data "aws_iam_policy_document" "test_assume_policy" {
  statement {
    actions = ["sts:AssumeRole"]

    principals {
      type = "AWS"
      // account
      identifiers = [123456]
    }
  }
}

resource "aws_iam_role" "test_role" {
  name               = "test_role"
  assume_role_policy = data.aws_iam_policy_document.test_assume_policy.json
}

resource "aws_iam_policy" "test_aws_permissions" {
  name        = "test_aws_permissions"
  description = ""
  policy      = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": [
        "s3:Get*",
        "s3:List*"
      ],
      "Effect": "Allow",
      "Resource": "*"
    }
  ]
}
EOF
}

resource "aws_iam_role_policy_attachment" "test_aws_policy_attach" {
  role       = aws_iam_role.test_role.name
  policy_arn = aws_iam_policy.test_aws_permissions.arn
}

Dockerfile

FROM --platform=linux/x86_64 amazoncorretto:8

RUN yum update -y \
  && yum install -y curl awscli jq \
  && rm -rf /var/cache/yum/* \
  && yum clean all -y

RUN export EMBULK_VERSION="0.9.25" \
  && curl --create-dirs -o /usr/local/embulk/bin/embulk -L "https://github.com/embulk/embulk/releases/download/v${EMBULK_VERSION}/embulk-${EMBULK_VERSION}.jar" \
  && chmod +x /usr/local/embulk/bin/embulk

ENV PATH $PATH:/usr/local/embulk/bin

RUN mkdir /usr/local/embulk/bundle

COPY bundle/Gemfile /usr/local/embulk/bundle
COPY bundle/Gemfile.lock /usr/local/embulk/bundle

RUN cd /usr/local/embulk/bundle \
 && embulk bundle install --path=/usr/local/embulk/bundle/vendor/bundle

COPY cmd.sh /usr/local/files/cmd.sh
RUN chmod +x /usr/local/files/cmd.sh

COPY config.yml.liquid /usr/local/files/config.yml.liquid

CMD ["/usr/local/files/cmd.sh"]

cmd.sh

#!/bin/bash

load_table() {

    RESPONSE=`aws sts assume-role \
      --role-arn arn:aws:iam::xxx:role/xxx_role \
      --role-session-name sts-session \
      --query '[Credentials]' \
      --output json`
    
    export ACCESS_KEY_ID=`echo $RESPONSE | jq -r '.[].AccessKeyId'`
    export SECRET_ACCESS_KEY=`echo $RESPONSE | jq -r '.[].SecretAccessKey'`
    export SESSION_TOKEN=`echo $RESPONSE | jq -r '.[].SessionToken'`

    java -jar /usr/local/embulk/bin/embulk run -b /usr/local/embulk/bundle /usr/local/files/config.yml.liquid 2>&1
}
 
 
main() {
 load_table
}
 
main "$@"

config.yml.liquid

in:
  type: s3
  bucket: foobar
  path_prefix: foobar-log/
  endpoint: s3.ap-northeast-1.amazonaws.com
  auth_method: session
  access_key_id: {{ env.ACCESS_KEY_ID }}
  secret_access_key: {{ env.SECRET_ACCESS_KEY }}
  session_token: {{ env.SESSION_TOKEN }}
  parser:
    charset: UTF-8
    newline: LF
    type: json
    columns:
    - {name: id, type: string}
    - {name: value, type: string}

out:
  type: stdout

bundle/Gemfile

source 'https://rubygems.org/'
gem 'embulk', '< 0.10'
gem 'embulk-input-s3'