ECS Task で動かすEmbulkでAssume Roleを利用し他アカウントのS3にアクセス

当然Embulkにそんな機能はないので、涙ぐましいことをする羽目になる、大変。

見ればわかるけど、 aws sts assume-role で assume role を利用したアクセスが可能な状態にAWSのrole/policyを設定しておく必要はある。

Dockerfile

FROM --platform=linux/x86_64 amazoncorretto:8

RUN yum update -y \
  && yum install -y curl awscli jq \
  && rm -rf /var/cache/yum/* \
  && yum clean all -y

RUN export EMBULK_VERSION="0.9.25" \
  && curl --create-dirs -o /usr/local/embulk/bin/embulk -L "https://github.com/embulk/embulk/releases/download/v${EMBULK_VERSION}/embulk-${EMBULK_VERSION}.jar" \
  && chmod +x /usr/local/embulk/bin/embulk

ENV PATH $PATH:/usr/local/embulk/bin

RUN mkdir /usr/local/embulk/bundle

COPY bundle/Gemfile /usr/local/embulk/bundle
COPY bundle/Gemfile.lock /usr/local/embulk/bundle

RUN cd /usr/local/embulk/bundle \
 && embulk bundle install --path=/usr/local/embulk/bundle/vendor/bundle

COPY cmd.sh /usr/local/files/cmd.sh
RUN chmod +x /usr/local/files/cmd.sh

COPY config.yml.liquid /usr/local/files/config.yml.liquid

CMD ["/usr/local/files/cmd.sh"]

cmd.sh

#!/bin/bash

load_table() {

    RESPONSE=`aws sts assume-role \
      --role-arn arn:aws:iam::xxx:role/xxx_role \
      --role-session-name sts-session \
      --query '[Credentials]' \
      --output json`
    
    export ACCESS_KEY_ID=`echo $RESPONSE | jq -r '.[].AccessKeyId'`
    export SECRET_ACCESS_KEY=`echo $RESPONSE | jq -r '.[].SecretAccessKey'`
    export SESSION_TOKEN=`echo $RESPONSE | jq -r '.[].SessionToken'`

    java -jar /usr/local/embulk/bin/embulk run -b /usr/local/embulk/bundle /usr/local/files/config.yml.liquid 2>&1
}
 
 
main() {
 load_table
}
 
main "$@"

config.yml.liquid

in:
  type: s3
  bucket: foobar
  path_prefix: foobar-log/
  endpoint: s3.ap-northeast-1.amazonaws.com
  auth_method: session
  access_key_id: {{ env.ACCESS_KEY_ID }}
  secret_access_key: {{ env.SECRET_ACCESS_KEY }}
  session_token: {{ env.SESSION_TOKEN }}
  parser:
    charset: UTF-8
    newline: LF
    type: json
    columns:
    - {name: id, type: string}
    - {name: value, type: string}

out:
  type: stdout

bundle/Gemfile

source 'https://rubygems.org/'
gem 'embulk', '< 0.10'
gem 'embulk-input-s3'