lambdaでシンプルにmysqldump to s3

$ docker run -it public.ecr.aws/lambda/provided:al2.2022.11.15.11 bash
Unable to find image 'public.ecr.aws/lambda/provided:al2.2022.11.15.11' locally
al2.2022.11.15.11: Pulling from lambda/provided
625fcf1e5e5f: Pull complete
8df5433d9fb6: Pull complete
bae685131515: Pull complete
94130148b8a0: Pull complete
Digest: sha256:023dd5403feaffcb92f8585c664444111723cf89ca42b0ebbec50a7acd080581
Status: Downloaded newer image for public.ecr.aws/lambda/provided:al2.2022.11.15.11
23 Nov 2022 03:30:25,642 [INFO] (rapid) exec '/var/runtime/bootstrap' (cwd=/var/task, handler=)

^C23 Nov 2022 03:30:43,926 [INFO] (rapid) Received signal signal=interrupt
23 Nov 2022 03:30:43,926 [INFO] (rapid) Shutting down...
23 Nov 2022 03:30:43,926 [WARNING] (rapid) Reset initiated: SandboxTerminated

$ docker run -it --entrypoint bash public.ecr.aws/lambda/provided:al2.2022.11.15
.11
bash-4.2# pwd
/var/task
bash-4.2# cat /etc/os-release
NAME="Amazon Linux"
VERSION="2"
ID="amzn"
ID_LIKE="centos rhel fedora"
VERSION_ID="2"
PRETTY_NAME="Amazon Linux 2"
ANSI_COLOR="0;33"
CPE_NAME="cpe:2.3:o:amazon:amazon_linux:2"
HOME_URL="https://amazonlinux.com/"
VARIANT_ID="202211081638-2.0.979.0"

bash-4.2# cat /lambda-entrypoint.sh
#!/bin/sh
# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.

if [ $# -ne 1 ]; then
  echo "entrypoint requires the handler name to be the first argument" 1>&2
  exit 142
fi
export _HANDLER="$1"

RUNTIME_ENTRYPOINT=/var/runtime/bootstrap
if [ -z "${AWS_LAMBDA_RUNTIME_API}" ]; then
  exec /usr/local/bin/aws-lambda-rie $RUNTIME_ENTRYPOINT
else
  exec $RUNTIME_ENTRYPOINT
fi

bash-4.2# which gzip
bash: which: command not found
bash-4.2# gzip
bash: gzip: command not found
bash-4.2# yum search gzip
Loaded plugins: ovl
=========================================== N/S matched: gzip ============================================
gzip.x86_64 : The GNU data compression program
pigz.x86_64 : Parallel implementation of gzip

  Name and summary matches only, use "search all" for everything.

bash-4.2# yum search awscli
Loaded plugins: ovl
========================================== N/S matched: awscli ===========================================
awscli.noarch : Universal Command Line Environment for AWS

  Name and summary matches only, use "search all" for everything.
bash-4.2# yum info awscli
Loaded plugins: ovl
Available Packages
Name        : awscli
Arch        : noarch
Version     : 1.18.147
Release     : 1.amzn2.0.1
Size        : 2.1 M
Repo        : amzn2-core/2/x86_64
Summary     : Universal Command Line Environment for AWS
URL         : http://aws.amazon.com/cli
License     : ASL 2.0 and MIT
Description : This package provides a unified
            : command line interface to Amazon Web Services.

bash-4.2# yum search mysql
Loaded plugins: ovl
=========================================== N/S matched: mysql ===========================================
MySQL-python.x86_64 : An interface to MySQL
apr-util-mysql.x86_64 : APR utility library MySQL DBD driver
dovecot-mysql.x86_64 : MySQL back end for dovecot
freeradius-mysql.x86_64 : MySQL support for freeradius
libdbi-dbd-mysql.x86_64 : MySQL plugin for libdbi
mysql-connector-java.noarch : Official JDBC driver for MySQL
mysql-connector-odbc.x86_64 : ODBC driver for MySQL
pcp-pmda-mysql.x86_64 : Performance Co-Pilot (PCP) metrics for MySQL
perl-DBD-MySQL.x86_64 : A MySQL interface for Perl
php-mysql.x86_64 : A module for PHP applications that use MySQL databases
php-mysqlnd.x86_64 : A module for PHP applications that use MySQL databases
qt-mysql.x86_64 : MySQL driver for Qt's SQL classes
qt-mysql.i686 : MySQL driver for Qt's SQL classes
qt3-MySQL.x86_64 : MySQL drivers for Qt 3's SQL classes
qt3-MySQL.i686 : MySQL drivers for Qt 3's SQL classes
qt5-qtbase-mysql.x86_64 : MySQL driver for Qt5's SQL classes
qt5-qtbase-mysql.i686 : MySQL driver for Qt5's SQL classes
redland-mysql.x86_64 : MySQL storage support for Redland
rsyslog-mysql.x86_64 : MySQL support for rsyslog
mariadb.x86_64 : A community developed branch of MySQL
mariadb-devel.x86_64 : Files for development of MariaDB/MySQL applications
mariadb-libs.x86_64 : The shared libraries required for MariaDB/MySQL clients
mariadb-libs.i686 : The shared libraries required for MariaDB/MySQL clients

  Name and summary matches only, use "search all" for everything.

bash-4.2# yum search mariadb
Loaded plugins: ovl
========================================== N/S matched: mariadb ==========================================
mariadb-bench.x86_64 : MariaDB benchmark scripts and data
mariadb-devel.x86_64 : Files for development of MariaDB/MySQL applications
mariadb-embedded.x86_64 : MariaDB as an embeddable library
mariadb-embedded-devel.x86_64 : Development files for MariaDB as an embeddable library
mariadb-libs.x86_64 : The shared libraries required for MariaDB/MySQL clients
mariadb-libs.i686 : The shared libraries required for MariaDB/MySQL clients
mariadb-server.x86_64 : The MariaDB server and related files
mariadb.x86_64 : A community developed branch of MySQL
mariadb-test.x86_64 : The test suite distributed with MariaD

  Name and summary matches only, use "search all" for everything.

bash-4.2# repoquery --list mariadb-devel | grep mysqldump
bash-4.2# repoquery --list mariadb-libs | grep mysqldump
bash-4.2# repoquery --list mariadb-server | grep mysqldump
/usr/bin/mysqldumpslow
/usr/share/man/man1/mysqldumpslow.1.gz
bash-4.2# repoquery --list mariadb | grep mysqldump
/usr/bin/mysqldump
/usr/share/man/man1/mysqldump.1.gz

#!/bin/sh

set -euo pipefail

# Initialization - load function handler
source $LAMBDA_TASK_ROOT/"$(echo $_HANDLER | cut -d. -f1).sh"

# Processing
while true
do
  HEADERS="$(mktemp)"
  # Get an event. The HTTP request will block until one is received
  EVENT_DATA=$(curl -sS -LD "$HEADERS" -X GET "http://${AWS_LAMBDA_RUNTIME_API}/2018-06-01/runtime/invocation/next")

  # Extract request ID by scraping response headers received above
  REQUEST_ID=$(grep -Fi Lambda-Runtime-Aws-Request-Id "$HEADERS" | tr -d '[:space:]' | cut -d: -f2)

  # Run the handler function from the script
  RESPONSE=$($(echo "$_HANDLER" | cut -d. -f2) "$EVENT_DATA")

  # Send the response
  curl -X POST "http://${AWS_LAMBDA_RUNTIME_API}/2018-06-01/runtime/invocation/$REQUEST_ID/response"  -d "$RESPONSE"
done

function handler () {
  EVENT_DATA=$1
  echo "$EVENT_DATA" 1>&2;
  RESPONSE="Echoing request: '$EVENT_DATA'"

  echo $RESPONSE
}

FROM public.ecr.aws/lambda/provided:al2

# Copy custom runtime bootstrap
COPY bootstrap ${LAMBDA_RUNTIME_DIR}
# Copy function code
COPY function.sh ${LAMBDA_TASK_ROOT}

# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
CMD [ "function.handler" ]

$ docker build -t lambda-local .
(出力略)
$  docker run -p 9000:8080 lambda-local
23 Nov 2022 04:31:54,086 [INFO] (rapid) exec '/var/runtime/bootstrap' (cwd=/var/task, handler=)

$ curl -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations"
-d '{"payload":"hello world!"}'

$ docker run -p 9000:8080 lambda-local
23 Nov 2022 04:31:54,086 [INFO] (rapid) exec '/var/runtime/bootstrap' (cwd=/var/task, handler=)
23 Nov 2022 04:32:51,598 [INFO] (rapid) extensionsDisabledByLayer(/opt/disable-extensions-jwigqn8j) -> stat /opt/disable-extensions-jwigqn8j: no such file or directory
23 Nov 2022 04:32:51,598 [WARNING] (rapid) Cannot list external agents error=open /opt/extensions: no such file or directory
START RequestId: 4641b1fe-fc97-438d-b846-f5098c5b682f Version: $LATEST
23 Nov 2022 04:32:51,598 [WARNING] (rapid) First fatal error stored in appctx: Runtime.InvalidEntrypoint
23 Nov 2022 04:32:51,598 [ERROR] (rapid) Init failed error=fork/exec /var/runtime/bootstrap: permission denied InvokeID=
23 Nov 2022 04:32:51,599 [WARNING] (rapid) Reset initiated: ReserveFail
23 Nov 2022 04:32:51,599 [WARNING] (rapid) Cannot list external agents error=open /opt/extensions: no such file or directory
23 Nov 2022 04:32:51,599 [WARNING] (rapid) First fatal error stored in appctx: Runtime.InvalidEntrypoint

$ docker run -p 9000:8080 lambda-local
23 Nov 2022 04:34:52,088 [INFO] (rapid) exec '/var/runtime/bootstrap' (cwd=/var/task, handler=)
23 Nov 2022 04:34:53,583 [INFO] (rapid) extensionsDisabledByLayer(/opt/disable-extensions-jwigqn8j) -> stat /opt/disable-extensions-jwigqn8j: no such file or directory
23 Nov 2022 04:34:53,583 [WARNING] (rapid) Cannot list external agents error=open /opt/extensions: no such file or directory
START RequestId: 8958d653-4fd5-4083-a4a7-fb03400dbc25 Version: $LATEST
{"payload":"hello world!"}
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100    61  100    16  100    45  22598  63559 --:--:-- --:--:-- --:--:-- 61000
{"status":"OK"}
END RequestId: 8958d653-4fd5-4083-a4a7-fb03400dbc25
REPORT RequestId: 8958d653-4fd5-4083-a4a7-fb03400dbc25  Init Duration: 0.28 ms Duration: 23.11 ms       Billed Duration: 24 ms    Memory Size: 3008 MB    Max Memory Used: 3008 MB

$ curl -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations"
-d '{"payload":"hello world!"}'
Echoing request: '{"payload":"hello world!"}'

version: "3"

services:
  lambda:
    image: public.ecr.aws/lambda/provided:al2.2022.11.15.11
    command: function.handler
    volumes:
      - ./runtime:/var/runtime
      - ./task:/var/task

  invoker:
    image: curlimages/curl
    command: sh -c 'curl -s -XPOST "$$URL" -d "$$DATA"; echo ""'
    environment:
      - URL=http://lambda:8080/2015-03-31/functions/function/invocations
      - DATA={"payload":"hello world!"}
    depends_on:
      - lambda

  db:
    image: mysql:8.0
    environment:
      MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
    volumes:
      - db:/var/lib/mysql

volumes:
  db:

 # Initialization - load function handler
 source $LAMBDA_TASK_ROOT/"$(echo $_HANDLER | cut -d. -f1).sh"

+yum install -y gzip awscli mariadb >/dev/null
+
 # Processing
 while true

bash-4.2# mysqladmin ping -h db -u root
mysqladmin: connect to server at 'db' failed
error: 'Authentication plugin 'caching_sha2_password' cannot be loaded: /usr/lib64/mysql/plugin/caching_sha2_password.so: cannot open shared object file: No such file or directory'

   db:
     image: mysql:8.0
+    command: --default-authentication-plugin=mysql_native_password
     environment:
       MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
     volumes:

bash-4.2# mysqladmin ping -h db -u root
mysqld is alive

  lambda:
    image: public.ecr.aws/lambda/provided:al2.2022.11.15.11
    command: function.handler
     volumes:
       - ./runtime:/var/runtime
       - ./task:/var/task
+    environment:
+      - AWS_ACCESS_KEY_ID
+      - AWS_SECRET_ACCESS_KEY
+      - AWS_SESSION_TOKEN

function handler () {
  filename=$(date '+%Y-%m-%d_%H%M%S').sql.gz

  mysqldump -u root -h db db | \
    gzip | \
    aws s3 cp - s3://sbox-lambda-simple-mysqldump-s3/dump/$filename

  echo $filename
}

     image: mysql:8.0
     command: --default-authentication-plugin=mysql_native_password
     environment:
-      MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
+      MYSQL_ROOT_PASSWORD: password
     volumes:
       - db:/var/lib/mysql

 function handler () {
   filename=$(date '+%Y-%m-%d_%H%M%S').sql.gz

-  mysqldump -u root -h db db | \
+  mysqldump -u $DBUSER -h $DBHOST $DBNAME -p$DBPASS | \
     gzip | \
     aws s3 cp - s3://sbox-lambda-simple-mysqldump-s3/dump/$filename

 yum install -y gzip awscli mariadb >/dev/null

+function getparam() {
+  aws ssm get-parameter --with-decryption --region ap-northeast-1 --name $1 --query 'Parameter.Value' --output text
+}
+export DBUSER=$(getparam sbox-mysqldump-dbuser)
+export DBHOST=$(getparam sbox-mysqldump-dbhost)
+export DBNAME=$(getparam sbox-mysqldump-dbname)
+export DBPASS=$(getparam sbox-mysqldump-dbpass)
+
 # Processing
 while true
 do

function handler () {
  mysqladmin ping -u $DBUSER -h $DBHOST -p$DBPASS
}

FROM public.ecr.aws/lambda/provided:al2.2022.11.15.11 as build

RUN yum install -y gzip mariadb unzip
ADD https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip /opt/awscli.zip
RUN cd /opt && unzip awscli.zip

FROM public.ecr.aws/lambda/provided:al2.2022.11.15.11

COPY --from=build /usr/bin/gzip /usr/bin/mysqldump /usr/local/bin/
COPY --from=build /opt/aws /opt/aws
RUN ln -s /opt/aws/dist/aws /usr/local/bin/

#!/bin/bash

set -eu

rm -rf dist
mkdir dist
cp runtime/bootstrap task/function.sh dist/

tag=$(docker build --quiet --target build . | cut -d: -f2)
container=$(docker create $tag)
docker cp $container:/usr/bin/gzip dist/gzip
docker cp $container:/usr/bin/mysqldump dist/mysqldump
docker cp $container:/opt/aws dist/aws
docker rm $container

cd dist
zip -qr ../function.zip ./

FROM public.ecr.aws/lambda/provided:al2 as build

RUN yum install -y gzip mariadb unzip

ADD https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip /opt/awscli.zip
RUN cd /opt && unzip -q awscli.zip

FROM public.ecr.aws/lambda/provided:al2

COPY --from=build /usr/bin/gzip /usr/bin/mysqldump /usr/local/bin/
COPY --from=build /opt/aws /opt/aws
RUN ln -s /opt/aws/dist/aws /usr/local/bin/

$ docker history lambda-local
IMAGE          CREATED             CREATED BY                                      SIZE      COMMENT
3b5f2bd5f2ad   About an hour ago   /bin/sh -c ln -s /opt/aws/dist/aws /usr/loca…   17B
d32fee162833   About an hour ago   /bin/sh -c #(nop) COPY dir:71e7a6526e5cb2598…   168MB
028557407f88   About an hour ago   /bin/sh -c #(nop) COPY multi:b3e5c2b0ab7627b…   3.27MB
bdd84b649c1d   11 days ago         ENTRYPOINT [ "/lambda-entrypoint.sh" ]          0B
...

FROM public.ecr.aws/lambda/provided:al2

RUN yum install -y gzip mariadb awscli && \
  yum clean all && rm -rf /var/cache/yum

$ docker history lambda-local
IMAGE          CREATED         CREATED BY                                      SIZE      COMMENT
a0c7ad338830   3 seconds ago   /bin/sh -c yum install -y gzip mariadb awscl…   250MB
bdd84b649c1d   11 days ago     ENTRYPOINT [ "/lambda-entrypoint.sh" ]          0B
...