CodeCommitでファイル変更をトリガー条件に含める方法

CodeCommitを使っていると他のリポジトリサービスだとできるのに。。。ということがたまにあります。その中でも個人的に変更を検知するときにファイルパスでフィルタできないのは微妙だなと思っていたので、解決策を考えてみた。

まとめ

• CodeCommitのget_differencesを使う
• CodeCommitのイベントをLambdaで受けてファイルパスでフィルタ

EventBridgeにはファイルパスは含まれない

イベントにファイルパスは含まれないので、EventBridgeのイベントルールではフィルタできないですがBranchや更新前後のCommitIdが含まれているため、この情報を使ってファイルパスの差分をとっていきます。

{
   "version": "0",
   "id": "01234567-EXAMPLE",
   "detail-type": "CodeCommit Repository State Change",
   "source": "aws.codecommit",
   "account": "123456789012",
   "time": "2019-06-12T10:23:43Z",
   "region": "us-east-2",
   "resources": [
     "arn:aws:codecommit:us-east-2:123456789012:MyDemoRepo"
   ],
   "detail": {
     "event": "referenceUpdated",
     "repositoryName": "MyDemoRepo",
     "repositoryId": "12345678-1234-5678-abcd-12345678abcd",
     "referenceType": "branch",
     "referenceName": "myBranch",
     "referenceFullName": "refs/heads/myBranch",
     "commitId": "7f0103fMERGE",
     "oldCommitId": "3e5983DESTINATION",
     "baseCommitId": "3e5a9bf1BASE",
     "sourceCommitId": "26a8f2SOURCE",
     "destinationCommitId": "3e5983DESTINATION",
     "mergeOption": "THREE_WAY_MERGE",
     "conflictDetailsLevel": "LINE_LEVEL",
     "conflictResolutionStrategy": "AUTOMERGE"
   }
}

https://docs.aws.amazon.com/ja_jp/codecommit/latest/userguide/monitoring-events.html#referenceUpdated

構成

EventBridgeとCodeBuildやCodePipelineの間にLambdaを挟み、ファイルパスの差分取得とフィルタを行います。

Lambdaのコード

さきにLambdaで処理しているコードの全量を乗せておきます。

import re
from os import environ
from typing import Optional
from dataclasses import dataclass

import boto3
import yaml
from aws_lambda_powertools import Logger
from aws_lambda_powertools.utilities.data_classes import (EventBridgeEvent,
                                                          event_source)
from aws_lambda_powertools.utilities.typing import LambdaContext

logger = Logger()
commit = boto3.client("codecommit")
build = boto3.client("codebuild")
pipeline = boto3.client("codepipeline")
ssm = boto3.client("ssm")

@dataclass
class CommitRefEvent:
    callerUserArn: str
    event: str
    referenceType: str
    repositoryId: str
    repositoryName: str
    referenceName: str
    referenceFullName: str
    commitId: str
    mergeOption: str = ""
    sourceCommitId: str = ""
    destinationCommitId: str = ""
    oldCommitId: str = ""
    conflictDetailsLevel: str = ""
    conflictResolutionStrategy: str = ""

def get_differences_paths(commit_detail: CommitRefEvent) -> Optional[dict]:
    res = []
    paginator = commit.get_paginator('get_differences')
    if commit_detail.event == "referenceCreated":
        before_commit_specifier=commit_detail.referenceFullName
    elif commit_detail.event == "referenceUpdated":
        before_commit_specifier=commit_detail.oldCommitId
    else:
        return None
    page_iterator = paginator.paginate(
        repositoryName=commit_detail.repositoryName,
        afterCommitSpecifier=commit_detail.commitId,
        beforeCommitSpecifier=before_commit_specifier
    )

    try:
        for page in page_iterator:
            res.extend(page["differences"])
    except commit.exceptions.CommitDoesNotExistException:
        return None
    return {"change_paths": [{"path": r["afterBlob"]["path"], "change_type": r["changeType"]}
                             for r in res if "afterBlob" in r.keys()]}


def get_ssm_param(key_name, decryption=True) -> str:
    response = ssm.get_parameter(
        Name=key_name,
        WithDecryption=decryption
    )
    return response['Parameter']['Value']

def _is_rule_met(commit_diff: dict, ref_fullname: str, _rule: dict):
    _refs = _rule.get("refs")
    _changes = _rule.get("changes")
    if _refs is None:
        _ref_result = True
    else:
        _ref_result = True if [ref for ref in _refs
                               if ref in ref_fullname] else False
    if _changes is None:
        _change_result = True
    else:
        _change_result = False
        for p in commit_diff["change_paths"]:
            for pattern in _changes:
                _r = re.search(pattern, p["path"])
                if not _r is None:
                    _change_result = True
                    break
            if _change_result:
                break
    if _ref_result and _change_result:
        return True
    return False

def is_trigger_rule_met(commit_diff: dict, ref_fullname: str, job_trigger_rule: dict) -> bool:
    _only_rule = job_trigger_rule.get("only")
    _except_rule = job_trigger_rule.get("except")
    if _only_rule is None and _except_rule is None:
        return True
    if not _only_rule is None and not _except_rule is None:
        logger.warning("Only and except conditionals cannot be duplicated in a single definition")
        return False

    if not _only_rule is None:
        return _is_rule_met(commit_diff, ref_fullname, _only_rule)
    return not(_is_rule_met(commit_diff, ref_fullname, _except_rule))


def start_build(commit_detail:CommitRefEvent, project: str, repo_region:str, env_var_override: list = []) -> None:
    build.start_build(
        projectName=project,
        sourceTypeOverride="CODECOMMIT",
        sourceLocationOverride=f"https://git-codecommit.{repo_region}.amazonaws.com/v1/repos/{commit_detail.repositoryName}",
        sourceVersion=commit_detail.referenceFullName + "^{" + commit_detail.commitId + "}",
        environmentVariablesOverride=env_var_override + [
            {
                "name": "REPOSITORY_REGION",
                "value": repo_region,
                "type": "PLAINTEXT"
            },
            {
                "name": "REPOSITORY_NAME",
                "value": commit_detail.repositoryName,
                "type": "PLAINTEXT"
            },
            {
                "name": "REFERENCE_NAME",
                "value": commit_detail.referenceName,
                "type": "PLAINTEXT"
            },
            {
                "name": "COMMIT_ID",
                "value": commit_detail.commitId,
                "type": "PLAINTEXT"
            },
        ]
    )
    return None


def start_pipeline(commit_detail:CommitRefEvent, pipeline_name: str, repo_region:str, pipeline_var_override: list = []) -> None:
    pipeline.start_pipeline_execution(
        name=pipeline_name,
        variables=pipeline_var_override + [
            {
                "name": "REPOSITORY_REGION",
                "value": repo_region
            },
            {
                "name": "REPOSITORY_NAME",
                "value": commit_detail.repositoryName
            },
            {
                "name": "REFERENCE_NAME",
                "value": commit_detail.referenceName
            },
            {
                "name": "COMMIT_ID",
                "value": commit_detail.commitId
            },
        ]
    )
    return None

def main(event: EventBridgeEvent) -> None:
    commit_detail = CommitRefEvent(**event.detail)
    commit_diff = get_differences_paths(commit_detail)
    logger.debug(commit_diff)
    if commit_diff is None:
        logger.info("commit does not exist.")
        return None
    if not commit_diff["change_paths"]:
        logger.info("commit no changes exist.")
        return None

    jobs_trigger_rule = yaml.safe_load(get_ssm_param(environ.get("SSM_PARAM_KEY")))
    for job_trigger_rule in jobs_trigger_rule["jobs"]:
        if not is_trigger_rule_met(commit_diff, commit_detail.referenceFullName, job_trigger_rule):
            continue
        if job_trigger_rule["invoke_type"] == "CODE_BUILD":
            logger.info("start build")
            start_build(commit_detail,
                        job_trigger_rule["target_arn"].split("/")[-1],
                        event.region,
                        job_trigger_rule["env_var_override"])
        if job_trigger_rule["invoke_type"] == "CODE_PIPELINE":
            logger.info("start pipeline")
            start_pipeline(commit_detail,
                           job_trigger_rule["target_arn"].split(":")[-1],
                           event.region,
                           job_trigger_rule["pipeline_var_override"])
    return None

@logger.inject_lambda_context()
@event_source(data_class=EventBridgeEvent)
def handler(event: EventBridgeEvent, context: LambdaContext):
    logger.info(event.raw_event)
    main(event)

ファイルパスレベルの差分を取得

EventBridgeから受け取ったcommitIdとoldCommitIdなどから、get_differencesを使って変更のあったファイルパス情報を取得しています

def get_differences_paths(commit_detail: CommitRefEvent) -> Optional[dict]:
    res = []
    paginator = commit.get_paginator('get_differences')
    if commit_detail.event == "referenceCreated":
        before_commit_specifier=commit_detail.referenceFullName
    elif commit_detail.event == "referenceUpdated":
        before_commit_specifier=commit_detail.oldCommitId
    else:
        return None
    page_iterator = paginator.paginate(
        repositoryName=commit_detail.repositoryName,
        afterCommitSpecifier=commit_detail.commitId,
        beforeCommitSpecifier=before_commit_specifier
    )

    try:
        for page in page_iterator:
            res.extend(page["differences"])
    except commit.exceptions.CommitDoesNotExistException:
        return None
    return {"change_paths": [{"path": r["afterBlob"]["path"], "change_type": r["changeType"]}
                             for r in res if "afterBlob" in r.keys()]}

フィルタ条件

トリガー条件を別途Yaml形式で作成しParameterStoreに保存。
条件式はほんのり一昔前のGitLabっぽく。

• only： ブランチが合致し、かつ、changesの正規表現のいずれかに合致している場合にトリガー
• except： ブランチが合致し、かつ、changesの正規表現のいずれかに合致している場合、以外でトリガー

jobs:
  - invoke_type: CODE_PIPELINE
    target_arn: arn:aws:codepipeline:us-east-1:012345678901:test-pipeline
    pipeline_var_override:
      - name: BUILD_FLAG
        value: test
    only:
      refs:
        - main
      changes:
        - ^terraform/.*\.tf$
        - ^app/.*\.py$
        - ^terraform/config/.*\.yaml$
  - invoke_type: CODE_BUILD
    target_arn: arn:aws:codebuild:us-east-1:012345678901:project/test-build
    env_var_override:
      -  name: BUILD_FLAG
         value: test
         type: PLAINTEXT
    except:
      refs:
        - main
      changes:
        - '^state/.*'

条件式を取得しフィルタ

LambdaはParameterStoreから取得してフィルタするようにしています。

def get_ssm_param(key_name, decryption=True) -> str:
    response = ssm.get_parameter(
        Name=key_name,
        WithDecryption=decryption
    )
    return response['Parameter']['Value']
    
def _is_rule_met(commit_diff: dict, ref_fullname: str, _rule: dict):
    _refs = _rule.get("refs")
    _changes = _rule.get("changes")
    if _refs is None:
        _ref_result = True
    else:
        _ref_result = True if [ref for ref in _refs
                               if ref in ref_fullname] else False
    if _changes is None:
        _change_result = True
    else:
        _change_result = False
        for p in commit_diff["change_paths"]:
            for pattern in _changes:
                _r = re.search(pattern, p["path"])
                if not _r is None:
                    _change_result = True
                    break
            if _change_result:
                break
    if _ref_result and _change_result:
        return True
    return False

def is_trigger_rule_met(commit_diff: dict, ref_fullname: str, job_trigger_rule: dict) -> bool:
    _only_rule = job_trigger_rule.get("only")
    _except_rule = job_trigger_rule.get("except")
    if _only_rule is None and _except_rule is None:
        return True
    if not _only_rule is None and not _except_rule is None:
        logger.warning("Only and except conditionals cannot be duplicated in a single definition")
        return False

    if not _only_rule is None:
        return _is_rule_met(commit_diff, ref_fullname, _only_rule)
    return not(_is_rule_met(commit_diff, ref_fullname, _except_rule))
    
def main(event: EventBridgeEvent) -> None:
    # 一部省略
    jobs_trigger_rule = yaml.safe_load(get_ssm_param(environ.get("SSM_PARAM_KEY")))
    for job_trigger_rule in jobs_trigger_rule["jobs"]:
        if not is_trigger_rule_met(commit_diff, commit_detail.referenceFullName, job_trigger_rule):
            continue

Pipelineのv2タイプが最近でたのでせっかくなので条件ごとにBuildとPipelineを起動するときに変数を渡してあげる

CodeBuildやCodePipeline v2タイプを起動するときに変数を渡してあげる。

def start_build(commit_detail:CommitRefEvent, project: str, repo_region:str, env_var_override: list = []) -> None:
    build.start_build(
        projectName=project,
        sourceTypeOverride="CODECOMMIT",
        sourceLocationOverride=f"https://git-codecommit.{repo_region}.amazonaws.com/v1/repos/{commit_detail.repositoryName}",
        sourceVersion=commit_detail.referenceFullName + "^{" + commit_detail.commitId + "}",
        environmentVariablesOverride=env_var_override + [
            {
                "name": "REPOSITORY_REGION",
                "value": repo_region,
                "type": "PLAINTEXT"
            },
            {
                "name": "REPOSITORY_NAME",
                "value": commit_detail.repositoryName,
                "type": "PLAINTEXT"
            },
            {
                "name": "REFERENCE_NAME",
                "value": commit_detail.referenceName,
                "type": "PLAINTEXT"
            },
            {
                "name": "COMMIT_ID",
                "value": commit_detail.commitId,
                "type": "PLAINTEXT"
            },
        ]
    )
    return None


def start_pipeline(commit_detail:CommitRefEvent, pipeline_name: str, repo_region:str, pipeline_var_override: list = []) -> None:
    pipeline.start_pipeline_execution(
        name=pipeline_name,
        variables=pipeline_var_override + [
            {
                "name": "REPOSITORY_REGION",
                "value": repo_region
            },
            {
                "name": "REPOSITORY_NAME",
                "value": commit_detail.repositoryName
            },
            {
                "name": "REFERENCE_NAME",
                "value": commit_detail.referenceName
            },
            {
                "name": "COMMIT_ID",
                "value": commit_detail.commitId
            },
        ]
    )
    return None

※ start_pipelineで変数を渡すにはboto3の1.28.70以降が必要なので注意。試す際にはLambdaの既定バージョンでは現時点では実行できないので、LambdaLayerに最新のboto3を入れる必用があります。
https://docs.aws.amazon.com/ja_jp/lambda/latest/dg/lambda-runtimes.html

感想

やっぱりここら辺を個別に作りたくないですね。。。
CodeCatalystではファイルパスの変更もトリガー条件に入れれるようになったので、そっちをつかうべきか。

おまけでCodePipelineでpipelineのｖ2タイプにより起動時に変数を渡すことができるようになったので試してみました。ソースアクションにも変数使えるようにしてほしい。

どの種類の Source アクションでもパイプラインレベルの変数を使用することはサポートされていないことに注意してください。

https://docs.aws.amazon.com/ja_jp/codepipeline/latest/userguide/reference-variables.html#reference-variables-workflow