Zenn
Closed5

ISUCON12

Yuuki TakahashiYuuki Takahashi

ローカル環境構築

インストール

Golandだけ買うと月3000円程度？らしい
https://www.jetbrains.com/go/download/?source=google&medium=cpc&campaign=10160681737&term=goland&gclid=Cj0KCQjwzLCVBhD3ARIsAPKYTcSD83hvGfr6ciw_GQHEOA2G1SI3FLNxS3P7wYtl4JzcD5G1DBCXCbUaAtkbEALw_wcB#section=linux

.ssh/config

インフラ構築担当から、 isucon.pem と各サーバのIPが記入されたconfig設定をもらう

Host c1
  HostName xx.xx.xx.xx
  User isucon
  IdentityFile ~/.ssh/isucon.pem
  ServerAliveInterval 30
  ServerAliveCountMax 120
  TCPKeepAlive yes

入れることを確認

ssh c1

Goまわり

Go Mudules の設定でEnable Go modules integration にチェック

Sync dependencies ~ にチェック

DB周り

DB接続設定はコードから手に入れる
それらを以下の要領で指定

SSHで接続先のサーバ（c1とか）を指定

All schemas にチェック

SQL Dialect を MySQL に

補完が効くようになる

ローカルCI

ローカルでビルドできることを確認するコマンド

cd "$HOME/lapras/repos/yktakaha4/isucon11-qualify-test/webapp/go" && (echo "- fmt -" && go fmt . && echo "- vet -" && go vet . && echo "- build -" && go build -o "$(mktemp)" -a main.go && echo "complete."); rc="$?"; cd - >/dev/null && return "$rc"

ビルドが通せるならPRを出してデプロイ担当にご報告

GitHub ActionsのCI

以下を.github/workflows/check.yml に貼り付ける

.github/workflows/check.yml
name: Check
on: pull_request
jobs:
  fmt:
    timeout-minutes: 5
    runs-on: ubuntu-20.04
    defaults:
      run:
        working-directory: ./webapp/go
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-go@v3
        with:
          go-version: '^1.16.5'
      - run: go fmt main.go

  vet:
    timeout-minutes: 5
    runs-on: ubuntu-20.04
    defaults:
      run:
        working-directory: ./webapp/go
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-go@v3
        with:
          go-version: '^1.16.5'
      - run: go vet main.go

  build:
    timeout-minutes: 5
    runs-on: ubuntu-20.04
    defaults:
      run:
        working-directory: ./webapp/go
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-go@v3
        with:
          go-version: '^1.16.5'
      - run: go build -a main.go

よさげ

Yuuki TakahashiYuuki Takahashi

Goのスニペット

バルクインサート

	type InsertParams struct {
		JIAIsuUUID   string    `db:"jia_isu_uuid"`
		Timestamp    time.Time `db:"timestamp"`
		IsSitting    bool      `db:"is_sitting"`
		Condition    string    `db:"condition"`
		Message      string    `db:"message"`
		CreatedAt    time.Time `db:"created_at"`
		IsBroken     bool      `db:"is_broken"`
		IsDirty      bool      `db:"is_dirty"`
		IsOverweight bool      `db:"is_overweight"`
	}

	var params []InsertParams
	for _, cond := range req {
		timestamp := time.Unix(cond.Timestamp, 0)

		if !isValidConditionFormat(cond.Condition) {
			return c.String(http.StatusBadRequest, "bad request body")
		}

		isBroken := isBrokenRegex.MatchString(cond.Condition)
		isDirty := isDirtyRegex.MatchString(cond.Condition)
		isOverweight := isOverweightRegex.MatchString(cond.Condition)

		params = append(params, InsertParams{
			JIAIsuUUID:   jiaIsuUUID,
			Timestamp:    timestamp,
			IsSitting:    cond.IsSitting,
			Condition:    cond.Condition,
			Message:      cond.Message,
			IsBroken:     isBroken,
			IsDirty:      isDirty,
			IsOverweight: isOverweight,
		})
	}

	_, err = tx.NamedExec("INSERT INTO `isu_condition` (`jia_isu_uuid`, `timestamp`, `is_sitting`, `condition`, `message`, `is_broken`, `is_dirty`, `is_overweight`) VALUES (:jia_isu_uuid, :timestamp, :is_sitting, :condition, :message, :is_broken, :is_dirty, :is_overweight)", params)
	if err != nil {
		c.Logger().Errorf("db error: %v", err)
		return c.NoContent(http.StatusInternalServerError)
	}

	err = tx.Commit()
	if err != nil {
		c.Logger().Errorf("db error: %v", err)
		return c.NoContent(http.StatusInternalServerError)
	}

正規表現

var conditionFormat = regexp.MustCompile("^is_dirty=(true|false),is_overweight=(true|false),is_broken=(true|false)$")

func isValidConditionFormat(conditionStr string) bool {
	return conditionFormat.MatchString(conditionStr)
}

ログをオフる(echo)

	e := echo.New()
	e.Debug = true
	e.Logger.SetLevel(log.ERROR)

	//e.Use(middleware.Logger())

インメモリキャッシュ

import (
	"github.com/patrickmn/go-cache"
)

var configCache = cache.New(time.Hour, time.Hour)

func getJIAServiceURL(tx *sqlx.Tx) string {
	configValue, ok := configCache.Get("jia_service_url")
	if !ok {
		log.Fatal("failed to get Config")
	}
	return configValue.(string)
}

func postInitialize(c echo.Context) error {
	configCache.Set("jia_service_url", request.JIAServiceURL, cache.DefaultExpiration)
}

環境変数から値を取得

	// 一定割合リクエストを落としてしのぐようにしたが、本来は全量さばけるようにすべき
	dropProbablityStr := os.Getenv("DROP_PROBABILITY")
	if dropProbablityStr == "" {
		dropProbablityStr = "0.9"
	}
	dropProbability, parseErr := strconv.ParseFloat(dropProbablityStr, 64)
	if parseErr != nil {
		return c.String(http.StatusInternalServerError, fmt.Sprintf("invalid DROP_PROBABILITY: %v", dropProbablityStr))
	}

Redisジョブ

// ----------------------------------------------------------------
// Redisによるポーリングキュー
// import ("go.uber.org/zap")
var appLogger *zap.Logger
var batchLogger *zap.Logger
var ctx = context.Background()

func getRedisClient() *redis.Client {
	rdb := redis.NewClient(&redis.Options{
		Addr:     os.Getenv("REDIS_HOST") + ":" + os.Getenv("REDIS_PORT"),
		Password: os.Getenv("REDIS_PASSWORD"),
		DB:       0,
	})
	return rdb
}

type RedisBatchRequest struct {
	QueuedAt                 string                    `json:"queuedAt"`
	JIAIsuUUID               string                    `json:"jiaIsuUUID"`
	PostIsuConditionRequests []PostIsuConditionRequest `json:"postIsuConditionRequest"`
}

const (
	redisBatchRequestHigh   = "redisBatchRequest:high"
	redisBatchRequestNormal = "redisBatchRequest:normal"
)

func runRedisBatchMainLoop() {
	batchLogger.Info("start runRedisBatchMainLoop")
	rdb := getRedisClient()

	statusCmd := rdb.FlushAll(ctx)
	if statusCmd.Err() != nil {
		//batchLogger.Error("failed to FlushAll", zap.Error(statusCmd.Err()))
		return
	}

	for {
		highLen := rdb.LLen(ctx, redisBatchRequestHigh).Val()
		normalLen := rdb.LLen(ctx, redisBatchRequestNormal).Val()
		batchLogger.Debug("queue length", zap.Int64("high", highLen), zap.Int64("normal", normalLen))

		blPopCmd := rdb.BLPop(ctx, 0, redisBatchRequestHigh, redisBatchRequestNormal)
		if blPopCmd.Err() != nil {
			batchLogger.Error("failed to BLPop", zap.Error(blPopCmd.Err()))
			continue
		}
		batchLogger.Debug("fetch request", zap.Strings("val", blPopCmd.Val()))
		requestStr := blPopCmd.Val()[1]

		var request RedisBatchRequest
		if err := json.Unmarshal([]byte(requestStr), &request); err != nil {
			batchLogger.Error("failed to unmarshal", zap.String("requestStr", requestStr), zap.Error(err))
			continue
		}
		if err := processRedisBatchRequest(request); err != nil {
			batchLogger.Error("failed to processRedisBatchRequest", zap.Error(err))
			continue
		}
	}
}
func putRedisBatchRequest(request RedisBatchRequest, isHighPriority bool) {
	client := getRedisClient()
	defer client.Close()
	requestBytes, err := json.Marshal(request)
	if err != nil {
		//batchLogger.Error("failed to marshal", zap.ByteString("requestBytes", requestBytes), zap.Error(err))
		return
	}
	queueName := redisBatchRequestNormal
	if isHighPriority {
		queueName = redisBatchRequestHigh
	}
	request.QueuedAt = time.Now().String()
	requestStr := string(requestBytes)
	batchLogger.Debug("try RPush", zap.String("queueName", queueName), zap.String("requestStr", requestStr))
	rPushCmd := client.RPush(ctx, queueName, requestStr)
	if rPushCmd.Err() != nil {
		//batchLogger.Error("failed to RPush", zap.Error(rPushCmd.Err()))
		return
	}
	batchLogger.Debug("RPush complete")
}
func processRedisBatchRequest(request RedisBatchRequest) error {
	batchLogger.Debug("process", zap.String("at", request.QueuedAt))

	// ここで主処理
	tx, err := db.Beginx()
	if err != nil {
		return err
	}
	defer tx.Rollback()

	var params []IsuCondition
	for _, cond := range request.PostIsuConditionRequests {
		timestamp := time.Unix(cond.Timestamp, 0)

		params = append(params, IsuCondition{
			JIAIsuUUID: request.JIAIsuUUID,
			Timestamp:  timestamp,
			IsSitting:  cond.IsSitting,
			Condition:  cond.Condition,
			Message:    cond.Message,
		})

	}

	_, err = tx.NamedExec("INSERT INTO `isu_condition` (`jia_isu_uuid`, `timestamp`, `is_sitting`, `condition`, `message`) "+
		"VALUES (:jia_isu_uuid, :timestamp, :is_sitting, :condition, :message)", params)

	if err != nil {
		return err
	}

	err = tx.Commit()
	if err != nil {
		return err
	}

	return nil
}

// ----------------------------------------------------------------

func main() {
	// Redisによるポーリングキュー
	batchLoggerConfig := zap.NewDevelopmentConfig()
	batchLoggerConfig.Level.SetLevel(zap.DebugLevel)
	bLogger, _ := batchLoggerConfig.Build()
	batchLogger = bLogger
	defer func(l *zap.Logger) {
		_ = l.Sync()
	}(batchLogger)

	if _, ok := os.LookupEnv("SHINTARO_BATCH_MODE"); ok {
		//batchLogger.Info("start batch mode", zap.String("SHINTARO_BATCH_MODE", mode))

		mySQLConnectionData = NewMySQLConnectionEnv()

		var err error
		db, err = mySQLConnectionData.ConnectDB()
		if err != nil {
			return
		}
		db.SetMaxOpenConns(10)
		defer db.Close()

		// 変数設定があった場合はバッチとして起動
		runRedisBatchMainLoop()
		return
	}
}

Redisキャッシュ

// ----------------------------------------------------------------
// Redisによるメモリキャッシュ
// Redisジョブを定義していることが前提
func deleteRedisCache(key string) error {
	rdb := getRedisClient()
	err := rdb.Del(ctx, key).Err()
	if err != nil {
		return err
	}
	return nil
}
func resetRedisCache() error {
	rdb := getRedisClient()
	err := rdb.FlushAll(ctx).Err()
	if err != nil {
		return err
	}
	return nil
}
// RedisCacheHoge ----------------------------------------------------------------
type RedisCacheHoge struct {
	Value string `json:"value"`
}
func putRedisCacheHoge(key string, data RedisCacheHoge) error {
	dataStr, err := json.Marshal(data)
	if err != nil {
		return err
	}

	rdb := getRedisClient()
	err = rdb.Set(ctx, key, dataStr, time.Duration(0)).Err()
	if err != nil {
		return err
	}
	return nil
}
func getRedisCacheHoge(key string) (RedisCacheHoge, error) {
	var val RedisCacheHoge

	rdb := getRedisClient()
	getCmd := rdb.Get(ctx, key)
	if getCmd.Err() != nil {
		return val, getCmd.Err()
	}

	valBytes := getCmd.Val()
	err := json.Unmarshal([]byte(valBytes), &val)
	if err != nil {
		return val, err
	}

	return val, nil
}
// ----------------------------------------------------------------
このスクラップは2023/02/26にクローズされました