👏
Dockerでprotocを行って開発を楽にする
概要
Protocol Buffersを使ってスキーマ駆動開発をするチームは多いと思う。
しかし、protocで自動生成する時に各々のローカルで行うと以下のような問題がある。
- メンバーごとによってprotocのversionが異なって生成物に差が生まれてしまう
- versionを上げる時に、slack等で「protocのversionを3.xx.xにあげてください」など周知をする必要がある
そこで今回はDockerを使ってprotocを行う方法について書いてく。
Dockerを使用すれば各々のローカル環境に依存しなくなるので上記の問題が解決出来る。
※今回はProtocol Buffersとは、のような基礎的な話は割愛している。
前提
- 言語はGo
- 通信はgRPC
- gRPC Gatewayを使用してREST APIも叩けるようにする
- リポジトリ名は「todo-server」とする
protoファイルを用意
今回使うprotoファイルを用意する。
protobuf/api/item.proto
syntax = "proto3";
package api;
# ここは出力先のフォルダを指定する
option go_package = "github.com/todo-server/pkg/domain/proto/api";
# gRPC Gatewayを使用する時に必要なoptionをimport
import "google/api/annotations.proto";
service Item {
rpc GetItem(GetItemRequest) returns (GetItemResponse) {
option (google.api.http) = {
post: "/item/get"
body: "*"
};
}
}
message GetItemRequest {
string item_id = 1;
}
message GetItemResponse {
string item_name = 2;
}
下記のファイルをプロジェクトに追加。コピペで大丈夫。
protobuf/google/api/annotations.proto
// Copyright 2015 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package google.api;
import "google/api/http.proto";
import "google/protobuf/descriptor.proto";
option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations";
option java_multiple_files = true;
option java_outer_classname = "AnnotationsProto";
option java_package = "com.google.api";
option objc_class_prefix = "GAPI";
extend google.protobuf.MethodOptions {
// See `HttpRule`.
HttpRule http = 72295728;
}
protobuf/google/api/http.proto
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package google.api;
option cc_enable_arenas = true;
option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations";
option java_multiple_files = true;
option java_outer_classname = "HttpProto";
option java_package = "com.google.api";
option objc_class_prefix = "GAPI";
// Defines the HTTP configuration for an API service. It contains a list of
// [HttpRule][google.api.HttpRule], each specifying the mapping of an RPC method
// to one or more HTTP REST API methods.
message Http {
// A list of HTTP configuration rules that apply to individual API methods.
//
// **NOTE:** All service configuration rules follow "last one wins" order.
repeated HttpRule rules = 1;
// When set to true, URL path parmeters will be fully URI-decoded except in
// cases of single segment matches in reserved expansion, where "%2F" will be
// left encoded.
//
// The default behavior is to not decode RFC 6570 reserved characters in multi
// segment matches.
bool fully_decode_reserved_expansion = 2;
}
// `HttpRule` defines the mapping of an RPC method to one or more HTTP
// REST API methods. The mapping specifies how different portions of the RPC
// request message are mapped to URL path, URL query parameters, and
// HTTP request body. The mapping is typically specified as an
// `google.api.http` annotation on the RPC method,
// see "google/api/annotations.proto" for details.
//
// The mapping consists of a field specifying the path template and
// method kind. The path template can refer to fields in the request
// message, as in the example below which describes a REST GET
// operation on a resource collection of messages:
//
//
// service Messaging {
// rpc GetMessage(GetMessageRequest) returns (Message) {
// option (google.api.http).get =
// "/v1/messages/{message_id}/{sub.subfield}";
// }
// }
// message GetMessageRequest {
// message SubMessage {
// string subfield = 1;
// }
// string message_id = 1; // mapped to the URL
// SubMessage sub = 2; // `sub.subfield` is url-mapped
// }
// message Message {
// string text = 1; // content of the resource
// }
//
// The same http annotation can alternatively be expressed inside the
// `GRPC API Configuration` YAML file.
//
// http:
// rules:
// - selector: <proto_package_name>.Messaging.GetMessage
// get: /v1/messages/{message_id}/{sub.subfield}
//
// This definition enables an automatic, bidrectional mapping of HTTP
// JSON to RPC. Example:
//
// HTTP | RPC
// -----|-----
// `GET /v1/messages/123456/foo` | `GetMessage(message_id: "123456" sub:
// SubMessage(subfield: "foo"))`
//
// In general, not only fields but also field paths can be referenced
// from a path pattern. Fields mapped to the path pattern cannot be
// repeated and must have a primitive (non-message) type.
//
// Any fields in the request message which are not bound by the path
// pattern automatically become (optional) HTTP query
// parameters. Assume the following definition of the request message:
//
//
// service Messaging {
// rpc GetMessage(GetMessageRequest) returns (Message) {
// option (google.api.http).get = "/v1/messages/{message_id}";
// }
// }
// message GetMessageRequest {
// message SubMessage {
// string subfield = 1;
// }
// string message_id = 1; // mapped to the URL
// int64 revision = 2; // becomes a parameter
// SubMessage sub = 3; // `sub.subfield` becomes a parameter
// }
//
//
// This enables a HTTP JSON to RPC mapping as below:
//
// HTTP | RPC
// -----|-----
// `GET /v1/messages/123456?revision=2&sub.subfield=foo` |
// `GetMessage(message_id: "123456" revision: 2 sub: SubMessage(subfield:
// "foo"))`
//
// Note that fields which are mapped to HTTP parameters must have a
// primitive type or a repeated primitive type. Message types are not
// allowed. In the case of a repeated type, the parameter can be
// repeated in the URL, as in `...?param=A¶m=B`.
//
// For HTTP method kinds which allow a request body, the `body` field
// specifies the mapping. Consider a REST update method on the
// message resource collection:
//
//
// service Messaging {
// rpc UpdateMessage(UpdateMessageRequest) returns (Message) {
// option (google.api.http) = {
// put: "/v1/messages/{message_id}"
// body: "message"
// };
// }
// }
// message UpdateMessageRequest {
// string message_id = 1; // mapped to the URL
// Message message = 2; // mapped to the body
// }
//
//
// The following HTTP JSON to RPC mapping is enabled, where the
// representation of the JSON in the request body is determined by
// protos JSON encoding:
//
// HTTP | RPC
// -----|-----
// `PUT /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id:
// "123456" message { text: "Hi!" })`
//
// The special name `*` can be used in the body mapping to define that
// every field not bound by the path template should be mapped to the
// request body. This enables the following alternative definition of
// the update method:
//
// service Messaging {
// rpc UpdateMessage(Message) returns (Message) {
// option (google.api.http) = {
// put: "/v1/messages/{message_id}"
// body: "*"
// };
// }
// }
// message Message {
// string message_id = 1;
// string text = 2;
// }
//
//
// The following HTTP JSON to RPC mapping is enabled:
//
// HTTP | RPC
// -----|-----
// `PUT /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id:
// "123456" text: "Hi!")`
//
// Note that when using `*` in the body mapping, it is not possible to
// have HTTP parameters, as all fields not bound by the path end in
// the body. This makes this option more rarely used in practice of
// defining REST APIs. The common usage of `*` is in custom methods
// which don't use the URL at all for transferring data.
//
// It is possible to define multiple HTTP methods for one RPC by using
// the `additional_bindings` option. Example:
//
// service Messaging {
// rpc GetMessage(GetMessageRequest) returns (Message) {
// option (google.api.http) = {
// get: "/v1/messages/{message_id}"
// additional_bindings {
// get: "/v1/users/{user_id}/messages/{message_id}"
// }
// };
// }
// }
// message GetMessageRequest {
// string message_id = 1;
// string user_id = 2;
// }
//
//
// This enables the following two alternative HTTP JSON to RPC
// mappings:
//
// HTTP | RPC
// -----|-----
// `GET /v1/messages/123456` | `GetMessage(message_id: "123456")`
// `GET /v1/users/me/messages/123456` | `GetMessage(user_id: "me" message_id:
// "123456")`
//
// # Rules for HTTP mapping
//
// The rules for mapping HTTP path, query parameters, and body fields
// to the request message are as follows:
//
// 1. The `body` field specifies either `*` or a field path, or is
// omitted. If omitted, it indicates there is no HTTP request body.
// 2. Leaf fields (recursive expansion of nested messages in the
// request) can be classified into three types:
// (a) Matched in the URL template.
// (b) Covered by body (if body is `*`, everything except (a) fields;
// else everything under the body field)
// (c) All other fields.
// 3. URL query parameters found in the HTTP request are mapped to (c) fields.
// 4. Any body sent with an HTTP request can contain only (b) fields.
//
// The syntax of the path template is as follows:
//
// Template = "/" Segments [ Verb ] ;
// Segments = Segment { "/" Segment } ;
// Segment = "*" | "**" | LITERAL | Variable ;
// Variable = "{" FieldPath [ "=" Segments ] "}" ;
// FieldPath = IDENT { "." IDENT } ;
// Verb = ":" LITERAL ;
//
// The syntax `*` matches a single path segment. The syntax `**` matches zero
// or more path segments, which must be the last part of the path except the
// `Verb`. The syntax `LITERAL` matches literal text in the path.
//
// The syntax `Variable` matches part of the URL path as specified by its
// template. A variable template must not contain other variables. If a variable
// matches a single path segment, its template may be omitted, e.g. `{var}`
// is equivalent to `{var=*}`.
//
// If a variable contains exactly one path segment, such as `"{var}"` or
// `"{var=*}"`, when such a variable is expanded into a URL path, all characters
// except `[-_.~0-9a-zA-Z]` are percent-encoded. Such variables show up in the
// Discovery Document as `{var}`.
//
// If a variable contains one or more path segments, such as `"{var=foo/*}"`
// or `"{var=**}"`, when such a variable is expanded into a URL path, all
// characters except `[-_.~/0-9a-zA-Z]` are percent-encoded. Such variables
// show up in the Discovery Document as `{+var}`.
//
// NOTE: While the single segment variable matches the semantics of
// [RFC 6570](https://tools.ietf.org/html/rfc6570) Section 3.2.2
// Simple String Expansion, the multi segment variable **does not** match
// RFC 6570 Reserved Expansion. The reason is that the Reserved Expansion
// does not expand special characters like `?` and `#`, which would lead
// to invalid URLs.
//
// NOTE: the field paths in variables and in the `body` must not refer to
// repeated fields or map fields.
message HttpRule {
// Selects methods to which this rule applies.
//
// Refer to [selector][google.api.DocumentationRule.selector] for syntax
// details.
string selector = 1;
// Determines the URL pattern is matched by this rules. This pattern can be
// used with any of the {get|put|post|delete|patch} methods. A custom method
// can be defined using the 'custom' field.
oneof pattern {
// Used for listing and getting information about resources.
string get = 2;
// Used for updating a resource.
string put = 3;
// Used for creating a resource.
string post = 4;
// Used for deleting a resource.
string delete = 5;
// Used for updating a resource.
string patch = 6;
// The custom pattern is used for specifying an HTTP method that is not
// included in the `pattern` field, such as HEAD, or "*" to leave the
// HTTP method unspecified for this rule. The wild-card rule is useful
// for services that provide content to Web (HTML) clients.
CustomHttpPattern custom = 8;
}
// The name of the request field whose value is mapped to the HTTP body, or
// `*` for mapping all fields not captured by the path pattern to the HTTP
// body. NOTE: the referred field must not be a repeated field and must be
// present at the top-level of request message type.
string body = 7;
// Optional. The name of the response field whose value is mapped to the HTTP
// body of response. Other response fields are ignored. When
// not set, the response message will be used as HTTP body of response.
string response_body = 12;
// Additional HTTP bindings for the selector. Nested bindings must
// not contain an `additional_bindings` field themselves (that is,
// the nesting may only be one level deep).
repeated HttpRule additional_bindings = 11;
}
// A custom pattern is used for defining custom HTTP verb.
message CustomHttpPattern {
// The name of this custom HTTP verb.
string kind = 1;
// The path matched by this custom verb.
string path = 2;
}
Dockerfileを用意
build/docker/protoc/Dockerfile
# Goやprotocのバージョンをアプデしたい時はここをいじるだけ↓
FROM golang:1.16.5
ARG PROTOBUF_VERSION=3.17.3
# はじめにインデックスファイルをダウンロード（更新）する必要がある(apt-get update)
# protocをインストールするためにunzipを用意
RUN apt-get update && apt-get install unzip
# protocのダウンロード
WORKDIR /tmp/protoc
RUN curl -L https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip -o protoc.zip && \
unzip protoc.zip && \
mv bin/* /usr/local/bin/ && \
mv include/* /usr/local/include/
WORKDIR /usr/src/todo-server
COPY go.mod .
COPY go.sum .
# プラグインの導入
RUN go get google.golang.org/protobuf/cmd/protoc-gen-go && \
go get google.golang.org/grpc/cmd/protoc-gen-go-grpc && \
go get github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway
protocコマンドのシェルスクリプトを用意
scripts/protoc.sh
#!/bin/sh
proto_file_dir=./protobuf
api_proto_files=$(find ${proto_file_dir}/api -type f -name '*.proto')
proto_out_dir=./pkg/domain/proto
# If the paths=source_relative flag is specified,
# the output file is placed in the same relative directory as the input file.
# For example, an input file protos/buzz.proto results in an output file at protos/buzz.pb.go.
protoc \
-I=${proto_file_dir} \
--go_out=paths=source_relative:${proto_out_dir} \
--go-grpc_out=paths=source_relative,require_unimplemented_servers=false:${proto_out_dir} \
--grpc-gateway_out=paths=source_relative,logtostderr=true:${proto_out_dir} \
${api_proto_files};
楽に実行出来るようにMakefileとdocker-compose.yamlを用意
docker-compose.yaml
version: "3"
services:
protoc:
build:
context: .
dockerfile: ./build/docker/protoc/Dockerfile
working_dir: /usr/src/todo-server
volumes:
- ./:/usr/src/todo-server
Makefile
.PHONY: protoc
protoc: ## gRPCのstubコードの生成
# protoc
docker-compose run --rm --entrypoint sh protoc ./scripts/protoc.sh
# gofmt,goimportsで忘れないように整形 (意外と大事)
gofmt -s -w pkg/domain/proto/
gofmt -s -w protobuf/
goimports -w -local "github.com/todo-server" pkg/domain/proto/
実行する
これだけでOK。
$ make protoc
# 下記が生成される。
# pkg/domain/proto/api/item.pb.go
# pkg/domain/proto/api/item.pb.gw.go
# pkg/domain/proto/api/item_grpc.pb.go
Discussion