♻️

FlatBuffersのチュートリアルやってみた (C++, Windows)

2020/09/26に公開

FlatBuffersとは

https://google.github.io/flatbuffers/index.html#flatbuffers_overview

  • バイナリシリアライズフォーマット
  • Google製 (2014年公開)
  • Apache License v2
  • C++, C#, C, Go, Java, JavaScript, PHP, Python, ...etc
  • スキーマ定義で型安全
  • オプションフィールドによりフォーマットに前方、後方互換性を持たせることが可能
  • スキーマレスVerもありJSONっぽく使用することも可?
  • unpackが超高速 (バッファ領域に直接オフセット指定アクセスするためデコード時のメモリアロケートコストなし)
  • ストリーミング処理とかmmapとかと相性がいいらしい
  • Cocos2d-x, Facebook, ...etc

インストール (v1.8.0, Windows)

以下からWindows用バイナリをダウンロードしてパスを通す
https://github.com/google/flatbuffers/releases

チュートリアル (C++)

https://google.github.io/flatbuffers/flatbuffers_guide_tutorial.html

  • 準備
$ mkdir monster_sample
$ cd monster_sample
$ git init
$ git submodule add https://github.com/google/flatbuffers.git
$ cd flatbuffers
$ git checkout v1.8.0
$ cd ../
  • monster.fbsを作成

https://google.github.io/flatbuffers/flatbuffers_guide_writing_schema.html
<details><summary>展開</summary><div>

// Example IDL file for our monster's schema.

namespace MyGame.Sample;

enum Color:byte { Red = 0, Green, Blue = 2 }

union Equipment { Weapon } // Optionally add more tables.

struct Vec3 {
  x:float;
  y:float;
  z:float;
}

table Monster {
  pos:Vec3; // Struct.
  mana:short = 150;
  hp:short = 100;
  name:string;
  friendly:bool = false (deprecated);
  inventory:[ubyte];  // Vector of scalars.
  color:Color = Blue; // Enum.
  weapons:[Weapon];   // Vector of tables.
  equipped:Equipment; // Union.
  path:[Vec3];        // Vector of structs.
}

table Weapon {
  name:string;
  damage:short;
}

root_type Monster;

</div></details>

  • monster.fbsをコンパイル
$ flatc --cpp monster.fbs

以下のmonster_generated.hが生成される
<details><summary>展開</summary><div>

monster_generated.h
// automatically generated by the FlatBuffers compiler, do not modify


#ifndef FLATBUFFERS_GENERATED_MONSTER_MYGAME_SAMPLE_H_
#define FLATBUFFERS_GENERATED_MONSTER_MYGAME_SAMPLE_H_

#include "flatbuffers/flatbuffers.h"

namespace MyGame {
namespace Sample {

struct Vec3;

struct Monster;

struct Weapon;

enum Color {
  Color_Red = 0,
  Color_Green = 1,
  Color_Blue = 2,
  Color_MIN = Color_Red,
  Color_MAX = Color_Blue
};

inline Color (&EnumValuesColor())[3] {
  static Color values[] = {
    Color_Red,
    Color_Green,
    Color_Blue
  };
  return values;
}

inline const char **EnumNamesColor() {
  static const char *names[] = {
    "Red",
    "Green",
    "Blue",
    nullptr
  };
  return names;
}

inline const char *EnumNameColor(Color e) {
  const size_t index = static_cast<int>(e);
  return EnumNamesColor()[index];
}

enum Equipment {
  Equipment_NONE = 0,
  Equipment_Weapon = 1,
  Equipment_MIN = Equipment_NONE,
  Equipment_MAX = Equipment_Weapon
};

inline Equipment (&EnumValuesEquipment())[2] {
  static Equipment values[] = {
    Equipment_NONE,
    Equipment_Weapon
  };
  return values;
}

inline const char **EnumNamesEquipment() {
  static const char *names[] = {
    "NONE",
    "Weapon",
    nullptr
  };
  return names;
}

inline const char *EnumNameEquipment(Equipment e) {
  const size_t index = static_cast<int>(e);
  return EnumNamesEquipment()[index];
}

template<typename T> struct EquipmentTraits {
  static const Equipment enum_value = Equipment_NONE;
};

template<> struct EquipmentTraits<Weapon> {
  static const Equipment enum_value = Equipment_Weapon;
};

bool VerifyEquipment(flatbuffers::Verifier &verifier, const void *obj, Equipment type);
bool VerifyEquipmentVector(flatbuffers::Verifier &verifier, const flatbuffers::Vector<flatbuffers::Offset<void>> *values, const flatbuffers::Vector<uint8_t> *types);

MANUALLY_ALIGNED_STRUCT(4) Vec3 FLATBUFFERS_FINAL_CLASS {
 private:
  float x_;
  float y_;
  float z_;

 public:
  Vec3() {
    memset(this, 0, sizeof(Vec3));
  }
  Vec3(float _x, float _y, float _z)
      : x_(flatbuffers::EndianScalar(_x)),
        y_(flatbuffers::EndianScalar(_y)),
        z_(flatbuffers::EndianScalar(_z)) {
  }
  float x() const {
    return flatbuffers::EndianScalar(x_);
  }
  float y() const {
    return flatbuffers::EndianScalar(y_);
  }
  float z() const {
    return flatbuffers::EndianScalar(z_);
  }
};
STRUCT_END(Vec3, 12);

struct Monster FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
  enum {
    VT_POS = 4,
    VT_MANA = 6,
    VT_HP = 8,
    VT_NAME = 10,
    VT_INVENTORY = 14,
    VT_COLOR = 16,
    VT_WEAPONS = 18,
    VT_EQUIPPED_TYPE = 20,
    VT_EQUIPPED = 22,
    VT_PATH = 24
  };
  const Vec3 *pos() const {
    return GetStruct<const Vec3 *>(VT_POS);
  }
  int16_t mana() const {
    return GetField<int16_t>(VT_MANA, 150);
  }
  int16_t hp() const {
    return GetField<int16_t>(VT_HP, 100);
  }
  const flatbuffers::String *name() const {
    return GetPointer<const flatbuffers::String *>(VT_NAME);
  }
  const flatbuffers::Vector<uint8_t> *inventory() const {
    return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_INVENTORY);
  }
  Color color() const {
    return static_cast<Color>(GetField<int8_t>(VT_COLOR, 2));
  }
  const flatbuffers::Vector<flatbuffers::Offset<Weapon>> *weapons() const {
    return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<Weapon>> *>(VT_WEAPONS);
  }
  Equipment equipped_type() const {
    return static_cast<Equipment>(GetField<uint8_t>(VT_EQUIPPED_TYPE, 0));
  }
  const void *equipped() const {
    return GetPointer<const void *>(VT_EQUIPPED);
  }
  template<typename T> const T *equipped_as() const;
  const Weapon *equipped_as_Weapon() const {
    return equipped_type() == Equipment_Weapon ? static_cast<const Weapon *>(equipped()) : nullptr;
  }
  const flatbuffers::Vector<const Vec3 *> *path() const {
    return GetPointer<const flatbuffers::Vector<const Vec3 *> *>(VT_PATH);
  }
  bool Verify(flatbuffers::Verifier &verifier) const {
    return VerifyTableStart(verifier) &&
           VerifyField<Vec3>(verifier, VT_POS) &&
           VerifyField<int16_t>(verifier, VT_MANA) &&
           VerifyField<int16_t>(verifier, VT_HP) &&
           VerifyOffset(verifier, VT_NAME) &&
           verifier.Verify(name()) &&
           VerifyOffset(verifier, VT_INVENTORY) &&
           verifier.Verify(inventory()) &&
           VerifyField<int8_t>(verifier, VT_COLOR) &&
           VerifyOffset(verifier, VT_WEAPONS) &&
           verifier.Verify(weapons()) &&
           verifier.VerifyVectorOfTables(weapons()) &&
           VerifyField<uint8_t>(verifier, VT_EQUIPPED_TYPE) &&
           VerifyOffset(verifier, VT_EQUIPPED) &&
           VerifyEquipment(verifier, equipped(), equipped_type()) &&
           VerifyOffset(verifier, VT_PATH) &&
           verifier.Verify(path()) &&
           verifier.EndTable();
  }
};

template<> inline const Weapon *Monster::equipped_as<Weapon>() const {
  return equipped_as_Weapon();
}

struct MonsterBuilder {
  flatbuffers::FlatBufferBuilder &fbb_;
  flatbuffers::uoffset_t start_;
  void add_pos(const Vec3 *pos) {
    fbb_.AddStruct(Monster::VT_POS, pos);
  }
  void add_mana(int16_t mana) {
    fbb_.AddElement<int16_t>(Monster::VT_MANA, mana, 150);
  }
  void add_hp(int16_t hp) {
    fbb_.AddElement<int16_t>(Monster::VT_HP, hp, 100);
  }
  void add_name(flatbuffers::Offset<flatbuffers::String> name) {
    fbb_.AddOffset(Monster::VT_NAME, name);
  }
  void add_inventory(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> inventory) {
    fbb_.AddOffset(Monster::VT_INVENTORY, inventory);
  }
  void add_color(Color color) {
    fbb_.AddElement<int8_t>(Monster::VT_COLOR, static_cast<int8_t>(color), 2);
  }
  void add_weapons(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Weapon>>> weapons) {
    fbb_.AddOffset(Monster::VT_WEAPONS, weapons);
  }
  void add_equipped_type(Equipment equipped_type) {
    fbb_.AddElement<uint8_t>(Monster::VT_EQUIPPED_TYPE, static_cast<uint8_t>(equipped_type), 0);
  }
  void add_equipped(flatbuffers::Offset<void> equipped) {
    fbb_.AddOffset(Monster::VT_EQUIPPED, equipped);
  }
  void add_path(flatbuffers::Offset<flatbuffers::Vector<const Vec3 *>> path) {
    fbb_.AddOffset(Monster::VT_PATH, path);
  }
  explicit MonsterBuilder(flatbuffers::FlatBufferBuilder &_fbb)
        : fbb_(_fbb) {
    start_ = fbb_.StartTable();
  }
  MonsterBuilder &operator=(const MonsterBuilder &);
  flatbuffers::Offset<Monster> Finish() {
    const auto end = fbb_.EndTable(start_);
    auto o = flatbuffers::Offset<Monster>(end);
    return o;
  }
};

inline flatbuffers::Offset<Monster> CreateMonster(
    flatbuffers::FlatBufferBuilder &_fbb,
    const Vec3 *pos = 0,
    int16_t mana = 150,
    int16_t hp = 100,
    flatbuffers::Offset<flatbuffers::String> name = 0,
    flatbuffers::Offset<flatbuffers::Vector<uint8_t>> inventory = 0,
    Color color = Color_Blue,
    flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Weapon>>> weapons = 0,
    Equipment equipped_type = Equipment_NONE,
    flatbuffers::Offset<void> equipped = 0,
    flatbuffers::Offset<flatbuffers::Vector<const Vec3 *>> path = 0) {
  MonsterBuilder builder_(_fbb);
  builder_.add_path(path);
  builder_.add_equipped(equipped);
  builder_.add_weapons(weapons);
  builder_.add_inventory(inventory);
  builder_.add_name(name);
  builder_.add_pos(pos);
  builder_.add_hp(hp);
  builder_.add_mana(mana);
  builder_.add_equipped_type(equipped_type);
  builder_.add_color(color);
  return builder_.Finish();
}

inline flatbuffers::Offset<Monster> CreateMonsterDirect(
    flatbuffers::FlatBufferBuilder &_fbb,
    const Vec3 *pos = 0,
    int16_t mana = 150,
    int16_t hp = 100,
    const char *name = nullptr,
    const std::vector<uint8_t> *inventory = nullptr,
    Color color = Color_Blue,
    const std::vector<flatbuffers::Offset<Weapon>> *weapons = nullptr,
    Equipment equipped_type = Equipment_NONE,
    flatbuffers::Offset<void> equipped = 0,
    const std::vector<const Vec3 *> *path = nullptr) {
  return MyGame::Sample::CreateMonster(
      _fbb,
      pos,
      mana,
      hp,
      name ? _fbb.CreateString(name) : 0,
      inventory ? _fbb.CreateVector<uint8_t>(*inventory) : 0,
      color,
      weapons ? _fbb.CreateVector<flatbuffers::Offset<Weapon>>(*weapons) : 0,
      equipped_type,
      equipped,
      path ? _fbb.CreateVector<const Vec3 *>(*path) : 0);
}

struct Weapon FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
  enum {
    VT_NAME = 4,
    VT_DAMAGE = 6
  };
  const flatbuffers::String *name() const {
    return GetPointer<const flatbuffers::String *>(VT_NAME);
  }
  int16_t damage() const {
    return GetField<int16_t>(VT_DAMAGE, 0);
  }
  bool Verify(flatbuffers::Verifier &verifier) const {
    return VerifyTableStart(verifier) &&
           VerifyOffset(verifier, VT_NAME) &&
           verifier.Verify(name()) &&
           VerifyField<int16_t>(verifier, VT_DAMAGE) &&
           verifier.EndTable();
  }
};

struct WeaponBuilder {
  flatbuffers::FlatBufferBuilder &fbb_;
  flatbuffers::uoffset_t start_;
  void add_name(flatbuffers::Offset<flatbuffers::String> name) {
    fbb_.AddOffset(Weapon::VT_NAME, name);
  }
  void add_damage(int16_t damage) {
    fbb_.AddElement<int16_t>(Weapon::VT_DAMAGE, damage, 0);
  }
  explicit WeaponBuilder(flatbuffers::FlatBufferBuilder &_fbb)
        : fbb_(_fbb) {
    start_ = fbb_.StartTable();
  }
  WeaponBuilder &operator=(const WeaponBuilder &);
  flatbuffers::Offset<Weapon> Finish() {
    const auto end = fbb_.EndTable(start_);
    auto o = flatbuffers::Offset<Weapon>(end);
    return o;
  }
};

inline flatbuffers::Offset<Weapon> CreateWeapon(
    flatbuffers::FlatBufferBuilder &_fbb,
    flatbuffers::Offset<flatbuffers::String> name = 0,
    int16_t damage = 0) {
  WeaponBuilder builder_(_fbb);
  builder_.add_name(name);
  builder_.add_damage(damage);
  return builder_.Finish();
}

inline flatbuffers::Offset<Weapon> CreateWeaponDirect(
    flatbuffers::FlatBufferBuilder &_fbb,
    const char *name = nullptr,
    int16_t damage = 0) {
  return MyGame::Sample::CreateWeapon(
      _fbb,
      name ? _fbb.CreateString(name) : 0,
      damage);
}

inline bool VerifyEquipment(flatbuffers::Verifier &verifier, const void *obj, Equipment type) {
  switch (type) {
    case Equipment_NONE: {
      return true;
    }
    case Equipment_Weapon: {
      auto ptr = reinterpret_cast<const Weapon *>(obj);
      return verifier.VerifyTable(ptr);
    }
    default: return false;
  }
}

inline bool VerifyEquipmentVector(flatbuffers::Verifier &verifier, const flatbuffers::Vector<flatbuffers::Offset<void>> *values, const flatbuffers::Vector<uint8_t> *types) {
  if (values->size() != types->size()) return false;
  for (flatbuffers::uoffset_t i = 0; i < values->size(); ++i) {
    if (!VerifyEquipment(
        verifier,  values->Get(i), types->GetEnum<Equipment>(i))) {
      return false;
    }
  }
  return true;
}

inline const MyGame::Sample::Monster *GetMonster(const void *buf) {
  return flatbuffers::GetRoot<MyGame::Sample::Monster>(buf);
}

inline bool VerifyMonsterBuffer(
    flatbuffers::Verifier &verifier) {
  return verifier.VerifyBuffer<MyGame::Sample::Monster>(nullptr);
}

inline void FinishMonsterBuffer(
    flatbuffers::FlatBufferBuilder &fbb,
    flatbuffers::Offset<MyGame::Sample::Monster> root) {
  fbb.Finish(root);
}

}  // namespace Sample
}  // namespace MyGame

#endif  // FLATBUFFERS_GENERATED_MONSTER_MYGAME_SAMPLE_H_

</div></details>

  • エンコードするプログラムの作成

<details><summary>展開</summary><div>

encoder.cpp
#include <iostream>
#include <fstream>
using namespace std;

#include "monster_generated.h"
using namespace MyGame::Sample;

int main() {

	// 作業領域確保 (初期値1KB)
	flatbuffers::FlatBufferBuilder builder(1024);

	// name, hp, mana, pos
	auto name = builder.CreateString("Orc");
	int hp = 300;
	int mana = 150;
	auto pos = Vec3(1.0f, 2.0f, 3.0f);

	// Weapon 生成
	auto weapon_one_name = builder.CreateString("Sword");
	short weapon_one_damage = 3;
	auto sword = CreateWeapon(builder, weapon_one_name, weapon_one_damage);

	auto weapon_two_name = builder.CreateString("Axe");
	short weapon_two_damage = 5;
	auto axe = CreateWeapon(builder, weapon_two_name, weapon_two_damage);

	std::vector<flatbuffers::Offset<Weapon>> weapons_vector;
	weapons_vector.push_back(sword);
	weapons_vector.push_back(axe);
	auto weapons = builder.CreateVector(weapons_vector);

	// inventory
	unsigned char treasure[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
	auto inventory = builder.CreateVector(treasure, 10);

	// path
	Vec3 points[] = { Vec3(1.0f, 2.0f, 3.0f), Vec3(4.0f, 5.0f, 6.0f) };
	auto path = builder.CreateVectorOfStructs(points, 2);

	// Monster 生成
#if 1 // どちらでも可
	auto orc = CreateMonster(builder, &pos, mana, hp, name,
							inventory, Color_Red, weapons,
							Equipment_Weapon, axe.Union(), path);
#else
	MonsterBuilder monster_builder(builder);
	monster_builder.add_pos(&pos);
	monster_builder.add_mana(mana);
	monster_builder.add_hp(hp);
	monster_builder.add_name(name);
	monster_builder.add_inventory(inventory);
	monster_builder.add_color(Color_Red);
	monster_builder.add_weapons(weapons);
	monster_builder.add_equipped_type(Equipment_Weapon);
	monster_builder.add_equipped(axe.Union());
	monster_builder.add_path(path);
	auto orc = monster_builder.Finish();
#endif

#if 1 // どちらでも可
	builder.Finish(orc);
#else
	FinishMonsterBuffer(builder, orc);
#endif

	// バッファ取得 ※This must be called after `Finish()`.
	uint8_t *buf = builder.GetBufferPointer();
	size_t size = builder.GetSize();

	// ファイルに書き込み
 	ofstream fout;
    fout.open("monster.bin", ios::out|ios::binary|ios::trunc);
    if (!fout) {
        return 1;
    }
    fout.write((char*)buf, size);
    fout.close();
}

</div></details>

  • デコードするプログラムの作成

<details><summary>展開</summary><div>

decoder.cpp
#include <iostream>
#include <fstream>
using namespace std;

#include "monster_generated.h"
using namespace MyGame::Sample;

int main() {

	// ファイル読み込み
 	ifstream fin("monster.bin", ios::in | ios::binary);
    if (!fin) {
        return 1;
    }
	auto begin = fin.tellg();
	fin.seekg(0, fin.end);
	auto end = fin.tellg();
	fin.clear();
	fin.seekg(0, fin.beg);
	auto len = end - begin;
	auto buf = new char[len + 1];
	fin.read(buf, len);
    fin.close();

	// バッファ設定
	auto monster = GetMonster((uint8_t*)buf);

	// hp, mana, name
	printf("hp: %d\n", monster->hp());
	printf("mana: %d\n", monster->mana());
	printf("name: %s\n", monster->name()->c_str());

	// inventory
	printf("inventory: [");
	for (auto val: *monster->inventory()) {
		printf("%d, ", val);
	}
	printf("\b\b]\n");

	// weapons
	printf("weapons: [");
	for (auto wp: *monster->weapons()) {
		printf("{\n");
		printf("\tname: %s\n", wp->name()->c_str());
		printf("\tdamage: %d\n", wp->damage());
		printf("}, ");
	}
	printf("\b\b]\n");

	// equipped
	auto union_type = monster->equipped_type();
	if (union_type == Equipment_Weapon) {
		// Requires `static_cast` to type `const Weapon*`.
		auto weapon = static_cast<const Weapon*>(monster->equipped());
		printf("equipped: Weapon{\n");
		printf("\tname: %s\n", weapon->name()->c_str());
		printf("\tdamage: %d\n", weapon->damage());
		printf("}\n");
	}

	delete[] buf;
}

</div></details>

  • コンパイル (g++.exe (x86_64-win32-seh-rev1, Built by MinGW-W64 project) 7.2.0)
$ g++ -I flatbuffers/include -o encoder.exe encoder.cpp
$ g++ -I flatbuffers/include -o decoder.exe decoder.cpp
  • エンコード
$ ./encode.exe

monster.bin.png

  • デコード
$ ./decode.exe
hp: 300
mana: 150
name: Orc
inventory: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
weapons: [{
        name: Sword
        damage: 3
}, {
        name: Axe
        damage: 5
}]
equipped: Weapon{
        name: Axe
        damage: 5
}

Discussion