LiveKitの本番運用に向けて気になったとこ色々調べる

# ここを設定
redis:
  address: redis.mine.net:6379
  db: 0
  username: myredis
  password: xxxxxxxxx
  # To use sentinel remove the address key above and add the following
  # sentinel_master_name: livekit
  # sentinel_addresses:
  # - livekit-redis-node-0.livekit-redis-headless:26379
  # - livekit-redis-node-1.livekit-redis-headless:26379
  # If you use a different set of credentials for sentinel add
  # sentinel_username: user
  # sentinel_password: pass

  # To use TLS with redis
  tls:
    enabled: true
    # when set to true, LiveKit will not verify the server's certificate, defaults to true
    insecure: true
    # server_name: myserver.com
    # file containing trusted root certificates for verification
    # ca_cert_file: /path/to/ca.crt
    # client_cert_file: /path/to/client.crt
    # client_key_file: /path/to/client.key

  # To use cluster remove the address key above and add the following
  # cluster_addresses:
  # - livekit-redis-node-0.livekit-redis-headless:6379
  # - livekit-redis-node-1.livekit-redis-headless:6380
  # And it will use the password key above as cluster password
  # And the db key will not be used due to cluster mode not support it.

type RedisConfig struct {
	Address  string `yaml:"address,omitempty"`
	Username string `yaml:"username,omitempty"`
	Password string `yaml:"password,omitempty"`
	DB       int    `yaml:"db,omitempty"`
	// Deprecated: use TLS instead of UseTLS
	UseTLS            bool         `yaml:"use_tls,omitempty"`
	TLS               *xtls.Config `yaml:"tls,omitempty"`
	MasterName        string       `yaml:"sentinel_master_name,omitempty"`
	SentinelUsername  string       `yaml:"sentinel_username,omitempty"`
	SentinelPassword  string       `yaml:"sentinel_password,omitempty"`
	SentinelAddresses []string     `yaml:"sentinel_addresses,omitempty"`
	ClusterAddresses  []string     `yaml:"cluster_addresses,omitempty"`
	DialTimeout       int          `yaml:"dial_timeout,omitempty"`
	ReadTimeout       int          `yaml:"read_timeout,omitempty"`
	WriteTimeout      int          `yaml:"write_timeout,omitempty"`
	// for clustererd mode only, number of redirects to follow, defaults to 2
	MaxRedirects *int          `yaml:"max_redirects,omitempty"`
	PoolTimeout  time.Duration `yaml:"pool_timeout,omitempty"`
	PoolSize     int           `yaml:"pool_size,omitempty"`
}

// NewUniversalClient returns a new multi client. The type of the returned client depends
// on the following conditions:
//
// 1. If the MasterName option is specified, a sentinel-backed FailoverClient is returned.
// 2. if the number of Addrs is two or more, a ClusterClient is returned.
// 3. Otherwise, a single-node Client is returned.
func NewUniversalClient(opts *UniversalOptions) UniversalClient {
	if opts.MasterName != "" {
		return NewFailoverClient(opts.Failover())
	} else if len(opts.Addrs) > 1 {
		return NewClusterClient(opts.Cluster())
	}
	return NewClient(opts.Simple())
}

	httpGroup := &errgroup.Group{}
	for _, ln := range listeners {
		l := ln
		httpGroup.Go(func() error {
			return s.httpServer.Serve(l)
		})
	}

クライアントがLiveKitへのシグナル接続を確立すると、インスタンスの1つと持続的なWebSocket接続が作成されます。
そのインスタンスは、ルームがホストされているノードとクライアント間のメッセージをプロキシするシグナルブリッジとして動作します。

マルチノードのセットアップでは、LiveKitは多数の同時ルームをサポートすることができます。
ただし、今のところルームは1つのノードに収まる必要があるため、ルームの参加者数には制限があります。

# # node selector is (possibly) for multi region deployment
# node_selector:
#   # default: any. valid values: any, sysload, cpuload, regionaware
#   kind: sysload
#   # priority used for selection of node when multiple are available
#   # default: random. valid values: random, sysload, cpuload, rooms, clients, tracks, bytespersec
#   sort_by: sysload
#   # used in sysload and regionaware
#   # do not assign room to node if load per CPU exceeds sysload_limit
#   sysload_limit: 0.7
#   # used in regionaware
#   # list of regions and their lat/lon coordinates
#   regions:
#     - name: us-west-2
#       lat: 44.19434095976287
#       lon: -123.0674908379146

AnySelector: 利用可能なノードを制限なく選択する
CPULoadSelector: CPU使用率がCPULoadLimitより高いノードを排除し、その後、過負荷でないノードからノードを選択する
SystemLoadSelector: SysloadLimitより高いCPUあたりのノードを持つノードを排除し、その後、過負荷でないノードからノードを選択します。
RegionAwareSelector: 現在のインスタンスのリージョンに最も近い利用可能なノードを優先する。

// NodeSelector selects an appropriate node to run the current session
type NodeSelector interface {
	SelectNode(nodes []*livekit.Node) (*livekit.Node, error)
}

func (s *SystemLoadSelector) filterNodes(nodes []*livekit.Node) ([]*livekit.Node, error) {
	nodes = GetAvailableNodes(nodes)
	if len(nodes) == 0 {
		return nil, ErrNoAvailableNodes
	}

	nodesLowLoad := make([]*livekit.Node, 0)
	for _, node := range nodes {
		if GetNodeSysload(node) < s.SysloadLimit {
			nodesLowLoad = append(nodesLowLoad, node)
		}
	}
	if len(nodesLowLoad) > 0 {
		nodes = nodesLowLoad
	}
	return nodes, nil
}

func (s *SystemLoadSelector) SelectNode(nodes []*livekit.Node) (*livekit.Node, error) {
	nodes, err := s.filterNodes(nodes)
	if err != nil {
		return nil, err
	}

	return SelectSortedNode(nodes, s.SortBy)
}

func SelectSortedNode(nodes []*livekit.Node, sortBy string) (*livekit.Node, error) {
	if sortBy == "" {
		return nil, ErrSortByNotSet
	}

	// Return a node based on what it should be sorted by for priority
	switch sortBy {
	case "random":
		idx := funk.RandomInt(0, len(nodes))
		return nodes[idx], nil
	case "sysload":
		sort.Slice(nodes, func(i, j int) bool {
			return GetNodeSysload(nodes[i]) < GetNodeSysload(nodes[j])
		})
		return nodes[0], nil
	case "cpuload":
		sort.Slice(nodes, func(i, j int) bool {
			return nodes[i].Stats.CpuLoad < nodes[j].Stats.CpuLoad
		})
		return nodes[0], nil
	case "rooms":
		sort.Slice(nodes, func(i, j int) bool {
			return nodes[i].Stats.NumRooms < nodes[j].Stats.NumRooms
		})
		return nodes[0], nil
	case "clients":
		sort.Slice(nodes, func(i, j int) bool {
			return nodes[i].Stats.NumClients < nodes[j].Stats.NumClients
		})
		return nodes[0], nil
	case "tracks":
		sort.Slice(nodes, func(i, j int) bool {
			return nodes[i].Stats.NumTracksIn+nodes[i].Stats.NumTracksOut < nodes[j].Stats.NumTracksIn+nodes[j].Stats.NumTracksOut
		})
		return nodes[0], nil
	case "bytespersec":
		sort.Slice(nodes, func(i, j int) bool {
			return nodes[i].Stats.BytesInPerSec+nodes[i].Stats.BytesOutPerSec < nodes[j].Stats.BytesInPerSec+nodes[j].Stats.BytesOutPerSec
		})
		return nodes[0], nil
	default:
		return nil, ErrSortByUnknown
	}
}

lk load-test \
  --url <YOUR-SERVER-URL> \
  --api-key <YOUR-KEY> \
  --api-secret <YOUR-SECRET> \
  --room load-test \
  --audio-publishers 10 \
  --subscribers 10 \
  --duration 1m

Summary | Tester | Tracks | Bitrate                 | Total Dropped | Error
        | Sub 10 | 6/10   | 121.7kbps               | 0 (0%)        | -
        | Sub 11 | 6/10   | 121.7kbps               | 0 (0%)        | -
        | Sub 12 | 6/10   | 121.6kbps               | 0 (0%)        | -
        | Sub 13 | 6/10   | 121.5kbps               | 0 (0%)        | -
        | Sub 14 | 6/10   | 121.4kbps               | 0 (0%)        | -
        | Sub 15 | 6/10   | 121.5kbps               | 0 (0%)        | -
        | Sub 16 | 6/10   | 119.5kbps               | 0 (0%)        | -
        | Sub 17 | 6/10   | 121.1kbps               | 0 (0%)        | -
        | Sub 18 | 6/10   | 121.2kbps               | 0 (0%)        | -
        | Sub 19 | 6/10   | 120.8kbps               | 0 (0%)        | -
        | Total  | 60/100 | 1.2mbps (119.5kbps avg) | 0 (0%)        | 0

# WebRTC configuration
rtc:
  # node_ip: 106.168.167.96
  # UDP ports to use for client traffic.
  # this port range should be open for inbound traffic on the firewall
  port_range_start: 40000
  port_range_end: 65535
  # when set, LiveKit enable WebRTC ICE over TCP when UDP isn't available
  # this port *cannot* be behind load balancer or TLS, and must be exposed on the node
  # WebRTC transports are encrypted and do not require additional encryption
  # only 80/443 on public IP are allowed if less than 1024
  tcp_port: 7881
  # when set to true, attempts to discover the host's public IP via STUN
  # this is useful for cloud environments such as AWS & Google where hosts have an internal IP
  # that maps to an external one
  use_external_ip: true
  # # when set, LiveKit will attempt to use a UDP mux so all UDP traffic goes through
  # # listed port(s). To maximize system performance, we recommend using a range of ports
  # # greater or equal to the number of vCPUs on the machine.
  # # port_range_start & end must not be set for this config to take effect
  # udp_port: 7882-7892
  # # when set to true, server will use a lite ice agent, that will speed up ice connection, but
  # # might cause connect issue if server running behind NAT.
  # use_ice_lite: true
  # # optional STUN servers for LiveKit clients to use. Clients will be configured to use these STUN servers automatically.
  # # by default LiveKit clients use Google's public STUN servers
  # stun_servers:    <-- ここ⭐️
  #    - 

// pkg/config.go

var DefaultStunServers = []string{
	"global.stun.twilio.com:3478",
	"stun.l.google.com:19302",
	"stun1.l.google.com:19302",
}

// pkg/config.go

func (conf *RTCConfig) Validate(development bool) error {
	// set defaults for ports if none are set
	if !conf.UDPPort.Valid() && conf.ICEPortRangeStart == 0 {
		// to make it easier to run in dev mode/docker, default to single port
		if development {
			conf.UDPPort = PortRange{Start: 7882}
		} else {
			conf.ICEPortRangeStart = 50000
			conf.ICEPortRangeEnd = 60000
		}
	}

	var err error
	if conf.NodeIP == "" || conf.UseExternalIP {
		conf.NodeIP, err = conf.determineIP()
		if err != nil {
			logger.Warnw("could not determine node ip", err)
			return err
		}
		conf.NodeIPAutoGenerated = true
	}

	return nil
}

func NewWebRTCConfig(rtcConf *RTCConfig, development bool) (*WebRTCConfig, error) {
...
..
	if rtcConf.UseICELite {
		s.SetLite(true)
	} else if (rtcConf.NodeIP == "" || rtcConf.NodeIPAutoGenerated) && !rtcConf.UseExternalIP {
		// use STUN servers for server to support NAT
		// when deployed in production, we expect UseExternalIP to be used, and ports accessible
		// this is not compatible with ICE Lite
		// Do not automatically add STUN servers if nodeIP is set
		if len(rtcConf.STUNServers) > 0 {
			c.ICEServers = []webrtc.ICEServer{iceServerForStunServers(rtcConf.STUNServers)}
		} else {
			c.ICEServers = []webrtc.ICEServer{iceServerForStunServers(DefaultStunServers)}
		}
	}
...
..