Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add support to emit Metrics same as Traces via OTEL #456

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ clean:

test: all
go test -race -v ./...

protos: ${OPENSAVES_GO_PROTOS}

${OPENSAVES_GO_PROTOS}: ${API_DIR}/open_saves.proto
Expand Down
5 changes: 4 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,12 @@ require (
github.com/vmihailenco/msgpack/v5 v5.3.5
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0
go.opentelemetry.io/otel v1.28.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.26.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.26.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.26.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0
go.opentelemetry.io/otel/sdk v1.28.0
go.opentelemetry.io/otel/sdk/metric v1.28.0
gocloud.dev v0.28.0
google.golang.org/api v0.193.0
google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142
Expand Down Expand Up @@ -58,7 +61,7 @@ require (
github.com/google/wire v0.5.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.13.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/huandu/xstrings v1.0.0 // indirect
github.com/imdario/mergo v0.3.12 // indirect
Expand Down
14 changes: 10 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1162,8 +1162,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.1/go.mod h1:G+WkljZi4mflcqVxYSgvt8MNctRQHjEH8ubKtt1Ka3w=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k=
github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok=
github.com/hanwen/go-fuse/v2 v2.1.0/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc=
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
Expand Down Expand Up @@ -1635,8 +1635,8 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
Expand Down Expand Up @@ -1842,6 +1842,10 @@ go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKi
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4=
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.6.1/go.mod h1:NEu79Xo32iVb+0gVNV8PMd7GoWqnyDXRlj04yFjqz40=
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1/go.mod h1:i8vjiSzbiUC7wOQplijSXMYUpNM93DtlS5CbUT+C6oQ=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.26.0 h1:+hm+I+KigBy3M24/h1p/NHkUx/evbLH0PNcjpMyCHc4=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.26.0/go.mod h1:NjC8142mLvvNT6biDpaMjyz78kyEHIwAJlSX0N9P5KI=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.26.0 h1:HGZWGmCVRCVyAs2GQaiHQPbDHo+ObFWeUEOd+zDnp64=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.26.0/go.mod h1:SaH+v38LSCHddyk7RGlU9uZyQoRrKao6IBnJw6Kbn+c=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.6.1/go.mod h1:YJ/JbY5ag/tSQFXzH3mtDmHqzF3aFn3DI/aB1n7pt4w=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.1/go.mod h1:19O5I2U5iys38SsmT2uDJja/300woyzE1KPIQxEUBUc=
Expand Down Expand Up @@ -1871,6 +1875,8 @@ go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBq
go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg=
go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE=
go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE=
go.opentelemetry.io/otel/sdk/metric v1.28.0 h1:OkuaKgKrgAbYrrY0t92c+cC+2F6hsFNnCQArXCKlg08=
go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg=
go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk=
go.opentelemetry.io/otel/trace v1.6.0/go.mod h1:qs7BrU5cZ8dXQHBGxHMOxwME/27YH2qEp4/+tZLLwJE=
Expand Down
30 changes: 25 additions & 5 deletions internal/app/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,10 @@ package server

import (
"context"
"github.com/googleforgames/open-saves/internal/pkg/metrics"
"github.com/googleforgames/open-saves/internal/pkg/tracing"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/trace"
"google.golang.org/grpc/keepalive"
"net"
Expand Down Expand Up @@ -62,15 +64,28 @@ func Run(ctx context.Context, network string, cfg *config.ServiceConfig) error {
}),
}

var meterProvider *metric.MeterProvider
if cfg.EnableMetrics {
log.Println("Enabling Metrics exporter")
meterProvider, err = metrics.InitMetrics(cfg.MetricsEnableGRPCCollector, cfg.MetricsEnableHTTPCollector)
if err != nil {
return err
}
}

defer func() {
if meterProvider != nil {
if err := meterProvider.Shutdown(context.Background()); err != nil {
log.Fatalf("Error shutting down meter provider: %v", err)
}
}
}()

var tracer *trace.TracerProvider
if cfg.EnableTrace {
log.Printf("Enabling CloudTrace exporter with sample rate: %f\n", cfg.ServerConfig.TraceSampleRate)

grpcOptions = append(grpcOptions,
grpc.UnaryInterceptor(otelgrpc.UnaryServerInterceptor()),
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

UnaryServerInterceptor and StreamServerInterceptor have been deprecated in favour of NewServerHandler()

grpc.StreamInterceptor(otelgrpc.StreamServerInterceptor()))

tracer, err = tracing.InitTracer(cfg.ServerConfig.TraceSampleRate, cfg.ServerConfig.EnableGRPCCollector, cfg.ServerConfig.EnableHTTPCollector, cfg.TraceServiceName)
tracer, err = tracing.InitTracer(cfg.ServerConfig.TraceSampleRate, cfg.ServerConfig.TraceEnableGRPCCollector, cfg.ServerConfig.TraceEnableHTTPCollector, cfg.TraceServiceName)
if err != nil {
return err
}
Expand All @@ -84,6 +99,11 @@ func Run(ctx context.Context, network string, cfg *config.ServiceConfig) error {
}
}()

// grpc otel support uses the same option for both metrics and traces.
if cfg.EnableMetrics || cfg.EnableTrace {
grpcOptions = append(grpcOptions, grpc.StatsHandler(otelgrpc.NewServerHandler()))
}

s := grpc.NewServer(grpcOptions...)

healthcheck := health.NewServer()
Expand Down
23 changes: 13 additions & 10 deletions internal/pkg/config/loader.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,16 +96,19 @@ func Load(path string) (*ServiceConfig, error) {
}

serverConfig := ServerConfig{
Address: fmt.Sprintf(":%d", viper.GetUint(OpenSavesPort)),
Cloud: viper.GetString(OpenSavesCloud),
Bucket: viper.GetString(OpenSavesBucket),
Project: viper.GetString(OpenSavesProject),
ShutdownGracePeriod: viper.GetDuration(ShutdownGracePeriod),
EnableTrace: viper.GetBool(EnableTrace),
TraceSampleRate: viper.GetFloat64(TraceSampleRate),
TraceServiceName: viper.GetString(TraceServiceName),
EnableGRPCCollector: viper.GetBool(TraceEnableGRPCCollector),
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've added the Trace prefix for both EnableGRPCCollector and EnableHTTPCollector to differentiate them from the counter parts of Metrics.

It is important to mention that the OTLP endpoint for Traces and Metrics could or could not be the same.

EnableHTTPCollector: viper.GetBool(TraceEnableHTTPCollector),
Address: fmt.Sprintf(":%d", viper.GetUint(OpenSavesPort)),
Cloud: viper.GetString(OpenSavesCloud),
Bucket: viper.GetString(OpenSavesBucket),
Project: viper.GetString(OpenSavesProject),
ShutdownGracePeriod: viper.GetDuration(ShutdownGracePeriod),
EnableMetrics: viper.GetBool(EnableTrace),
MetricsEnableGRPCCollector: viper.GetBool(MetricsEnableGRPCCollector),
MetricsEnableHTTPCollector: viper.GetBool(MetricsEnableHTTPCollector),
EnableTrace: viper.GetBool(EnableTrace),
TraceSampleRate: viper.GetFloat64(TraceSampleRate),
TraceServiceName: viper.GetString(TraceServiceName),
TraceEnableGRPCCollector: viper.GetBool(TraceEnableGRPCCollector),
TraceEnableHTTPCollector: viper.GetBool(TraceEnableHTTPCollector),
}

// Cloud Run environment populates the PORT env var, so check for it here.
Expand Down
22 changes: 16 additions & 6 deletions internal/pkg/config/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,10 @@ const (
GRPCKeepAliveTime = "grpc_keepalive_time"
GRPCKeepAliveTimeout = "grpc_keepalive_timeout"

EnableMetrics = "enable_metrics"
MetricsEnableGRPCCollector = "metrics_enable_grpc_collector"
MetricsEnableHTTPCollector = "metrics_enable_http_collector"

EnableTrace = "enable_trace"
TraceSampleRate = "trace_sample_rate"
TraceServiceName = "trace_service_name"
Expand All @@ -67,14 +71,20 @@ type ServerConfig struct {
Project string
ShutdownGracePeriod time.Duration

// The following enables OpenTelemetry Metrics
// See https://github.com/open-telemetry/opentelemetry-go/tree/main/exporters/otlp/otlpmetric for how to configure the exporters with env variables
EnableMetrics bool
MetricsEnableGRPCCollector bool
MetricsEnableHTTPCollector bool

// The following enables OpenTelemetry Tracing
// It is EXPERIMENTAL and subject to change or removal without notice.
// See https://github.com/open-telemetry/opentelemetry-go/tree/main/exporters/otlp/otlptrace for how to configure the exporters with env variables
EnableTrace bool
TraceSampleRate float64
TraceServiceName string
EnableGRPCCollector bool
EnableHTTPCollector bool
EnableTrace bool
TraceSampleRate float64
TraceServiceName string
TraceEnableGRPCCollector bool
TraceEnableHTTPCollector bool
}

// CacheConfig has configurations for caching control (not Redis specific).
Expand All @@ -85,7 +95,7 @@ type CacheConfig struct {

// RedisConfig as defined in https://pkg.go.dev/github.com/redis/go-redis/v9#Options
type RedisConfig struct {
Address string
Address string
RedisMode string

MaxRetries int
Expand Down
55 changes: 55 additions & 0 deletions internal/pkg/metrics/metric.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package metrics

import (
"context"

"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
"go.opentelemetry.io/otel/sdk/metric"
sdkresource "go.opentelemetry.io/otel/sdk/resource"
)

func InitMetrics(enableGRPCCollector bool, enableHTTPCollector bool) (*metric.MeterProvider, error) {
extraResources, _ := sdkresource.New(
context.Background(),
sdkresource.WithOS(),
sdkresource.WithProcess(),
sdkresource.WithContainer(),
sdkresource.WithHost(),
sdkresource.WithAttributes(),
)
resource, _ := sdkresource.Merge(
sdkresource.Default(),
extraResources,
)

options := []metric.Option{metric.WithResource(resource)}

if enableGRPCCollector {
exporter, err := otlpmetricgrpc.New(context.Background())
if err != nil {
return nil, err
}
options = append(options, metric.WithReader(metric.NewPeriodicReader(exporter)))
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are options we can configure at the PeriodicReader, like the frequency, but I didn't want to make configuration too complex, and the defaults are sensible.

}

if enableHTTPCollector {
exporter, err := otlpmetrichttp.New(context.Background())
if err != nil {
return nil, err
}
options = append(options, metric.WithReader(metric.NewPeriodicReader(exporter)))
}

meterProvider := metric.NewMeterProvider(options...)

// Register as global meter provider so that it can be used via otel.Meter
// and accessed using otel.GetMeterProvider.
// Most instrumentation libraries use the global meter provider as default.
// If the global meter provider is not set then a no-op implementation
// is used, which fails to generate data.
otel.SetMeterProvider(meterProvider)

return meterProvider, nil
}