diff --git a/chain/indexer.go b/chain/indexer.go index da2aae663..a9a96ea69 100644 --- a/chain/indexer.go +++ b/chain/indexer.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/sha256" + "fmt" "sync" "time" @@ -158,9 +159,15 @@ func NewTipSetIndexer(node lens.API, d model.Storage, window time.Duration, name // TipSet is called when a new tipset has been discovered func (t *TipSetIndexer) TipSet(ctx context.Context, ts *types.TipSet) error { - ctx, span := otel.Tracer("").Start(ctx, "Indexer.TipSet") + ctx, span := otel.Tracer("").Start(ctx, "TipSetIndexer.TipSet") if span.IsRecording() { - span.SetAttributes(attribute.String("tipset", ts.String()), attribute.Int64("height", int64(ts.Height()))) + span.SetAttributes( + attribute.String("tipset", ts.String()), + attribute.Int64("height", int64(ts.Height())), + attribute.String("name", t.name), + attribute.String("window", t.window.String()), + attribute.StringSlice("tasks", t.tasks), + ) } defer span.End() @@ -211,6 +218,15 @@ func (t *TipSetIndexer) TipSet(ctx context.Context, ts *types.TipSet) error { return nil } + if span.IsRecording() { + span.SetAttributes( + attribute.String("next_tipset", next.String()), + attribute.Int64("next_height", int64(next.Height())), + attribute.String("current_tipset", current.String()), + attribute.Int64("current_height", int64(current.Height())), + ) + } + ll := log.With("current", int64(current.Height()), "next", int64(next.Height())) ll.Debugw("indexing tipset") @@ -386,12 +402,14 @@ func (t *TipSetIndexer) TipSet(ctx context.Context, ts *types.TipSet) error { if _, complete := completed[name]; !complete { taskOutputs[name] = model.PersistableList{t.buildSkippedTipsetReport(ts, name, start, "indexer not ready")} ll.Infow("task skipped", "task", name, "reason", "indexer not ready") + span.AddEvent(fmt.Sprintf("skipped task: %s", res.Task)) } } stats.Record(ctx, metrics.TipSetSkip.M(1)) goto persist case res = <-results: } + span.AddEvent(fmt.Sprintf("completed task: %s", res.Task)) inFlight-- llt := ll.With("task", res.Task) @@ -450,8 +468,16 @@ persist: // Persist all results go func() { + ctx, persistSpan := otel.Tracer("").Start(ctx, "TipSetIndexer.Persist") + if persistSpan.IsRecording() { + persistSpan.SetAttributes( + attribute.String("tipset", ts.String()), + attribute.Int64("height", int64(ts.Height())), + ) + } // free up the slot when done defer func() { + persistSpan.End() <-t.persistSlot }() @@ -477,11 +503,13 @@ persist: wg.Wait() ll.Infow("tasks complete", "total_time", time.Since(start)) }() - return nil } func (t *TipSetIndexer) runProcessor(ctx context.Context, p TipSetProcessor, name string, ts *types.TipSet, results chan *TaskResult) { + ctx, span := otel.Tracer("").Start(ctx, fmt.Sprintf("TipSetIndexer.Processor.%s", name)) + defer span.End() + ctx, _ = tag.New(ctx, tag.Upsert(metrics.TaskType, name)) stats.Record(ctx, metrics.TipsetHeight.M(int64(ts.Height()))) stop := metrics.Timer(ctx, metrics.ProcessingDuration) @@ -540,9 +568,12 @@ func (t *TipSetIndexer) getGenesisActors(ctx context.Context) (map[string]lens.A // and applies it to versions of state tress supporting it. These include Version 2 and 3 of the lotus state tree implementation. // stateChangedActors will fall back to the lotus API method when the optimized diffing cannot be applied. func (t *TipSetIndexer) stateChangedActors(ctx context.Context, old, new cid.Cid) (map[string]lens.ActorStateChange, error) { - ctx, span := otel.Tracer("").Start(ctx, "StateChangedActors") + ctx, span := otel.Tracer("").Start(ctx, "TipSetIndexer.StateChangedActors") if span.IsRecording() { - span.SetAttributes(attribute.String("old", old.String()), attribute.String("new", new.String())) + span.SetAttributes( + attribute.String("old", old.String()), + attribute.String("new", new.String()), + ) } defer span.End() @@ -561,9 +592,7 @@ func (t *TipSetIndexer) stateChangedActors(ctx context.Context, old, new cid.Cid } if newVersion == oldVersion && (newVersion != types.StateTreeVersion0 && newVersion != types.StateTreeVersion1) { - if span.IsRecording() { - span.SetAttributes(attribute.String("diff", "fast")) - } + span.SetAttributes(attribute.String("diff", "fast")) // TODO: replace hamt.UseTreeBitWidth and hamt.UseHashFunction with values based on network version changes, err := hamt.Diff(ctx, t.node.Store(), t.node.Store(), oldRoot, newRoot, hamt.UseTreeBitWidth(5), hamt.UseHashFunction(func(input []byte) []byte { res := sha256.Sum256(input) @@ -609,6 +638,7 @@ func (t *TipSetIndexer) stateChangedActors(ctx context.Context, old, new cid.Cid return out, nil } } + span.SetAttributes(attribute.String("diff", "slow")) log.Debug("using slow state diff") actors, err := t.node.StateChangedActors(ctx, old, new) if err != nil { @@ -626,6 +656,9 @@ func (t *TipSetIndexer) stateChangedActors(ctx context.Context, old, new cid.Cid } func (t *TipSetIndexer) runMessageProcessor(ctx context.Context, p MessageProcessor, name string, ts, pts *types.TipSet, emsgs []*lens.ExecutedMessage, blkMsgs []*lens.BlockMessages, results chan *TaskResult) { + ctx, span := otel.Tracer("").Start(ctx, fmt.Sprintf("TipSetIndexer.MessageProcessor.%s", name)) + defer span.End() + ctx, _ = tag.New(ctx, tag.Upsert(metrics.TaskType, name)) stats.Record(ctx, metrics.TipsetHeight.M(int64(ts.Height()))) stop := metrics.Timer(ctx, metrics.ProcessingDuration) @@ -653,6 +686,9 @@ func (t *TipSetIndexer) runMessageProcessor(ctx context.Context, p MessageProces } func (t *TipSetIndexer) runConsensusProcessor(ctx context.Context, p TipSetsProcessor, name string, ts, pts *types.TipSet, results chan *TaskResult) { + ctx, span := otel.Tracer("").Start(ctx, fmt.Sprintf("TipSetIndexer.ConsensusProcessor.%s", name)) + defer span.End() + ctx, _ = tag.New(ctx, tag.Upsert(metrics.TaskType, name)) stats.Record(ctx, metrics.TipsetHeight.M(int64(ts.Height()))) stop := metrics.Timer(ctx, metrics.ProcessingDuration) @@ -680,6 +716,9 @@ func (t *TipSetIndexer) runConsensusProcessor(ctx context.Context, p TipSetsProc } func (t *TipSetIndexer) runActorProcessor(ctx context.Context, p ActorProcessor, name string, ts, pts *types.TipSet, actors map[string]lens.ActorStateChange, emsgs []*lens.ExecutedMessage, results chan *TaskResult) { + ctx, span := otel.Tracer("").Start(ctx, fmt.Sprintf("TipSetIndexer.ActorProcessor.%s", name)) + defer span.End() + ctx, _ = tag.New(ctx, tag.Upsert(metrics.TaskType, name)) stats.Record(ctx, metrics.TipsetHeight.M(int64(ts.Height()))) stop := metrics.Timer(ctx, metrics.ProcessingDuration) @@ -707,6 +746,9 @@ func (t *TipSetIndexer) runActorProcessor(ctx context.Context, p ActorProcessor, } func (t *TipSetIndexer) runMessageExecutionProcessor(ctx context.Context, p MessageExecutionProcessor, name string, ts, pts *types.TipSet, imsgs []*lens.MessageExecution, results chan *TaskResult) { + ctx, span := otel.Tracer("").Start(ctx, fmt.Sprintf("TipSetIndexer.MessageExecutionProcessor.%s", name)) + defer span.End() + ctx, _ = tag.New(ctx, tag.Upsert(metrics.TaskType, name)) stats.Record(ctx, metrics.TipsetHeight.M(int64(ts.Height()))) stop := metrics.Timer(ctx, metrics.ProcessingDuration) diff --git a/chain/walker.go b/chain/walker.go index abd1ade8b..8343d416e 100644 --- a/chain/walker.go +++ b/chain/walker.go @@ -2,12 +2,10 @@ package chain import ( "context" - "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" "golang.org/x/xerrors" "github.com/filecoin-project/lily/lens" @@ -72,11 +70,20 @@ func (c *Walker) Done() <-chan struct{} { } func (c *Walker) WalkChain(ctx context.Context, node lens.API, ts *types.TipSet) error { - ctx, span := otel.Tracer("").Start(ctx, "Walker.WalkChain", trace.WithAttributes(attribute.Int64("height", c.maxHeight))) + ctx, span := otel.Tracer("").Start(ctx, "Walker.WalkChain") + if span.IsRecording() { + span.SetAttributes( + attribute.Int64("height", int64(ts.Height())), + attribute.String("tipset", ts.String()), + attribute.Int64("min_height", c.minHeight), + attribute.Int64("max_height", c.maxHeight), + ) + } defer span.End() log.Debugw("found tipset", "height", ts.Height()) if err := c.obs.TipSet(ctx, ts); err != nil { + span.RecordError(err) return xerrors.Errorf("notify tipset: %w", err) } @@ -90,6 +97,7 @@ func (c *Walker) WalkChain(ctx context.Context, node lens.API, ts *types.TipSet) ts, err = node.ChainGetTipSet(ctx, ts.Parents()) if err != nil { + span.RecordError(err) return xerrors.Errorf("get tipset: %w", err) } @@ -99,6 +107,7 @@ func (c *Walker) WalkChain(ctx context.Context, node lens.API, ts *types.TipSet) log.Debugw("found tipset", "height", ts.Height()) if err := c.obs.TipSet(ctx, ts); err != nil { + span.RecordError(err) return xerrors.Errorf("notify tipset: %w", err) } diff --git a/chain/watcher.go b/chain/watcher.go index 85bb3f377..0ff6e744f 100644 --- a/chain/watcher.go +++ b/chain/watcher.go @@ -3,9 +3,10 @@ package chain import ( "context" "errors" - "github.com/filecoin-project/lotus/chain/types" "github.com/gammazero/workerpool" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" "golang.org/x/xerrors" "github.com/filecoin-project/lily/metrics" @@ -115,6 +116,14 @@ func (c *Watcher) index(ctx context.Context, he *HeadEvent) error { // maybeIndexTipSet is called when a new tipset has been discovered func (c *Watcher) maybeIndexTipSet(ctx context.Context, ts *types.TipSet) error { + ctx, span := otel.Tracer("").Start(ctx, "Watcher.maybeIndexTipSet") + if span.IsRecording() { + span.SetAttributes( + attribute.String("tipset", ts.String()), + attribute.Int64("height", int64(ts.Height())), + ) + } + defer span.End() // Process the tipset if we can, otherwise skip it so we don't block if indexing is too slow select { case <-ctx.Done(): diff --git a/commands/daemon.go b/commands/daemon.go index a833e8c8b..d99833be9 100644 --- a/commands/daemon.go +++ b/commands/daemon.go @@ -19,8 +19,6 @@ import ( "github.com/mitchellh/go-homedir" "github.com/multiformats/go-multiaddr" "github.com/urfave/cli/v2" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/trace" "golang.org/x/xerrors" "github.com/filecoin-project/lily/commands/util" @@ -178,14 +176,8 @@ Note that jobs are not persisted between restarts of the daemon. See return xerrors.Errorf("setup metrics: %w", err) } - if VisorTracingFlags.Tracing { - tp, err := NewJaegerTraceProvider(VisorTracingFlags) - if err != nil { - return xerrors.Errorf("setup tracing: %w", err) - } - otel.SetTracerProvider(tp) - } else { - otel.SetTracerProvider(trace.NewNoopTracerProvider()) + if err := setupTracing(VisorTracingFlags); err != nil { + return xerrors.Errorf("setup tracing: %w", err) } ctx := context.Background() diff --git a/commands/setup.go b/commands/setup.go index 46ad77a19..9a0f19505 100644 --- a/commands/setup.go +++ b/commands/setup.go @@ -1,7 +1,9 @@ package commands import ( - "fmt" + octrace "go.opencensus.io/trace" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/bridge/opencensus" "net/http" "net/http/pprof" "strings" @@ -16,10 +18,6 @@ import ( "github.com/prometheus/client_golang/prometheus/collectors" "go.opencensus.io/stats/view" "go.opencensus.io/zpages" - "go.opentelemetry.io/otel/exporters/trace/jaeger" - "go.opentelemetry.io/otel/sdk/resource" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - "go.opentelemetry.io/otel/semconv" "golang.org/x/xerrors" "github.com/filecoin-project/lily/metrics" @@ -36,11 +34,9 @@ type VisorLogOpts struct { var VisorLogFlags VisorLogOpts type VisorTracingOpts struct { - Tracing bool - JaegerHost string - JaegerPort int - JaegerName string - JaegerSampleType string + Enabled bool + ServiceName string + ProviderURL string JaegerSamplerParam float64 } @@ -52,36 +48,6 @@ type VisorMetricOpts struct { var VisorMetricFlags VisorMetricOpts -func NewJaegerTraceProvider(flags VisorTracingOpts) (*sdktrace.TracerProvider, error) { - serviceName := flags.JaegerName - sampleRatio := flags.JaegerSamplerParam - agentEndpoint := fmt.Sprintf("http://%s:%d/api/traces", flags.JaegerHost, flags.JaegerPort) - - log.Infow("creating jaeger trace provider", "name", serviceName, "ratio", sampleRatio, "endpoint", agentEndpoint) - var sampler sdktrace.Sampler - if sampleRatio < 1 && sampleRatio > 0 { - sampler = sdktrace.ParentBased(sdktrace.TraceIDRatioBased(sampleRatio)) - } else if sampleRatio == 1 { - sampler = sdktrace.AlwaysSample() - } else { - sampler = sdktrace.NeverSample() - } - - exp, err := jaeger.NewRawExporter(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(agentEndpoint))) - if err != nil { - return nil, err - } - tp := sdktrace.NewTracerProvider( - // Always be sure to batch in production. - sdktrace.WithBatcher(exp), - sdktrace.WithSampler(sampler), - sdktrace.WithResource(resource.NewWithAttributes( - semconv.ServiceNameKey.String(serviceName), - )), - ) - return tp, nil -} - func setupLogging(flags VisorLogOpts) error { ll := flags.LogLevel if err := logging.SetLogLevel("*", ll); err != nil { @@ -161,3 +127,20 @@ func setupMetrics(flags VisorMetricOpts) error { }() return nil } + +func setupTracing(flags VisorTracingOpts) error { + if !flags.Enabled { + return nil + } + + tp, err := metrics.NewJaegerTraceProvider(VisorTracingFlags.ServiceName, VisorTracingFlags.ProviderURL, VisorTracingFlags.JaegerSamplerParam) + if err != nil { + return xerrors.Errorf("setup tracing: %w", err) + } + otel.SetTracerProvider(tp) + // upgrades libraries (lotus) that use OpenCensus to OpenTelemetry to facilitate a migration. + tracer := tp.Tracer(VisorTracingFlags.ServiceName) + octrace.DefaultTracer = opencensus.NewTracer(tracer) + + return nil +} diff --git a/docker-compose.yml b/docker-compose.yml index 24c940f06..269b45286 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,7 @@ services: - "6831:6831/udp" - "5778:5778" - "16686:16686" + - "14268:14268" prometheus: image: prom/prometheus:v2.1.0 diff --git a/go.mod b/go.mod index cb24d6d89..14fc85890 100644 --- a/go.mod +++ b/go.mod @@ -52,10 +52,10 @@ require ( github.com/urfave/cli/v2 v2.3.0 github.com/whyrusleeping/cbor-gen v0.0.0-20210713220151-be142a5ae1a8 go.opencensus.io v0.23.0 - go.opentelemetry.io/otel v0.20.0 - go.opentelemetry.io/otel/exporters/trace/jaeger v0.20.0 - go.opentelemetry.io/otel/sdk v0.20.0 - go.opentelemetry.io/otel/trace v0.20.0 + go.opentelemetry.io/otel v1.2.0 + go.opentelemetry.io/otel/bridge/opencensus v0.25.0 + go.opentelemetry.io/otel/exporters/jaeger v1.2.0 + go.opentelemetry.io/otel/sdk v1.2.0 go.uber.org/fx v1.15.0 go.uber.org/zap v1.19.0 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 diff --git a/go.sum b/go.sum index 70f848e24..2dc1e5add 100644 --- a/go.sum +++ b/go.sum @@ -106,8 +106,9 @@ github.com/beevik/ntp v0.2.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NR github.com/benbjohnson/clock v1.0.1/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.0.2/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.2.0 h1:9Re3G2TWxkE06LdMWMpcY6KV81GLXMGiYpPYUPkFAws= +github.com/benbjohnson/clock v1.2.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -508,8 +509,9 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -1816,21 +1818,33 @@ go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.6-0.20201102222123-380f4078db9f/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/otel v0.11.0/go.mod h1:G8UCk+KooF2HLkgo8RHX9epABH/aRGYET7gQOqBVdB0= -go.opentelemetry.io/otel v0.20.0 h1:eaP0Fqu7SXHwvjiqDq83zImeehOHX8doTvU9AwXON8g= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel/exporters/trace/jaeger v0.20.0 h1:FoclOadJNul1vUiKnZU0sKFWOZtZQq3jUzSbrX2jwNM= -go.opentelemetry.io/otel/exporters/trace/jaeger v0.20.0/go.mod h1:10qwvAmKpvwRO5lL3KQ8EWznPp89uGfhcbK152LFWsQ= -go.opentelemetry.io/otel/metric v0.20.0 h1:4kzhXFP+btKm4jwxpjIqjs41A7MakRFUS86bqLHTIw8= +go.opentelemetry.io/otel v1.2.0 h1:YOQDvxO1FayUcT9MIhJhgMyNO1WqoduiyvQHzGN0kUQ= +go.opentelemetry.io/otel v1.2.0/go.mod h1:aT17Fk0Z1Nor9e0uisf98LrntPGMnk4frBO9+dkf69I= +go.opentelemetry.io/otel/bridge/opencensus v0.25.0 h1:18Ww8TpCEGes12HZJzB2nEbUglvMLzPxqgZypsrKiNc= +go.opentelemetry.io/otel/bridge/opencensus v0.25.0/go.mod h1:dkZDdaNwLlIutxK2Kc2m3jwW2M1ISaNf8/rOYVwuVHs= +go.opentelemetry.io/otel/exporters/jaeger v1.2.0 h1:C/5Egj3MJBXRJi22cSl07suqPqtZLnLFmH//OxETUEc= +go.opentelemetry.io/otel/exporters/jaeger v1.2.0/go.mod h1:KJLFbEMKTNPIfOxcg/WikIozEoKcPgJRz3Ce1vLlM8E= +go.opentelemetry.io/otel/internal/metric v0.25.0 h1:w/7RXe16WdPylaIXDgcYM6t/q0K5lXgSdZOEbIEyliE= +go.opentelemetry.io/otel/internal/metric v0.25.0/go.mod h1:Nhuw26QSX7d6n4duoqAFi5KOQR4AuzyMcl5eXOgwxtc= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/oteltest v0.20.0 h1:HiITxCawalo5vQzdHfKeZurV8x7ljcqAgiWzF6Vaeaw= +go.opentelemetry.io/otel/metric v0.25.0 h1:7cXOnCADUsR3+EOqxPaSKwhEuNu0gz/56dRN1hpIdKw= +go.opentelemetry.io/otel/metric v0.25.0/go.mod h1:E884FSpQfnJOMMUaq+05IWlJ4rjZpk2s/F1Ju+TEEm8= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= -go.opentelemetry.io/otel/sdk v0.20.0 h1:JsxtGXd06J8jrnya7fdI/U/MR6yXA5DtbZy+qoHQlr8= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/trace v0.20.0 h1:1DL6EXUdcg95gukhuRRvLDO/4X5THh/5dIV52lqtnbw= +go.opentelemetry.io/otel/sdk v1.2.0 h1:wKN260u4DesJYhyjxDa7LRFkuhH7ncEVKU37LWcyNIo= +go.opentelemetry.io/otel/sdk v1.2.0/go.mod h1:jNN8QtpvbsKhgaC6V5lHiejMoKD+V8uadoSafgHPx1U= +go.opentelemetry.io/otel/sdk/export/metric v0.25.0 h1:6UjAFmVB5Fza3K5qUJpYWGrk8QMPIqlSnya5FI46VBY= +go.opentelemetry.io/otel/sdk/export/metric v0.25.0/go.mod h1:Ej7NOa+WpN49EIcr1HMUYRvxXXCCnQCg2+ovdt2z8Pk= +go.opentelemetry.io/otel/sdk/metric v0.25.0 h1:J+Ta+4IAA5W9AdWhGQLfciEpavBqqSkBzTDeYvJLFNU= +go.opentelemetry.io/otel/sdk/metric v0.25.0/go.mod h1:G4xzj4LvC6xDDSsVXpvRVclQCbofGGg4ZU2VKKtDRfg= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/otel/trace v1.2.0 h1:Ys3iqbqZhcf28hHzrm5WAquMkDHNZTUkw7KHbuNjej0= +go.opentelemetry.io/otel/trace v1.2.0/go.mod h1:N5FLswTubnxKxOJHM7XZC074qpeEdLy3CgAVsdMucK0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -2134,6 +2148,7 @@ golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426080607-c94f62235c83/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210511113859-b0526f3d8744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/lens/util/repo.go b/lens/util/repo.go index 6f3890932..8ef515382 100644 --- a/lens/util/repo.go +++ b/lens/util/repo.go @@ -3,6 +3,7 @@ package util import ( "bytes" "context" + "go.opentelemetry.io/otel" "strings" "github.com/filecoin-project/go-address" @@ -29,6 +30,8 @@ var log = logging.Logger("lily/lens") // No attempt at deduplication of messages is made. A list of blocks with their corresponding messages is also returned - it contains all messages // in the block regardless if they were applied during the state change. func GetExecutedAndBlockMessagesForTipset(ctx context.Context, cs *store.ChainStore, next, current *types.TipSet) (*lens.TipSetMessages, error) { + ctx, span := otel.Tracer("").Start(ctx, "GetExecutedAndBlockMessagesForTipSet") + defer span.End() if !types.CidArrsEqual(next.Parents().Cids(), current.Cids()) { return nil, xerrors.Errorf("child tipset (%s) is not on the same chain as parent (%s)", next.Key(), current.Key()) } @@ -53,14 +56,16 @@ func GetExecutedAndBlockMessagesForTipset(ctx context.Context, cs *store.ChainSt for _, c := range secpkcids { messageBlocks[c] = append(messageBlocks[c], current.Cids()[blockIdx]) } - } + span.AddEvent("read block message metadata") bmsgs, err := cs.BlockMsgsForTipset(current) if err != nil { return nil, xerrors.Errorf("block messages for tipset: %w", err) } + span.AddEvent("read block messages for tipset") + pblocks := current.Blocks() if len(bmsgs) != len(pblocks) { // logic error somewhere @@ -129,6 +134,7 @@ func GetExecutedAndBlockMessagesForTipset(ctx context.Context, cs *store.ChainSt } } + span.AddEvent("built executed message list") // Retrieve receipts using a block from the child tipset rs, err := adt.AsArray(cs.ActorStore(ctx), next.Blocks()[0].ParentMessageReceipts) @@ -156,6 +162,7 @@ func GetExecutedAndBlockMessagesForTipset(ctx context.Context, cs *store.ChainSt if err != nil { return nil, xerrors.Errorf("load state tree: %w", err) } + span.AddEvent("loaded parent state tree") // Receipts are in same order as BlockMsgsForTipset for _, em := range emsgs { @@ -175,6 +182,8 @@ func GetExecutedAndBlockMessagesForTipset(ctx context.Context, cs *store.ChainSt em.GasOutputs = vm.ComputeGasOutputs(em.Receipt.GasUsed, em.Message.GasLimit, em.BlockHeader.ParentBaseFee, em.Message.GasFeeCap, em.Message.GasPremium, burn) } + span.AddEvent("computed executed message gas usage") + blkMsgs := make([]*lens.BlockMessages, len(next.Blocks())) for idx, blk := range next.Blocks() { msgs, smsgs, err := cs.MessagesForBlock(blk) @@ -188,6 +197,8 @@ func GetExecutedAndBlockMessagesForTipset(ctx context.Context, cs *store.ChainSt } } + span.AddEvent("read messages for next block") + return &lens.TipSetMessages{ Executed: emsgs, Block: blkMsgs, diff --git a/main.go b/main.go index 52216403c..85b398ba1 100644 --- a/main.go +++ b/main.go @@ -91,39 +91,28 @@ func main() { Destination: &commands.VisorLogFlags.LogLevelNamed, }, &cli.BoolFlag{ - Name: "tracing", - EnvVars: []string{"LILY_TRACING"}, + Name: "jaeger-tracing", + EnvVars: []string{"LILY_JAEGER_TRACING"}, Value: false, - Destination: &commands.VisorTracingFlags.Tracing, - }, - &cli.StringFlag{ - Name: "jaeger-agent-host", - EnvVars: []string{"JAEGER_AGENT_HOST"}, - Value: "localhost", - Destination: &commands.VisorTracingFlags.JaegerHost, - }, - &cli.IntFlag{ - Name: "jaeger-agent-port", - EnvVars: []string{"JAEGER_AGENT_PORT"}, - Value: 6831, - Destination: &commands.VisorTracingFlags.JaegerPort, + Destination: &commands.VisorTracingFlags.Enabled, }, &cli.StringFlag{ Name: "jaeger-service-name", - EnvVars: []string{"JAEGER_SERVICE_NAME"}, + EnvVars: []string{"LILY_JAEGER_SERVICE_NAME"}, Value: "lily", - Destination: &commands.VisorTracingFlags.JaegerName, + Destination: &commands.VisorTracingFlags.ServiceName, }, &cli.StringFlag{ - Name: "jaeger-sampler-type", - EnvVars: []string{"JAEGER_SAMPLER_TYPE"}, - Value: "probabilistic", - Destination: &commands.VisorTracingFlags.JaegerSampleType, + Name: "jaeger-provider-url", + EnvVars: []string{"LILY_JAEGER_PROVIDER_URL"}, + Value: "http://localhost:14268/api/traces", + Destination: &commands.VisorTracingFlags.ProviderURL, }, &cli.Float64Flag{ - Name: "jaeger-sampler-param", - EnvVars: []string{"JAEGER_SAMPLER_PARAM"}, - Value: 0.0001, + Name: "jaeger-sampler-ratio", + EnvVars: []string{"LILY_JAEGER_SAMPLER_RATIO"}, + Usage: "If less than 1 probabilistic metrics will be used.", + Value: 1, Destination: &commands.VisorTracingFlags.JaegerSamplerParam, }, &cli.StringFlag{ diff --git a/metrics/jaeger.go b/metrics/jaeger.go new file mode 100644 index 000000000..d480b442f --- /dev/null +++ b/metrics/jaeger.go @@ -0,0 +1,41 @@ +package metrics + +import ( + logging "github.com/ipfs/go-log/v2" + "go.opentelemetry.io/otel/exporters/jaeger" + "go.opentelemetry.io/otel/sdk/resource" + tracesdk "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.7.0" +) + +var log = logging.Logger("metrics") + +// NewJaegerTraceProvider returns a new and configured TracerProvider backed by Jaeger. +func NewJaegerTraceProvider(serviceName, agentEndpoint string, sampleRatio float64) (*tracesdk.TracerProvider, error) { + log.Infow("creating jaeger trace provider", "serviceName", serviceName, "ratio", sampleRatio, "endpoint", agentEndpoint) + var sampler tracesdk.Sampler + if sampleRatio < 1 && sampleRatio > 0 { + sampler = tracesdk.ParentBased(tracesdk.TraceIDRatioBased(sampleRatio)) + } else if sampleRatio == 1 { + sampler = tracesdk.AlwaysSample() + } else { + sampler = tracesdk.NeverSample() + } + + exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(agentEndpoint))) + if err != nil { + return nil, err + } + tp := tracesdk.NewTracerProvider( + // Always be sure to batch in production. + tracesdk.WithBatcher(exp), + // Use the provided sampling ratio. + tracesdk.WithSampler(sampler), + // Record information about this application in an Resource. + tracesdk.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(serviceName), + )), + ) + return tp, nil +} diff --git a/model/actors/common/actors.go b/model/actors/common/actors.go index 314bf3b80..feb395e57 100644 --- a/model/actors/common/actors.go +++ b/model/actors/common/actors.go @@ -3,13 +3,11 @@ package common import ( "context" + "github.com/filecoin-project/lily/metrics" + "github.com/filecoin-project/lily/model" "go.opencensus.io/tag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - - "github.com/filecoin-project/lily/metrics" - "github.com/filecoin-project/lily/model" ) type Actor struct { @@ -43,7 +41,10 @@ func (a *Actor) Persist(ctx context.Context, s model.StorageBatch, version model type ActorList []*Actor func (actors ActorList) Persist(ctx context.Context, s model.StorageBatch, version model.Version) error { - ctx, span := otel.Tracer("").Start(ctx, "ActorList.Persist", trace.WithAttributes(attribute.Int("count", len(actors)))) + ctx, span := otel.Tracer("").Start(ctx, "ActorList.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(actors))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "actors")) @@ -85,7 +86,10 @@ func (as *ActorState) Persist(ctx context.Context, s model.StorageBatch, version type ActorStateList []*ActorState func (states ActorStateList) Persist(ctx context.Context, s model.StorageBatch, version model.Version) error { - ctx, span := otel.Tracer("").Start(ctx, "ActorStateList.Persist", trace.WithAttributes(attribute.Int("count", len(states)))) + ctx, span := otel.Tracer("").Start(ctx, "ActorStateList.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(states))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "actor_states")) diff --git a/model/actors/init/idaddress.go b/model/actors/init/idaddress.go index e375f18eb..ed5468130 100644 --- a/model/actors/init/idaddress.go +++ b/model/actors/init/idaddress.go @@ -6,7 +6,6 @@ import ( "go.opencensus.io/tag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" "golang.org/x/xerrors" "github.com/filecoin-project/lily/metrics" @@ -64,7 +63,10 @@ func (ia *IdAddress) Persist(ctx context.Context, s model.StorageBatch, version type IdAddressList []*IdAddress func (ias IdAddressList) Persist(ctx context.Context, s model.StorageBatch, version model.Version) error { - ctx, span := otel.Tracer("").Start(ctx, "IdAddressList.PersistWithTx", trace.WithAttributes(attribute.Int("count", len(ias)))) + ctx, span := otel.Tracer("").Start(ctx, "IdAddressList.PersistWithTx") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(ias))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "id_addresses")) diff --git a/model/actors/market/dealproposal.go b/model/actors/market/dealproposal.go index c98bbe585..7b98122e9 100644 --- a/model/actors/market/dealproposal.go +++ b/model/actors/market/dealproposal.go @@ -2,14 +2,12 @@ package market import ( "context" - - "go.opencensus.io/tag" - "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" "github.com/filecoin-project/lily/metrics" "github.com/filecoin-project/lily/model" + "go.opencensus.io/tag" + "go.opentelemetry.io/otel" ) type MarketDealProposal struct { @@ -46,7 +44,10 @@ func (dp *MarketDealProposal) Persist(ctx context.Context, s model.StorageBatch, type MarketDealProposals []*MarketDealProposal func (dps MarketDealProposals) Persist(ctx context.Context, s model.StorageBatch, version model.Version) error { - ctx, span := otel.Tracer("").Start(ctx, "MarketDealProposals.Persist", trace.WithAttributes(attribute.Int("count", len(dps)))) + ctx, span := otel.Tracer("").Start(ctx, "MarketDealProposals.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(dps))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "market_deal_proposals")) diff --git a/model/actors/market/dealstate.go b/model/actors/market/dealstate.go index 8523d480d..5da903ac2 100644 --- a/model/actors/market/dealstate.go +++ b/model/actors/market/dealstate.go @@ -3,13 +3,11 @@ package market import ( "context" + "github.com/filecoin-project/lily/metrics" + "github.com/filecoin-project/lily/model" "go.opencensus.io/tag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - - "github.com/filecoin-project/lily/metrics" - "github.com/filecoin-project/lily/model" ) type MarketDealState struct { @@ -34,7 +32,10 @@ func (ds *MarketDealState) Persist(ctx context.Context, s model.StorageBatch, ve type MarketDealStates []*MarketDealState func (dss MarketDealStates) Persist(ctx context.Context, s model.StorageBatch, version model.Version) error { - ctx, span := otel.Tracer("").Start(ctx, "MarketDealStates.PersistWithTx", trace.WithAttributes(attribute.Int("count", len(dss)))) + ctx, span := otel.Tracer("").Start(ctx, "MarketDealStates.PersistWithTx") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(dss))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "market_deal_states")) diff --git a/model/actors/miner/precommit.go b/model/actors/miner/precommit.go index 2e931d705..4cf7a46fc 100644 --- a/model/actors/miner/precommit.go +++ b/model/actors/miner/precommit.go @@ -2,11 +2,10 @@ package miner import ( "context" + "go.opentelemetry.io/otel/attribute" "go.opencensus.io/tag" "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" "golang.org/x/xerrors" "github.com/filecoin-project/lily/metrics" @@ -105,7 +104,10 @@ func (mpi *MinerPreCommitInfo) Persist(ctx context.Context, s model.StorageBatch type MinerPreCommitInfoList []*MinerPreCommitInfo func (ml MinerPreCommitInfoList) Persist(ctx context.Context, s model.StorageBatch, version model.Version) error { - ctx, span := otel.Tracer("").Start(ctx, "MinerPreCommitInfoList.Persist", trace.WithAttributes(attribute.Int("count", len(ml)))) + ctx, span := otel.Tracer("").Start(ctx, "MinerPreCommitInfoList.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(ml))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "miner_pre_commit_infos")) diff --git a/model/actors/miner/sector.go b/model/actors/miner/sector.go index cbe23021c..559c5707e 100644 --- a/model/actors/miner/sector.go +++ b/model/actors/miner/sector.go @@ -6,7 +6,6 @@ import ( "go.opencensus.io/tag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" "golang.org/x/xerrors" "github.com/filecoin-project/lily/metrics" @@ -100,7 +99,10 @@ type ( ) func (ml MinerSectorInfoList) Persist(ctx context.Context, s model.StorageBatch, version model.Version) error { - ctx, span := otel.Tracer("").Start(ctx, "MinerSectorInfoList.Persist", trace.WithAttributes(attribute.Int("count", len(ml)))) + ctx, span := otel.Tracer("").Start(ctx, "MinerSectorInfoList.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(ml))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "miner_sector_infos")) diff --git a/model/actors/miner/sectordeals.go b/model/actors/miner/sectordeals.go index e8298d0cd..a70797a31 100644 --- a/model/actors/miner/sectordeals.go +++ b/model/actors/miner/sectordeals.go @@ -3,13 +3,11 @@ package miner import ( "context" + "github.com/filecoin-project/lily/metrics" + "github.com/filecoin-project/lily/model" "go.opencensus.io/tag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - - "github.com/filecoin-project/lily/metrics" - "github.com/filecoin-project/lily/model" ) type MinerSectorDeal struct { @@ -31,7 +29,10 @@ func (ds *MinerSectorDeal) Persist(ctx context.Context, s model.StorageBatch, ve type MinerSectorDealList []*MinerSectorDeal func (ml MinerSectorDealList) Persist(ctx context.Context, s model.StorageBatch, version model.Version) error { - ctx, span := otel.Tracer("").Start(ctx, "MinerSectorDealList.Persist", trace.WithAttributes(attribute.Int("count", len(ml)))) + ctx, span := otel.Tracer("").Start(ctx, "MinerSectorDealList.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(ml))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "miner_sector_deals")) diff --git a/model/actors/miner/sectorevents.go b/model/actors/miner/sectorevents.go index 442121049..237e1ec1d 100644 --- a/model/actors/miner/sectorevents.go +++ b/model/actors/miner/sectorevents.go @@ -3,13 +3,11 @@ package miner import ( "context" + "github.com/filecoin-project/lily/metrics" + "github.com/filecoin-project/lily/model" "go.opencensus.io/tag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - - "github.com/filecoin-project/lily/metrics" - "github.com/filecoin-project/lily/model" ) const ( @@ -52,7 +50,10 @@ func (mse *MinerSectorEvent) Persist(ctx context.Context, s model.StorageBatch, type MinerSectorEventList []*MinerSectorEvent func (l MinerSectorEventList) Persist(ctx context.Context, s model.StorageBatch, version model.Version) error { - ctx, span := otel.Tracer("").Start(ctx, "MinerSectorEventList.Persist", trace.WithAttributes(attribute.Int("count", len(l)))) + ctx, span := otel.Tracer("").Start(ctx, "MinerSectorEventList.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(l))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "miner_sector_events")) diff --git a/model/actors/miner/sectorposts.go b/model/actors/miner/sectorposts.go index 19ff3587f..278f82a0d 100644 --- a/model/actors/miner/sectorposts.go +++ b/model/actors/miner/sectorposts.go @@ -3,13 +3,11 @@ package miner import ( "context" + "github.com/filecoin-project/lily/metrics" + "github.com/filecoin-project/lily/model" "go.opencensus.io/tag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - - "github.com/filecoin-project/lily/metrics" - "github.com/filecoin-project/lily/model" ) type MinerSectorPost struct { @@ -32,7 +30,10 @@ func (msp *MinerSectorPost) Persist(ctx context.Context, s model.StorageBatch, v } func (ml MinerSectorPostList) Persist(ctx context.Context, s model.StorageBatch, version model.Version) error { - ctx, span := otel.Tracer("").Start(ctx, "MinerSectorPostList.Persist", trace.WithAttributes(attribute.Int("count", len(ml)))) + ctx, span := otel.Tracer("").Start(ctx, "MinerSectorPostList.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(ml))) + } defer span.End() if len(ml) == 0 { return nil diff --git a/model/actors/miner/task.go b/model/actors/miner/task.go index e24a0ab79..4ff43c903 100644 --- a/model/actors/miner/task.go +++ b/model/actors/miner/task.go @@ -6,7 +6,6 @@ import ( "github.com/filecoin-project/lily/model" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" ) type MinerTaskResult struct { @@ -74,7 +73,10 @@ func (res *MinerTaskResult) Persist(ctx context.Context, s model.StorageBatch, v type MinerTaskResultList []*MinerTaskResult func (ml MinerTaskResultList) Persist(ctx context.Context, s model.StorageBatch, version model.Version) error { - ctx, span := otel.Tracer("").Start(ctx, "MinerTaskResultList.Persist", trace.WithAttributes(attribute.Int("count", len(ml)))) + ctx, span := otel.Tracer("").Start(ctx, "MinerTaskResultList.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(ml))) + } defer span.End() for _, res := range ml { diff --git a/model/actors/power/chainpower.go b/model/actors/power/chainpower.go index b8d0afea8..d929669c6 100644 --- a/model/actors/power/chainpower.go +++ b/model/actors/power/chainpower.go @@ -6,7 +6,6 @@ import ( "go.opencensus.io/tag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" "golang.org/x/xerrors" "github.com/filecoin-project/lily/metrics" @@ -103,7 +102,10 @@ type ChainPowerList []*ChainPower // PersistWithTx makes a batch insertion of the list using the given // transaction. func (cpl ChainPowerList) Persist(ctx context.Context, s model.StorageBatch, version model.Version) error { - ctx, span := otel.Tracer("").Start(ctx, "ChainPowerList.PersistWithTx", trace.WithAttributes(attribute.Int("count", len(cpl)))) + ctx, span := otel.Tracer("").Start(ctx, "ChainPowerList.PersistWithTx") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(cpl))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "chain_powers")) diff --git a/model/blocks/drand.go b/model/blocks/drand.go index 37858cf71..4a45bc213 100644 --- a/model/blocks/drand.go +++ b/model/blocks/drand.go @@ -2,15 +2,13 @@ package blocks import ( "context" - - "github.com/filecoin-project/lotus/chain/types" - "go.opencensus.io/tag" - "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" "github.com/filecoin-project/lily/metrics" "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lotus/chain/types" + "go.opencensus.io/tag" + "go.opentelemetry.io/otel" ) func NewDrandBlockEntries(header *types.BlockHeader) DrandBlockEntries { @@ -44,7 +42,10 @@ func (dbes DrandBlockEntries) Persist(ctx context.Context, s model.StorageBatch, if len(dbes) == 0 { return nil } - ctx, span := otel.Tracer("").Start(ctx, "DrandBlockEntries.Persist", trace.WithAttributes(attribute.Int("count", len(dbes)))) + ctx, span := otel.Tracer("").Start(ctx, "DrandBlockEntries.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(dbes))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "drand_block_entries")) diff --git a/model/blocks/header.go b/model/blocks/header.go index 3e462e28f..f8d5aec96 100644 --- a/model/blocks/header.go +++ b/model/blocks/header.go @@ -3,14 +3,12 @@ package blocks import ( "context" + "github.com/filecoin-project/lily/metrics" + "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lotus/chain/types" "go.opencensus.io/tag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - - "github.com/filecoin-project/lily/metrics" - "github.com/filecoin-project/lily/model" ) type BlockHeader struct { @@ -55,7 +53,10 @@ func (bhl BlockHeaders) Persist(ctx context.Context, s model.StorageBatch, versi if len(bhl) == 0 { return nil } - ctx, span := otel.Tracer("").Start(ctx, "BlockHeaders.Persist", trace.WithAttributes(attribute.Int("count", len(bhl)))) + ctx, span := otel.Tracer("").Start(ctx, "BlockHeaders.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(bhl))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "block_headers")) diff --git a/model/blocks/parent.go b/model/blocks/parent.go index 58084a810..eb156303f 100644 --- a/model/blocks/parent.go +++ b/model/blocks/parent.go @@ -3,14 +3,12 @@ package blocks import ( "context" + "github.com/filecoin-project/lily/metrics" + "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lotus/chain/types" "go.opencensus.io/tag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - - "github.com/filecoin-project/lily/metrics" - "github.com/filecoin-project/lily/model" ) type BlockParent struct { @@ -46,7 +44,10 @@ func (bps BlockParents) Persist(ctx context.Context, s model.StorageBatch, versi if len(bps) == 0 { return nil } - ctx, span := otel.Tracer("").Start(ctx, "BlockParents.Persist", trace.WithAttributes(attribute.Int("count", len(bps)))) + ctx, span := otel.Tracer("").Start(ctx, "BlockParents.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(bps))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "block_parents")) diff --git a/model/chain/consensus.go b/model/chain/consensus.go index b6b4b1193..143b784c8 100644 --- a/model/chain/consensus.go +++ b/model/chain/consensus.go @@ -2,13 +2,11 @@ package chain import ( "context" - "github.com/filecoin-project/lily/metrics" "github.com/filecoin-project/lily/model" "go.opencensus.io/tag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" ) type ChainConsensus struct { @@ -33,7 +31,10 @@ func (c ChainConsensus) Persist(ctx context.Context, s model.StorageBatch, versi type ChainConsensusList []*ChainConsensus func (c ChainConsensusList) Persist(ctx context.Context, s model.StorageBatch, version model.Version) error { - ctx, span := otel.Tracer("").Start(ctx, "ChainConsensusList.Persist", trace.WithAttributes(attribute.Int("count", len(c)))) + ctx, span := otel.Tracer("").Start(ctx, "ChainConsensusList.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(c))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "chain_consensus")) diff --git a/model/chain/economics.go b/model/chain/economics.go index a1f279df3..a3f5744e6 100644 --- a/model/chain/economics.go +++ b/model/chain/economics.go @@ -6,7 +6,6 @@ import ( "go.opencensus.io/tag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" "golang.org/x/xerrors" "github.com/filecoin-project/lily/metrics" @@ -79,7 +78,10 @@ func (l ChainEconomicsList) Persist(ctx context.Context, s model.StorageBatch, v if len(l) == 0 { return nil } - ctx, span := otel.Tracer("").Start(ctx, "ChainEconomicsList.Persist", trace.WithAttributes(attribute.Int("count", len(l)))) + ctx, span := otel.Tracer("").Start(ctx, "ChainEconomicsList.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(l))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "chain_economics")) diff --git a/model/derived/gasoutputs.go b/model/derived/gasoutputs.go index f91d8de0e..a743ed2f7 100644 --- a/model/derived/gasoutputs.go +++ b/model/derived/gasoutputs.go @@ -6,7 +6,6 @@ import ( "go.opencensus.io/tag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" "golang.org/x/xerrors" "github.com/filecoin-project/lily/metrics" @@ -129,7 +128,10 @@ func (l GasOutputsList) Persist(ctx context.Context, s model.StorageBatch, versi if len(l) == 0 { return nil } - ctx, span := otel.Tracer("").Start(ctx, "GasOutputsList.Persist", trace.WithAttributes(attribute.Int("count", len(l)))) + ctx, span := otel.Tracer("").Start(ctx, "GasOutputsList.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(l))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "derived_gas_outputs")) diff --git a/model/messages/blockmessage.go b/model/messages/blockmessage.go index 854330b35..fc05b8132 100644 --- a/model/messages/blockmessage.go +++ b/model/messages/blockmessage.go @@ -3,13 +3,11 @@ package messages import ( "context" + "github.com/filecoin-project/lily/metrics" + "github.com/filecoin-project/lily/model" "go.opencensus.io/tag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - - "github.com/filecoin-project/lily/metrics" - "github.com/filecoin-project/lily/model" ) type BlockMessage struct { @@ -33,7 +31,10 @@ func (bms BlockMessages) Persist(ctx context.Context, s model.StorageBatch, vers if len(bms) == 0 { return nil } - ctx, span := otel.Tracer("").Start(ctx, "BlockMessages.Persist", trace.WithAttributes(attribute.Int("count", len(bms)))) + ctx, span := otel.Tracer("").Start(ctx, "BlockMessages.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(bms))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "block_messages")) diff --git a/model/messages/internal.go b/model/messages/internal.go index c2e76aa94..428ab8105 100644 --- a/model/messages/internal.go +++ b/model/messages/internal.go @@ -3,13 +3,11 @@ package messages import ( "context" + "github.com/filecoin-project/lily/metrics" + "github.com/filecoin-project/lily/model" "go.opencensus.io/tag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - - "github.com/filecoin-project/lily/metrics" - "github.com/filecoin-project/lily/model" ) type InternalMessage struct { @@ -44,7 +42,10 @@ func (l InternalMessageList) Persist(ctx context.Context, s model.StorageBatch, if len(l) == 0 { return nil } - ctx, span := otel.Tracer("").Start(ctx, "InternalMessageList.Persist", trace.WithAttributes(attribute.Int("count", len(l)))) + ctx, span := otel.Tracer("").Start(ctx, "InternalMessageList.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(l))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "internal_messages")) @@ -82,7 +83,10 @@ func (l InternalParsedMessageList) Persist(ctx context.Context, s model.StorageB if len(l) == 0 { return nil } - ctx, span := otel.Tracer("").Start(ctx, "InternalParsedMessageList.Persist", trace.WithAttributes(attribute.Int("count", len(l)))) + ctx, span := otel.Tracer("").Start(ctx, "InternalParsedMessageList.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(l))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "internal_parsed_messages")) diff --git a/model/messages/message.go b/model/messages/message.go index 8c805194e..6abee671d 100644 --- a/model/messages/message.go +++ b/model/messages/message.go @@ -2,11 +2,9 @@ package messages import ( "context" - "go.opencensus.io/tag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" "golang.org/x/xerrors" "github.com/filecoin-project/lily/metrics" @@ -94,7 +92,10 @@ func (ms Messages) Persist(ctx context.Context, s model.StorageBatch, version mo if len(ms) == 0 { return nil } - ctx, span := otel.Tracer("").Start(ctx, "Messages.Persist", trace.WithAttributes(attribute.Int("count", len(ms)))) + ctx, span := otel.Tracer("").Start(ctx, "Messages.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(ms))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "messages")) diff --git a/model/messages/parsedmessage.go b/model/messages/parsedmessage.go index b78add691..6554219d0 100644 --- a/model/messages/parsedmessage.go +++ b/model/messages/parsedmessage.go @@ -6,7 +6,6 @@ import ( "go.opencensus.io/tag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" "golang.org/x/xerrors" "github.com/filecoin-project/lily/metrics" @@ -78,7 +77,10 @@ func (pms ParsedMessages) Persist(ctx context.Context, s model.StorageBatch, ver if len(pms) == 0 { return nil } - ctx, span := otel.Tracer("").Start(ctx, "ParsedMessages.Persist", trace.WithAttributes(attribute.Int("count", len(pms)))) + ctx, span := otel.Tracer("").Start(ctx, "ParsedMessages.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(pms))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "parsed_messages")) diff --git a/model/messages/receipt.go b/model/messages/receipt.go index c69493c8d..63725949d 100644 --- a/model/messages/receipt.go +++ b/model/messages/receipt.go @@ -3,13 +3,11 @@ package messages import ( "context" + "github.com/filecoin-project/lily/metrics" + "github.com/filecoin-project/lily/model" "go.opencensus.io/tag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - - "github.com/filecoin-project/lily/metrics" - "github.com/filecoin-project/lily/model" ) type Receipt struct { @@ -37,7 +35,10 @@ func (rs Receipts) Persist(ctx context.Context, s model.StorageBatch, version mo if len(rs) == 0 { return nil } - ctx, span := otel.Tracer("").Start(ctx, "Receipts.Persist", trace.WithAttributes(attribute.Int("count", len(rs)))) + ctx, span := otel.Tracer("").Start(ctx, "Receipts.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(rs))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "receipts")) diff --git a/model/surveyed/peeragent.go b/model/surveyed/peeragent.go index bb344b686..be67c9e20 100644 --- a/model/surveyed/peeragent.go +++ b/model/surveyed/peeragent.go @@ -2,15 +2,13 @@ package observed import ( "context" - "time" - - "go.opencensus.io/tag" - "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" + "time" "github.com/filecoin-project/lily/metrics" "github.com/filecoin-project/lily/model" + "go.opencensus.io/tag" + "go.opentelemetry.io/otel" ) type PeerAgent struct { @@ -47,7 +45,10 @@ func (l PeerAgentList) Persist(ctx context.Context, s model.StorageBatch, versio if len(l) == 0 { return nil } - ctx, span := otel.Tracer("").Start(ctx, "PeerAgentList.Persist", trace.WithAttributes(attribute.Int("count", len(l)))) + ctx, span := otel.Tracer("").Start(ctx, "PeerAgentList.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(l))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "surveyed_peer_agents")) diff --git a/model/visor/gapreport.go b/model/visor/gapreport.go index 5280fcea8..fdfb325c2 100644 --- a/model/visor/gapreport.go +++ b/model/visor/gapreport.go @@ -9,7 +9,6 @@ import ( "go.opencensus.io/tag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" ) type GapReport struct { @@ -40,7 +39,10 @@ func (pl GapReportList) Persist(ctx context.Context, s model.StorageBatch, versi if len(pl) == 0 { return nil } - ctx, span := otel.Tracer("").Start(ctx, "GapReportList.Persist", trace.WithAttributes(attribute.Int("count", len(pl)))) + ctx, span := otel.Tracer("").Start(ctx, "GapReportList.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(pl))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "visor_gap_reports")) diff --git a/model/visor/processingreport.go b/model/visor/processingreport.go index bffe3c5ef..ba2ae6b30 100644 --- a/model/visor/processingreport.go +++ b/model/visor/processingreport.go @@ -4,13 +4,11 @@ import ( "context" "time" + "github.com/filecoin-project/lily/metrics" + "github.com/filecoin-project/lily/model" "go.opencensus.io/tag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - - "github.com/filecoin-project/lily/metrics" - "github.com/filecoin-project/lily/model" ) const ( @@ -62,7 +60,10 @@ func (pl ProcessingReportList) Persist(ctx context.Context, s model.StorageBatch if len(pl) == 0 { return nil } - ctx, span := otel.Tracer("").Start(ctx, "ProcessingReportList.Persist", trace.WithAttributes(attribute.Int("count", len(pl)))) + ctx, span := otel.Tracer("").Start(ctx, "ProcessingReportList.Persist") + if span.IsRecording() { + span.SetAttributes(attribute.Int("count", len(pl))) + } defer span.End() ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "visor_processing_reports")) diff --git a/tasks/actorstate/task.go b/tasks/actorstate/task.go index ccbb170db..5fa3b79aa 100644 --- a/tasks/actorstate/task.go +++ b/tasks/actorstate/task.go @@ -9,8 +9,6 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" "go.opencensus.io/tag" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" "golang.org/x/xerrors" "github.com/filecoin-project/lily/chain/actors/builtin" @@ -36,11 +34,6 @@ func NewTask(node lens.API, extracterMap ActorExtractorMap) *Task { } func (t *Task) ProcessActors(ctx context.Context, ts *types.TipSet, pts *types.TipSet, candidates map[string]lens.ActorStateChange, emsgs []*lens.ExecutedMessage) (model.Persistable, *visormodel.ProcessingReport, error) { - ctx, span := otel.Tracer("").Start(ctx, "ProcessActors") - if span.IsRecording() { - span.SetAttributes(attribute.String("tipset", ts.String()), attribute.String("parent_tipset", pts.String()), attribute.Int64("height", int64(ts.Height()))) - } - defer span.End() log.Debugw("processing actor state changes", "height", ts.Height(), "parent_height", pts.Height()) report := &visormodel.ProcessingReport{