From 7bd8d0cadb67390be392f38e823d2c9580695507 Mon Sep 17 00:00:00 2001 From: Frank van Lankvelt Date: Fri, 26 Apr 2024 16:18:25 +0200 Subject: [PATCH] STAC-21209: export relations as well --- exporter/ststopologyexporter/exporter.go | 65 +++++++++++++---- exporter/ststopologyexporter/exporter_test.go | 15 ++++ .../ststopologyexporter/internal/topology.go | 73 +++++++++++++++++-- .../ststopologyexporter/internal/types.go | 16 +++- 4 files changed, 149 insertions(+), 20 deletions(-) diff --git a/exporter/ststopologyexporter/exporter.go b/exporter/ststopologyexporter/exporter.go index 9311ed3..a907419 100644 --- a/exporter/ststopologyexporter/exporter.go +++ b/exporter/ststopologyexporter/exporter.go @@ -38,6 +38,16 @@ func newTopologyExporter(logger *zap.Logger, cfg component.Config) (*topologyExp return &topologyExporter{logger: logger, httpClient: httpClient, cfg: stsCfg}, nil } + +func getOrDefault(componentsByApiKey map[string]*internal.ComponentsCollection, sts_api_key string) *internal.ComponentsCollection { + collection, has_siblings := componentsByApiKey[sts_api_key] + if !has_siblings { + collection = internal.NewCollection() + componentsByApiKey[sts_api_key] = collection + } + return collection +} + func (t *topologyExporter) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) error { log := t.logger @@ -46,21 +56,48 @@ func (t *topologyExporter) ConsumeMetrics(ctx context.Context, md pmetric.Metric for i := 0; i < rms.Len(); i++ { rs := rms.At(i) resource := rs.Resource() - sts_api_key_value, key_exists := resource.Attributes().Get("sts_api_key") - if !key_exists { - log.Warn("No sts_api_key attribute found on resource") - continue - } - sts_api_key := sts_api_key_value.AsString() attrs := resource.Attributes() - attrs.Remove("sts_api_key") - collection, has_siblings := componentsByApiKey[sts_api_key] - if !has_siblings { - collection = internal.NewCollection() - componentsByApiKey[sts_api_key] = collection - } - if !collection.AddResource(&attrs) { - log.Warn("Skipping resource without necessary attributes") + sts_api_key_value, key_exists := attrs.Get("sts_api_key") + if key_exists { + sts_api_key := sts_api_key_value.AsString() + attrs.Remove("sts_api_key") + collection := getOrDefault(componentsByApiKey, sts_api_key) + if !collection.AddResource(&attrs) { + log.Warn("Skipping resource without necessary attributes") + } + } else { + // look for servicegraph metrics for relations + ilms := rs.ScopeMetrics() + for j := 0; j < ilms.Len(); j++ { + ilm := ilms.At(j) + scope := ilm.Scope() + if scope.Name() != "traces_service_graph" { + continue + } + + metrics := ilm.Metrics() + for k := 0; k < metrics.Len(); k++ { + m := metrics.At(k) + if m.Name() != "traces_service_graph_request_total" { + continue + } + connAttrs := m.Sum().DataPoints().At(0).Attributes() + sts_api_key_value, key_exists := connAttrs.Get("client_sts_api_key") + if !key_exists { + log.Error("Configuration error - no sts_api_key available on servicegraph metric") + return errInternal + } + sts_api_key := sts_api_key_value.AsString() + connAttrs.Remove("client_sts_api_key") + connAttrs.Remove("server_sts_api_key") + collection := getOrDefault(componentsByApiKey, sts_api_key) + if !collection.AddConnection(&connAttrs) { + log.Warn("Unable to add connection from servicegraphconnector") + } + } + break + } + } } diff --git a/exporter/ststopologyexporter/exporter_test.go b/exporter/ststopologyexporter/exporter_test.go index 06e2799..fcb482b 100644 --- a/exporter/ststopologyexporter/exporter_test.go +++ b/exporter/ststopologyexporter/exporter_test.go @@ -24,6 +24,7 @@ func TestExporter_pushResourcesData(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, len(payload.Topologies)) require.Equal(t, 3, len(payload.Topologies[0].Components)) + require.Equal(t, 1, len(payload.Topologies[0].Relations)) res.WriteHeader(200) })) exporter := newTestExporter(t, testServer.URL) @@ -45,6 +46,20 @@ func simpleMetrics() pmetric.Metrics { rm.Resource().Attributes().PutStr("service.name", "demo 2") rm.Resource().Attributes().PutStr("service.namespace", "demo") rm.Resource().Attributes().PutStr("Resource Attributes 2", "value2") + + rm = metrics.ResourceMetrics().AppendEmpty() + sc := rm.ScopeMetrics().AppendEmpty() + sc.Scope().SetName("traces_service_graph") + ms := sc.Metrics().AppendEmpty() + ms.SetName("traces_service_graph_request_total") + ms.SetEmptySum().SetIsMonotonic(true) + ma := ms.Sum().DataPoints().AppendEmpty().Attributes() + ma.PutStr("client_sts_api_key", "APIKEY") + ma.PutStr("client", "client") + ma.PutStr("client_service.namespace", "clientns") + ma.PutStr("server", "server") + ma.PutStr("server_service.namespace", "serverns") + ma.PutStr("connection_type", "unknown") return metrics } diff --git a/exporter/ststopologyexporter/internal/topology.go b/exporter/ststopologyexporter/internal/topology.go index c406db0..2da2284 100644 --- a/exporter/ststopologyexporter/internal/topology.go +++ b/exporter/ststopologyexporter/internal/topology.go @@ -10,6 +10,7 @@ type ComponentsCollection struct { namespaces map[string]*Component services []*Component serviceInstances []*Component + relations map[string]*Relation } func NewCollection() *ComponentsCollection { @@ -17,6 +18,7 @@ func NewCollection() *ComponentsCollection { make(map[string]*Component, 0), make([]*Component, 0), make([]*Component, 0), + make(map[string]*Relation, 0), } } @@ -43,7 +45,7 @@ func (c *ComponentsCollection) AddResource(attrs *pcommon.Map) bool { ComponentType{ "namespace", }, - newData(). + newComponentData(). withLayer("urn:stackpack:common:layer:applications"). withEnvironment(attrs). withName(attrs, "service.namespace"), @@ -54,7 +56,7 @@ func (c *ComponentsCollection) AddResource(attrs *pcommon.Map) bool { ComponentType{ "service", }, - newData(). + newComponentData(). withLayer("urn:stackpack:common:layer:services"). withEnvironment(attrs). withName(attrs, "service.name"). @@ -66,7 +68,7 @@ func (c *ComponentsCollection) AddResource(attrs *pcommon.Map) bool { ComponentType{ "service_instance", }, - newData(). + newComponentData(). withLayer("urn:stackpack:common:layer:containers"). withEnvironment(attrs). withName(attrs, "service.instance.id"). @@ -77,6 +79,52 @@ func (c *ComponentsCollection) AddResource(attrs *pcommon.Map) bool { return true } +func (c *ComponentsCollection) AddConnection(attrs *pcommon.Map) bool { + reqAttrs := make(map[string]string, 4) + for _, key := range []string{ + "client", + "client_service.namespace", + "server", + "server_service.namespace", + "connection_type", + } { + value, ok := attrs.Get(key) + if !ok { + return false + } + reqAttrs[key] = value.AsString() + } + + instanceId, ok := attrs.Get("client_service.instance.id") + var clientInstanceId string + if !ok { + clientInstanceId = reqAttrs["client"] + } else { + clientInstanceId = instanceId.AsString() + } + sourceId := fmt.Sprintf("urn:opentelemetry:namespace/%s:service/%s:serviceInstance/%s", reqAttrs["client_service.namespace"], reqAttrs["client"], clientInstanceId) + + instanceId, ok = attrs.Get("server_service.instance.id") + var serverInstanceId string + if !ok { + serverInstanceId = reqAttrs["server"] + } else { + serverInstanceId = instanceId.AsString() + } + targetId := fmt.Sprintf("urn:opentelemetry:namespace/%s:service/%s:serviceInstance/%s", reqAttrs["server_service.namespace"], reqAttrs["server"], serverInstanceId) + + relationId := fmt.Sprintf("%s-%s", sourceId, targetId) + c.relations[relationId] = &Relation{ + ExternalId: fmt.Sprintf("%s-%s", sourceId, targetId), + SourceId: sourceId, + TargetId: targetId, + Type: RelationType{ + Name: reqAttrs["connection_type"], + }, + } + return true +} + func (c *ComponentsCollection) GetComponents() []*Component { namespaces := make([]*Component, 0, len(c.namespaces)) for _, namespace := range c.namespaces { @@ -92,10 +140,14 @@ func (c *ComponentsCollection) GetComponents() []*Component { } func (c *ComponentsCollection) GetRelations() []*Relation { - return make([]*Relation, 0) + relations := make([]*Relation, 0, len(c.relations)) + for _, relation := range c.relations { + relations = append(relations, relation) + } + return relations } -func newData() *ComponentData { +func newComponentData() *ComponentData { return &ComponentData{ Name: "", Version: "", @@ -153,3 +205,14 @@ func (c *ComponentData) withTags(attrs *pcommon.Map) *ComponentData { }) return c } + +func newRelationData() *ComponentData { + return &ComponentData{ + Name: "", + Version: "", + Layer: "", + Domain: "", + Environment: "", + Tags: map[string]string{}, + } +} diff --git a/exporter/ststopologyexporter/internal/types.go b/exporter/ststopologyexporter/internal/types.go index f4937ad..b64b687 100644 --- a/exporter/ststopologyexporter/internal/types.go +++ b/exporter/ststopologyexporter/internal/types.go @@ -24,7 +24,21 @@ type Component struct { Data *ComponentData `json:"data"` } -type Relation struct{} +type RelationType struct { + Name string `json:"name"` +} + +type RelationData struct { + Tags map[string]string `json:"tags"` +} + +type Relation struct { + ExternalId string `json:"externalId"` + SourceId string `json:"sourceId"` + TargetId string `json:"targetId"` + Type RelationType `json:"type"` + Data *RelationData `json:"data"` +} type Topology struct { Instance Instance `json:"instance"`