Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature: multi-org detection #1

Draft
wants to merge 7 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 25 additions & 1 deletion api/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,25 @@ import (
"encoding/json"
"fmt"
"net/http"
"strconv"
"strings"
)

var ErrBadStatusCode = fmt.Errorf("bad status code")

type orgCtxKeyType string

const orgCtxKey orgCtxKeyType = "orgID"

func NewOrgContext(ctx context.Context, orgID int) context.Context {
return context.WithValue(ctx, orgCtxKey, orgID)
}

func OrgFromContext(ctx context.Context) (int, bool) {
org, ok := ctx.Value(orgCtxKey).(int)
return org, ok
}

type Client struct {
BaseURL string

Expand Down Expand Up @@ -75,9 +89,15 @@ func (cl Client) newRequest(ctx context.Context, method, url string) (*http.Requ
// API let's use it first
if cl.token != "" {
req.Header.Add("Authorization", "Bearer "+cl.token)
} else if cl.basicAuthUser != "" && cl.basicAuthPassword != "" {
} else if cl.UsingBasicAuth() {
req.SetBasicAuth(cl.basicAuthUser, cl.basicAuthPassword)
}

orgID, ok := OrgFromContext(ctx)
if ok {
req.Header.Add("X-Grafana-Org-Id", strconv.Itoa(orgID))
}

return req, err
}

Expand All @@ -101,3 +121,7 @@ func (cl Client) Request(ctx context.Context, method, url string, out interface{
}
return nil
}

func (cl Client) UsingBasicAuth() bool {
return cl.basicAuthUser != "" && cl.basicAuthPassword != ""
}
6 changes: 6 additions & 0 deletions api/grafana/grafana.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,12 @@ func (cl APIClient) GetOrgs(ctx context.Context) ([]Org, error) {
return orgs, err
}

func (cl APIClient) GetCurrentOrg(ctx context.Context) (Org, error) {
var org Org
err := cl.Request(ctx, http.MethodGet, "org", &org)
return org, err
}

func (cl APIClient) UserSwitchContext(ctx context.Context, orgID string) error {
return cl.Request(ctx, http.MethodPost, "user/using/"+orgID, nil)
}
Expand Down
132 changes: 73 additions & 59 deletions detector/detector.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"strings"
"sync"

"github.com/grafana/detect-angular-dashboards/api"
"github.com/grafana/detect-angular-dashboards/api/gcom"
"github.com/grafana/detect-angular-dashboards/api/grafana"
"github.com/grafana/detect-angular-dashboards/logger"
Expand Down Expand Up @@ -54,7 +55,7 @@ func NewDetector(log *logger.LeveledLogger, grafanaClient GrafanaDetectorAPIClie
}

// Run runs the angular detector tool against the specified Grafana instance.
func (d *Detector) Run(ctx context.Context) ([]output.Dashboard, error) {
func (d *Detector) Run(ctx context.Context, orgs ...grafana.Org) ([]output.Dashboard, error) {
var (
finalOutput []output.Dashboard
// Determine if we should use GCOM or frontendsettings
Expand Down Expand Up @@ -142,72 +143,85 @@ func (d *Detector) Run(ctx context.Context) ([]output.Dashboard, error) {
d.log.Verbose().Log("Plugin %q angular %t", p, isAngular)
}

// Map ds name -> ds plugin id, to resolve legacy dashboards that have ds name
apiDs, err := d.grafanaClient.GetDatasourcePluginIDs(ctx)
if err != nil {
return []output.Dashboard{}, fmt.Errorf("get datasource plugin ids: %w", err)
}
d.datasourcePluginIDs = make(map[string]string, len(apiDs))
for _, ds := range apiDs {
d.datasourcePluginIDs[ds.Name] = ds.Type
}
// Org specific checks
for _, org := range orgs {
// orgID in context is used by request from the GrafanaClient
ctx = api.NewOrgContext(ctx, org.ID)
d.log.Verbose().Log("Running detection for organization %q (%d)", org.Name, org.ID)

dashboards, err := d.grafanaClient.GetDashboards(ctx, 1)
if err != nil {
return []output.Dashboard{}, fmt.Errorf("get dashboards: %w", err)
}
// Map ds name -> ds plugin id, to resolve legacy dashboards that have ds name
apiDs, err := d.grafanaClient.GetDatasourcePluginIDs(ctx)
if err != nil {
return []output.Dashboard{}, fmt.Errorf("get datasource plugin ids: %w", err)
}
d.datasourcePluginIDs = make(map[string]string, len(apiDs))
for _, ds := range apiDs {
d.datasourcePluginIDs[ds.Name] = ds.Type
}

// Create a semaphore to limit concurrency
semaphore := make(chan struct{}, d.maxConcurrency)
var wg sync.WaitGroup
var mu sync.Mutex
var downloadErrors []error
dashboards, err := d.grafanaClient.GetDashboards(ctx, 1)
if err != nil {
return []output.Dashboard{}, fmt.Errorf("get dashboards: %w", err)
}

for _, dash := range dashboards {
wg.Add(1)
go func(dash grafana.ListedDashboard) {
defer wg.Done()
semaphore <- struct{}{} // Acquire semaphore
defer func() { <-semaphore }() // Release semaphore
orgID, ok := api.OrgFromContext(ctx)
if !ok {
return []output.Dashboard{}, fmt.Errorf("could not get org ID from context")
}

dashboardAbsURL, err := url.JoinPath(strings.TrimSuffix(d.grafanaClient.BaseURL(), "/api"), dash.URL)
if err != nil {
dashboardAbsURL = ""
}
dashboardDefinition, err := d.grafanaClient.GetDashboard(ctx, dash.UID)
if err != nil {
mu.Lock()
downloadErrors = append(downloadErrors, fmt.Errorf("get dashboard %q: %w", dash.UID, err))
mu.Unlock()
return
}
dashboardOutput := output.Dashboard{
Detections: []output.Detection{},
URL: dashboardAbsURL,
Title: dash.Title,
Folder: dashboardDefinition.Meta.FolderTitle,
CreatedBy: dashboardDefinition.Meta.CreatedBy,
UpdatedBy: dashboardDefinition.Meta.UpdatedBy,
Created: dashboardDefinition.Meta.Created,
Updated: dashboardDefinition.Meta.Updated,
}
dashboardOutput.Detections, err = d.checkPanels(dashboardDefinition, dashboardDefinition.Dashboard.Panels)
if err != nil {
// Create a semaphore to limit concurrency
semaphore := make(chan struct{}, d.maxConcurrency)
var wg sync.WaitGroup
var mu sync.Mutex
var downloadErrors []error

for _, dash := range dashboards {
wg.Add(1)
go func(dash grafana.ListedDashboard) {
defer wg.Done()
semaphore <- struct{}{} // Acquire semaphore
defer func() { <-semaphore }() // Release semaphore

dashboardAbsURL, err := url.JoinPath(strings.TrimSuffix(d.grafanaClient.BaseURL(), "/api"), dash.URL)
if err != nil {
dashboardAbsURL = ""
}
dashboardDefinition, err := d.grafanaClient.GetDashboard(ctx, dash.UID)
if err != nil {
mu.Lock()
downloadErrors = append(downloadErrors, fmt.Errorf("get dashboard %q: %w", dash.UID, err))
mu.Unlock()
return
}
dashboardOutput := output.Dashboard{
Detections: []output.Detection{},
URL: dashboardAbsURL,
Title: dash.Title,
Folder: dashboardDefinition.Meta.FolderTitle,
CreatedBy: dashboardDefinition.Meta.CreatedBy,
UpdatedBy: dashboardDefinition.Meta.UpdatedBy,
Created: dashboardDefinition.Meta.Created,
Updated: dashboardDefinition.Meta.Updated,
OrgID: orgID,
}
dashboardOutput.Detections, err = d.checkPanels(dashboardDefinition, dashboardDefinition.Dashboard.Panels)
if err != nil {
mu.Lock()
downloadErrors = append(downloadErrors, fmt.Errorf("check panels: %w", err))
mu.Unlock()
return
}
mu.Lock()
downloadErrors = append(downloadErrors, fmt.Errorf("check panels: %w", err))
finalOutput = append(finalOutput, dashboardOutput)
mu.Unlock()
return
}
mu.Lock()
finalOutput = append(finalOutput, dashboardOutput)
mu.Unlock()
}(dash)
}
}(dash)
}

wg.Wait()
wg.Wait()

if len(downloadErrors) > 0 {
return finalOutput, fmt.Errorf("errors occurred during dashboard download: %v", downloadErrors)
if len(downloadErrors) > 0 {
return finalOutput, fmt.Errorf("errors occurred during dashboard download: %v", downloadErrors)
}
}

return finalOutput, nil
Expand Down
2 changes: 2 additions & 0 deletions flags/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ type Flags struct {
Server string
Interval time.Duration
MaxConcurrency int
AllOrgs bool
}

// Parse parses the command-line flags.
Expand All @@ -26,6 +27,7 @@ func Parse() Flags {
flag.DurationVar(&flags.Interval, "interval", 5*time.Minute, "detection refresh interval when running in HTTP server mode")
flag.StringVar(&flags.Server, "server", "", "Run as HTTP server instead of CLI. Value must be a listen address (e.g.: 0.0.0.0:5000. Output is exposed as JSON at /detections.")
flag.IntVar(&flags.MaxConcurrency, "max-concurrency", 10, "maximum number of concurrent dashboard downloads")
flag.BoolVar(&flags.AllOrgs, "all-orgs", false, "run detection for all organizations")
flag.Parse()

return flags
Expand Down
31 changes: 28 additions & 3 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,28 @@ func main() {
}
client := initializeClient(token, &f)

if !client.UsingBasicAuth() && f.AllOrgs {
log.Errorf("all-orgs flag can only be used with basic authentication")
os.Exit(1)
}

// orgs to run detection on
var orgs []grafana.Org
if f.AllOrgs {
orgs, err = client.GetOrgs(context.Background())
if err != nil {
log.Errorf("Failed to get orgs: %s\n", err)
os.Exit(1)
}
} else {
currentOrg, err := client.GetCurrentOrg(context.Background())
if err != nil {
log.Errorf("Failed to get current org: %s\n", err)
os.Exit(1)
}
orgs = append(orgs, currentOrg)
}

d := detector.NewDetector(log, client, gcom.NewAPIClient(), f.MaxConcurrency)

if f.Server != "" {
Expand All @@ -57,7 +79,7 @@ func main() {
return
}

if err := runCLIMode(&f, log, d); err != nil {
if err := runCLIMode(&f, log, d, orgs); err != nil {
log.Errorf("%s\n", err)
os.Exit(1)
}
Expand Down Expand Up @@ -154,21 +176,24 @@ func runServer(flags *flags.Flags, log *logger.LeveledLogger) error {
}

// runCLIMode runs the program in CLI mode.
func runCLIMode(flags *flags.Flags, log *logger.LeveledLogger, d *detector.Detector) error {
func runCLIMode(flags *flags.Flags, log *logger.LeveledLogger, d *detector.Detector, orgs []grafana.Org) error {
log.Log("Detecting Angular dashboards")
var out output.Outputter
if flags.JSONOutput {
out = output.NewJSONOutputter(os.Stdout)
} else {
out = output.NewLoggerReadableOutput(log)
}
data, err := d.Run(context.Background())

ctx := context.Background()
data, err := d.Run(ctx, orgs...)
if err != nil {
return fmt.Errorf("run detector: %w", err)
}
if err := out.Output(data); err != nil {
return fmt.Errorf("output: %w", err)
}

return nil
}

Expand Down
1 change: 1 addition & 0 deletions output/output.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ type Dashboard struct {
CreatedBy string
Created string
Updated string
OrgID int
}

type Outputter interface {
Expand Down