From 75360527e5ea62f6b5f04a2cdc7e376385fc1945 Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Wed, 2 Jun 2021 18:39:33 +0530 Subject: [PATCH] Add env substitution Signed-off-by: Saswata Mukherjee --- main.go | 2 +- pkg/extflag/pathorcontent.go | 38 ++++++++++++++++---- pkg/mdformatter/linktransformer/config.go | 2 ++ pkg/mdformatter/linktransformer/validator.go | 6 ++-- 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/main.go b/main.go index 16c8f5a..39e185f 100644 --- a/main.go +++ b/main.go @@ -127,7 +127,7 @@ This directive runs executable with arguments and put its stderr and stdout outp "Absolute path links will be converted to relative links to anchor dir as well.").Regexp() // TODO(bwplotka): Add cache in file? linksValidateEnabled := cmd.Flag("links.validate", "If true, all links will be validated").Short('l').Bool() - linksValidateConfig := extflag.RegisterPathOrContent(cmd, "links.validate.config", "YAML file for skipping link check, with spec defined in github.com/bwplotka/mdox/pkg/linktransformer.Config", false) + linksValidateConfig := extflag.RegisterPathOrContent(cmd, "links.validate.config", "YAML file for skipping link check, with spec defined in github.com/bwplotka/mdox/pkg/linktransformer.Config", false, true) cmd.Run(func(ctx context.Context, logger log.Logger) (err error) { var opts []mdformatter.Option diff --git a/pkg/extflag/pathorcontent.go b/pkg/extflag/pathorcontent.go index 84eb0fd..db73b10 100644 --- a/pkg/extflag/pathorcontent.go +++ b/pkg/extflag/pathorcontent.go @@ -10,6 +10,9 @@ package extflag import ( "fmt" "io/ioutil" + "os" + "regexp" + "strings" "github.com/pkg/errors" "gopkg.in/alecthomas/kingpin.v2" @@ -19,7 +22,8 @@ import ( type PathOrContent struct { flagName string - required bool + envSubstitution bool + required bool path *string content *string @@ -30,7 +34,7 @@ type FlagClause interface { } // RegisterPathOrContent registers PathOrContent flag in kingpinCmdClause. -func RegisterPathOrContent(cmd FlagClause, flagName string, help string, required bool) *PathOrContent { +func RegisterPathOrContent(cmd FlagClause, flagName string, help string, required bool, envSubstitution bool) *PathOrContent { fileFlagName := fmt.Sprintf("%s-file", flagName) contentFlagName := flagName @@ -41,10 +45,11 @@ func RegisterPathOrContent(cmd FlagClause, flagName string, help string, require contentFlag := cmd.Flag(contentFlagName, contentHelp).PlaceHolder("").String() return &PathOrContent{ - flagName: flagName, - required: required, - path: fileFlag, - content: contentFlag, + flagName: flagName, + required: required, + path: fileFlag, + content: contentFlag, + envSubstitution: envSubstitution, } } @@ -74,6 +79,25 @@ func (p *PathOrContent) Content() ([]byte, error) { if len(content) == 0 && p.required { return nil, errors.Errorf("flag %s or %s is required for running this command and content cannot be empty.", fileFlagName, p.flagName) } - + if p.envSubstitution { + content = substituteEnvVars(string(content)) + } return content, nil } + +// substituteEnvVars returns content of YAML file with substituted environment variables. +// Will be substituted with empty string if env var isn't set. +// Follows K8s convention, i.e $(...), as mentioned here https://kubernetes.io/docs/tasks/inject-data-application/define-interdependent-environment-variables/. +func substituteEnvVars(content string) []byte { + var replaceWithEnv []string + // Match env variable syntax. + envVarName := regexp.MustCompile(`\$\((?P[a-zA-Z_]+[a-zA-Z0-9_]*)\)`) + loc := envVarName.FindAllStringSubmatchIndex(content, -1) + for i := range loc { + // Add pair to be replaced. + replaceWithEnv = append(replaceWithEnv, content[loc[i][0]:loc[i][1]], os.Getenv(content[loc[i][2]:loc[i][3]])) + } + replacer := strings.NewReplacer(replaceWithEnv...) + contentWithEnv := replacer.Replace(content) + return []byte(contentWithEnv) +} diff --git a/pkg/mdformatter/linktransformer/config.go b/pkg/mdformatter/linktransformer/config.go index 725c644..debacfd 100644 --- a/pkg/mdformatter/linktransformer/config.go +++ b/pkg/mdformatter/linktransformer/config.go @@ -28,6 +28,8 @@ type Validator struct { Regex string `yaml:"regex"` // By default type is `roundtrip`. Could be `github`. Type ValidatorType `yaml:"type"` + // GitHub repo token to avoid getting rate limited. + Token string `yaml:"token"` } type ValidatorType string diff --git a/pkg/mdformatter/linktransformer/validator.go b/pkg/mdformatter/linktransformer/validator.go index d726532..afab67d 100644 --- a/pkg/mdformatter/linktransformer/validator.go +++ b/pkg/mdformatter/linktransformer/validator.go @@ -8,7 +8,6 @@ import ( "fmt" "math" "net/http" - "os" "regexp" "strconv" "strings" @@ -64,7 +63,7 @@ func (v Config) validateGH() error { v.Validators[i]._maxNum = math.MaxInt64 continue } - regex, maxNum, err := getGitHubRegex(v.Validators[i].Regex) + regex, maxNum, err := getGitHubRegex(v.Validators[i].Regex, v.Validators[i].Token) if err != nil { return err } @@ -75,7 +74,7 @@ func (v Config) validateGH() error { } // getGitHubRegex returns GitHub pulls/issues regex from repo name. -func getGitHubRegex(repoRe string) (*regexp.Regexp, int, error) { +func getGitHubRegex(repoRe string, repoToken string) (*regexp.Regexp, int, error) { // Get reponame from regex. getRepo := regexp.MustCompile(`(?P[A-Za-z0-9_.-]+)\\\/(?P[A-Za-z0-9_.-]+)`) match := getRepo.FindStringSubmatch(repoRe) @@ -89,7 +88,6 @@ func getGitHubRegex(repoRe string) (*regexp.Regexp, int, error) { max := 0 // All GitHub API reqs need to have User-Agent: https://docs.github.com/en/rest/overview/resources-in-the-rest-api#user-agent-required. client := &http.Client{} - repoToken := os.Getenv("GITHUB_TOKEN") // Check latest pull request number. reqPull, err := http.NewRequest("GET", fmt.Sprintf(gitHubAPIURL, reponame, "pulls"), nil)