Add env substitution

Signed-off-by: Saswata Mukherjee <saswataminsta@yahoo.com>

main.go

@@ -127,7 +127,7 @@ This directive runs executable with arguments and put its stderr and stdout outp
 		"Absolute path links will be converted to relative links to anchor dir as well.").Regexp()
 	// TODO(bwplotka): Add cache in file?
 	linksValidateEnabled := cmd.Flag("links.validate", "If true, all links will be validated").Short('l').Bool()
-	linksValidateConfig := extflag.RegisterPathOrContent(cmd, "links.validate.config", "YAML file for skipping link check, with spec defined in github.com/bwplotka/mdox/pkg/linktransformer.Config", false)
+	linksValidateConfig := extflag.RegisterPathOrContent(cmd, "links.validate.config", "YAML file for skipping link check, with spec defined in github.com/bwplotka/mdox/pkg/linktransformer.Config", false, true)
 
 	cmd.Run(func(ctx context.Context, logger log.Logger) (err error) {
 		var opts []mdformatter.Option

pkg/extflag/pathorcontent.go

@@ -10,6 +10,9 @@ package extflag
 import (
 	"fmt"
 	"io/ioutil"
+	"os"
+	"regexp"
+	"strings"
 
 	"github.com/pkg/errors"
 	"gopkg.in/alecthomas/kingpin.v2"
@@ -19,7 +22,8 @@ import (
 type PathOrContent struct {
 	flagName string
 
-	required bool
+	envSubstitution bool
+	required        bool
 
 	path    *string
 	content *string
@@ -30,7 +34,7 @@ type FlagClause interface {
 }
 
 // RegisterPathOrContent registers PathOrContent flag in kingpinCmdClause.
-func RegisterPathOrContent(cmd FlagClause, flagName string, help string, required bool) *PathOrContent {
+func RegisterPathOrContent(cmd FlagClause, flagName string, help string, required bool, envSubstitution bool) *PathOrContent {
 	fileFlagName := fmt.Sprintf("%s-file", flagName)
 	contentFlagName := flagName
 
@@ -41,10 +45,11 @@ func RegisterPathOrContent(cmd FlagClause, flagName string, help string, require
 	contentFlag := cmd.Flag(contentFlagName, contentHelp).PlaceHolder("<content>").String()
 
 	return &PathOrContent{
-		flagName: flagName,
-		required: required,
-		path:     fileFlag,
-		content:  contentFlag,
+		flagName:        flagName,
+		required:        required,
+		path:            fileFlag,
+		content:         contentFlag,
+		envSubstitution: envSubstitution,
 	}
 }
 
@@ -74,6 +79,25 @@ func (p *PathOrContent) Content() ([]byte, error) {
 	if len(content) == 0 && p.required {
 		return nil, errors.Errorf("flag %s or %s is required for running this command and content cannot be empty.", fileFlagName, p.flagName)
 	}
-
+	if p.envSubstitution {
+		content = substituteEnvVars(string(content))
+	}
 	return content, nil
 }
+
+// substituteEnvVars returns content of YAML file with substituted environment variables.
+// Will be substituted with empty string if env var isn't set.
+// Follows K8s convention, i.e $(...), as mentioned here https://kubernetes.io/docs/tasks/inject-data-application/define-interdependent-environment-variables/.
+func substituteEnvVars(content string) []byte {
+	var replaceWithEnv []string
+	// Match env variable syntax.
+	envVarName := regexp.MustCompile(`\$\((?P<var>[a-zA-Z_]+[a-zA-Z0-9_]*)\)`)
+	loc := envVarName.FindAllStringSubmatchIndex(content, -1)
+	for i := range loc {
+		// Add pair to be replaced.
+		replaceWithEnv = append(replaceWithEnv, content[loc[i][0]:loc[i][1]], os.Getenv(content[loc[i][2]:loc[i][3]]))
+	}
+	replacer := strings.NewReplacer(replaceWithEnv...)
+	contentWithEnv := replacer.Replace(content)
+	return []byte(contentWithEnv)
+}

pkg/mdformatter/linktransformer/config.go

@@ -28,6 +28,8 @@ type Validator struct {
 	Regex string `yaml:"regex"`
 	// By default type is `roundtrip`. Could be `github`.
 	Type ValidatorType `yaml:"type"`
+	// GitHub repo token to avoid getting rate limited.
+	Token string `yaml:"token"`
 }
 
 type ValidatorType string

pkg/mdformatter/linktransformer/validator.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"math"
 	"net/http"
-	"os"
 	"regexp"
 	"strconv"
 	"strings"
@@ -64,7 +63,7 @@ func (v Config) validateGH() error {
 			v.Validators[i]._maxNum = math.MaxInt64
 			continue
 		}
-		regex, maxNum, err := getGitHubRegex(v.Validators[i].Regex)
+		regex, maxNum, err := getGitHubRegex(v.Validators[i].Regex, v.Validators[i].Token)
 		if err != nil {
 			return err
 		}
@@ -75,7 +74,7 @@ func (v Config) validateGH() error {
 }
 
 // getGitHubRegex returns GitHub pulls/issues regex from repo name.
-func getGitHubRegex(repoRe string) (*regexp.Regexp, int, error) {
+func getGitHubRegex(repoRe string, repoToken string) (*regexp.Regexp, int, error) {
 	// Get reponame from regex.
 	getRepo := regexp.MustCompile(`(?P<org>[A-Za-z0-9_.-]+)\\\/(?P<repo>[A-Za-z0-9_.-]+)`)
 	match := getRepo.FindStringSubmatch(repoRe)
@@ -89,7 +88,6 @@ func getGitHubRegex(repoRe string) (*regexp.Regexp, int, error) {
 	max := 0
 	// All GitHub API reqs need to have User-Agent: https://docs.github.com/en/rest/overview/resources-in-the-rest-api#user-agent-required.
 	client := &http.Client{}
-	repoToken := os.Getenv("GITHUB_TOKEN")
 
 	// Check latest pull request number.
 	reqPull, err := http.NewRequest("GET", fmt.Sprintf(gitHubAPIURL, reponame, "pulls"), nil)