adding support for vault import from json/yaml formats

cli/internal/commands/commands.go

@@ -39,6 +39,7 @@ var (
 	vaultShareCmd  *cobra.Command
 	vaultInfoCmd   *cobra.Command
 	vaultPutCmd    *cobra.Command
+	vaultImportCmd *cobra.Command
 	vaultGetCmd    *cobra.Command
 	vaultExportCmd *cobra.Command
 	vaultRefCmd    *cobra.Command

cli/internal/commands/flags.go

@@ -125,8 +125,6 @@ var (
 		usage: "Specify a name for the K8s SLV object if the vault is to be used in a K8s environment",
 	}
 
-	// Secret Command Flags
-
 	secretNameFlag = FlagDef{
 		name:      "name",
 		shorthand: "n",
@@ -138,31 +136,41 @@ var (
 		usage: "Secret to be added to the vault",
 	}
 
+	vaultImportFileFlag = FlagDef{
+		name:  "file",
+		usage: "Path to the YAML/JSON file to be imported",
+	}
+
 	secretForceUpdateFlag = FlagDef{
 		name:  "force",
 		usage: "Replaces the secret if it exists already",
 	}
 
-	secretListFormatFlag = FlagDef{
+	vaultExportFormatFlag = FlagDef{
 		name:  "format",
-		usage: "List secrets as one of [json, yaml, table, envar]. Defaults to envar",
+		usage: "List secrets as one of [json, yaml, envar]. Defaults to envar",
 	}
 
 	secretEncodeBase64Flag = FlagDef{
 		name:  "base64",
 		usage: "Encode the returned secret as base64",
 	}
 
-	secretRefFileFlag = FlagDef{
-		name:  "ref-file",
+	vaultRefFileFlag = FlagDef{
+		name:  "file",
 		usage: "Path to the YAML/JSON file to be referenced",
 	}
 
-	secretRefTypeFlag = FlagDef{
-		name:  "ref-format",
+	vaultRefTypeFlag = FlagDef{
+		name:  "format",
 		usage: "Data serialization format of the referenced file",
 	}
 
+	vaultDerefPathFlag = FlagDef{
+		name:  "path",
+		usage: "Path to a file/directory to dereference secrets",
+	}
+
 	secretRefPreviewOnlyFlag = FlagDef{
 		name:  "preview",
 		usage: "Preview only mode",

cli/internal/commands/vault.go

@@ -70,6 +70,7 @@ func vaultCommand() *cobra.Command {
 	vaultCmd.AddCommand(vaultShareCommand())
 	vaultCmd.AddCommand(vaultInfoCommand())
 	vaultCmd.AddCommand(vaultPutCommand())
+	vaultCmd.AddCommand(vaultImportCommand())
 	vaultCmd.AddCommand(vaultGetCommand())
 	vaultCmd.AddCommand(vaultExportCommand())
 	vaultCmd.AddCommand(vaultRefCommand())
@@ -344,6 +345,43 @@ func vaultPutCommand() *cobra.Command {
 	return vaultPutCmd
 }
 
+func vaultImportCommand() *cobra.Command {
+	if vaultImportCmd != nil {
+		return vaultImportCmd
+	}
+	vaultImportCmd = &cobra.Command{
+		Use:     "import",
+		Aliases: []string{"load", "put-all", "add-all", "set-all", "create-all"},
+		Short:   "Imports secrets into the vault from YAML or JSON",
+		Run: func(cmd *cobra.Command, args []string) {
+			vaultFile := cmd.Flag(vaultFileFlag.name).Value.String()
+			vault, err := getVault(vaultFile)
+			if err != nil {
+				exitOnError(err)
+			}
+			forceUpdate, _ := cmd.Flags().GetBool(secretForceUpdateFlag.name)
+			importFile := cmd.Flag(vaultImportFileFlag.name).Value.String()
+			var importData []byte
+			if importFile == "" {
+				importData, err = input.GetHiddenInput("Enter the YAML/JSON data to be imported: ")
+			} else {
+				importData, err = os.ReadFile(importFile)
+			}
+			if err != nil {
+				exitOnError(err)
+			}
+			if err = vault.ImportSecrets(importData, forceUpdate); err != nil {
+				exitOnError(err)
+			}
+			fmt.Printf("Successfully imported secrets from %s into the vault %s\n", color.GreenString(importFile), color.GreenString(vaultFile))
+			safeExit()
+		},
+	}
+	vaultImportCmd.Flags().StringP(vaultImportFileFlag.name, vaultImportFileFlag.shorthand, "", vaultImportFileFlag.usage)
+	vaultImportCmd.Flags().Bool(secretForceUpdateFlag.name, false, secretForceUpdateFlag.usage)
+	return vaultImportCmd
+}
+
 func vaultGetCommand() *cobra.Command {
 	if vaultGetCmd != nil {
 		return vaultGetCmd
@@ -421,17 +459,11 @@ func vaultExportCommand() *cobra.Command {
 					secretOutputMap[name] = string(secret)
 				}
 			}
-			exportFormat := cmd.Flag(secretListFormatFlag.name).Value.String()
+			exportFormat := cmd.Flag(vaultExportFormatFlag.name).Value.String()
 			if exportFormat == "" {
-				exportFormat = "env"
+				exportFormat = "envar"
 			}
 			switch exportFormat {
-			case "table":
-				tw := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0)
-				for key, value := range secretOutputMap {
-					fmt.Fprintf(tw, "%s\t%s\n", key, value)
-				}
-				tw.Flush()
 			case "json":
 				jsonData, err := json.MarshalIndent(secretOutputMap, "", "  ")
 				if err != nil {
@@ -444,7 +476,7 @@ func vaultExportCommand() *cobra.Command {
 					exitOnError(err)
 				}
 				fmt.Println(string(yamlData))
-			case "envars", "envar", "env":
+			case "envars", "envar", ".env":
 				for key, value := range secretOutputMap {
 					value = strings.ReplaceAll(value, "\\", "\\\\")
 					value = strings.ReplaceAll(value, "\"", "\\\"")
@@ -456,7 +488,7 @@ func vaultExportCommand() *cobra.Command {
 			safeExit()
 		},
 	}
-	vaultExportCmd.Flags().StringP(secretListFormatFlag.name, secretListFormatFlag.shorthand, "", secretListFormatFlag.usage)
+	vaultExportCmd.Flags().StringP(vaultExportFormatFlag.name, vaultExportFormatFlag.shorthand, "", vaultExportFormatFlag.usage)
 	vaultExportCmd.Flags().BoolP(secretEncodeBase64Flag.name, secretEncodeBase64Flag.shorthand, false, secretEncodeBase64Flag.usage)
 	return vaultExportCmd
 }
@@ -475,13 +507,13 @@ func vaultRefCommand() *cobra.Command {
 			if err != nil {
 				exitOnError(err)
 			}
-			refFile := cmd.Flag(secretRefFileFlag.name).Value.String()
+			refFile := cmd.Flag(vaultRefFileFlag.name).Value.String()
 			secretNamePrefix := cmd.Flag(secretNameFlag.name).Value.String()
-			refType := strings.ToLower(cmd.Flag(secretRefTypeFlag.name).Value.String())
+			refType := strings.ToLower(cmd.Flag(vaultRefTypeFlag.name).Value.String())
 			previewOnly, _ := cmd.Flags().GetBool(secretRefPreviewOnlyFlag.name)
 			forceUpdate, _ := cmd.Flags().GetBool(secretForceUpdateFlag.name)
 			if secretNamePrefix == "" && refType == "" {
-				exitOnErrorWithMessage("please provide at least one of --" + secretNameFlag.name + " or --" + secretRefTypeFlag.name + " flag")
+				exitOnErrorWithMessage("please provide at least one of --" + secretNameFlag.name + " or --" + vaultRefTypeFlag.name + " flag")
 			}
 			if refType != "" && refType != "yaml" {
 				exitOnErrorWithMessage("only yaml auto reference is supported at the moment")
@@ -500,12 +532,12 @@ func vaultRefCommand() *cobra.Command {
 			safeExit()
 		},
 	}
-	vaultRefCmd.Flags().StringP(secretRefFileFlag.name, secretRefFileFlag.shorthand, "", secretRefFileFlag.usage)
+	vaultRefCmd.Flags().StringP(vaultRefFileFlag.name, vaultRefFileFlag.shorthand, "", vaultRefFileFlag.usage)
 	vaultRefCmd.Flags().StringP(secretNameFlag.name, secretNameFlag.shorthand, "", secretNameFlag.usage)
-	vaultRefCmd.Flags().StringP(secretRefTypeFlag.name, secretRefTypeFlag.shorthand, "", secretRefTypeFlag.usage)
+	vaultRefCmd.Flags().StringP(vaultRefTypeFlag.name, vaultRefTypeFlag.shorthand, "", vaultRefTypeFlag.usage)
 	vaultRefCmd.Flags().BoolP(secretRefPreviewOnlyFlag.name, secretRefPreviewOnlyFlag.shorthand, false, secretRefPreviewOnlyFlag.usage)
 	vaultRefCmd.Flags().BoolP(secretForceUpdateFlag.name, secretForceUpdateFlag.shorthand, false, secretForceUpdateFlag.usage)
-	vaultRefCmd.MarkFlagRequired(secretRefFileFlag.name)
+	vaultRefCmd.MarkFlagRequired(vaultRefFileFlag.name)
 	return vaultRefCmd
 }
 
@@ -525,14 +557,10 @@ func vaultDerefCommand() *cobra.Command {
 			if err != nil {
 				exitOnError(err)
 			}
-			files, err := cmd.Flags().GetStringSlice(secretRefFileFlag.name)
+			paths, err := cmd.Flags().GetStringSlice(vaultDerefPathFlag.name)
 			if err != nil {
 				exitOnError(err)
 			}
-			previewOnly := false
-			if len(vaultFiles) > 1 || len(files) > 1 {
-				previewOnly, _ = cmd.Flags().GetBool(secretRefPreviewOnlyFlag.name)
-			}
 			for _, vaultFile := range vaultFiles {
 				vault, err := getVault(vaultFile)
 				if err != nil {
@@ -542,23 +570,18 @@ func vaultDerefCommand() *cobra.Command {
 				if err != nil {
 					exitOnError(err)
 				}
-				for _, file := range files {
-					result, err := vault.DeRefSecrets(file, previewOnly)
-					if err != nil {
+				for _, path := range paths {
+					if err = vault.DeRefSecrets(path); err != nil {
 						exitOnError(err)
 					}
-					if previewOnly {
-						fmt.Println(result)
-					} else {
-						fmt.Println("Dereferenced ", color.GreenString(file), "with the vault", color.GreenString(vaultFile))
-					}
+					fmt.Println("Dereferenced", color.GreenString(path), "with the vault", color.GreenString(vaultFile))
 				}
 			}
 			safeExit()
 		},
 	}
-	vaultDerefCmd.Flags().StringSliceP(secretRefFileFlag.name, secretRefFileFlag.shorthand, []string{}, secretRefFileFlag.usage)
-	vaultDerefCmd.Flags().BoolP(secretRefPreviewOnlyFlag.name, secretRefPreviewOnlyFlag.shorthand, false, secretRefPreviewOnlyFlag.usage)
-	vaultDerefCmd.MarkFlagRequired(secretRefFileFlag.name)
+	vaultDerefCmd.Flags().StringSliceP(vaultFileFlag.name, vaultFileFlag.shorthand, []string{}, vaultFileFlag.usage)
+	vaultDerefCmd.Flags().StringSliceP(vaultDerefPathFlag.name, vaultDerefPathFlag.shorthand, []string{}, vaultDerefPathFlag.usage)
+	vaultDerefCmd.MarkFlagRequired(vaultDerefPathFlag.name)
 	return vaultDerefCmd
 }

core/vaults/const.go

@@ -33,4 +33,5 @@ var (
 	errVaultSecretNotFound          = errors.New("no secret found for the given name")
 	errVaultPublicKeyNotFound       = errors.New("vault public key not found")
 	errInvalidReferenceFormat       = errors.New("invalid reference format. references must follow the pattern {{SLV_VSR_VAULTID.secretName}} to allow dereferencing")
+	errInvalidImportDataFormat      = errors.New("invalid import data format - expected a map of string to string [secretName: secretValue] in YAML/JSON format")
 )

core/vaults/deref.go

@@ -2,6 +2,7 @@ package vaults
 
 import (
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 )
@@ -39,20 +40,25 @@ func (vlt *Vault) deRefSecretsFromContent(content string) ([]byte, error) {
 	return []byte(content), nil
 }
 
-func (vlt *Vault) DeRefSecrets(file string, previewOnly bool) (dereferncedBytes []byte, err error) {
+func (vlt *Vault) DeRefSecrets(path string) error {
 	if vlt.IsLocked() {
-		return nil, errVaultLocked
+		return errVaultLocked
 	}
-	data, err := os.ReadFile(file)
-	if err != nil {
-		return nil, err
-	}
-	dereferncedBytes, err = vlt.deRefSecretsFromContent(string(data))
-	if err != nil {
-		return nil, err
-	}
-	if !previewOnly {
-		err = os.WriteFile(file, dereferncedBytes, 0644)
-	}
-	return
+	return filepath.WalkDir(path, func(currentPath string, d os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		data, err := os.ReadFile(currentPath)
+		if err != nil {
+			return err
+		}
+		dereferncedBytes, err := vlt.deRefSecretsFromContent(string(data))
+		if err != nil {
+			return err
+		}
+		return os.WriteFile(currentPath, dereferncedBytes, 0644)
+	})
 }

core/vaults/secrets.go → core/vaults/vaultsecrets.go

@@ -1,6 +1,9 @@
 package vaults
 
 import (
+	"fmt"
+
+	"gopkg.in/yaml.v3"
 	"savesecrets.org/slv/core/crypto"
 )
 
@@ -30,6 +33,26 @@ func (vlt *Vault) PutSecret(secretName string, secretValue []byte) (err error) {
 	return
 }
 
+func (vlt *Vault) ImportSecrets(importData []byte, force bool) (err error) {
+	secretsMap := make(map[string]string)
+	if err = yaml.Unmarshal(importData, &secretsMap); err != nil {
+		return errInvalidImportDataFormat
+	}
+	if !force {
+		for secretName := range secretsMap {
+			if vlt.SecretExists(secretName) {
+				return fmt.Errorf("secret %s already exists", secretName)
+			}
+		}
+	}
+	for secretName, secretValue := range secretsMap {
+		if err = vlt.putSecretWithoutCommit(secretName, []byte(secretValue)); err != nil {
+			return err
+		}
+	}
+	return vlt.commit()
+}
+
 func (vlt *Vault) SecretExists(secretName string) (exists bool) {
 	if vlt.Secrets != nil {
 		_, exists = vlt.Secrets[secretName]

go.mod

@@ -4,15 +4,15 @@ go 1.21
 
 require (
 	dev.shib.me/xipher v0.9.1
-	github.com/aws/aws-sdk-go v1.50.30
+	github.com/aws/aws-sdk-go v1.50.31
 	github.com/fatih/color v1.16.0
 	github.com/go-git/go-git/v5 v5.11.0
 	github.com/go-logr/logr v1.4.1
-	github.com/onsi/ginkgo/v2 v2.15.0
+	github.com/onsi/ginkgo/v2 v2.16.0
 	github.com/onsi/gomega v1.31.1
 	github.com/spf13/cobra v1.8.0
-	golang.org/x/crypto v0.20.0
-	golang.org/x/term v0.17.0
+	golang.org/x/crypto v0.21.0
+	golang.org/x/term v0.18.0
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.29.2
 	k8s.io/apimachinery v0.29.2
@@ -21,7 +21,7 @@ require (
 )
 
 require (
-	cloud.google.com/go/compute v1.24.0 // indirect
+	cloud.google.com/go/compute v1.25.0 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/iam v1.1.6 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
@@ -36,10 +36,10 @@ require (
 	go.opentelemetry.io/otel/metric v1.24.0 // indirect
 	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
-	google.golang.org/api v0.167.0 // indirect
-	google.golang.org/genproto v0.0.0-20240228224816-df926f6c8641 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240228224816-df926f6c8641 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240228224816-df926f6c8641 // indirect
+	google.golang.org/api v0.168.0 // indirect
+	google.golang.org/genproto v0.0.0-20240304161311-37d4d3c04a78 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240304161311-37d4d3c04a78 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78 // indirect
 	google.golang.org/grpc v1.62.0 // indirect
 )
 
@@ -60,9 +60,9 @@ require (
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.5.0 // indirect
 	github.com/go-logr/zapr v1.3.0 // indirect
-	github.com/go-openapi/jsonpointer v0.20.2 // indirect
-	github.com/go-openapi/jsonreference v0.20.4 // indirect
-	github.com/go-openapi/swag v0.22.9 // indirect
+	github.com/go-openapi/jsonpointer v0.20.3 // indirect
+	github.com/go-openapi/jsonreference v0.20.5 // indirect
+	github.com/go-openapi/swag v0.22.10 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -98,10 +98,10 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect
-	golang.org/x/mod v0.15.0 // indirect
-	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/mod v0.16.0 // indirect
+	golang.org/x/net v0.22.0 // indirect
 	golang.org/x/oauth2 v0.17.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.18.0 // indirect