actually update branched image tag in all jobs, not just parent config (

#5592)

* actually update to branched image tags in all jobs, not just parent config

* filter duplicate env vars

* log images needing retagging if running with --skip-gar-tagging

* update OS function calls deprecated since Go 1.16

---------

Co-authored-by: Mike Morris <1149913+mikemorris@users.noreply.github.com>

tools/prowgen/cmd/prowgen/main.go

@@ -15,9 +15,9 @@
 package main
 
 import (
+	"errors"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -28,6 +28,7 @@ import (
 
 	"github.com/hashicorp/go-multierror"
 	shell "github.com/kballard/go-shellquote"
+	v1 "k8s.io/api/core/v1"
 	k8sProwConfig "sigs.k8s.io/prow/pkg/config"
 	"sigs.k8s.io/yaml"
 
@@ -80,7 +81,9 @@ func main() {
 			}
 			cli := pkg.Client{BaseConfig: baseConfig, LongJobNamesAllowed: *longJobNamesAllowed}
 
-			files, _ := ioutil.ReadDir(path)
+			imagesToTag := make(map[string]string)
+
+			files, _ := os.ReadDir(path)
 			for _, file := range files {
 				if file.IsDir() {
 					continue
@@ -93,53 +96,65 @@ func main() {
 				}
 
 				src := filepath.Join(path, file.Name())
-				jobs := cli.ReadJobsConfig(src)
-				jobs.Jobs = pkg.FilterReleaseBranchingJobs(jobs.Jobs)
+				cfg := cli.ReadJobsConfig(src)
+				cfg.Jobs = pkg.FilterReleaseBranchingJobs(cfg.Jobs)
+
+				if cfg.SupportReleaseBranching {
+					cfg.Env = filterDuplicateEnvVars(cfg.Env)
 
-				if jobs.SupportReleaseBranching {
-					match := tagRegex.FindStringSubmatch(jobs.Image)
 					branch := "release-" + flag.Arg(1)
-					if len(match) == 4 {
-						// HACK: replacing the branch name in the image tag and
-						// adding it as a new tag.
-						// For example, if the test image in the current Prow job
-						// config is
-						// `gcr.io/istio-testing/build-tools:release-1.10-2021-08-09T16-46-08`,
-						// and the Prow job config for release-1.11 branch is
-						// supposed to be generated, the image will be added a
-						// new `release-1.11-2021-08-09T16-46-08` tag.
-						// This is only needed for creating Prow jobs for a new
-						// release branch for the first time, and the image tag
-						// will be overwritten by Automator the next time the
-						// image for the new branch is updated.
-						newImage := fmt.Sprintf("%s:%s-%s", match[1], branch, match[3])
-						jobs.Image = newImage
-						if !*skipGarTagging {
-							if err := exec.Command("gcloud", "container", "images", "add-tag", match[0], newImage).Run(); err != nil {
-								log.Fatalf("Unable to add image tag %q: %v", newImage, err)
-							}
+
+					err, newImage, matchedImage := branchedImageName(cfg.Image, branch)
+					if err != nil {
+						log.Fatalf("Error matching config image: %v", err)
+					}
+
+					cfg.Image = newImage
+					if !*skipGarTagging {
+						if err := exec.Command("gcloud", "container", "images", "add-tag", matchedImage, newImage).Run(); err != nil {
+							log.Fatalf("Unable to add image tag %q: %v", newImage, err)
+						}
+					} else {
+						imagesToTag[matchedImage] = newImage
+					}
+
+					for index, job := range cfg.Jobs {
+						job.Env = filterDuplicateEnvVars(job.Env)
+
+						err, newImage, _ := branchedImageName(job.Image, branch)
+						if err != nil {
+							log.Fatalf("Error matching job image: %v", err)
 						}
+
+						cfg.Jobs[index].Image = newImage
 					}
-					jobs.Branches = []string{branch}
-					jobs.SupportReleaseBranching = false
+
+					cfg.Branches = []string{branch}
+					cfg.SupportReleaseBranching = false
 
 					name := file.Name()
 					ext := filepath.Ext(name)
 					name = name[:len(name)-len(ext)] + "-" + flag.Arg(1) + ext
 
 					dst := filepath.Join(*inputDir, name)
-					bytes, err := yaml.Marshal(jobs)
+					bytes, err := yaml.Marshal(cfg)
 					if err != nil {
 						log.Fatalf("Error marshaling jobs config: %v", err)
 					}
 
 					// Writes the job yaml
-					if err := ioutil.WriteFile(dst, bytes, 0o644); err != nil {
+					if err := os.WriteFile(dst, bytes, 0o644); err != nil {
 						log.Fatalf("Error writing branches config: %v", err)
 					}
 				}
 			}
 
+			if *skipGarTagging {
+				for matchedImage, newImage := range imagesToTag {
+					log.Printf("Please find a maintainer with sufficient permissions and have them run `gcloud container image add-tag %s %s`", matchedImage, newImage)
+				}
+			}
+
 			return nil
 		}); err != nil {
 			log.Fatalf("Walking through the meta config files failed: %v", err)
@@ -174,7 +189,7 @@ func main() {
 			}
 			cli := pkg.Client{BaseConfig: baseConfig, LongJobNamesAllowed: *longJobNamesAllowed}
 
-			files, _ := ioutil.ReadDir(path)
+			files, _ := os.ReadDir(path)
 			for _, file := range files {
 				if file.IsDir() {
 					continue
@@ -187,18 +202,18 @@ func main() {
 				}
 
 				src := filepath.Join(path, file.Name())
-				jobs := cli.ReadJobsConfig(src)
-				for _, branch := range jobs.Branches {
-					output, err := cli.ConvertJobConfig(file.Name(), jobs, branch)
+				cfg := cli.ReadJobsConfig(src)
+				for _, branch := range cfg.Branches {
+					output, err := cli.ConvertJobConfig(file.Name(), cfg, branch)
 					if err != nil {
 						log.Fatal(err)
 					}
-					rf := ref{jobs.Org, jobs.Repo, branch}
+					rf := ref{cfg.Org, cfg.Repo, branch}
 					if _, ok := cachedOutput[rf]; !ok {
 						cachedOutput[rf] = output
 					} else {
 						cachedOutput[rf] = combineJobConfigs(cachedOutput[rf], output,
-							fmt.Sprintf("%s/%s", jobs.Org, jobs.Repo))
+							fmt.Sprintf("%s/%s", cfg.Org, cfg.Repo))
 					}
 				}
 			}
@@ -235,6 +250,44 @@ func main() {
 	}
 }
 
+func filterDuplicateEnvVars(env []v1.EnvVar) (filtered []v1.EnvVar) {
+	m := make(map[string]string)
+
+	for _, v := range env {
+		m[v.Name] = v.Value
+	}
+
+	for key, value := range m {
+		filtered = append(filtered, v1.EnvVar{Name: key, Value: value})
+	}
+
+	return filtered
+}
+
+func branchedImageName(image string, branch string) (error, string, string) {
+	match := tagRegex.FindStringSubmatch(image)
+
+	if len(match) == 4 {
+		// HACK: replacing the branch name in the image tag and
+		// adding it as a new tag.
+		// For example, if the test image in the current Prow job
+		// config is
+		// `gcr.io/istio-testing/build-tools:master-gitsha`,
+		// and the Prow job config for release-1.25 branch is
+		// supposed to be generated, the image will be added a
+		// new `release-1.25-gitsha` tag.
+		// This is only needed for creating Prow jobs for a new
+		// release branch for the first time, and the image tag
+		// will be overwritten by Automator the next time the
+		// image for the new branch is updated.
+		newImage := fmt.Sprintf("%s:%s-%s", match[1], branch, match[3])
+
+		return nil, newImage, match[0]
+	}
+
+	return errors.New("no match found"), "", ""
+}
+
 func runProcessCommand(rawCommand string) error {
 	log.Printf("⚙️ %s", rawCommand)
 	cmdSplit, err := shell.Split(rawCommand)