Update Porter to handle relocation mapping on pull/operations (#719)

* Update Porter to handle relocation mapping on pull/operations

This PR updates Porter to handle a relocation mapping if it is generated on a bundle pull. If generated, it is stored in the bundle cache and then passed along via an oper config to add the file to an operation before it is run. Also overwrites the invocation image, if it is found in the relocation mapping

Author: jeremyrickard

pkg/cache/cache.go

@@ -3,17 +3,19 @@ package cache
 import (
 	"crypto/md5"
 	"encoding/hex"
+	"encoding/json"
 	"os"
 	"path/filepath"
 
 	"github.com/deislabs/cnab-go/bundle"
 	"github.com/deislabs/porter/pkg/config"
+	"github.com/docker/cnab-to-oci/relocation"
 	"github.com/pkg/errors"
 )
 
 type BundleCache interface {
-	FindBundle(string) (string, bool, error)
-	StoreBundle(string, *bundle.Bundle) (string, error)
+	FindBundle(string) (string, string, bool, error)
+	StoreBundle(string, *bundle.Bundle, relocation.ImageRelocationMap) (string, string, error)
 	GetCacheDir() (string, error)
 }
 
@@ -29,42 +31,73 @@ func New(cfg *config.Config) BundleCache {
 
 // FindBundle looks for a given bundle tag in the Porter bundle cache and
 // returns the path to the bundle if it exists. If it is not found, an
-// empty string and the boolean false value are returned.
-func (c *cache) FindBundle(bundleTag string) (string, bool, error) {
+// empty string and the boolean false value are returned. If the bundle is found,
+// and a relocation mapping file is present, it will be returned as well. If the relocation
+// is not found, an empty string is returned.
+func (c *cache) FindBundle(bundleTag string) (string, string, bool, error) {
 	bid := getBundleID(bundleTag)
 	bundleCnabDir, err := c.getCachedBundleCNABDir(bid)
 	cachedBundlePath := filepath.Join(bundleCnabDir, "bundle.json")
+	cachedReloPath := filepath.Join(bundleCnabDir, "relocation-mapping.json")
 	bExists, err := c.FileSystem.Exists(cachedBundlePath)
 	if err != nil {
-		return "", false, errors.Wrapf(err, "unable to read bundle %s at %s", bundleTag, cachedBundlePath)
+		return "", "", false, errors.Wrapf(err, "unable to read bundle %s at %s", bundleTag, cachedBundlePath)
 	}
 	if !bExists {
-		return "", false, nil
+		return "", "", false, nil
 	}
-	return cachedBundlePath, true, nil
+	//check for a relocation mapping next to it
+	rExists, err := c.FileSystem.Exists(cachedReloPath)
+	if err != nil {
+		return "", "", false, errors.Wrapf(err, "unable to read relocation mapping %s at %s", bundleTag, cachedReloPath)
+	}
+	if rExists {
+		return cachedBundlePath, cachedReloPath, true, nil
+	}
+	return cachedBundlePath, "", true, nil
 
 }
 
 // StoreBundle will write a given bundle to the bundle cache, in a location derived
-// from the bundleTag.
-func (c *cache) StoreBundle(bundleTag string, bun *bundle.Bundle) (string, error) {
+// from the bundleTag. If a relocation mapping is provided, it will be stored along side
+// the bundle. If successful, returns the path to the bundle, along with the path to a
+// relocation mapping, if provided. Otherwise, returns an error.
+func (c *cache) StoreBundle(bundleTag string, bun *bundle.Bundle, reloMap relocation.ImageRelocationMap) (string, string, error) {
 	bid := getBundleID(bundleTag)
 	bundleCnabDir, err := c.getCachedBundleCNABDir(bid)
 	cachedBundlePath := filepath.Join(bundleCnabDir, "bundle.json")
 	err = c.FileSystem.MkdirAll(bundleCnabDir, os.ModePerm)
 	if err != nil {
-		return "", errors.Wrap(err, "unable to create cache directory")
+		return "", "", errors.Wrap(err, "unable to create cache directory")
 	}
 	f, err := c.FileSystem.OpenFile(cachedBundlePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
 	defer f.Close()
 	if err != nil {
-		return "", errors.Wrapf(err, "error creating cnab/bundle.json for %s", bundleTag)
+		return "", "", errors.Wrapf(err, "error creating cnab/bundle.json for %s", bundleTag)
 	}
 	_, err = bun.WriteTo(f)
 	if err != nil {
-		return "", errors.Wrapf(err, "error writing to cnab/bundle.json for %s", bundleTag)
+		return "", "", errors.Wrapf(err, "error writing to cnab/bundle.json for %s", bundleTag)
+	}
+	// we wrote the bundle, now lets store a relocation mapping in cnab/ and return the path
+	if len(reloMap) > 0 {
+		cachedReloPath := filepath.Join(bundleCnabDir, "relocation-mapping.json")
+		f, err = c.FileSystem.OpenFile(cachedReloPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+		defer f.Close()
+		if err != nil {
+			return "", "", errors.Wrapf(err, "error creating cnab/relocation-mapping.json for %s", bundleTag)
+		}
+		b, err := json.Marshal(reloMap)
+		if err != nil {
+			return "", "", errors.Wrapf(err, "couldn't marshall relocation mapping for %s", bundleTag)
+		}
+		_, err = f.Write(b)
+		if err != nil {
+			return "", "", errors.Wrapf(err, "couldn't write relocation mapping for %s", bundleTag)
+		}
+		return cachedBundlePath, cachedReloPath, nil
 	}
-	return cachedBundlePath, nil
+	return cachedBundlePath, "", nil
 }
 
 func (c *cache) GetCacheDir() (string, error) {

pkg/cache/cache_test.go

@@ -2,11 +2,13 @@ package cache
 
 import (
 	"bytes"
+	"fmt"
 	"path/filepath"
 	"testing"
 
 	"github.com/deislabs/cnab-go/bundle"
 	"github.com/deislabs/porter/pkg/config"
+	"github.com/docker/cnab-to-oci/relocation"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -36,7 +38,7 @@ func TestFindBundleCacheExists(t *testing.T) {
 	cfg.TestContext.AddTestDirectory("testdata", cacheDir)
 	c := New(cfg.Config)
 
-	_, ok, err := c.FindBundle("deislabs/kubekahn:latest")
+	_, _, ok, err := c.FindBundle("deislabs/kubekahn:latest")
 	assert.NoError(t, err, "the cache dir should exist, no error should have happened")
 	assert.False(t, ok, "the bundle shouldn't exist")
 }
@@ -49,7 +51,7 @@ func TestFindBundleCacheDoesNotExist(t *testing.T) {
 	cfg.TestContext.AddTestDirectory("testdata", cacheDir)
 	c := New(cfg.Config)
 
-	_, ok, err := c.FindBundle("deislabs/kubekahn:latest")
+	_, _, ok, err := c.FindBundle("deislabs/kubekahn:latest")
 	assert.NoError(t, err, "the cache dir doesn't exist, but this shouldn't be an error")
 	assert.False(t, ok, "the bundle shouldn't exist")
 }
@@ -66,7 +68,7 @@ func TestFindBundleBundleCached(t *testing.T) {
 	foundIt, err := cfg.Config.FileSystem.Exists(expectedCacheFile)
 	require.True(t, foundIt, "test data not loaded")
 	c := New(cfg.Config)
-	path, ok, err := c.FindBundle(kahn1dot01)
+	path, _, ok, err := c.FindBundle(kahn1dot01)
 	assert.NoError(t, err, "the cache dir should exist, no error should have happened")
 	assert.True(t, ok, "the bundle should exist")
 	assert.Equal(t, expectedCacheFile, path)
@@ -75,7 +77,7 @@ func TestFindBundleBundleCached(t *testing.T) {
 func TestFindBundleBundleNotCached(t *testing.T) {
 	cfg := config.NewTestConfig(t)
 	c := New(cfg.Config)
-	path, ok, err := c.FindBundle(kahnlatest)
+	path, _, ok, err := c.FindBundle(kahnlatest)
 	assert.NoError(t, err, "the cache dir should exist, no error should have happened")
 	assert.False(t, ok, "the bundle should not exist")
 	assert.Empty(t, path, "should not have a path")
@@ -89,7 +91,8 @@ func TestCacheWriteNoCacheDir(t *testing.T) {
 	require.NoError(t, err, "bundle should have been valid")
 
 	c := New(cfg.Config)
-	path, err := c.StoreBundle(kahn1dot01, &bun)
+	var reloMap relocation.ImageRelocationMap
+	path, _, err := c.StoreBundle(kahn1dot01, &bun, reloMap)
 
 	home, err := cfg.Config.GetHomeDir()
 	require.NoError(t, err, "should have had a porter home dir")
@@ -114,7 +117,8 @@ func TestCacheWriteCacheDirExists(t *testing.T) {
 	require.NoError(t, err, "bundle should have been valid")
 
 	c := New(cfg.Config)
-	path, err := c.StoreBundle(kahn1dot01, &bun)
+	var reloMap relocation.ImageRelocationMap
+	path, _, err := c.StoreBundle(kahn1dot01, &bun, reloMap)
 
 	expectedCacheDirectory := filepath.Join(cacheDir, kahn1dot0Hash)
 	expectedCacheCNABDirectory := filepath.Join(expectedCacheDirectory, "cnab")
@@ -123,3 +127,45 @@ func TestCacheWriteCacheDirExists(t *testing.T) {
 	assert.Equal(t, expectedCacheFile, path)
 	assert.NoError(t, err, "storing bundle should have succeeded")
 }
+
+func TestStoreRelocationMapping(t *testing.T) {
+
+	cfg := config.NewTestConfig(t)
+	home, _ := cfg.Config.GetHomeDir()
+	cacheDir := filepath.Join(home, "cache")
+
+	tests := []struct {
+		name              string
+		relocationMapping relocation.ImageRelocationMap
+		tag               string
+		bundle            *bundle.Bundle
+		wantedReloPath    string
+		err               error
+	}{
+		{
+			name:   "relocation file gets a path",
+			bundle: &bundle.Bundle{},
+			tag:    kahn1dot01,
+			relocationMapping: relocation.ImageRelocationMap{
+				"asd": "asdf",
+			},
+			wantedReloPath: filepath.Join(cacheDir, kahn1dot0Hash, "cnab", "relocation-mapping.json"),
+		},
+		{
+			name:           "no relocation file gets no path",
+			tag:            kahnlatest,
+			bundle:         &bundle.Bundle{},
+			wantedReloPath: "",
+		},
+	}
+
+	c := New(cfg.Config)
+	for _, test := range tests {
+		_, reloPath, err := c.StoreBundle(test.tag, test.bundle, test.relocationMapping)
+		assert.NoError(t, err, fmt.Sprintf("didn't expect storage error for test %s", test.name))
+		assert.Equal(t, test.wantedReloPath, reloPath, "didn't get expected path for store on test: %s", test.name)
+		_, fetchedPath, _, err := c.FindBundle(test.tag)
+		assert.Equal(t, test.wantedReloPath, fetchedPath, "didn't get expected path for load on test: %s", test.name)
+
+	}
+}

pkg/cache/testdata/cnab/bundle.json

@@ -5,7 +5,13 @@
 		}
 	},
 	"description": "",
-	"images": null,
+	"images": { 
+		"my-microservice":{ 
+		   "contentDigest":"sha256:cca460afa270d4c527981ef9ca4989346c56cf9b20217dcea37df1ece8120687",
+		   "description":"my microservice",
+		   "image":"gabrtv/microservice@sha256:cca460afa270d4c527981ef9ca4989346c56cf9b20217dcea37df1ece8120687"
+		}
+	},
 	"invocationImages": [{
 		"digest": "sha256:f858bc025ad34099fe67ebe6152e03b4c91b34cc7a77d1aa10aaf1dc1389c2c2",
 		"image": "jeremyrickard/porter-mysql@sha256:f858bc025ad34099fe67ebe6152e03b4c91b34cc7a77d1aa10aaf1dc1389c2c2",

pkg/cache/testdata/cnab/relocation-mapping.json

@@ -0,0 +1,3 @@
+{
+    "gabrtv/microservice@sha256:cca460afa270d4c527981ef9ca4989346c56cf9b20217dcea37df1ece8120687": "my.registry/microservice@sha256:cca460afa270d4c527981ef9ca4989346c56cf9b20217dcea37df1ece8120687"
+  }

pkg/cnab/cnab-to-oci/registry.go

@@ -3,13 +3,15 @@ package cnabtooci
 import (
 	"context"
 	"fmt"
+
 	"strings"
 
 	containerdRemotes "github.com/containerd/containerd/remotes"
 	"github.com/deislabs/cnab-go/bundle"
 	"github.com/docker/cli/cli/command"
 	dockerconfig "github.com/docker/cli/cli/config"
 	cliflags "github.com/docker/cli/cli/flags"
+	"github.com/docker/cnab-to-oci/relocation"
 	"github.com/docker/cnab-to-oci/remotes"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
@@ -31,11 +33,11 @@ func NewRegistry(c *portercontext.Context) *Registry {
 	}
 }
 
-// PullBundle pulls a bundle from an OCI registry.
-func (r *Registry) PullBundle(tag string, insecureRegistry bool) (*bundle.Bundle, error) {
+// PullBundle pulls a bundle from an OCI registry. Returns the bundle, and an optional image relocation mapping, if applicable.
+func (r *Registry) PullBundle(tag string, insecureRegistry bool) (*bundle.Bundle, relocation.ImageRelocationMap, error) {
 	ref, err := reference.ParseNormalizedNamed(tag)
 	if err != nil {
-		return nil, errors.Wrap(err, "invalid bundle tag format, expected REGISTRY/name:tag")
+		return nil, nil, errors.Wrap(err, "invalid bundle tag format, expected REGISTRY/name:tag")
 	}
 
 	var insecureRegistries []string
@@ -44,11 +46,11 @@ func (r *Registry) PullBundle(tag string, insecureRegistry bool) (*bundle.Bundle
 		insecureRegistries = append(insecureRegistries, reg)
 	}
 
-	bun, _, err := remotes.Pull(context.Background(), ref, r.createResolver(insecureRegistries))
+	bun, reloMap, err := remotes.Pull(context.Background(), ref, r.createResolver(insecureRegistries))
 	if err != nil {
-		return nil, errors.Wrap(err, "unable to pull remote bundle")
+		return nil, nil, errors.Wrap(err, "unable to pull remote bundle")
 	}
-	return bun, nil
+	return bun, reloMap, nil
 }
 
 func (r *Registry) PushBundle(bun *bundle.Bundle, tag string, insecureRegistry bool) error {

pkg/cnab/provider/action.go

@@ -1,8 +1,12 @@
 package cnabprovider
 
 import (
+	"encoding/json"
+
 	"github.com/deislabs/cnab-go/action"
 	"github.com/deislabs/cnab-go/driver"
+	"github.com/docker/cnab-to-oci/relocation"
+	"github.com/pkg/errors"
 )
 
 // Shared arguments for all CNAB actions
@@ -28,12 +32,16 @@ type ActionArguments struct {
 
 	// Driver is the CNAB-compliant driver used to run bundle actions.
 	Driver string
+
+	// Path to an optional relocation mapping file
+	RelocationMapping string
 }
 
 func (d *Runtime) ApplyConfig(args ActionArguments) action.OperationConfigs {
 	return action.OperationConfigs{
 		d.SetOutput(),
 		d.AddFiles(args),
+		d.AddRelocation(args),
 	}
 }
 
@@ -52,3 +60,27 @@ func (d *Runtime) AddFiles(args ActionArguments) action.OperationConfigFunc {
 		return nil
 	}
 }
+
+// AddRelocation operates on an ActionArguments and adds any provided relocation mapping
+// to the operation's files.
+func (d *Runtime) AddRelocation(args ActionArguments) action.OperationConfigFunc {
+	return func(op *driver.Operation) error {
+		if args.RelocationMapping != "" {
+			b, err := d.FileSystem.ReadFile(args.RelocationMapping)
+			if err != nil {
+				return errors.Wrap(err, "unable to add relocation mapping")
+			}
+			op.Files["/cnab/app/relocation-mapping.json"] = string(b)
+			var reloMap relocation.ImageRelocationMap
+			err = json.Unmarshal(b, &reloMap)
+			// If the invocation image is present in the relocation mapping, we need
+			// to update the operation and set the new image reference. Unfortunately,
+			// the relocation mapping is just reference => reference, so there isn't a
+			// great way to check for the invocation image.
+			if mappedInvo, ok := reloMap[op.Image.Image]; ok {
+				op.Image.Image = mappedInvo
+			}
+		}
+		return nil
+	}
+}

pkg/cnab/provider/action_test.go

@@ -0,0 +1,46 @@
+package cnabprovider
+
+import (
+	"github.com/deislabs/cnab-go/bundle"
+	"io/ioutil"
+	"testing"
+
+	"github.com/deislabs/cnab-go/driver"
+	"github.com/deislabs/porter/pkg/config"
+	instancestorageprovider "github.com/deislabs/porter/pkg/instance-storage/provider"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAddReloccation(t *testing.T) {
+	data, err := ioutil.ReadFile("testdata/relocation-mapping.json")
+	require.NoError(t, err)
+
+	c := config.NewTestConfig(t)
+	instanceStorage := instancestorageprovider.NewPluginDelegator(c.Config)
+	d := NewRuntime(c.Config, instanceStorage)
+
+	args := ActionArguments{
+		RelocationMapping: "/cnab/app/relocation-mapping.json",
+	}
+
+	c.TestContext.AddTestFile("testdata/relocation-mapping.json", "/cnab/app/relocation-mapping.json")
+
+	opConf := d.AddRelocation(args)
+
+	invoImage := bundle.InvocationImage{}
+	invoImage.Image = "gabrtv/microservice@sha256:cca460afa270d4c527981ef9ca4989346c56cf9b20217dcea37df1ece8120687"
+
+	op := &driver.Operation{
+		Files: make(map[string]string),
+		Image: invoImage,
+	}
+	err = opConf(op)
+	assert.NoError(t, err)
+
+	mapping, ok := op.Files["/cnab/app/relocation-mapping.json"]
+	assert.True(t, ok)
+	assert.Equal(t, string(data), mapping)
+	assert.Equal(t, "my.registry/microservice@sha256:cca460afa270d4c527981ef9ca4989346c56cf9b20217dcea37df1ece8120687", op.Image.Image)
+
+}

pkg/cnab/provider/testdata/relocation-mapping.json

@@ -0,0 +1,4 @@
+{
+    "gabrtv/microservice@sha256:cca460afa270d4c527981ef9ca4989346c56cf9b20217dcea37df1ece8120687": "my.registry/microservice@sha256:cca460afa270d4c527981ef9ca4989346c56cf9b20217dcea37df1ece8120687",
+    "technosophos/helloworld:0.1.0": "my.registry/helloworld:0.1.0"
+  }