Merge pull request #168 from balena-os/a2o-migrate

Aufs to overlay2 migration utility

daemon/daemon.go

@@ -56,6 +56,7 @@ import (
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/locker"
 	"github.com/docker/docker/pkg/plugingetter"
+	"github.com/docker/docker/pkg/storagemigration"
 	"github.com/docker/docker/pkg/sysinfo"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/docker/pkg/truncindex"
@@ -933,6 +934,41 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		return nil, err
 	}
 
+	// attempt to run the aufs-to-overlay2 graphdriver migration on the
+	// state directory. since this is happening before we even initialize
+	// the graphdrivers it should be safe to do here.
+	_, doStorageMigration := os.LookupEnv("BALENA_MIGRATE_OVERLAY")
+	if config.GraphDriver == "overlay2" && doStorageMigration {
+		logrus.Info("Starting storage migration from aufs to overlay2")
+		start := time.Now()
+		var err error
+		err = storagemigration.Migrate(config.Root)
+		if err != nil {
+			if err == storagemigration.ErrAuFSRootNotExists ||
+				err == storagemigration.ErrOverlayRootExists {
+				// gracefully handle missing aufs root or existing
+				// overlay root - we have nothing to migrate from
+				logrus.Infof("Storage migration skipped: %s", err)
+			} else {
+				// rollback partial migration
+				if err := storagemigration.FailCleanup(config.Root); err != nil {
+					// if this fails abort daemon startup
+					return nil, errors.Wrap(err, "failed to cleanup after failed storage migration")
+				}
+				// even though the migration failed with an error
+				// we continue daemon startup on aufs. This allows
+				// for debugging the failed migration by hand.
+				logrus.Errorf("Storage migration failed: %s", err)
+			}
+		} else {
+			// only commit if migration succeded
+			if err := storagemigration.Commit(config.Root); err != nil {
+				return nil, errors.Wrap(err, "failed to commit storage migration")
+			}
+			logrus.Infof("Finished storage migration from aufs to overlay2, took %s", time.Now().Sub(start))
+		}
+	}
+
 	for operatingSystem, gd := range d.graphDrivers {
 		layerStores[operatingSystem], err = layer.NewStoreFromOptions(layer.StoreOptions{
 			Root:                      config.Root,

daemon/graphdriver/overlay2/overlay.go

@@ -77,14 +77,14 @@ const (
 	lowerFile     = "lower"
 	maxDepth      = 128
 
-	// idLength represents the number of random characters
+	// IDLength represents the number of random characters
 	// which can be used to create the unique link identifier
 	// for every layer. If this value is too long then the
 	// page size limit for the mount command may be exceeded.
-	// The idLength should be selected such that following equation
+	// The IDLength should be selected such that following equation
 	// is true (512 is a buffer for label metadata).
-	// ((idLength + len(linkDir) + 1) * maxDepth) <= (pageSize - 512)
-	idLength = 26
+	// ((IDLength + len(linkDir) + 1) * maxDepth) <= (pageSize - 512)
+	IDLength = 26
 )
 
 type overlayOptions struct {
@@ -371,7 +371,7 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts) (retErr
 		return err
 	}
 
-	lid := generateID(idLength)
+	lid := GenerateID(IDLength)
 	if err := os.Symlink(path.Join("..", id, diffDirName), path.Join(d.home, linkDir, lid)); err != nil {
 		return err
 	}

daemon/graphdriver/overlay2/randomid.go

@@ -14,8 +14,8 @@ import (
 	"golang.org/x/sys/unix"
 )
 
-// generateID creates a new random string identifier with the given length
-func generateID(l int) string {
+// GenerateID creates a new random string identifier with the given length
+func GenerateID(l int) string {
 	const (
 		// ensures we backoff for less than 450ms total. Use the following to
 		// select new value, in units of 10ms:

integration/storagemigration/aufs_to_overlay2_test.go

@@ -0,0 +1,81 @@
+package storagemigration
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+
+	"github.com/docker/docker/internal/test/daemon"
+
+	"gotest.tools/assert"
+	"gotest.tools/fs"
+	"gotest.tools/skip"
+)
+
+func TestAufsToOverlay2Migration(t *testing.T) {
+	skip.If(t, testEnv.DaemonInfo.OSType != "linux")
+	skip.If(t, testEnv.DaemonInfo.Driver != "overlay2")
+	defer setupTest(t)()
+
+	var err error
+
+	root := fs.NewDir(t, t.Name())
+	defer root.Remove()
+
+	{
+		// aufs.tar.gz contains a snapshot of /var/lib/docker after
+		// building testdata/Dockerfile using dockerd which uses aufs
+		// as default storage driver
+		tar := exec.Command("tar", "-xzf", filepath.Join("testdata", "aufs.tar.gz"), "-C", root.Path())
+		tar.Stdout = os.Stdout
+		tar.Stderr = os.Stderr
+		assert.NilError(t, tar.Run())
+	}
+
+	// if testing.Verbose() {
+	// 	logrus.SetLevel(logrus.DebugLevel)
+	// }
+	// err = storagemigration.Migrate(root.Path())
+	// assert.NilError(t, err)
+
+	err = os.Setenv("BALENA_MIGRATE_OVERLAY", "1")
+	assert.NilError(t, err)
+
+	d := daemon.New(t)
+	d.Root = root.Path()
+	d.Start(t)
+	defer d.Stop(t)
+
+	ctx := context.Background()
+
+	cl := d.NewClientT(t)
+
+	ctr, err := cl.ContainerCreate(ctx,
+		&container.Config{
+			Image: "a2o-test",
+		},
+		nil,
+		nil,
+		"",
+	)
+	assert.NilError(t, err)
+
+	err = cl.ContainerStart(ctx, ctr.ID, types.ContainerStartOptions{})
+	assert.NilError(t, err)
+
+	// original f1 should be removed (.wh.)
+	_, err = cl.ContainerStatPath(ctx, ctr.ID, "/tmp/f1")
+	assert.ErrorContains(t, err, "No such container:path")
+	// original d1 should be opaque (.wh..wh.opq)
+	_, err = cl.ContainerStatPath(ctx, ctr.ID, "/tmp/d1/d1f2")
+	assert.NilError(t, err)
+	_, err = cl.ContainerStatPath(ctx, ctr.ID, "/tmp/hlkn")
+	assert.NilError(t, err)
+	_, err = cl.ContainerStatPath(ctx, ctr.ID, "/tmp/slkn")
+	assert.NilError(t, err)
+}

integration/storagemigration/main_test.go

@@ -0,0 +1,33 @@
+package storagemigration
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/docker/docker/internal/test/environment"
+)
+
+var testEnv *environment.Execution
+
+func TestMain(m *testing.M) {
+	var err error
+	testEnv, err = environment.New()
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	err = environment.EnsureFrozenImagesLinux(testEnv)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	testEnv.Print()
+	os.Exit(m.Run())
+}
+
+func setupTest(t *testing.T) func() {
+	environment.ProtectAll(t, testEnv)
+	return func() { testEnv.Clean(t) }
+}

integration/storagemigration/testdata/Dockerfile

@@ -0,0 +1,5 @@
+FROM busybox:1.33.0
+RUN mkdir /tmp/d1 && touch /tmp/d1/d1f1 && touch /tmp/f1 && touch /tmp/f2
+RUN rm -R /tmp/d1 && mkdir /tmp/d1 && touch /tmp/d1/d1f2 && rm /tmp/f1
+RUN ln -s /tmp/d1/d1f2 /tmp/slkn
+RUN ln /tmp/d1/d1f2 /tmp/hlkn

pkg/storagemigration/aufsutils.go

@@ -0,0 +1,97 @@
+// TODO: do we need to handle .wh..wh.plnk layer hardlinks?
+package storagemigration
+
+import (
+	"bufio"
+	"errors"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/docker/docker/pkg/archive"
+)
+
+var (
+	// ErrAuFSRootNotExists indicates the aufs root directory wasn't found
+	ErrAuFSRootNotExists = errors.New("Aufs root doesn't exists")
+)
+
+// CheckRootExists checks for the aufs storage root directory
+func CheckAufsRootExists(engineDir string) error {
+	root := filepath.Join(engineDir, "aufs")
+	logrus.WithField("aufs_root", root).Debug("checking if aufs root exists")
+	ok, err := exists(root, true)
+	if err != nil {
+		return err
+	}
+	if !ok {
+		return ErrAuFSRootNotExists
+	}
+	return nil
+}
+
+func IsWhiteout(filename string) bool {
+	return strings.HasPrefix(filename, archive.WhiteoutPrefix)
+}
+
+func IsWhiteoutMeta(filename string) bool {
+	return strings.HasPrefix(filename, archive.WhiteoutMetaPrefix)
+}
+
+func IsOpaqueParentDir(filename string) bool {
+	return filename == archive.WhiteoutOpaqueDir
+}
+
+func StripWhiteoutPrefix(filename string) string {
+	out := filename
+	for IsWhiteout(out) && !IsWhiteoutMeta(out) {
+		out = strings.TrimPrefix(out, archive.WhiteoutPrefix)
+	}
+	return out
+}
+
+// Return all the directories
+//
+// from daemon/graphdriver/aufs/dirs.go
+func LoadIDs(root string) ([]string, error) {
+	dirs, err := ioutil.ReadDir(root)
+	if err != nil {
+		return nil, err
+	}
+	var out []string
+	for _, d := range dirs {
+		if d.IsDir() {
+			out = append(out, d.Name())
+		}
+	}
+	return out, nil
+}
+
+// Read the layers file for the current id and return all the
+// layers represented by new lines in the file
+//
+// If there are no lines in the file then the id has no parent
+// and an empty slice is returned.
+//
+// from daemon/graphdriver/aufs/dirs.go
+func GetParentIDs(root, id string) ([]string, error) {
+	f, err := os.Open(path.Join(root, "layers", id))
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	var out []string
+	s := bufio.NewScanner(f)
+
+	for s.Scan() {
+		if t := s.Text(); t != "" {
+			out = append(out, s.Text())
+		}
+	}
+	return out, s.Err()
+}

pkg/storagemigration/aufsutils_test.go

@@ -0,0 +1,58 @@
+package storagemigration
+
+import "testing"
+
+func TestIsWhiteout(t *testing.T) {
+	var tcs = map[string]bool{
+		".wh.foo.txt":  true,
+		"bar.txt":      false,
+		".wh..wh.plnk": true,
+		".wh..wh..opq": true,
+	}
+	for file, expect := range tcs {
+		if IsWhiteout(file) != expect {
+			t.Fatalf("did not detect %v", file)
+		}
+	}
+}
+func TestIsWhiteoutMeta(t *testing.T) {
+	var tcs = map[string]bool{
+		".wh.foo.txt":  false,
+		"bar.txt":      false,
+		".wh..wh.plnk": true,
+		".wh..wh..opq": true,
+	}
+	for file, expect := range tcs {
+		if IsWhiteoutMeta(file) != expect {
+			t.Fatalf("did not detect %v", file)
+		}
+	}
+}
+
+func TestIsOpaque(t *testing.T) {
+	var tcs = map[string]bool{
+		".wh.foo.txt":  false,
+		"bar.txt":      false,
+		".wh..wh.plnk": false,
+		".wh..wh..opq": true,
+	}
+	for file, expect := range tcs {
+		if IsOpaqueParentDir(file) != expect {
+			t.Fatalf("did not detect %v", file)
+		}
+	}
+}
+
+func TestStripWhiteoutPrefix(t *testing.T) {
+	var tcs = map[string]string{
+		".wh.foo.txt":  "foo.txt",
+		"bar.txt":      "bar.txt",
+		".wh..wh.plnk": ".wh..wh.plnk",
+		".wh..wh..opq": ".wh..wh..opq",
+	}
+	for file, expect := range tcs {
+		if result := StripWhiteoutPrefix(file); result != expect {
+			t.Fatalf("stripping filename failed, got: %q wanted: %q", result, expect)
+		}
+	}
+}

pkg/storagemigration/commit.go

@@ -0,0 +1,27 @@
+package storagemigration
+
+import (
+	"path/filepath"
+
+	"github.com/sirupsen/logrus"
+)
+
+// Commit finalises the migration by deleting aufs storage root and images.
+func Commit(root string) error {
+	logrus.WithField("storage_root", root).Debug("committing changes")
+
+	// remove aufs layer data
+	err := removeDirIfExists(aufsRoot(root))
+	if err != nil {
+		return err
+	}
+
+	// remove images
+	aufsImageDir := filepath.Join(root, "image", "aufs")
+	err = removeDirIfExists(aufsImageDir)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}