From ebe66b0000f35441298d1b8da3ec8789de12a44b Mon Sep 17 00:00:00 2001 From: Robert Glonek Date: Mon, 8 Jan 2024 13:48:08 -0800 Subject: [PATCH] add multiarch support --- CHANGELOG.md | 1 + LATEST.md | 1 + docs/docker_multiarch.md | 70 +++++++++++++++++++++ src/backendDocker.go | 129 +++++++++++++++++++++++++++++---------- src/cmdConfig.go | 10 +++ 5 files changed, 178 insertions(+), 33 deletions(-) create mode 100644 docs/docker_multiarch.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 92898373..af0a0f84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ #### 7.4.0 * FEATURE: Web UI. +* FRATURE: DOCKER: Add support for multiarch. See [this page](https://github.com/aerospike/aerolab/tree/master/docs/docker_multiarch.md) for details. * FIX: GCP: Many commands would fail during template creation, making parallel use imposible. Fixed. * FIX: GCP: Delete `arm` templates was not working at all. * FIX: The `net list` command does not work when client has same name as server. diff --git a/LATEST.md b/LATEST.md index 371a8c74..5d28396d 100644 --- a/LATEST.md +++ b/LATEST.md @@ -4,6 +4,7 @@ _Release Date: UNDEF_ **Release Notes:** * FEATURE: Web UI. +* FRATURE: DOCKER: Add support for multiarch. See [this page](https://github.com/aerospike/aerolab/tree/master/docs/docker_multiarch.md) for details. * FIX: GCP: Many commands would fail during template creation, making parallel use imposible. Fixed. * FIX: GCP: Delete `arm` templates was not working at all. * FIX: The `net list` command does not work when client has same name as server. diff --git a/docs/docker_multiarch.md b/docs/docker_multiarch.md new file mode 100644 index 00000000..7ddcbdc7 --- /dev/null +++ b/docs/docker_multiarch.md @@ -0,0 +1,70 @@ +# Docker - Multiarch + +AeroLab supports forcing a specific architecture on docker/podman(desktop). This allows running `arm64` images on `amd64` and vice-versa. + +In order for this to work, the docker-host must have a `qemu-user-static` dependent library installed. This library performs a live translation of syscalls between incompatible architectures. + +Docker Desktop already has this automatically installed in newest versions. No prerequisite actions are reququired for this; scroll down to `AeroLab Usage` section. + +## Installing prerequisites + +### Updating + +Whether using podman/docker/podman-desktop/docker-desktop, please first install all the relevant software updates to ensure the latest stable is installed. + +### Podman desktop +```bash +podman machine ssh -- rpm-ostree install qemu-user-static +podman machine ssh -- sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/selinux/config +podman machine ssh -- systemctl reboot +podman machine ssh -- podman run --rm --privileged multiarch/qemu-user-static --reset -p yes +``` + +### Docker-Desktop + +Docker Desktop automatically enables the required `qemu-user-static` and `binfmt` extensions. No actions are required. + +### Docker/Podman on Linux + +First install `qemu-user-static` package from your repository. For example, on debian/ubuntu: + +```bash +apt update && apt -y install qemu-user-static +``` + +Next, enable emulation for `binfmt` on docker or podman: +* docker - `docker run --rm --privileged multiarch/qemu-user-static --reset -p yes` +* podman - `podman run --rm --privileged multiarch/qemu-user-static --reset -p yes` + +## Enabling multiarch support on reboot + +NOTE: Do not execute this step on docker desktop. Docker Desktop already has the required extensions auto-enabled. + +After system reboot, or restart of podman-desktop, multiarch support must be enabled again, as follows: + +* docker - `docker run --rm --privileged multiarch/qemu-user-static --reset -p yes` +* podman - `podman run --rm --privileged multiarch/qemu-user-static --reset -p yes` + +## AeroLab Usage + +By default, `aerolab` will use whichever architecture is native to docker. To force `aerolab` to use a specific architecture, specify it during the backend setup. Once an architecture has been selected, proceed as normal. + +Note: this can be switched back and forth as required without any issues. + +### Force arm64 builds + +```bash +aerolab config backend -t docker -a arm64 +``` + +### Force amd64 builds + +```bash +aerolab config backend -t docker -a amd64 +``` + +### Reset to use native builds + +```bash +aerolab config backend -t docker -a unset +``` diff --git a/src/backendDocker.go b/src/backendDocker.go index c6e28d04..ffcb9d6b 100644 --- a/src/backendDocker.go +++ b/src/backendDocker.go @@ -236,14 +236,20 @@ func (d *backendDocker) Inventory(owner string, inventoryItems []int) (inventory if len(i2) > 1 { i3 = strings.Split(i2[1], ":") } - if len(i3) > 1 { - asdVer = i3[1] + i4 := strings.Split(tt[3], ":") + if len(i4) > 1 { + asdVer = i4[1] } } else { i2 = strings.Split(tt[3], ":") if len(i2) > 1 { i3[0] = i2[1] } + ix := strings.Split(i2[0], "/") + if len(ix) > 1 { + i2[0] = ix[1] + arch = ix[0] + } } clientType := "" if len(tt) > 4 { @@ -268,6 +274,11 @@ func (d *backendDocker) Inventory(owner string, inventoryItems []int) (inventory if intPorts == "" { intPorts = intPort } + if strings.Contains(tt[3], "_amd64") { + arch = "amd64" + } else if strings.Contains(tt[3], "_arm64") { + arch = "arm64" + } invLock.Lock() defer invLock.Unlock() if i == 1 { @@ -435,11 +446,19 @@ func (d *backendDocker) ListTemplates() ([]backendVersion, error) { if len(repo) > len(dockerNameHeader)+2 { repo = repo[len(dockerNameHeader):] distVer := strings.Split(repo, "_") - if len(distVer) == 2 { + if len(distVer) > 1 { + isArm := d.isArm + if len(distVer) > 2 { + if distVer[2] == "arm64" { + isArm = true + } else { + isArm = false + } + } tagList := strings.Split(t, ";") if len(tagList) > 1 { tag := strings.Trim(tagList[1], "'\"") - templateList = append(templateList, backendVersion{distVer[0], distVer[1], tag, d.isArm}) + templateList = append(templateList, backendVersion{distVer[0], distVer[1], tag, isArm}) } } } @@ -461,22 +480,29 @@ func (d *backendDocker) WorkOnServers() { } func (d *backendDocker) Init() error { - ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*10) defer ctxCancel() out, err := exec.CommandContext(ctx, "docker", "info").CombinedOutput() if err != nil { return fmt.Errorf("docker command not found, or docker appears to be unreachable or down: %s", string(out)) } - for _, line := range strings.Split(string(out), "\n") { - line = strings.Trim(line, "\r\n\t ") - if !strings.HasPrefix(line, "Architecture: ") { - continue - } - arch := strings.Split(line, ": ")[1] - if strings.Contains(arch, "arm") || strings.Contains(arch, "aarch") { - d.isArm = true + switch a.opts.Config.Backend.Arch { + case "amd64": + d.isArm = false + case "arm64": + d.isArm = true + default: + for _, line := range strings.Split(string(out), "\n") { + line = strings.Trim(line, "\r\n\t ") + if !strings.HasPrefix(line, "Architecture: ") { + continue + } + arch := strings.Split(line, ": ")[1] + if strings.Contains(arch, "arm") || strings.Contains(arch, "aarch") { + d.isArm = true + } + break } - break } d.WorkOnServers() return nil @@ -527,7 +553,11 @@ func (d *backendDocker) VacuumTemplate(v backendVersion) error { if err := d.versionToReal(&v); err != nil { return err } - templName := fmt.Sprintf("aerotmpl-%s-%s-%s", v.distroName, v.distroVersion, v.aerospikeVersion) + arch := "amd64" + if v.isArm { + arch = "arm64" + } + templName := fmt.Sprintf("aerotmpl-%s-%s-%s-%s", v.distroName, v.distroVersion, v.aerospikeVersion, arch) out, err := exec.Command("docker", "stop", "-t", "1", templName).CombinedOutput() if err != nil { return fmt.Errorf("could not stop temporary template container: %s;%s", out, err) @@ -545,7 +575,11 @@ func (d *backendDocker) DeployTemplate(v backendVersion, script string, files [] if err := d.versionToReal(&v); err != nil { return err } - templName := fmt.Sprintf("aerotmpl-%s-%s-%s", v.distroName, v.distroVersion, v.aerospikeVersion) + arch := "amd64" + if v.isArm { + arch = "arm64" + } + templName := fmt.Sprintf("aerotmpl-%s-%s-%s-%s", v.distroName, v.distroVersion, v.aerospikeVersion, arch) addShutdownHandler("deployTemplate", func(os.Signal) { for len(deployTemplateShutdownMaking) > 0 { time.Sleep(time.Second) @@ -555,7 +589,7 @@ func (d *backendDocker) DeployTemplate(v backendVersion, script string, files [] defer delShutdownHandler("deployTemplate") // deploy container with os deployTemplateShutdownMaking <- 1 - out, err := exec.Command("docker", "run", "-td", "--name", templName, d.centosNaming(v)).CombinedOutput() + out, err := exec.Command("docker", "run", "-td", "--name", templName, d.imageNaming(v)).CombinedOutput() <-deployTemplateShutdownMaking if err != nil { return fmt.Errorf("could not start vanilla container: %s;%s", out, err) @@ -582,7 +616,7 @@ func (d *backendDocker) DeployTemplate(v backendVersion, script string, files [] return fmt.Errorf("failed stopping container: %s;%s", out, err) } // docker container commit container_name dist_ver:aeroVer - templImg := fmt.Sprintf(dockerNameHeader+"%s_%s:%s", v.distroName, v.distroVersion, v.aerospikeVersion) + templImg := fmt.Sprintf(dockerNameHeader+"%s_%s_%s:%s", v.distroName, v.distroVersion, arch, v.aerospikeVersion) out, err = exec.Command("docker", "container", "commit", templName, templImg).CombinedOutput() if err != nil { return fmt.Errorf("failed to commit container to image: %s;%s", out, err) @@ -599,10 +633,18 @@ func (d *backendDocker) TemplateDestroy(v backendVersion) error { if v.distroName == "el" { v.distroName = "centos" } - name := fmt.Sprintf(dockerNameHeader+"%s_%s:%s", v.distroName, v.distroVersion, v.aerospikeVersion) + arch := "amd64" + if v.isArm { + arch = "arm64" + } + name := fmt.Sprintf(dockerNameHeader+"%s_%s_%s:%s", v.distroName, v.distroVersion, arch, v.aerospikeVersion) out, err := exec.Command("docker", "image", "list", "--format", "{{json .ID}}", fmt.Sprintf("--filter=reference=%s", name)).CombinedOutput() - if err != nil { - return fmt.Errorf("failed to get image list: %s;%s", string(out), err) + if err != nil || len(strings.Trim(string(out), "\t\r\n ")) == 0 { + name = fmt.Sprintf(dockerNameHeader+"%s_%s:%s", v.distroName, v.distroVersion, v.aerospikeVersion) + out, err = exec.Command("docker", "image", "list", "--format", "{{json .ID}}", fmt.Sprintf("--filter=reference=%s", name)).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to get image list: %s;%s", string(out), err) + } } imageId := strings.Trim(string(out), "\"' \n\r") out, err = exec.Command("docker", "rmi", imageId).CombinedOutput() @@ -762,9 +804,13 @@ func (d *backendDocker) DeployCluster(v backendVersion, name string, nodeCount i for _, newlabel := range extra.labels { exposeList = append(exposeList, "--label", newlabel) } - tmplName := fmt.Sprintf(dockerNameHeader+"%s_%s:%s", v.distroName, v.distroVersion, v.aerospikeVersion) + arch := "amd64" + if v.isArm { + arch = "arm64" + } + tmplName := fmt.Sprintf(dockerNameHeader+"%s_%s_%s:%s", v.distroName, v.distroVersion, arch, v.aerospikeVersion) if d.client { - tmplName = d.centosNaming(v) + tmplName = d.imageNaming(v) } if extra.dockerHostname { exposeList = append(exposeList, "--hostname", name+"-"+strconv.Itoa(node)) @@ -808,17 +854,34 @@ func (d *backendDocker) DeployCluster(v backendVersion, name string, nodeCount i return nil } -func (d *backendDocker) centosNaming(v backendVersion) (templName string) { - if v.distroName != "centos" { - return fmt.Sprintf("%s:%s", v.distroName, v.distroVersion) - } - switch v.distroVersion { - case "6": - return "quay.io/centos/centos:6" - case "7": - return "quay.io/centos/centos:7" +func (d *backendDocker) imageNaming(v backendVersion) (templName string) { + switch v.distroName { + case "centos": + switch v.distroVersion { + case "6": + return "quay.io/centos/centos:6" + case "7": + return "quay.io/centos/centos:7" + default: + switch a.opts.Config.Backend.Arch { + case "amd64": + return "quay.io/centos/amd64:stream" + v.distroVersion + case "arm64": + return "quay.io/centos/arm64v8:stream" + v.distroVersion + default: + return "quay.io/centos/centos:stream" + v.distroVersion + } + } + case "ubuntu", "debian": + switch a.opts.Config.Backend.Arch { + case "amd64": + return fmt.Sprintf("amd64/%s:%s", v.distroName, v.distroVersion) + case "arm64": + return fmt.Sprintf("arm64v8/%s:%s", v.distroName, v.distroVersion) + } + fallthrough default: - return "quay.io/centos/centos:stream" + v.distroVersion + return fmt.Sprintf("%s:%s", v.distroName, v.distroVersion) } } diff --git a/src/cmdConfig.go b/src/cmdConfig.go index b1a61caa..484c6993 100644 --- a/src/cmdConfig.go +++ b/src/cmdConfig.go @@ -38,6 +38,7 @@ type configBackendCmd struct { Region string `short:"r" long:"region" description:"AWS backend: override default aws configured region" default:""` AWSProfile string `short:"P" long:"aws-profile" description:"AWS backend: provide a profile to use; setting this ignores the AWS_PROFILE env variable"` Project string `short:"o" long:"project" description:"GCP backend: override default gcp configured project" default:""` + Arch string `short:"a" long:"docker-arch" description:"set to either amd64 or arm64 to force a particular architecture on docker; see https://github.com/aerospike/aerolab/tree/master/docs/docker_multiarch.md"` TmpDir flags.Filename `short:"d" long:"temp-dir" description:"use a non-default temporary directory" default:""` Help helpCmd `command:"help" subcommands-optional:"true" description:"Print help"` typeSet string @@ -61,6 +62,12 @@ func (c *configBackendCmd) Execute(args []string) error { a.forceFileOptional = true return nil } + if c.Arch != "" && c.Arch != "amd64" && c.Arch != "arm64" && c.Arch != "unset" { + return errors.New("docker-arch must be one of: unset, amd64, arm64") + } + if c.Arch == "unset" { + c.Arch = "" + } if c.typeSet != "" { err := c.ExecTypeSet(args) if err != nil { @@ -74,6 +81,9 @@ func (c *configBackendCmd) Execute(args []string) error { if c.Type == "gcp" { fmt.Printf("Config.Backend.Project = %s\n", c.Project) } + if c.Type == "docker" && c.Arch != "" { + fmt.Printf("Config.Backend.Arch = %s\n", c.Arch) + } fmt.Printf("Config.Backend.TmpDir = %s\n", c.TmpDir) return nil }