From f6e22dc96747e8d36452da6f0bb2da5b2bee57fd Mon Sep 17 00:00:00 2001 From: Michel Laterman <82832767+michel-laterman@users.noreply.github.com> Date: Wed, 5 Mar 2025 08:12:52 -0800 Subject: [PATCH] FIPS Build (#42402) Enable FIPS compliant builds when the env var FIPS=true is set. Artifacts are built with the microsfot/go toolchain with the env var GOEXPERIMENT=systemcrypto and the build tag "-tags=requirefips". In order to run the resulting binary, the system must have a FIPS compliant crypto provider. (cherry picked from commit 83e0ec0cd6e887eac7a9c79d4c3ea156ff324d1a) --- dev-tools/mage/build.go | 44 ++++++++++++++- dev-tools/mage/build_test.go | 72 ++++++++++++++++++++++++ dev-tools/mage/common.go | 6 ++ dev-tools/mage/crossbuild.go | 4 ++ dev-tools/mage/dockerbuilder.go | 3 + dev-tools/mage/gotest.go | 4 +- dev-tools/mage/pkg.go | 11 +++- dev-tools/mage/pkgtypes.go | 7 ++- dev-tools/mage/settings.go | 11 +++- dev-tools/packaging/package_test.go | 87 ++++++++++++++++++++++++++++- 10 files changed, 236 insertions(+), 13 deletions(-) create mode 100644 dev-tools/mage/build_test.go diff --git a/dev-tools/mage/build.go b/dev-tools/mage/build.go index 263299671fd7..24f78e493e34 100644 --- a/dev-tools/mage/build.go +++ b/dev-tools/mage/build.go @@ -24,6 +24,7 @@ import ( "log" "os" "path/filepath" + "regexp" "strings" "github.com/josephspurrier/goversioninfo" @@ -46,6 +47,39 @@ type BuildArgs struct { WinMetadata bool // Add resource metadata to Windows binaries (like add the version number to the .exe properties). } +// buildTagRE is a regexp to match strings like "-tags=abcd" +// but does not match "-tags= " +var buildTagRE = regexp.MustCompile(`-tags=([\S]+)?`) + +// ParseBuildTags returns the ExtraFlags param where all flags that are go build tags are joined by a comma. +// +// For example if given -someflag=val1 -tags=buildtag1 -tags=buildtag2 +// It will return -someflag=val1 -tags=buildtag1,buildtag2 +func (b BuildArgs) ParseBuildTags() []string { + flags := make([]string, 0) + if len(b.ExtraFlags) == 0 { + return flags + } + + buildTags := make([]string, 0) + for _, flag := range b.ExtraFlags { + if buildTagRE.MatchString(flag) { + arr := buildTagRE.FindStringSubmatch(flag) + if len(arr) != 2 || arr[1] == "" { + log.Printf("Parsing buildargs.ExtraFlags found strange flag %q ignoring value", flag) + continue + } + buildTags = append(buildTags, arr[1]) + } else { + flags = append(flags, flag) + } + } + if len(buildTags) > 0 { + flags = append(flags, "-tags="+strings.Join(buildTags, ",")) + } + return flags +} + // DefaultBuildArgs returns the default BuildArgs for use in builds. func DefaultBuildArgs() BuildArgs { args := BuildArgs{ @@ -74,6 +108,10 @@ func DefaultBuildArgs() BuildArgs { // Remove all file system paths from the compiled executable, to improve build reproducibility args.ExtraFlags = append(args.ExtraFlags, "-trimpath") } + if FIPSBuild { + args.ExtraFlags = append(args.ExtraFlags, "-tags=requirefips") + args.CGO = true + } return args } @@ -175,6 +213,10 @@ func Build(params BuildArgs) error { if params.CGO { cgoEnabled = "1" } + if FIPSBuild { + cgoEnabled = "1" + env["GOEXPERIMENT"] = "systemcrypto" + } env["CGO_ENABLED"] = cgoEnabled // Spec @@ -186,7 +228,7 @@ func Build(params BuildArgs) error { if params.BuildMode != "" { args = append(args, "-buildmode", params.BuildMode) } - args = append(args, params.ExtraFlags...) + args = append(args, params.ParseBuildTags()...) // ldflags ldflags := params.LDFlags diff --git a/dev-tools/mage/build_test.go b/dev-tools/mage/build_test.go new file mode 100644 index 000000000000..39ba435cf1a6 --- /dev/null +++ b/dev-tools/mage/build_test.go @@ -0,0 +1,72 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mage + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_BuildArgs_ParseBuildTags(t *testing.T) { + tests := []struct { + name string + input []string + expect []string + }{{ + name: "no flags", + input: nil, + expect: []string{}, + }, { + name: "multiple flags with no tags", + input: []string{"-a", "-b", "-key=value"}, + expect: []string{"-a", "-b", "-key=value"}, + }, { + name: "one build tag", + input: []string{"-tags=example"}, + expect: []string{"-tags=example"}, + }, { + name: "multiple build tags", + input: []string{"-tags=example", "-tags=test"}, + expect: []string{"-tags=example,test"}, + }, { + name: "joined build tags", + input: []string{"-tags=example,test"}, + expect: []string{"-tags=example,test"}, + }, { + name: "multiple build tags with other flags", + input: []string{"-tags=example", "-tags=test", "-key=value", "-a"}, + expect: []string{"-key=value", "-a", "-tags=example,test"}, + }, { + name: "incorrectly formatted tag", + input: []string{"-tags= example"}, + expect: []string{}, + }, { + name: "incorrectly formatted tag with valid tag", + input: []string{"-tags= example", "-tags=test"}, + expect: []string{"-tags=test"}, + }} + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + args := BuildArgs{ExtraFlags: tc.input} + flags := args.ParseBuildTags() + assert.EqualValues(t, tc.expect, flags) + }) + } +} diff --git a/dev-tools/mage/common.go b/dev-tools/mage/common.go index 01c683c11e7b..07dffd15fba8 100644 --- a/dev-tools/mage/common.go +++ b/dev-tools/mage/common.go @@ -511,6 +511,12 @@ func untar(sourceFile, destinationDir string) error { return err } case tar.TypeReg: + // create containing folder if it doesn't exist yet + targetContainingDir := filepath.Dir(filepath.FromSlash(path)) + if mkDirErr := os.MkdirAll(targetContainingDir, 0755); mkDirErr != nil { + return fmt.Errorf("creating container directory for file %s: %w", header.Name, mkDirErr) + } + writer, err := os.Create(path) if err != nil { return err diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index ede35e08d8a3..7c73bbb34fcc 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -248,6 +248,9 @@ func CrossBuildImage(platform string) (string, error) { if err != nil { return "", err } + if FIPSBuild { + tagSuffix += "-fips" + } return BeatsCrossBuildImage + ":" + goVersion + "-" + tagSuffix, nil } @@ -331,6 +334,7 @@ func (b GolangCrossBuilder) Build() error { "--env", "MAGEFILE_VERBOSE="+verbose, "--env", "MAGEFILE_TIMEOUT="+EnvOr("MAGEFILE_TIMEOUT", ""), "--env", fmt.Sprintf("SNAPSHOT=%v", Snapshot), + "--env", fmt.Sprintf("FIPS=%v", FIPSBuild), "-v", repoInfo.RootDir+":"+mountPoint, "-w", workDir, ) diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 2066670dc808..ccc532ad0591 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -194,6 +194,9 @@ func (b *dockerBuilder) dockerBuild() (string, error) { if b.Snapshot { tag = tag + "-SNAPSHOT" } + if b.FIPS { + tag = tag + "-fips" + } if repository, _ := b.ExtraVars["repository"]; repository != "" { tag = fmt.Sprintf("%s/%s", repository, tag) } diff --git a/dev-tools/mage/gotest.go b/dev-tools/mage/gotest.go index ecc8f277b941..f1b977b6ac65 100644 --- a/dev-tools/mage/gotest.go +++ b/dev-tools/mage/gotest.go @@ -292,9 +292,9 @@ func GoTest(ctx context.Context, params GoTestArgs) error { } } if len(params.Tags) > 0 { - params := strings.Join(params.Tags, " ") + params := strings.Join(params.Tags, ",") if params != "" { - testArgs = append(testArgs, "-tags", params) + testArgs = append(testArgs, "-tags="+params) } } if params.CoverageProfileFile != "" { diff --git a/dev-tools/mage/pkg.go b/dev-tools/mage/pkg.go index 757f857265f4..6d6e2aff535d 100644 --- a/dev-tools/mage/pkg.go +++ b/dev-tools/mage/pkg.go @@ -88,6 +88,7 @@ func Package() error { spec.OS = target.GOOS() spec.Arch = packageArch spec.Snapshot = Snapshot + spec.FIPS = FIPSBuild spec.evalContext = map[string]interface{}{ "GOOS": target.GOOS(), "GOARCH": target.GOARCH(), @@ -246,11 +247,11 @@ type packageBuilder struct { } func (b packageBuilder) Build() error { - fmt.Printf(">> package: Building %v type=%v for platform=%v\n", b.Spec.Name, b.Type, b.Platform.Name) + fmt.Printf(">> package: Building %v type=%v for platform=%v fips=%v\n", b.Spec.Name, b.Type, b.Platform.Name, b.Spec.FIPS) log.Printf("Package spec: %+v", b.Spec) if err := b.Type.Build(b.Spec); err != nil { - return fmt.Errorf("failed building %v type=%v for platform=%v: %w", - b.Spec.Name, b.Type, b.Platform.Name, err) + return fmt.Errorf("failed building %v type=%v for platform=%v fips=%v: %w", + b.Spec.Name, b.Type, b.Platform.Name, b.Spec.FIPS, err) } return nil } @@ -344,6 +345,10 @@ func TestPackages(options ...TestPackagesOption) error { args = append(args, "-root-owner") } + if FIPSBuild { + args = append(args, "-fips") + } + args = append(args, "-files", MustExpand("{{.PWD}}/build/distributions/*")) if out, err := goTest(args...); err != nil { diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index 249612f8feb8..4ef64d1959d6 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -47,7 +47,7 @@ const ( packageStagingDir = "build/package" // defaultBinaryName specifies the output file for zip and tar.gz. - defaultBinaryName = "{{.Name}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}" + defaultBinaryName = "{{.Name}}{{if .FIPS}}-fips{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}" ) // PackageType defines the file format of the package (e.g. zip, rpm, etc). @@ -79,6 +79,7 @@ type PackageSpec struct { Arch string `yaml:"arch,omitempty"` Vendor string `yaml:"vendor,omitempty"` Snapshot bool `yaml:"snapshot"` + FIPS bool `yaml:"fips"` Version string `yaml:"version,omitempty"` License string `yaml:"license,omitempty"` URL string `yaml:"url,omitempty"` @@ -528,7 +529,7 @@ func (s PackageSpec) rootDir() string { // NOTE: This uses .BeatName instead of .Name because we wanted the internal // directory to not include "-oss". - return s.MustExpand("{{.BeatName}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}") + return s.MustExpand("{{.BeatName}}{{if .FIPS}}-fips{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}") } // PackageZip packages a zip file. @@ -727,7 +728,7 @@ func runFPM(spec PackageSpec, packageType PackageType) error { } defer os.Remove(inputTar) - outputFile, err := spec.Expand("{{.Name}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.Arch}}") + outputFile, err := spec.Expand("{{.Name}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.Arch}}{{if .FIPS}}-fips{{end}}") if err != nil { return err } diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index 22c081351e8b..b1394e66c4bd 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -79,8 +79,9 @@ var ( BeatProjectType ProjectType - Snapshot bool - DevBuild bool + Snapshot bool + DevBuild bool + FIPSBuild bool versionQualified bool versionQualifier string @@ -128,6 +129,11 @@ func init() { panic(fmt.Errorf("failed to parse DEV env value: %w", err)) } + FIPSBuild, err = strconv.ParseBool(EnvOr("FIPS", "false")) + if err != nil { + panic(fmt.Errorf("failed to parse FIPS env value: %w", err)) + } + versionQualifier, versionQualified = os.LookupEnv("VERSION_QUALIFIER") } @@ -179,6 +185,7 @@ func varMap(args ...map[string]interface{}) map[string]interface{} { "BeatUser": BeatUser, "Snapshot": Snapshot, "DEV": DevBuild, + "FIPS": FIPSBuild, "Qualifier": versionQualifier, "CI": CI, } diff --git a/dev-tools/packaging/package_test.go b/dev-tools/packaging/package_test.go index 6a9a72a8facd..7acc16affc7e 100644 --- a/dev-tools/packaging/package_test.go +++ b/dev-tools/packaging/package_test.go @@ -27,6 +27,8 @@ import ( "bytes" "compress/gzip" "context" + "debug/buildinfo" + "debug/elf" "encoding/json" "errors" "flag" @@ -45,6 +47,9 @@ import ( "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/strslice" "github.com/docker/docker/client" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/dev-tools/mage" ) const ( @@ -75,6 +80,7 @@ var ( monitorsd = flag.Bool("monitors.d", false, "check monitors.d folder contents") rootOwner = flag.Bool("root-owner", false, "expect root to own package files") rootUserContainer = flag.Bool("root-user-container", false, "expect root in container user") + fips = flag.Bool("fips", false, "check agent binary for FIPS compliance") ) func TestRPM(t *testing.T) { @@ -96,7 +102,15 @@ func TestTar(t *testing.T) { // Regexp matches *-arch.tar.gz, but not *-arch.docker.tar.gz tars := getFiles(t, regexp.MustCompile(`-\w+\.tar\.gz$`)) for _, tar := range tars { - checkTar(t, tar) + checkTar(t, tar, false) + } +} + +func TestFIPSTar(t *testing.T) { + // Regexp matches *-arch.tar.gz, but not *-arch.docker.tar.gz + tars := getFiles(t, regexp.MustCompile(`-\w+-fips\.tar\.gz$`)) + for _, tar := range tars { + checkTar(t, tar, *fips) } } @@ -159,7 +173,7 @@ func checkDeb(t *testing.T, file string, buf *bytes.Buffer) { checkSystemdUnitPermissions(t, p) } -func checkTar(t *testing.T, file string) { +func checkTar(t *testing.T, file string, fipsCheck bool) { p, err := readTar(file) if err != nil { t.Error(err) @@ -174,6 +188,29 @@ func checkTar(t *testing.T, file string) { checkModulesPermissions(t, p) checkModulesOwner(t, p, true) checkLicensesPresent(t, "", p) + if fipsCheck { + t.Run(p.Name+"_fips_test", func(t *testing.T) { + extractDir := t.TempDir() + t.Logf("Extracting file %s into %s", file, extractDir) + err := mage.Extract(file, extractDir) + require.NoError(t, err) + containingDir := strings.TrimSuffix(filepath.Base(file), ".tar.gz") + beatName := extractBeatNameFromTarName(t, filepath.Base(file)) + checkFIPS(t, beatName, filepath.Join(extractDir, containingDir)) + }) + } +} + +func extractBeatNameFromTarName(t *testing.T, fileName string) string { + // TODO check if cutting at the first '-' is an acceptable shortcut + t.Logf("Extracting beat name from filename %s", fileName) + const sep = "-" + beatName, _, found := strings.Cut(fileName, sep) + if !found { + t.Logf("separator %s not found in filename %s: beatName may be incorrect", sep, fileName) + } + + return beatName } func checkZip(t *testing.T, file string) { @@ -773,6 +810,52 @@ func readTarContents(tarName string, data io.Reader) (*packageFile, error) { return p, nil } +func checkFIPS(t *testing.T, beatName, path string) { + t.Logf("Checking %s for FIPS compliance", beatName) + binaryPath := filepath.Join(path, beatName) // TODO eventually we'll need to support checking a .exe + require.FileExistsf(t, binaryPath, "Unable to find beat executable %s", binaryPath) + + info, err := buildinfo.ReadFile(binaryPath) + require.NoError(t, err) + + foundTags := false + foundExperiment := false + for _, setting := range info.Settings { + switch setting.Key { + case "-tags": + foundTags = true + require.Contains(t, setting.Value, "requirefips") + continue + case "GOEXPERIMENT": + foundExperiment = true + require.Contains(t, setting.Value, "systemcrypto") + continue + } + } + + require.True(t, foundTags, "Did not find -tags within binary version information") + require.True(t, foundExperiment, "Did not find GOEXPERIMENT within binary version information") + + // TODO only elf is supported at the moment, in the future we will need to use macho (darwin) and pe (windows) + f, err := elf.Open(binaryPath) + require.NoError(t, err, "unable to open ELF file") + + symbols, err := f.Symbols() + if err != nil { + t.Logf("no symbols present in %q: %v", binaryPath, err) + return + } + + hasOpenSSL := false + for _, symbol := range symbols { + if strings.Contains(symbol.Name, "OpenSSL_version") { + hasOpenSSL = true + break + } + } + require.True(t, hasOpenSSL, "unable to find OpenSSL_version symbol") +} + // inspector is a file contents inspector. It vets the contents of the file // within a package for a requirement and returns an error if it is not met. type inspector func(pkg, file string, contents io.Reader) error