Build Swift SDKs for a Linux host if the --host parameter matches a Linux OS (*-unknown-linux-gnu)

.gitignore

@@ -1,5 +1,5 @@
 .DS_Store
-/.build
+.build*
 /.index-build
 /Packages
 /*.xcodeproj

Package.resolved

@@ -5,8 +5,17 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/swift-server/async-http-client.git",
       "state" : {
-        "revision" : "16f7e62c08c6969899ce6cc277041e868364e5cf",
-        "version" : "1.19.0"
+        "revision" : "2119f0d9cc1b334e25447fe43d3693c0e60e6234",
+        "version" : "1.24.0"
+      }
+    },
+    {
+      "identity" : "swift-algorithms",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-algorithms.git",
+      "state" : {
+        "revision" : "f6919dfc309e7f1b56224378b11e28bab5bccc42",
+        "version" : "1.2.0"
       }
     },
     {
@@ -18,6 +27,15 @@
         "version" : "1.5.0"
       }
     },
+    {
+      "identity" : "swift-asn1",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-asn1.git",
+      "state" : {
+        "revision" : "7faebca1ea4f9aaf0cda1cef7c43aecd2311ddf6",
+        "version" : "1.3.0"
+      }
+    },
     {
       "identity" : "swift-async-algorithms",
       "kind" : "remoteSourceControl",
@@ -41,89 +59,98 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-collections.git",
       "state" : {
-        "revision" : "9bf03ff58ce34478e66aaee630e491823326fd06",
-        "version" : "1.1.3"
+        "revision" : "671108c96644956dddcd89dd59c203dcdb36cec7",
+        "version" : "1.1.4"
       }
     },
     {
       "identity" : "swift-crypto",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-crypto.git",
       "state" : {
-        "revision" : "b51f1d6845b353a2121de1c6a670738ec33561a6",
-        "version" : "3.1.0"
+        "revision" : "ff0f781cf7c6a22d52957e50b104f5768b50c779",
+        "version" : "3.10.0"
       }
     },
     {
       "identity" : "swift-http-types",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-http-types",
       "state" : {
-        "revision" : "12358d55a3824bd5fed310b999ea8cf83a9a1a65",
-        "version" : "1.0.3"
+        "revision" : "ef18d829e8b92d731ad27bb81583edd2094d1ce3",
+        "version" : "1.3.1"
       }
     },
     {
       "identity" : "swift-log",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-log.git",
       "state" : {
-        "revision" : "9cb486020ebf03bfa5b5df985387a14a98744537",
-        "version" : "1.6.1"
+        "revision" : "96a2f8a0fa41e9e09af4585e2724c4e825410b91",
+        "version" : "1.6.2"
       }
     },
     {
       "identity" : "swift-nio",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-nio.git",
       "state" : {
-        "revision" : "4c4453b489cf76e6b3b0f300aba663eb78182fad",
-        "version" : "2.70.0"
+        "revision" : "dca6594f65308c761a9c409e09fbf35f48d50d34",
+        "version" : "2.77.0"
       }
     },
     {
       "identity" : "swift-nio-extras",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-nio-extras.git",
       "state" : {
-        "revision" : "363da63c1966405764f380c627409b2f9d9e710b",
-        "version" : "1.21.0"
+        "revision" : "2e9746cfc57554f70b650b021b6ae4738abef3e6",
+        "version" : "1.24.1"
       }
     },
     {
       "identity" : "swift-nio-http2",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-nio-http2.git",
       "state" : {
-        "revision" : "0904bf0feb5122b7e5c3f15db7df0eabe623dd87",
-        "version" : "1.30.0"
+        "revision" : "170f4ca06b6a9c57b811293cebcb96e81b661310",
+        "version" : "1.35.0"
       }
     },
     {
       "identity" : "swift-nio-ssl",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-nio-ssl.git",
       "state" : {
-        "revision" : "4fb7ead803e38949eb1d6fabb849206a72c580f3",
-        "version" : "2.23.0"
+        "revision" : "c7e95421334b1068490b5d41314a50e70bab23d1",
+        "version" : "2.29.0"
       }
     },
     {
       "identity" : "swift-nio-transport-services",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-nio-transport-services.git",
       "state" : {
-        "revision" : "e7403c35ca6bb539a7ca353b91cc2d8ec0362d58",
-        "version" : "1.19.0"
+        "revision" : "bbd5e63cf949b7db0c9edaf7a21e141c52afe214",
+        "version" : "1.23.0"
+      }
+    },
+    {
+      "identity" : "swift-numerics",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/apple/swift-numerics.git",
+      "state" : {
+        "revision" : "0a5bc04095a675662cf24757cc0640aa2204253b",
+        "version" : "1.0.2"
       }
     },
     {
       "identity" : "swift-system",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-system",
       "state" : {
-        "revision" : "d2ba781702a1d8285419c15ee62fd734a9437ff5",
-        "version" : "1.3.2"
+        "revision" : "c8a44d836fe7913603e246acab7c528c2e780168",
+        "version" : "1.4.0"
       }
     }
   ],

README.md

@@ -37,13 +37,16 @@ Linux distributions officially supported by the Swift project.
 | -:             | :-                        | :-                          |
 | macOS (arm64)  | ✅ macOS 13.0+            | ❌                         |
 | macOS (x86_64) | ✅ macOS 13.0+[^1]        | ❌                         |
-| Ubuntu         | ⚠️ (WIP)                  | ✅ 20.04 / 22.04           |
-| RHEL           | ⚠️ (WIP)                  | ✅ UBI 9                   |
-
+| Ubuntu         | ✅ 20.04+                 | ✅ 20.04 / 22.04           |
+| RHEL           | ✅ Fedora 39[^2], UBI 9   | ✅ UBI 9                   |
+| Amazon Linux 2 | ✅ Supported              | ✅ Supported[^3]           |
+| Debian 12      | ✅ Supported[^2]          | ✅ Supported[^2][^3]       |
 
 [^1]: Since LLVM project doesn't provide pre-built binaries of `lld` for macOS on x86_64, it will be automatically built
 from sources by the generator, which will increase its run by at least 15 minutes on recent hardware. You will also
 need CMake and Ninja preinstalled (e.g. via `brew install cmake ninja`).
+[^2]: These distributions are only supported by Swift 5.10.1 and later on both the host and target.
+[^3]: These versions are technically supported but require custom commands and a Docker container to build the Swift SDK, as the generator will not download dependencies for these distributions automatically. See [issue 138](https://github.com/swiftlang/swift-sdk-generator/issues/138).
 
 ## How to use it
 

Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift

@@ -51,16 +51,31 @@ struct DownloadableArtifacts: Sendable {
     self.versions = versions
     self.paths = paths
 
-    self.hostSwift = .init(
-      remoteURL: versions.swiftDownloadURL(
-        subdirectory: "xcode",
-        platform: "osx",
-        fileExtension: "pkg"
-      ),
-      localPath: paths.artifactsCachePath
-        .appending("host_swift_\(versions.swiftVersion)_\(hostTriple.triple).pkg"),
-      isPrebuilt: true
-    )
+    if hostTriple.os == .linux {
+      // Amazon Linux 2 is chosen for its best compatibility with all Swift-supported Linux hosts
+      let linuxArchSuffix = hostTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : ""
+      self.hostSwift = .init(
+        remoteURL: versions.swiftDownloadURL(
+          subdirectory: "amazonlinux2\(linuxArchSuffix)",
+          platform: "amazonlinux2\(linuxArchSuffix)",
+          fileExtension: "tar.gz"
+        ),
+        localPath: paths.artifactsCachePath
+          .appending("host_swift_\(versions.swiftVersion)_\(hostTriple.triple).tar.gz"),
+        isPrebuilt: true
+      )
+    } else {
+      self.hostSwift = .init(
+        remoteURL: versions.swiftDownloadURL(
+          subdirectory: "xcode",
+          platform: "osx",
+          fileExtension: "pkg"
+        ),
+        localPath: paths.artifactsCachePath
+          .appending("host_swift_\(versions.swiftVersion)_\(hostTriple.triple).pkg"),
+        isPrebuilt: true
+      )
+    }
 
     self.hostLLVM = .init(
       remoteURL: URL(

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift

@@ -114,6 +114,7 @@ extension SwiftSDKGenerator {
       report(downloadedFiles: downloadedFiles)
 
       for fileName in urls.map(\.lastPathComponent) {
+        print("Extracting \(fileName)...")
         try await fs.unpack(file: tmpDir.appending(fileName), into: sdkDirPath)
       }
     }

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift

@@ -49,9 +49,13 @@ extension SwiftSDKGenerator {
   }
 
   func symlinkClangHeaders() throws {
-    try self.createSymlink(
-      at: self.pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static/clang"),
-      pointingTo: "../swift/clang"
-    )
+    let swiftStaticClangPath = self.pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static/clang")
+    if !doesFileExist(at: swiftStaticClangPath) {
+      logGenerationStep("Symlinking clang headers...")
+      try self.createSymlink(
+        at: self.pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static/clang"),
+        pointingTo: "../swift/clang"
+      )
+    }
   }
 }

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift

@@ -14,7 +14,7 @@ import Helpers
 
 import struct SystemPackage.FilePath
 
-let unusedDarwinPlatforms = [
+let unusedTargetPlatforms = [
   "appletvos",
   "appletvsimulator",
   "embedded",
@@ -34,11 +34,13 @@ let unusedHostBinaries = [
   "swift-format",
   "swift-package",
   "swift-package-collection",
+  "lldb*",
 ]
 
 let unusedHostLibraries = [
   "sourcekitd.framework",
   "libsourcekitdInProc.so",
+  "liblldb.so*",
 ]
 
 extension SwiftSDKGenerator {
@@ -49,23 +51,32 @@ extension SwiftSDKGenerator {
     try self.createDirectoryIfNeeded(at: pathsConfiguration.toolchainDirPath)
 
     let excludes =
-      unusedDarwinPlatforms.map { "--exclude usr/lib/swift/\($0)" } +
-      unusedDarwinPlatforms.map { "--exclude usr/lib/swift_static/\($0)" } +
+      unusedTargetPlatforms.map { "--exclude usr/lib/swift/\($0)" } +
+      unusedTargetPlatforms.map { "--exclude usr/lib/swift_static/\($0)" } +
       unusedHostBinaries.map { "--exclude usr/bin/\($0)" } +
       unusedHostLibraries.map { "--exclude usr/lib/\($0)" }
 
-    try await Shell.run(
-      #"""
-      tar -x --to-stdout -f \#(hostSwiftPackagePath) \*.pkg/Payload |
-      tar -C "\#(pathsConfiguration.toolchainDirPath)" -x \#(excludes.joined(separator: " ")) --include usr
-      """#,
-      shouldLogCommands: isVerbose
-    )
+    if hostSwiftPackagePath.string.contains("tar.gz") {
+      try await Shell.run(
+        #"""
+        tar -xzf \#(hostSwiftPackagePath) -C "\#(pathsConfiguration.toolchainDirPath)" -x \#(excludes.joined(separator: " ")) --strip-components=1
+        """#,
+        shouldLogCommands: isVerbose
+      )
+    } else {
+      try await Shell.run(
+        #"""
+        tar -x --to-stdout -f \#(hostSwiftPackagePath) \*.pkg/Payload |
+        tar -C "\#(pathsConfiguration.toolchainDirPath)" -x \#(excludes.joined(separator: " ")) --include usr
+        """#,
+        shouldLogCommands: isVerbose
+      )
+    }
   }
 
   func removeToolchainComponents(
     _ packagePath: FilePath,
-    platforms: [String] = unusedDarwinPlatforms,
+    platforms: [String] = unusedTargetPlatforms,
     libraries: [String] = unusedHostLibraries,
     binaries: [String] = unusedHostBinaries
   ) async throws {

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift

@@ -253,7 +253,9 @@ public actor SwiftSDKGenerator {
     let isVerbose = self.isVerbose
     try await self.inTemporaryDirectory { _, tmp in
       try await Shell.run(#"cd "\#(tmp)" && ar -x "\#(debFile)""#, shouldLogCommands: isVerbose)
-      try await print(Shell.readStdout("ls \(tmp)"))
+      if isVerbose {
+        try await print(Shell.readStdout("ls \(tmp)"))
+      }
 
       try await Shell.run(
         #"tar -C "\#(directoryPath)" -xf "\#(tmp)"/data.tar.*"#,

Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift

@@ -164,7 +164,9 @@ public struct LinuxRecipe: SwiftSDKRecipe {
 
   func itemsToDownload(from artifacts: DownloadableArtifacts) -> [DownloadableArtifacts.Item] {
     var items: [DownloadableArtifacts.Item] = []
-    if self.hostSwiftSource != .preinstalled && !self.versionsConfiguration.swiftVersion.hasPrefix("6.0") {
+    if self.hostSwiftSource != .preinstalled
+      && self.mainHostTriple.os != .linux
+      && !self.versionsConfiguration.swiftVersion.hasPrefix("6.0") {
       items.append(artifacts.hostLLVM)
     }
 
@@ -279,12 +281,12 @@ public struct LinuxRecipe: SwiftSDKRecipe {
     try await generator.fixAbsoluteSymlinks(sdkDirPath: sdkDirPath)
 
     if self.hostSwiftSource != .preinstalled {
-      if !self.versionsConfiguration.swiftVersion.hasPrefix("6.0") {
+      if self.mainHostTriple.os != .linux && !self.versionsConfiguration.swiftVersion.hasPrefix("6.0") {
         try await generator.prepareLLDLinker(engine, llvmArtifact: downloadableArtifacts.hostLLVM)
       }
 
       if self.versionsConfiguration.swiftVersion.hasPrefix("5.9") ||
-          self.versionsConfiguration.swiftVersion .hasPrefix("5.10") {
+          self.versionsConfiguration.swiftVersion.hasPrefix("5.10") {
         try await generator.symlinkClangHeaders()
       }
 

Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift

@@ -115,7 +115,7 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe {
       }()
       try await generator.removeToolchainComponents(
         pathsConfiguration.toolchainDirPath,
-        platforms: unusedDarwinPlatforms + ["embedded"],
+        platforms: unusedTargetPlatforms,
         libraries: unusedHostLibraries + liblldbNames,
         binaries: unusedHostBinaries + ["lldb", "lldb-argdumper", "lldb-server"]
       )