Run integration tests with both Scala 3 LTS & Next versions

.github/workflows/ci.yml

@@ -129,6 +129,31 @@ jobs:
         name: test-results-jvm-tests-3
         path: test-report.xml
 
+  jvm-tests-4:
+    timeout-minutes: 120
+    runs-on: "ubuntu-20.04"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+      - uses: VirtusLab/scala-cli-setup@5b1a6c5ca98b5642996812781f6ad9b2d5935f85
+        with:
+          jvm: "temurin:17"
+      - name: JVM integration tests
+        run: ./mill -i integration.test.jvm
+        env:
+          SCALA_CLI_IT_GROUP: 4
+      - name: Convert Mill test reports to JUnit XML format
+        if: success() || failure()
+        run: .github/scripts/generate-junit-reports.sc jvm-tests-4 'Scala CLI JVM Tests (4)' test-report.xml out/
+      - name: Upload test report
+        uses: actions/upload-artifact@v3
+        if: success() || failure()
+        with:
+          name: test-results-jvm-tests-4
+          path: test-report.xml
+
   generate-linux-launcher:
     timeout-minutes: 120
     runs-on: "ubuntu-20.04"
@@ -254,6 +279,39 @@ jobs:
         name: test-results-linux-tests-3
         path: test-report.xml
 
+  native-linux-tests-4:
+    needs: generate-linux-launcher
+    timeout-minutes: 120
+    runs-on: "ubuntu-20.04"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+      - uses: VirtusLab/scala-cli-setup@5b1a6c5ca98b5642996812781f6ad9b2d5935f85
+        with:
+          jvm: "temurin:17"
+      - uses: actions/download-artifact@v3
+        with:
+          name: linux-launchers
+          path: artifacts/
+      - name: Native integration tests
+        run: ./mill -i nativeIntegrationTests
+        env:
+          UPDATE_GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SCALA_CLI_IT_FORCED_LAUNCHER_DIRECTORY: artifacts/
+          SCALA_CLI_IT_GROUP: 4
+          SCALA_CLI_SODIUM_JNI_ALLOW: false
+      - name: Convert Mill test reports to JUnit XML format
+        if: success() || failure()
+        run: .github/scripts/generate-junit-reports.sc linux-tests-4 'Scala CLI Linux Tests (4)' test-report.xml out/
+      - name: Upload test report
+        uses: actions/upload-artifact@v3
+        if: success() || failure()
+        with:
+          name: test-results-linux-tests-4
+          path: test-report.xml
+
   generate-linux-arm64-native-launcher:
     runs-on: "macOS-m1"
     if: github.event_name == 'push'
@@ -402,6 +460,39 @@ jobs:
         name: test-results-macos-tests-3
         path: test-report.xml
 
+  native-macos-tests-4:
+    needs: generate-macos-launcher
+    timeout-minutes: 120
+    runs-on: "macOS-13"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+      - uses: VirtusLab/scala-cli-setup@5b1a6c5ca98b5642996812781f6ad9b2d5935f85
+        with:
+          jvm: "temurin:17"
+      - uses: actions/download-artifact@v3
+        with:
+          name: macos-launchers
+          path: artifacts/
+      - name: Native integration tests
+        run: ./mill -i nativeIntegrationTests
+        env:
+          UPDATE_GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SCALA_CLI_IT_FORCED_LAUNCHER_DIRECTORY: artifacts/
+          SCALA_CLI_IT_GROUP: 4
+          SCALA_CLI_SODIUM_JNI_ALLOW: false
+      - name: Convert Mill test reports to JUnit XML format
+        if: success() || failure()
+        run: .github/scripts/generate-junit-reports.sc macos-tests-4 'Scala CLI MacOS Tests (4)' test-report.xml out/
+      - name: Upload test report
+        uses: actions/upload-artifact@v3
+        if: success() || failure()
+        with:
+          name: test-results-macos-tests-4
+          path: test-report.xml
+
   generate-macos-m1-launcher:
     timeout-minutes: 120
     runs-on: "macOS-m1"
@@ -619,6 +710,47 @@ jobs:
         name: test-results-windows-tests-3
         path: test-report.xml
 
+  native-windows-tests-4:
+    needs: generate-windows-launcher
+    timeout-minutes: 120
+    runs-on: "windows-2019"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - uses: VirtusLab/scala-cli-setup@5b1a6c5ca98b5642996812781f6ad9b2d5935f85
+        with:
+          jvm: "temurin:17"
+      - name: Get latest coursier launcher
+        run: .github/scripts/get-latest-cs.sh
+        shell: bash
+      - uses: actions/download-artifact@v3
+        with:
+          name: windows-launchers
+          path: artifacts/
+      - name: Native integration tests
+        run: ./mill -i nativeIntegrationTests
+        env:
+          COURSIER_JNI: force
+          UPDATE_GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SCALA_CLI_IT_FORCED_LAUNCHER_DIRECTORY: artifacts/
+          SCALA_CLI_IT_GROUP: 4
+          SCALA_CLI_SODIUM_JNI_ALLOW: false
+      - name: Convert Mill test reports to JUnit XML format
+        if: success() || failure()
+        run:  scala-cli shebang .github/scripts/generate-junit-reports.sc windows-tests-4 'Scala CLI Windows Tests (4)' test-report.xml out/
+      - name: Upload test report
+        uses: actions/upload-artifact@v3
+        if: success() || failure()
+        with:
+          name: test-results-windows-tests-4
+          path: test-report.xml
+
   generate-mostly-static-launcher:
     timeout-minutes: 120
     runs-on: ubuntu-20.04
@@ -755,6 +887,39 @@ jobs:
         name: test-results-native-mostly-static-tests-3
         path: test-report.xml
 
+  native-mostly-static-tests-4:
+    needs: generate-mostly-static-launcher
+    timeout-minutes: 120
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+      - uses: VirtusLab/scala-cli-setup@5b1a6c5ca98b5642996812781f6ad9b2d5935f85
+        with:
+          jvm: "temurin:17"
+      - uses: actions/download-artifact@v3
+        with:
+          name: mostly-static-launchers
+          path: artifacts/
+      - name: Native integration tests
+        run: ./mill -i integration.test.nativeMostlyStatic
+        env:
+          UPDATE_GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SCALA_CLI_IT_FORCED_LAUNCHER_DIRECTORY: artifacts/
+          SCALA_CLI_IT_GROUP: 4
+          SCALA_CLI_SODIUM_JNI_ALLOW: false
+      - name: Convert Mill test reports to JUnit XML format
+        if: success() || failure()
+        run: .github/scripts/generate-junit-reports.sc native-mostly-static-tests-4 'Scala CLI Native Mostly Static Tests (4)' test-report.xml out/
+      - name: Upload test report
+        uses: actions/upload-artifact@v3
+        if: success() || failure()
+        with:
+          name: test-results-native-mostly-static-tests-4
+          path: test-report.xml
+
   generate-static-launcher:
     timeout-minutes: 120
     runs-on: ubuntu-20.04
@@ -895,6 +1060,41 @@ jobs:
         name: test-results-native-static-tests-3
         path: test-report.xml
 
+  native-static-tests-4:
+    needs: generate-static-launcher
+    timeout-minutes: 120
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+      - uses: VirtusLab/scala-cli-setup@5b1a6c5ca98b5642996812781f6ad9b2d5935f85
+        with:
+          jvm: "temurin:17"
+      - uses: actions/download-artifact@v3
+        with:
+          name: static-launchers
+          path: artifacts/
+      - name: Build docker image
+        run: .github/scripts/generate-docker-image.sh
+      - name: Native integration tests
+        run: ./mill -i integration.test.nativeStatic
+        env:
+          UPDATE_GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SCALA_CLI_IT_FORCED_LAUNCHER_DIRECTORY: artifacts/
+          SCALA_CLI_IT_GROUP: 4
+          SCALA_CLI_SODIUM_JNI_ALLOW: false
+      - name: Convert Mill test reports to JUnit XML format
+        if: success() || failure()
+        run: .github/scripts/generate-junit-reports.sc native-static-tests-4 'Scala CLI Native Static Tests (4)' test-report.xml out/
+      - name: Upload test report
+        uses: actions/upload-artifact@v3
+        if: success() || failure()
+        with:
+          name: test-results-native-static-tests-4
+          path: test-report.xml
+
   docs-tests:
     # for now, let's run those tests only on ubuntu
     runs-on: ubuntu-20.04
@@ -1039,7 +1239,7 @@ jobs:
         retention-days: 2
 
   publish:
-    needs: [unit-tests, jvm-tests-1, jvm-tests-2, jvm-tests-3, format, checks, reference-doc]
+    needs: [unit-tests, jvm-tests-1, jvm-tests-2, jvm-tests-3, jvm-tests-4, format, checks, reference-doc]
     if: github.event_name == 'push' && github.repository == 'VirtusLab/scala-cli'
     runs-on: ubuntu-20.04
     steps:
@@ -1082,22 +1282,28 @@ jobs:
       - jvm-tests-1
       - jvm-tests-2
       - jvm-tests-3
+      - jvm-tests-4
       - native-linux-tests-1
       - native-linux-tests-2
       - native-linux-tests-3
+      - native-linux-tests-4
       - native-macos-tests-1
       - native-macos-tests-2
       - native-macos-tests-3
+      - native-macos-tests-4
       - native-macos-m1-tests
       - native-windows-tests-1
       - native-windows-tests-2
       - native-windows-tests-3
+      - native-windows-tests-4
       - native-mostly-static-tests-1
       - native-mostly-static-tests-2
       - native-mostly-static-tests-3
+      - native-mostly-static-tests-4
       - native-static-tests-1
       - native-static-tests-2
       - native-static-tests-3
+      - native-static-tests-4
       - vc-redist
       - format
       - checks

build.sc

@@ -958,6 +958,7 @@ trait CliIntegration extends SbtModule with ScalaCliPublishModule with HasTests
            |  def scala212                   = "${Scala.scala212}"
            |  def scala213                   = "${Scala.scala213}"
            |  def scalaSnapshot213           = "${TestDeps.scalaSnapshot213}"
+           |  def scala3LtsPrefix            = "${Scala.scala3LtsPrefix}"
            |  def scala3Lts                  = "${Scala.scala3Lts}"
            |  def scala3Next                 = "${Scala.scala3Next}"
            |  def defaultScala               = "${Scala.defaultUser}"

modules/integration/src/test/scala/scala/cli/integration/BloopTests.scala

@@ -8,7 +8,6 @@ import scala.concurrent.duration.Duration
 import scala.util.Properties
 
 class BloopTests extends ScalaCliSuite {
-
   def runScalaCli(args: String*): os.proc = os.proc(TestUtil.cli, args)
 
   private lazy val bloopDaemonDir =

modules/integration/src/test/scala/scala/cli/integration/BspTestDefinitions.scala

@@ -22,9 +22,8 @@ import scala.jdk.CollectionConverters.*
 import scala.util.control.NonFatal
 import scala.util.{Failure, Properties, Success, Try}
 
-abstract class BspTestDefinitions(val scalaVersionOpt: Option[String])
-    extends ScalaCliSuite with TestScalaVersionArgs {
-
+abstract class BspTestDefinitions extends ScalaCliSuite with TestScalaVersionArgs {
+  _: TestScalaVersion =>
   private lazy val extraOptions = scalaVersionArgs ++ TestUtil.extraOptions
 
   import BspTestDefinitions.*

modules/integration/src/test/scala/scala/cli/integration/BspTests212.scala

@@ -1,7 +1,3 @@
 package scala.cli.integration
 
-// format: off
-class BspTests212 extends BspTestDefinitions(
-  scalaVersionOpt = Some(Constants.scala212)
-)
-// format: on
+class BspTests212 extends BspTestDefinitions with Test212

modules/integration/src/test/scala/scala/cli/integration/BspTests213.scala

@@ -1,7 +1,3 @@
 package scala.cli.integration
 
-// format: off
-class BspTests213 extends BspTestDefinitions(
-  scalaVersionOpt = Some(Constants.scala213)
-)
-// format: on
+class BspTests213 extends BspTestDefinitions with Test213

modules/integration/src/test/scala/scala/cli/integration/BspTests3Lts.scala

@@ -0,0 +1,3 @@
+package scala.cli.integration
+
+class BspTests3Lts extends BspTestDefinitions with Test3Lts

modules/integration/src/test/scala/scala/cli/integration/BspTestsDefault.scala

@@ -1,3 +1,3 @@
 package scala.cli.integration
 
-class BspTestsDefault extends BspTestDefinitions(scalaVersionOpt = None)
+class BspTestsDefault extends BspTestDefinitions with TestDefault

modules/integration/src/test/scala/scala/cli/integration/CleanTests.scala

@@ -3,7 +3,6 @@ package scala.cli.integration
 import com.eed3si9n.expecty.Expecty.expect
 
 class CleanTests extends ScalaCliSuite {
-
   override def group: ScalaCliSuite.TestGroup = ScalaCliSuite.TestGroup.First
 
   test("simple") {

modules/integration/src/test/scala/scala/cli/integration/CompileTestDefinitions.scala

@@ -6,12 +6,11 @@ import java.io.File
 
 import scala.cli.integration.util.BloopUtil
 
-abstract class CompileTestDefinitions(val scalaVersionOpt: Option[String])
+abstract class CompileTestDefinitions
     extends ScalaCliSuite
     with TestScalaVersionArgs
     with CompilerPluginTestDefinitions
-    with SemanticDbTestDefinitions {
-
+    with SemanticDbTestDefinitions { _: TestScalaVersion =>
   protected lazy val extraOptions: Seq[String] = scalaVersionArgs ++ TestUtil.extraOptions
 
   private lazy val bloopDaemonDir = BloopUtil.bloopDaemonDir {
@@ -693,13 +692,9 @@ abstract class CompileTestDefinitions(val scalaVersionOpt: Option[String])
       )
         .call(cwd = root, check = false, mergeErrIntoOut = true)
       expect(res.exitCode == 1)
-      assertNoDiff(
-        res.out.text(),
-        """Error occurred during initialization of VM
-          |Too small maximum heap
-          |Compilation failed
-          |""".stripMargin
-      )
+      val out = res.out.text()
+      expect(out.contains("Error occurred during initialization of VM"))
+      expect(out.contains("Too small maximum heap"))
     }
   }
 }

modules/integration/src/test/scala/scala/cli/integration/CompileTests212.scala

@@ -1,11 +1,6 @@
 package scala.cli.integration
 
-// format: off
-class CompileTests212 extends CompileTestDefinitions(
-  scalaVersionOpt = Some(Constants.scala212)
-) {
-// format: on
-
+class CompileTests212 extends CompileTestDefinitions with Test212 {
   val pluginInputs: TestInputs = TestInputs(
     os.rel / "Plugin.scala" ->
       // Copied from (https://github.com/typelevel/kind-projector/blob/00bf25cef1b7d01d61a3555cccb6cf38fe30e117/src/test/scala/polylambda.scala)

modules/integration/src/test/scala/scala/cli/integration/CompileTests213.scala

@@ -1,7 +1,3 @@
 package scala.cli.integration
 
-// format: off
-class CompileTests213 extends CompileTestDefinitions(
-  scalaVersionOpt = Some(Constants.scala213)
-)
-// format: on
+class CompileTests213 extends CompileTestDefinitions with Test213