Merge remote-tracking branch 'origin/master' into sql_heuristics_for_…

…subqueries

.github/workflows/release.yml

@@ -45,6 +45,23 @@ jobs:
           path: core/target/evomaster.jar
           retention-days: ${{env.retention-days}}
           if-no-files-found: error
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: webfuzzing/evomaster
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+      - name: Build and push Docker
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
 
   installer-for-windows:
     needs: build-base
@@ -129,7 +146,6 @@ jobs:
     runs-on: "ubuntu-latest"
 
     steps:
-
       - name: Download fat jar
         uses: actions/download-artifact@v4
         with:
@@ -146,19 +162,6 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: evomaster.msi
-
-
-## Doesn't seem any longer maintained... plus usability issues
-#      - uses: "marvinpinto/action-automatic-releases@latest"
-#        with:
-#          repo_token: "${{ secrets.GITHUB_TOKEN }}"
-#          prerelease: false
-#          files: |
-#            evomaster.jar
-#            ${{env.installer-windows}}
-#            ${{env.installer-osx}}
-#            ${{env.installer-debian}}
-
       - name: Release
         uses: softprops/action-gh-release@v1
         with:

Dockerfile

@@ -0,0 +1,46 @@
+FROM amazoncorretto:21-alpine-jdk
+
+COPY core/target/evomaster.jar .
+
+ENTRYPOINT [  \
+    "java", \
+    "-Xmx4G", \
+    "-jar", "evomaster.jar", \
+    "--runningInDocker", "true" \
+]
+
+
+###################
+###### NOTES ######
+###################
+# Build
+# docker build -t webfuzzing/evomaster  .
+#
+# Run
+# docker run webfuzzing/evomaster  <options>
+#
+# Publish (latest, otherwise tag with :TAG)
+# docker login
+# docker push webfuzzing/evomaster
+#
+# Example remote BB
+# docker run -v "/$(pwd)/generated_tests":/generated_tests webfuzzing/evomaster --blackBox true --bbSwaggerUrl https://api.apis.guru/v2/openapi.yaml  --outputFormat JAVA_JUNIT_4 --maxTime 10s --ratePerMinute 60
+#
+# Example local BB
+# docker run -v "/$(pwd)/generated_tests":/generated_tests  webfuzzing/evomaster  --blackBox true --bbSwaggerUrl http://host.docker.internal:8080/v3/api-docs --maxTime 5s
+#
+# Example WB  (NOT IMPLEMENTED YET)
+# docker run -v "/$(pwd)/generated_tests":/generated_tests  webfuzzing/evomaster --dockerLocalhost true
+#
+# Setting for existing em.yaml
+# -v "/$(pwd)/em.yaml":/em.yaml
+#
+# Debugging
+# docker run -it --entrypoint sh  webfuzzing/evomaster
+#
+#
+#
+#
+#
+#
+#

README.md

@@ -118,6 +118,17 @@ __Known limitations__:
 -->
 
 
+### Use in Industry
+
+Several enterprises use _EvoMaster_ to fuzz their Web APIs.
+We do few academia-industry collaborations ([see more info here](docs/contribute.md)), where we help test engineers to apply _EvoMaster_ on their systems, as long as we can then report on such experience.
+Example of Fortune 500 companies using _EvoMaster_ are:
+
+* [Meituan](https://www.meituan.com): see [TOSEM'23](docs/publications/2023_tosem_rpc.pdf), [ASE'24](docs/publications/2024_ase.pdf).
+
+* [Volkswagen](https://www.volkswagen.com): see [AUSE'24](docs/publications/2024_ause_vw.pdf), [ICST'25](docs/publications/2025_icst.pdf).
+
+
 ### Videos
 
 ![](docs/img/video-player-flaticon.png)

core/src/main/kotlin/org/evomaster/core/EMConfig.kt

@@ -1,9 +1,6 @@
 package org.evomaster.core
 
-import joptsimple.BuiltinHelpFormatter
-import joptsimple.OptionDescriptor
-import joptsimple.OptionParser
-import joptsimple.OptionSet
+import joptsimple.*
 import org.evomaster.client.java.controller.api.ControllerConstants
 import org.evomaster.client.java.controller.api.dto.auth.AuthenticationDto
 import org.evomaster.client.java.instrumentation.shared.ExternalServiceSharedUtils
@@ -585,6 +582,18 @@ class EMConfig {
             throw ConfigProblemException("The use of 'prematureStop' is meaningful only if the stopping criterion" +
                     " 'stoppingCriterion' is based on time")
         }
+
+        if(blackBox){
+            if(sutControllerHost != ControllerConstants.DEFAULT_CONTROLLER_HOST){
+                throw ConfigProblemException("Changing 'sutControllerHost' has no meaning in black-box testing, as no controller is used")
+            }
+            if(!overrideOpenAPIUrl.isNullOrBlank()){
+                throw ConfigProblemException("Changing 'overrideOpenAPIUrl' has no meaning in black-box testing, as no controller is used")
+            }
+        }
+        if(dockerLocalhost && !runningInDocker){
+            throw ConfigProblemException("Specifying 'dockerLocalhost' only makes sense when running EvoMaster inside Docker.")
+        }
     }
 
     private fun checkPropertyConstraints(m: KMutableProperty<*>) {
@@ -696,8 +705,11 @@ class EMConfig {
             return
         }
 
-        val opt = options.valueOf(m.name)?.toString()
-                ?: throw ConfigProblemException("Value not found for property ${m.name}")
+        val opt = try{
+            options.valueOf(m.name)?.toString()
+        } catch (e: OptionException){
+          throw  ConfigProblemException("Error in parsing configuration option '${m.name}'. Library message: ${e.message}")
+        } ?: throw ConfigProblemException("Value not found for property '${m.name}'")
 
         updateValue(opt, m)
     }
@@ -1048,9 +1060,43 @@ class EMConfig {
             " If no tag is specified here, then such filter is not applied.")
     var endpointTagFilter: String? = null
 
+    @Important(6.0)
+    @Cfg("Host name or IP address of where the SUT EvoMaster Controller Driver is listening on." +
+            " This option is only needed for white-box testing.")
+    var sutControllerHost = ControllerConstants.DEFAULT_CONTROLLER_HOST
+
+
+    @Important(6.1)
+    @Cfg("TCP port of where the SUT EvoMaster Controller Driver is listening on." +
+            " This option is only needed for white-box testing.")
+    @Min(0.0)
+    @Max(maxTcpPort)
+    var sutControllerPort = ControllerConstants.DEFAULT_CONTROLLER_PORT
+
+
+    @Important(7.0)
+    @Url
+    @Cfg("If specified, override the OpenAPI URL location given by the EvoMaster Driver." +
+        " This option is only needed for white-box testing.")
+    var overrideOpenAPIUrl = ""
 
     //-------- other options -------------
 
+    @Cfg("Inform EvoMaster process that it is running inside Docker." +
+            " Users should not modify this parameter, as it is set automatically in the Docker image of EvoMaster.")
+    var runningInDocker = false
+
+    /**
+     * TODO this is currently not implemented.
+     * Even if did, there would still be major issues with handling WireMock.
+     * Until we can think of a good solution there, no point in implementing this.
+     */
+    @Experimental
+    @Cfg("Replace references to 'localhost' to point to the actual host machine." +
+            " Only needed when running EvoMaster inside Docker.")
+    var dockerLocalhost = false
+
+
     @FilePath
     @Cfg("When generating tests in JavaScript, there is the need to know where the driver is located in respect to" +
             " the generated tests")
@@ -1137,14 +1183,6 @@ class EMConfig {
             "A negative value means the CPU clock time will be rather used as seed")
     var seed: Long = -1
 
-    @Cfg("TCP port of where the SUT REST controller is listening on")
-    @Min(0.0)
-    @Max(maxTcpPort)
-    var sutControllerPort = ControllerConstants.DEFAULT_CONTROLLER_PORT
-
-    @Cfg("Host name or IP address of where the SUT REST controller is listening on")
-    var sutControllerHost = ControllerConstants.DEFAULT_CONTROLLER_HOST
-
     @Cfg("Limit of number of individuals per target to keep in the archive")
     @Min(1.0)
     var archiveTargetLimit = 10
@@ -2398,9 +2436,8 @@ class EMConfig {
             " instead of OFF.")
     val probabilityOfOnVsOffInAllOptionals = 0.8
 
-    @Experimental
     @Cfg("Add summary comments on each test")
-    var addTestComments = false
+    var addTestComments = true
 
     @Min(1.0)
     @Cfg("Max length for test comments. Needed when enumerating some names/values, making comments too long to be" +

core/src/main/kotlin/org/evomaster/core/Main.kt

@@ -77,11 +77,39 @@ class Main {
                     return
                 }
 
-                if (parser.parse(*args).has("help")) {
+                val options = parser.parse(*args)
+
+                if (options.has("help")) {
                     parser.printHelpOn(System.out)
                     return
                 }
 
+                val config = EMConfig().apply { updateProperties(options) }
+
+                if(config.runningInDocker){
+                    if(config.blackBox) {
+                        LoggingUtil.getInfoLogger().info(
+                            inGreen("You are running EvoMaster inside Docker." +
+                                    " To access the generated test suite under '/generated_tests', you will need to mount a folder" +
+                                    " or volume." +
+                                    " Also references to host machine on 'localhost' would need to be replaced with" +
+                                    " 'host.docker.internal'." +
+                                    " If this is the first time you run EvoMaster in Docker, you are strongly recommended to first" +
+                                    " check the documentation at:") +
+                                    " ${inBlue("https://github.com/WebFuzzing/EvoMaster/blob/master/docs/docker.md")}"
+                        )
+                    } else {
+                        LoggingUtil.getInfoLogger().warn(
+                            inYellow(
+                            "White-box testing (default in EvoMaster) is currently not supported / not recommended in Docker." +
+                                    " To run EvoMaster in black-box mode, you can use '--blackBox true'." +
+                                    " If you need to run in white-box mode, it is recommended to download an OS installer or" +
+                                    " the uber JAR file from the release-page on GitHub."
+                            )
+                        )
+                    }
+                }
+
                 initAndRun(args)
 
                 LoggingUtil.getInfoLogger().apply {

core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt

@@ -6,6 +6,7 @@ import org.evomaster.core.output.Lines
 import org.evomaster.core.output.SqlWriter
 import org.evomaster.core.output.TestCase
 import org.evomaster.core.output.TestWriterUtils
+import org.evomaster.core.problem.enterprise.EnterpriseActionResult
 import org.evomaster.core.problem.httpws.HttpWsAction
 import org.evomaster.core.problem.httpws.HttpWsCallResult
 import org.evomaster.core.problem.rest.*
@@ -512,6 +513,25 @@ class RestTestCaseWriter : HttpWsTestCaseWriter {
             }
         }
 
+        //TODO move up when adding test comments to other problem types as well
+        //faults
+        val faults = ea.map { it.result }
+            .filterIsInstance<EnterpriseActionResult>()
+            .flatMap { it.getFaults() }
+        if(faults.isNotEmpty()){
+            if(faults.size == 1){
+                lines.addBlockCommentLine("Found 1 potential fault of type-code ${faults.first().category.code}")
+            } else {
+                val codes = faults.asSequence().map { it.category.code }.toSet().toList().sorted()
+                val codeInfo = if (codes.size == 1) {
+                    " of type-code ${codes[0]}"
+                } else {
+                    ". Type-codes: ${codes.joinToString(", ")}"
+                }
+                lines.addBlockCommentLine("Found ${faults.size} potential faults$codeInfo")
+            }
+        }
+
         //examples
         val examples = getAllUsedExamples(ind.individual as RestIndividual)
             .toSet().sorted()

core/src/main/kotlin/org/evomaster/core/problem/rest/service/AbstractRestSampler.kt

@@ -91,7 +91,9 @@ abstract class AbstractRestSampler : HttpWsSampler<RestIndividual>() {
         // set up authentications moved up since we are going to get authentication info from HttpWsSampler
         setupAuthentication(infoDto)
 
-        if(!openApiURL.isNullOrBlank()) {
+        if(!config.overrideOpenAPIUrl.isNullOrBlank()){
+            retrieveSwagger(config.overrideOpenAPIUrl)
+        }else if(!openApiURL.isNullOrBlank()) {
             retrieveSwagger(openApiURL)
         } else if(! openApiSchema.isNullOrBlank()){
             swagger = OpenApiAccess.getOpenApi(openApiSchema)

core/src/test/kotlin/org/evomaster/core/output/PythonTestCaseWriterTest.kt

@@ -15,15 +15,8 @@ import org.junit.jupiter.api.Assertions.*
 import org.junit.jupiter.api.Test
 import javax.ws.rs.core.MediaType
 
-class PythonTestCaseWriterTest {
-
-    private fun getConfig(format: OutputFormat): EMConfig {
-        val config = EMConfig()
-        config.outputFormat = format
-        //config.expectationsActive = false
-        config.testTimeout = -1
-        return config
-    }
+class PythonTestCaseWriterTest : WriterTestBase(){
+
 
     @Test
     fun testEmptyTest() {

core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt

@@ -29,14 +29,9 @@ import org.junit.jupiter.api.Assertions.*
 import org.junit.jupiter.api.Test
 import javax.ws.rs.core.MediaType
 
-class TestCaseWriterTest {
+class TestCaseWriterTest : WriterTestBase(){
+
 
-    private fun getConfig(format: OutputFormat): EMConfig {
-        val config = EMConfig()
-        config.outputFormat = format
-        config.testTimeout = -1
-        return config
-    }
 
     @Test
     fun testEmptyDbInitialization() {

core/src/test/kotlin/org/evomaster/core/output/WriteJsonTest.kt

@@ -19,15 +19,8 @@ import org.evomaster.core.search.gene.sql.SqlPrimaryKeyGene
 import org.junit.jupiter.api.Assertions
 import org.junit.jupiter.api.Test
 
-class WriteJsonTest {
-
-    private fun getConfig(format: OutputFormat): EMConfig {
-        val config = EMConfig()
-        config.outputFormat = format
-        //config.expectationsActive = false
-        config.testTimeout  = -1
-        return config
-    }
+class WriteJsonTest : WriterTestBase() {
+
 
     @Test
     fun testJSONBEmpty() {

core/src/test/kotlin/org/evomaster/core/output/WriteXMLTest.kt

@@ -15,15 +15,8 @@ import org.evomaster.core.search.gene.string.StringGene
 import org.junit.jupiter.api.Assertions
 import org.junit.jupiter.api.Test
 
-class WriteXMLTest {
-
-    private fun getConfig(format: OutputFormat): EMConfig {
-        val config = EMConfig()
-        config.outputFormat = format
-        //config.expectationsActive = false
-        config.testTimeout = -1
-        return config
-    }
+class WriteXMLTest : WriterTestBase() {
+
 
     @Test
     fun testEmptyXML() {