Merge branch 'master' into feat/integrate-with-builder

core/src/main/kotlin/org/evomaster/core/EMConfig.kt

@@ -12,6 +12,7 @@ import org.evomaster.core.config.ConfigsFromFile
 import org.evomaster.core.logging.LoggingUtil
 import org.evomaster.core.output.OutputFormat
 import org.evomaster.core.output.naming.NamingStrategy
+import org.evomaster.core.output.sorting.SortingStrategy
 import org.evomaster.core.search.impact.impactinfocollection.GeneMutationSelectionMethod
 import org.evomaster.core.search.service.IdMapper
 import org.slf4j.LoggerFactory
@@ -83,6 +84,8 @@ class EMConfig {
 
         private val defaultTestCaseNamingStrategy = NamingStrategy.NUMBERED
 
+        private val defaultTestCaseSortingStrategy = SortingStrategy.COVERED_TARGETS
+
         fun validateOptions(args: Array<String>): OptionParser {
 
             val config = EMConfig() // tmp config object used only for validation.
@@ -2432,6 +2435,8 @@ class EMConfig {
             " Used for test case naming disambiguation. Only valid for Action based naming strategy.")
     var nameWithQueryParameters = false
 
+    @Cfg("Specify the test case sorting strategy")
+    var testCaseSortingStrategy = defaultTestCaseSortingStrategy
 
     @Experimental
     @Probability(true)

core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt

@@ -1,14 +1,22 @@
 package org.evomaster.core.output
 
 import org.evomaster.core.Lazy
-import org.evomaster.core.output.service.PartialOracles
 import org.evomaster.core.output.naming.TestCaseNamingStrategy
+import org.evomaster.core.output.service.PartialOracles
+import org.evomaster.core.output.sorting.SortingStrategy
+import org.evomaster.core.problem.graphql.GraphQLAction
+import org.evomaster.core.problem.graphql.GraphQLIndividual
 import org.evomaster.core.problem.httpws.HttpWsCallResult
 import org.evomaster.core.problem.rest.HttpVerb
 import org.evomaster.core.problem.rest.RestCallAction
 import org.evomaster.core.problem.rest.RestIndividual
+import org.evomaster.core.problem.rpc.RPCCallAction
+import org.evomaster.core.problem.rpc.RPCIndividual
+import org.evomaster.core.problem.webfrontend.WebIndividual
 import org.evomaster.core.search.EvaluatedIndividual
 import org.evomaster.core.search.Solution
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
 import kotlin.reflect.KFunction1
 
 /**
@@ -28,10 +36,10 @@ class TestSuiteOrganizer {
 
     private val defaultSorting = listOf(0, 1)
 
-    fun sortTests(solution: Solution<*>, namingStrategy: TestCaseNamingStrategy): List<TestCase> {
+    fun sortTests(solution: Solution<*>, namingStrategy: TestCaseNamingStrategy, testCaseSortingStrategy: SortingStrategy): List<TestCase> {
         //sortingHelper.selectCriteriaByIndex(defaultSorting)
         //TODO here in the future we will have something a bit smarter
-        return sortingHelper.sort(solution, namingStrategy)
+        return sortingHelper.sort(solution, namingStrategy, testCaseSortingStrategy)
     }
 
 //    fun setPartialOracles(partialOracles: PartialOracles){
@@ -135,6 +143,10 @@ class NamingHelper {
 
 class SortingHelper {
 
+    companion object {
+        private val log: Logger = LoggerFactory.getLogger(SortingHelper::class.java)
+    }
+
     /** [maxStatusCode] sorts Evaluated individuals based on the highest status code (e.g., 500s are first).
      *
      * **/
@@ -196,6 +208,40 @@ class SortingHelper {
      */
 
     var comparatorList = listOf(statusCode, coveredTargets)
+
+    val restComparator: Comparator<EvaluatedIndividual<*>> = compareBy<EvaluatedIndividual<*>> { ind ->
+            (ind.evaluatedMainActions().last().action as RestCallAction).path.levels()
+        }
+        .thenBy { ind ->
+            val min = ind.seeResults().filterIsInstance<HttpWsCallResult>().minByOrNull {
+                it.getStatusCode()?.rem(500) ?: 0
+            }
+            (min?.getStatusCode())?.rem(500) ?: 0
+        }
+        .thenBy { ind ->
+            (ind.evaluatedMainActions().last().action as RestCallAction).verb
+        }
+
+    val graphQLComparator: Comparator<EvaluatedIndividual<*>> = compareBy<EvaluatedIndividual<*>> { ind ->
+            (ind.evaluatedMainActions().last().action as GraphQLAction).methodName
+        }
+        .thenBy { ind ->
+            (ind.evaluatedMainActions().last().action as GraphQLAction).methodType
+        }
+        .thenBy { ind ->
+            (ind.evaluatedMainActions().last().action as GraphQLAction).parameters.size
+        }
+
+    val rpcComparator: Comparator<EvaluatedIndividual<*>> = compareBy<EvaluatedIndividual<*>> { ind ->
+            (ind.evaluatedMainActions().last().action as RPCCallAction).getSimpleClassName()
+        }
+        .thenBy { ind ->
+            (ind.evaluatedMainActions().last().action as RPCCallAction).getExecutedFunctionName()
+        }
+        .thenBy { ind ->
+            (ind.evaluatedMainActions().last().action as RPCCallAction).parameters.size
+        }
+
     private val availableSortCriteria = listOf(statusCode, minActions, coveredTargets, maxStatusCode, maxActions, dbInitSize)
 
 
@@ -249,11 +295,30 @@ class SortingHelper {
          */
 
         return namingStrategy.getSortedTestCases(comparators)
+    }
 
+    private fun sortByTargetIncremental(solution: Solution<*>, namingStrategy: TestCaseNamingStrategy): List<TestCase> {
+        val individuals = solution.individuals
+        val comparator = when {
+            individuals.any { it.individual is RestIndividual } -> restComparator
+            individuals.any { it.individual is GraphQLIndividual } -> graphQLComparator
+            individuals.any { it.individual is RPCIndividual } -> rpcComparator
+            individuals.any { it.individual is WebIndividual } -> {
+                log.warn("Web individuals do not have action based test case naming yet. Defaulting to Numbered strategy.")
+                statusCode
+            }
+            else -> throw IllegalStateException("Unrecognized test individuals with no target incremental based sorting strategy set.")
+        }
+
+        return namingStrategy.getSortedTestCases(comparator)
     }
 
-    fun sort(solution: Solution<*>, namingStrategy: TestCaseNamingStrategy): List<TestCase> {
-        val newSort = sortByComparatorList(comparatorList, namingStrategy)
+    fun sort(solution: Solution<*>, namingStrategy: TestCaseNamingStrategy, testCaseSortingStrategy: SortingStrategy): List<TestCase> {
+        val newSort = when (testCaseSortingStrategy) {
+            SortingStrategy.COVERED_TARGETS -> sortByComparatorList(comparatorList, namingStrategy)
+            SortingStrategy.TARGET_INCREMENTAL -> sortByTargetIncremental(solution, namingStrategy)
+            else -> throw IllegalStateException("Unrecognized sorting strategy $testCaseSortingStrategy")
+        }
 
         Lazy.assert { solution.individuals.toSet() == newSort.map { it.test }.toSet()}
         return newSort

core/src/main/kotlin/org/evomaster/core/output/naming/NumberedTestCaseNamingStrategy.kt

@@ -4,6 +4,7 @@ import org.evomaster.core.output.TestCase
 import org.evomaster.core.search.EvaluatedIndividual
 import org.evomaster.core.search.Solution
 import org.evomaster.core.search.action.Action
+import java.util.Collections.singletonList
 
 open class NumberedTestCaseNamingStrategy(
     solution: Solution<*>
@@ -13,11 +14,17 @@ open class NumberedTestCaseNamingStrategy(
         return generateNames(solution.individuals)
     }
 
+    override fun getSortedTestCases(comparator: Comparator<EvaluatedIndividual<*>>): List<TestCase> {
+        return getSortedTestCases(singletonList(comparator))
+    }
+
     override fun getSortedTestCases(comparators: List<Comparator<EvaluatedIndividual<*>>>): List<TestCase> {
         val inds = solution.individuals
+
         comparators.asReversed().forEach {
             inds.sortWith(it)
         }
+
         return generateNames(inds)
     }
 

core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategy.kt

@@ -18,6 +18,13 @@ abstract class TestCaseNamingStrategy(
      */
     abstract fun getTestCases(): List<TestCase>
 
+    /**
+     * @param comparator used to sort the test cases
+     *
+     * @return the list of sorted TestCase with the generated name given the naming strategy
+     */
+    abstract fun getSortedTestCases(comparator: Comparator<EvaluatedIndividual<*>>): List<TestCase>
+
     /**
      * @param comparators used to sort the test cases
      *

core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt

@@ -144,7 +144,7 @@ class TestSuiteWriter {
         //catch any sorting problems (see NPE is SortingHelper on Trello)
         val tests = try {
             // TODO skip to sort RPC for the moment
-                testSuiteOrganizer.sortTests(solution, namingStrategy)
+                testSuiteOrganizer.sortTests(solution, namingStrategy, config.testCaseSortingStrategy)
         } catch (ex: Exception) {
             log.warn(
                 "A failure has occurred with the test sorting. Reverting to default settings. \n"

core/src/main/kotlin/org/evomaster/core/output/sorting/SortingStrategy.kt

@@ -0,0 +1,30 @@
+package org.evomaster.core.output.sorting
+
+import org.evomaster.core.search.EvaluatedIndividual
+
+/**
+ * Sorting strategies will determine the order of the written test cases in the resulting suite.
+ */
+enum class SortingStrategy {
+
+    /**
+     * Will sort by priority:
+     * - First: status codes. 500 responses go first, then 2xx and last 4xx
+     * - Second: [EvaluatedIndividual] objects on based on the higher number of covered targets. The  purpose is to
+     * give an example of sorting based on fitness information.
+      */
+    COVERED_TARGETS,
+
+    /**
+     * Will sort depending on the [ProblemType], using the following characteristics:
+     * - REST: amount of path segments, status code (like in [COVERED_TARGETS]), and last [HttpVerb].
+     * - GraphQL: method name, method type (query/mutation), and last amount of parameters.
+     * - RPC: class name of the function being tested, function name, and last amount of parameters.
+     *
+     * The goal is to provide an incremental way of reading tests. For example in REST, a developer would usually test
+     * the first (least path segments) and then go moving up the most complex paths.
+     */
+    TARGET_INCREMENTAL
+    ;
+
+}