DynamoDB insertion integration plus e2e test

core-tests/e2e-tests/spring/spring-rest-dynamodb/src/test/java/com/foo/spring/rest/dynamodb/WorldCupPlayersEmptyController.java

@@ -0,0 +1,39 @@
+package com.foo.spring.rest.dynamodb;
+
+import com.dynamodb.players.WorldCupPlayersApp;
+import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
+import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
+import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
+import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;
+import software.amazon.awssdk.services.dynamodb.model.KeyType;
+import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput;
+import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
+
+/** Embedded controller for an empty World Cup players DynamoDB table. */
+public class WorldCupPlayersEmptyController extends DynamoDbController {
+
+    /** Creates the table used by the read-only player endpoint without seed items. */
+    @Override
+    protected void initializeDatabase(DynamoDbAsyncClient client) {
+        client.createTable(CreateTableRequest.builder()
+                .tableName("WorldCupPlayers")
+                .attributeDefinitions(AttributeDefinition.builder()
+                        .attributeName("country")
+                        .attributeType(ScalarAttributeType.S)
+                        .build())
+                .keySchema(KeySchemaElement.builder().attributeName("country").keyType(KeyType.HASH).build())
+                .provisionedThroughput(ProvisionedThroughput.builder()
+                        .readCapacityUnits(1L).writeCapacityUnits(1L).build())
+                .build()).join();
+    }
+
+    @Override
+    protected Class<?> getApplicationClass() {
+        return WorldCupPlayersApp.class;
+    }
+
+    @Override
+    public String getPackagePrefixesToCover() {
+        return "com.dynamodb.players";
+    }
+}

core-tests/e2e-tests/spring/spring-rest-dynamodb/src/test/java/org/evomaster/e2etests/spring/rest/dynamodb/DynamoDbInsertionsEMTest.java

@@ -0,0 +1,45 @@
+package org.evomaster.e2etests.spring.rest.dynamodb;
+
+import com.foo.spring.rest.dynamodb.WorldCupPlayersEmptyController;
+import org.evomaster.core.problem.rest.data.HttpVerb;
+import org.evomaster.core.problem.rest.data.RestIndividual;
+import org.evomaster.core.search.Solution;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/** Verifies that a generated DynamoDB insertion satisfies a failed conditional player read. */
+public class DynamoDbInsertionsEMTest extends DynamoDbTestBase {
+
+    /** Starts the empty-table application. */
+    @BeforeAll
+    public static void initClass() throws Exception {
+        initDynamoDbTest(new WorldCupPlayersEmptyController());
+    }
+
+    /** Compares the empty-table baseline with insertion-enabled DynamoDB generation. */
+    @Test
+    public void testInsertionSatisfiesConditionalQuery() throws Throwable {
+        runTestHandlingFlakyAndCompilation(
+                "DynamoDbInsertionsEM",
+                "org.foo.spring.rest.dynamodb.DynamoDbInsertionsEM",
+                1000,
+                false,
+                args -> {
+                    List<String> baselineArgs = new ArrayList<>(args);
+                    configureDynamoDbInsertions(baselineArgs, false);
+                    Solution<RestIndividual> baseline = initAndRun(baselineArgs);
+                    assertHasAtLeastOne(baseline, HttpVerb.GET, 404, "/players/{fifaId}", null);
+                    assertNone(baseline, HttpVerb.GET, 200, "/players/{fifaId}", null);
+
+                    List<String> insertionArgs = new ArrayList<>(args);
+                    configureDynamoDbInsertions(insertionArgs, true);
+                    Solution<RestIndividual> generated = initAndRun(insertionArgs);
+                    assertHasAtLeastOne(generated, HttpVerb.GET, 200, "/players/{fifaId}",
+                            "Lionel Messi plays for Argentina");
+                },
+                5);
+    }
+}

core-tests/e2e-tests/spring/spring-rest-dynamodb/src/test/java/org/evomaster/e2etests/spring/rest/dynamodb/DynamoDbTestBase.java

@@ -13,6 +13,8 @@ public abstract class DynamoDbTestBase extends RestTestBase {
 
     private static final String INSTRUMENT_DYNAMODB_OPTION = "instrumentMR_DYNAMODB";
     private static final String DYNAMODB_HEURISTICS_OPTION = "heuristicsForDynamoDb";
+    private static final String DYNAMODB_EXTRACTION_OPTION = "extractDynamoDbExecutionInfo";
+    private static final String DYNAMODB_GENERATION_OPTION = "generateDynamoDbData";
 
     /**
      * Starts an instrumented DynamoDB SUT for a concrete test class.
@@ -36,4 +38,17 @@ protected void configureDynamoDbHeuristics(List<String> args, boolean heuristics
         setOption(args, INSTRUMENT_DYNAMODB_OPTION, "true");
         setOption(args, DYNAMODB_HEURISTICS_OPTION, Boolean.toString(heuristicsEnabled));
     }
+
+    /**
+     * Configures DynamoDB failed-read extraction and generated initialization data.
+     *
+     * @param args EvoMaster command-line arguments
+     * @param enabled whether insertion generation should be enabled
+     */
+    protected void configureDynamoDbInsertions(List<String> args, boolean enabled) {
+        setOption(args, INSTRUMENT_DYNAMODB_OPTION, "true");
+        setOption(args, DYNAMODB_HEURISTICS_OPTION, Boolean.toString(enabled));
+        setOption(args, DYNAMODB_EXTRACTION_OPTION, Boolean.toString(enabled));
+        setOption(args, DYNAMODB_GENERATION_OPTION, Boolean.toString(enabled));
+    }
 }

core/src/main/kotlin/org/evomaster/core/EMConfig.kt

@@ -780,6 +780,16 @@ class EMConfig {
                     "extracting Mongo execution info with 'extractMongoExecutionInfo'")
         }
 
+        if (shouldGenerateDynamoDbData() && !heuristicsForDynamoDb) {
+            throw ConfigProblemException("Cannot generate DynamoDB data without enabling " +
+                    "'heuristicsForDynamoDb'")
+        }
+
+        if (shouldGenerateDynamoDbData() && !extractDynamoDbExecutionInfo) {
+            throw ConfigProblemException("Cannot generate DynamoDB data without enabling " +
+                    "'extractDynamoDbExecutionInfo'")
+        }
+
         if (enableTrackEvaluatedIndividual && enableTrackIndividual) {
             throw ConfigProblemException("When tracking EvaluatedIndividual, it is not necessary to track individual")
         }
@@ -1016,6 +1026,8 @@ class EMConfig {
 
     fun shouldGenerateRedisData() = generateRedisData
 
+    fun shouldGenerateDynamoDbData() = generateDynamoDbData
+
     fun dtoSupportedForPayload() =  dtoForRequestPayload && couldSupportDtoForPayload()
 
     fun couldSupportDtoForPayload() = problemType == ProblemType.REST && outputFormat.isJavaOrKotlin()
@@ -2010,6 +2022,11 @@ class EMConfig {
     @DependsOnFalseFor("blackBox")
     var extractRedisExecutionInfo = false
 
+    @Experimental
+    @Cfg("Enable extracting DynamoDB execution info")
+    @DependsOnFalseFor("blackBox")
+    var extractDynamoDbExecutionInfo = false
+
     @Experimental
     @Cfg("Enable EvoMaster to generate SQL data with direct accesses to the database. Use the Z3 SMT solver")
     @DependsOnFalseFor("blackBox")
@@ -2070,6 +2087,11 @@ class EMConfig {
     @DependsOnFalseFor("blackBox")
     var generateRedisData = false
 
+    @Experimental
+    @Cfg("Enable EvoMaster to generate DynamoDB data with direct database access")
+    @DependsOnFalseFor("blackBox")
+    var generateDynamoDbData = false
+
     @Cfg("When generating SQL data, how many new rows (max) to generate for each specific SQL Select")
     @Min(1.0)
     @DependsOnFalseFor("blackBox")

core/src/main/kotlin/org/evomaster/core/problem/api/service/ApiWsStructureMutator.kt

@@ -1,6 +1,7 @@
 package org.evomaster.core.problem.api.service
 
 import com.google.inject.Inject
+import org.evomaster.client.java.controller.api.dto.database.execution.DynamoDbFailedQuery
 import org.evomaster.client.java.controller.api.dto.database.execution.MongoFailedQuery
 import org.evomaster.client.java.controller.api.dto.database.execution.RedisFailedCommand
 import org.evomaster.client.java.instrumentation.shared.ExternalServiceSharedUtils
@@ -16,6 +17,8 @@ import org.evomaster.core.problem.externalservice.httpws.service.HarvestActualHt
 import org.evomaster.core.problem.externalservice.httpws.service.HttpWsExternalServiceHandler
 import org.evomaster.core.database.redis.RedisDbAction
 import org.evomaster.core.database.redis.RedisInsertBuilder
+import org.evomaster.core.database.dynamodb.DynamoDbAction
+import org.evomaster.core.database.dynamodb.DynamoDbInsertBuilder
 import org.evomaster.core.search.EvaluatedIndividual
 import org.evomaster.core.search.GroupsOfChildren
 import org.evomaster.core.search.Individual
@@ -195,6 +198,7 @@ abstract class ApiWsStructureMutator : StructureMutator() {
         addInitializingSqlActions(individual, mutatedGenes, sampler)
         addInitializingMongoDbActions(individual, mutatedGenes, sampler)
         addInitializingRedisDbActions(individual, mutatedGenes, sampler)
+        addInitializingDynamoDbActions(individual, mutatedGenes, sampler)
         addInitializingHostnameResolutionActions(individual, mutatedGenes, sampler)
         // TODO if we handle schedule actions with structure mutator
     }
@@ -269,6 +273,40 @@ abstract class ApiWsStructureMutator : StructureMutator() {
         }
     }
 
+    private fun <T: ApiWsIndividual> addInitializingDynamoDbActions(
+        individual: EvaluatedIndividual<*>,
+        mutatedGenes: MutatedGeneSpecification?,
+        sampler: ApiWsSampler<T>
+    ) {
+        if (!config.shouldGenerateDynamoDbData()) {
+            return
+        }
+
+        val ind = individual.individual as? T
+            ?: throw IllegalArgumentException("Invalid individual type")
+
+        val failedQueries = individual.fitness.getViewOfAggregatedFailedDynamoDbQueries()
+
+        if (failedQueries.isEmpty()) {
+            return
+        }
+
+        val oldDynamoDbActions = mutableListOf<EnvironmentAction>().plus(ind.seeInitializingActions())
+
+        val addedDynamoDbInsertions = handleFailedDynamoDbQueries(ind, failedQueries)
+            .let { if (it.isEmpty()) emptyList() else listOf(it) }
+
+        if (mutatedGenes != null && config.isEnabledArchiveGeneSelection()) {
+            individual.updateImpactGeneDueToAddedInitializationGenes(
+                mutatedGenes,
+                oldDynamoDbActions,
+                addedDynamoDbInsertions,
+                ImpactsOfIndividual.DYNAMODB_ACTION_KEY,
+                config
+            )
+        }
+    }
+
     private fun <T : ApiWsIndividual> addInitializingHostnameResolutionActions(
         individual: EvaluatedIndividual<*>,
         mutatedGenes: MutatedGeneSpecification?,
@@ -537,6 +575,31 @@ abstract class ApiWsStructureMutator : StructureMutator() {
         return addedActions
     }
 
+    private fun <T : ApiWsIndividual> handleFailedDynamoDbQueries(
+        ind: T,
+        failedQueries: List<DynamoDbFailedQuery>
+    ): List<EnvironmentAction> {
+
+        val existingKeys = ind.seeInitializingActions()
+            .filterIsInstance<DynamoDbAction>()
+            .map { it.insertionKey() }
+            .toSet()
+
+        val addedActions = DynamoDbInsertBuilder.buildInsertActions(
+            failedQueries,
+            existingKeys
+        )
+
+        if (addedActions.isNotEmpty()) {
+            ind.addInitializingActions(actions = addedActions)
+            addedActions.forEach { action ->
+                action.seeTopGenes().forEach { gene -> gene.markAllAsInitialized() }
+            }
+        }
+
+        return addedActions
+    }
+
     private fun findMissing(
         fw: Map<TableId, Set<String>>,
         dbactions: List<SqlAction>

core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt

@@ -14,6 +14,10 @@ import org.evomaster.core.database.redis.RedisDbAction
 import org.evomaster.core.database.redis.RedisDbActionResult
 import org.evomaster.core.database.redis.RedisDbActionTransformer
 import org.evomaster.core.database.redis.RedisExecution
+import org.evomaster.core.database.dynamodb.DynamoDbAction
+import org.evomaster.core.database.dynamodb.DynamoDbActionResult
+import org.evomaster.core.database.dynamodb.DynamoDbActionTransformer
+import org.evomaster.core.database.dynamodb.DynamoDbExecution
 import org.evomaster.core.database.sql.DatabaseExecution
 import org.evomaster.core.database.sql.SqlAction
 import org.evomaster.core.database.sql.SqlActionResult
@@ -269,6 +273,20 @@ abstract class EnterpriseFitness<T> : FitnessFunction<T>() where T : Individual
         return true
     }
 
+    fun doDynamoDbCalls(
+        allDynamoDbActions: List<DynamoDbAction>,
+        actionResults: MutableList<ActionResult>
+    ): Boolean {
+        if (allDynamoDbActions.isEmpty()) return true
+        val results = allDynamoDbActions.map { DynamoDbActionResult(it.getLocalId()) }
+        actionResults.addAll(results)
+        val execution = rc.executeDynamoDbInsertions(DynamoDbActionTransformer.transform(allDynamoDbActions))
+        execution?.executionResults?.forEachIndexed { index, success ->
+            results.getOrNull(index)?.setInsertExecutionResult(success)
+        }
+        return execution?.executionResults?.all { it } ?: false
+    }
+
     protected fun registerNewAction(action: Action, index: Int){
         rc.registerNewAction(getActionDto(action, index))
     }
@@ -414,6 +432,13 @@ abstract class EnterpriseFitness<T> : FitnessFunction<T>() where T : Individual
             }
             fv.aggregateRedisDatabaseData()
         }
+
+        if (configuration.extractDynamoDbExecutionInfo) {
+            for (i in 0 until dto.extraHeuristics.size) {
+                fv.setDynamoDbExecution(i, DynamoDbExecution.fromDto(dto.extraHeuristics[i].dynamoDbExecutionsDto))
+            }
+            fv.aggregateDynamoDbData()
+        }
     }
 
     private fun handleSqlHeuristics(

core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/ResourceRestFitness.kt

@@ -16,6 +16,7 @@ import org.evomaster.core.problem.rest.resource.RestResourceCalls
 import org.evomaster.core.problem.rest.service.ResourceDepManageService
 import org.evomaster.core.problem.rest.service.ResourceManageService
 import org.evomaster.core.database.redis.RedisDbAction
+import org.evomaster.core.database.dynamodb.DynamoDbAction
 import org.evomaster.core.search.action.ActionFilter
 import org.evomaster.core.search.action.ActionResult
 import org.evomaster.core.search.EvaluatedIndividual
@@ -86,6 +87,8 @@ class ResourceRestFitness : AbstractRestFitness() {
 
         doRedisDbCalls(individual.seeInitializingActions().filterIsInstance<RedisDbAction>(), actionResults)
 
+        doDynamoDbCalls(individual.seeInitializingActions().filterIsInstance<DynamoDbAction>(), actionResults)
+
         //used for things like chaining "location" paths
         val chainState = mutableMapOf<String, String>()
 

core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/RestFitness.kt

@@ -6,6 +6,7 @@ import org.evomaster.core.problem.httpws.auth.AuthUtils
 import org.evomaster.core.problem.rest.data.RestCallResult
 import org.evomaster.core.problem.rest.data.RestIndividual
 import org.evomaster.core.database.redis.RedisDbAction
+import org.evomaster.core.database.dynamodb.DynamoDbAction
 import org.evomaster.core.search.action.ActionFilter
 import org.evomaster.core.search.action.ActionResult
 import org.evomaster.core.search.EvaluatedIndividual
@@ -46,6 +47,7 @@ open class RestFitness : AbstractRestFitness() {
         doDbCalls(individual.seeInitializingActions().filterIsInstance<SqlAction>(), actionResults = actionResults)
         doMongoDbCalls(individual.seeInitializingActions().filterIsInstance<MongoDbAction>(), actionResults = actionResults)
         doRedisDbCalls(individual.seeInitializingActions().filterIsInstance<RedisDbAction>(), actionResults = actionResults)
+        doDynamoDbCalls(individual.seeInitializingActions().filterIsInstance<DynamoDbAction>(), actionResults = actionResults)
 
 
         val fv = FitnessValue(individual.size().toDouble())

core/src/main/kotlin/org/evomaster/core/search/EvaluatedIndividual.kt

@@ -19,6 +19,7 @@ import org.evomaster.core.problem.rest.data.RestCallResult
 import org.evomaster.core.problem.rest.data.RestIndividual
 import org.evomaster.core.problem.rest.resource.ResourceImpactOfIndividual
 import org.evomaster.core.database.redis.RedisDbAction
+import org.evomaster.core.database.dynamodb.DynamoDbAction
 import org.evomaster.core.scheduletask.ScheduleTaskAction
 import org.evomaster.core.search.action.*
 import org.evomaster.core.search.action.ActionFilter.*
@@ -1007,7 +1008,7 @@ class EvaluatedIndividual<T>(
     }
 
     private fun initializingActionClasses(): List<KClass<*>> {
-        return listOf(MongoDbAction::class, SqlAction::class, RedisDbAction::class, ScheduleTaskAction::class)
+        return listOf(MongoDbAction::class, SqlAction::class, RedisDbAction::class, DynamoDbAction::class, ScheduleTaskAction::class)
     }
 
     fun hasAnyPotentialFault() = this.fitness.hasAnyPotentialFault(this.individual.searchGlobalState!!.idMapper)