diff --git a/sdlf-stage-glue/src/glue.yaml b/sdlf-stage-glue/src/glue.yaml index 56e75a07..8013c296 100644 --- a/sdlf-stage-glue/src/glue.yaml +++ b/sdlf-stage-glue/src/glue.yaml @@ -112,6 +112,10 @@ Parameters: pEnableTracing: Description: Flag for whether XRay tracing is enabled Type: String + pMaxItemsPerBatch: + Description: Maximum number of items to process per batch - each Glue job execution will process that many items, if available + Type: Number + Default: 5 pGlueJobName: Description: Glue job name Type: String @@ -523,6 +527,7 @@ Resources: DefinitionSubstitutions: lPostMetadata: !GetAtt rLambdaPostMetadataStep.Arn lError: !GetAtt rLambdaErrorStep.Arn + lMaxItemsPerBatch: !Ref pMaxItemsPerBatch lTransform: !Ref pGlueJobName lWorkerType: !Ref pGlueWorkerType lNumberOfWorkers: !Ref pGlueNumberOfWorkers diff --git a/sdlf-stage-glue/src/state-machine/stage-glue.asl.json b/sdlf-stage-glue/src/state-machine/stage-glue.asl.json index afd99103..9c85336f 100644 --- a/sdlf-stage-glue/src/state-machine/stage-glue.asl.json +++ b/sdlf-stage-glue/src/state-machine/stage-glue.asl.json @@ -41,7 +41,7 @@ "MaxConcurrency": 50, "ToleratedFailurePercentage": 100, "ItemBatcher": { - "MaxItemsPerBatch": 1 + "MaxItemsPerBatch": ${lMaxItemsPerBatch} }, "InputPath": "$.Items", "Next": "Run Glue Crawler"