diff --git a/sdlf-utils/workshop-examples/10-demo/sdlf-workshop/dataset-legislators.yaml b/sdlf-utils/workshop-examples/10-demo/sdlf-workshop/dataset-legislators.yaml new file mode 100644 index 00000000..2fb1a5d5 --- /dev/null +++ b/sdlf-utils/workshop-examples/10-demo/sdlf-workshop/dataset-legislators.yaml @@ -0,0 +1,17 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Example datasets + +Parameters: + pPipelineReference: + Type: String + Default: none + +Resources: + rLegislators: + Type: awslabs::sdlf::dataset::MODULE + Properties: + pPipelineReference: !Ref pPipelineReference + pDatasetName: legislators + pS3Prefix: legislators + pDeploymentInstance: dev + pStorageDeploymentInstance: dev diff --git a/sdlf-utils/workshop-examples/10-demo/sdlf-workshop/foundations-datalake-dev.yaml b/sdlf-utils/workshop-examples/10-demo/sdlf-workshop/foundations-datalake-dev.yaml new file mode 100644 index 00000000..42746203 --- /dev/null +++ b/sdlf-utils/workshop-examples/10-demo/sdlf-workshop/foundations-datalake-dev.yaml @@ -0,0 +1,18 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: SDLF Foundations in datalake domain, dev environment + +Parameters: + pPipelineReference: + Type: String + Default: none + +Resources: + rAnycompany: + Type: awslabs::sdlf::foundations::MODULE + Properties: + pPipelineReference: !Ref pPipelineReference + pChildAccountId: !Ref AWS::AccountId + pOrg: anycompany + pDomain: datalake + pDeploymentInstance: dev + pCicdRole: Admin diff --git a/sdlf-utils/workshop-examples/10-demo/sdlf-workshop/pipeline-main.yaml b/sdlf-utils/workshop-examples/10-demo/sdlf-workshop/pipeline-main.yaml new file mode 100644 index 00000000..b72a47b0 --- /dev/null +++ b/sdlf-utils/workshop-examples/10-demo/sdlf-workshop/pipeline-main.yaml @@ -0,0 +1,70 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: Main pipeline + +Parameters: + pPipelineReference: + Type: String + Default: none + +Resources: + rMainA: + Type: awslabs::sdlf::stagelambda::MODULE + Properties: + pStageName: A + pPipeline: main + pDataset: legislators + pDeploymentInstance: maina + pStorageDeploymentInstance: dev + pDatasetDeploymentInstance: dev + pTriggerType: event + pEventPattern: !Sub >- + { + "source": ["aws.s3"], + "detail-type": ["Object Created"], + "detail": { + "bucket": { + "name": ["{{resolve:ssm:/sdlf/storage/rRawBucket/dev}}"] + }, + "object": { + "key": [{ "prefix": "legislators/" }] + } + } + } + pEnableTracing: false + + rMainB: + Type: awslabs::sdlf::stageglue::MODULE + Properties: + pStageName: B + pPipeline: main + pDataset: legislators + pDeploymentInstance: mainb + pStorageDeploymentInstance: dev + pDatasetDeploymentInstance: dev + pGlueJobName: sdlf-legislators-glue-job + pGlueNumberOfWorkers: 10 + pGlueWorkerType: G.1X + pTriggerType: schedule + pEventPattern: !Sub >- + { + "source": ["aws.s3"], + "detail-type": ["Object Created"], + "detail": { + "bucket": { + "name": ["{{resolve:ssm:/sdlf/storage/rStageBucket/dev}}"] + }, + "object": { + "key": [{ "prefix": "legislators/main/A/" }] + } + } + } + pSchedule: "cron(*/5 * * * ? *)" + pEnableTracing: false + pGlueArguments: >- + { + "--job-bookmark-option": "job-bookmark-enable", + "--enable-metrics": "", + "--enable-auto-scaling": "true", + "--SOURCE_LOCATION": !Sub "s3://{{resolve:ssm:/sdlf/storage/rStageBucket/dev}}/legislators/main/A", + "--OUTPUT_LOCATION": !Sub "s3://{{resolve:ssm:/sdlf/storage/rAnalyticsBucket/dev}}/legislators/main/B" + } diff --git a/sdlf-utils/workshop-examples/10-demo/sdlf-workshop/tags.json b/sdlf-utils/workshop-examples/10-demo/sdlf-workshop/tags.json new file mode 100644 index 00000000..9a7d8c7d --- /dev/null +++ b/sdlf-utils/workshop-examples/10-demo/sdlf-workshop/tags.json @@ -0,0 +1,5 @@ +{ + "Tags" : { + "Framework" : "sdlf" + } +} diff --git a/sdlf-utils/workshop-examples/10-demo/sdlf-workshop/team-datalake-engineering-dev.yaml b/sdlf-utils/workshop-examples/10-demo/sdlf-workshop/team-datalake-engineering-dev.yaml new file mode 100644 index 00000000..cc8abec9 --- /dev/null +++ b/sdlf-utils/workshop-examples/10-demo/sdlf-workshop/team-datalake-engineering-dev.yaml @@ -0,0 +1,16 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Engineering SDLF Team in datalake domain, dev environment + +Parameters: + pPipelineReference: + Type: String + Default: none + +Resources: + rEngineering: + Type: awslabs::sdlf::team::MODULE + Properties: + pPipelineReference: !Ref pPipelineReference + pTeamName: engineering + pStorageDeploymentInstance: dev + pSNSNotificationsEmail: nobody@amazon.com