diff --git a/.github/workflows/upload-and-deploy.yaml b/.github/workflows/upload-and-deploy.yaml index 8d1b4dc6..c3cab9a3 100755 --- a/.github/workflows/upload-and-deploy.yaml +++ b/.github/workflows/upload-and-deploy.yaml @@ -11,6 +11,7 @@ env: DEV_INPUT_BUCKET: recover-dev-input-data DEV_PROCESSED_BUCKET: recover-dev-processed-data PROD_INPUT_BUCKET: recover-input-data + INTEGRATION_TEST_NUM_EXPORTS: 28 jobs: @@ -287,7 +288,7 @@ jobs: aws s3api list-objects-v2 \ --bucket $DEV_INPUT_BUCKET \ --prefix $EXPORT_S3_KEY_PREFIX \ - --query '((sort_by(Contents[? !ends_with(Key, `/`) && !contains(Key, `owner.txt`)], &LastModified)[::-1])[:28])[*].Key' | + --query '((sort_by(Contents[? !ends_with(Key, `/`) && !contains(Key, `owner.txt`)], &LastModified)[::-1])[:${{ env.INTEGRATION_TEST_NUM_EXPORTS }}])[*].Key' | jq -c )" >> "$GITHUB_OUTPUT" @@ -363,7 +364,7 @@ jobs: aws s3api list-objects-v2 \ --bucket $PROD_INPUT_BUCKET \ --prefix "$EXPORT_S3_KEY_PREFIX/" \ - --query '((sort_by(Contents[? !ends_with(Key, `/`) && !contains(Key, `owner.txt`)], &LastModified)[::-1])[:28])[*].Key' | + --query '((sort_by(Contents[? !ends_with(Key, `/`) && !contains(Key, `owner.txt`)], &LastModified)[::-1])[:${{ env.INTEGRATION_TEST_NUM_EXPORTS }}])[*].Key' | jq -c )" >> "$GITHUB_OUTPUT" diff --git a/src/glue/jobs/json_to_parquet.py b/src/glue/jobs/json_to_parquet.py index f632e877..3d0741a0 100644 --- a/src/glue/jobs/json_to_parquet.py +++ b/src/glue/jobs/json_to_parquet.py @@ -39,7 +39,7 @@ INDEX_FIELD_MAP = { "enrolledparticipants": ["ParticipantIdentifier"], "fitbitprofiles": ["ParticipantIdentifier", "ModifiedDate"], - "fitbitdevices": ["ParticipantIdentifier", "Date"], + "fitbitdevices": ["ParticipantIdentifier", "Date", "Device"], "fitbitactivitylogs": ["ParticipantIdentifier", "LogId"], "fitbitdailydata": ["ParticipantIdentifier", "Date"], "fitbitecg": ["ParticipantIdentifier", "FitbitEcgKey"],