From 2120a72f5e844730d4b8a66cb24d1dde90ac9995 Mon Sep 17 00:00:00 2001 From: Chris Wright Date: Wed, 28 Aug 2024 10:26:53 +0100 Subject: [PATCH] Add `cloudtrail query` command * Adds a command which allows querying the CloudTrail logs via Athena, given a Dalmatian account name and SQL query * The dalmatian account bootstrap creates the Trail and Athena database/table to query the S3 bucket storing the logs. The naming of the Athena resources are consitent, which allows this command to target the correct Athena workgroup * To allow for easier querying, the keyword 'CLOUDTRAIL' is to be used within the given query, which will be replaced with the actual table name, eg: ``` dalmatian cloudtrail query \ -a 123456789112-eu-west-2-my-account \ -Q "select * from CLOUDTRAIL limit 10;" ``` * Currently the output is the full JSON Athena reult. We can expand on this to provide common useful CloudTrail queries --- bin/cloudtrail/v2/query | 81 +++++++++++++++++++++++ lib/bash-functions/resolve_aws_profile.sh | 47 ++++++++----- 2 files changed, 113 insertions(+), 15 deletions(-) create mode 100755 bin/cloudtrail/v2/query diff --git a/bin/cloudtrail/v2/query b/bin/cloudtrail/v2/query new file mode 100755 index 0000000..a2ec3f3 --- /dev/null +++ b/bin/cloudtrail/v2/query @@ -0,0 +1,81 @@ +#!/bin/bash + +# exit on failures +set -e +set -o pipefail + +usage() { + echo "Usage: $(basename "$0") [OPTIONS]" 1>&2 + echo " -h - help" + echo " -a - Dalmatian account name" + echo " -Q - Athena Query to run against CloudTrail logs" + echo " Format the query, using 'CLOUDTRAIL' in place of the full table name, which will be" + echo " evaulated and replaced within the given query that is sent to Athena. eg:" + echo " select * from CLOUDTRAIL limit 50;" + exit 1 +} + +while getopts "a:Q:h" opt; do + case $opt in + a) + DALMATIAN_ACCOUNT=$OPTARG + ;; + Q) + ATHENA_QUERY=$OPTARG + ;; + h) + usage + ;; + *) + usage + ;; + esac +done + +if [[ + -z "$DALMATIAN_ACCOUNT" + || -z "$ATHENA_QUERY" +]] +then + usage +fi + +PROFILE="$(resolve_aws_profile -a "$DALMATIAN_ACCOUNT")" +ACCOUNT_NUMBER="$(echo "$DALMATIAN_ACCOUNT" | cut -d'-' -f1)" +PROJECT_NAME="$(jq -r '.project_name' < "$CONFIG_SETUP_JSON_FILE")" +PROJECT_NAME_SNAKE="$(echo "$PROJECT_NAME" | tr '-' '_')" +TABLE_NAME="cloudtrail_logs_${ACCOUNT_NUMBER}_${PROJECT_NAME_SNAKE}_cloudtrail_cloudtrail" +DATABASE="${PROJECT_NAME_SNAKE}_cloudtrail" +WORKGROUP="${PROJECT_NAME}-cloudtrail" +ATHENA_QUERY="${ATHENA_QUERY/CLOUDTRAIL/$TABLE_NAME}" + +EXECUTION_ID="$( + "$APP_ROOT/bin/dalmatian" aws-sso run-command \ + -p "$PROFILE" \ + athena start-query-execution \ + --query-string "$ATHENA_QUERY" \ + --query-execution-context Database="$DATABASE" \ + --work-group "$WORKGROUP" \ + | jq -r '.QueryExecutionId' +)" + +log_info -l "Execution ID: $EXECUTION_ID" -q "$QUIET_MODE" + +EXECUTION_STATUS="" +while [ "$EXECUTION_STATUS" != "SUCCEEDED" ] +do + EXECUTION_STATUS="$( + "$APP_ROOT/bin/dalmatian" aws-sso run-command \ + -p "$PROFILE" \ + athena get-query-execution \ + --query-execution-id "$EXECUTION_ID" \ + | jq -r '.QueryExecution.Status.State' + )" + log_info -l "Execution status: $EXECUTION_STATUS" -q "$QUIET_MODE" + sleep 1 +done + +"$APP_ROOT/bin/dalmatian" aws-sso run-command \ + -p "$PROFILE" \ + athena get-query-results \ + --query-execution-id "$EXECUTION_ID" | jq diff --git a/lib/bash-functions/resolve_aws_profile.sh b/lib/bash-functions/resolve_aws_profile.sh index fc34bfc..bdc5732 100644 --- a/lib/bash-functions/resolve_aws_profile.sh +++ b/lib/bash-functions/resolve_aws_profile.sh @@ -3,14 +3,15 @@ set -e set -o pipefail # Dalmatian specific function to resolve the aws-sso profile name -# from a given infrastructure name and environment +# from a given infrastructure name and environment, or a Dalmatian +# account name # -# @usage log_info -l 'Something happened :)'" # @param -i An infrastructure's friendly name # @param -e An infrastructure's environment name +# @param -a A Dalmatian Account name function resolve_aws_profile { OPTIND=1 - while getopts "i:e:" opt; do + while getopts "i:e:a:" opt; do case $opt in i) INFRASTRUCTURE_NAME="$OPTARG" @@ -18,24 +19,40 @@ function resolve_aws_profile { e) ENVIRONMENT_NAME="$OPTARG" ;; + a) + DALMATIAN_ACCOUNT="$OPTARG" + ;; *) echo "Invalid \`resolve_aws_profile\` function usage" >&2 exit 1 ;; esac done - ACCOUNT_INFRASTRUCTURES="$("$APP_ROOT/bin/dalmatian" deploy list-infrastructures)" - ACCOUNT_WORKSPACE="$(echo "$ACCOUNT_INFRASTRUCTURES" | jq -r \ - --arg infrastructure_name "$INFRASTRUCTURE_NAME" \ - --arg environment_name "$ENVIRONMENT_NAME" \ - '.accounts | - to_entries | - map(select( - (.value.infrastructures | has($infrastructure_name) ) and - ( .value.infrastructures[$infrastructure_name].environments | index($environment_name) ) - )) | - from_entries | - keys[0]')" + if [[ + -n "$INFRASTRUCTURE_NAME" + && -n "$ENVIRONMENT_NAME" + ]] + then + ACCOUNT_INFRASTRUCTURES="$("$APP_ROOT/bin/dalmatian" deploy list-infrastructures)" + ACCOUNT_WORKSPACE="$(echo "$ACCOUNT_INFRASTRUCTURES" | jq -r \ + --arg infrastructure_name "$INFRASTRUCTURE_NAME" \ + --arg environment_name "$ENVIRONMENT_NAME" \ + '.accounts | + to_entries | + map(select( + (.value.infrastructures | has($infrastructure_name) ) and + ( .value.infrastructures[$infrastructure_name].environments | index($environment_name) ) + )) | + from_entries | + keys[0]')" + elif [[ + -n "$DALMATIAN_ACCOUNT" + ]] + then + ACCOUNT_WORKSPACE="$DALMATIAN_ACCOUNT" + else + echo "Invalid \`resolve_aws_profile\` function usage" >&2 + fi PROFILE_NAME="$(echo "$ACCOUNT_WORKSPACE" | cut -d'-' -f5-)" echo "$PROFILE_NAME"