add basic scripts

.dockerignore

@@ -0,0 +1,2 @@
+.gitignore
+docker-compose.yml

.env.example

@@ -0,0 +1,24 @@
+# Base64 encoded Google service account key
+GOOGLE_CREDENTIALS=
+
+# Google Cloud Storage bucket name
+GOOGLE_BUCKET_NAME=
+
+# Google Cloud Storage backup path
+DATABASE_BACKUP_PATH=
+
+# Base64 enoded private key
+ENCRYPTION_PRIVATE_KEY=
+
+# Private key passphrase
+ENCRYPTION_PASSPHRASE=
+
+# Base64 encoded public key
+ENCRYPTION_PUBLIC_KEY=
+
+# Database connection
+POSTGRES_HOST=
+POSTGRES_PORT=
+POSTGRES_USER=
+POSTGRES_PASSWORD=
+POSTGRES_DB=

.gitignore

@@ -0,0 +1 @@
+.env

Dockerfile

@@ -0,0 +1,19 @@
+FROM google/cloud-sdk:321.0.0-alpine
+
+RUN apk add --no-cache postgresql-client openssl tini
+
+COPY ./scripts /usr/local/bin/scripts
+RUN chmod +x /usr/local/bin/scripts/*
+RUN mv /usr/local/bin/scripts/* /usr/local/bin \
+    && rmdir /usr/local/bin/scripts
+
+COPY ./entrypoint /entrypoint
+RUN sed -i 's/\r$//g' /entrypoint
+RUN chmod +x /entrypoint
+
+# https://crontab.guru/#*_*/8_*_*_*
+RUN echo '* * */8 * * cd /app && bash /app/scripts/backup.sh >> /var/log/pg_backup.log' >> /etc/crontabs/root
+
+WORKDIR /app
+ENTRYPOINT ["/entrypoint"]
+CMD ["/sbin/tini", "--", "/usr/sbin/crond", "-f", "-l", "8"]

docker-compose.yml

@@ -0,0 +1,43 @@
+version: "3.7"
+
+services:
+  db:
+    container_name: db
+    image: postgis/postgis:11-2.5-alpine
+    networks:
+      - inner
+    volumes:
+      - data:/var/lib/postgresql/data
+    environment:
+      - "POSTGRES_USER=${POSTGRES_USER}"
+      - "POSTGRES_PASSWORD=${POSTGRES_PASSWORD}"
+      - "POSTGRES_DB=${POSTGRES_DB}"
+
+  pg_backup:
+    container_name: pg_backup
+    build: .
+    env_file:
+      - ./.env
+    environment:
+      - "GOOGLE_CREDENTIALS=${GOOGLE_CREDENTIALS}"
+      - "GOOGLE_BUCKET_NAME=${GOOGLE_BUCKET_NAME}"
+      - "ENCRYPTION_PRIVATE_KEY=${ENCRYPTION_PRIVATE_KEY}"
+      - "ENCRYPTION_PUBLIC_KEY=${ENCRYPTION_PUBLIC_KEY}"
+      - "ENCRYPTION_PASSPHRASE=${ENCRYPTION_PASSPHRASE}"
+      - "DATABASE_BACKUP_PATH=${DATABASE_BACKUP_PATH}"
+      - "POSTGRES_HOST=${POSTGRES_HOST}"
+      - "POSTGRES_PORT=${POSTGRES_PORT}"
+      - "POSTGRES_USER=${POSTGRES_USER}"
+      - "POSTGRES_PASSWORD={POSTGRES_PASSWORD}"
+      - "POSTGRES_DB=${POSTGRES_DB}"
+    volumes:
+      - .:/app
+    tty: true
+    networks:
+      - inner
+
+volumes:
+  data: {}
+
+networks:
+  inner:

entrypoint

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+if [[ -z $GOOGLE_CREDENTIALS ]]; then
+    echo "Google SDK credentials not set. Ensure GOOGLE_CREDENTIALS contains a valid, B64 encoded, service account key."
+    exit 1
+fi
+
+if [[ -z $GOOGLE_BUCKET_NAME ]]; then
+    echo "Google Storage bucket name not set. Ensure GOOGLE_BUCKET_NAME contains a valid bucket name."
+    exit 1
+fi
+
+if [[ -z $DATABASE_BACKUP_PATH ]]; then
+    echo "Database backups path not set. Ensure DATABASE_BACKUP_PATH contains a valid path for database backups."
+    exit 1
+fi
+
+# Setup Google SDK credentials
+echo $GOOGLE_CREDENTIALS | base64 -d > /google_credentials.json
+export GOOGLE_APPLICATION_CREDENTIALS=/google_credentials.json
+gcloud auth activate-service-account --key-file $GOOGLE_APPLICATION_CREDENTIALS
+
+exec "$@"

scripts/_sourced/messages.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Source: cookiecutter-django template
+# https://github.com/pydanny/cookiecutter-django/blob/master/%7B%7Bcookiecutter.project_slug%7D%7D/compose/production/postgres/maintenance/_sourced/messages.sh
+
+message_newline() {
+    echo
+}
+
+message_debug()
+{
+    echo -e "DEBUG: ${@}"
+}
+
+message_welcome()
+{
+    echo -e "\e[1m${@}\e[0m"
+}
+
+message_warning()
+{
+    echo -e "\e[33mWARNING\e[0m: ${@}"
+}
+
+message_error()
+{
+    echo -e "\e[31mERROR\e[0m: ${@}"
+}
+
+message_info()
+{
+    echo -e "\e[37mINFO\e[0m: ${@}"
+}
+
+message_suggestion()
+{
+    echo -e "\e[33mSUGGESTION\e[0m: ${@}"
+}
+
+message_success()
+{
+    echo -e "\e[32mSUCCESS\e[0m: ${@}"
+}

scripts/backup

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# Based on cookiecutter-django template
+# https://github.com/pydanny/cookiecutter-django/blob/master/%7B%7Bcookiecutter.project_slug%7D%7D/compose/production/postgres/maintenance/_sourced/messages.sh
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+
+working_dir="$(dirname ${0})"
+source "${working_dir}/_sourced/messages.sh"
+
+
+message_welcome "Backing up the '${POSTGRES_DB}' database..."
+
+
+if [[ "${POSTGRES_USER}" == "postgres" ]]; then
+    message_error "Backing up as 'postgres' user is not supported. Assign 'POSTGRES_USER' env with another one and try again."
+    exit 1
+fi
+
+export PGHOST="${POSTGRES_HOST}"
+export PGPORT="${POSTGRES_PORT}"
+export PGUSER="${POSTGRES_USER}"
+export PGPASSWORD="${POSTGRES_PASSWORD}"
+export PGDATABASE="${POSTGRES_DB}"  
+
+random_nonce="$(openssl rand -hex 6)"
+timestamp="$(date +'%Y_%m_%dT%H_%M_%S')"
+backup_directory="${timestamp}_${random_nonce}"
+backup_filename="${backup_directory}/backup_${timestamp}_${random_nonce}.gz"
+checksum_filename="${backup_filename}.sha256"
+unique_passphrase_file_path="${backup_filename}.pass"
+
+# Create backup directory
+mkdir "${backup_directory}"
+
+
+# Generate a unique passphrase for encrypting the databse dump
+openssl rand -hex 128 > "${unique_passphrase_file_path}"
+
+# Dump database
+pg_dump -O -x --exclude-table-data vehicles_tripwaypoint -v | gzip > "${backup_filename}"
+sha256sum "${backup_filename}" > "${checksum_filename}"
+
+# Encrypt the databse dump using the unique private key
+cat "${unique_passphrase_file_path}" | gpg --passphrase-fd 0 --batch --yes --output "${backup_filename}.enc" --symmetric --cipher-algo AES256 "${backup_filename}"
+rm "${backup_filename}"
+
+# Encrypt the unique passphrase
+public_encryption_key_path="public_encryption_key.key"
+echo ${ENCRYPTION_PUBLIC_KEY} | base64 -d > "${public_encryption_key_path}"
+private_encryption_key_path="private_encryption_key.key"
+echo ${ENCRYPTION_PRIVATE_KEY} | base64 -d > "${private_encryption_key_path}"
+encription_passphrase="${ENCRYPTION_PASSPHRASE}"
+openssl rsautl -encrypt -inkey "${public_encryption_key_path}" -pubin -in "${unique_passphrase_file_path}" -out "${unique_passphrase_file_path}.enc"
+rm -f "${unique_passphrase_file_path}"
+
+# Upload encrypted unique passphrase, encrypted database dump and checksum, zipped
+tar c -zvf "${backup_directory}.tar.gz" "${backup_directory}"
+set +e
+gsutil cp "${backup_directory}.tar.gz" "gs://${GOOGLE_BUCKET_NAME}${DATABASE_BACKUP_PATH}"; exit_code=$?
+set -e
+
+# Cleanup
+rm -f "${public_encryption_key_path}"
+rm -f "${private_encryption_key_path}"
+rm -rf ${backup_directory}
+rm -f "${backup_directory}.tar.gz"
+
+if [ $exit_code -ne 0 ]; then
+    message_error "'${POSTGRES_DB}' database backup '${backup_directory}.tar.gz' has been created but upload failed."
+else
+    message_success "'${POSTGRES_DB}' database backup '${backup_directory}.tar.gz' has been created and uploaded."
+fi;

scripts/backups

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Based on cookiecutter-django template
+# https://github.com/pydanny/cookiecutter-django/blob/master/%7B%7Bcookiecutter.project_slug%7D%7D/compose/production/postgres/maintenance/_sourced/messages.sh
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+
+working_dir="$(dirname ${0})"
+source "${working_dir}/_sourced/messages.sh"
+
+
+message_welcome "These are the backups you have got:"
+backups_path="gs://${GOOGLE_BUCKET_NAME}${DATABASE_BACKUP_PATH}"
+escaped_backups_path="$(echo $backups_path | sed 's/\//\\\//g')"
+
+gsutil ls -lh "${backups_path}" | grep gs:// | sed "s/$escaped_backups_path//g" | sed 's/.tar.gz//g' | sort -r -k 2

scripts/restore

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+# Based on cookiecutter-django template
+# https://github.com/pydanny/cookiecutter-django/blob/master/%7B%7Bcookiecutter.project_slug%7D%7D/compose/production/postgres/maintenance/_sourced/messages.sh
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+
+working_dir="$(dirname ${0})"
+source "${working_dir}/_sourced/messages.sh"
+
+
+if [[ -z ${1+x} ]]; then
+    message_error "Backup filename is not specified yet it is a required parameter. Make sure you provide one and try again."
+    exit 1
+fi
+
+backup_remote_full_path="gs://${GOOGLE_BUCKET_NAME}${DATABASE_BACKUP_PATH}${1}.tar.gz"
+echo $backup_remote_full_path
+
+if ! gsutil -q stat "${backup_remote_full_path}"; then
+    message_error "No backup with the specified filename found. Check out the 'backups' scripts script output to see if there is one and try again."
+    exit 1
+fi
+
+message_welcome "Restoring the '${POSTGRES_DB}' database from the '${1}' backup..."
+
+if [[ "${POSTGRES_USER}" == "postgres" ]]; then
+    message_error "Restoring as 'postgres' user is not supported. Assign 'POSTGRES_USER' env with another one and try again."
+    exit 1
+fi
+
+export PGHOST="${POSTGRES_HOST}"
+export PGPORT="${POSTGRES_PORT}"
+export PGUSER="${POSTGRES_USER}"
+export PGPASSWORD="${POSTGRES_PASSWORD}"
+export PGDATABASE="${POSTGRES_DB}"
+
+
+message_info "Downloading backup file..."
+gsutil cp "${backup_remote_full_path}" .
+
+message_info "Unzipping backup file..."
+tar x -zvf "${1}.tar.gz"
+rm -r "${1}.tar.gz"
+
+backup_directory="${1}"
+encrypted_backup_filename="${backup_directory}/backup_${1}.gz.enc"
+decrypted_backup_filename="${backup_directory}/backup_${1}.gz"
+encrypted_pass_filename="${backup_directory}/backup_${1}.gz.pass.enc"
+checksum_filename="${backup_directory}/backup_${1}.gz.sha256"
+
+message_info "Decrypting passphrase..."
+private_encryption_key_path="private_encryption_key.key"
+echo ${ENCRYPTION_PRIVATE_KEY} | base64 -d > "${private_encryption_key_path}"
+encription_passphrase="${ENCRYPTION_PASSPHRASE}"
+openssl rsautl -decrypt -inkey "${private_encryption_key_path}" -in "${encrypted_pass_filename}" -out "${encrypted_pass_filename}.dec"
+
+message_info "Decrypting backup file..."
+cat "${encrypted_pass_filename}.dec" | gpg --passphrase-fd 0 --batch --yes --output "${decrypted_backup_filename}" --decrypt "${encrypted_backup_filename}"
+
+message_info "Validating checksum..."
+if ! cat "${checksum_filename}" | sha256sum -c; then
+    message_error "Backup SHA256 checksum invalid!"
+    exit 1
+fi
+
+message_info "Dropping the database..."
+dropdb "${PGDATABASE}"
+
+message_info "Creating a new database..."
+createdb --owner="${POSTGRES_USER}" "${PGDATABASE}"
+
+message_info "Applying the backup to the new database..."
+gunzip -c "${decrypted_backup_filename}" | psql "${POSTGRES_DB}"
+
+message_info "Cleaning up..."
+rm -f "${private_encryption_key_path}"
+rm -rf "${backup_directory}"
+
+message_success "The '${POSTGRES_DB}' database has been restored from the '${1}' backup."