From b86d46b0b594a435f60169bf27af44c8c3d741db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Filipe=20Caba=C3=A7o?= <filipecabaco@gmail.com>
Date: Tue, 14 Nov 2023 00:03:41 +0000
Subject: [PATCH] fix: Send crash dump to S3 bucket

Currently we're not storing the Erlang Crash dumps. This changes allows to trap exist signals, fetch the crash dump and send it to an S3 bucket.

Can be emulated locally by running:
1. `docker compose up`
2. `docker exec -it realtime-server bin/realtime remote`
3. `Process.whereis(:application_controller) |> Process.exit(:kill)`
---
 Dockerfile           |  2 +-
 docker-compose.yml   | 38 ++++++++++++++++++++++++++++++++++--
 mix.exs              |  2 +-
 tailscale/wrapper.sh | 46 ++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 84 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index d83325f7d..e8b87bc99 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -93,7 +93,7 @@ RUN mix release
 # the compiled release and other runtime necessities
 FROM ${RUNNER_IMAGE}
 
-RUN apt-get update -y && apt-get install -y libstdc++6 openssl libncurses5 locales iptables sudo tini \
+RUN apt-get update -y && apt-get install -y libstdc++6 openssl libncurses5 locales iptables sudo tini curl \
   && apt-get clean && rm -f /var/lib/apt/lists/*_*
 
 # Set the locale
diff --git a/docker-compose.yml b/docker-compose.yml
index df916802c..776dcf1c1 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -8,7 +8,7 @@ services:
       - "5432:5432"
     volumes:
       - ./dev/postgres:/docker-entrypoint-initdb.d/
-    command: postgres -c config_file=/etc/postgresql/postgresql.conf 
+    command: postgres -c config_file=/etc/postgresql/postgresql.conf
     environment:
       POSTGRES_HOST: /var/run/postgresql
       POSTGRES_PASSWORD: postgres
@@ -37,4 +37,38 @@ services:
       ERL_AFLAGS: -proto_dist inet_tcp
       ENABLE_TAILSCALE: "false"
       DNS_NODES: "''"
-    command: sh -c "/app/bin/migrate && /app/bin/realtime eval 'Realtime.Release.seeds(Realtime.Repo)' && /app/bin/server"
+      ENABLE_ERL_CRASH_DUMP: true
+      ERL_CRASH_DUMP_FOLDER: /tmp
+      ERL_CRASH_DUMP_FILE_NAME: erl_crash.dump
+      ERL_CRASH_DUMP_S3_ENDPOINT: http://minio:9000
+      ERL_CRASH_DUMP_S3_REGION: us-east-1
+      ERL_CRASH_DUMP_S3_BUCKET: realtime
+      ERL_CRASH_DUMP_S3_KEY: minio-access-key
+      ERL_CRASH_DUMP_S3_SECRET: minio-secret-key
+      ERL_CRASH_DUMP_S3_HOST: minio
+      ERL_CRASH_DUMP_S3_PORT: 9000
+      ERL_CRASH_DUMP_S3_SSL: false
+      command: sh -c "/app/bin/migrate && /app/bin/realtime eval 'Realtime.Release.seeds(Realtime.Repo)' && /app/bin/server"
+
+  minio:
+    image: minio/minio
+    container_name: realtime-minio
+    ports:
+      - "9000:9000"
+    environment:
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: minio123
+    command: server /data
+
+  createbuckets:
+    image: minio/mc
+    depends_on:
+      - minio
+    entrypoint: >
+      /bin/sh -c "
+      mc alias set realtime-minio http://minio:9000 minio minio123;
+      mc admin user add realtime-minio user minio-secret-key;
+      mc admin user svcacct add --access-key minio-access-key --secret-key minio-secret-key realtime-minio user;
+      mc mb realtime-minio/realtime;
+      mc admin policy attach realtime-minio readwrite --user user
+      "
\ No newline at end of file
diff --git a/mix.exs b/mix.exs
index 22af70e23..9e6ca77ed 100644
--- a/mix.exs
+++ b/mix.exs
@@ -4,7 +4,7 @@ defmodule Realtime.MixProject do
   def project do
     [
       app: :realtime,
-      version: "2.25.28",
+      version: "2.25.29",
       elixir: "~> 1.14.0",
       elixirc_paths: elixirc_paths(Mix.env()),
       start_permanent: Mix.env() == :prod,
diff --git a/tailscale/wrapper.sh b/tailscale/wrapper.sh
index 620029ba1..527f10bec 100755
--- a/tailscale/wrapper.sh
+++ b/tailscale/wrapper.sh
@@ -10,6 +10,52 @@ if [ "${ENABLE_TAILSCALE-}" = true ]; then
     /tailscale/tailscale up --authkey=${TAILSCALE_AUTHKEY} --hostname="${TAILSCALE_APP_NAME}" --accept-routes=true
 fi
 
+export ERL_CRASH_DUMP=/tmp/erl_crash.dump
+
+function upload_crash_dump_to_s3 {
+    EXIT_CODE=${?:-0}
+
+    bucket=$ERL_CRASH_DUMP_S3_BUCKET
+    s3Key=$ERL_CRASH_DUMP_S3_KEY
+    s3Secret=$ERL_CRASH_DUMP_S3_SECRET
+    s3Host=$ERL_CRASH_DUMP_S3_HOST
+    s3Port=$ERL_CRASH_DUMP_S3_PORT
+
+    filePath=${ERL_CRASH_DUMP_FOLDER:-tmp}/$(date +%s)_${ERL_CRASH_DUMP_FILE_NAME:-erl_crash.dump}
+
+    if [ -f "${ERL_CRASH_DUMP_FOLDER:-tmp}/${ERL_CRASH_DUMP_FILE_NAME:-erl_crash.dump}" ]; then
+        mv ${ERL_CRASH_DUMP_FOLDER:-tmp}/${ERL_CRASH_DUMP_FILE_NAME:-erl_crash.dump} $filePath
+
+        resource="/${bucket}${filePath}"
+
+        contentType="application/octet-stream"
+        dateValue=$(date -R)
+        stringToSign="PUT\n\n${contentType}\n${dateValue}\n${resource}"
+
+        signature=$(echo -en ${stringToSign} | openssl sha1 -hmac ${s3Secret} -binary | base64)
+
+        if [ "${S3_USE_HTTPS:-}" = true ]; then
+            protocol="https"
+        else
+            protocol="http"
+        fi
+
+        curl -v -X PUT -T "${filePath}" \
+            -H "Host: ${s3Host}" \
+            -H "Date: ${dateValue}" \
+            -H "Content-Type: ${contentType}" \
+            -H "Authorization: AWS ${s3Key}:${signature}" \
+            ${protocol}://${s3Host}:${s3Port}${resource}
+    fi
+
+    exit "$EXIT_CODE"
+}
+
+if [ "${ENABLE_ERL_CRASH_DUMP-}" = true ]; then
+    echo "trap?"
+    trap upload_crash_dump_to_s3 SIGINT SIGTERM SIGKILL EXIT
+fi
+
 echo "Starting Realtime"
 
 if [ "${AWS_EXECUTION_ENV:=none}" = "AWS_ECS_FARGATE" ]; then