fix: Send crash dump to S3 bucket

Dockerfile

@@ -93,7 +93,7 @@ RUN mix release
 # the compiled release and other runtime necessities
 FROM ${RUNNER_IMAGE}
 
-RUN apt-get update -y && apt-get install -y libstdc++6 openssl libncurses5 locales iptables sudo tini \
+RUN apt-get update -y && apt-get install -y libstdc++6 openssl libncurses5 locales iptables sudo tini curl \
   && apt-get clean && rm -f /var/lib/apt/lists/*_*
 
 # Set the locale

docker-compose.yml

@@ -8,7 +8,7 @@ services:
       - "5432:5432"
     volumes:
       - ./dev/postgres:/docker-entrypoint-initdb.d/
-    command: postgres -c config_file=/etc/postgresql/postgresql.conf 
+    command: postgres -c config_file=/etc/postgresql/postgresql.conf
     environment:
       POSTGRES_HOST: /var/run/postgresql
       POSTGRES_PASSWORD: postgres
@@ -37,4 +37,38 @@ services:
       ERL_AFLAGS: -proto_dist inet_tcp
       ENABLE_TAILSCALE: "false"
       DNS_NODES: "''"
-    command: sh -c "/app/bin/migrate && /app/bin/realtime eval 'Realtime.Release.seeds(Realtime.Repo)' && /app/bin/server"
+      ENABLE_ERL_CRASH_DUMP: true
+      ERL_CRASH_DUMP_FOLDER: /tmp
+      ERL_CRASH_DUMP_FILE_NAME: erl_crash.dump
+      ERL_CRASH_DUMP_S3_ENDPOINT: http://minio:9000
+      ERL_CRASH_DUMP_S3_REGION: us-east-1
+      ERL_CRASH_DUMP_S3_BUCKET: realtime
+      ERL_CRASH_DUMP_S3_KEY: minio-access-key
+      ERL_CRASH_DUMP_S3_SECRET: minio-secret-key
+      ERL_CRASH_DUMP_S3_HOST: minio
+      ERL_CRASH_DUMP_S3_PORT: 9000
+      ERL_CRASH_DUMP_S3_SSL: false
+      command: sh -c "/app/bin/migrate && /app/bin/realtime eval 'Realtime.Release.seeds(Realtime.Repo)' && /app/bin/server"
+
+  minio:
+    image: minio/minio
+    container_name: realtime-minio
+    ports:
+      - "9000:9000"
+    environment:
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: minio123
+    command: server /data
+
+  createbuckets:
+    image: minio/mc
+    depends_on:
+      - minio
+    entrypoint: >
+      /bin/sh -c "
+      mc alias set realtime-minio http://minio:9000 minio minio123;
+      mc admin user add realtime-minio user minio-secret-key;
+      mc admin user svcacct add --access-key minio-access-key --secret-key minio-secret-key realtime-minio user;
+      mc mb realtime-minio/realtime;
+      mc admin policy attach realtime-minio readwrite --user user
+      "

mix.exs

@@ -4,7 +4,7 @@ defmodule Realtime.MixProject do
   def project do
     [
       app: :realtime,
-      version: "2.25.28",
+      version: "2.25.29",
       elixir: "~> 1.14.0",
       elixirc_paths: elixirc_paths(Mix.env()),
       start_permanent: Mix.env() == :prod,

tailscale/wrapper.sh

@@ -10,6 +10,52 @@ if [ "${ENABLE_TAILSCALE-}" = true ]; then
     /tailscale/tailscale up --authkey=${TAILSCALE_AUTHKEY} --hostname="${TAILSCALE_APP_NAME}" --accept-routes=true
 fi
 
+export ERL_CRASH_DUMP=/tmp/erl_crash.dump
+
+function upload_crash_dump_to_s3 {
+    EXIT_CODE=${?:-0}
+
+    bucket=$ERL_CRASH_DUMP_S3_BUCKET
+    s3Key=$ERL_CRASH_DUMP_S3_KEY
+    s3Secret=$ERL_CRASH_DUMP_S3_SECRET
+    s3Host=$ERL_CRASH_DUMP_S3_HOST
+    s3Port=$ERL_CRASH_DUMP_S3_PORT
+
+    filePath=${ERL_CRASH_DUMP_FOLDER:-tmp}/$(date +%s)_${ERL_CRASH_DUMP_FILE_NAME:-erl_crash.dump}
+
+    if [ -f "${ERL_CRASH_DUMP_FOLDER:-tmp}/${ERL_CRASH_DUMP_FILE_NAME:-erl_crash.dump}" ]; then
+        mv ${ERL_CRASH_DUMP_FOLDER:-tmp}/${ERL_CRASH_DUMP_FILE_NAME:-erl_crash.dump} $filePath
+
+        resource="/${bucket}${filePath}"
+
+        contentType="application/octet-stream"
+        dateValue=$(date -R)
+        stringToSign="PUT\n\n${contentType}\n${dateValue}\n${resource}"
+
+        signature=$(echo -en ${stringToSign} | openssl sha1 -hmac ${s3Secret} -binary | base64)
+
+        if [ "${S3_USE_HTTPS:-}" = true ]; then
+            protocol="https"
+        else
+            protocol="http"
+        fi
+
+        curl -v -X PUT -T "${filePath}" \
+            -H "Host: ${s3Host}" \
+            -H "Date: ${dateValue}" \
+            -H "Content-Type: ${contentType}" \
+            -H "Authorization: AWS ${s3Key}:${signature}" \
+            ${protocol}://${s3Host}:${s3Port}${resource}
+    fi
+
+    exit "$EXIT_CODE"
+}
+
+if [ "${ENABLE_ERL_CRASH_DUMP-}" = true ]; then
+    echo "trap?"
+    trap upload_crash_dump_to_s3 SIGINT SIGTERM SIGKILL EXIT
+fi
+
 echo "Starting Realtime"
 
 if [ "${AWS_EXECUTION_ENV:=none}" = "AWS_ECS_FARGATE" ]; then