Merge pull request #1669 from serlo/redis-harderning

feat: Use the database while trying to reconnect to Redis
serlo · Jul 18, 2024 · 1f27bb7 · 1f27bb7
2 parents cf8422f + 8e0dfad
commit 1f27bb7
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 9 deletions.
diff --git a/packages/server/src/cache.ts b/packages/server/src/cache.ts
@@ -5,6 +5,7 @@ import createMsgpack from 'msgpack5'
 import * as R from 'ramda'
 import Redlock from 'redlock'
 
+import { captureErrorEvent } from './error-event'
 import { Priority, Cache, CacheEntry } from '~/context/cache'
 import { timeToMilliseconds, Time, Timer } from '~/timer'
 import { FunctionOrValue, isUpdateFunction } from '~/utils'
@@ -27,14 +28,22 @@ export function createCache({ timer }: { timer: Timer }): Cache {
       if (times * delay > 300_000) throw new Error('Redis connection timed out')
       return delay
     },
+    maxRetriesPerRequest: 3,
+  })
+
+  client.on('error', (error) => {
+    captureErrorEvent({
+      error,
+      location: 'cache',
+    })
   })
 
   const lockManagers: Record<Priority, LockManager> = {
     [Priority.Low]: createLockManager({
       retryCount: 0,
     }),
     [Priority.High]: createLockManager({
-      retryCount: 10,
+      retryCount: 5,
     }),
   }
 
@@ -178,6 +187,13 @@ function createLockManager({
   retryCount: number
 }): LockManager {
   const client = new Redis(process.env.REDIS_URL)
+  client.on('error', (error) => {
+    captureErrorEvent({
+      error,
+      location: 'Cache lockManager',
+    })
+    client.disconnect()
+  })
   const redlock = new Redlock([client], { retryCount })
 
   return {

diff --git a/packages/server/src/cached-resolver.ts b/packages/server/src/cached-resolver.ts
@@ -15,7 +15,12 @@ export function createCachedResolver<P, R>(
     __typename: 'CachedResolver',
     async resolveWithDecoder(customDecoder, payload, context) {
       const key = spec.getKey(payload)
-      const cacheValue = await context.cache.get<R>({ key, maxAge })
+      const cacheValue = await context.cache
+        .get<R>({ key, maxAge })
+        .catch(() => {
+          // probably we don't have a connection to redis
+          return O.none
+        })
 
       if (O.isSome(cacheValue)) {
         const cacheEntry = cacheValue.value
@@ -38,13 +43,16 @@ export function createCachedResolver<P, R>(
       const value = await spec.getCurrentValue(payload, context)
 
       if (customDecoder.is(value)) {
-        await context.cache.set({
-          key,
-          value,
-          source: 'API: From a call to a data source',
-          ttlInSeconds,
-        })
-
+        await context.cache
+          .set({
+            key,
+            value,
+            source: 'API: From a call to a data source',
+            ttlInSeconds,
+          })
+          .catch(() => {
+            // probably we don't have a connection to redis
+          })
         return value
       } else {
         throw new InvalidCurrentValueError({

diff --git a/packages/server/src/internals/swr-queue.ts b/packages/server/src/internals/swr-queue.ts
@@ -64,6 +64,7 @@ export function createSwrQueue({
     isWorker: false,
     removeOnFailure: true,
     removeOnSuccess: true,
+    autoConnect: false,
   })
 
   queue.on('error', (error) => {