fix: keep trying to acquire lock in cache invalidator

Informatievlaanderen · Sep 30, 2024 · bc6a423 · bc6a423
1 parent 2b72da2
commit bc6a423
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 16 deletions.
diff --git a/src/BuildingRegistry.Cache.Invalidator/Infrastructure/Program.cs b/src/BuildingRegistry.Cache.Invalidator/Infrastructure/Program.cs
@@ -3,20 +3,19 @@ namespace BuildingRegistry.Cache.Invalidator.Infrastructure
     using System;
     using System.IO;
     using System.Linq;
+    using System.Threading;
     using System.Threading.Tasks;
-    using System.Timers;
     using Autofac;
     using Autofac.Extensions.DependencyInjection;
     using Be.Vlaanderen.Basisregisters.Aws.DistributedMutex;
-    using BuildingRegistry.Infrastructure;
     using Consumer.Read.Parcel;
     using Consumer.Read.Parcel.Infrastructure.Modules;
     using Destructurama;
-    using Microsoft.EntityFrameworkCore;
     using Microsoft.Extensions.Configuration;
     using Microsoft.Extensions.DependencyInjection;
     using Microsoft.Extensions.Hosting;
     using Microsoft.Extensions.Logging;
+    using Polly;
     using Serilog;
     using Serilog.Debugging;
     using Serilog.Extensions.Logging;
@@ -65,9 +64,7 @@ public static async Task Main(string[] args)
                     builder.ClearProviders();
                     builder.AddSerilog(Log.Logger);
                 })
-                .ConfigureServices((hostContext, services) =>
-                {
-                })
+                .ConfigureServices((hostContext, services) => { })
                 .UseServiceProviderFactory(new AutofacServiceProviderFactory())
                 .ConfigureContainer<ContainerBuilder>((hostContext, builder) =>
                 {
@@ -103,17 +100,27 @@ public static async Task Main(string[] args)
                 if (consumerParcelContext.BuildingsToInvalidate.Any())
                 {
                     var distributedLockTableName = configuration.GetValue<string>("DistributedLock:LockName")
-                        ?? throw new Exception("No 'LockName' configuration found");
+                                                   ?? throw new Exception("No 'LockName' configuration found");
 
                     var distributedLockOptions = DistributedLockOptions.LoadFromConfiguration(configuration);
-                    var distributedLock = new DistributedLock<Program>(distributedLockOptions, distributedLockTableName, loggerFactory.CreateLogger<Program>());
-
-                    await distributedLock.RunAsync(
-                            async () =>
-                            {
-                                await host.RunAsync().ConfigureAwait(false);
-                            })
-                        .ConfigureAwait(false);
+                    var distributedLock = new DistributedLock<Program>(
+                        distributedLockOptions,
+                        distributedLockTableName,
+                        loggerFactory.CreateLogger<Program>());
+
+                    await Policy
+                        .Handle<AcquireLockFailedException>()
+                        .WaitAndRetryAsync(5, _ =>
+                        {
+                            Log.Information("Failed to acquire lock. Trying again within 1 minute.");
+                            return TimeSpan.FromMinutes(1);
+                        })
+                        .ExecuteAsync(async ct =>
+                        {
+                            await distributedLock.RunAsync(
+                                    async () => { await host.RunAsync(token: ct).ConfigureAwait(false); })
+                                .ConfigureAwait(false);
+                        }, CancellationToken.None);
                 }
             }
             catch (AggregateException aggregateException)

diff --git a/src/BuildingRegistry.Cache.Invalidator/appsettings.json b/src/BuildingRegistry.Cache.Invalidator/appsettings.json
@@ -12,8 +12,9 @@
     "Region": "eu-west-1",
     "TableName": "__DistributedLocks__",
     "LeasePeriodInMinutes": 5,
+    "ThrowOnFailedAcquire": true,
     "TerminateApplicationOnFailedRenew": true,
-    "TerminateApplicationOnFailedAcquire": true,
+    "TerminateApplicationOnFailedAcquire": false,
     "LockName": "Be.Vlaanderen.Basisregisters.Redis.Populator.Program"
   },