Merge pull request #502 from radixdlt/release/babylon

Release/babylon -> main

.editorconfig

@@ -276,6 +276,7 @@ dotnet_diagnostic.SA1024.severity = none # SA1024ColonsMustBeSpacedCorrectly - b
 dotnet_diagnostic.SA1600.severity = none # SA1600ElementsMustBeDocumented - Duplicates CS1591 - let's just use CS1591 instead
 dotnet_diagnostic.SA1615.severity = none # SA1615ElementReturnValueMustBeDocumented
 dotnet_diagnostic.SA1611.severity = none # SA1611ElementParametersMustBeDocumented
+dotnet_diagnostic.SA1618.severity = none # SA1618GenericTypeParametersMustBeDocumented
 dotnet_diagnostic.SA1602.severity = none # SA1602EnumerationItemsMustBeDocumented
 dotnet_diagnostic.SA1601.severity = none # SA1601PartialElementsMustBeDocumented
 

.github/workflows/releases.yml

@@ -264,9 +264,9 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - setup-tags
-      - docker-database-migrations-dockerhub
-      - docker-data-aggregator-dockerhub
-      - docker-gateway-api-dockerhub
+      - join-gateway-images
+      - join-aggregator-images
+      - join-migrations-images
     permissions:
       id-token: write
       pull-requests: read

CHANGELOG.md

@@ -1,11 +1,12 @@
-## Next release (name TBC)
+## 1.0.0 - Babylon Launch
 Release Date: _unreleased_
 
 ### Breaking changes
 _None_
 
 ### What’s new?
-_N/A_
+- log warning if sql query takes longer than configured threshold (default to 250ms) for both entity framework and dapper queries.
+- gather execution time metrics for all sql queries (both entity framework and dapper).
 
 ## 1.0.0-rc3 - Babylon Launch (Release Candidate 3)
 Release Date: Friday 22nd September 2023

Directory.Build.props

@@ -13,7 +13,7 @@
   </PropertyGroup>
 
   <PropertyGroup>
-    <VersionPrefix>0.5.4</VersionPrefix>
+    <VersionPrefix>1.0.0</VersionPrefix>
     <VersionSuffix>develop</VersionSuffix>
   </PropertyGroup>
 

Directory.Packages.props

@@ -22,7 +22,7 @@
     <PackageVersion Include="prometheus-net" Version="8.0.1" />
     <PackageVersion Include="prometheus-net.AspNetCore" Version="8.0.1" />
     <PackageVersion Include="prometheus-net.AspNetCore.HealthChecks" Version="8.0.1" />
-    <PackageVersion Include="RadixDlt.RadixEngineToolkit" Version="1.0.0-commit-dc89718" />
+    <PackageVersion Include="RadixDlt.RadixEngineToolkit" Version="1.0.0" />
     <PackageVersion Include="Swashbuckle.AspNetCore" Version="6.5.0" />
     <!-- build time dependencies -->
     <PackageVersion Include="coverlet.collector" Version="6.0.0" />
@@ -33,4 +33,4 @@
     <PackageVersion Include="xunit" Version="2.5.1" />
     <PackageVersion Include="xunit.runner.visualstudio" Version="2.5.1" />
   </ItemGroup>
-</Project>
+</Project>

apps/GatewayApi/GatewayApiStartup.cs

@@ -64,18 +64,15 @@
 
 using GatewayApi.SlowRequestLogging;
 using Microsoft.AspNetCore.Builder;
-using Microsoft.AspNetCore.HttpOverrides;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Newtonsoft.Json;
-using Newtonsoft.Json.Converters;
 using Prometheus;
 using RadixDlt.NetworkGateway.GatewayApi;
 using RadixDlt.NetworkGateway.PostgresIntegration;
 using RadixDlt.NetworkGateway.PrometheusIntegration;
 using System;
-using System.Globalization;
 
 namespace GatewayApi;
 

apps/GatewayApi/SlowRequestLogging/BufferingStream.cs

@@ -70,7 +70,6 @@
 using System.Buffers;
 using System.Diagnostics;
 using System.IO;
-using System.IO.Pipelines;
 using System.Runtime.CompilerServices;
 using System.Text;
 using System.Threading;

src/RadixDlt.NetworkGateway.Abstractions/Configuration/SlowQueryLoggingOptions.cs

@@ -0,0 +1,17 @@
+using FluentValidation;
+using System;
+
+namespace RadixDlt.NetworkGateway.Abstractions.Configuration;
+
+public sealed class SlowQueryLoggingOptions
+{
+    public TimeSpan SlowQueryThreshold { get; set; } = TimeSpan.FromMilliseconds(250);
+}
+
+public class SlowQueryLoggingOptionsValidator : AbstractOptionsValidator<SlowQueryLoggingOptions>
+{
+    public SlowQueryLoggingOptionsValidator()
+    {
+        RuleFor(x => x.SlowQueryThreshold).GreaterThan(TimeSpan.Zero);
+    }
+}

src/RadixDlt.NetworkGateway.Abstractions/Extensions/ExceptionExtensions.cs

@@ -62,7 +62,6 @@
  * permissions under this License.
  */
 
-using RadixDlt.NetworkGateway.Abstractions.CoreCommunications;
 using RadixDlt.NetworkGateway.Abstractions.Exceptions;
 using System;
 using System.Linq;

src/RadixDlt.NetworkGateway.DataAggregator/Configuration/NodeWorkersOptions.cs

@@ -0,0 +1,23 @@
+using FluentValidation;
+using Microsoft.Extensions.Configuration;
+using RadixDlt.NetworkGateway.Abstractions.Configuration;
+
+namespace RadixDlt.NetworkGateway.DataAggregator.Configuration;
+
+public sealed class NodeWorkersOptions
+{
+    [ConfigurationKeyName("ErrorStartupBlockTimeSeconds")]
+    public int ErrorStartupBlockTimeSeconds { get; set; } = 20;
+
+    [ConfigurationKeyName("GraceSecondsBeforeMarkingStalled")]
+    public int GraceSecondsBeforeMarkingStalled { get; set; } = 10;
+}
+
+internal class NodeWorkersOptionsValidator : AbstractOptionsValidator<NodeWorkersOptions>
+{
+    public NodeWorkersOptionsValidator()
+    {
+        RuleFor(x => x.ErrorStartupBlockTimeSeconds).GreaterThan(0);
+        RuleFor(x => x.GraceSecondsBeforeMarkingStalled).GreaterThan(0);
+    }
+}

src/RadixDlt.NetworkGateway.DataAggregator/NodeServices/ApiReaders/NetworkStatusReader.cs

@@ -62,7 +62,6 @@
  * permissions under this License.
  */
 
-using RadixDlt.NetworkGateway.Abstractions.CoreCommunications;
 using RadixDlt.NetworkGateway.Abstractions.Extensions;
 using RadixDlt.NetworkGateway.DataAggregator.Services;
 using System;

src/RadixDlt.NetworkGateway.DataAggregator/NodeServices/NodeConfigProvider.cs

@@ -62,7 +62,6 @@
  * permissions under this License.
  */
 
-using RadixDlt.NetworkGateway.Abstractions.Configuration;
 using RadixDlt.NetworkGateway.DataAggregator.Configuration;
 
 namespace RadixDlt.NetworkGateway.DataAggregator.NodeServices;

src/RadixDlt.NetworkGateway.DataAggregator/NodeServices/NodeWorkersRunner.cs

@@ -64,9 +64,11 @@
 
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 using RadixDlt.NetworkGateway.Abstractions;
 using RadixDlt.NetworkGateway.Abstractions.Exceptions;
 using RadixDlt.NetworkGateway.Abstractions.Extensions;
+using RadixDlt.NetworkGateway.DataAggregator.Configuration;
 using RadixDlt.NetworkGateway.DataAggregator.Workers.NodeWorkers;
 using System;
 using System.Collections.Generic;
@@ -107,6 +109,7 @@ private set
     private readonly object _statusLock = new();
     private readonly ILogger<NodeWorkersRunner> _logger;
     private readonly IClock _clock;
+    private readonly IOptionsMonitor<NodeWorkersOptions> _nodeWorkersOptions;
 
     private IServiceScope? _nodeDependencyInjectionScope;
 
@@ -119,12 +122,13 @@ private set
     private IDisposable? _logScope;
     private NodeWorkersRunnerStatus _status;
 
-    public NodeWorkersRunner(ILogger<NodeWorkersRunner> logger, IServiceScope nodeDependencyInjectionScope, IDisposable logScope, IClock clock)
+    public NodeWorkersRunner(ILogger<NodeWorkersRunner> logger, IServiceScope nodeDependencyInjectionScope, IDisposable logScope, IClock clock, IOptionsMonitor<NodeWorkersOptions> nodeWorkersOptions)
     {
         _logger = logger;
         _nodeDependencyInjectionScope = nodeDependencyInjectionScope;
         _logScope = logScope;
         _clock = clock;
+        _nodeWorkersOptions = nodeWorkersOptions;
         _cancellationTokenSource = new CancellationTokenSource();
         _initializers = nodeDependencyInjectionScope.ServiceProvider.GetServices<INodeInitializer>().ToList();
         _workers = nodeDependencyInjectionScope.ServiceProvider.GetServices<INodeWorker>().ToList();
@@ -136,16 +140,16 @@ public NodeWorkersRunner(ILogger<NodeWorkersRunner> logger, IServiceScope nodeDe
     /// </summary>
     public bool IsHealthy()
     {
-        const int GraceSecondsBeforeMarkingStalled = 10;
+        var graceSecondsBeforeMarkingStalled = _nodeWorkersOptions.CurrentValue.GraceSecondsBeforeMarkingStalled;
 
-        var isRunningOrNotStalled = Status == NodeWorkersRunnerStatus.Running || _lastStatusChange.WithinPeriodOfNow(TimeSpan.FromSeconds(GraceSecondsBeforeMarkingStalled), _clock);
+        var isRunningOrNotStalled = Status == NodeWorkersRunnerStatus.Running || _lastStatusChange.WithinPeriodOfNow(TimeSpan.FromSeconds(graceSecondsBeforeMarkingStalled), _clock);
         if (!isRunningOrNotStalled)
         {
             _logger.LogWarning(
                 "Marked as unhealthy because current status is {Status} and last status changes was at {LastStatusChange}, longer than {GracePeriod}s ago - suggesting that the WorkersRegistry hasn't properly handled something, and these NodeWorkers should be restarted",
                 Status,
                 _lastStatusChange,
-                GraceSecondsBeforeMarkingStalled
+                graceSecondsBeforeMarkingStalled
             );
             return false;
         }

src/RadixDlt.NetworkGateway.DataAggregator/ServiceCollectionExtensions.cs

@@ -99,8 +99,10 @@ public static DataAggregatorBuilder AddNetworkGatewayDataAggregatorCore(this ISe
 
         services
             .AddValidatableOptionsAtSection<NetworkOptions, NetworkOptionsValidator>("DataAggregator:Network")
+            .AddValidatableOptionsAtSection<SlowQueryLoggingOptions, SlowQueryLoggingOptionsValidator>("DataAggregator:SlowQueryLogging")
             .AddValidatableOptionsAtSection<MonitoringOptions, MonitoringOptionsValidator>("DataAggregator:Monitoring")
             .AddValidatableOptionsAtSection<MempoolOptions, MempoolOptionsValidator>("DataAggregator:Mempool")
+            .AddValidatableOptionsAtSection<NodeWorkersOptions, NodeWorkersOptionsValidator>("DataAggregator:NodeWorkers")
             .AddValidatableOptionsAtSection<LedgerConfirmationOptions, LedgerConfirmationOptionsValidator>("DataAggregator:LedgerConfirmation")
             .AddValidatableOptionsAtSection<TransactionAssertionsOptions, TransactionAssertionsOptionsValidator>("DataAggregator:TransactionAssertions");
 

src/RadixDlt.NetworkGateway.DataAggregator/Services/FetchedTransactionStore.cs

@@ -84,7 +84,7 @@ public interface IFetchedTransactionStore
 
     void StoreNodeTransactions(string nodeName, List<CoreModel.CommittedTransaction> transactions, int responseSize);
 
-    bool ShouldFetchNewTransactions(string nodeName, long fromStateVersion);
+    bool ShouldFetchNewTransactions(string nodeName, long committedStateVersion);
 
     long GetFirstStateVersionToFetch(string nodeName, long lastCommittedStateVersion);
 }
@@ -172,12 +172,12 @@ public void RemoveProcessedTransactions(long committedStateVersion)
             .ToList();
     }
 
-    public bool ShouldFetchNewTransactions(string nodeName, long fromStateVersion)
+    public bool ShouldFetchNewTransactions(string nodeName, long committedStateVersion)
     {
         var transactionStore = GetTransactionsStoreForNode(nodeName);
 
         var storedTransactionsSize = transactionStore
-            .Where(x => x.Key >= fromStateVersion)
+            .Where(x => x.Key > committedStateVersion)
             .Aggregate(0, (sum, current) => sum + current.Value.EstimatedSize);
 
         var shouldFetchTransactions = storedTransactionsSize < Config.MaxEstimatedTransactionPipelineByteSizePerNode;

src/RadixDlt.NetworkGateway.DataAggregator/Services/IPendingTransactionPrunerServiceObserver.cs

@@ -62,7 +62,6 @@
  * permissions under this License.
  */
 
-using RadixDlt.NetworkGateway.Abstractions.Model;
 using System.Collections.Generic;
 using System.Threading.Tasks;
 

src/RadixDlt.NetworkGateway.DataAggregator/Services/LedgerTransactionsProcessor.cs

@@ -71,7 +71,6 @@
 using RadixDlt.NetworkGateway.DataAggregator.Exceptions;
 using RadixDlt.NetworkGateway.DataAggregator.Monitoring;
 using System.Collections.Generic;
-using System.Diagnostics;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;

src/RadixDlt.NetworkGateway.DataAggregator/Services/NetworkConfigurationProvider.cs

@@ -62,8 +62,6 @@
  * permissions under this License.
  */
 
-using RadixDlt.NetworkGateway.Abstractions;
-using RadixDlt.NetworkGateway.Abstractions.Addressing;
 using RadixDlt.NetworkGateway.Abstractions.CoreCommunications;
 using System.Threading;
 using System.Threading.Tasks;

src/RadixDlt.NetworkGateway.DataAggregator/Services/NodeStatusProvider.cs

@@ -64,12 +64,9 @@
 
 using RadixDlt.NetworkGateway.Abstractions.Extensions;
 using RadixDlt.NetworkGateway.DataAggregator.NodeServices;
-using RadixDlt.NetworkGateway.DataAggregator.NodeServices.ApiReaders;
-using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
-using System.Threading;
 using CoreModel = RadixDlt.CoreApiSdk.Model;
 
 namespace RadixDlt.NetworkGateway.DataAggregator.Services;

src/RadixDlt.NetworkGateway.DataAggregator/Services/NodeWorkersRunnerFactory.cs

@@ -64,6 +64,7 @@
 
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 using RadixDlt.NetworkGateway.Abstractions;
 using RadixDlt.NetworkGateway.DataAggregator.Configuration;
 using RadixDlt.NetworkGateway.DataAggregator.NodeServices;
@@ -97,6 +98,7 @@ public NodeWorkersRunner CreateWorkersForNode(CoreApiNode initialCoreApiNode)
         var nodeScope = _services.CreateScope();
         nodeScope.ServiceProvider.GetRequiredService<INodeConfigProvider>().CoreApiNode = initialCoreApiNode;
         var logScope = _logger.BeginScope($"[NODE: {initialCoreApiNode.Name}]");
-        return new NodeWorkersRunner(_logger, nodeScope, logScope!, nodeScope.ServiceProvider.GetRequiredService<IClock>());
+        return new NodeWorkersRunner(_logger, nodeScope, logScope!, nodeScope.ServiceProvider.GetRequiredService<IClock>(),
+            nodeScope.ServiceProvider.GetRequiredService<IOptionsMonitor<NodeWorkersOptions>>());
     }
 }

src/RadixDlt.NetworkGateway.DataAggregator/Services/NodeWorkersRunnerRegistry.cs

@@ -63,6 +63,7 @@
  */
 
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 using RadixDlt.NetworkGateway.Abstractions.Extensions;
 using RadixDlt.NetworkGateway.DataAggregator.Configuration;
 using RadixDlt.NetworkGateway.DataAggregator.NodeServices;
@@ -83,18 +84,18 @@ public interface INodeWorkersRunnerRegistry
 
 internal class NodeWorkersRunnerRegistry : INodeWorkersRunnerRegistry
 {
-    private const int ErrorStartupBlockTimeSeconds = 20;
-
     private readonly ILogger<INodeWorkersRunnerRegistry> _logger;
+    private readonly IOptionsMonitor<NodeWorkersOptions> _nodeWorkersOptions;
     private readonly INodeWorkersRunnerFactory _nodeWorkersRunnerFactory;
     private readonly Dictionary<CoreApiNode, NodeWorkersRunner> _servicesMap = new();
     private readonly Dictionary<string, Task> _startupBlocklist = new();
     private readonly object _servicesMapLock = new();
 
-    public NodeWorkersRunnerRegistry(ILogger<NodeWorkersRunnerRegistry> logger, INodeWorkersRunnerFactory nodeWorkersRunnerFactory)
+    public NodeWorkersRunnerRegistry(ILogger<NodeWorkersRunnerRegistry> logger, INodeWorkersRunnerFactory nodeWorkersRunnerFactory, IOptionsMonitor<NodeWorkersOptions> nodeWorkersOptions)
     {
         _logger = logger;
         _nodeWorkersRunnerFactory = nodeWorkersRunnerFactory;
+        _nodeWorkersOptions = nodeWorkersOptions;
     }
 
     public async Task EnsureCorrectNodeServicesRunning(List<CoreApiNode> enabledNodes, CancellationToken cancellationToken)
@@ -185,16 +186,18 @@ private async Task CreateAndStartNodeWorkersIfNotExists(CoreApiNode coreApiNode,
                 throw;
             }
 
+            var errorStartupBlockTimeSeconds = _nodeWorkersOptions.CurrentValue.ErrorStartupBlockTimeSeconds;
+
             _logger.LogError(
                 ex,
                 "Error initializing or starting up services for node: {NodeName}. We won't try again for {ErrorStartupBlockTimeSeconds} seconds. Now clearing up...",
                 coreApiNode.Name,
-                ErrorStartupBlockTimeSeconds
+                errorStartupBlockTimeSeconds
             );
 
             lock (_servicesMapLock)
             {
-                _startupBlocklist[coreApiNode.Name] = Task.Delay(TimeSpan.FromSeconds(ErrorStartupBlockTimeSeconds), cancellationToken);
+                _startupBlocklist[coreApiNode.Name] = Task.Delay(TimeSpan.FromSeconds(errorStartupBlockTimeSeconds), cancellationToken);
             }
 
             await StopNodeWorkers(coreApiNode, cancellationToken);

src/RadixDlt.NetworkGateway.GatewayApi/Configuration/EndpointOptions.cs

@@ -74,6 +74,12 @@ public sealed class EndpointOptions
     [ConfigurationKeyName("MaxPageSize")]
     public int MaxPageSize { get; set; } = 100;
 
+    [ConfigurationKeyName("DefaultTransactionsStreamPageSize")]
+    public int DefaultTransactionsStreamPageSize { get; set; } = 10;
+
+    [ConfigurationKeyName("DefaultNonFungibleIdsPageSize")]
+    public int DefaultNonFungibleIdsPageSize { get; set; } = 100;
+
     [ConfigurationKeyName("RequestTimeout")]
     public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(10);
 
@@ -98,6 +104,8 @@ internal class EndpointOptionsValidator : AbstractOptionsValidator<EndpointOptio
     public EndpointOptionsValidator()
     {
         RuleFor(x => x.MaxPageSize).GreaterThan(0);
+        RuleFor(x => x.DefaultNonFungibleIdsPageSize).GreaterThan(0);
+        RuleFor(x => x.DefaultTransactionsStreamPageSize).GreaterThan(0);
         RuleFor(x => x.RequestTimeout).GreaterThan(TimeSpan.Zero);
         RuleFor(x => x.DefaultPageSize).GreaterThan(0);
         RuleFor(x => x.ValidatorsPageSize).GreaterThan(0);

src/RadixDlt.NetworkGateway.GatewayApi/Exceptions/InvalidTransactionException.cs

@@ -62,7 +62,6 @@
  * permissions under this License.
  */
 
-using System;
 using CoreModel = RadixDlt.CoreApiSdk.Model;
 using GatewayModel = RadixDlt.NetworkGateway.GatewayApiSdk.Model;
 

src/RadixDlt.NetworkGateway.GatewayApi/Exceptions/NotSyncedUpException.cs

@@ -64,6 +64,7 @@
 
 using RadixDlt.NetworkGateway.Abstractions.Extensions;
 using System;
+using System.Net;
 using GatewayModel = RadixDlt.NetworkGateway.GatewayApiSdk.Model;
 
 namespace RadixDlt.NetworkGateway.GatewayApi.Exceptions;
@@ -76,8 +77,10 @@ public enum NotSyncedUpRequestType
 
 public sealed class NotSyncedUpException : KnownGatewayErrorException
 {
+    private const int InternalServerErrorCode = (int)HttpStatusCode.InternalServerError;
+
     private NotSyncedUpException(GatewayModel.NotSyncedUpError gatewayError, string userFacingMessage)
-        : base(500, gatewayError, userFacingMessage)
+        : base(InternalServerErrorCode, gatewayError, userFacingMessage)
     {
     }