From 4e88c8cfe2dde7d04eb1adb179bf41ed3b1d50c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Mon, 25 Sep 2023 09:44:35 +0200 Subject: [PATCH 01/22] Add missing configuration options. --- .../Configuration/NodeWorkersOptions.cs | 23 +++++++++++++++++++ .../NodeServices/NodeWorkersRunner.cs | 12 ++++++---- .../ServiceCollectionExtensions.cs | 1 + .../Services/NodeWorkersRunnerFactory.cs | 4 +++- .../Services/NodeWorkersRunnerRegistry.cs | 13 +++++++---- .../Configuration/EndpointOptions.cs | 8 +++++++ .../Handlers/DefaultNonFungibleHandler.cs | 10 ++++---- .../Handlers/DefaultTransactionHandler.cs | 14 +++++------ 8 files changed, 64 insertions(+), 21 deletions(-) create mode 100644 src/RadixDlt.NetworkGateway.DataAggregator/Configuration/NodeWorkersOptions.cs diff --git a/src/RadixDlt.NetworkGateway.DataAggregator/Configuration/NodeWorkersOptions.cs b/src/RadixDlt.NetworkGateway.DataAggregator/Configuration/NodeWorkersOptions.cs new file mode 100644 index 000000000..76a5b11d9 --- /dev/null +++ b/src/RadixDlt.NetworkGateway.DataAggregator/Configuration/NodeWorkersOptions.cs @@ -0,0 +1,23 @@ +using FluentValidation; +using Microsoft.Extensions.Configuration; +using RadixDlt.NetworkGateway.Abstractions.Configuration; + +namespace RadixDlt.NetworkGateway.DataAggregator.Configuration; + +public sealed class NodeWorkersOptions +{ + [ConfigurationKeyName("ErrorStartupBlockTimeSeconds")] + public int ErrorStartupBlockTimeSeconds { get; set; } = 20; + + [ConfigurationKeyName("GraceSecondsBeforeMarkingStalled")] + public int GraceSecondsBeforeMarkingStalled { get; set; } = 10; +} + +internal class NodeWorkersOptionsValidator : AbstractOptionsValidator +{ + public NodeWorkersOptionsValidator() + { + RuleFor(x => x.ErrorStartupBlockTimeSeconds).GreaterThan(0); + RuleFor(x => x.GraceSecondsBeforeMarkingStalled).GreaterThan(0); + } +} diff --git a/src/RadixDlt.NetworkGateway.DataAggregator/NodeServices/NodeWorkersRunner.cs b/src/RadixDlt.NetworkGateway.DataAggregator/NodeServices/NodeWorkersRunner.cs index 23753ef52..b13d0a04c 100644 --- a/src/RadixDlt.NetworkGateway.DataAggregator/NodeServices/NodeWorkersRunner.cs +++ b/src/RadixDlt.NetworkGateway.DataAggregator/NodeServices/NodeWorkersRunner.cs @@ -64,9 +64,11 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Exceptions; using RadixDlt.NetworkGateway.Abstractions.Extensions; +using RadixDlt.NetworkGateway.DataAggregator.Configuration; using RadixDlt.NetworkGateway.DataAggregator.Workers.NodeWorkers; using System; using System.Collections.Generic; @@ -107,6 +109,7 @@ private set private readonly object _statusLock = new(); private readonly ILogger _logger; private readonly IClock _clock; + private readonly IOptionsMonitor _nodeWorkersOptions; private IServiceScope? _nodeDependencyInjectionScope; @@ -119,12 +122,13 @@ private set private IDisposable? _logScope; private NodeWorkersRunnerStatus _status; - public NodeWorkersRunner(ILogger logger, IServiceScope nodeDependencyInjectionScope, IDisposable logScope, IClock clock) + public NodeWorkersRunner(ILogger logger, IServiceScope nodeDependencyInjectionScope, IDisposable logScope, IClock clock, IOptionsMonitor nodeWorkersOptions) { _logger = logger; _nodeDependencyInjectionScope = nodeDependencyInjectionScope; _logScope = logScope; _clock = clock; + _nodeWorkersOptions = nodeWorkersOptions; _cancellationTokenSource = new CancellationTokenSource(); _initializers = nodeDependencyInjectionScope.ServiceProvider.GetServices().ToList(); _workers = nodeDependencyInjectionScope.ServiceProvider.GetServices().ToList(); @@ -136,16 +140,16 @@ public NodeWorkersRunner(ILogger logger, IServiceScope nodeDe /// public bool IsHealthy() { - const int GraceSecondsBeforeMarkingStalled = 10; + var graceSecondsBeforeMarkingStalled = _nodeWorkersOptions.CurrentValue.GraceSecondsBeforeMarkingStalled; - var isRunningOrNotStalled = Status == NodeWorkersRunnerStatus.Running || _lastStatusChange.WithinPeriodOfNow(TimeSpan.FromSeconds(GraceSecondsBeforeMarkingStalled), _clock); + var isRunningOrNotStalled = Status == NodeWorkersRunnerStatus.Running || _lastStatusChange.WithinPeriodOfNow(TimeSpan.FromSeconds(graceSecondsBeforeMarkingStalled), _clock); if (!isRunningOrNotStalled) { _logger.LogWarning( "Marked as unhealthy because current status is {Status} and last status changes was at {LastStatusChange}, longer than {GracePeriod}s ago - suggesting that the WorkersRegistry hasn't properly handled something, and these NodeWorkers should be restarted", Status, _lastStatusChange, - GraceSecondsBeforeMarkingStalled + graceSecondsBeforeMarkingStalled ); return false; } diff --git a/src/RadixDlt.NetworkGateway.DataAggregator/ServiceCollectionExtensions.cs b/src/RadixDlt.NetworkGateway.DataAggregator/ServiceCollectionExtensions.cs index 405613ada..d16f4d4b7 100644 --- a/src/RadixDlt.NetworkGateway.DataAggregator/ServiceCollectionExtensions.cs +++ b/src/RadixDlt.NetworkGateway.DataAggregator/ServiceCollectionExtensions.cs @@ -101,6 +101,7 @@ public static DataAggregatorBuilder AddNetworkGatewayDataAggregatorCore(this ISe .AddValidatableOptionsAtSection("DataAggregator:Network") .AddValidatableOptionsAtSection("DataAggregator:Monitoring") .AddValidatableOptionsAtSection("DataAggregator:Mempool") + .AddValidatableOptionsAtSection("DataAggregator:NodeWorkers") .AddValidatableOptionsAtSection("DataAggregator:LedgerConfirmation") .AddValidatableOptionsAtSection("DataAggregator:TransactionAssertions"); diff --git a/src/RadixDlt.NetworkGateway.DataAggregator/Services/NodeWorkersRunnerFactory.cs b/src/RadixDlt.NetworkGateway.DataAggregator/Services/NodeWorkersRunnerFactory.cs index 1004660f7..9cc683210 100644 --- a/src/RadixDlt.NetworkGateway.DataAggregator/Services/NodeWorkersRunnerFactory.cs +++ b/src/RadixDlt.NetworkGateway.DataAggregator/Services/NodeWorkersRunnerFactory.cs @@ -64,6 +64,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.DataAggregator.Configuration; using RadixDlt.NetworkGateway.DataAggregator.NodeServices; @@ -97,6 +98,7 @@ public NodeWorkersRunner CreateWorkersForNode(CoreApiNode initialCoreApiNode) var nodeScope = _services.CreateScope(); nodeScope.ServiceProvider.GetRequiredService().CoreApiNode = initialCoreApiNode; var logScope = _logger.BeginScope($"[NODE: {initialCoreApiNode.Name}]"); - return new NodeWorkersRunner(_logger, nodeScope, logScope!, nodeScope.ServiceProvider.GetRequiredService()); + return new NodeWorkersRunner(_logger, nodeScope, logScope!, nodeScope.ServiceProvider.GetRequiredService(), + nodeScope.ServiceProvider.GetRequiredService>()); } } diff --git a/src/RadixDlt.NetworkGateway.DataAggregator/Services/NodeWorkersRunnerRegistry.cs b/src/RadixDlt.NetworkGateway.DataAggregator/Services/NodeWorkersRunnerRegistry.cs index d13e0bf73..90596015c 100644 --- a/src/RadixDlt.NetworkGateway.DataAggregator/Services/NodeWorkersRunnerRegistry.cs +++ b/src/RadixDlt.NetworkGateway.DataAggregator/Services/NodeWorkersRunnerRegistry.cs @@ -63,6 +63,7 @@ */ using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using RadixDlt.NetworkGateway.Abstractions.Extensions; using RadixDlt.NetworkGateway.DataAggregator.Configuration; using RadixDlt.NetworkGateway.DataAggregator.NodeServices; @@ -83,18 +84,18 @@ public interface INodeWorkersRunnerRegistry internal class NodeWorkersRunnerRegistry : INodeWorkersRunnerRegistry { - private const int ErrorStartupBlockTimeSeconds = 20; - private readonly ILogger _logger; + private readonly IOptionsMonitor _nodeWorkersOptions; private readonly INodeWorkersRunnerFactory _nodeWorkersRunnerFactory; private readonly Dictionary _servicesMap = new(); private readonly Dictionary _startupBlocklist = new(); private readonly object _servicesMapLock = new(); - public NodeWorkersRunnerRegistry(ILogger logger, INodeWorkersRunnerFactory nodeWorkersRunnerFactory) + public NodeWorkersRunnerRegistry(ILogger logger, INodeWorkersRunnerFactory nodeWorkersRunnerFactory, IOptionsMonitor nodeWorkersOptions) { _logger = logger; _nodeWorkersRunnerFactory = nodeWorkersRunnerFactory; + _nodeWorkersOptions = nodeWorkersOptions; } public async Task EnsureCorrectNodeServicesRunning(List enabledNodes, CancellationToken cancellationToken) @@ -185,16 +186,18 @@ private async Task CreateAndStartNodeWorkersIfNotExists(CoreApiNode coreApiNode, throw; } + var errorStartupBlockTimeSeconds = _nodeWorkersOptions.CurrentValue.ErrorStartupBlockTimeSeconds; + _logger.LogError( ex, "Error initializing or starting up services for node: {NodeName}. We won't try again for {ErrorStartupBlockTimeSeconds} seconds. Now clearing up...", coreApiNode.Name, - ErrorStartupBlockTimeSeconds + errorStartupBlockTimeSeconds ); lock (_servicesMapLock) { - _startupBlocklist[coreApiNode.Name] = Task.Delay(TimeSpan.FromSeconds(ErrorStartupBlockTimeSeconds), cancellationToken); + _startupBlocklist[coreApiNode.Name] = Task.Delay(TimeSpan.FromSeconds(errorStartupBlockTimeSeconds), cancellationToken); } await StopNodeWorkers(coreApiNode, cancellationToken); diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/EndpointOptions.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/EndpointOptions.cs index eaec91288..d221f3261 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/EndpointOptions.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/EndpointOptions.cs @@ -74,6 +74,12 @@ public sealed class EndpointOptions [ConfigurationKeyName("MaxPageSize")] public int MaxPageSize { get; set; } = 100; + [ConfigurationKeyName("DefaultTransactionsStreamPageSize")] + public int DefaultTransactionsStreamPageSize { get; set; } = 10; + + [ConfigurationKeyName("DefaultNonFungibleIdsPageSize")] + public int DefaultNonFungibleIdsPageSize { get; set; } = 100; + [ConfigurationKeyName("RequestTimeout")] public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(10); @@ -98,6 +104,8 @@ internal class EndpointOptionsValidator : AbstractOptionsValidator x.MaxPageSize).GreaterThan(0); + RuleFor(x => x.DefaultNonFungibleIdsPageSize).GreaterThan(0); + RuleFor(x => x.DefaultTransactionsStreamPageSize).GreaterThan(0); RuleFor(x => x.RequestTimeout).GreaterThan(TimeSpan.Zero); RuleFor(x => x.DefaultPageSize).GreaterThan(0); RuleFor(x => x.ValidatorsPageSize).GreaterThan(0); diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Handlers/DefaultNonFungibleHandler.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Handlers/DefaultNonFungibleHandler.cs index 7a53db6c4..743d7ad12 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/Handlers/DefaultNonFungibleHandler.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/Handlers/DefaultNonFungibleHandler.cs @@ -62,7 +62,9 @@ * permissions under this License. */ +using Microsoft.Extensions.Options; using RadixDlt.NetworkGateway.Abstractions; +using RadixDlt.NetworkGateway.GatewayApi.Configuration; using RadixDlt.NetworkGateway.GatewayApi.Services; using System.Threading; using System.Threading.Tasks; @@ -72,15 +74,15 @@ namespace RadixDlt.NetworkGateway.GatewayApi.Handlers; internal class DefaultNonFungibleHandler : INonFungibleHandler { - private const int DefaultPageLimit = 100; // TODO make it configurable - private readonly ILedgerStateQuerier _ledgerStateQuerier; private readonly IEntityStateQuerier _entityStateQuerier; + private readonly IOptionsSnapshot _endpointConfiguration; - public DefaultNonFungibleHandler(ILedgerStateQuerier ledgerStateQuerier, IEntityStateQuerier entityStateQuerier) + public DefaultNonFungibleHandler(ILedgerStateQuerier ledgerStateQuerier, IEntityStateQuerier entityStateQuerier, IOptionsSnapshot endpointConfiguration) { _ledgerStateQuerier = ledgerStateQuerier; _entityStateQuerier = entityStateQuerier; + _endpointConfiguration = endpointConfiguration; } public async Task Ids(GatewayModel.StateNonFungibleIdsRequest request, CancellationToken token = default) @@ -91,7 +93,7 @@ public DefaultNonFungibleHandler(ILedgerStateQuerier ledgerStateQuerier, IEntity var pageRequest = new IEntityStateQuerier.PageRequest( Address: (EntityAddress)request.ResourceAddress, Offset: cursor?.Offset ?? 0, - Limit: request.LimitPerPage ?? DefaultPageLimit + Limit: request.LimitPerPage ?? _endpointConfiguration.Value.DefaultNonFungibleIdsPageSize ); return await _entityStateQuerier.NonFungibleIds(pageRequest, ledgerState, token); diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Handlers/DefaultTransactionHandler.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Handlers/DefaultTransactionHandler.cs index 5944eb3d2..07810c5e2 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/Handlers/DefaultTransactionHandler.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/Handlers/DefaultTransactionHandler.cs @@ -62,15 +62,14 @@ * permissions under this License. */ +using Microsoft.Extensions.Options; using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Model; using RadixDlt.NetworkGateway.Abstractions.Numerics; +using RadixDlt.NetworkGateway.GatewayApi.Configuration; using RadixDlt.NetworkGateway.GatewayApi.Exceptions; using RadixDlt.NetworkGateway.GatewayApi.Services; -using System; -using System.Collections.Generic; using System.Diagnostics; -using System.Linq; using System.Threading; using System.Threading.Tasks; using GatewayModel = RadixDlt.NetworkGateway.GatewayApiSdk.Model; @@ -79,23 +78,24 @@ namespace RadixDlt.NetworkGateway.GatewayApi.Handlers; internal class DefaultTransactionHandler : ITransactionHandler { - private const int DefaultPageLimit = 10; // TODO make it configurable - private readonly ILedgerStateQuerier _ledgerStateQuerier; private readonly ITransactionQuerier _transactionQuerier; private readonly IPreviewService _previewService; private readonly ISubmissionService _submissionService; + private readonly IOptionsSnapshot _endpointConfiguration; public DefaultTransactionHandler( ILedgerStateQuerier ledgerStateQuerier, ITransactionQuerier transactionQuerier, IPreviewService previewService, - ISubmissionService submissionService) + ISubmissionService submissionService, + IOptionsSnapshot endpointConfiguration) { _ledgerStateQuerier = ledgerStateQuerier; _transactionQuerier = transactionQuerier; _previewService = previewService; _submissionService = submissionService; + _endpointConfiguration = endpointConfiguration; } public async Task Construction(CancellationToken token = default) @@ -186,7 +186,7 @@ public DefaultTransactionHandler( var transactionsPageRequest = new TransactionStreamPageRequest( FromStateVersion: fromLedgerState?.StateVersion, Cursor: GatewayModel.LedgerTransactionsCursor.FromCursorString(request.Cursor), - PageSize: request.LimitPerPage ?? DefaultPageLimit, + PageSize: request.LimitPerPage ?? _endpointConfiguration.Value.DefaultPageSize, AscendingOrder: request.Order == GatewayModel.StreamTransactionsRequest.OrderEnum.Asc, SearchCriteria: searchCriteria, OptIns: request.OptIns ?? GatewayModel.TransactionDetailsOptIns.Default From da8988fa93fc1de9b4cd0e8837fa8c67c2a1e0e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Mon, 25 Sep 2023 09:45:07 +0200 Subject: [PATCH 02/22] bumped version to 1.0.0 --- Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Build.props b/Directory.Build.props index 8ec963941..aad66eb02 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -13,7 +13,7 @@ - 0.5.4 + 1.0.0 develop From 7b5d09e3f182490a6bf7ee03b39c140bcea8dd20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Mon, 25 Sep 2023 09:45:16 +0200 Subject: [PATCH 03/22] cleanup. --- apps/GatewayApi/GatewayApiStartup.cs | 3 --- apps/GatewayApi/SlowRequestLogging/BufferingStream.cs | 1 - .../Extensions/ExceptionExtensions.cs | 1 - .../NodeServices/ApiReaders/NetworkStatusReader.cs | 1 - .../NodeServices/NodeConfigProvider.cs | 1 - .../Services/IPendingTransactionPrunerServiceObserver.cs | 1 - .../Services/LedgerTransactionsProcessor.cs | 1 - .../Services/NetworkConfigurationProvider.cs | 2 -- .../Services/NodeStatusProvider.cs | 3 --- .../Exceptions/InvalidTransactionException.cs | 1 - .../Services/ISubmissionServiceObserver.cs | 1 - .../Services/NetworkConfigurationProvider.cs | 1 - .../Validators/StateKeyValueStoreDataRequestValidator.cs | 1 - .../StreamTransactionsRequestEventFilterItemValidator.cs | 2 -- .../TransactionCommittedDetailsRequestValidator.cs | 1 - .../Validators/TransactionStatusRequestValidator.cs | 1 - .../Services/EventDecoder.cs | 2 -- .../Services/RoleAssignmentsKeyProvider.cs | 6 ------ .../Services/RoleAssignmentsMapper.cs | 2 -- .../PostgresIntegration/ScryptoSborUtilsTests.cs | 1 - 20 files changed, 33 deletions(-) diff --git a/apps/GatewayApi/GatewayApiStartup.cs b/apps/GatewayApi/GatewayApiStartup.cs index e22b6bb1f..35d9a7171 100644 --- a/apps/GatewayApi/GatewayApiStartup.cs +++ b/apps/GatewayApi/GatewayApiStartup.cs @@ -64,18 +64,15 @@ using GatewayApi.SlowRequestLogging; using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.HttpOverrides; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Newtonsoft.Json; -using Newtonsoft.Json.Converters; using Prometheus; using RadixDlt.NetworkGateway.GatewayApi; using RadixDlt.NetworkGateway.PostgresIntegration; using RadixDlt.NetworkGateway.PrometheusIntegration; using System; -using System.Globalization; namespace GatewayApi; diff --git a/apps/GatewayApi/SlowRequestLogging/BufferingStream.cs b/apps/GatewayApi/SlowRequestLogging/BufferingStream.cs index 4080a3a21..253e3c20e 100644 --- a/apps/GatewayApi/SlowRequestLogging/BufferingStream.cs +++ b/apps/GatewayApi/SlowRequestLogging/BufferingStream.cs @@ -70,7 +70,6 @@ using System.Buffers; using System.Diagnostics; using System.IO; -using System.IO.Pipelines; using System.Runtime.CompilerServices; using System.Text; using System.Threading; diff --git a/src/RadixDlt.NetworkGateway.Abstractions/Extensions/ExceptionExtensions.cs b/src/RadixDlt.NetworkGateway.Abstractions/Extensions/ExceptionExtensions.cs index 6f2673c86..4fd42a19f 100644 --- a/src/RadixDlt.NetworkGateway.Abstractions/Extensions/ExceptionExtensions.cs +++ b/src/RadixDlt.NetworkGateway.Abstractions/Extensions/ExceptionExtensions.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using RadixDlt.NetworkGateway.Abstractions.CoreCommunications; using RadixDlt.NetworkGateway.Abstractions.Exceptions; using System; using System.Linq; diff --git a/src/RadixDlt.NetworkGateway.DataAggregator/NodeServices/ApiReaders/NetworkStatusReader.cs b/src/RadixDlt.NetworkGateway.DataAggregator/NodeServices/ApiReaders/NetworkStatusReader.cs index 31f740dbb..0dc778eae 100644 --- a/src/RadixDlt.NetworkGateway.DataAggregator/NodeServices/ApiReaders/NetworkStatusReader.cs +++ b/src/RadixDlt.NetworkGateway.DataAggregator/NodeServices/ApiReaders/NetworkStatusReader.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using RadixDlt.NetworkGateway.Abstractions.CoreCommunications; using RadixDlt.NetworkGateway.Abstractions.Extensions; using RadixDlt.NetworkGateway.DataAggregator.Services; using System; diff --git a/src/RadixDlt.NetworkGateway.DataAggregator/NodeServices/NodeConfigProvider.cs b/src/RadixDlt.NetworkGateway.DataAggregator/NodeServices/NodeConfigProvider.cs index eba0ac151..ba9d3ca3d 100644 --- a/src/RadixDlt.NetworkGateway.DataAggregator/NodeServices/NodeConfigProvider.cs +++ b/src/RadixDlt.NetworkGateway.DataAggregator/NodeServices/NodeConfigProvider.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using RadixDlt.NetworkGateway.Abstractions.Configuration; using RadixDlt.NetworkGateway.DataAggregator.Configuration; namespace RadixDlt.NetworkGateway.DataAggregator.NodeServices; diff --git a/src/RadixDlt.NetworkGateway.DataAggregator/Services/IPendingTransactionPrunerServiceObserver.cs b/src/RadixDlt.NetworkGateway.DataAggregator/Services/IPendingTransactionPrunerServiceObserver.cs index aad8899f6..7369bfbd9 100644 --- a/src/RadixDlt.NetworkGateway.DataAggregator/Services/IPendingTransactionPrunerServiceObserver.cs +++ b/src/RadixDlt.NetworkGateway.DataAggregator/Services/IPendingTransactionPrunerServiceObserver.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using RadixDlt.NetworkGateway.Abstractions.Model; using System.Collections.Generic; using System.Threading.Tasks; diff --git a/src/RadixDlt.NetworkGateway.DataAggregator/Services/LedgerTransactionsProcessor.cs b/src/RadixDlt.NetworkGateway.DataAggregator/Services/LedgerTransactionsProcessor.cs index 40b18d95d..983115a3d 100644 --- a/src/RadixDlt.NetworkGateway.DataAggregator/Services/LedgerTransactionsProcessor.cs +++ b/src/RadixDlt.NetworkGateway.DataAggregator/Services/LedgerTransactionsProcessor.cs @@ -71,7 +71,6 @@ using RadixDlt.NetworkGateway.DataAggregator.Exceptions; using RadixDlt.NetworkGateway.DataAggregator.Monitoring; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; diff --git a/src/RadixDlt.NetworkGateway.DataAggregator/Services/NetworkConfigurationProvider.cs b/src/RadixDlt.NetworkGateway.DataAggregator/Services/NetworkConfigurationProvider.cs index 053af6664..e9a8efda3 100644 --- a/src/RadixDlt.NetworkGateway.DataAggregator/Services/NetworkConfigurationProvider.cs +++ b/src/RadixDlt.NetworkGateway.DataAggregator/Services/NetworkConfigurationProvider.cs @@ -62,8 +62,6 @@ * permissions under this License. */ -using RadixDlt.NetworkGateway.Abstractions; -using RadixDlt.NetworkGateway.Abstractions.Addressing; using RadixDlt.NetworkGateway.Abstractions.CoreCommunications; using System.Threading; using System.Threading.Tasks; diff --git a/src/RadixDlt.NetworkGateway.DataAggregator/Services/NodeStatusProvider.cs b/src/RadixDlt.NetworkGateway.DataAggregator/Services/NodeStatusProvider.cs index d0e77beea..537f25d49 100644 --- a/src/RadixDlt.NetworkGateway.DataAggregator/Services/NodeStatusProvider.cs +++ b/src/RadixDlt.NetworkGateway.DataAggregator/Services/NodeStatusProvider.cs @@ -64,12 +64,9 @@ using RadixDlt.NetworkGateway.Abstractions.Extensions; using RadixDlt.NetworkGateway.DataAggregator.NodeServices; -using RadixDlt.NetworkGateway.DataAggregator.NodeServices.ApiReaders; -using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; -using System.Threading; using CoreModel = RadixDlt.CoreApiSdk.Model; namespace RadixDlt.NetworkGateway.DataAggregator.Services; diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Exceptions/InvalidTransactionException.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Exceptions/InvalidTransactionException.cs index 06406cb35..aeaee5b05 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/Exceptions/InvalidTransactionException.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/Exceptions/InvalidTransactionException.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using System; using CoreModel = RadixDlt.CoreApiSdk.Model; using GatewayModel = RadixDlt.NetworkGateway.GatewayApiSdk.Model; diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Services/ISubmissionServiceObserver.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Services/ISubmissionServiceObserver.cs index 9b3da6b8f..89f646ff0 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/Services/ISubmissionServiceObserver.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/Services/ISubmissionServiceObserver.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using System; using System.Threading.Tasks; using CoreModel = RadixDlt.CoreApiSdk.Model; using GatewayModel = RadixDlt.NetworkGateway.GatewayApiSdk.Model; diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Services/NetworkConfigurationProvider.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Services/NetworkConfigurationProvider.cs index 0af32cd2b..5e44513fb 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/Services/NetworkConfigurationProvider.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/Services/NetworkConfigurationProvider.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Addressing; using RadixDlt.NetworkGateway.Abstractions.Configuration; using RadixDlt.NetworkGateway.Abstractions.CoreCommunications; diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Validators/StateKeyValueStoreDataRequestValidator.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Validators/StateKeyValueStoreDataRequestValidator.cs index 2978168e0..4fbf21646 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/Validators/StateKeyValueStoreDataRequestValidator.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/Validators/StateKeyValueStoreDataRequestValidator.cs @@ -63,7 +63,6 @@ */ using FluentValidation; -using FluentValidation.Validators; using Microsoft.Extensions.Options; using RadixDlt.NetworkGateway.GatewayApi.Configuration; using GatewayModel = RadixDlt.NetworkGateway.GatewayApiSdk.Model; diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Validators/StreamTransactionsRequestEventFilterItemValidator.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Validators/StreamTransactionsRequestEventFilterItemValidator.cs index 0e78aac88..3bb12ec73 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/Validators/StreamTransactionsRequestEventFilterItemValidator.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/Validators/StreamTransactionsRequestEventFilterItemValidator.cs @@ -63,8 +63,6 @@ */ using FluentValidation; -using Microsoft.Extensions.Options; -using RadixDlt.NetworkGateway.GatewayApi.Configuration; using GatewayModel = RadixDlt.NetworkGateway.GatewayApiSdk.Model; namespace RadixDlt.NetworkGateway.GatewayApi.Validators; diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Validators/TransactionCommittedDetailsRequestValidator.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Validators/TransactionCommittedDetailsRequestValidator.cs index 52a286295..dcb2a0947 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/Validators/TransactionCommittedDetailsRequestValidator.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/Validators/TransactionCommittedDetailsRequestValidator.cs @@ -63,7 +63,6 @@ */ using FluentValidation; -using RadixDlt.NetworkGateway.Abstractions; using GatewayModel = RadixDlt.NetworkGateway.GatewayApiSdk.Model; namespace RadixDlt.NetworkGateway.GatewayApi.Validators; diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Validators/TransactionStatusRequestValidator.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Validators/TransactionStatusRequestValidator.cs index a17b39a98..5e2f8e753 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/Validators/TransactionStatusRequestValidator.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/Validators/TransactionStatusRequestValidator.cs @@ -63,7 +63,6 @@ */ using FluentValidation; -using RadixDlt.NetworkGateway.Abstractions; using GatewayModel = RadixDlt.NetworkGateway.GatewayApiSdk.Model; namespace RadixDlt.NetworkGateway.GatewayApi.Validators; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EventDecoder.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EventDecoder.cs index c4b3bd62b..4151d9962 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EventDecoder.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EventDecoder.cs @@ -63,9 +63,7 @@ */ using System; -using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; -using System.Linq; using CoreModel = RadixDlt.CoreApiSdk.Model; using ToolkitModel = RadixEngineToolkit; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/RoleAssignmentsKeyProvider.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/RoleAssignmentsKeyProvider.cs index f76472ac5..248cd2819 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/RoleAssignmentsKeyProvider.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/RoleAssignmentsKeyProvider.cs @@ -62,15 +62,9 @@ * permissions under this License. */ -using Microsoft.EntityFrameworkCore; -using Newtonsoft.Json; using RadixDlt.NetworkGateway.Abstractions.Model; -using RadixDlt.NetworkGateway.PostgresIntegration.Models; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; -using System.Threading; -using System.Threading.Tasks; using CoreModel = RadixDlt.CoreApiSdk.Model; using GatewayModel = RadixDlt.NetworkGateway.Abstractions.Model; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/RoleAssignmentsMapper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/RoleAssignmentsMapper.cs index 524fa0a32..cf89a8591 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/RoleAssignmentsMapper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/RoleAssignmentsMapper.cs @@ -67,8 +67,6 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; -using System.Threading; -using System.Threading.Tasks; using CoreApiModel = RadixDlt.CoreApiSdk.Model; using GatewayModel = RadixDlt.NetworkGateway.GatewayApiSdk.Model; diff --git a/tests/RadixDlt.NetworkGateway.UnitTests/PostgresIntegration/ScryptoSborUtilsTests.cs b/tests/RadixDlt.NetworkGateway.UnitTests/PostgresIntegration/ScryptoSborUtilsTests.cs index d8678d721..d0ee39d61 100644 --- a/tests/RadixDlt.NetworkGateway.UnitTests/PostgresIntegration/ScryptoSborUtilsTests.cs +++ b/tests/RadixDlt.NetworkGateway.UnitTests/PostgresIntegration/ScryptoSborUtilsTests.cs @@ -63,7 +63,6 @@ */ using FluentAssertions; -using Microsoft.Extensions.Logging.Abstractions; using RadixDlt.NetworkGateway.PostgresIntegration; using System; using System.Collections.Generic; From c85f2c4d176abd5c31d5ebbd24749ea7c90a81b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Mon, 25 Sep 2023 12:38:38 +0200 Subject: [PATCH 04/22] update toolkit version to stable 1.0.0. --- Directory.Packages.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index b210a3eb3..41ed23812 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -22,7 +22,7 @@ - + @@ -33,4 +33,4 @@ - + \ No newline at end of file From ffd4b123a47a591605894d4f08f10613a99167eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20=C5=81abu=C5=9B?= Date: Mon, 25 Sep 2023 17:18:21 +0200 Subject: [PATCH 05/22] Fixed erroneous HTTP response codes --- .../Exceptions/NotSyncedUpException.cs | 5 ++++- .../Exceptions/ValidationException.cs | 3 ++- .../Services/ExceptionHandler.cs | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Exceptions/NotSyncedUpException.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Exceptions/NotSyncedUpException.cs index b000f06aa..c8577a0c7 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/Exceptions/NotSyncedUpException.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/Exceptions/NotSyncedUpException.cs @@ -64,6 +64,7 @@ using RadixDlt.NetworkGateway.Abstractions.Extensions; using System; +using System.Net; using GatewayModel = RadixDlt.NetworkGateway.GatewayApiSdk.Model; namespace RadixDlt.NetworkGateway.GatewayApi.Exceptions; @@ -76,8 +77,10 @@ public enum NotSyncedUpRequestType public sealed class NotSyncedUpException : KnownGatewayErrorException { + private const int InternalServerErrorCode = (int)HttpStatusCode.InternalServerError; + private NotSyncedUpException(GatewayModel.NotSyncedUpError gatewayError, string userFacingMessage) - : base(500, gatewayError, userFacingMessage) + : base(InternalServerErrorCode, gatewayError, userFacingMessage) { } diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Exceptions/ValidationException.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Exceptions/ValidationException.cs index aa2f9de40..13b7536e0 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/Exceptions/ValidationException.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/Exceptions/ValidationException.cs @@ -62,13 +62,14 @@ * permissions under this License. */ +using System.Net; using GatewayModel = RadixDlt.NetworkGateway.GatewayApiSdk.Model; namespace RadixDlt.NetworkGateway.GatewayApi.Exceptions; public abstract class ValidationException : KnownGatewayErrorException { - private const int ValidationErrorStatusCode = 400; + private const int ValidationErrorStatusCode = (int)HttpStatusCode.BadRequest; public ValidationException(GatewayModel.GatewayError gatewayError, string userFacingMessage, string internalMessage) : base(ValidationErrorStatusCode, gatewayError, userFacingMessage, internalMessage) diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Services/ExceptionHandler.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Services/ExceptionHandler.cs index e044db21e..72140b15c 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/Services/ExceptionHandler.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/Services/ExceptionHandler.cs @@ -108,7 +108,7 @@ public ActionResult CreateAndLogApiResultFromException(ActionContext actionConte traceId: traceId )) { - StatusCode = (int)HttpStatusCode.InternalServerError, + StatusCode = gatewayErrorException.StatusCode, }; } From 39142b3f746e9126215857b3d6ffff1bc4cfefc5 Mon Sep 17 00:00:00 2001 From: Kim Fehrs <122281269+kofrdx@users.noreply.github.com> Date: Tue, 26 Sep 2023 11:02:06 +0200 Subject: [PATCH 06/22] fix / wait for multiarch image (#504) * wait for multiarch image * fix wrong needs update --- .github/workflows/releases.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 0a38ad40a..b790241f4 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -264,9 +264,9 @@ jobs: runs-on: ubuntu-latest needs: - setup-tags - - docker-database-migrations-dockerhub - - docker-data-aggregator-dockerhub - - docker-gateway-api-dockerhub + - join-gateway-images + - join-aggregator-images + - join-migrations-images permissions: id-token: write pull-requests: read From b422d50e99a4271b554f504a00828ef3b4b8d117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Tue, 26 Sep 2023 19:53:39 +0200 Subject: [PATCH 07/22] added metrics to sql queries. log warning if sql query is slower than configured threshold. --- .../SlowQueriesLoggingOptions.cs | 18 +++ .../ServiceCollectionExtensions.cs | 1 + .../SlowQueriesLoggingOptions.cs | 18 +++ .../ServiceCollectionExtensions.cs | 1 + .../Services/ISqlQueryObserver.cs | 8 ++ .../CommonDbContext.cs | 7 +- .../DataAggregatorBuilderExtensions.cs | 6 +- .../GatewayApiBuilderExtensions.cs | 14 ++- .../Interceptors/ForcedDistinctInterceptor.cs | 2 +- .../Interceptors/MetricsInterceptor.cs | 106 ++++++++++++++++++ .../PostgresLedgerExtenderService.cs | 2 + .../LedgerExtension/ReadHelper.cs | 20 ++-- .../Metrics/SqlQueryMetricsHelper.cs | 27 +++++ .../QueryableExtensions.cs | 21 ++++ ....NetworkGateway.PostgresIntegration.csproj | 1 + .../Services/BlueprintProvider.cs | 2 + .../Services/DapperWrapper.cs | 104 +++++++++++++++++ .../Services/EntityStateQuerier.cs | 52 +++++++-- .../Services/EntityStateQuerier.fungibles.cs | 4 +- .../EntityStateQuerier.nonfungibles.cs | 10 +- .../PendingTransactionPrunerService.cs | 1 + .../PendingTransactionResubmissionService.cs | 1 + .../Services/RoleAssignmentQuerier.cs | 4 + .../Services/TransactionQuerier.cs | 6 + .../Services/ValidatorQuerier.cs | 7 +- .../DataAggregatorBuilderExtensions.cs | 4 +- .../DataAggregatorMetricsObserver.cs | 13 ++- .../GatewayApiBuilderExtensions.cs | 3 +- .../GatewayApiMetricObserver.cs | 12 +- 29 files changed, 434 insertions(+), 41 deletions(-) create mode 100644 src/RadixDlt.NetworkGateway.DataAggregator/Configuration/SlowQueriesLoggingOptions.cs create mode 100644 src/RadixDlt.NetworkGateway.GatewayApi/Configuration/SlowQueriesLoggingOptions.cs create mode 100644 src/RadixDlt.NetworkGateway.GatewayApi/Services/ISqlQueryObserver.cs create mode 100644 src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs create mode 100644 src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs create mode 100644 src/RadixDlt.NetworkGateway.PostgresIntegration/QueryableExtensions.cs create mode 100644 src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs diff --git a/src/RadixDlt.NetworkGateway.DataAggregator/Configuration/SlowQueriesLoggingOptions.cs b/src/RadixDlt.NetworkGateway.DataAggregator/Configuration/SlowQueriesLoggingOptions.cs new file mode 100644 index 000000000..7b386e552 --- /dev/null +++ b/src/RadixDlt.NetworkGateway.DataAggregator/Configuration/SlowQueriesLoggingOptions.cs @@ -0,0 +1,18 @@ +using FluentValidation; +using RadixDlt.NetworkGateway.Abstractions.Configuration; +using System; + +namespace RadixDlt.NetworkGateway.DataAggregator.Configuration; + +public sealed class SlowQueriesLoggingOptions +{ + public TimeSpan SlowQueriesThreshold { get; set; } = TimeSpan.FromMilliseconds(250); +} + +internal class SlowQueriesLoggingOptionsValidator : AbstractOptionsValidator +{ + public SlowQueriesLoggingOptionsValidator() + { + RuleFor(x => x.SlowQueriesThreshold).GreaterThan(TimeSpan.Zero); + } +} diff --git a/src/RadixDlt.NetworkGateway.DataAggregator/ServiceCollectionExtensions.cs b/src/RadixDlt.NetworkGateway.DataAggregator/ServiceCollectionExtensions.cs index d16f4d4b7..ecd1402b8 100644 --- a/src/RadixDlt.NetworkGateway.DataAggregator/ServiceCollectionExtensions.cs +++ b/src/RadixDlt.NetworkGateway.DataAggregator/ServiceCollectionExtensions.cs @@ -99,6 +99,7 @@ public static DataAggregatorBuilder AddNetworkGatewayDataAggregatorCore(this ISe services .AddValidatableOptionsAtSection("DataAggregator:Network") + .AddValidatableOptionsAtSection("DataAggregator:SlowQueriesLogging") .AddValidatableOptionsAtSection("DataAggregator:Monitoring") .AddValidatableOptionsAtSection("DataAggregator:Mempool") .AddValidatableOptionsAtSection("DataAggregator:NodeWorkers") diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/SlowQueriesLoggingOptions.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/SlowQueriesLoggingOptions.cs new file mode 100644 index 000000000..609a062fb --- /dev/null +++ b/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/SlowQueriesLoggingOptions.cs @@ -0,0 +1,18 @@ +using FluentValidation; +using RadixDlt.NetworkGateway.Abstractions.Configuration; +using System; + +namespace RadixDlt.NetworkGateway.GatewayApi.Configuration; + +public sealed class SlowQueriesLoggingOptions +{ + public TimeSpan SlowQueriesThreshold { get; set; } = TimeSpan.FromMilliseconds(250); +} + +internal class SlowQueriesLoggingOptionsValidator : AbstractOptionsValidator +{ + public SlowQueriesLoggingOptionsValidator() + { + RuleFor(x => x.SlowQueriesThreshold).GreaterThan(TimeSpan.Zero); + } +} diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/ServiceCollectionExtensions.cs b/src/RadixDlt.NetworkGateway.GatewayApi/ServiceCollectionExtensions.cs index 2aa67cd0b..b5a802d21 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/ServiceCollectionExtensions.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/ServiceCollectionExtensions.cs @@ -99,6 +99,7 @@ public static GatewayApiBuilder AddNetworkGatewayApiCore(this IServiceCollection services .AddValidatableOptionsAtSection("GatewayApi:Endpoint") + .AddValidatableOptionsAtSection("GatewayApi:SlowQueriesLogging") .AddValidatableOptionsAtSection("GatewayApi:CoreApiIntegration") .AddValidatableOptionsAtSection("GatewayApi:Network") .AddValidatableOptionsAtSection("GatewayApi:AcceptableLedgerLag"); diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Services/ISqlQueryObserver.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Services/ISqlQueryObserver.cs new file mode 100644 index 000000000..3149944f8 --- /dev/null +++ b/src/RadixDlt.NetworkGateway.GatewayApi/Services/ISqlQueryObserver.cs @@ -0,0 +1,8 @@ +using System; + +namespace RadixDlt.NetworkGateway.GatewayApi.Services; + +public interface ISqlQueryObserver +{ + void OnSqlQueryExecuted(string queryName, TimeSpan duration); +} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs index cf8be2d49..dfda16852 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs @@ -66,9 +66,9 @@ using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Model; using RadixDlt.NetworkGateway.Abstractions.Numerics; -using RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; using RadixDlt.NetworkGateway.PostgresIntegration.Models; using RadixDlt.NetworkGateway.PostgresIntegration.ValueConverters; +using System; namespace RadixDlt.NetworkGateway.PostgresIntegration; @@ -180,11 +180,6 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) HookupStatistics(modelBuilder); } - protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) - { - optionsBuilder.AddInterceptors(new ForceDistinctInterceptor()); - } - protected override void ConfigureConventions(ModelConfigurationBuilder configurationBuilder) { configurationBuilder diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/DataAggregatorBuilderExtensions.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/DataAggregatorBuilderExtensions.cs index d0549d897..4c23c44dd 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/DataAggregatorBuilderExtensions.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/DataAggregatorBuilderExtensions.cs @@ -66,6 +66,7 @@ using Microsoft.Extensions.DependencyInjection; using RadixDlt.NetworkGateway.DataAggregator; using RadixDlt.NetworkGateway.DataAggregator.Services; +using RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; using RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; using RadixDlt.NetworkGateway.PostgresIntegration.Services; @@ -91,7 +92,9 @@ public static DataAggregatorBuilder AddPostgresPersistenceCore(this DataAggregat .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton(); + .AddSingleton() + .AddSingleton() + .AddSingleton(); CustomTypes.EnsureConfigured(); @@ -101,6 +104,7 @@ public static DataAggregatorBuilder AddPostgresPersistenceCore(this DataAggregat .AddDbContextFactory((serviceProvider, options) => { options.UseNpgsql(serviceProvider.GetRequiredService>().NpgsqlDataSource); + options.AddInterceptors(serviceProvider.GetRequiredService()); }); return builder; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayApiBuilderExtensions.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayApiBuilderExtensions.cs index 7d031b4ec..cc97c0693 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayApiBuilderExtensions.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayApiBuilderExtensions.cs @@ -63,9 +63,15 @@ */ using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using RadixDlt.NetworkGateway.GatewayApi; +using RadixDlt.NetworkGateway.GatewayApi.Configuration; using RadixDlt.NetworkGateway.GatewayApi.Services; +using RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; +using RadixDlt.NetworkGateway.PostgresIntegration.Metrics; using RadixDlt.NetworkGateway.PostgresIntegration.Services; namespace RadixDlt.NetworkGateway.PostgresIntegration; @@ -94,7 +100,9 @@ public static GatewayApiBuilder AddPostgresPersistenceCore(this GatewayApiBuilde .AddScoped() .AddScoped() .AddScoped() - .AddScoped(); + .AddScoped() + .AddSingleton() + .AddSingleton(); CustomTypes.EnsureConfigured(); @@ -104,11 +112,15 @@ public static GatewayApiBuilder AddPostgresPersistenceCore(this GatewayApiBuilde .AddDbContext((serviceProvider, options) => { options.UseNpgsql(serviceProvider.GetRequiredService>().NpgsqlDataSource); + options.AddInterceptors(serviceProvider.GetRequiredService()); + options.AddInterceptors(new ForceDistinctInterceptor()); }) .AddNpgsqlDataSourceHolder(PostgresIntegrationConstants.Configuration.ReadWriteConnectionStringName) .AddDbContext((serviceProvider, options) => { options.UseNpgsql(serviceProvider.GetRequiredService>().NpgsqlDataSource); + options.AddInterceptors(serviceProvider.GetRequiredService()); + options.AddInterceptors(new ForceDistinctInterceptor()); }); return builder; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/ForcedDistinctInterceptor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/ForcedDistinctInterceptor.cs index b9b56a9e2..afc64332d 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/ForcedDistinctInterceptor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/ForcedDistinctInterceptor.cs @@ -97,7 +97,7 @@ public override ValueTask> ReaderExecutingAsync private static void ModifyCommand(DbCommand command) { // something slightly more sophisticated needed as this is going to fail if multiple tags were applied or query with more than one SELECT clause was used - if (command.CommandText.StartsWith($"-- {Apply}", StringComparison.Ordinal)) + if (command.CommandText.Contains($"-- {Apply}", StringComparison.Ordinal)) { command.CommandText = command.CommandText.Replace("SELECT ", "SELECT DISTINCT "); } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs new file mode 100644 index 000000000..49fd79da5 --- /dev/null +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs @@ -0,0 +1,106 @@ +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Storage.Internal; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using RadixDlt.NetworkGateway.GatewayApi.Configuration; +using RadixDlt.NetworkGateway.GatewayApi.Services; +using RadixDlt.NetworkGateway.PostgresIntegration.Metrics; +using System.Data.Common; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; + +namespace RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; + +internal class MetricsInterceptor : DbCommandInterceptor +{ + private const string QueryNameTag = "QueryName"; + private const string UnknownQueryName = "UNKNOWN"; + + private readonly IOptionsMonitor _slowQueriesLoggingOptions; + private readonly ILogger _logger; + private readonly ISqlQueryObserver _sqlQueryObserver; + + public MetricsInterceptor(ILoggerFactory loggerFactory, IOptionsMonitor slowQueriesLoggingOptions, ISqlQueryObserver sqlQueryObserver) + { + _slowQueriesLoggingOptions = slowQueriesLoggingOptions; + _sqlQueryObserver = sqlQueryObserver; + _logger = loggerFactory.CreateLogger(); + } + + public static string GetQueryNameTag(string queryName) + { + return $"{QueryNameTag}={queryName};"; + } + + public override DbDataReader ReaderExecuted(DbCommand command, CommandExecutedEventData eventData, DbDataReader result) + { + Observe(command, eventData); + return result; + } + + public override ValueTask ReaderExecutedAsync( + DbCommand command, + CommandExecutedEventData eventData, + DbDataReader result, + CancellationToken cancellationToken = default) + { + Observe(command, eventData); + return new ValueTask(result); + } + + private void Observe( + DbCommand command, + CommandExecutedEventData eventData) + { + var queryName = GetQueryName(command) ?? UnknownQueryName; + + _sqlQueryObserver.OnSqlQueryExecuted(queryName, eventData.Duration); + + var logQueriesLongerThan = _slowQueriesLoggingOptions.CurrentValue.SlowQueriesThreshold; + if (eventData.Duration > logQueriesLongerThan) + { +#pragma warning disable EF1001 + var parameters = command.Parameters.FormatParameters(true); +#pragma warning restore EF1001 + + _logger.LogWarning( + "Long running query: {query}, parameters: {queryParameters} duration: {queryDuration} milliseconds", + command.CommandText, parameters, eventData.Duration.TotalMilliseconds); + } + } + + private string? GetQueryName(DbCommand dbCommand) + { + const string QueryNameGroup = "queryName"; + var matches = Regex.Matches(dbCommand.CommandText, $"(?:{QueryNameTag}=)(?<{QueryNameGroup}>\\b.*?\\b);"); + + switch (matches.Count) + { + case 0: + _logger.LogDebug("Missing query name for: {commandText}", dbCommand.CommandText); + return null; + case 1: + { + var hasQueryName = matches.First().Groups.TryGetValue(QueryNameGroup, out var queryName); + + if (!hasQueryName || string.IsNullOrEmpty(queryName?.Value)) + { + _logger.LogDebug("Missing query name for: {commandText}", dbCommand.CommandText); + } + + return queryName!.Value; + } + + case > 1: + { + var foundTags = string.Join(',', matches.Select(x => x.Groups.Values.ToString())); + _logger.LogDebug("Query name provided multiple times: {foundTags}, in query: {commandText}", foundTags, dbCommand.CommandText); + return null; + } + + default: return null; + } + } +} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs index fe0321a85..c2bc14b23 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs @@ -71,6 +71,7 @@ using RadixDlt.NetworkGateway.Abstractions.Numerics; using RadixDlt.NetworkGateway.DataAggregator; using RadixDlt.NetworkGateway.DataAggregator.Services; +using RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; using RadixDlt.NetworkGateway.PostgresIntegration.Models; using RadixDlt.NetworkGateway.PostgresIntegration.Services; using System; @@ -171,6 +172,7 @@ private async Task UpdatePendingTransactions(ReadWriteDbContext dbContext, pt.LedgerDetails.LatestRejectionTimestamp, pt.LedgerDetails.LatestRejectionReason, }) + .WithQueryName() .ToListAsync(token); foreach (var details in pendingTransactions) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs index f35fbe64b..e8f991c4e 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs @@ -132,6 +132,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) emh ON true;") .AsNoTracking() + .WithQueryName() .ToDictionaryAsync(e => new MetadataLookup(e.EntityId, e.Key), token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentEntityMetadataHistoryFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -165,6 +166,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) emah ON true;") .AsNoTracking() + .WithQueryName() .ToDictionaryAsync(e => e.EntityId, token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentEntityAggregateMetadataHistoryFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -218,6 +220,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) eareh ON true;") .AsNoTracking() + .WithQueryName() .ToDictionaryAsync(e => new RoleAssignmentEntryLookup(e.EntityId, e.KeyRole, e.KeyModule), token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentEntityRoleAssignmentsEntryHistoryFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -253,6 +256,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) earah ON true;") .AsNoTracking() + .WithQueryName() .ToDictionaryAsync(e => e.EntityId, token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentEntityRoleAssignmentsAggregateHistoryFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -294,6 +298,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) erah ON true;") .AsNoTracking() + .WithQueryName() .ToDictionaryAsync(e => e.EntityId, token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentEntityResourceAggregateHistoryFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -336,6 +341,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) eravh ON true;") .AsNoTracking() + .WithQueryName() .ToDictionaryAsync(e => new EntityResourceLookup(e.EntityId, e.ResourceEntityId), token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentEntityResourceAggregatedVaultsHistoryFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -386,6 +392,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) ervah ON true;") .AsNoTracking() + .WithQueryName() .ToDictionaryAsync(e => new EntityResourceVaultLookup(e.EntityId, e.ResourceEntityId), token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentEntityResourceVaultAggregateHistoryFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -419,6 +426,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) evh ON true;") .AsNoTracking() + .WithQueryName() .ToDictionaryAsync(e => e.VaultEntityId, e => (EntityNonFungibleVaultHistory)e, token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentEntityNonFungibleVaultHistory), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -426,13 +434,6 @@ LIMIT 1 return result; } - // public async Task<> MostRecentPackageDefinitionHistory(List packageChanges, CancellationToken token) - // { - // var packageIds = packageChanges.Select(x => x.PackageEntityId).Distinct().ToList(); - // - // return await _dbContext. - // } - public async Task> MostRecentNonFungibleIdStoreHistoryFor(List nonFungibleIdStoreChanges, CancellationToken token) { if (!nonFungibleIdStoreChanges.Any()) @@ -459,6 +460,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) emh ON true;") .AsNoTracking() + .WithQueryName() .ToDictionaryAsync(e => e.NonFungibleResourceEntityId, token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentNonFungibleIdStoreHistoryFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -492,6 +494,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) rmesh ON true;") .AsNoTracking() + .WithQueryName() .ToDictionaryAsync(e => e.ResourceEntityId, token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentResourceEntitySupplyHistoryFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -520,6 +523,7 @@ FROM entities WHERE address = ANY({entityAddressesParameter}) )") .AsNoTracking() + .WithQueryName() .ToDictionaryAsync(e => e.Address, token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(ExistingEntitiesFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -565,6 +569,7 @@ public async Task> ExistingNo SELECT UNNEST({resourceEntityIds}), UNNEST({nonFungibleIds}) )") .AsNoTracking() + .WithQueryName() .ToDictionaryAsync(e => new NonFungibleIdLookup(e.NonFungibleResourceEntityId, e.NonFungibleId), token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(ExistingNonFungibleIdDataFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -615,6 +620,7 @@ LIMIT 1 ) vpkh ON true; ") .AsNoTracking() + .WithQueryName() .ToDictionaryAsync(e => new ValidatorKeyLookup(e.ValidatorEntityId, e.KeyType, e.Key), token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(ExistingValidatorKeysFor), Stopwatch.GetElapsedTime(sw), result.Count)); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs new file mode 100644 index 000000000..7e1687707 --- /dev/null +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs @@ -0,0 +1,27 @@ +using Prometheus; +using RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; +using System; +using System.IO; + +namespace RadixDlt.NetworkGateway.PostgresIntegration.Metrics; + +internal class SqlQueryMetricsHelper +{ + public static string GetQueryNameValue( + string operationName = "", + string filePath = "", + string methodName = "") + { + var fileName = Path.GetFileNameWithoutExtension(filePath); + + if (!string.IsNullOrEmpty(operationName) && + !string.Equals(operationName, methodName, StringComparison.InvariantCultureIgnoreCase)) + { + return $"{fileName}_{methodName}_{operationName}"; + } + else + { + return $"{fileName}_{methodName}"; + } + } +} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/QueryableExtensions.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/QueryableExtensions.cs new file mode 100644 index 000000000..1e4977422 --- /dev/null +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/QueryableExtensions.cs @@ -0,0 +1,21 @@ +using Microsoft.EntityFrameworkCore; +using RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; +using RadixDlt.NetworkGateway.PostgresIntegration.Metrics; +using System.Linq; +using System.Runtime.CompilerServices; + +namespace RadixDlt.NetworkGateway.PostgresIntegration; + +public static class QueryableExtensions +{ + public static IQueryable WithQueryName( + this IQueryable source, + string operationName = "", + [CallerFilePath] string filePath = "", + [CallerMemberName] string methodName = "") + { + var queryNameValue = SqlQueryMetricsHelper.GetQueryNameValue(operationName, filePath, methodName); + var queryName = MetricsInterceptor.GetQueryNameTag(queryNameValue); + return source.TagWith(queryName); + } +} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/RadixDlt.NetworkGateway.PostgresIntegration.csproj b/src/RadixDlt.NetworkGateway.PostgresIntegration/RadixDlt.NetworkGateway.PostgresIntegration.csproj index 70de143d4..389575040 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/RadixDlt.NetworkGateway.PostgresIntegration.csproj +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/RadixDlt.NetworkGateway.PostgresIntegration.csproj @@ -21,4 +21,5 @@ + diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/BlueprintProvider.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/BlueprintProvider.cs index aed5d1d53..ae589b77b 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/BlueprintProvider.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/BlueprintProvider.cs @@ -63,6 +63,7 @@ */ using Microsoft.EntityFrameworkCore; +using RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; using RadixDlt.NetworkGateway.PostgresIntegration.Models; using System.Collections.Generic; using System.Linq; @@ -109,6 +110,7 @@ INNER JOIN package_blueprint_history pbh ON pbh.name = v.blueprint_name AND pbh.version = v.blueprint_version and pbh.package_entity_id = v.package_entity_id WHERE from_state_version <= {ledgerState.StateVersion} ") + .WithQueryName() .ToDictionaryAsync( x => new BlueprintDefinitionIdentifier(x.Name, x.Version, x.PackageEntityId), x => x, diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs new file mode 100644 index 000000000..a044579bb --- /dev/null +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs @@ -0,0 +1,104 @@ +using Dapper; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Newtonsoft.Json; +using RadixDlt.NetworkGateway.GatewayApi.Configuration; +using RadixDlt.NetworkGateway.GatewayApi.Services; +using RadixDlt.NetworkGateway.PostgresIntegration.Metrics; +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; + +namespace RadixDlt.NetworkGateway.PostgresIntegration.Services; + +public interface IDapperWrapper +{ + Task> QueryAsync( + IDbConnection cnn, + CommandDefinition command, + string operationName = "", + [CallerFilePath] string filePath = "", + [CallerMemberName] string methodName = ""); + + Task QueryFirstOrDefaultAsync( + IDbConnection cnn, + CommandDefinition command, + string operationName = "", + [CallerFilePath] string filePath = "", + [CallerMemberName] string methodName = ""); +} + +public class DapperWrapper : IDapperWrapper +{ + private readonly IOptionsMonitor _slowQueriesLoggingOptions; + private readonly ILogger _logger; + private readonly ISqlQueryObserver _sqlQueryObserver; + + public DapperWrapper( + IOptionsMonitor slowQueriesLoggingOptions, + ILogger logger, + ISqlQueryObserver sqlQueryObserver) + { + _slowQueriesLoggingOptions = slowQueriesLoggingOptions; + _logger = logger; + _sqlQueryObserver = sqlQueryObserver; + } + + public async Task> QueryAsync( + IDbConnection cnn, + CommandDefinition command, + string operationName = "", + [CallerFilePath] string filePath = "", + [CallerMemberName] string methodName = "") + { + var stopwatch = Stopwatch.StartNew(); + + var result = await cnn.QueryAsync(command); + + var elapsed = stopwatch.Elapsed; + var queryName = SqlQueryMetricsHelper.GetQueryNameValue(operationName, filePath, methodName); + + _sqlQueryObserver.OnSqlQueryExecuted(queryName, elapsed); + + var logQueriesLongerThan = _slowQueriesLoggingOptions.CurrentValue.SlowQueriesThreshold; + if (elapsed > logQueriesLongerThan) + { + var parameters = JsonConvert.SerializeObject(command.Parameters); + _logger.LogWarning( + "Long running query: {query}, parameters: {queryParameters} duration: {queryDuration} milliseconds", + command.CommandText, parameters, elapsed); + } + + return result; + } + + public async Task QueryFirstOrDefaultAsync( + IDbConnection cnn, + CommandDefinition command, + string operationName = "", + [CallerFilePath] string filePath = "", + [CallerMemberName] string methodName = "") + { + var stopwatch = Stopwatch.StartNew(); + + var result = await cnn.QueryFirstOrDefaultAsync(command); + + var elapsed = stopwatch.Elapsed; + var queryName = SqlQueryMetricsHelper.GetQueryNameValue(operationName, filePath, methodName); + _sqlQueryObserver.OnSqlQueryExecuted(queryName, elapsed); + + var logQueriesLongerThan = _slowQueriesLoggingOptions.CurrentValue.SlowQueriesThreshold; + if (elapsed > logQueriesLongerThan) + { + var parameters = JsonConvert.SerializeObject(command.Parameters); + _logger.LogWarning( + "Long running query: {query}, parameters: {queryParameters} duration: {queryDuration} milliseconds", + command.CommandText, parameters, elapsed); + } + + return result; + } +} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs index 191b45685..0f39fa30f 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs @@ -73,6 +73,7 @@ using RadixDlt.NetworkGateway.GatewayApi.Configuration; using RadixDlt.NetworkGateway.GatewayApi.Exceptions; using RadixDlt.NetworkGateway.GatewayApi.Services; +using RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; using RadixDlt.NetworkGateway.PostgresIntegration.Models; using System; using System.Collections.Generic; @@ -117,20 +118,22 @@ private record struct ExplicitMetadataLookup(long EntityId, string MetadataKey); private readonly ReadOnlyDbContext _dbContext; private readonly IVirtualEntityDataProvider _virtualEntityDataProvider; private readonly IRoleAssignmentQuerier _roleAssignmentQuerier; + private readonly IDapperWrapper _dapperWrapper; public EntityStateQuerier( INetworkConfigurationProvider networkConfigurationProvider, ReadOnlyDbContext dbContext, IOptionsSnapshot endpointConfiguration, IVirtualEntityDataProvider virtualEntityDataProvider, - IRoleAssignmentsMapper roleAssignmentsMapper, - IRoleAssignmentQuerier roleAssignmentQuerier) + IRoleAssignmentQuerier roleAssignmentQuerier, + IDapperWrapper dapperWrapper) { _networkConfigurationProvider = networkConfigurationProvider; _dbContext = dbContext; _endpointConfiguration = endpointConfiguration; _virtualEntityDataProvider = virtualEntityDataProvider; _roleAssignmentQuerier = roleAssignmentQuerier; + _dapperWrapper = dapperWrapper; } public async Task EntityDetails( @@ -530,7 +533,7 @@ FROM most_recent_non_fungible_id_store_history_slice hs long totalCount = 0; - var items = (await _dbContext.Database.GetDbConnection().QueryAsync(cd)) + var items = (await _dapperWrapper.QueryAsync(_dbContext.Database.GetDbConnection(), cd)) .ToList() .Select(vm => { @@ -574,7 +577,9 @@ FROM non_fungible_schema_history nfsh }, cancellationToken: token); - var nonFungibleDataSchema = await _dbContext.Database.GetDbConnection().QueryFirstOrDefaultAsync(nonFungibleDataSchemaQuery); + var nonFungibleDataSchema = await _dapperWrapper.QueryFirstOrDefaultAsync( + _dbContext.Database.GetDbConnection(), nonFungibleDataSchemaQuery, "GetNonFungibleDataSchema" + ); if (nonFungibleDataSchema == null) { @@ -605,7 +610,12 @@ ORDER BY nfid.from_state_version DESC var items = new List(); - foreach (var vm in await _dbContext.Database.GetDbConnection().QueryAsync(cd)) + var result = await _dapperWrapper.QueryAsync( + _dbContext.Database.GetDbConnection(), + cd, + "GetNonFungibleData"); + + foreach (var vm in result) { var programmaticJson = !vm.IsDeleted ? ScryptoSborUtils.DataToProgrammaticJson(vm.Data, nonFungibleDataSchema.Schema, @@ -676,7 +686,7 @@ LIMIT 1 }, cancellationToken: token); - var result = await _dbContext.Database.GetDbConnection().QueryAsync(cd); + var result = await _dapperWrapper.QueryAsync(_dbContext.Database.GetDbConnection(), cd); return new GatewayModel.StateNonFungibleLocationResponse( ledgerState: ledgerState, @@ -707,6 +717,7 @@ LIMIT 1 .OrderBy(e => e.FromStateVersion) .ThenBy(e => e.Id) .Take(validatorsPageSize + 1) + .WithQueryName("GetValidators") .ToListAsync(token); var findEpochSubquery = _dbContext @@ -721,6 +732,7 @@ LIMIT 1 .ValidatorActiveSetHistory .Include(e => e.PublicKey) .Where(e => e.Epoch == findEpochSubquery.First()) + .WithQueryName("GetValidatorActiveSet") .ToDictionaryAsync(e => e.PublicKey.ValidatorEntityId, token); var totalStake = activeSetById @@ -801,11 +813,15 @@ LIMIT 1 }, cancellationToken: token); - var validatorsDetails = (await _dbContext.Database.GetDbConnection().QueryAsync(cd)).ToList(); + var validatorsDetails = (await _dapperWrapper.QueryAsync( + _dbContext.Database.GetDbConnection(), cd, "GetValidatorDetails") + ).ToList(); + var vaultAddresses = await _dbContext .Entities .Where(e => validatorVaultIds.Contains(e.Id)) .Select(e => new { e.Id, e.Address }) + .WithQueryName("GetVaultAddresses") .ToDictionaryAsync(e => e.Id, e => e.Address, token); var metadataById = await GetMetadataSlices(validatorIds, 0, _endpointConfiguration.Value.DefaultPageSize, ledgerState, token); @@ -887,7 +903,9 @@ ORDER BY kvssh.from_state_version DESC }, cancellationToken: token); - var keyValueStoreSchema = await _dbContext.Database.GetDbConnection().QueryFirstOrDefaultAsync(keyValueStoreSchemaQuery); + var keyValueStoreSchema = await _dapperWrapper.QueryFirstOrDefaultAsync( + _dbContext.Database.GetDbConnection(), keyValueStoreSchemaQuery, "GetKeyValueStoreSchema" + ); if (keyValueStoreSchema == null) { @@ -911,6 +929,7 @@ FROM key_value_store_entry_history ORDER BY from_state_version DESC LIMIT 1 ) kvseh ON TRUE;") + .WithQueryName("GetKeyValueStores") .ToListAsync(token); var items = new List(); @@ -964,8 +983,7 @@ LIMIT 1 }, cancellationToken: token); - var result = await _dbContext.Database.GetDbConnection().QueryAsync(cd); - + var result = await _dapperWrapper.QueryAsync(_dbContext.Database.GetDbConnection(), cd); return result.ToList(); } @@ -1008,7 +1026,7 @@ INNER JOIN LATERAL UNNEST(metadata_slice) WITH ORDINALITY AS metadata_join(id, o }, cancellationToken: token); - foreach (var vm in await _dbContext.Database.GetDbConnection().QueryAsync(cd)) + foreach (var vm in await _dapperWrapper.QueryAsync(_dbContext.Database.GetDbConnection(), cd)) { if (!result.ContainsKey(vm.EntityId)) { @@ -1070,6 +1088,7 @@ FROM entity_metadata_history ORDER BY from_state_version DESC LIMIT 1 ) emh ON TRUE;") + .WithQueryName() .ToListAsync(token); var result = new Dictionary(); @@ -1153,7 +1172,7 @@ order by ah.ord var items = new List(); - foreach (var vm in await _dbContext.Database.GetDbConnection().QueryAsync(cd)) + foreach (var vm in await _dapperWrapper.QueryAsync(_dbContext.Database.GetDbConnection(), cd)) { totalCount = vm.ResourcesTotalCount; @@ -1186,6 +1205,7 @@ FROM resource_entity_supply_history ORDER BY from_state_version DESC LIMIT 1 ) resh ON true") + .WithQueryName() .ToDictionaryAsync(e => e.ResourceEntityId, token); foreach (var missing in entityIds.Except(result.Keys)) @@ -1247,6 +1267,7 @@ private async Task> GetEntities(List addresse var entities = await _dbContext .Entities .Where(e => e.FromStateVersion <= ledgerState.StateVersion && addresses.Contains(e.Address)) + .WithQueryName() .ToListAsync(token); foreach (var address in addresses) @@ -1291,6 +1312,7 @@ FROM state_history ORDER BY from_state_version DESC LIMIT 1 ) esh ON TRUE;") + .WithQueryName("GetStateHistory") .ToListAsync(token); var schemasToLoad = states @@ -1320,6 +1342,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) esh ON TRUE; ") + .WithQueryName("GetSchemas") .ToDictionaryAsync( x => new SchemaIdentifier(x.SchemaHash, x.EntityId), x => x.Schema, @@ -1378,6 +1401,7 @@ INNER JOIN LATERAL( FROM package_blueprint_history WHERE package_entity_id = variables.entity_id AND from_state_version <= {ledgerState.StateVersion} ) pbh ON true") + .WithQueryName() .ToListAsync(token)) .GroupBy(b => b.PackageEntityId) .ToDictionary(g => g.Key, g => g.ToArray()); @@ -1403,6 +1427,7 @@ FROM package_code_history ORDER BY from_state_version DESC LIMIT 1 ) pch ON true") + .WithQueryName() .ToDictionaryAsync(e => e.PackageEntityId, token); } @@ -1424,6 +1449,7 @@ INNER JOIN LATERAL( FROM schema_history WHERE entity_id = variables.entity_id AND from_state_version <= {ledgerState.StateVersion} ) psh ON true") + .WithQueryName() .ToListAsync(token)) .GroupBy(b => b.EntityId) .ToDictionary(g => g.Key, g => g.ToArray()); @@ -1464,6 +1490,7 @@ private async Task> GetCorrelatedEntityAddresses .Entities .Where(e => ids.Contains(e.Id) && e.FromStateVersion <= ledgerState.StateVersion) .Select(e => new { e.Id, GlobalAddress = e.Address }) + .WithQueryName() .ToDictionaryAsync(e => e.Id, e => e.GlobalAddress, token); } @@ -1488,6 +1515,7 @@ private async Task> ResolveResourceEntityIds( .Entities .Where(e => addresses.Contains(e.Address)) .Select(e => new { e.Id, e.Address }) + .WithQueryName() .ToDictionaryAsync(e => e.Address, e => e.Id, token); } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.fungibles.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.fungibles.cs index bdda89902..fecc8f0bd 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.fungibles.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.fungibles.cs @@ -167,7 +167,7 @@ LIMIT 1 var resources = new Dictionary(); - foreach (var vm in await _dbContext.Database.GetDbConnection().QueryAsync(cd)) + foreach (var vm in await _dapperWrapper.QueryAsync(_dbContext.Database.GetDbConnection(), cd)) { resourcesTotalCount = vm.ResourceTotalCount; @@ -250,7 +250,7 @@ LIMIT 1 }, cancellationToken: token); - var result = (await _dbContext.Database.GetDbConnection().QueryAsync(cd)).ToList(); + var result = (await _dapperWrapper.QueryAsync(_dbContext.Database.GetDbConnection(), cd)).ToList(); var vaultsTotalCount = result.FirstOrDefault()?.VaultTotalCount ?? 0; var castedResult = result .Select(x => diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.nonfungibles.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.nonfungibles.cs index e9ee001b9..5420a07ed 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.nonfungibles.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.nonfungibles.cs @@ -159,7 +159,7 @@ LIMIT 1 var items = new List(); - foreach (var vm in await _dbContext.Database.GetDbConnection().QueryAsync(cd)) + foreach (var vm in await _dapperWrapper.QueryAsync(_dbContext.Database.GetDbConnection(), cd)) { totalCount = vm.ResourcesTotalCount; @@ -249,7 +249,7 @@ LIMIT 1 }, cancellationToken: token); - return (await _dbContext.Database.GetDbConnection().QueryAsync(cd)).ToList(); + return (await _dapperWrapper.QueryAsync(_dbContext.Database.GetDbConnection(), cd)).ToList(); } private async Task> GetNonFungibleResourceVaults( @@ -314,7 +314,7 @@ LIMIT 1 }, cancellationToken: token); - return (await _dbContext.Database.GetDbConnection().QueryAsync(cd)).ToList(); + return (await _dapperWrapper.QueryAsync(_dbContext.Database.GetDbConnection(), cd)).ToList(); } private async Task GetNonFungibleIdsSlice( @@ -357,7 +357,7 @@ order by ord var totalCount = 0; - var items = (await _dbContext.Database.GetDbConnection().QueryAsync(cd)) + var items = (await _dapperWrapper.QueryAsync(_dbContext.Database.GetDbConnection(), cd)) .ToList() .Select(vm => { @@ -435,7 +435,7 @@ FROM unnested_nfids un }, cancellationToken: token); - var result = (await _dbContext.Database.GetDbConnection().QueryAsync(cd)).ToList(); + var result = (await _dapperWrapper.QueryAsync(_dbContext.Database.GetDbConnection(), cd)).ToList(); return result; } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/PendingTransactionPrunerService.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/PendingTransactionPrunerService.cs index 802817706..93bce3394 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/PendingTransactionPrunerService.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/PendingTransactionPrunerService.cs @@ -117,6 +117,7 @@ public async Task PrunePendingTransactions(CancellationToken token = default) .PendingTransactions .GroupBy(canPruneExpression) .Select(g => new PendingTransactionCountByPruneStatus(g.Key, g.Count())) + .WithQueryName("GetTransactionsToPrune") .ToListAsync(token); await _observers.ForEachAsync(x => x.PrePendingTransactionPrune(countByStatus)); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/PendingTransactionResubmissionService.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/PendingTransactionResubmissionService.cs index 1c6f7c7a5..f720dd6b2 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/PendingTransactionResubmissionService.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/PendingTransactionResubmissionService.cs @@ -168,6 +168,7 @@ await GetPendingTransactionsNeedingResubmission(instantForTransactionChoosing, d .OrderBy(mt => mt.GatewayHandling.ResubmitFromTimestamp) .Include(t => t.Payload) .Take(batchSize) + .WithQueryName() .ToListAsync(token); var totalTransactionsNeedingResubmission = transactionsToResubmit.Count < batchSize diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/RoleAssignmentQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/RoleAssignmentQuerier.cs index 274b84591..fa9e6d929 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/RoleAssignmentQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/RoleAssignmentQuerier.cs @@ -64,6 +64,7 @@ using Microsoft.EntityFrameworkCore; using Newtonsoft.Json; + using RadixDlt.NetworkGateway.PostgresIntegration.Models; using System.Collections.Generic; using System.Diagnostics; @@ -116,12 +117,14 @@ public RoleAssignmentQuerier( var ownerRoles = await _dbContext .EntityRoleAssignmentsOwnerHistory .Where(e => ownerRoleIds.Contains(e.Id)) + .WithQueryName("GetOwnerRoles") .ToListAsync(token); var entries = await _dbContext .EntityRoleAssignmentsEntryHistory .Where(e => roleAssignmentsHistory.Contains(e.Id)) .Where(e => !e.IsDeleted) + .WithQueryName("GetRoleEntires") .ToListAsync(token); return _roleAssignmentsMapper.GetEffectiveRoleAssignments(componentEntities, blueprintAuthConfigs, ownerRoles, entries); @@ -172,6 +175,7 @@ FROM entity_role_assignments_aggregate_history ORDER BY from_state_version DESC LIMIT 1 ) earah ON TRUE;") + .WithQueryName() .ToListAsync(token); return aggregates; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs index 04830a455..98b0c92df 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs @@ -300,6 +300,7 @@ public async Task GetTransactionStream(TransactionS var stateVersions = await searchQuery .TagWith(ForceDistinctInterceptor.Apply) .Take(request.PageSize + 1) + .WithQueryName("GetTransactionsStateVersions") .ToListAsync(token); var transactions = await GetTransactions(stateVersions.Take(request.PageSize).ToList(), request.OptIns, token); @@ -562,6 +563,7 @@ private async Task> LookupPendingTransact pt.NetworkDetails.LastSubmitErrorTitle ) ) + .WithQueryName() .ToListAsync(token); } @@ -573,6 +575,7 @@ private async Task> LookupPendingTransact var transactions = await _dbContext .LedgerTransactions .Where(ult => transactionStateVersions.Contains(ult.StateVersion)) + .WithQueryName("GetTransactions") .ToListAsync(token); var entityIdToAddressMap = await GetEntityAddresses(transactions.SelectMany(x => x.AffectedGlobalEntities).ToList(), token); @@ -597,6 +600,7 @@ SELECT UNNEST({entityIds}), UNNEST({schemaHashes}) SELECT sh.* FROM variables var INNER JOIN schema_history sh ON sh.entity_id = var.entity_id AND sh.schema_hash = var.schema_hash") + .WithQueryName("GetEventSchemas") .ToDictionaryAsync(x => new SchemaLookup(x.EntityId, x.SchemaHash), x => x.Schema, token); } @@ -642,6 +646,7 @@ private async Task> GetEntityIds(List addresses .Entities .Where(e => addresses.Contains(e.Address)) .Select(e => new { e.Id, e.Address }) + .WithQueryName() .ToDictionaryAsync(e => e.Address.ToString(), e => e.Id, token); } @@ -656,6 +661,7 @@ private async Task> GetEntityAddresses(List entit .Entities .Where(e => entityIds.Contains(e.Id)) .Select(e => new { e.Id, e.Address }) + .WithQueryName() .ToDictionaryAsync(e => e.Id, e => e.Address.ToString(), token); } } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/ValidatorQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/ValidatorQuerier.cs index 3f0ecaf29..dd6f682d8 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/ValidatorQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/ValidatorQuerier.cs @@ -80,10 +80,12 @@ internal class ValidatorQuerier : IValidatorQuerier private record ValidatorUptimeViewModel(long ValidatorEntityId, long? ProposalsMadeSum, long? ProposalsMissedSum, long EpochsActiveIn); private readonly ReadOnlyDbContext _dbContext; + private readonly IDapperWrapper _dapperWrapper; - public ValidatorQuerier(ReadOnlyDbContext dbContext) + public ValidatorQuerier(ReadOnlyDbContext dbContext, IDapperWrapper dapperWrapper) { _dbContext = dbContext; + _dapperWrapper = dapperWrapper; } public async Task ValidatorsUptimeStatistics( @@ -97,6 +99,7 @@ public ValidatorQuerier(ReadOnlyDbContext dbContext) var validators = await _dbContext .Entities .Where(e => addresses.Contains(e.Address) && e.FromStateVersion <= ledgerState.StateVersion) + .WithQueryName("GetValidators") .ToDictionaryAsync(e => e.Id, e => e.Address, token); var cd = new CommandDefinition( @@ -117,7 +120,7 @@ FROM validator_emission_statistics }, cancellationToken: token); - var validatorUptime = (await _dbContext.Database.GetDbConnection().QueryAsync(cd)) + var validatorUptime = (await _dapperWrapper.QueryAsync(_dbContext.Database.GetDbConnection(), cd, "GetValidatorEmissionStatistics")) .ToDictionary(e => e.ValidatorEntityId); var items = validators diff --git a/src/RadixDlt.NetworkGateway.PrometheusIntegration/DataAggregatorBuilderExtensions.cs b/src/RadixDlt.NetworkGateway.PrometheusIntegration/DataAggregatorBuilderExtensions.cs index 38bc3c272..3ea06b86d 100644 --- a/src/RadixDlt.NetworkGateway.PrometheusIntegration/DataAggregatorBuilderExtensions.cs +++ b/src/RadixDlt.NetworkGateway.PrometheusIntegration/DataAggregatorBuilderExtensions.cs @@ -71,6 +71,7 @@ using RadixDlt.NetworkGateway.DataAggregator.Services; using RadixDlt.NetworkGateway.DataAggregator.Workers.GlobalWorkers; using RadixDlt.NetworkGateway.DataAggregator.Workers.NodeWorkers; +using RadixDlt.NetworkGateway.GatewayApi.Services; namespace RadixDlt.NetworkGateway.PrometheusIntegration; @@ -101,7 +102,8 @@ public static DataAggregatorBuilder AddPrometheusObserverMetrics(this DataAggreg .AddSingleton(provider => provider.GetRequiredService()) .AddSingleton(provider => provider.GetRequiredService()) .AddSingleton(provider => provider.GetRequiredService()) - .AddSingleton(provider => provider.GetRequiredService()); + .AddSingleton(provider => provider.GetRequiredService()) + .AddSingleton(provider => provider.GetRequiredService()); return builder; } diff --git a/src/RadixDlt.NetworkGateway.PrometheusIntegration/DataAggregatorMetricsObserver.cs b/src/RadixDlt.NetworkGateway.PrometheusIntegration/DataAggregatorMetricsObserver.cs index 6de428712..ce0e8b1cb 100644 --- a/src/RadixDlt.NetworkGateway.PrometheusIntegration/DataAggregatorMetricsObserver.cs +++ b/src/RadixDlt.NetworkGateway.PrometheusIntegration/DataAggregatorMetricsObserver.cs @@ -71,6 +71,7 @@ using RadixDlt.NetworkGateway.DataAggregator.Services; using RadixDlt.NetworkGateway.DataAggregator.Workers.GlobalWorkers; using RadixDlt.NetworkGateway.DataAggregator.Workers.NodeWorkers; +using RadixDlt.NetworkGateway.GatewayApi.Services; using System; using System.Collections.Generic; using System.Threading.Tasks; @@ -92,8 +93,13 @@ internal class DataAggregatorMetricsObserver : ILedgerExtenderServiceObserver, INetworkConfigurationReaderObserver, INetworkStatusReaderObserver, - ITransactionStreamReaderObserver + ITransactionStreamReaderObserver, + ISqlQueryObserver { + private static readonly Histogram _sqlQueryDuration = Prometheus.Metrics + .CreateHistogram("sql_query_duration", "The duration of SQL queries processed by this app.", + new HistogramConfiguration { LabelNames = new[] { "query_name" } }); + private static readonly Counter _globalWorkerErrorsCount = Metrics .CreateCounter( "ng_workers_global_error_count", @@ -573,4 +579,9 @@ ValueTask ITransactionStreamReaderObserver.GetTransactionsFailed(string nodeName return ValueTask.CompletedTask; } + + public void OnSqlQueryExecuted(string queryName, TimeSpan duration) + { + _sqlQueryDuration.WithLabels(queryName).Observe(duration.TotalMilliseconds); + } } diff --git a/src/RadixDlt.NetworkGateway.PrometheusIntegration/GatewayApiBuilderExtensions.cs b/src/RadixDlt.NetworkGateway.PrometheusIntegration/GatewayApiBuilderExtensions.cs index 7edc14bc7..65ae974e7 100644 --- a/src/RadixDlt.NetworkGateway.PrometheusIntegration/GatewayApiBuilderExtensions.cs +++ b/src/RadixDlt.NetworkGateway.PrometheusIntegration/GatewayApiBuilderExtensions.cs @@ -87,7 +87,8 @@ public static GatewayApiBuilder AddPrometheusObserverMetrics(this GatewayApiBuil .AddSingleton(provider => provider.GetRequiredService()) .AddSingleton(provider => provider.GetRequiredService()) .AddSingleton(provider => provider.GetRequiredService()) - .AddSingleton(provider => provider.GetRequiredService()); + .AddSingleton(provider => provider.GetRequiredService()) + .AddSingleton(provider => provider.GetRequiredService()); return builder; } diff --git a/src/RadixDlt.NetworkGateway.PrometheusIntegration/GatewayApiMetricObserver.cs b/src/RadixDlt.NetworkGateway.PrometheusIntegration/GatewayApiMetricObserver.cs index 888419028..fef9f2833 100644 --- a/src/RadixDlt.NetworkGateway.PrometheusIntegration/GatewayApiMetricObserver.cs +++ b/src/RadixDlt.NetworkGateway.PrometheusIntegration/GatewayApiMetricObserver.cs @@ -85,8 +85,13 @@ internal class GatewayApiMetricObserver : ISubmissionServiceObserver, IPreviewServiceObserver, ILedgerStateQuerierObserver, - ISubmissionTrackingServiceObserver + ISubmissionTrackingServiceObserver, + ISqlQueryObserver { + private static readonly Histogram _sqlQueryDuration = Metrics + .CreateHistogram("sql_query_duration", "The duration of SQL queries processed by this app.", + new HistogramConfiguration { LabelNames = new[] { "query_name" } }); + private static readonly Counter _apiResponseErrorCount = Metrics .CreateCounter( "ng_gateway_response_error_count", @@ -169,6 +174,11 @@ internal class GatewayApiMetricObserver : "Number of mempool transactions submitted to the gateway which were already being tracked" ); + public void OnSqlQueryExecuted(string queryName, TimeSpan duration) + { + _sqlQueryDuration.WithLabels(queryName).Observe(duration.TotalMilliseconds); + } + void IExceptionObserver.OnException(ActionContext actionContext, Exception exception, KnownGatewayErrorException gatewayErrorException) { // actionContext.HttpContext.Request.Method - GET or POST From bb1eabfa563c527968d849bd459f0571c14c0089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Tue, 26 Sep 2023 20:00:25 +0200 Subject: [PATCH 08/22] add changelog entry. cleanup usings. --- CHANGELOG.md | 5 +++-- .../CommonDbContext.cs | 1 - .../GatewayApiBuilderExtensions.cs | 5 ----- .../Interceptors/MetricsInterceptor.cs | 9 +-------- .../PostgresLedgerExtenderService.cs | 1 - .../Metrics/SqlQueryMetricsHelper.cs | 14 ++++++++++---- .../Migrations/MigrationsDbContextModelSnapshot.cs | 3 --- .../QueryableExtensions.cs | 6 ++---- .../Services/BlueprintProvider.cs | 1 - .../Services/DapperWrapper.cs | 1 - .../Services/EntityStateQuerier.cs | 1 - .../Services/LedgerStateQuerier.cs | 1 - 12 files changed, 16 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03edadc96..3c02a62c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,12 @@ -## Next release (name TBC) +## 1.0.0 - Babylon Launch Release Date: _unreleased_ ### Breaking changes _None_ ### What’s new? -_N/A_ +- log warning if sql query takes longer than configured threshold (default to 250ms) for both entity framework and dapper queries. +- gather execution time metrics for all sql queries (both entity framework and dapper). ## 1.0.0-rc3 - Babylon Launch (Release Candidate 3) Release Date: Friday 22nd September 2023 diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs index dfda16852..5d64b29a0 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs @@ -68,7 +68,6 @@ using RadixDlt.NetworkGateway.Abstractions.Numerics; using RadixDlt.NetworkGateway.PostgresIntegration.Models; using RadixDlt.NetworkGateway.PostgresIntegration.ValueConverters; -using System; namespace RadixDlt.NetworkGateway.PostgresIntegration; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayApiBuilderExtensions.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayApiBuilderExtensions.cs index cc97c0693..e623c8a87 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayApiBuilderExtensions.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayApiBuilderExtensions.cs @@ -63,15 +63,10 @@ */ using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; using RadixDlt.NetworkGateway.GatewayApi; -using RadixDlt.NetworkGateway.GatewayApi.Configuration; using RadixDlt.NetworkGateway.GatewayApi.Services; using RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; -using RadixDlt.NetworkGateway.PostgresIntegration.Metrics; using RadixDlt.NetworkGateway.PostgresIntegration.Services; namespace RadixDlt.NetworkGateway.PostgresIntegration; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs index 49fd79da5..a386dd6c4 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs @@ -15,9 +15,7 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; internal class MetricsInterceptor : DbCommandInterceptor { - private const string QueryNameTag = "QueryName"; private const string UnknownQueryName = "UNKNOWN"; - private readonly IOptionsMonitor _slowQueriesLoggingOptions; private readonly ILogger _logger; private readonly ISqlQueryObserver _sqlQueryObserver; @@ -29,11 +27,6 @@ public MetricsInterceptor(ILoggerFactory loggerFactory, IOptionsMonitor(); } - public static string GetQueryNameTag(string queryName) - { - return $"{QueryNameTag}={queryName};"; - } - public override DbDataReader ReaderExecuted(DbCommand command, CommandExecutedEventData eventData, DbDataReader result) { Observe(command, eventData); @@ -74,7 +67,7 @@ private void Observe( private string? GetQueryName(DbCommand dbCommand) { const string QueryNameGroup = "queryName"; - var matches = Regex.Matches(dbCommand.CommandText, $"(?:{QueryNameTag}=)(?<{QueryNameGroup}>\\b.*?\\b);"); + var matches = Regex.Matches(dbCommand.CommandText, $"(?:{SqlQueryMetricsHelper.QueryNameTag}=)(?<{QueryNameGroup}>\\b.*?\\b);"); switch (matches.Count) { diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs index c2bc14b23..b954a67e6 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs @@ -71,7 +71,6 @@ using RadixDlt.NetworkGateway.Abstractions.Numerics; using RadixDlt.NetworkGateway.DataAggregator; using RadixDlt.NetworkGateway.DataAggregator.Services; -using RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; using RadixDlt.NetworkGateway.PostgresIntegration.Models; using RadixDlt.NetworkGateway.PostgresIntegration.Services; using System; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs index 7e1687707..9857486cd 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs @@ -1,12 +1,12 @@ -using Prometheus; -using RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; -using System; +using System; using System.IO; namespace RadixDlt.NetworkGateway.PostgresIntegration.Metrics; -internal class SqlQueryMetricsHelper +public static class SqlQueryMetricsHelper { + public const string QueryNameTag = "QueryName"; + public static string GetQueryNameValue( string operationName = "", string filePath = "", @@ -24,4 +24,10 @@ public static string GetQueryNameValue( return $"{fileName}_{methodName}"; } } + + public static string GenerateQueryNameTag(string operationName = "", string filePath = "", string methodName = "") + { + var queryName = GetQueryNameValue(operationName, filePath, methodName); + return $"{QueryNameTag}={queryName};"; + } } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs index 617905d36..8768f6b83 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs @@ -68,11 +68,8 @@ using System.Numerics; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; using RadixDlt.NetworkGateway.Abstractions.Addressing; using RadixDlt.NetworkGateway.Abstractions.Model; -using RadixDlt.NetworkGateway.PostgresIntegration; using RadixDlt.NetworkGateway.PostgresIntegration.Models; #nullable disable diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/QueryableExtensions.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/QueryableExtensions.cs index 1e4977422..10be9b6d9 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/QueryableExtensions.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/QueryableExtensions.cs @@ -1,5 +1,4 @@ using Microsoft.EntityFrameworkCore; -using RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; using RadixDlt.NetworkGateway.PostgresIntegration.Metrics; using System.Linq; using System.Runtime.CompilerServices; @@ -14,8 +13,7 @@ public static IQueryable WithQueryName( [CallerFilePath] string filePath = "", [CallerMemberName] string methodName = "") { - var queryNameValue = SqlQueryMetricsHelper.GetQueryNameValue(operationName, filePath, methodName); - var queryName = MetricsInterceptor.GetQueryNameTag(queryNameValue); - return source.TagWith(queryName); + var queryNameTag = SqlQueryMetricsHelper.GenerateQueryNameTag(operationName, filePath, methodName); + return source.TagWith(queryNameTag); } } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/BlueprintProvider.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/BlueprintProvider.cs index ae589b77b..c9994e441 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/BlueprintProvider.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/BlueprintProvider.cs @@ -63,7 +63,6 @@ */ using Microsoft.EntityFrameworkCore; -using RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; using RadixDlt.NetworkGateway.PostgresIntegration.Models; using System.Collections.Generic; using System.Linq; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs index a044579bb..beed794b8 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs @@ -5,7 +5,6 @@ using RadixDlt.NetworkGateway.GatewayApi.Configuration; using RadixDlt.NetworkGateway.GatewayApi.Services; using RadixDlt.NetworkGateway.PostgresIntegration.Metrics; -using System; using System.Collections.Generic; using System.Data; using System.Diagnostics; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs index 0f39fa30f..cd460717e 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs @@ -73,7 +73,6 @@ using RadixDlt.NetworkGateway.GatewayApi.Configuration; using RadixDlt.NetworkGateway.GatewayApi.Exceptions; using RadixDlt.NetworkGateway.GatewayApi.Services; -using RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; using RadixDlt.NetworkGateway.PostgresIntegration.Models; using System; using System.Collections.Generic; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/LedgerStateQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/LedgerStateQuerier.cs index 95d145285..3209ddbf2 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/LedgerStateQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/LedgerStateQuerier.cs @@ -65,7 +65,6 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Npgsql; using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Extensions; using RadixDlt.NetworkGateway.GatewayApi.Configuration; From 6c34381131a86ba7220ee83635b304e9ef907cb5 Mon Sep 17 00:00:00 2001 From: David Edey Date: Wed, 27 Sep 2023 01:47:32 +0100 Subject: [PATCH 09/22] fix: PermanentRejection status is returned if expiry epoch is past --- .../Services/TransactionQuerier.cs | 69 ++++++++++++++----- 1 file changed, 50 insertions(+), 19 deletions(-) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs index 04830a455..009d67c9a 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs @@ -343,6 +343,7 @@ private record CommittedTransactionSummary( private record PendingTransactionSummary( string PayloadHash, + ulong EndEpochExclusive, DateTime? ResubmitFromTimestamp, string? HandlingStatusReason, PendingTransactionPayloadLedgerStatus PayloadStatus, @@ -358,8 +359,9 @@ private class PendingTransactionResponseAggregator private readonly long? _committedStateVersion; private readonly List _knownPayloads = new(); private readonly GatewayModel.TransactionIntentStatus? _committedIntentStatus; + private readonly string? _committedErrorMessage; private PendingTransactionIntentLedgerStatus _mostAccuratePendingTransactionIntentLedgerStatus = PendingTransactionIntentLedgerStatus.Unknown; - private string? _errorMessageForMostAccurateIntentLedgerStatus; + private string? _rejectionReasonForMostAccurateIntentLedgerStatus; internal PendingTransactionResponseAggregator(GatewayModel.LedgerState ledgerState, CommittedTransactionSummary? committedTransactionSummary) { @@ -380,7 +382,7 @@ internal PendingTransactionResponseAggregator(GatewayModel.LedgerState ledgerSta }; _committedIntentStatus = intentStatus; - _errorMessageForMostAccurateIntentLedgerStatus = committedTransactionSummary.ErrorMessage; + _committedErrorMessage = committedTransactionSummary.ErrorMessage; _knownPayloads.Add(new GatewayModel.TransactionStatusResponseKnownPayloadItem( payloadHash: committedTransactionSummary.PayloadHash, @@ -421,20 +423,43 @@ internal void AddPendingTransaction(PendingTransactionSummary pendingTransaction if (pendingTransactionSummary.IntentStatus.AggregationPriorityAcrossKnownPayloads() >= _mostAccuratePendingTransactionIntentLedgerStatus.AggregationPriorityAcrossKnownPayloads()) { _mostAccuratePendingTransactionIntentLedgerStatus = pendingTransactionSummary.IntentStatus; - _errorMessageForMostAccurateIntentLedgerStatus = pendingTransactionSummary.LatestRejectionReason; + _rejectionReasonForMostAccurateIntentLedgerStatus = pendingTransactionSummary.LatestRejectionReason; } - var latestErrorMessage = pendingTransactionSummary.LatestRejectionReason == pendingTransactionSummary.InitialRejectionReason - ? null - : pendingTransactionSummary.LatestRejectionReason; + var initialRejectionReason = pendingTransactionSummary.InitialRejectionReason; + var latestRejectionReason = pendingTransactionSummary.LatestRejectionReason; + + // If the intent's EndEpochExclusive has been reached, then the payload and intent must be permanently rejected (assuming they're not already committed). + if ((ulong)_ledgerState.Epoch >= pendingTransactionSummary.EndEpochExclusive) + { + // The if statement is defence-in-depth to avoid replacing a Committed status with PermanentlyRejected. + // This shouldn't matter, because we'd see a committed transaction in this case, which would trump the _mostAccuratePendingTransactionIntentLedgerStatus when we + // resolve the status. But best to be defensive anyway. + if (PendingTransactionIntentLedgerStatus.PermanentRejection.AggregationPriorityAcrossKnownPayloads() >= + _mostAccuratePendingTransactionIntentLedgerStatus.AggregationPriorityAcrossKnownPayloads()) + { + _mostAccuratePendingTransactionIntentLedgerStatus = PendingTransactionIntentLedgerStatus.PermanentRejection; + _rejectionReasonForMostAccurateIntentLedgerStatus = $"Transaction has expired, as its expiry epoch of {pendingTransactionSummary.EndEpochExclusive} has been reached."; + } + + // The if statement is defence-in-depth to avoid replacing a Committed status with PermanentlyRejected. + // (Even though this shouldn't be possible because of the early return if pendingTransactionSummary.PayloadHash == _committedPayloadHash) + if (PendingTransactionPayloadLedgerStatus.PermanentlyRejected.ReplacementPriority() >= pendingTransactionSummary.PayloadStatus.ReplacementPriority()) + { + payloadStatus = GatewayModel.TransactionPayloadStatus.PermanentlyRejected; + legacyStatus = GatewayModel.TransactionStatus.Rejected; + latestRejectionReason = $"Transaction has expired, as its expiry epoch of {pendingTransactionSummary.EndEpochExclusive} has been reached."; + initialRejectionReason ??= latestRejectionReason; + } + } _knownPayloads.Add(new GatewayModel.TransactionStatusResponseKnownPayloadItem( payloadHash: pendingTransactionSummary.PayloadHash, status: legacyStatus, payloadStatus: payloadStatus, payloadStatusDescription: GetPayloadStatusDescription(payloadStatus), - errorMessage: pendingTransactionSummary.InitialRejectionReason, - latestErrorMessage: latestErrorMessage, + errorMessage: initialRejectionReason, + latestErrorMessage: latestRejectionReason == initialRejectionReason ? null : latestRejectionReason, handlingStatus: handlingStatus, handlingStatusReason: pendingTransactionSummary.HandlingStatusReason, submissionError: pendingTransactionSummary.LastSubmissionError @@ -443,15 +468,20 @@ internal void AddPendingTransaction(PendingTransactionSummary pendingTransaction internal GatewayModel.TransactionStatusResponse IntoResponse() { - var intentStatus = _committedIntentStatus ?? _mostAccuratePendingTransactionIntentLedgerStatus switch - { - PendingTransactionIntentLedgerStatus.Unknown => GatewayModel.TransactionIntentStatus.Unknown, - PendingTransactionIntentLedgerStatus.Committed => GatewayModel.TransactionIntentStatus.CommitPendingOutcomeUnknown, - PendingTransactionIntentLedgerStatus.CommitPending => GatewayModel.TransactionIntentStatus.CommitPendingOutcomeUnknown, - PendingTransactionIntentLedgerStatus.PermanentRejection => GatewayModel.TransactionIntentStatus.PermanentlyRejected, - PendingTransactionIntentLedgerStatus.PossibleToCommit => GatewayModel.TransactionIntentStatus.Pending, - PendingTransactionIntentLedgerStatus.LikelyButNotCertainRejection => GatewayModel.TransactionIntentStatus.LikelyButNotCertainRejection, - }; + var (intentStatus, errorMessage) = _committedIntentStatus != null + ? (_committedIntentStatus.Value, _committedErrorMessage) + : ( + _mostAccuratePendingTransactionIntentLedgerStatus switch + { + PendingTransactionIntentLedgerStatus.Unknown => GatewayModel.TransactionIntentStatus.Unknown, + PendingTransactionIntentLedgerStatus.Committed => GatewayModel.TransactionIntentStatus.CommitPendingOutcomeUnknown, + PendingTransactionIntentLedgerStatus.CommitPending => GatewayModel.TransactionIntentStatus.CommitPendingOutcomeUnknown, + PendingTransactionIntentLedgerStatus.PermanentRejection => GatewayModel.TransactionIntentStatus.PermanentlyRejected, + PendingTransactionIntentLedgerStatus.PossibleToCommit => GatewayModel.TransactionIntentStatus.Pending, + PendingTransactionIntentLedgerStatus.LikelyButNotCertainRejection => GatewayModel.TransactionIntentStatus.LikelyButNotCertainRejection, + }, + _rejectionReasonForMostAccurateIntentLedgerStatus + ); var legacyIntentStatus = intentStatus switch { @@ -467,11 +497,11 @@ internal GatewayModel.TransactionStatusResponse IntoResponse() return new GatewayModel.TransactionStatusResponse( ledgerState: _ledgerState, status: legacyIntentStatus, - intentStatus: intentStatus, + intentStatus, intentStatusDescription: GetIntentStatusDescription(intentStatus), knownPayloads: _knownPayloads, committedStateVersion: _committedStateVersion, - errorMessage: _errorMessageForMostAccurateIntentLedgerStatus + errorMessage ); } @@ -553,6 +583,7 @@ private async Task> LookupPendingTransact .Select(pt => new PendingTransactionSummary( pt.PayloadHash, + pt.EndEpochExclusive, pt.GatewayHandling.ResubmitFromTimestamp, pt.GatewayHandling.HandlingStatusReason, pt.LedgerDetails.PayloadLedgerStatus, From e5e6c4545811f09ede042e7800ed77fd8f602b99 Mon Sep 17 00:00:00 2001 From: David Edey Date: Wed, 27 Sep 2023 02:48:05 +0100 Subject: [PATCH 10/22] fix: Fix issue with transaction store size accounting when race condition encountered --- .../Services/FetchedTransactionStore.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/RadixDlt.NetworkGateway.DataAggregator/Services/FetchedTransactionStore.cs b/src/RadixDlt.NetworkGateway.DataAggregator/Services/FetchedTransactionStore.cs index 92a9965bc..2a7056248 100644 --- a/src/RadixDlt.NetworkGateway.DataAggregator/Services/FetchedTransactionStore.cs +++ b/src/RadixDlt.NetworkGateway.DataAggregator/Services/FetchedTransactionStore.cs @@ -84,7 +84,7 @@ public interface IFetchedTransactionStore void StoreNodeTransactions(string nodeName, List transactions, int responseSize); - bool ShouldFetchNewTransactions(string nodeName, long fromStateVersion); + bool ShouldFetchNewTransactions(string nodeName, long committedStateVersion); long GetFirstStateVersionToFetch(string nodeName, long lastCommittedStateVersion); } @@ -172,12 +172,12 @@ public void RemoveProcessedTransactions(long committedStateVersion) .ToList(); } - public bool ShouldFetchNewTransactions(string nodeName, long fromStateVersion) + public bool ShouldFetchNewTransactions(string nodeName, long committedStateVersion) { var transactionStore = GetTransactionsStoreForNode(nodeName); var storedTransactionsSize = transactionStore - .Where(x => x.Key >= fromStateVersion) + .Where(x => x.Key > committedStateVersion) .Aggregate(0, (sum, current) => sum + current.Value.EstimatedSize); var shouldFetchTransactions = storedTransactionsSize < Config.MaxEstimatedTransactionPipelineByteSizePerNode; From 7e43f2918017f79ec39a77fc0c3dd3b93602509d Mon Sep 17 00:00:00 2001 From: David Edey Date: Wed, 27 Sep 2023 02:48:41 +0100 Subject: [PATCH 11/22] fix: Fix performance issues when querying ledger state --- .editorconfig | 1 + .../DbQueryExtensions.cs | 49 ++++++++++++++++--- .../Services/LedgerStateQuerier.cs | 33 ++++++++----- .../Services/TopOfLedgerProvider.cs | 24 +++++++-- 4 files changed, 83 insertions(+), 24 deletions(-) diff --git a/.editorconfig b/.editorconfig index 1f72bfe9a..fda65ad3f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -276,6 +276,7 @@ dotnet_diagnostic.SA1024.severity = none # SA1024ColonsMustBeSpacedCorrectly - b dotnet_diagnostic.SA1600.severity = none # SA1600ElementsMustBeDocumented - Duplicates CS1591 - let's just use CS1591 instead dotnet_diagnostic.SA1615.severity = none # SA1615ElementReturnValueMustBeDocumented dotnet_diagnostic.SA1611.severity = none # SA1611ElementParametersMustBeDocumented +dotnet_diagnostic.SA1618.severity = none # SA1618GenericTypeParametersMustBeDocumented dotnet_diagnostic.SA1602.severity = none # SA1602EnumerationItemsMustBeDocumented dotnet_diagnostic.SA1601.severity = none # SA1601PartialElementsMustBeDocumented diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/DbQueryExtensions.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/DbQueryExtensions.cs index 54256ae40..e237bcd63 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/DbQueryExtensions.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/DbQueryExtensions.cs @@ -70,7 +70,12 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration; internal static class DbQueryExtensions { - public static IQueryable GetTopLedgerTransaction(this TDbContext dbContext) + /// + /// The long/horrible/verbose name is on purpose, to draw your attention and prevent people foot-gunning themselves. + /// A LedgerTransaction row contains large blobs, so you must SELECT the fields you need after using this, and not pull down the whole + /// ledger transaction row, to avoid possible performance issues. + /// + public static IQueryable GetTopLedgerTransaction__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(this TDbContext dbContext) where TDbContext : CommonDbContext { return dbContext @@ -79,7 +84,12 @@ public static IQueryable GetTopLedgerTransaction( .Take(1); } - public static IQueryable GetLatestLedgerTransactionBeforeStateVersion(this TDbContext dbContext, long beforeStateVersion) + /// + /// The long/horrible/verbose name is on purpose, to draw your attention and prevent people foot-gunning themselves. + /// A LedgerTransaction row contains large blobs, so you must SELECT the fields you need after using this, and not pull down the whole + /// ledger transaction row, to avoid possible performance issues. + /// + public static IQueryable GetLatestLedgerTransactionBeforeStateVersion__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(this TDbContext dbContext, long beforeStateVersion) where TDbContext : CommonDbContext { return dbContext @@ -89,7 +99,12 @@ public static IQueryable GetLatestLedgerTransactionBeforeStat .Take(1); } - public static IQueryable GetFirstLedgerTransactionAfterStateVersion(this TDbContext dbContext, long afterStateVersion) + /// + /// The long/horrible/verbose name is on purpose, to draw your attention and prevent people foot-gunning themselves. + /// A LedgerTransaction row contains large blobs, so you must SELECT the fields you need after using this, and not pull down the whole + /// ledger transaction row, to avoid possible performance issues. + /// + public static IQueryable GetFirstLedgerTransactionAfterStateVersion__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(this TDbContext dbContext, long afterStateVersion) where TDbContext : CommonDbContext { return dbContext @@ -99,7 +114,12 @@ public static IQueryable GetFirstLedgerTransactionAfterStateV .Take(1); } - public static IQueryable GetLatestLedgerTransactionBeforeTimestamp(this TDbContext dbContext, DateTime timestamp) + /// + /// The long/horrible/verbose name is on purpose, to draw your attention and prevent people foot-gunning themselves. + /// A LedgerTransaction row contains large blobs, so you must SELECT the fields you need after using this, and not pull down the whole + /// ledger transaction row, to avoid possible performance issues. + /// + public static IQueryable GetLatestLedgerTransactionBeforeTimestamp__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(this TDbContext dbContext, DateTime timestamp) where TDbContext : CommonDbContext { return dbContext @@ -110,7 +130,12 @@ public static IQueryable GetLatestLedgerTransactionBeforeTime .Take(1); } - public static IQueryable GetFirstLedgerTransactionAfterTimestamp(this TDbContext dbContext, DateTime timestamp) + /// + /// The long/horrible/verbose name is on purpose, to draw your attention and prevent people foot-gunning themselves. + /// A LedgerTransaction row contains large blobs, so you must SELECT the fields you need after using this, and not pull down the whole + /// ledger transaction row, to avoid possible performance issues. + /// + public static IQueryable GetFirstLedgerTransactionAfterTimestamp__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(this TDbContext dbContext, DateTime timestamp) where TDbContext : CommonDbContext { return dbContext @@ -121,7 +146,12 @@ public static IQueryable GetFirstLedgerTransactionAfterTimest .Take(1); } - public static IQueryable GetLatestLedgerTransactionAtEpochRound(this TDbContext dbContext, long epoch, long round) + /// + /// The long/horrible/verbose name is on purpose, to draw your attention and prevent people foot-gunning themselves. + /// A LedgerTransaction row contains large blobs, so you must SELECT the fields you need after using this, and not pull down the whole + /// ledger transaction row, to avoid possible performance issues. + /// + public static IQueryable GetLatestLedgerTransactionAtEpochRound__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(this TDbContext dbContext, long epoch, long round) where TDbContext : CommonDbContext { return dbContext @@ -131,7 +161,12 @@ public static IQueryable GetLatestLedgerTransactionAtEpochRou .Take(1); } - public static IQueryable GetFirstLedgerTransactionAtEpochRound(this TDbContext dbContext, long epoch, long round) + /// + /// The long/horrible/verbose name is on purpose, to draw your attention and prevent people foot-gunning themselves. + /// A LedgerTransaction row contains large blobs, so you must SELECT the fields you need after using this, and not pull down the whole + /// ledger transaction row, to avoid possible performance issues. + /// + public static IQueryable GetFirstLedgerTransactionAtEpochRound__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(this TDbContext dbContext, long epoch, long round) where TDbContext : CommonDbContext { return dbContext diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/LedgerStateQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/LedgerStateQuerier.cs index 95d145285..a9a956a22 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/LedgerStateQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/LedgerStateQuerier.cs @@ -65,7 +65,6 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Npgsql; using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Extensions; using RadixDlt.NetworkGateway.GatewayApi.Configuration; @@ -124,7 +123,7 @@ public LedgerStateQuerier( public async Task GetGatewayStatus(CancellationToken token) { - var topLedgerTransaction = await GetTopLedgerTransaction(token); + var topLedgerTransaction = await GetTopLedgerTransactionSummary(token); return new GatewayModel.GatewayStatusResponse( new GatewayModel.LedgerState( @@ -234,7 +233,7 @@ public LedgerStateQuerier( public async Task GetTopOfLedgerStateVersion(CancellationToken token = default) { - var topLedgerTransaction = await GetTopLedgerTransaction(token); + var topLedgerTransaction = await GetTopLedgerTransactionSummary(token); return topLedgerTransaction.StateVersion; } @@ -263,10 +262,18 @@ private static string GetOpenApiSchemaVersion() return match.Groups[1].Value; } - private async Task GetTopLedgerTransaction(CancellationToken token) + private record LedgerTransactionSummary(long StateVersion, DateTime RoundTimestamp, long Epoch, long RoundInEpoch); + + private async Task GetTopLedgerTransactionSummary(CancellationToken token) { var topLedgerTransaction = await _dbContext - .GetTopLedgerTransaction() + .GetTopLedgerTransaction__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig() + .Select(lt => new LedgerTransactionSummary( + lt.StateVersion, + lt.RoundTimestamp, + lt.Epoch, + lt.RoundInEpoch + )) .FirstOrDefaultAsync(token); if (topLedgerTransaction == null) @@ -305,7 +312,7 @@ private async Task GetLedgerState(GatewayModel.LedgerStateSel private async Task GetTopOfLedgerStateReport(CancellationToken token) { - var ledgerState = await GetLedgerStateFromQuery(_dbContext.GetTopLedgerTransaction(), true, token); + var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetTopLedgerTransaction__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(), true, token); if (ledgerState == null) { @@ -317,7 +324,7 @@ private async Task GetTopOfLedgerStateReport(CancellationToke private async Task GetLedgerStateBeforeStateVersion(long stateVersion, CancellationToken token) { - var ledgerState = await GetLedgerStateFromQuery(_dbContext.GetLatestLedgerTransactionBeforeStateVersion(stateVersion), false, token); + var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetLatestLedgerTransactionBeforeStateVersion__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(stateVersion), false, token); if (ledgerState == null) { @@ -329,7 +336,7 @@ private async Task GetLedgerStateBeforeStateVersion(long stat private async Task GetLedgerStateAfterStateVersion(long stateVersion, CancellationToken token) { - var ledgerState = await GetLedgerStateFromQuery(_dbContext.GetFirstLedgerTransactionAfterStateVersion(stateVersion), false, token); + var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetFirstLedgerTransactionAfterStateVersion__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(stateVersion), false, token); if (ledgerState == null) { @@ -341,7 +348,7 @@ private async Task GetLedgerStateAfterStateVersion(long state private async Task GetLedgerStateBeforeTimestamp(DateTime timestamp, CancellationToken token) { - var ledgerState = await GetLedgerStateFromQuery(_dbContext.GetLatestLedgerTransactionBeforeTimestamp(timestamp), false, token); + var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetLatestLedgerTransactionBeforeTimestamp__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(timestamp), false, token); if (ledgerState == null) { @@ -353,7 +360,7 @@ private async Task GetLedgerStateBeforeTimestamp(DateTime tim private async Task GetLedgerStateAfterTimestamp(DateTime timestamp, CancellationToken token) { - var ledgerState = await GetLedgerStateFromQuery(_dbContext.GetFirstLedgerTransactionAfterTimestamp(timestamp), false, token); + var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetFirstLedgerTransactionAfterTimestamp__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(timestamp), false, token); if (ledgerState == null) { @@ -365,7 +372,7 @@ private async Task GetLedgerStateAfterTimestamp(DateTime time private async Task GetLedgerStateAtEpochAndRound(long epoch, long round, CancellationToken token) { - var ledgerState = await GetLedgerStateFromQuery(_dbContext.GetLatestLedgerTransactionAtEpochRound(epoch, round), false, token); + var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetLatestLedgerTransactionAtEpochRound__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(epoch, round), false, token); if (ledgerState == null) { @@ -377,7 +384,7 @@ private async Task GetLedgerStateAtEpochAndRound(long epoch, private async Task GetLedgerStateAfterEpochAndRound(long epoch, long round, CancellationToken token) { - var ledgerState = await GetLedgerStateFromQuery(_dbContext.GetFirstLedgerTransactionAtEpochRound(epoch, round), false, token); + var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetFirstLedgerTransactionAtEpochRound__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(epoch, round), false, token); if (ledgerState == null) { @@ -387,7 +394,7 @@ private async Task GetLedgerStateAfterEpochAndRound(long epoc return ledgerState; } - private async Task GetLedgerStateFromQuery(IQueryable query, bool resolvesTopOfLedger, CancellationToken token) + private async Task SelectLedgerStateFromQuery(IQueryable query, bool resolvesTopOfLedger, CancellationToken token) { var lt = await query .Select(lt => new diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TopOfLedgerProvider.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TopOfLedgerProvider.cs index 0214a22ca..9e86750fc 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TopOfLedgerProvider.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TopOfLedgerProvider.cs @@ -65,6 +65,7 @@ using Microsoft.EntityFrameworkCore; using RadixDlt.NetworkGateway.DataAggregator.Services; using System; +using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -87,16 +88,31 @@ public async Task GetLastCommittedStateVersion(CancellationToken token) { var dbContext = await _dbContextFactory.CreateDbContextAsync(token); - var lastTransaction = await dbContext.GetTopLedgerTransaction().FirstOrDefaultAsync(token); - - return lastTransaction?.StateVersion ?? 0; + return await dbContext.GetTopLedgerTransaction__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig() + .AsNoTracking() + .Select(t => t.StateVersion) + .FirstOrDefaultAsync(token); // Defaults to 0, which is perfect } public async Task GetTopOfLedger(CancellationToken token) { var dbContext = await _dbContextFactory.CreateDbContextAsync(token); - var lastTransaction = await dbContext.GetTopLedgerTransaction().FirstOrDefaultAsync(token); + var lastTransaction = await dbContext.GetTopLedgerTransaction__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig() + .AsNoTracking() + .Select(lt => new + { + lt.StateVersion, + lt.LedgerHashes, + lt.RoundTimestamp, + lt.NormalizedRoundTimestamp, + lt.CreatedTimestamp, + lt.Epoch, + lt.RoundInEpoch, + lt.IndexInEpoch, + lt.IndexInRound, + }) + .FirstOrDefaultAsync(token); return lastTransaction == null ? PreGenesisTransactionSummary() From 9eaa31be686ab97a6922f637b6bd054834355412 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Wed, 27 Sep 2023 08:46:46 +0200 Subject: [PATCH 12/22] add queryname to singleOrDefault firstOrDefault and anyAsync. --- .../DbQueryExtensions.cs | 21 ++++++++++++------- .../Services/CapturedConfigProvider.cs | 2 +- .../CommittedStateIdentifiersReader.cs | 1 + .../Services/EntityStateQuerier.cs | 1 + .../Services/NetworkConfigurationProvider.cs | 2 +- .../Services/SubmissionTrackingService.cs | 1 + .../Services/TransactionQuerier.cs | 2 ++ 7 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/DbQueryExtensions.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/DbQueryExtensions.cs index 54256ae40..6c6344f34 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/DbQueryExtensions.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/DbQueryExtensions.cs @@ -76,7 +76,8 @@ public static IQueryable GetTopLedgerTransaction( return dbContext .LedgerTransactions .OrderByDescending(lt => lt.StateVersion) - .Take(1); + .Take(1) + .WithQueryName(); } public static IQueryable GetLatestLedgerTransactionBeforeStateVersion(this TDbContext dbContext, long beforeStateVersion) @@ -86,7 +87,8 @@ public static IQueryable GetLatestLedgerTransactionBeforeStat .LedgerTransactions .Where(lt => lt.StateVersion <= beforeStateVersion) .OrderByDescending(lt => lt.StateVersion) - .Take(1); + .Take(1) + .WithQueryName(); } public static IQueryable GetFirstLedgerTransactionAfterStateVersion(this TDbContext dbContext, long afterStateVersion) @@ -96,7 +98,8 @@ public static IQueryable GetFirstLedgerTransactionAfterStateV .LedgerTransactions .Where(lt => lt.StateVersion >= afterStateVersion) .OrderBy(lt => lt.StateVersion) - .Take(1); + .Take(1) + .WithQueryName(); } public static IQueryable GetLatestLedgerTransactionBeforeTimestamp(this TDbContext dbContext, DateTime timestamp) @@ -107,7 +110,8 @@ public static IQueryable GetLatestLedgerTransactionBeforeTime .Where(lt => lt.RoundTimestamp <= timestamp) .OrderByDescending(lt => lt.RoundTimestamp) .ThenByDescending(lt => lt.StateVersion) - .Take(1); + .Take(1) + .WithQueryName(); } public static IQueryable GetFirstLedgerTransactionAfterTimestamp(this TDbContext dbContext, DateTime timestamp) @@ -118,7 +122,8 @@ public static IQueryable GetFirstLedgerTransactionAfterTimest .Where(lt => lt.RoundTimestamp >= timestamp) .OrderBy(lt => lt.RoundTimestamp) .ThenBy(lt => lt.StateVersion) - .Take(1); + .Take(1) + .WithQueryName(); } public static IQueryable GetLatestLedgerTransactionAtEpochRound(this TDbContext dbContext, long epoch, long round) @@ -128,7 +133,8 @@ public static IQueryable GetLatestLedgerTransactionAtEpochRou .LedgerTransactions .Where(lt => lt.Epoch == epoch && lt.RoundInEpoch >= round && lt.IndexInRound == 0) .OrderByDescending(lt => lt.StateVersion) - .Take(1); + .Take(1) + .WithQueryName(); } public static IQueryable GetFirstLedgerTransactionAtEpochRound(this TDbContext dbContext, long epoch, long round) @@ -138,6 +144,7 @@ public static IQueryable GetFirstLedgerTransactionAtEpochRoun .LedgerTransactions .Where(lt => lt.Epoch >= epoch && lt.RoundInEpoch >= round && lt.IndexInRound == 0) .OrderBy(lt => lt.StateVersion) - .Take(1); + .Take(1) + .WithQueryName(); } } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/CapturedConfigProvider.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/CapturedConfigProvider.cs index a0a23f8d6..c383387c3 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/CapturedConfigProvider.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/CapturedConfigProvider.cs @@ -81,7 +81,7 @@ public CapturedConfigProvider(ReadOnlyDbContext dbContext) public async Task CaptureConfiguration() { - var networkConfiguration = await _dbContext.NetworkConfiguration.AsNoTracking().SingleOrDefaultAsync(); + var networkConfiguration = await _dbContext.NetworkConfiguration.AsNoTracking().WithQueryName().SingleOrDefaultAsync(); if (networkConfiguration == null) { diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/CommittedStateIdentifiersReader.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/CommittedStateIdentifiersReader.cs index 06523703c..8f785f437 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/CommittedStateIdentifiersReader.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/CommittedStateIdentifiersReader.cs @@ -84,6 +84,7 @@ public CommittedStateIdentifiersReader(IDbContextFactory dbC var dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); var transaction = await dbContext.LedgerTransactions + .WithQueryName() .FirstOrDefaultAsync(x => x.StateVersion == stateVersion, cancellationToken); if (transaction == null) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs index cd460717e..32864b8c3 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs @@ -1221,6 +1221,7 @@ private async Task GetEntity(EntityAddress address, GatewayMod var entity = await _dbContext .Entities .Where(e => e.FromStateVersion <= ledgerState.StateVersion) + .WithQueryName() .FirstOrDefaultAsync(e => e.Address == address, token); if (entity == null) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/NetworkConfigurationProvider.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/NetworkConfigurationProvider.cs index bd9f62085..7d321b3e2 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/NetworkConfigurationProvider.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/NetworkConfigurationProvider.cs @@ -141,7 +141,7 @@ public async Task SaveLedgerNetworkConfigurationToDatabaseOnInitIfNotExist { await using var dbContext = await _dbContextFactory.CreateDbContextAsync(token); - if (await dbContext.NetworkConfiguration.AsNoTracking().AnyAsync(token)) + if (await dbContext.NetworkConfiguration.AsNoTracking().WithQueryName().AnyAsync(token)) { return false; } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/SubmissionTrackingService.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/SubmissionTrackingService.cs index 7c41c6635..93ef741bd 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/SubmissionTrackingService.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/SubmissionTrackingService.cs @@ -169,6 +169,7 @@ private async Task TrackSubmission( var existingPendingTransaction = await _dbContext .PendingTransactions .Where(t => t.PayloadHash == payloadHash) + .WithQueryName() .SingleOrDefaultAsync(token); if (existingPendingTransaction != null) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs index 98b0c92df..837d4551e 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs @@ -324,6 +324,7 @@ public async Task GetTransactionStream(TransactionS .OfType() .Where(ult => ult.StateVersion <= ledgerState.StateVersion && ult.IntentHash == intentHash) .Select(ult => ult.StateVersion) + .WithQueryName() .FirstOrDefaultAsync(token); if (stateVersion == default) @@ -530,6 +531,7 @@ private string GetIntentStatusDescription(GatewayModel.TransactionIntentStatus i ult.PayloadHash, ult.EngineReceipt.Status, ult.EngineReceipt.ErrorMessage)) + .WithQueryName() .FirstOrDefaultAsync(token); var aggregator = new PendingTransactionResponseAggregator(ledgerState, maybeCommittedTransactionSummary); From 9cfdec593e3619da1ccb5ef06820e1982c71622f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20=C5=81abu=C5=9B?= Date: Wed, 27 Sep 2023 10:58:00 +0200 Subject: [PATCH 13/22] Code renames / docs added --- .../CommonDbContext.cs | 7 +++ .../DbQueryExtensions.cs | 56 ++++++++++++------- .../Services/LedgerStateQuerier.cs | 16 +++--- .../Services/TopOfLedgerProvider.cs | 4 +- 4 files changed, 52 insertions(+), 31 deletions(-) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs index cf8be2d49..c809bed4d 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs @@ -83,6 +83,13 @@ internal abstract class CommonDbContext : DbContext public DbSet NetworkConfiguration => Set(); + /// + /// Gets LedgerTransactions. + /// + /// + /// A LedgerTransaction row contains large blobs, so you must SELECT the fields you need after using this, and not pull down the whole + /// ledger transaction row, to avoid possible performance issues. + /// public DbSet LedgerTransactions => Set(); public DbSet LedgerTransactionMarkers => Set(); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/DbQueryExtensions.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/DbQueryExtensions.cs index e237bcd63..c0cdfc98b 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/DbQueryExtensions.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/DbQueryExtensions.cs @@ -71,11 +71,13 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration; internal static class DbQueryExtensions { /// - /// The long/horrible/verbose name is on purpose, to draw your attention and prevent people foot-gunning themselves. + /// Returns most recently committed ledger transaction. + /// + /// /// A LedgerTransaction row contains large blobs, so you must SELECT the fields you need after using this, and not pull down the whole /// ledger transaction row, to avoid possible performance issues. - /// - public static IQueryable GetTopLedgerTransaction__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(this TDbContext dbContext) + /// + public static IQueryable GetTopLedgerTransaction(this TDbContext dbContext) where TDbContext : CommonDbContext { return dbContext @@ -85,11 +87,13 @@ public static IQueryable GetTopLedgerTransaction__EnsureYouSe } /// - /// The long/horrible/verbose name is on purpose, to draw your attention and prevent people foot-gunning themselves. + /// Returns most recently committed ledger transaction at or before given state version. + /// + /// /// A LedgerTransaction row contains large blobs, so you must SELECT the fields you need after using this, and not pull down the whole /// ledger transaction row, to avoid possible performance issues. - /// - public static IQueryable GetLatestLedgerTransactionBeforeStateVersion__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(this TDbContext dbContext, long beforeStateVersion) + /// + public static IQueryable GetLatestLedgerTransactionBeforeStateVersion(this TDbContext dbContext, long beforeStateVersion) where TDbContext : CommonDbContext { return dbContext @@ -100,11 +104,13 @@ public static IQueryable GetLatestLedgerTransactionBeforeStat } /// - /// The long/horrible/verbose name is on purpose, to draw your attention and prevent people foot-gunning themselves. + /// Returns the first committed ledger transaction at or after given state version. + /// + /// /// A LedgerTransaction row contains large blobs, so you must SELECT the fields you need after using this, and not pull down the whole /// ledger transaction row, to avoid possible performance issues. - /// - public static IQueryable GetFirstLedgerTransactionAfterStateVersion__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(this TDbContext dbContext, long afterStateVersion) + /// + public static IQueryable GetFirstLedgerTransactionAfterStateVersion(this TDbContext dbContext, long afterStateVersion) where TDbContext : CommonDbContext { return dbContext @@ -115,11 +121,13 @@ public static IQueryable GetFirstLedgerTransactionAfterStateV } /// - /// The long/horrible/verbose name is on purpose, to draw your attention and prevent people foot-gunning themselves. + /// Returns most recently committed ledger transaction at or before given timestamp. + /// + /// /// A LedgerTransaction row contains large blobs, so you must SELECT the fields you need after using this, and not pull down the whole /// ledger transaction row, to avoid possible performance issues. - /// - public static IQueryable GetLatestLedgerTransactionBeforeTimestamp__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(this TDbContext dbContext, DateTime timestamp) + /// + public static IQueryable GetLatestLedgerTransactionBeforeTimestamp(this TDbContext dbContext, DateTime timestamp) where TDbContext : CommonDbContext { return dbContext @@ -131,11 +139,13 @@ public static IQueryable GetLatestLedgerTransactionBeforeTime } /// - /// The long/horrible/verbose name is on purpose, to draw your attention and prevent people foot-gunning themselves. + /// Returns the first committed ledger transaction at or after given timestamp. + /// + /// /// A LedgerTransaction row contains large blobs, so you must SELECT the fields you need after using this, and not pull down the whole /// ledger transaction row, to avoid possible performance issues. - /// - public static IQueryable GetFirstLedgerTransactionAfterTimestamp__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(this TDbContext dbContext, DateTime timestamp) + /// + public static IQueryable GetFirstLedgerTransactionAfterTimestamp(this TDbContext dbContext, DateTime timestamp) where TDbContext : CommonDbContext { return dbContext @@ -147,11 +157,13 @@ public static IQueryable GetFirstLedgerTransactionAfterTimest } /// - /// The long/horrible/verbose name is on purpose, to draw your attention and prevent people foot-gunning themselves. + /// Returns most recently committed ledger transaction at or before given epoch and round. + /// + /// /// A LedgerTransaction row contains large blobs, so you must SELECT the fields you need after using this, and not pull down the whole /// ledger transaction row, to avoid possible performance issues. - /// - public static IQueryable GetLatestLedgerTransactionAtEpochRound__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(this TDbContext dbContext, long epoch, long round) + /// + public static IQueryable GetLatestLedgerTransactionAtEpochRound(this TDbContext dbContext, long epoch, long round) where TDbContext : CommonDbContext { return dbContext @@ -162,11 +174,13 @@ public static IQueryable GetLatestLedgerTransactionAtEpochRou } /// - /// The long/horrible/verbose name is on purpose, to draw your attention and prevent people foot-gunning themselves. + /// Returns the first committed ledger transaction at or after given epoch and round. + /// + /// /// A LedgerTransaction row contains large blobs, so you must SELECT the fields you need after using this, and not pull down the whole /// ledger transaction row, to avoid possible performance issues. - /// - public static IQueryable GetFirstLedgerTransactionAtEpochRound__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(this TDbContext dbContext, long epoch, long round) + /// + public static IQueryable GetFirstLedgerTransactionAtEpochRound(this TDbContext dbContext, long epoch, long round) where TDbContext : CommonDbContext { return dbContext diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/LedgerStateQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/LedgerStateQuerier.cs index a9a956a22..02e737d23 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/LedgerStateQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/LedgerStateQuerier.cs @@ -267,7 +267,7 @@ private record LedgerTransactionSummary(long StateVersion, DateTime RoundTimesta private async Task GetTopLedgerTransactionSummary(CancellationToken token) { var topLedgerTransaction = await _dbContext - .GetTopLedgerTransaction__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig() + .GetTopLedgerTransaction() .Select(lt => new LedgerTransactionSummary( lt.StateVersion, lt.RoundTimestamp, @@ -312,7 +312,7 @@ private async Task GetLedgerState(GatewayModel.LedgerStateSel private async Task GetTopOfLedgerStateReport(CancellationToken token) { - var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetTopLedgerTransaction__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(), true, token); + var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetTopLedgerTransaction(), true, token); if (ledgerState == null) { @@ -324,7 +324,7 @@ private async Task GetTopOfLedgerStateReport(CancellationToke private async Task GetLedgerStateBeforeStateVersion(long stateVersion, CancellationToken token) { - var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetLatestLedgerTransactionBeforeStateVersion__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(stateVersion), false, token); + var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetLatestLedgerTransactionBeforeStateVersion(stateVersion), false, token); if (ledgerState == null) { @@ -336,7 +336,7 @@ private async Task GetLedgerStateBeforeStateVersion(long stat private async Task GetLedgerStateAfterStateVersion(long stateVersion, CancellationToken token) { - var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetFirstLedgerTransactionAfterStateVersion__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(stateVersion), false, token); + var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetFirstLedgerTransactionAfterStateVersion(stateVersion), false, token); if (ledgerState == null) { @@ -348,7 +348,7 @@ private async Task GetLedgerStateAfterStateVersion(long state private async Task GetLedgerStateBeforeTimestamp(DateTime timestamp, CancellationToken token) { - var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetLatestLedgerTransactionBeforeTimestamp__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(timestamp), false, token); + var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetLatestLedgerTransactionBeforeTimestamp(timestamp), false, token); if (ledgerState == null) { @@ -360,7 +360,7 @@ private async Task GetLedgerStateBeforeTimestamp(DateTime tim private async Task GetLedgerStateAfterTimestamp(DateTime timestamp, CancellationToken token) { - var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetFirstLedgerTransactionAfterTimestamp__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(timestamp), false, token); + var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetFirstLedgerTransactionAfterTimestamp(timestamp), false, token); if (ledgerState == null) { @@ -372,7 +372,7 @@ private async Task GetLedgerStateAfterTimestamp(DateTime time private async Task GetLedgerStateAtEpochAndRound(long epoch, long round, CancellationToken token) { - var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetLatestLedgerTransactionAtEpochRound__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(epoch, round), false, token); + var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetLatestLedgerTransactionAtEpochRound(epoch, round), false, token); if (ledgerState == null) { @@ -384,7 +384,7 @@ private async Task GetLedgerStateAtEpochAndRound(long epoch, private async Task GetLedgerStateAfterEpochAndRound(long epoch, long round, CancellationToken token) { - var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetFirstLedgerTransactionAtEpochRound__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig(epoch, round), false, token); + var ledgerState = await SelectLedgerStateFromQuery(_dbContext.GetFirstLedgerTransactionAtEpochRound(epoch, round), false, token); if (ledgerState == null) { diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TopOfLedgerProvider.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TopOfLedgerProvider.cs index 9e86750fc..41bfa49da 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TopOfLedgerProvider.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TopOfLedgerProvider.cs @@ -88,7 +88,7 @@ public async Task GetLastCommittedStateVersion(CancellationToken token) { var dbContext = await _dbContextFactory.CreateDbContextAsync(token); - return await dbContext.GetTopLedgerTransaction__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig() + return await dbContext.GetTopLedgerTransaction() .AsNoTracking() .Select(t => t.StateVersion) .FirstOrDefaultAsync(token); // Defaults to 0, which is perfect @@ -98,7 +98,7 @@ public async Task GetTopOfLedger(CancellationToken token) { var dbContext = await _dbContextFactory.CreateDbContextAsync(token); - var lastTransaction = await dbContext.GetTopLedgerTransaction__EnsureYouSelectRequiredFieldsAsRowsCanBeVeryBig() + var lastTransaction = await dbContext.GetTopLedgerTransaction() .AsNoTracking() .Select(lt => new { From 724c483ca299ba744ede404a9e5adce2455426c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Wed, 27 Sep 2023 10:59:17 +0200 Subject: [PATCH 14/22] code refactoring. --- ...gOptions.cs => SlowQueryLoggingOptions.cs} | 6 +-- .../ServiceCollectionExtensions.cs | 2 +- ...gOptions.cs => SlowQueryLoggingOptions.cs} | 6 +-- .../ServiceCollectionExtensions.cs | 2 +- .../DbQueryExtensions.cs | 14 ++--- .../Interceptors/MetricsInterceptor.cs | 52 +++++++------------ .../PostgresLedgerExtenderService.cs | 2 +- .../LedgerExtension/ReadHelper.cs | 26 +++++----- .../Metrics/SqlQueryMetricsHelper.cs | 26 ++++++---- .../QueryableExtensions.cs | 5 +- .../Services/BlueprintProvider.cs | 2 +- .../Services/CapturedConfigProvider.cs | 2 +- .../CommittedStateIdentifiersReader.cs | 2 +- .../Services/DapperWrapper.cs | 26 ++++------ .../Services/EntityStateQuerier.cs | 30 +++++------ .../Services/NetworkConfigurationProvider.cs | 2 +- .../PendingTransactionPrunerService.cs | 2 +- .../PendingTransactionResubmissionService.cs | 2 +- .../Services/RoleAssignmentQuerier.cs | 6 +-- .../Services/SubmissionTrackingService.cs | 2 +- .../Services/TransactionQuerier.cs | 16 +++--- .../Services/ValidatorQuerier.cs | 2 +- .../DataAggregatorMetricsObserver.cs | 4 +- .../GatewayApiMetricObserver.cs | 2 +- 24 files changed, 116 insertions(+), 125 deletions(-) rename src/RadixDlt.NetworkGateway.DataAggregator/Configuration/{SlowQueriesLoggingOptions.cs => SlowQueryLoggingOptions.cs} (63%) rename src/RadixDlt.NetworkGateway.GatewayApi/Configuration/{SlowQueriesLoggingOptions.cs => SlowQueryLoggingOptions.cs} (63%) diff --git a/src/RadixDlt.NetworkGateway.DataAggregator/Configuration/SlowQueriesLoggingOptions.cs b/src/RadixDlt.NetworkGateway.DataAggregator/Configuration/SlowQueryLoggingOptions.cs similarity index 63% rename from src/RadixDlt.NetworkGateway.DataAggregator/Configuration/SlowQueriesLoggingOptions.cs rename to src/RadixDlt.NetworkGateway.DataAggregator/Configuration/SlowQueryLoggingOptions.cs index 7b386e552..ec6be035b 100644 --- a/src/RadixDlt.NetworkGateway.DataAggregator/Configuration/SlowQueriesLoggingOptions.cs +++ b/src/RadixDlt.NetworkGateway.DataAggregator/Configuration/SlowQueryLoggingOptions.cs @@ -4,14 +4,14 @@ namespace RadixDlt.NetworkGateway.DataAggregator.Configuration; -public sealed class SlowQueriesLoggingOptions +public sealed class SlowQueryLoggingOptions { public TimeSpan SlowQueriesThreshold { get; set; } = TimeSpan.FromMilliseconds(250); } -internal class SlowQueriesLoggingOptionsValidator : AbstractOptionsValidator +internal class SlowQueryLoggingOptionsValidator : AbstractOptionsValidator { - public SlowQueriesLoggingOptionsValidator() + public SlowQueryLoggingOptionsValidator() { RuleFor(x => x.SlowQueriesThreshold).GreaterThan(TimeSpan.Zero); } diff --git a/src/RadixDlt.NetworkGateway.DataAggregator/ServiceCollectionExtensions.cs b/src/RadixDlt.NetworkGateway.DataAggregator/ServiceCollectionExtensions.cs index ecd1402b8..5734aefd3 100644 --- a/src/RadixDlt.NetworkGateway.DataAggregator/ServiceCollectionExtensions.cs +++ b/src/RadixDlt.NetworkGateway.DataAggregator/ServiceCollectionExtensions.cs @@ -99,7 +99,7 @@ public static DataAggregatorBuilder AddNetworkGatewayDataAggregatorCore(this ISe services .AddValidatableOptionsAtSection("DataAggregator:Network") - .AddValidatableOptionsAtSection("DataAggregator:SlowQueriesLogging") + .AddValidatableOptionsAtSection("DataAggregator:SlowQueryLogging") .AddValidatableOptionsAtSection("DataAggregator:Monitoring") .AddValidatableOptionsAtSection("DataAggregator:Mempool") .AddValidatableOptionsAtSection("DataAggregator:NodeWorkers") diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/SlowQueriesLoggingOptions.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/SlowQueryLoggingOptions.cs similarity index 63% rename from src/RadixDlt.NetworkGateway.GatewayApi/Configuration/SlowQueriesLoggingOptions.cs rename to src/RadixDlt.NetworkGateway.GatewayApi/Configuration/SlowQueryLoggingOptions.cs index 609a062fb..d50e7dde6 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/SlowQueriesLoggingOptions.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/SlowQueryLoggingOptions.cs @@ -4,14 +4,14 @@ namespace RadixDlt.NetworkGateway.GatewayApi.Configuration; -public sealed class SlowQueriesLoggingOptions +public sealed class SlowQueryLoggingOptions { public TimeSpan SlowQueriesThreshold { get; set; } = TimeSpan.FromMilliseconds(250); } -internal class SlowQueriesLoggingOptionsValidator : AbstractOptionsValidator +internal class SlowQueryLoggingValidator : AbstractOptionsValidator { - public SlowQueriesLoggingOptionsValidator() + public SlowQueryLoggingValidator() { RuleFor(x => x.SlowQueriesThreshold).GreaterThan(TimeSpan.Zero); } diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/ServiceCollectionExtensions.cs b/src/RadixDlt.NetworkGateway.GatewayApi/ServiceCollectionExtensions.cs index b5a802d21..61d5d19ea 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/ServiceCollectionExtensions.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/ServiceCollectionExtensions.cs @@ -99,7 +99,7 @@ public static GatewayApiBuilder AddNetworkGatewayApiCore(this IServiceCollection services .AddValidatableOptionsAtSection("GatewayApi:Endpoint") - .AddValidatableOptionsAtSection("GatewayApi:SlowQueriesLogging") + .AddValidatableOptionsAtSection("GatewayApi:SlowQueryLogging") .AddValidatableOptionsAtSection("GatewayApi:CoreApiIntegration") .AddValidatableOptionsAtSection("GatewayApi:Network") .AddValidatableOptionsAtSection("GatewayApi:AcceptableLedgerLag"); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/DbQueryExtensions.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/DbQueryExtensions.cs index 6c6344f34..edaa689e4 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/DbQueryExtensions.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/DbQueryExtensions.cs @@ -77,7 +77,7 @@ public static IQueryable GetTopLedgerTransaction( .LedgerTransactions .OrderByDescending(lt => lt.StateVersion) .Take(1) - .WithQueryName(); + .AnnotateMetricName(); } public static IQueryable GetLatestLedgerTransactionBeforeStateVersion(this TDbContext dbContext, long beforeStateVersion) @@ -88,7 +88,7 @@ public static IQueryable GetLatestLedgerTransactionBeforeStat .Where(lt => lt.StateVersion <= beforeStateVersion) .OrderByDescending(lt => lt.StateVersion) .Take(1) - .WithQueryName(); + .AnnotateMetricName(); } public static IQueryable GetFirstLedgerTransactionAfterStateVersion(this TDbContext dbContext, long afterStateVersion) @@ -99,7 +99,7 @@ public static IQueryable GetFirstLedgerTransactionAfterStateV .Where(lt => lt.StateVersion >= afterStateVersion) .OrderBy(lt => lt.StateVersion) .Take(1) - .WithQueryName(); + .AnnotateMetricName(); } public static IQueryable GetLatestLedgerTransactionBeforeTimestamp(this TDbContext dbContext, DateTime timestamp) @@ -111,7 +111,7 @@ public static IQueryable GetLatestLedgerTransactionBeforeTime .OrderByDescending(lt => lt.RoundTimestamp) .ThenByDescending(lt => lt.StateVersion) .Take(1) - .WithQueryName(); + .AnnotateMetricName(); } public static IQueryable GetFirstLedgerTransactionAfterTimestamp(this TDbContext dbContext, DateTime timestamp) @@ -123,7 +123,7 @@ public static IQueryable GetFirstLedgerTransactionAfterTimest .OrderBy(lt => lt.RoundTimestamp) .ThenBy(lt => lt.StateVersion) .Take(1) - .WithQueryName(); + .AnnotateMetricName(); } public static IQueryable GetLatestLedgerTransactionAtEpochRound(this TDbContext dbContext, long epoch, long round) @@ -134,7 +134,7 @@ public static IQueryable GetLatestLedgerTransactionAtEpochRou .Where(lt => lt.Epoch == epoch && lt.RoundInEpoch >= round && lt.IndexInRound == 0) .OrderByDescending(lt => lt.StateVersion) .Take(1) - .WithQueryName(); + .AnnotateMetricName(); } public static IQueryable GetFirstLedgerTransactionAtEpochRound(this TDbContext dbContext, long epoch, long round) @@ -145,6 +145,6 @@ public static IQueryable GetFirstLedgerTransactionAtEpochRoun .Where(lt => lt.Epoch >= epoch && lt.RoundInEpoch >= round && lt.IndexInRound == 0) .OrderBy(lt => lt.StateVersion) .Take(1) - .WithQueryName(); + .AnnotateMetricName(); } } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs index a386dd6c4..63bc95539 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs @@ -5,9 +5,8 @@ using RadixDlt.NetworkGateway.GatewayApi.Configuration; using RadixDlt.NetworkGateway.GatewayApi.Services; using RadixDlt.NetworkGateway.PostgresIntegration.Metrics; +using System; using System.Data.Common; -using System.Linq; -using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; @@ -15,16 +14,15 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; internal class MetricsInterceptor : DbCommandInterceptor { - private const string UnknownQueryName = "UNKNOWN"; - private readonly IOptionsMonitor _slowQueriesLoggingOptions; + private readonly IOptionsMonitor _slowQueriesLoggingOptions; private readonly ILogger _logger; private readonly ISqlQueryObserver _sqlQueryObserver; - public MetricsInterceptor(ILoggerFactory loggerFactory, IOptionsMonitor slowQueriesLoggingOptions, ISqlQueryObserver sqlQueryObserver) + public MetricsInterceptor(ILogger logger, IOptionsMonitor slowQueriesLoggingOptions, ISqlQueryObserver sqlQueryObserver) { _slowQueriesLoggingOptions = slowQueriesLoggingOptions; _sqlQueryObserver = sqlQueryObserver; - _logger = loggerFactory.CreateLogger(); + _logger = logger; } public override DbDataReader ReaderExecuted(DbCommand command, CommandExecutedEventData eventData, DbDataReader result) @@ -47,7 +45,7 @@ private void Observe( DbCommand command, CommandExecutedEventData eventData) { - var queryName = GetQueryName(command) ?? UnknownQueryName; + var queryName = GetQueryName(command); _sqlQueryObserver.OnSqlQueryExecuted(queryName, eventData.Duration); @@ -64,36 +62,26 @@ private void Observe( } } - private string? GetQueryName(DbCommand dbCommand) + private string GetQueryName(DbCommand dbCommand) { - const string QueryNameGroup = "queryName"; - var matches = Regex.Matches(dbCommand.CommandText, $"(?:{SqlQueryMetricsHelper.QueryNameTag}=)(?<{QueryNameGroup}>\\b.*?\\b);"); + const string UnknownQueryName = "UNKNOWN"; - switch (matches.Count) - { - case 0: - _logger.LogDebug("Missing query name for: {commandText}", dbCommand.CommandText); - return null; - case 1: - { - var hasQueryName = matches.First().Groups.TryGetValue(QueryNameGroup, out var queryName); - - if (!hasQueryName || string.IsNullOrEmpty(queryName?.Value)) - { - _logger.LogDebug("Missing query name for: {commandText}", dbCommand.CommandText); - } + var startOfTag = dbCommand.CommandText.IndexOf($"{SqlQueryMetricsHelper.QueryNameTag}<", StringComparison.InvariantCultureIgnoreCase); + var endOfTag = dbCommand.CommandText.IndexOf(">", StringComparison.InvariantCultureIgnoreCase); - return queryName!.Value; - } + if (startOfTag < 0 || endOfTag < 0) + { + _logger.LogDebug("Missing query name for: {commandText}", dbCommand.CommandText); + return UnknownQueryName; + } - case > 1: - { - var foundTags = string.Join(',', matches.Select(x => x.Groups.Values.ToString())); - _logger.LogDebug("Query name provided multiple times: {foundTags}, in query: {commandText}", foundTags, dbCommand.CommandText); - return null; - } + var queryName = dbCommand.CommandText.Substring(startOfTag, endOfTag); - default: return null; + if (string.IsNullOrEmpty(queryName)) + { + throw new ArgumentNullException(queryName, "Query name extracted from query tag is empty."); } + + return queryName; } } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs index b954a67e6..166204c26 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs @@ -171,7 +171,7 @@ private async Task UpdatePendingTransactions(ReadWriteDbContext dbContext, pt.LedgerDetails.LatestRejectionTimestamp, pt.LedgerDetails.LatestRejectionReason, }) - .WithQueryName() + .AnnotateMetricName() .ToListAsync(token); foreach (var details in pendingTransactions) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs index e8f991c4e..067573b4b 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs @@ -132,7 +132,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) emh ON true;") .AsNoTracking() - .WithQueryName() + .AnnotateMetricName() .ToDictionaryAsync(e => new MetadataLookup(e.EntityId, e.Key), token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentEntityMetadataHistoryFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -166,7 +166,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) emah ON true;") .AsNoTracking() - .WithQueryName() + .AnnotateMetricName() .ToDictionaryAsync(e => e.EntityId, token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentEntityAggregateMetadataHistoryFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -220,7 +220,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) eareh ON true;") .AsNoTracking() - .WithQueryName() + .AnnotateMetricName() .ToDictionaryAsync(e => new RoleAssignmentEntryLookup(e.EntityId, e.KeyRole, e.KeyModule), token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentEntityRoleAssignmentsEntryHistoryFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -256,7 +256,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) earah ON true;") .AsNoTracking() - .WithQueryName() + .AnnotateMetricName() .ToDictionaryAsync(e => e.EntityId, token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentEntityRoleAssignmentsAggregateHistoryFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -298,7 +298,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) erah ON true;") .AsNoTracking() - .WithQueryName() + .AnnotateMetricName() .ToDictionaryAsync(e => e.EntityId, token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentEntityResourceAggregateHistoryFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -341,7 +341,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) eravh ON true;") .AsNoTracking() - .WithQueryName() + .AnnotateMetricName() .ToDictionaryAsync(e => new EntityResourceLookup(e.EntityId, e.ResourceEntityId), token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentEntityResourceAggregatedVaultsHistoryFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -392,7 +392,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) ervah ON true;") .AsNoTracking() - .WithQueryName() + .AnnotateMetricName() .ToDictionaryAsync(e => new EntityResourceVaultLookup(e.EntityId, e.ResourceEntityId), token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentEntityResourceVaultAggregateHistoryFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -426,7 +426,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) evh ON true;") .AsNoTracking() - .WithQueryName() + .AnnotateMetricName() .ToDictionaryAsync(e => e.VaultEntityId, e => (EntityNonFungibleVaultHistory)e, token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentEntityNonFungibleVaultHistory), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -460,7 +460,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) emh ON true;") .AsNoTracking() - .WithQueryName() + .AnnotateMetricName() .ToDictionaryAsync(e => e.NonFungibleResourceEntityId, token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentNonFungibleIdStoreHistoryFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -494,7 +494,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) rmesh ON true;") .AsNoTracking() - .WithQueryName() + .AnnotateMetricName() .ToDictionaryAsync(e => e.ResourceEntityId, token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentResourceEntitySupplyHistoryFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -523,7 +523,7 @@ FROM entities WHERE address = ANY({entityAddressesParameter}) )") .AsNoTracking() - .WithQueryName() + .AnnotateMetricName() .ToDictionaryAsync(e => e.Address, token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(ExistingEntitiesFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -569,7 +569,7 @@ public async Task> ExistingNo SELECT UNNEST({resourceEntityIds}), UNNEST({nonFungibleIds}) )") .AsNoTracking() - .WithQueryName() + .AnnotateMetricName() .ToDictionaryAsync(e => new NonFungibleIdLookup(e.NonFungibleResourceEntityId, e.NonFungibleId), token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(ExistingNonFungibleIdDataFor), Stopwatch.GetElapsedTime(sw), result.Count)); @@ -620,7 +620,7 @@ LIMIT 1 ) vpkh ON true; ") .AsNoTracking() - .WithQueryName() + .AnnotateMetricName() .ToDictionaryAsync(e => new ValidatorKeyLookup(e.ValidatorEntityId, e.KeyType, e.Key), token); await _observers.ForEachAsync(x => x.StageCompleted(nameof(ExistingValidatorKeysFor), Stopwatch.GetElapsedTime(sw), result.Count)); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs index 9857486cd..9ec20efc3 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Linq; namespace RadixDlt.NetworkGateway.PostgresIntegration.Metrics; @@ -8,26 +9,33 @@ public static class SqlQueryMetricsHelper public const string QueryNameTag = "QueryName"; public static string GetQueryNameValue( - string operationName = "", - string filePath = "", - string methodName = "") + string operationName, + string methodName) { - var fileName = Path.GetFileNameWithoutExtension(filePath); + if (!operationName.All(char.IsLetterOrDigit)) + { + throw new ArgumentException($"{nameof(operationName)} is expected to be alphanumeric. Got: {operationName}"); + } + + if (!methodName.All(char.IsLetterOrDigit)) + { + throw new ArgumentException($"{nameof(methodName)} is expected to be alphanumeric. Got: {operationName}"); + } if (!string.IsNullOrEmpty(operationName) && !string.Equals(operationName, methodName, StringComparison.InvariantCultureIgnoreCase)) { - return $"{fileName}_{methodName}_{operationName}"; + return $"{methodName}_{operationName}"; } else { - return $"{fileName}_{methodName}"; + return $"{methodName}"; } } - public static string GenerateQueryNameTag(string operationName = "", string filePath = "", string methodName = "") + public static string GenerateQueryNameTag(string operationName, string methodName) { - var queryName = GetQueryNameValue(operationName, filePath, methodName); - return $"{QueryNameTag}={queryName};"; + var queryName = GetQueryNameValue(operationName, methodName); + return $"{QueryNameTag}<{queryName}>;"; } } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/QueryableExtensions.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/QueryableExtensions.cs index 10be9b6d9..79c8b1b2c 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/QueryableExtensions.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/QueryableExtensions.cs @@ -7,13 +7,12 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration; public static class QueryableExtensions { - public static IQueryable WithQueryName( + public static IQueryable AnnotateMetricName( this IQueryable source, string operationName = "", - [CallerFilePath] string filePath = "", [CallerMemberName] string methodName = "") { - var queryNameTag = SqlQueryMetricsHelper.GenerateQueryNameTag(operationName, filePath, methodName); + var queryNameTag = SqlQueryMetricsHelper.GenerateQueryNameTag(operationName, methodName); return source.TagWith(queryNameTag); } } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/BlueprintProvider.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/BlueprintProvider.cs index c9994e441..56a587bbe 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/BlueprintProvider.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/BlueprintProvider.cs @@ -109,7 +109,7 @@ INNER JOIN package_blueprint_history pbh ON pbh.name = v.blueprint_name AND pbh.version = v.blueprint_version and pbh.package_entity_id = v.package_entity_id WHERE from_state_version <= {ledgerState.StateVersion} ") - .WithQueryName() + .AnnotateMetricName() .ToDictionaryAsync( x => new BlueprintDefinitionIdentifier(x.Name, x.Version, x.PackageEntityId), x => x, diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/CapturedConfigProvider.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/CapturedConfigProvider.cs index c383387c3..b80503701 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/CapturedConfigProvider.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/CapturedConfigProvider.cs @@ -81,7 +81,7 @@ public CapturedConfigProvider(ReadOnlyDbContext dbContext) public async Task CaptureConfiguration() { - var networkConfiguration = await _dbContext.NetworkConfiguration.AsNoTracking().WithQueryName().SingleOrDefaultAsync(); + var networkConfiguration = await _dbContext.NetworkConfiguration.AsNoTracking().AnnotateMetricName().SingleOrDefaultAsync(); if (networkConfiguration == null) { diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/CommittedStateIdentifiersReader.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/CommittedStateIdentifiersReader.cs index 8f785f437..b1cf45375 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/CommittedStateIdentifiersReader.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/CommittedStateIdentifiersReader.cs @@ -84,7 +84,7 @@ public CommittedStateIdentifiersReader(IDbContextFactory dbC var dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); var transaction = await dbContext.LedgerTransactions - .WithQueryName() + .AnnotateMetricName() .FirstOrDefaultAsync(x => x.StateVersion == stateVersion, cancellationToken); if (transaction == null) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs index beed794b8..5490525a4 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs @@ -16,28 +16,26 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.Services; public interface IDapperWrapper { Task> QueryAsync( - IDbConnection cnn, + IDbConnection connection, CommandDefinition command, string operationName = "", - [CallerFilePath] string filePath = "", [CallerMemberName] string methodName = ""); Task QueryFirstOrDefaultAsync( - IDbConnection cnn, + IDbConnection connection, CommandDefinition command, string operationName = "", - [CallerFilePath] string filePath = "", [CallerMemberName] string methodName = ""); } public class DapperWrapper : IDapperWrapper { - private readonly IOptionsMonitor _slowQueriesLoggingOptions; + private readonly IOptionsMonitor _slowQueriesLoggingOptions; private readonly ILogger _logger; private readonly ISqlQueryObserver _sqlQueryObserver; public DapperWrapper( - IOptionsMonitor slowQueriesLoggingOptions, + IOptionsMonitor slowQueriesLoggingOptions, ILogger logger, ISqlQueryObserver sqlQueryObserver) { @@ -47,18 +45,17 @@ public DapperWrapper( } public async Task> QueryAsync( - IDbConnection cnn, + IDbConnection connection, CommandDefinition command, string operationName = "", - [CallerFilePath] string filePath = "", [CallerMemberName] string methodName = "") { var stopwatch = Stopwatch.StartNew(); - var result = await cnn.QueryAsync(command); + var result = await connection.QueryAsync(command); var elapsed = stopwatch.Elapsed; - var queryName = SqlQueryMetricsHelper.GetQueryNameValue(operationName, filePath, methodName); + var queryName = SqlQueryMetricsHelper.GetQueryNameValue(operationName, methodName); _sqlQueryObserver.OnSqlQueryExecuted(queryName, elapsed); @@ -67,7 +64,7 @@ public async Task> QueryAsync( { var parameters = JsonConvert.SerializeObject(command.Parameters); _logger.LogWarning( - "Long running query: {query}, parameters: {queryParameters} duration: {queryDuration} milliseconds", + "Long running query: {query}, parameters: {queryParameters} duration: {queryDuration} seconds", command.CommandText, parameters, elapsed); } @@ -75,18 +72,17 @@ public async Task> QueryAsync( } public async Task QueryFirstOrDefaultAsync( - IDbConnection cnn, + IDbConnection connection, CommandDefinition command, string operationName = "", - [CallerFilePath] string filePath = "", [CallerMemberName] string methodName = "") { var stopwatch = Stopwatch.StartNew(); - var result = await cnn.QueryFirstOrDefaultAsync(command); + var result = await connection.QueryFirstOrDefaultAsync(command); var elapsed = stopwatch.Elapsed; - var queryName = SqlQueryMetricsHelper.GetQueryNameValue(operationName, filePath, methodName); + var queryName = SqlQueryMetricsHelper.GetQueryNameValue(operationName, methodName); _sqlQueryObserver.OnSqlQueryExecuted(queryName, elapsed); var logQueriesLongerThan = _slowQueriesLoggingOptions.CurrentValue.SlowQueriesThreshold; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs index 32864b8c3..dab96ea42 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs @@ -716,7 +716,7 @@ LIMIT 1 .OrderBy(e => e.FromStateVersion) .ThenBy(e => e.Id) .Take(validatorsPageSize + 1) - .WithQueryName("GetValidators") + .AnnotateMetricName("GetValidators") .ToListAsync(token); var findEpochSubquery = _dbContext @@ -731,7 +731,7 @@ LIMIT 1 .ValidatorActiveSetHistory .Include(e => e.PublicKey) .Where(e => e.Epoch == findEpochSubquery.First()) - .WithQueryName("GetValidatorActiveSet") + .AnnotateMetricName("GetValidatorActiveSet") .ToDictionaryAsync(e => e.PublicKey.ValidatorEntityId, token); var totalStake = activeSetById @@ -820,7 +820,7 @@ LIMIT 1 .Entities .Where(e => validatorVaultIds.Contains(e.Id)) .Select(e => new { e.Id, e.Address }) - .WithQueryName("GetVaultAddresses") + .AnnotateMetricName("GetVaultAddresses") .ToDictionaryAsync(e => e.Id, e => e.Address, token); var metadataById = await GetMetadataSlices(validatorIds, 0, _endpointConfiguration.Value.DefaultPageSize, ledgerState, token); @@ -928,7 +928,7 @@ FROM key_value_store_entry_history ORDER BY from_state_version DESC LIMIT 1 ) kvseh ON TRUE;") - .WithQueryName("GetKeyValueStores") + .AnnotateMetricName("GetKeyValueStores") .ToListAsync(token); var items = new List(); @@ -1087,7 +1087,7 @@ FROM entity_metadata_history ORDER BY from_state_version DESC LIMIT 1 ) emh ON TRUE;") - .WithQueryName() + .AnnotateMetricName() .ToListAsync(token); var result = new Dictionary(); @@ -1204,7 +1204,7 @@ FROM resource_entity_supply_history ORDER BY from_state_version DESC LIMIT 1 ) resh ON true") - .WithQueryName() + .AnnotateMetricName() .ToDictionaryAsync(e => e.ResourceEntityId, token); foreach (var missing in entityIds.Except(result.Keys)) @@ -1221,7 +1221,7 @@ private async Task GetEntity(EntityAddress address, GatewayMod var entity = await _dbContext .Entities .Where(e => e.FromStateVersion <= ledgerState.StateVersion) - .WithQueryName() + .AnnotateMetricName() .FirstOrDefaultAsync(e => e.Address == address, token); if (entity == null) @@ -1267,7 +1267,7 @@ private async Task> GetEntities(List addresse var entities = await _dbContext .Entities .Where(e => e.FromStateVersion <= ledgerState.StateVersion && addresses.Contains(e.Address)) - .WithQueryName() + .AnnotateMetricName() .ToListAsync(token); foreach (var address in addresses) @@ -1312,7 +1312,7 @@ FROM state_history ORDER BY from_state_version DESC LIMIT 1 ) esh ON TRUE;") - .WithQueryName("GetStateHistory") + .AnnotateMetricName("GetStateHistory") .ToListAsync(token); var schemasToLoad = states @@ -1342,7 +1342,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) esh ON TRUE; ") - .WithQueryName("GetSchemas") + .AnnotateMetricName("GetSchemas") .ToDictionaryAsync( x => new SchemaIdentifier(x.SchemaHash, x.EntityId), x => x.Schema, @@ -1401,7 +1401,7 @@ INNER JOIN LATERAL( FROM package_blueprint_history WHERE package_entity_id = variables.entity_id AND from_state_version <= {ledgerState.StateVersion} ) pbh ON true") - .WithQueryName() + .AnnotateMetricName() .ToListAsync(token)) .GroupBy(b => b.PackageEntityId) .ToDictionary(g => g.Key, g => g.ToArray()); @@ -1427,7 +1427,7 @@ FROM package_code_history ORDER BY from_state_version DESC LIMIT 1 ) pch ON true") - .WithQueryName() + .AnnotateMetricName() .ToDictionaryAsync(e => e.PackageEntityId, token); } @@ -1449,7 +1449,7 @@ INNER JOIN LATERAL( FROM schema_history WHERE entity_id = variables.entity_id AND from_state_version <= {ledgerState.StateVersion} ) psh ON true") - .WithQueryName() + .AnnotateMetricName() .ToListAsync(token)) .GroupBy(b => b.EntityId) .ToDictionary(g => g.Key, g => g.ToArray()); @@ -1490,7 +1490,7 @@ private async Task> GetCorrelatedEntityAddresses .Entities .Where(e => ids.Contains(e.Id) && e.FromStateVersion <= ledgerState.StateVersion) .Select(e => new { e.Id, GlobalAddress = e.Address }) - .WithQueryName() + .AnnotateMetricName() .ToDictionaryAsync(e => e.Id, e => e.GlobalAddress, token); } @@ -1515,7 +1515,7 @@ private async Task> ResolveResourceEntityIds( .Entities .Where(e => addresses.Contains(e.Address)) .Select(e => new { e.Id, e.Address }) - .WithQueryName() + .AnnotateMetricName() .ToDictionaryAsync(e => e.Address, e => e.Id, token); } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/NetworkConfigurationProvider.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/NetworkConfigurationProvider.cs index 7d321b3e2..76abe7a0a 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/NetworkConfigurationProvider.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/NetworkConfigurationProvider.cs @@ -141,7 +141,7 @@ public async Task SaveLedgerNetworkConfigurationToDatabaseOnInitIfNotExist { await using var dbContext = await _dbContextFactory.CreateDbContextAsync(token); - if (await dbContext.NetworkConfiguration.AsNoTracking().WithQueryName().AnyAsync(token)) + if (await dbContext.NetworkConfiguration.AsNoTracking().AnnotateMetricName().AnyAsync(token)) { return false; } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/PendingTransactionPrunerService.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/PendingTransactionPrunerService.cs index 93bce3394..40ee704d9 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/PendingTransactionPrunerService.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/PendingTransactionPrunerService.cs @@ -117,7 +117,7 @@ public async Task PrunePendingTransactions(CancellationToken token = default) .PendingTransactions .GroupBy(canPruneExpression) .Select(g => new PendingTransactionCountByPruneStatus(g.Key, g.Count())) - .WithQueryName("GetTransactionsToPrune") + .AnnotateMetricName("GetTransactionsToPrune") .ToListAsync(token); await _observers.ForEachAsync(x => x.PrePendingTransactionPrune(countByStatus)); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/PendingTransactionResubmissionService.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/PendingTransactionResubmissionService.cs index f720dd6b2..9533ee628 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/PendingTransactionResubmissionService.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/PendingTransactionResubmissionService.cs @@ -168,7 +168,7 @@ await GetPendingTransactionsNeedingResubmission(instantForTransactionChoosing, d .OrderBy(mt => mt.GatewayHandling.ResubmitFromTimestamp) .Include(t => t.Payload) .Take(batchSize) - .WithQueryName() + .AnnotateMetricName() .ToListAsync(token); var totalTransactionsNeedingResubmission = transactionsToResubmit.Count < batchSize diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/RoleAssignmentQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/RoleAssignmentQuerier.cs index fa9e6d929..5fa8b56b6 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/RoleAssignmentQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/RoleAssignmentQuerier.cs @@ -117,14 +117,14 @@ public RoleAssignmentQuerier( var ownerRoles = await _dbContext .EntityRoleAssignmentsOwnerHistory .Where(e => ownerRoleIds.Contains(e.Id)) - .WithQueryName("GetOwnerRoles") + .AnnotateMetricName("GetOwnerRoles") .ToListAsync(token); var entries = await _dbContext .EntityRoleAssignmentsEntryHistory .Where(e => roleAssignmentsHistory.Contains(e.Id)) .Where(e => !e.IsDeleted) - .WithQueryName("GetRoleEntires") + .AnnotateMetricName("GetRoleEntires") .ToListAsync(token); return _roleAssignmentsMapper.GetEffectiveRoleAssignments(componentEntities, blueprintAuthConfigs, ownerRoles, entries); @@ -175,7 +175,7 @@ FROM entity_role_assignments_aggregate_history ORDER BY from_state_version DESC LIMIT 1 ) earah ON TRUE;") - .WithQueryName() + .AnnotateMetricName() .ToListAsync(token); return aggregates; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/SubmissionTrackingService.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/SubmissionTrackingService.cs index 93ef741bd..ae4654de4 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/SubmissionTrackingService.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/SubmissionTrackingService.cs @@ -169,7 +169,7 @@ private async Task TrackSubmission( var existingPendingTransaction = await _dbContext .PendingTransactions .Where(t => t.PayloadHash == payloadHash) - .WithQueryName() + .AnnotateMetricName() .SingleOrDefaultAsync(token); if (existingPendingTransaction != null) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs index 837d4551e..b5ce1044b 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs @@ -300,7 +300,7 @@ public async Task GetTransactionStream(TransactionS var stateVersions = await searchQuery .TagWith(ForceDistinctInterceptor.Apply) .Take(request.PageSize + 1) - .WithQueryName("GetTransactionsStateVersions") + .AnnotateMetricName("GetTransactionsStateVersions") .ToListAsync(token); var transactions = await GetTransactions(stateVersions.Take(request.PageSize).ToList(), request.OptIns, token); @@ -324,7 +324,7 @@ public async Task GetTransactionStream(TransactionS .OfType() .Where(ult => ult.StateVersion <= ledgerState.StateVersion && ult.IntentHash == intentHash) .Select(ult => ult.StateVersion) - .WithQueryName() + .AnnotateMetricName() .FirstOrDefaultAsync(token); if (stateVersion == default) @@ -531,7 +531,7 @@ private string GetIntentStatusDescription(GatewayModel.TransactionIntentStatus i ult.PayloadHash, ult.EngineReceipt.Status, ult.EngineReceipt.ErrorMessage)) - .WithQueryName() + .AnnotateMetricName() .FirstOrDefaultAsync(token); var aggregator = new PendingTransactionResponseAggregator(ledgerState, maybeCommittedTransactionSummary); @@ -565,7 +565,7 @@ private async Task> LookupPendingTransact pt.NetworkDetails.LastSubmitErrorTitle ) ) - .WithQueryName() + .AnnotateMetricName() .ToListAsync(token); } @@ -577,7 +577,7 @@ private async Task> LookupPendingTransact var transactions = await _dbContext .LedgerTransactions .Where(ult => transactionStateVersions.Contains(ult.StateVersion)) - .WithQueryName("GetTransactions") + .AnnotateMetricName("GetTransactions") .ToListAsync(token); var entityIdToAddressMap = await GetEntityAddresses(transactions.SelectMany(x => x.AffectedGlobalEntities).ToList(), token); @@ -602,7 +602,7 @@ SELECT UNNEST({entityIds}), UNNEST({schemaHashes}) SELECT sh.* FROM variables var INNER JOIN schema_history sh ON sh.entity_id = var.entity_id AND sh.schema_hash = var.schema_hash") - .WithQueryName("GetEventSchemas") + .AnnotateMetricName("GetEventSchemas") .ToDictionaryAsync(x => new SchemaLookup(x.EntityId, x.SchemaHash), x => x.Schema, token); } @@ -648,7 +648,7 @@ private async Task> GetEntityIds(List addresses .Entities .Where(e => addresses.Contains(e.Address)) .Select(e => new { e.Id, e.Address }) - .WithQueryName() + .AnnotateMetricName() .ToDictionaryAsync(e => e.Address.ToString(), e => e.Id, token); } @@ -663,7 +663,7 @@ private async Task> GetEntityAddresses(List entit .Entities .Where(e => entityIds.Contains(e.Id)) .Select(e => new { e.Id, e.Address }) - .WithQueryName() + .AnnotateMetricName() .ToDictionaryAsync(e => e.Id, e => e.Address.ToString(), token); } } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/ValidatorQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/ValidatorQuerier.cs index dd6f682d8..c69bd782f 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/ValidatorQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/ValidatorQuerier.cs @@ -99,7 +99,7 @@ public ValidatorQuerier(ReadOnlyDbContext dbContext, IDapperWrapper dapperWrappe var validators = await _dbContext .Entities .Where(e => addresses.Contains(e.Address) && e.FromStateVersion <= ledgerState.StateVersion) - .WithQueryName("GetValidators") + .AnnotateMetricName("GetValidators") .ToDictionaryAsync(e => e.Id, e => e.Address, token); var cd = new CommandDefinition( diff --git a/src/RadixDlt.NetworkGateway.PrometheusIntegration/DataAggregatorMetricsObserver.cs b/src/RadixDlt.NetworkGateway.PrometheusIntegration/DataAggregatorMetricsObserver.cs index ce0e8b1cb..c1259f0ea 100644 --- a/src/RadixDlt.NetworkGateway.PrometheusIntegration/DataAggregatorMetricsObserver.cs +++ b/src/RadixDlt.NetworkGateway.PrometheusIntegration/DataAggregatorMetricsObserver.cs @@ -96,7 +96,7 @@ internal class DataAggregatorMetricsObserver : ITransactionStreamReaderObserver, ISqlQueryObserver { - private static readonly Histogram _sqlQueryDuration = Prometheus.Metrics + private static readonly Histogram _sqlQueryDuration = Metrics .CreateHistogram("sql_query_duration", "The duration of SQL queries processed by this app.", new HistogramConfiguration { LabelNames = new[] { "query_name" } }); @@ -582,6 +582,6 @@ ValueTask ITransactionStreamReaderObserver.GetTransactionsFailed(string nodeName public void OnSqlQueryExecuted(string queryName, TimeSpan duration) { - _sqlQueryDuration.WithLabels(queryName).Observe(duration.TotalMilliseconds); + _sqlQueryDuration.WithLabels(queryName).Observe(duration.TotalSeconds); } } diff --git a/src/RadixDlt.NetworkGateway.PrometheusIntegration/GatewayApiMetricObserver.cs b/src/RadixDlt.NetworkGateway.PrometheusIntegration/GatewayApiMetricObserver.cs index fef9f2833..294f713c3 100644 --- a/src/RadixDlt.NetworkGateway.PrometheusIntegration/GatewayApiMetricObserver.cs +++ b/src/RadixDlt.NetworkGateway.PrometheusIntegration/GatewayApiMetricObserver.cs @@ -176,7 +176,7 @@ internal class GatewayApiMetricObserver : public void OnSqlQueryExecuted(string queryName, TimeSpan duration) { - _sqlQueryDuration.WithLabels(queryName).Observe(duration.TotalMilliseconds); + _sqlQueryDuration.WithLabels(queryName).Observe(duration.TotalSeconds); } void IExceptionObserver.OnException(ActionContext actionContext, Exception exception, KnownGatewayErrorException gatewayErrorException) From 27bf20375675b4f772a0f78268045c893172f9d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Wed, 27 Sep 2023 11:25:21 +0200 Subject: [PATCH 15/22] address pull request comments. --- .../Configuration/SlowQueryLoggingOptions.cs | 4 ++-- .../Configuration/SlowQueryLoggingOptions.cs | 4 ++-- .../Interceptors/MetricsInterceptor.cs | 2 +- .../Metrics/SqlQueryMetricsHelper.cs | 13 +++++++++---- .../Services/DapperWrapper.cs | 4 ++-- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/RadixDlt.NetworkGateway.DataAggregator/Configuration/SlowQueryLoggingOptions.cs b/src/RadixDlt.NetworkGateway.DataAggregator/Configuration/SlowQueryLoggingOptions.cs index ec6be035b..8aeeaf344 100644 --- a/src/RadixDlt.NetworkGateway.DataAggregator/Configuration/SlowQueryLoggingOptions.cs +++ b/src/RadixDlt.NetworkGateway.DataAggregator/Configuration/SlowQueryLoggingOptions.cs @@ -6,13 +6,13 @@ namespace RadixDlt.NetworkGateway.DataAggregator.Configuration; public sealed class SlowQueryLoggingOptions { - public TimeSpan SlowQueriesThreshold { get; set; } = TimeSpan.FromMilliseconds(250); + public TimeSpan SlowQueryThreshold { get; set; } = TimeSpan.FromMilliseconds(250); } internal class SlowQueryLoggingOptionsValidator : AbstractOptionsValidator { public SlowQueryLoggingOptionsValidator() { - RuleFor(x => x.SlowQueriesThreshold).GreaterThan(TimeSpan.Zero); + RuleFor(x => x.SlowQueryThreshold).GreaterThan(TimeSpan.Zero); } } diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/SlowQueryLoggingOptions.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/SlowQueryLoggingOptions.cs index d50e7dde6..a05bb6754 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/SlowQueryLoggingOptions.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/SlowQueryLoggingOptions.cs @@ -6,13 +6,13 @@ namespace RadixDlt.NetworkGateway.GatewayApi.Configuration; public sealed class SlowQueryLoggingOptions { - public TimeSpan SlowQueriesThreshold { get; set; } = TimeSpan.FromMilliseconds(250); + public TimeSpan SlowQueryThreshold { get; set; } = TimeSpan.FromMilliseconds(250); } internal class SlowQueryLoggingValidator : AbstractOptionsValidator { public SlowQueryLoggingValidator() { - RuleFor(x => x.SlowQueriesThreshold).GreaterThan(TimeSpan.Zero); + RuleFor(x => x.SlowQueryThreshold).GreaterThan(TimeSpan.Zero); } } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs index 63bc95539..90e049fc0 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs @@ -49,7 +49,7 @@ private void Observe( _sqlQueryObserver.OnSqlQueryExecuted(queryName, eventData.Duration); - var logQueriesLongerThan = _slowQueriesLoggingOptions.CurrentValue.SlowQueriesThreshold; + var logQueriesLongerThan = _slowQueriesLoggingOptions.CurrentValue.SlowQueryThreshold; if (eventData.Duration > logQueriesLongerThan) { #pragma warning disable EF1001 diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs index 9ec20efc3..3d4759045 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs @@ -12,14 +12,14 @@ public static string GetQueryNameValue( string operationName, string methodName) { - if (!operationName.All(char.IsLetterOrDigit)) + if (!IsValidTagPart(operationName)) { - throw new ArgumentException($"{nameof(operationName)} is expected to be alphanumeric. Got: {operationName}"); + throw new ArgumentException("Expected alphanumeric, got: {operationName}", nameof(operationName)); } - if (!methodName.All(char.IsLetterOrDigit)) + if (!IsValidTagPart(methodName)) { - throw new ArgumentException($"{nameof(methodName)} is expected to be alphanumeric. Got: {operationName}"); + throw new ArgumentException("Expected alphanumeric, got: {operationName}", nameof(methodName)); } if (!string.IsNullOrEmpty(operationName) && @@ -38,4 +38,9 @@ public static string GenerateQueryNameTag(string operationName, string methodNam var queryName = GetQueryNameValue(operationName, methodName); return $"{QueryNameTag}<{queryName}>;"; } + + private static bool IsValidTagPart(string value) + { + return value.All(c => char.IsLetterOrDigit(c) || c == '_' || c == '_'); + } } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs index 5490525a4..d281901cc 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs @@ -59,7 +59,7 @@ public async Task> QueryAsync( _sqlQueryObserver.OnSqlQueryExecuted(queryName, elapsed); - var logQueriesLongerThan = _slowQueriesLoggingOptions.CurrentValue.SlowQueriesThreshold; + var logQueriesLongerThan = _slowQueriesLoggingOptions.CurrentValue.SlowQueryThreshold; if (elapsed > logQueriesLongerThan) { var parameters = JsonConvert.SerializeObject(command.Parameters); @@ -85,7 +85,7 @@ public async Task QueryFirstOrDefaultAsync( var queryName = SqlQueryMetricsHelper.GetQueryNameValue(operationName, methodName); _sqlQueryObserver.OnSqlQueryExecuted(queryName, elapsed); - var logQueriesLongerThan = _slowQueriesLoggingOptions.CurrentValue.SlowQueriesThreshold; + var logQueriesLongerThan = _slowQueriesLoggingOptions.CurrentValue.SlowQueryThreshold; if (elapsed > logQueriesLongerThan) { var parameters = JsonConvert.SerializeObject(command.Parameters); From 7d59c26e9caba1167ade631948a7b27b558ef20e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Wed, 27 Sep 2023 11:37:30 +0200 Subject: [PATCH 16/22] pull request comments addressed. --- .../Configuration/SlowQueryLoggingOptions.cs | 5 ++--- .../Configuration/SlowQueryLoggingOptions.cs | 18 ------------------ .../ServiceCollectionExtensions.cs | 2 +- .../DataAggregatorBuilderExtensions.cs | 4 ++-- .../GatewayApiBuilderExtensions.cs | 4 ++-- .../Interceptors/MetricsInterceptor.cs | 9 +++++---- .../Metrics/SqlQueryMetricsHelper.cs | 2 +- .../Services/DapperWrapper.cs | 11 ++++++----- 8 files changed, 19 insertions(+), 36 deletions(-) rename src/{RadixDlt.NetworkGateway.DataAggregator => RadixDlt.NetworkGateway.Abstractions}/Configuration/SlowQueryLoggingOptions.cs (58%) delete mode 100644 src/RadixDlt.NetworkGateway.GatewayApi/Configuration/SlowQueryLoggingOptions.cs diff --git a/src/RadixDlt.NetworkGateway.DataAggregator/Configuration/SlowQueryLoggingOptions.cs b/src/RadixDlt.NetworkGateway.Abstractions/Configuration/SlowQueryLoggingOptions.cs similarity index 58% rename from src/RadixDlt.NetworkGateway.DataAggregator/Configuration/SlowQueryLoggingOptions.cs rename to src/RadixDlt.NetworkGateway.Abstractions/Configuration/SlowQueryLoggingOptions.cs index 8aeeaf344..d771c62de 100644 --- a/src/RadixDlt.NetworkGateway.DataAggregator/Configuration/SlowQueryLoggingOptions.cs +++ b/src/RadixDlt.NetworkGateway.Abstractions/Configuration/SlowQueryLoggingOptions.cs @@ -1,15 +1,14 @@ using FluentValidation; -using RadixDlt.NetworkGateway.Abstractions.Configuration; using System; -namespace RadixDlt.NetworkGateway.DataAggregator.Configuration; +namespace RadixDlt.NetworkGateway.Abstractions.Configuration; public sealed class SlowQueryLoggingOptions { public TimeSpan SlowQueryThreshold { get; set; } = TimeSpan.FromMilliseconds(250); } -internal class SlowQueryLoggingOptionsValidator : AbstractOptionsValidator +public class SlowQueryLoggingOptionsValidator : AbstractOptionsValidator { public SlowQueryLoggingOptionsValidator() { diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/SlowQueryLoggingOptions.cs b/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/SlowQueryLoggingOptions.cs deleted file mode 100644 index a05bb6754..000000000 --- a/src/RadixDlt.NetworkGateway.GatewayApi/Configuration/SlowQueryLoggingOptions.cs +++ /dev/null @@ -1,18 +0,0 @@ -using FluentValidation; -using RadixDlt.NetworkGateway.Abstractions.Configuration; -using System; - -namespace RadixDlt.NetworkGateway.GatewayApi.Configuration; - -public sealed class SlowQueryLoggingOptions -{ - public TimeSpan SlowQueryThreshold { get; set; } = TimeSpan.FromMilliseconds(250); -} - -internal class SlowQueryLoggingValidator : AbstractOptionsValidator -{ - public SlowQueryLoggingValidator() - { - RuleFor(x => x.SlowQueryThreshold).GreaterThan(TimeSpan.Zero); - } -} diff --git a/src/RadixDlt.NetworkGateway.GatewayApi/ServiceCollectionExtensions.cs b/src/RadixDlt.NetworkGateway.GatewayApi/ServiceCollectionExtensions.cs index 61d5d19ea..8aa2dcb2e 100644 --- a/src/RadixDlt.NetworkGateway.GatewayApi/ServiceCollectionExtensions.cs +++ b/src/RadixDlt.NetworkGateway.GatewayApi/ServiceCollectionExtensions.cs @@ -99,7 +99,7 @@ public static GatewayApiBuilder AddNetworkGatewayApiCore(this IServiceCollection services .AddValidatableOptionsAtSection("GatewayApi:Endpoint") - .AddValidatableOptionsAtSection("GatewayApi:SlowQueryLogging") + .AddValidatableOptionsAtSection("GatewayApi:SlowQueryLogging") .AddValidatableOptionsAtSection("GatewayApi:CoreApiIntegration") .AddValidatableOptionsAtSection("GatewayApi:Network") .AddValidatableOptionsAtSection("GatewayApi:AcceptableLedgerLag"); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/DataAggregatorBuilderExtensions.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/DataAggregatorBuilderExtensions.cs index 4c23c44dd..e90cc7632 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/DataAggregatorBuilderExtensions.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/DataAggregatorBuilderExtensions.cs @@ -93,8 +93,8 @@ public static DataAggregatorBuilder AddPostgresPersistenceCore(this DataAggregat .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton() - .AddSingleton(); + .AddScoped() + .AddScoped(); CustomTypes.EnsureConfigured(); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayApiBuilderExtensions.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayApiBuilderExtensions.cs index e623c8a87..c68915dc9 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayApiBuilderExtensions.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayApiBuilderExtensions.cs @@ -96,8 +96,8 @@ public static GatewayApiBuilder AddPostgresPersistenceCore(this GatewayApiBuilde .AddScoped() .AddScoped() .AddScoped() - .AddSingleton() - .AddSingleton(); + .AddScoped() + .AddScoped(); CustomTypes.EnsureConfigured(); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs index 90e049fc0..5728b7e63 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs @@ -2,6 +2,7 @@ using Microsoft.EntityFrameworkCore.Storage.Internal; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using RadixDlt.NetworkGateway.Abstractions.Configuration; using RadixDlt.NetworkGateway.GatewayApi.Configuration; using RadixDlt.NetworkGateway.GatewayApi.Services; using RadixDlt.NetworkGateway.PostgresIntegration.Metrics; @@ -14,13 +15,13 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; internal class MetricsInterceptor : DbCommandInterceptor { - private readonly IOptionsMonitor _slowQueriesLoggingOptions; + private readonly SlowQueryLoggingOptions _slowQueriesLoggingOptions; private readonly ILogger _logger; private readonly ISqlQueryObserver _sqlQueryObserver; - public MetricsInterceptor(ILogger logger, IOptionsMonitor slowQueriesLoggingOptions, ISqlQueryObserver sqlQueryObserver) + public MetricsInterceptor(ILogger logger, IOptionsSnapshot slowQueriesLoggingOptions, ISqlQueryObserver sqlQueryObserver) { - _slowQueriesLoggingOptions = slowQueriesLoggingOptions; + _slowQueriesLoggingOptions = slowQueriesLoggingOptions.Value; _sqlQueryObserver = sqlQueryObserver; _logger = logger; } @@ -49,7 +50,7 @@ private void Observe( _sqlQueryObserver.OnSqlQueryExecuted(queryName, eventData.Duration); - var logQueriesLongerThan = _slowQueriesLoggingOptions.CurrentValue.SlowQueryThreshold; + var logQueriesLongerThan = _slowQueriesLoggingOptions.SlowQueryThreshold; if (eventData.Duration > logQueriesLongerThan) { #pragma warning disable EF1001 diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs index 3d4759045..e16fb9f9c 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs @@ -6,7 +6,7 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.Metrics; public static class SqlQueryMetricsHelper { - public const string QueryNameTag = "QueryName"; + public const string QueryNameTag = "gcmKFRVsi3IDgKuTQT2z"; public static string GetQueryNameValue( string operationName, diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs index d281901cc..ffb2084d2 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Newtonsoft.Json; +using RadixDlt.NetworkGateway.Abstractions.Configuration; using RadixDlt.NetworkGateway.GatewayApi.Configuration; using RadixDlt.NetworkGateway.GatewayApi.Services; using RadixDlt.NetworkGateway.PostgresIntegration.Metrics; @@ -30,16 +31,16 @@ Task QueryFirstOrDefaultAsync( public class DapperWrapper : IDapperWrapper { - private readonly IOptionsMonitor _slowQueriesLoggingOptions; + private readonly SlowQueryLoggingOptions _slowQueriesLoggingOptions; private readonly ILogger _logger; private readonly ISqlQueryObserver _sqlQueryObserver; public DapperWrapper( - IOptionsMonitor slowQueriesLoggingOptions, + IOptionsSnapshot slowQueriesLoggingOptions, ILogger logger, ISqlQueryObserver sqlQueryObserver) { - _slowQueriesLoggingOptions = slowQueriesLoggingOptions; + _slowQueriesLoggingOptions = slowQueriesLoggingOptions.Value; _logger = logger; _sqlQueryObserver = sqlQueryObserver; } @@ -59,7 +60,7 @@ public async Task> QueryAsync( _sqlQueryObserver.OnSqlQueryExecuted(queryName, elapsed); - var logQueriesLongerThan = _slowQueriesLoggingOptions.CurrentValue.SlowQueryThreshold; + var logQueriesLongerThan = _slowQueriesLoggingOptions.SlowQueryThreshold; if (elapsed > logQueriesLongerThan) { var parameters = JsonConvert.SerializeObject(command.Parameters); @@ -85,7 +86,7 @@ public async Task QueryFirstOrDefaultAsync( var queryName = SqlQueryMetricsHelper.GetQueryNameValue(operationName, methodName); _sqlQueryObserver.OnSqlQueryExecuted(queryName, elapsed); - var logQueriesLongerThan = _slowQueriesLoggingOptions.CurrentValue.SlowQueryThreshold; + var logQueriesLongerThan = _slowQueriesLoggingOptions.SlowQueryThreshold; if (elapsed > logQueriesLongerThan) { var parameters = JsonConvert.SerializeObject(command.Parameters); From 8819627366ca10655fdcde7d10648a59b13ca54d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Wed, 27 Sep 2023 11:43:45 +0200 Subject: [PATCH 17/22] keep metrics interceptor as singleton. --- .../DataAggregatorBuilderExtensions.cs | 4 ++-- .../GatewayApiBuilderExtensions.cs | 2 +- .../Interceptors/MetricsInterceptor.cs | 9 ++++----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/DataAggregatorBuilderExtensions.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/DataAggregatorBuilderExtensions.cs index e90cc7632..d7e757b95 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/DataAggregatorBuilderExtensions.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/DataAggregatorBuilderExtensions.cs @@ -93,8 +93,8 @@ public static DataAggregatorBuilder AddPostgresPersistenceCore(this DataAggregat .AddSingleton() .AddSingleton() .AddSingleton() - .AddScoped() - .AddScoped(); + .AddSingleton() + .AddScoped(); CustomTypes.EnsureConfigured(); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayApiBuilderExtensions.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayApiBuilderExtensions.cs index c68915dc9..92ffedc71 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayApiBuilderExtensions.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayApiBuilderExtensions.cs @@ -97,7 +97,7 @@ public static GatewayApiBuilder AddPostgresPersistenceCore(this GatewayApiBuilde .AddScoped() .AddScoped() .AddScoped() - .AddScoped(); + .AddSingleton(); CustomTypes.EnsureConfigured(); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs index 5728b7e63..8812436ff 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs @@ -3,7 +3,6 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using RadixDlt.NetworkGateway.Abstractions.Configuration; -using RadixDlt.NetworkGateway.GatewayApi.Configuration; using RadixDlt.NetworkGateway.GatewayApi.Services; using RadixDlt.NetworkGateway.PostgresIntegration.Metrics; using System; @@ -15,13 +14,13 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; internal class MetricsInterceptor : DbCommandInterceptor { - private readonly SlowQueryLoggingOptions _slowQueriesLoggingOptions; + private readonly IOptionsMonitor _slowQueriesLoggingOptions; private readonly ILogger _logger; private readonly ISqlQueryObserver _sqlQueryObserver; - public MetricsInterceptor(ILogger logger, IOptionsSnapshot slowQueriesLoggingOptions, ISqlQueryObserver sqlQueryObserver) + public MetricsInterceptor(ILogger logger, IOptionsMonitor slowQueriesLoggingOptions, ISqlQueryObserver sqlQueryObserver) { - _slowQueriesLoggingOptions = slowQueriesLoggingOptions.Value; + _slowQueriesLoggingOptions = slowQueriesLoggingOptions; _sqlQueryObserver = sqlQueryObserver; _logger = logger; } @@ -50,7 +49,7 @@ private void Observe( _sqlQueryObserver.OnSqlQueryExecuted(queryName, eventData.Duration); - var logQueriesLongerThan = _slowQueriesLoggingOptions.SlowQueryThreshold; + var logQueriesLongerThan = _slowQueriesLoggingOptions.CurrentValue.SlowQueryThreshold; if (eventData.Duration > logQueriesLongerThan) { #pragma warning disable EF1001 From 64147d22051efd99341829080df9417ef3b85e49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Wed, 27 Sep 2023 11:55:05 +0200 Subject: [PATCH 18/22] fix extracting query name. --- .../Interceptors/MetricsInterceptor.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs index 8812436ff..b4bff6c2e 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs @@ -66,8 +66,9 @@ private string GetQueryName(DbCommand dbCommand) { const string UnknownQueryName = "UNKNOWN"; - var startOfTag = dbCommand.CommandText.IndexOf($"{SqlQueryMetricsHelper.QueryNameTag}<", StringComparison.InvariantCultureIgnoreCase); - var endOfTag = dbCommand.CommandText.IndexOf(">", StringComparison.InvariantCultureIgnoreCase); + var queryNameStartTag = $"{SqlQueryMetricsHelper.QueryNameTag}<"; + var startOfTag = dbCommand.CommandText.IndexOf(queryNameStartTag, StringComparison.InvariantCultureIgnoreCase); + var endOfTag = dbCommand.CommandText.IndexOf(">;", StringComparison.InvariantCultureIgnoreCase); if (startOfTag < 0 || endOfTag < 0) { @@ -75,7 +76,8 @@ private string GetQueryName(DbCommand dbCommand) return UnknownQueryName; } - var queryName = dbCommand.CommandText.Substring(startOfTag, endOfTag); + var from = startOfTag + queryNameStartTag.Length; + var queryName = dbCommand.CommandText[from..endOfTag]; if (string.IsNullOrEmpty(queryName)) { From 41a250a15e06b0a42d74a7b500bd75e41a5cf063 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20=C5=81abu=C5=9B?= Date: Wed, 27 Sep 2023 11:25:52 +0200 Subject: [PATCH 19/22] Optimized ledger_transactions table usage --- .../Services/CommittedStateIdentifiersReader.cs | 10 +++++++++- .../Services/LedgerStateQuerier.cs | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/CommittedStateIdentifiersReader.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/CommittedStateIdentifiersReader.cs index 06523703c..de43eedb0 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/CommittedStateIdentifiersReader.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/CommittedStateIdentifiersReader.cs @@ -64,6 +64,7 @@ using Microsoft.EntityFrameworkCore; using RadixDlt.NetworkGateway.DataAggregator.Services; +using System.Linq; using System.Threading; using System.Threading.Tasks; using GatewayModel = RadixDlt.NetworkGateway.Abstractions; @@ -84,7 +85,14 @@ public CommittedStateIdentifiersReader(IDbContextFactory dbC var dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); var transaction = await dbContext.LedgerTransactions - .FirstOrDefaultAsync(x => x.StateVersion == stateVersion, cancellationToken); + .Where(e => e.StateVersion == stateVersion) + .Select(e => new + { + e.StateVersion, + e.LedgerHashes, + }) + .AsNoTracking() + .FirstOrDefaultAsync(cancellationToken); if (transaction == null) { diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/LedgerStateQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/LedgerStateQuerier.cs index 02e737d23..ef37aa9cd 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/LedgerStateQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/LedgerStateQuerier.cs @@ -404,7 +404,7 @@ private async Task GetLedgerStateAfterEpochAndRound(long epoc lt.Epoch, lt.RoundInEpoch, }) - .SingleOrDefaultAsync(token); + .FirstOrDefaultAsync(token); return lt == null ? null From 81f38cbad7cbc724e758d05a17035e744e624abf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Wed, 27 Sep 2023 12:00:01 +0200 Subject: [PATCH 20/22] fix validating tag. --- .../Metrics/SqlQueryMetricsHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs index e16fb9f9c..412c077e5 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs @@ -41,6 +41,6 @@ public static string GenerateQueryNameTag(string operationName, string methodNam private static bool IsValidTagPart(string value) { - return value.All(c => char.IsLetterOrDigit(c) || c == '_' || c == '_'); + return value.All(c => char.IsLetterOrDigit(c) || c == '_' || c == '-'); } } From e42d05293ebc7717daff0ceb1341e6f05a6ee963 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20=C5=81abu=C5=9B?= Date: Wed, 27 Sep 2023 13:46:24 +0200 Subject: [PATCH 21/22] Dropped ORDER BY / LIMIT from pending TX payload queries --- .../Services/TransactionQuerier.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs index 07082047b..95a09e563 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs @@ -581,8 +581,6 @@ private async Task> LookupPendingTransact return await _rwDbContext .PendingTransactions .Where(pt => pt.IntentHash == intentHash) - .OrderBy(pt => pt.GatewayHandling.FirstSubmittedToGatewayTimestamp) - .Take(100) // Limit this just in case .Select(pt => new PendingTransactionSummary( pt.PayloadHash, @@ -597,6 +595,7 @@ private async Task> LookupPendingTransact ) ) .AnnotateMetricName() + .AsNoTracking() .ToListAsync(token); } From 8cf616601206a5fa5f134fd4b77d8b5b59f6df0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Wed, 27 Sep 2023 14:29:23 +0200 Subject: [PATCH 22/22] refactor tags for metrics interceptor. --- .../Interceptors/ForcedDistinctInterceptor.cs | 4 ++-- .../Interceptors/MetricsInterceptor.cs | 17 ++++++++++++----- .../Metrics/SqlQueryMetricsHelper.cs | 8 +++----- .../Services/DapperWrapper.cs | 1 - 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/ForcedDistinctInterceptor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/ForcedDistinctInterceptor.cs index afc64332d..b32409f55 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/ForcedDistinctInterceptor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/ForcedDistinctInterceptor.cs @@ -72,9 +72,9 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; internal class ForceDistinctInterceptor : DbCommandInterceptor { - /// Query marked used to trigger interceptor. + /// Query marker used to trigger interceptor. /// Value has no meaning at all, it should be understood as opaque query marker. - public const string Apply = nameof(ForceDistinctInterceptor) + ":3c49f785-0598-462a-ba88-bcdbed969709-f66acff3-fd40-4fbd-8eb9-151aefcc5711"; + public const string Apply = nameof(ForceDistinctInterceptor) + ":MelTww0Ylh4NEfxO0huHwXzbSmy1ymcQtbq0mRuy9zLFjIY6wP8dO1AVziGvI2if"; public override InterceptionResult ReaderExecuting(DbCommand command, CommandEventData eventData, InterceptionResult result) { diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs index b4bff6c2e..3d043ddc0 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Interceptors/MetricsInterceptor.cs @@ -4,7 +4,6 @@ using Microsoft.Extensions.Options; using RadixDlt.NetworkGateway.Abstractions.Configuration; using RadixDlt.NetworkGateway.GatewayApi.Services; -using RadixDlt.NetworkGateway.PostgresIntegration.Metrics; using System; using System.Data.Common; using System.Threading; @@ -14,6 +13,13 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; internal class MetricsInterceptor : DbCommandInterceptor { + public const string QueryNameStartTag = $"{nameof(MetricsInterceptor)}:{QueryNameTag}<"; + public const string QueryNameEndTag = $">{nameof(MetricsInterceptor)}:{QueryNameTag}"; + + /// EF tag used to provide query name for further metrics usage. + /// Value has no meaning at all, it should be understood as opaque query marker. + private const string QueryNameTag = "uSWntehehitXgROv7bzwhy2w8olWjn45ig1einU8sAHM75NbdkLUTn6xuChuzPBu"; + private readonly IOptionsMonitor _slowQueriesLoggingOptions; private readonly ILogger _logger; private readonly ISqlQueryObserver _sqlQueryObserver; @@ -66,9 +72,10 @@ private string GetQueryName(DbCommand dbCommand) { const string UnknownQueryName = "UNKNOWN"; - var queryNameStartTag = $"{SqlQueryMetricsHelper.QueryNameTag}<"; - var startOfTag = dbCommand.CommandText.IndexOf(queryNameStartTag, StringComparison.InvariantCultureIgnoreCase); - var endOfTag = dbCommand.CommandText.IndexOf(">;", StringComparison.InvariantCultureIgnoreCase); + var startOfTag = dbCommand.CommandText.IndexOf(QueryNameStartTag, StringComparison.InvariantCultureIgnoreCase); + + var searchForEndTagFrom = startOfTag + QueryNameStartTag.Length; + var endOfTag = dbCommand.CommandText.IndexOf(QueryNameEndTag, searchForEndTagFrom, StringComparison.InvariantCultureIgnoreCase); if (startOfTag < 0 || endOfTag < 0) { @@ -76,7 +83,7 @@ private string GetQueryName(DbCommand dbCommand) return UnknownQueryName; } - var from = startOfTag + queryNameStartTag.Length; + var from = startOfTag + QueryNameStartTag.Length; var queryName = dbCommand.CommandText[from..endOfTag]; if (string.IsNullOrEmpty(queryName)) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs index 412c077e5..f95f619c6 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Metrics/SqlQueryMetricsHelper.cs @@ -1,13 +1,11 @@ -using System; -using System.IO; +using RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; +using System; using System.Linq; namespace RadixDlt.NetworkGateway.PostgresIntegration.Metrics; public static class SqlQueryMetricsHelper { - public const string QueryNameTag = "gcmKFRVsi3IDgKuTQT2z"; - public static string GetQueryNameValue( string operationName, string methodName) @@ -36,7 +34,7 @@ public static string GetQueryNameValue( public static string GenerateQueryNameTag(string operationName, string methodName) { var queryName = GetQueryNameValue(operationName, methodName); - return $"{QueryNameTag}<{queryName}>;"; + return $"{MetricsInterceptor.QueryNameStartTag}{queryName}{MetricsInterceptor.QueryNameEndTag}"; } private static bool IsValidTagPart(string value) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs index ffb2084d2..a448fb894 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapper.cs @@ -3,7 +3,6 @@ using Microsoft.Extensions.Options; using Newtonsoft.Json; using RadixDlt.NetworkGateway.Abstractions.Configuration; -using RadixDlt.NetworkGateway.GatewayApi.Configuration; using RadixDlt.NetworkGateway.GatewayApi.Services; using RadixDlt.NetworkGateway.PostgresIntegration.Metrics; using System.Collections.Generic;