From 24e365a8fb56ef6ce1c6c4963a7c7a44c42ea2c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Wed, 5 Mar 2025 13:05:24 +0100 Subject: [PATCH 1/3] fix transaction stream performance edge case. --- .../Services/TransactionQuerier.cs | 109 +++++++++--------- 1 file changed, 57 insertions(+), 52 deletions(-) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs index f9f533fad..104ed849a 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs @@ -158,17 +158,7 @@ public async Task GetTransactionStream(TransactionS ? request.Cursor?.StateVersionBoundary ?? request.FromStateVersion : request.FromStateVersion; - var baseQuery = _dbContext - .LedgerTransactions - .Where(lt => lt.StateVersion <= upperStateVersion && lt.StateVersion >= (lowerStateVersion ?? lt.StateVersion)); - - var searchQuery = request.SearchCriteria.Status switch - { - LedgerTransactionStatusFilter.All => baseQuery.Select(lt => lt.StateVersion), - LedgerTransactionStatusFilter.Success => baseQuery.Where(x => x.ReceiptStatus == LedgerTransactionStatus.Succeeded).Select(lt => lt.StateVersion), - LedgerTransactionStatusFilter.Failure => baseQuery.Where(x => x.ReceiptStatus == LedgerTransactionStatus.Failed).Select(lt => lt.StateVersion), - _ => throw new NotSupportedException($"Not supported status: {request.SearchCriteria.Status}"), - }; + IQueryable? searchQuery = null; var userKindFilterImplicitlyApplied = false; @@ -196,12 +186,7 @@ public async Task GetTransactionStream(TransactionS return TransactionPageWithoutTotal.Empty; } - searchQuery = searchQuery - .Join( - _dbContext.LedgerTransactionMarkers, - stateVersion => stateVersion, - ledgerTransactionMarker => ledgerTransactionMarker.StateVersion, - (stateVersion, ledgerTransactionMarker) => ledgerTransactionMarker) + searchQuery = PrepareForNextFilter(searchQuery) .OfType() .Where(maltm => maltm.EntityId == entityId) .Where(maltm => maltm.StateVersion <= upperStateVersion && maltm.StateVersion >= (lowerStateVersion ?? maltm.StateVersion)) @@ -263,8 +248,7 @@ public async Task GetTransactionStream(TransactionS return TransactionPageWithoutTotal.Empty; } - searchQuery = searchQuery - .Join(_dbContext.LedgerTransactionMarkers, sv => sv, ltm => ltm.StateVersion, (sv, ltm) => ltm) + searchQuery = PrepareForNextFilter(searchQuery) .OfType() .Where(agetm => agetm.EntityId == entityId) .Where(agetm => agetm.StateVersion <= upperStateVersion && agetm.StateVersion >= (lowerStateVersion ?? agetm.StateVersion)) @@ -281,8 +265,7 @@ public async Task GetTransactionStream(TransactionS return TransactionPageWithoutTotal.Empty; } - searchQuery = searchQuery - .Join(_dbContext.LedgerTransactionMarkers, sv => sv, ltm => ltm.StateVersion, (sv, ltm) => ltm) + searchQuery = PrepareForNextFilter(searchQuery) .OfType() .Where(agetm => agetm.EntityId == entityId) .Where(agetm => agetm.StateVersion <= upperStateVersion && agetm.StateVersion >= (lowerStateVersion ?? agetm.StateVersion)) @@ -326,8 +309,7 @@ public async Task GetTransactionStream(TransactionS eventResourceEntityId = id; } - searchQuery = searchQuery - .Join(_dbContext.LedgerTransactionMarkers, sv => sv, ltm => ltm.StateVersion, (sv, ltm) => ltm) + searchQuery = PrepareForNextFilter(searchQuery) .OfType() .Where(eltm => eltm.EventType == eventType && eltm.EntityId == (eventEmitterEntityId ?? eltm.EntityId) && eltm.ResourceEntityId == (eventResourceEntityId ?? eltm.ResourceEntityId)) .Where(eltm => eltm.StateVersion <= upperStateVersion && eltm.StateVersion >= (lowerStateVersion ?? eltm.StateVersion)) @@ -352,8 +334,7 @@ public async Task GetTransactionStream(TransactionS _ => throw new UnreachableException($"Didn't expect {request.SearchCriteria.ManifestClassFilter.Class} value"), }; - searchQuery = searchQuery - .Join(_dbContext.LedgerTransactionMarkers, sv => sv, ltm => ltm.StateVersion, (sv, ltm) => ltm) + searchQuery = PrepareForNextFilter(searchQuery) .OfType() .Where(ttm => ttm.ManifestClass == manifestClass) .Where(ttm => (request.SearchCriteria.ManifestClassFilter.MatchOnlyMostSpecificType && ttm.IsMostSpecific) || !request.SearchCriteria.ManifestClassFilter.MatchOnlyMostSpecificType) @@ -392,6 +373,8 @@ public async Task GetTransactionStream(TransactionS .Where(maltm => maltm.StateVersion <= upperStateVersion && maltm.StateVersion >= (lowerStateVersion ?? maltm.StateVersion)) .Select(y => y.StateVersion); + searchQuery ??= _dbContext.LedgerTransactionMarkers.Select(x => x.StateVersion); + searchQuery = searchQuery.Where(x => withManifestOwnerCall.All(y => y != x)); } } @@ -406,8 +389,7 @@ public async Task GetTransactionStream(TransactionS } else if (request.SearchCriteria.Kind == LedgerTransactionKindFilter.UserOnly) { - searchQuery = searchQuery - .Join(_dbContext.LedgerTransactionMarkers, sv => sv, ltm => ltm.StateVersion, (sv, ltm) => ltm) + searchQuery = PrepareForNextFilter(searchQuery) .OfType() .Where(oltm => oltm.TransactionType == LedgerTransactionMarkerTransactionType.User) .Where(oltm => oltm.StateVersion <= upperStateVersion && oltm.StateVersion >= (lowerStateVersion ?? oltm.StateVersion)) @@ -415,22 +397,34 @@ public async Task GetTransactionStream(TransactionS } else if (request.SearchCriteria.Kind == LedgerTransactionKindFilter.EpochChangeOnly) { - searchQuery = searchQuery - .Join(_dbContext.LedgerTransactionMarkers, sv => sv, ltm => ltm.StateVersion, (sv, ltm) => ltm) + searchQuery = PrepareForNextFilter(searchQuery) .OfType() .Where(ecltm => ecltm.EpochChange) .Where(ecltm => ecltm.StateVersion <= upperStateVersion && ecltm.StateVersion >= (lowerStateVersion ?? ecltm.StateVersion)) .Select(ecltm => ecltm.StateVersion); } - if (request.AscendingOrder) + var query = searchQuery == null ? + _dbContext.LedgerTransactions + : searchQuery.Join( + _dbContext.LedgerTransactions, + stateVersion => stateVersion, + ledgerTransactionMarker => ledgerTransactionMarker.StateVersion, + (stateVersion, ledgerTransactionMarker) => ledgerTransactionMarker); + + searchQuery = request.SearchCriteria.Status switch { - searchQuery = searchQuery.OrderBy(sv => sv); - } - else + LedgerTransactionStatusFilter.All => query.Select(lt => lt.StateVersion), + LedgerTransactionStatusFilter.Success => query.Where(x => x.ReceiptStatus == LedgerTransactionStatus.Succeeded).Select(lt => lt.StateVersion), + LedgerTransactionStatusFilter.Failure => query.Where(x => x.ReceiptStatus == LedgerTransactionStatus.Failed).Select(lt => lt.StateVersion), + _ => throw new NotSupportedException($"Not supported status: {request.SearchCriteria.Status}"), + }; + + searchQuery = request.AscendingOrder switch { - searchQuery = searchQuery.OrderByDescending(sv => sv); - } + true => searchQuery.OrderBy(sv => sv), + _ => searchQuery.OrderByDescending(sv => sv), + }; var stateVersions = await searchQuery .TagWith(ForceDistinctInterceptor.Apply) @@ -446,19 +440,28 @@ public async Task GetTransactionStream(TransactionS return new TransactionPageWithoutTotal(nextCursor, transactions); - IQueryable ApplyLedgerTransactionMarkerOperationTypeFilter(long entityId, LedgerTransactionMarkerOperationType operationType, IQueryable query) + IQueryable ApplyLedgerTransactionMarkerOperationTypeFilter(long entityId, LedgerTransactionMarkerOperationType operationType, IQueryable? query) { - return query - .Join( - _dbContext.LedgerTransactionMarkers, - stateVersion => stateVersion, - ledgerTransactionMarker => ledgerTransactionMarker.StateVersion, - (stateVersion, ledgerTransactionMarker) => ledgerTransactionMarker) + return PrepareForNextFilter(query) .OfType() .Where(maltm => maltm.OperationType == operationType && maltm.EntityId == entityId) .Where(maltm => maltm.StateVersion <= upperStateVersion && maltm.StateVersion >= (lowerStateVersion ?? maltm.StateVersion)) .Select(maltm => maltm.StateVersion); } + + IQueryable PrepareForNextFilter(IQueryable? baseQuery) + { + if (baseQuery == null) + { + return _dbContext.LedgerTransactionMarkers; + } + + return baseQuery.Join( + _dbContext.LedgerTransactionMarkers, + stateVersion => stateVersion, + ledgerTransactionMarker => ledgerTransactionMarker.StateVersion, + (stateVersion, ledgerTransactionMarker) => ledgerTransactionMarker); + } } public async Task LookupCommittedTransaction( @@ -705,11 +708,12 @@ private async Task> LookupPendingTransact var detailedEventEmitter = new GatewayModel.PackageFunctionEventEmitter(new GatewayModel.FunctionEmitter(functionEventEmitterIdentifier.BlueprintName)); - detailedEvents.Add(new GatewayModel.DetailedEventsItem( - detailedIdentifier, - payloadTypeDefinition, - detailedEventEmitter, - payload)); + detailedEvents.Add( + new GatewayModel.DetailedEventsItem( + detailedIdentifier, + payloadTypeDefinition, + detailedEventEmitter, + payload)); } else if (parsedEmitter is CoreModel.MethodEventEmitterIdentifier methodEventEmitterIdentifier) { @@ -728,11 +732,12 @@ private async Task> LookupPendingTransact outerEmitter: eventDetails.OuterObjectAddress, globalEmitter: eventDetails.GlobalAncestorAddress ?? methodEventEmitterIdentifier.Entity.EntityAddress); - detailedEvents.Add(new GatewayModel.DetailedEventsItem( - detailedIdentifier, - payloadTypeDefinition, - detailedEventEmitter, - payload)); + detailedEvents.Add( + new GatewayModel.DetailedEventsItem( + detailedIdentifier, + payloadTypeDefinition, + detailedEventEmitter, + payload)); } else { From 69089d0da9165a46029a01ef1c3cc6f0a1990ad8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Wed, 5 Mar 2025 13:57:28 +0100 Subject: [PATCH 2/3] apply state version boundary. --- .../Services/TransactionQuerier.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs index 104ed849a..a778ac4b3 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs @@ -405,12 +405,13 @@ public async Task GetTransactionStream(TransactionS } var query = searchQuery == null ? - _dbContext.LedgerTransactions + _dbContext.LedgerTransactions.Where(lt => lt.StateVersion <= upperStateVersion && lt.StateVersion >= (lowerStateVersion ?? lt.StateVersion)) : searchQuery.Join( _dbContext.LedgerTransactions, stateVersion => stateVersion, ledgerTransactionMarker => ledgerTransactionMarker.StateVersion, - (stateVersion, ledgerTransactionMarker) => ledgerTransactionMarker); + (stateVersion, ledgerTransactionMarker) => ledgerTransactionMarker) + .Where(lt => lt.StateVersion <= upperStateVersion && lt.StateVersion >= (lowerStateVersion ?? lt.StateVersion)); searchQuery = request.SearchCriteria.Status switch { From 5aebd553517f546e6d4569b21a1ec9aa3ed5ffe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Wed, 5 Mar 2025 15:58:12 +0100 Subject: [PATCH 3/3] another micro optimization to remove extra join if possible. --- .../Services/TransactionQuerier.cs | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs index a778ac4b3..d5ce3ec6a 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs @@ -373,7 +373,9 @@ public async Task GetTransactionStream(TransactionS .Where(maltm => maltm.StateVersion <= upperStateVersion && maltm.StateVersion >= (lowerStateVersion ?? maltm.StateVersion)) .Select(y => y.StateVersion); - searchQuery ??= _dbContext.LedgerTransactionMarkers.Select(x => x.StateVersion); + searchQuery ??= _dbContext.LedgerTransactionMarkers + .Where(ltm => ltm.StateVersion <= upperStateVersion && ltm.StateVersion >= (lowerStateVersion ?? ltm.StateVersion)) + .Select(x => x.StateVersion); searchQuery = searchQuery.Where(x => withManifestOwnerCall.All(y => y != x)); } @@ -404,22 +406,25 @@ public async Task GetTransactionStream(TransactionS .Select(ecltm => ecltm.StateVersion); } - var query = searchQuery == null ? - _dbContext.LedgerTransactions.Where(lt => lt.StateVersion <= upperStateVersion && lt.StateVersion >= (lowerStateVersion ?? lt.StateVersion)) - : searchQuery.Join( - _dbContext.LedgerTransactions, - stateVersion => stateVersion, - ledgerTransactionMarker => ledgerTransactionMarker.StateVersion, - (stateVersion, ledgerTransactionMarker) => ledgerTransactionMarker) - .Where(lt => lt.StateVersion <= upperStateVersion && lt.StateVersion >= (lowerStateVersion ?? lt.StateVersion)); - - searchQuery = request.SearchCriteria.Status switch + if (searchQuery == null || request.SearchCriteria.Status != LedgerTransactionStatusFilter.All) { - LedgerTransactionStatusFilter.All => query.Select(lt => lt.StateVersion), - LedgerTransactionStatusFilter.Success => query.Where(x => x.ReceiptStatus == LedgerTransactionStatus.Succeeded).Select(lt => lt.StateVersion), - LedgerTransactionStatusFilter.Failure => query.Where(x => x.ReceiptStatus == LedgerTransactionStatus.Failed).Select(lt => lt.StateVersion), - _ => throw new NotSupportedException($"Not supported status: {request.SearchCriteria.Status}"), - }; + var query = searchQuery == null ? + _dbContext.LedgerTransactions.Where(lt => lt.StateVersion <= upperStateVersion && lt.StateVersion >= (lowerStateVersion ?? lt.StateVersion)) + : searchQuery.Join( + _dbContext.LedgerTransactions, + stateVersion => stateVersion, + ledgerTransactionMarker => ledgerTransactionMarker.StateVersion, + (stateVersion, ledgerTransactionMarker) => ledgerTransactionMarker) + .Where(lt => lt.StateVersion <= upperStateVersion && lt.StateVersion >= (lowerStateVersion ?? lt.StateVersion)); + + searchQuery = request.SearchCriteria.Status switch + { + LedgerTransactionStatusFilter.All => query.Select(lt => lt.StateVersion), + LedgerTransactionStatusFilter.Success => query.Where(x => x.ReceiptStatus == LedgerTransactionStatus.Succeeded).Select(lt => lt.StateVersion), + LedgerTransactionStatusFilter.Failure => query.Where(x => x.ReceiptStatus == LedgerTransactionStatus.Failed).Select(lt => lt.StateVersion), + _ => throw new NotSupportedException($"Not supported status: {request.SearchCriteria.Status}"), + }; + } searchQuery = request.AscendingOrder switch {