From cb604318552c4755afcd8757661f0fb5391d8687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Fri, 19 Jan 2024 11:04:59 +0100 Subject: [PATCH] add support to delete package code. --- .../LedgerExtension/PackageCodeAggregator.cs | 39 +++++++++++++------ .../PostgresLedgerExtenderService.cs | 26 +++++++++++++ .../LedgerExtension/WriteHelper.cs | 3 +- ...9095226_SupportProtocolUpdate.Designer.cs} | 6 ++- ...> 20240119095226_SupportProtocolUpdate.cs} | 25 ++++++++++-- .../Migrations/IdempotentApplyMigrations.sql | 25 +++++++----- .../MigrationsDbContextModelSnapshot.cs | 4 ++ .../Models/PackageCodeHistory.cs | 3 ++ .../ScryptoSborUtils.cs | 7 ++++ .../Services/EntityStateQuerier.cs | 10 ++--- 10 files changed, 117 insertions(+), 31 deletions(-) rename src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/{20240118114735_SupportProtocolUpdate.Designer.cs => 20240119095226_SupportProtocolUpdate.Designer.cs} (99%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/{20240118114735_SupportProtocolUpdate.cs => 20240119095226_SupportProtocolUpdate.cs} (94%) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PackageCodeAggregator.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PackageCodeAggregator.cs index 27e7de942..a08f99987 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PackageCodeAggregator.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PackageCodeAggregator.cs @@ -66,6 +66,7 @@ using RadixDlt.NetworkGateway.Abstractions.Extensions; using RadixDlt.NetworkGateway.PostgresIntegration.Models; using System.Collections.Generic; +using System.Diagnostics; using CoreModel = RadixDlt.CoreApiSdk.Model; namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; @@ -77,6 +78,10 @@ internal record PackageCodeChange(long StateVersion) public CoreModel.PackageCodeOriginalCodeEntrySubstate? PackageCodeOriginalCode { get; set; } public CoreModel.PackageCodeVmTypeEntrySubstate? PackageCodeVmType { get; set; } + + public bool CodeVmTypeIsDeleted { get; set; } = false; + + public bool PackageCodeIsDeleted { get; set; } = false; } internal static class PackageCodeAggregator @@ -144,17 +149,29 @@ public static (List PackageCodeHistoryToAdd, List ProcessTransactions(ReadWriteDbContext db vaultSnapshots.Add(new NonFungibleVaultSnapshot(referencedEntity, resourceEntity, simpleRep, true, stateVersion)); } + + if (substateId.SubstateType == CoreModel.SubstateType.PackageCodeVmTypeEntry) + { + var keyHex = (substateId.SubstateKey as CoreModel.MapSubstateKey)!.KeyHex; + var code_hash = ScryptoSborUtils.DataToProgrammaticScryptoSborValueBytes(keyHex.ConvertFromHex(), _networkConfigurationProvider.GetNetworkId()); + + packageCodeChanges + .GetOrAdd( + new PackageCodeLookup(referencedEntity.DatabaseId, (ValueBytes)code_hash.Hex.ConvertFromHex()), + _ => new PackageCodeChange(stateVersion) + ) + .CodeVmTypeIsDeleted = true; + } + + if (substateId.SubstateType == CoreModel.SubstateType.PackageCodeOriginalCodeEntry) + { + var keyHex = (substateId.SubstateKey as CoreModel.MapSubstateKey)!.KeyHex; + var code_hash = ScryptoSborUtils.DataToProgrammaticScryptoSborValueBytes(keyHex.ConvertFromHex(), _networkConfigurationProvider.GetNetworkId()); + + packageCodeChanges + .GetOrAdd( + new PackageCodeLookup(referencedEntity.DatabaseId, (ValueBytes)code_hash.Hex.ConvertFromHex()), + _ => new PackageCodeChange(stateVersion) + ) + .PackageCodeIsDeleted = true; + } } var transaction = ledgerTransactionsToAdd.Single(x => x.StateVersion == stateVersion); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/WriteHelper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/WriteHelper.cs index 31d93da39..63086801f 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/WriteHelper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/WriteHelper.cs @@ -1123,7 +1123,7 @@ public async Task CopyPackageCodeHistory(ICollection en var sw = Stopwatch.GetTimestamp(); - await using var writer = await _connection.BeginBinaryImportAsync("COPY package_code_history (id, from_state_version, package_entity_id, code_hash, code, vm_type) FROM STDIN (FORMAT BINARY)", token); + await using var writer = await _connection.BeginBinaryImportAsync("COPY package_code_history (id, from_state_version, package_entity_id, code_hash, code, vm_type, is_deleted) FROM STDIN (FORMAT BINARY)", token); foreach (var e in entities) { @@ -1134,6 +1134,7 @@ public async Task CopyPackageCodeHistory(ICollection en await writer.WriteAsync(e.CodeHash, NpgsqlDbType.Bytea, token); await writer.WriteAsync(e.Code, NpgsqlDbType.Bytea, token); await writer.WriteAsync(e.VmType, "package_vm_type", token); + await writer.WriteAsync(e.IsDeleted, NpgsqlDbType.Boolean, token); } await writer.CompleteAsync(token); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240118114735_SupportProtocolUpdate.Designer.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240119095226_SupportProtocolUpdate.Designer.cs similarity index 99% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240118114735_SupportProtocolUpdate.Designer.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240119095226_SupportProtocolUpdate.Designer.cs index 6e63427ad..86f49c952 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240118114735_SupportProtocolUpdate.Designer.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240119095226_SupportProtocolUpdate.Designer.cs @@ -81,7 +81,7 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.Migrations { [DbContext(typeof(MigrationsDbContext))] - [Migration("20240118114735_SupportProtocolUpdate")] + [Migration("20240119095226_SupportProtocolUpdate")] partial class SupportProtocolUpdate { /// @@ -1259,6 +1259,10 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("bigint") .HasColumnName("from_state_version"); + b.Property("IsDeleted") + .HasColumnType("boolean") + .HasColumnName("is_deleted"); + b.Property("PackageEntityId") .HasColumnType("bigint") .HasColumnName("package_entity_id"); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240118114735_SupportProtocolUpdate.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240119095226_SupportProtocolUpdate.cs similarity index 94% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240118114735_SupportProtocolUpdate.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240119095226_SupportProtocolUpdate.cs index 1376ed9ca..a0cd37ab0 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240118114735_SupportProtocolUpdate.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240119095226_SupportProtocolUpdate.cs @@ -90,6 +90,13 @@ protected override void Up(MigrationBuilder migrationBuilder) nullable: false, defaultValue: PackageVmType.Native); + migrationBuilder.AddColumn( + name: "is_deleted", + table: "package_code_history", + type: "boolean", + nullable: false, + defaultValue: false); + migrationBuilder.Sql("update package_code_history pch set vm_type = (select vm_type from entities e where e.id = pch.package_entity_id)"); migrationBuilder.AlterColumn( @@ -98,6 +105,12 @@ protected override void Up(MigrationBuilder migrationBuilder) oldDefaultValue: PackageVmType.Native, defaultValue: null); + migrationBuilder.AlterColumn( + name: "is_deleted", + table: "package_code_history", + oldDefaultValue: false, + defaultValue: null); + migrationBuilder.DropColumn( name: "vm_type", table: "entities"); @@ -166,19 +179,23 @@ protected override void Down(MigrationBuilder migrationBuilder) migrationBuilder.DropTable( name: "package_code_aggregate_history"); + migrationBuilder.DropColumn( + name: "is_deleted", + table: "package_code_history"); + migrationBuilder.DropColumn( name: "vm_type", table: "package_code_history"); + migrationBuilder.AlterDatabase() + .Annotation("Npgsql:Enum:ledger_transaction_type", "genesis,user,round_update") + .OldAnnotation("Npgsql:Enum:ledger_transaction_type", "genesis,user,round_update,flash"); + migrationBuilder.AddColumn( name: "vm_type", table: "entities", type: "package_vm_type", nullable: true); - - migrationBuilder.AlterDatabase() - .Annotation("Npgsql:Enum:ledger_transaction_type", "genesis,user,round_update") - .OldAnnotation("Npgsql:Enum:ledger_transaction_type", "genesis,user,round_update,flash"); } } } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/IdempotentApplyMigrations.sql b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/IdempotentApplyMigrations.sql index fd4cd0bde..b69e5d09c 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/IdempotentApplyMigrations.sql +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/IdempotentApplyMigrations.sql @@ -1099,28 +1099,35 @@ START TRANSACTION; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240118114735_SupportProtocolUpdate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240119095226_SupportProtocolUpdate') THEN ALTER TABLE entities DROP COLUMN vm_type; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240118114735_SupportProtocolUpdate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240119095226_SupportProtocolUpdate') THEN ALTER TYPE ledger_transaction_type ADD VALUE 'flash'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240118114735_SupportProtocolUpdate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240119095226_SupportProtocolUpdate') THEN + ALTER TABLE package_code_history ADD is_deleted boolean NOT NULL DEFAULT FALSE; + END IF; +END $EF$; + +DO $EF$ +BEGIN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240119095226_SupportProtocolUpdate') THEN ALTER TABLE package_code_history ADD vm_type package_vm_type NOT NULL DEFAULT 'native'::package_vm_type; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240118114735_SupportProtocolUpdate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240119095226_SupportProtocolUpdate') THEN CREATE TABLE package_blueprint_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -1133,7 +1140,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240118114735_SupportProtocolUpdate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240119095226_SupportProtocolUpdate') THEN CREATE TABLE package_code_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -1146,23 +1153,23 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240118114735_SupportProtocolUpdate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240119095226_SupportProtocolUpdate') THEN CREATE INDEX "IX_package_blueprint_aggregate_history_package_entity_id_from_~" ON package_blueprint_aggregate_history (package_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240118114735_SupportProtocolUpdate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240119095226_SupportProtocolUpdate') THEN CREATE INDEX "IX_package_code_aggregate_history_package_entity_id_from_state~" ON package_code_aggregate_history (package_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240118114735_SupportProtocolUpdate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240119095226_SupportProtocolUpdate') THEN INSERT INTO "__EFMigrationsHistory" ("MigrationId", "ProductVersion") - VALUES ('20240118114735_SupportProtocolUpdate', '7.0.11'); + VALUES ('20240119095226_SupportProtocolUpdate', '7.0.11'); END IF; END $EF$; COMMIT; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs index 17d3b3522..ac091e190 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs @@ -1256,6 +1256,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("bigint") .HasColumnName("from_state_version"); + b.Property("IsDeleted") + .HasColumnType("boolean") + .HasColumnName("is_deleted"); + b.Property("PackageEntityId") .HasColumnType("bigint") .HasColumnName("package_entity_id"); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/PackageCodeHistory.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/PackageCodeHistory.cs index d02551d8c..9e448a8ef 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/PackageCodeHistory.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/PackageCodeHistory.cs @@ -89,4 +89,7 @@ internal class PackageCodeHistory [Column("vm_type")] public PackageVmType VmType { get; set; } + + [Column("is_deleted")] + public bool IsDeleted { get; set; } } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/ScryptoSborUtils.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/ScryptoSborUtils.cs index c1394662a..6aa42f201 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/ScryptoSborUtils.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/ScryptoSborUtils.cs @@ -109,6 +109,13 @@ public static GatewayModel.ProgrammaticScryptoSborValue DataToProgrammaticJson(b return JsonConvert.DeserializeObject(json) ?? throw new ArgumentException("Invalid input Scrypto SBOR"); } + public static GatewayModel.ProgrammaticScryptoSborValueBytes DataToProgrammaticScryptoSborValueBytes(byte[] rawScryptoSbor, byte networkId) + { + var json = ToolkitModel.RadixEngineToolkitUniffiMethods.ScryptoSborDecodeToStringRepresentation(rawScryptoSbor, ToolkitModel.SerializationMode.PROGRAMMATIC, networkId, null); + + return JsonConvert.DeserializeObject(json) ?? throw new ArgumentException("Invalid input Scrypto SBOR"); + } + public static GatewayModel.MetadataTypedValue DecodeToGatewayMetadataItemValue(byte[] rawScryptoSbor, byte networkId) { using var metadataValue = ToolkitModel.RadixEngineToolkitUniffiMethods.MetadataSborDecode(rawScryptoSbor, networkId); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs index 1589a6f64..84bf37771 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs @@ -238,10 +238,10 @@ public EntityStateQuerier( if (packageCodeHistory.TryGetValue(pe.Id, out var packageCodes)) { codeItems.AddRange(packageCodes.Select(pb => new GatewayModel.PackageCodeCollectionItem( - vmType: pb.VmType.ToGatewayModel(), - codeHashHex: pb.CodeHash.ToHex(), - codeHex: pb.Code.ToHex() - ))); + vmType: pb.VmType.ToGatewayModel(), + codeHashHex: pb.CodeHash.ToHex(), + codeHex: pb.Code.ToHex() + ))); } if (packageSchemaHistory.TryGetValue(pe.Id, out var packageSchemas)) @@ -1506,7 +1506,7 @@ LIMIT 1 SELECT pch.* FROM most_recent_package_codes mrpc INNER JOIN package_code_history pch -ON pch.id = ANY(mrpc.package_code_ids) +ON pch.id = ANY(mrpc.package_code_ids) AND pch.is_deleted = false ORDER BY array_position(mrpc.package_code_ids, pch.id)") .AnnotateMetricName() .ToListAsync(token))