add support to delete package code.

src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PackageCodeAggregator.cs

@@ -66,6 +66,7 @@
 using RadixDlt.NetworkGateway.Abstractions.Extensions;
 using RadixDlt.NetworkGateway.PostgresIntegration.Models;
 using System.Collections.Generic;
+using System.Diagnostics;
 using CoreModel = RadixDlt.CoreApiSdk.Model;
 
 namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension;
@@ -77,6 +78,10 @@ internal record PackageCodeChange(long StateVersion)
     public CoreModel.PackageCodeOriginalCodeEntrySubstate? PackageCodeOriginalCode { get; set; }
 
     public CoreModel.PackageCodeVmTypeEntrySubstate? PackageCodeVmType { get; set; }
+
+    public bool CodeVmTypeIsDeleted { get; set; } = false;
+
+    public bool PackageCodeIsDeleted { get; set; } = false;
 }
 
 internal static class PackageCodeAggregator
@@ -144,17 +149,29 @@ public static (List<PackageCodeHistory> PackageCodeHistoryToAdd, List<PackageCod
 
                 mostRecentPackageCodeHistory[change.Key] = packageCodeHistory;
 
-                packageCodeAggregate.PackageCodeIds.Add(packageCodeHistory.Id);
-            }
-
-            if (change.Value.PackageCodeVmType != null)
-            {
-                packageCodeHistory.VmType = change.Value.PackageCodeVmType.Value.VmType.ToModel();
-            }
-
-            if (change.Value.PackageCodeOriginalCode != null)
-            {
-                packageCodeHistory.Code = change.Value.PackageCodeOriginalCode.Value.CodeHex.ConvertFromHex();
+                if (change.Value is { PackageCodeIsDeleted: true, CodeVmTypeIsDeleted: true })
+                {
+                    packageCodeHistory.IsDeleted = true;
+                }
+                else if (change.Value.PackageCodeIsDeleted != change.Value.CodeVmTypeIsDeleted)
+                {
+                    throw new UnreachableException(
+                        $"Unexpected situation where PackageCode was deleted but VmType wasn't. PackageId: {change.Key.PackageEntityId}, CodeHashHex: {change.Key.CodeHash.ToHex()}, StateVersion: {change.Value.StateVersion}");
+                }
+                else
+                {
+                    packageCodeAggregate.PackageCodeIds.Add(packageCodeHistory.Id);
+
+                    if (change.Value.PackageCodeVmType != null)
+                    {
+                        packageCodeHistory.VmType = change.Value.PackageCodeVmType.Value.VmType.ToModel();
+                    }
+
+                    if (change.Value.PackageCodeOriginalCode != null)
+                    {
+                        packageCodeHistory.Code = change.Value.PackageCodeOriginalCode.Value.CodeHex.ConvertFromHex();
+                    }
+                }
             }
 
             packageCodeHistoryToAdd.Add(packageCodeHistory);

src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs

@@ -1148,6 +1148,32 @@ private async Task<ExtendLedgerReport> ProcessTransactions(ReadWriteDbContext db
 
                             vaultSnapshots.Add(new NonFungibleVaultSnapshot(referencedEntity, resourceEntity, simpleRep, true, stateVersion));
                         }
+
+                        if (substateId.SubstateType == CoreModel.SubstateType.PackageCodeVmTypeEntry)
+                        {
+                            var keyHex = (substateId.SubstateKey as CoreModel.MapSubstateKey)!.KeyHex;
+                            var code_hash = ScryptoSborUtils.DataToProgrammaticScryptoSborValueBytes(keyHex.ConvertFromHex(), _networkConfigurationProvider.GetNetworkId());
+
+                            packageCodeChanges
+                                .GetOrAdd(
+                                    new PackageCodeLookup(referencedEntity.DatabaseId, (ValueBytes)code_hash.Hex.ConvertFromHex()),
+                                    _ => new PackageCodeChange(stateVersion)
+                                )
+                                .CodeVmTypeIsDeleted = true;
+                        }
+
+                        if (substateId.SubstateType == CoreModel.SubstateType.PackageCodeOriginalCodeEntry)
+                        {
+                            var keyHex = (substateId.SubstateKey as CoreModel.MapSubstateKey)!.KeyHex;
+                            var code_hash = ScryptoSborUtils.DataToProgrammaticScryptoSborValueBytes(keyHex.ConvertFromHex(), _networkConfigurationProvider.GetNetworkId());
+
+                            packageCodeChanges
+                                .GetOrAdd(
+                                    new PackageCodeLookup(referencedEntity.DatabaseId, (ValueBytes)code_hash.Hex.ConvertFromHex()),
+                                    _ => new PackageCodeChange(stateVersion)
+                                )
+                                .PackageCodeIsDeleted = true;
+                        }
                     }
 
                     var transaction = ledgerTransactionsToAdd.Single(x => x.StateVersion == stateVersion);

src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/WriteHelper.cs

@@ -1123,7 +1123,7 @@ public async Task<int> CopyPackageCodeHistory(ICollection<PackageCodeHistory> en
 
         var sw = Stopwatch.GetTimestamp();
 
-        await using var writer = await _connection.BeginBinaryImportAsync("COPY package_code_history (id, from_state_version, package_entity_id, code_hash, code, vm_type) FROM STDIN (FORMAT BINARY)", token);
+        await using var writer = await _connection.BeginBinaryImportAsync("COPY package_code_history (id, from_state_version, package_entity_id, code_hash, code, vm_type, is_deleted) FROM STDIN (FORMAT BINARY)", token);
 
         foreach (var e in entities)
         {
@@ -1134,6 +1134,7 @@ public async Task<int> CopyPackageCodeHistory(ICollection<PackageCodeHistory> en
             await writer.WriteAsync(e.CodeHash, NpgsqlDbType.Bytea, token);
             await writer.WriteAsync(e.Code, NpgsqlDbType.Bytea, token);
             await writer.WriteAsync(e.VmType, "package_vm_type", token);
+            await writer.WriteAsync(e.IsDeleted, NpgsqlDbType.Boolean, token);
         }
 
         await writer.CompleteAsync(token);

src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240118114735_SupportProtocolUpdate.cs → src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240119095226_SupportProtocolUpdate.cs

@@ -90,6 +90,13 @@ protected override void Up(MigrationBuilder migrationBuilder)
                 nullable: false,
                 defaultValue: PackageVmType.Native);
 
+            migrationBuilder.AddColumn<bool>(
+                name: "is_deleted",
+                table: "package_code_history",
+                type: "boolean",
+                nullable: false,
+                defaultValue: false);
+
             migrationBuilder.Sql("update package_code_history pch set vm_type = (select vm_type from entities e where e.id = pch.package_entity_id)");
 
             migrationBuilder.AlterColumn<PackageVmType>(
@@ -98,6 +105,12 @@ protected override void Up(MigrationBuilder migrationBuilder)
                 oldDefaultValue: PackageVmType.Native,
                 defaultValue: null);
 
+            migrationBuilder.AlterColumn<bool>(
+                name: "is_deleted",
+                table: "package_code_history",
+                oldDefaultValue: false,
+                defaultValue: null);
+
             migrationBuilder.DropColumn(
                 name: "vm_type",
                 table: "entities");
@@ -166,19 +179,23 @@ protected override void Down(MigrationBuilder migrationBuilder)
             migrationBuilder.DropTable(
                 name: "package_code_aggregate_history");
 
+            migrationBuilder.DropColumn(
+                name: "is_deleted",
+                table: "package_code_history");
+
             migrationBuilder.DropColumn(
                 name: "vm_type",
                 table: "package_code_history");
 
+            migrationBuilder.AlterDatabase()
+                .Annotation("Npgsql:Enum:ledger_transaction_type", "genesis,user,round_update")
+                .OldAnnotation("Npgsql:Enum:ledger_transaction_type", "genesis,user,round_update,flash");
+
             migrationBuilder.AddColumn<PackageVmType>(
                 name: "vm_type",
                 table: "entities",
                 type: "package_vm_type",
                 nullable: true);
-
-            migrationBuilder.AlterDatabase()
-                .Annotation("Npgsql:Enum:ledger_transaction_type", "genesis,user,round_update")
-                .OldAnnotation("Npgsql:Enum:ledger_transaction_type", "genesis,user,round_update,flash");
         }
     }
 }

src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/IdempotentApplyMigrations.sql

@@ -1099,28 +1099,35 @@ START TRANSACTION;
 
 DO $EF$
 BEGIN
-    IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240118114735_SupportProtocolUpdate') THEN
+    IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240119095226_SupportProtocolUpdate') THEN
     ALTER TABLE entities DROP COLUMN vm_type;
     END IF;
 END $EF$;
 
 DO $EF$
 BEGIN
-    IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240118114735_SupportProtocolUpdate') THEN
+    IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240119095226_SupportProtocolUpdate') THEN
     ALTER TYPE ledger_transaction_type ADD VALUE 'flash';
     END IF;
 END $EF$;
 
 DO $EF$
 BEGIN
-    IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240118114735_SupportProtocolUpdate') THEN
+    IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240119095226_SupportProtocolUpdate') THEN
+    ALTER TABLE package_code_history ADD is_deleted boolean NOT NULL DEFAULT FALSE;
+    END IF;
+END $EF$;
+
+DO $EF$
+BEGIN
+    IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240119095226_SupportProtocolUpdate') THEN
     ALTER TABLE package_code_history ADD vm_type package_vm_type NOT NULL DEFAULT 'native'::package_vm_type;
     END IF;
 END $EF$;
 
 DO $EF$
 BEGIN
-    IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240118114735_SupportProtocolUpdate') THEN
+    IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240119095226_SupportProtocolUpdate') THEN
     CREATE TABLE package_blueprint_aggregate_history (
         id bigint GENERATED BY DEFAULT AS IDENTITY,
         from_state_version bigint NOT NULL,
@@ -1133,7 +1140,7 @@ END $EF$;
 
 DO $EF$
 BEGIN
-    IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240118114735_SupportProtocolUpdate') THEN
+    IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240119095226_SupportProtocolUpdate') THEN
     CREATE TABLE package_code_aggregate_history (
         id bigint GENERATED BY DEFAULT AS IDENTITY,
         from_state_version bigint NOT NULL,
@@ -1146,23 +1153,23 @@ END $EF$;
 
 DO $EF$
 BEGIN
-    IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240118114735_SupportProtocolUpdate') THEN
+    IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240119095226_SupportProtocolUpdate') THEN
     CREATE INDEX "IX_package_blueprint_aggregate_history_package_entity_id_from_~" ON package_blueprint_aggregate_history (package_entity_id, from_state_version);
     END IF;
 END $EF$;
 
 DO $EF$
 BEGIN
-    IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240118114735_SupportProtocolUpdate') THEN
+    IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240119095226_SupportProtocolUpdate') THEN
     CREATE INDEX "IX_package_code_aggregate_history_package_entity_id_from_state~" ON package_code_aggregate_history (package_entity_id, from_state_version);
     END IF;
 END $EF$;
 
 DO $EF$
 BEGIN
-    IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240118114735_SupportProtocolUpdate') THEN
+    IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240119095226_SupportProtocolUpdate') THEN
     INSERT INTO "__EFMigrationsHistory" ("MigrationId", "ProductVersion")
-    VALUES ('20240118114735_SupportProtocolUpdate', '7.0.11');
+    VALUES ('20240119095226_SupportProtocolUpdate', '7.0.11');
     END IF;
 END $EF$;
 COMMIT;

src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs

@@ -1256,6 +1256,10 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                         .HasColumnType("bigint")
                         .HasColumnName("from_state_version");
 
+                    b.Property<bool>("IsDeleted")
+                        .HasColumnType("boolean")
+                        .HasColumnName("is_deleted");
+
                     b.Property<long>("PackageEntityId")
                         .HasColumnType("bigint")
                         .HasColumnName("package_entity_id");

src/RadixDlt.NetworkGateway.PostgresIntegration/Models/PackageCodeHistory.cs

@@ -89,4 +89,7 @@ internal class PackageCodeHistory
 
     [Column("vm_type")]
     public PackageVmType VmType { get; set; }
+
+    [Column("is_deleted")]
+    public bool IsDeleted { get; set; }
 }

src/RadixDlt.NetworkGateway.PostgresIntegration/ScryptoSborUtils.cs

@@ -109,6 +109,13 @@ public static GatewayModel.ProgrammaticScryptoSborValue DataToProgrammaticJson(b
         return JsonConvert.DeserializeObject<GatewayModel.ProgrammaticScryptoSborValue>(json) ?? throw new ArgumentException("Invalid input Scrypto SBOR");
     }
 
+    public static GatewayModel.ProgrammaticScryptoSborValueBytes DataToProgrammaticScryptoSborValueBytes(byte[] rawScryptoSbor, byte networkId)
+    {
+        var json = ToolkitModel.RadixEngineToolkitUniffiMethods.ScryptoSborDecodeToStringRepresentation(rawScryptoSbor, ToolkitModel.SerializationMode.PROGRAMMATIC, networkId, null);
+
+        return JsonConvert.DeserializeObject<GatewayModel.ProgrammaticScryptoSborValueBytes>(json) ?? throw new ArgumentException("Invalid input Scrypto SBOR");
+    }
+
     public static GatewayModel.MetadataTypedValue DecodeToGatewayMetadataItemValue(byte[] rawScryptoSbor, byte networkId)
     {
         using var metadataValue = ToolkitModel.RadixEngineToolkitUniffiMethods.MetadataSborDecode(rawScryptoSbor, networkId);

src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs

@@ -238,10 +238,10 @@ public EntityStateQuerier(
                     if (packageCodeHistory.TryGetValue(pe.Id, out var packageCodes))
                     {
                         codeItems.AddRange(packageCodes.Select(pb => new GatewayModel.PackageCodeCollectionItem(
-                            vmType: pb.VmType.ToGatewayModel(),
-                            codeHashHex: pb.CodeHash.ToHex(),
-                            codeHex: pb.Code.ToHex()
-                        )));
+                                vmType: pb.VmType.ToGatewayModel(),
+                                codeHashHex: pb.CodeHash.ToHex(),
+                                codeHex: pb.Code.ToHex()
+                            )));
                     }
 
                     if (packageSchemaHistory.TryGetValue(pe.Id, out var packageSchemas))
@@ -1506,7 +1506,7 @@ LIMIT 1
 SELECT pch.*
 FROM most_recent_package_codes mrpc
 INNER JOIN package_code_history pch
-ON pch.id = ANY(mrpc.package_code_ids)
+ON pch.id = ANY(mrpc.package_code_ids) AND pch.is_deleted = false
 ORDER BY array_position(mrpc.package_code_ids, pch.id)")
                 .AnnotateMetricName()
                 .ToListAsync(token))