From 0552d7b2e9b02b31493e8009e907bac7b34c5160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Fri, 20 Sep 2024 16:23:53 +0200 Subject: [PATCH 1/5] refactor ledger extender service into separate services. --- babylon-gateway.sln.DotSettings | 1 + .../LedgerExtension/IProcessor.cs | 46 ++ .../LedgerExtension/IWriteHelper.cs | 2 + .../PostgresLedgerExtenderService.cs | 757 ++++-------------- .../AccountAuthorizedDepositorsProcessor.cs | 68 +- .../AccountDefaultDepositRuleProcessor.cs | 25 +- .../AccountLockerProcessor.cs | 12 +- ...AccountResourcePreferenceRulesProcessor.cs | 24 +- .../ComponentMethodRoyaltyProcessor.cs | 27 +- .../EntityMetadataProcessor.cs | 6 +- .../EntityResourceProcessor.cs | 10 +- .../EntityRoleAssignmentProcessor.cs | 38 +- .../{ => Processors}/EntitySchemaProcessor.cs | 123 ++- .../{ => Processors}/EntityStateProcessor.cs | 126 +-- .../KeyValueStoreProcessor.cs | 35 +- .../AffectedGlobalEntitiesProcessor.cs | 42 +- .../EventLedgerTransactionMarkerProcessor.cs | 80 ++ .../GlobalEventEmitterProcessor.cs | 32 +- .../LedgerTransactionMarkersProcessor.cs | 168 ++++ .../LedgerTransactionProcessor.cs | 254 ++++++ .../ManifestProcessor.cs | 171 ++++ .../OriginLedgerTransactionMarkerProcessor.cs | 46 ++ .../PackageBlueprintProcessor.cs | 47 +- .../{ => Processors}/PackageCodeProcessor.cs | 51 +- .../Processors/ResourceSupplyProcessor.cs | 207 +++++ .../StandardMetadataProcessor.cs | 155 ++-- .../ValidatorEmissionProcessor.cs | 33 +- .../{ => Processors}/ValidatorProcessor.cs | 74 +- .../{ => Processors}/VaultProcessor.cs | 10 +- .../LedgerExtension/ReadHelper.cs | 34 - .../LedgerExtension/WriteHelper.cs | 292 +------ .../Models/Entity.cs | 1 + 32 files changed, 1696 insertions(+), 1301 deletions(-) create mode 100644 src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/IProcessor.cs rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/{ => Processors}/AccountAuthorizedDepositorsProcessor.cs (90%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/{ => Processors}/AccountDefaultDepositRuleProcessor.cs (90%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/{ => Processors}/AccountLockerProcessor.cs (97%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/{ => Processors}/AccountResourcePreferenceRulesProcessor.cs (94%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/{ => Processors}/ComponentMethodRoyaltyProcessor.cs (93%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/{ => Processors}/EntityMetadataProcessor.cs (98%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/{ => Processors}/EntityResourceProcessor.cs (98%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/{ => Processors}/EntityRoleAssignmentProcessor.cs (90%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/{ => Processors}/EntitySchemaProcessor.cs (61%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/{ => Processors}/EntityStateProcessor.cs (72%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/{ => Processors}/KeyValueStoreProcessor.cs (90%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/{ => Processors/LedgerTransactionMarkers}/AffectedGlobalEntitiesProcessor.cs (79%) create mode 100644 src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/EventLedgerTransactionMarkerProcessor.cs rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/{ => Processors/LedgerTransactionMarkers}/GlobalEventEmitterProcessor.cs (88%) create mode 100644 src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/LedgerTransactionMarkersProcessor.cs create mode 100644 src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/LedgerTransactionProcessor.cs create mode 100644 src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/ManifestProcessor.cs create mode 100644 src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/OriginLedgerTransactionMarkerProcessor.cs rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/{ => Processors}/PackageBlueprintProcessor.cs (85%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/{ => Processors}/PackageCodeProcessor.cs (86%) create mode 100644 src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/ResourceSupplyProcessor.cs rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/{ => Processors}/StandardMetadataProcessor.cs (72%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/{ => Processors}/ValidatorEmissionProcessor.cs (88%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/{ => Processors}/ValidatorProcessor.cs (77%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/{ => Processors}/VaultProcessor.cs (98%) diff --git a/babylon-gateway.sln.DotSettings b/babylon-gateway.sln.DotSettings index 99b35ffac..f32a80e43 100644 --- a/babylon-gateway.sln.DotSettings +++ b/babylon-gateway.sln.DotSettings @@ -3,5 +3,6 @@ True True True + True True True \ No newline at end of file diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/IProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/IProcessor.cs new file mode 100644 index 000000000..eb54b607d --- /dev/null +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/IProcessor.cs @@ -0,0 +1,46 @@ +using RadixDlt.NetworkGateway.PostgresIntegration.Models; +using System.Collections.Generic; +using System.Threading.Tasks; +using CoreModel = RadixDlt.CoreApiSdk.Model; +using ToolkitModel = RadixEngineToolkit; + +namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; + +internal interface IProcessorBase +{ + Task LoadDependenciesAsync(); + + void ProcessChanges(); + + Task SaveEntitiesAsync(); +} + +internal interface IDecodedEventProcessor +{ + void VisitDecodedEvent(ToolkitModel.TypedNativeEvent decodedEvent, ReferencedEntity eventEmitterEntity, long stateVersion); +} + +internal interface ITransactionProcessor +{ + void VisitTransaction(CoreModel.CommittedTransaction transaction, long stateVersion); +} + +internal interface IEventProcessor +{ + void VisitEvent(CoreModel.Event @event, long stateVersion); +} + +internal interface ISubstateUpsertProcessor +{ + void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion); +} + +internal interface ISubstateDeleteProcessor +{ + void VisitDelete(CoreModel.SubstateId substateId, ReferencedEntity referencedEntity, long stateVersion); +} + +internal interface ITransactionMarkerProcessor +{ + IEnumerable CreateTransactionMarkers(); +} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/IWriteHelper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/IWriteHelper.cs index 57aad406d..d99b49ff4 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/IWriteHelper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/IWriteHelper.cs @@ -76,4 +76,6 @@ internal interface IWriteHelper Task Copy(ICollection entities, string copy, Func callback, [CallerMemberName] string stageName = ""); T GetDiscriminator(Type type); + + ValueTask HandleMaxAggregateCounts(object? entity); } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs index b871d16d0..282cf8b3a 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs @@ -69,9 +69,10 @@ using RadixDlt.NetworkGateway.Abstractions.Extensions; using RadixDlt.NetworkGateway.Abstractions.Model; using RadixDlt.NetworkGateway.Abstractions.Network; -using RadixDlt.NetworkGateway.Abstractions.Numerics; using RadixDlt.NetworkGateway.DataAggregator; using RadixDlt.NetworkGateway.DataAggregator.Services; +using RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors; +using RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors.LedgerTransactionMarkers; using RadixDlt.NetworkGateway.PostgresIntegration.Models; using RadixDlt.NetworkGateway.PostgresIntegration.Utils; using System; @@ -80,9 +81,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Array = System.Array; using CoreModel = RadixDlt.CoreApiSdk.Model; -using ToolkitModel = RadixEngineToolkit; namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; @@ -161,13 +160,13 @@ private async Task UpdatePendingTransactions(ReadWriteDbContext dbContext, .Where(pt => pt.LedgerDetails.PayloadLedgerStatus == PendingTransactionPayloadLedgerStatus.PermanentlyRejected) .Select( pt => new - { - pt.PayloadHash, - pt.LedgerDetails.PayloadLedgerStatus, - pt.GatewayHandling.FirstSubmittedToGatewayTimestamp, - pt.LedgerDetails.LatestRejectionTimestamp, - pt.LedgerDetails.LatestRejectionReason, - }) + { + pt.PayloadHash, + pt.LedgerDetails.PayloadLedgerStatus, + pt.GatewayHandling.FirstSubmittedToGatewayTimestamp, + pt.LedgerDetails.LatestRejectionTimestamp, + pt.LedgerDetails.LatestRejectionReason, + }) .AnnotateMetricName() .ToListAsync(token); @@ -220,16 +219,10 @@ private async Task ProcessTransactions(ReadWriteDbContext db var outerStopwatch = Stopwatch.StartNew(); var referencedEntities = new ReferencedEntityDictionary(); var childToParentEntities = new Dictionary(); - var manifestExtractedAddresses = new Dictionary(); - var manifestClasses = new Dictionary>(); var readHelper = new ReadHelper(dbContext, _observers, token); var writeHelper = new WriteHelper(dbContext, _observers, token); - var lastTransactionSummary = ledgerExtension.LatestTransactionSummary; - - var ledgerTransactionsToAdd = new List(); - var ledgerTransactionMarkersToAdd = new List(); var entitiesToAdd = new List(); SequencesHolder sequences; @@ -257,6 +250,25 @@ private async Task ProcessTransactions(ReadWriteDbContext db var processorContext = new ProcessorContext(sequences, readHelper, writeHelper, networkConfiguration, token); var relationshipProcessor = new RelationshipProcessor(referencedEntities); + var manifestProcessor = new ManifestProcessor(processorContext, referencedEntities, networkConfiguration); + var affectedGlobalEntitiesProcessor = new AffectedGlobalEntitiesProcessor(processorContext, referencedEntities, networkConfiguration); + + var ledgerTransactionMarkersProcessor = new LedgerTransactionMarkersProcessor( + manifestProcessor, + affectedGlobalEntitiesProcessor, + processorContext, + referencedEntities, + writeHelper, + networkConfiguration); + + var ledgerTransactionProcessor = new LedgerTransactionProcessor( + processorContext, + _clock, + referencedEntities, + manifestProcessor, + affectedGlobalEntitiesProcessor, + writeHelper, + ledgerExtension.LatestTransactionSummary); // step: scan for any referenced entities { @@ -274,93 +286,37 @@ private async Task ProcessTransactions(ReadWriteDbContext db try { - long? epochUpdate = null; - long? roundInEpochUpdate = null; - DateTime? roundTimestampUpdate = null; - - if (committedTransaction.LedgerTransaction is CoreModel.UserLedgerTransaction userLedgerTransaction) - { - ledgerTransactionMarkersToAdd.Add( - new OriginLedgerTransactionMarker - { - Id = sequences.LedgerTransactionMarkerSequence++, - StateVersion = stateVersion, - OriginType = LedgerTransactionMarkerOriginType.User, - }); - - var coreInstructions = userLedgerTransaction.NotarizedTransaction.SignedIntent.Intent.Instructions; - var coreBlobs = userLedgerTransaction.NotarizedTransaction.SignedIntent.Intent.BlobsHex; - using var manifestInstructions = ToolkitModel.Instructions.FromString(coreInstructions, networkConfiguration.Id); - using var toolkitManifest = new ToolkitModel.TransactionManifest(manifestInstructions, coreBlobs.Values.Select(x => x.ConvertFromHex()).ToArray()); - - var extractedAddresses = ManifestAddressesExtractor.ExtractAddresses(toolkitManifest, networkConfiguration.Id); - - foreach (var address in extractedAddresses.All()) - { - referencedEntities.MarkSeenAddress(address); - } - - manifestExtractedAddresses[stateVersion] = extractedAddresses; - - var manifestSummary = toolkitManifest.Summary(networkConfiguration.Id); - - for (var i = 0; i < manifestSummary.classification.Length; ++i) - { - var manifestClass = manifestSummary.classification[i].ToModel(); - - manifestClasses - .GetOrAdd(stateVersion, _ => new List()) - .Add(manifestClass); - - ledgerTransactionMarkersToAdd.Add( - new ManifestClassMarker - { - Id = sequences.LedgerTransactionMarkerSequence++, - StateVersion = stateVersion, - ManifestClass = manifestClass, - IsMostSpecific = i == 0, - }); - } - } - - if (committedTransaction.LedgerTransaction is CoreModel.RoundUpdateLedgerTransaction rult) - { - epochUpdate = lastTransactionSummary.Epoch != rult.RoundUpdateTransaction.Epoch ? rult.RoundUpdateTransaction.Epoch : null; - roundInEpochUpdate = rult.RoundUpdateTransaction.RoundInEpoch; - roundTimestampUpdate = DateTimeOffset.FromUnixTimeMilliseconds(rult.RoundUpdateTransaction.ProposerTimestamp.UnixTimestampMs).UtcDateTime; - } - foreach (var newGlobalEntity in stateUpdates.NewGlobalEntities) { var referencedEntity = referencedEntities.GetOrAdd((EntityAddress)newGlobalEntity.EntityAddress, ea => new ReferencedEntity(ea, newGlobalEntity.EntityType, stateVersion)); referencedEntity.WithTypeHint( newGlobalEntity.EntityType switch - { - CoreModel.EntityType.GlobalPackage => typeof(GlobalPackageEntity), - CoreModel.EntityType.GlobalConsensusManager => typeof(GlobalConsensusManager), - CoreModel.EntityType.GlobalValidator => typeof(GlobalValidatorEntity), - CoreModel.EntityType.GlobalGenericComponent => typeof(GlobalGenericComponentEntity), - CoreModel.EntityType.GlobalAccount => typeof(GlobalAccountEntity), - CoreModel.EntityType.GlobalAccountLocker => typeof(GlobalAccountLockerEntity), - CoreModel.EntityType.GlobalIdentity => typeof(GlobalIdentityEntity), - CoreModel.EntityType.GlobalAccessController => typeof(GlobalAccessControllerEntity), - CoreModel.EntityType.GlobalVirtualSecp256k1Account => typeof(GlobalAccountEntity), - CoreModel.EntityType.GlobalVirtualSecp256k1Identity => typeof(GlobalIdentityEntity), - CoreModel.EntityType.GlobalVirtualEd25519Account => typeof(GlobalAccountEntity), - CoreModel.EntityType.GlobalVirtualEd25519Identity => typeof(GlobalIdentityEntity), - CoreModel.EntityType.GlobalFungibleResource => typeof(GlobalFungibleResourceEntity), - CoreModel.EntityType.InternalFungibleVault => typeof(InternalFungibleVaultEntity), - CoreModel.EntityType.GlobalNonFungibleResource => typeof(GlobalNonFungibleResourceEntity), - CoreModel.EntityType.InternalNonFungibleVault => typeof(InternalNonFungibleVaultEntity), - CoreModel.EntityType.InternalGenericComponent => typeof(InternalGenericComponentEntity), - CoreModel.EntityType.InternalKeyValueStore => typeof(InternalKeyValueStoreEntity), - CoreModel.EntityType.GlobalOneResourcePool => typeof(GlobalOneResourcePoolEntity), - CoreModel.EntityType.GlobalTwoResourcePool => typeof(GlobalTwoResourcePoolEntity), - CoreModel.EntityType.GlobalMultiResourcePool => typeof(GlobalMultiResourcePoolEntity), - CoreModel.EntityType.GlobalTransactionTracker => typeof(GlobalTransactionTrackerEntity), - _ => throw new ArgumentOutOfRangeException(nameof(newGlobalEntity.EntityType), newGlobalEntity.EntityType.ToString()), - }); + { + CoreModel.EntityType.GlobalPackage => typeof(GlobalPackageEntity), + CoreModel.EntityType.GlobalConsensusManager => typeof(GlobalConsensusManager), + CoreModel.EntityType.GlobalValidator => typeof(GlobalValidatorEntity), + CoreModel.EntityType.GlobalGenericComponent => typeof(GlobalGenericComponentEntity), + CoreModel.EntityType.GlobalAccount => typeof(GlobalAccountEntity), + CoreModel.EntityType.GlobalAccountLocker => typeof(GlobalAccountLockerEntity), + CoreModel.EntityType.GlobalIdentity => typeof(GlobalIdentityEntity), + CoreModel.EntityType.GlobalAccessController => typeof(GlobalAccessControllerEntity), + CoreModel.EntityType.GlobalVirtualSecp256k1Account => typeof(GlobalAccountEntity), + CoreModel.EntityType.GlobalVirtualSecp256k1Identity => typeof(GlobalIdentityEntity), + CoreModel.EntityType.GlobalVirtualEd25519Account => typeof(GlobalAccountEntity), + CoreModel.EntityType.GlobalVirtualEd25519Identity => typeof(GlobalIdentityEntity), + CoreModel.EntityType.GlobalFungibleResource => typeof(GlobalFungibleResourceEntity), + CoreModel.EntityType.InternalFungibleVault => typeof(InternalFungibleVaultEntity), + CoreModel.EntityType.GlobalNonFungibleResource => typeof(GlobalNonFungibleResourceEntity), + CoreModel.EntityType.InternalNonFungibleVault => typeof(InternalNonFungibleVaultEntity), + CoreModel.EntityType.InternalGenericComponent => typeof(InternalGenericComponentEntity), + CoreModel.EntityType.InternalKeyValueStore => typeof(InternalKeyValueStoreEntity), + CoreModel.EntityType.GlobalOneResourcePool => typeof(GlobalOneResourcePoolEntity), + CoreModel.EntityType.GlobalTwoResourcePool => typeof(GlobalTwoResourcePoolEntity), + CoreModel.EntityType.GlobalMultiResourcePool => typeof(GlobalMultiResourcePoolEntity), + CoreModel.EntityType.GlobalTransactionTracker => typeof(GlobalTransactionTrackerEntity), + _ => throw new ArgumentOutOfRangeException(nameof(newGlobalEntity.EntityType), newGlobalEntity.EntityType.ToString()), + }); } foreach (var substate in stateUpdates.UpsertedSubstates) @@ -369,11 +325,6 @@ private async Task ProcessTransactions(ReadWriteDbContext db var substateData = substate.Value.SubstateData; var referencedEntity = referencedEntities.GetOrAdd((EntityAddress)substateId.EntityAddress, ea => new ReferencedEntity(ea, substateId.EntityType, stateVersion)); - if (substateData is CoreModel.ConsensusManagerFieldCurrentTimeSubstate currentTime) - { - roundTimestampUpdate = DateTimeOffset.FromUnixTimeMilliseconds(currentTime.Value.ProposerTimestamp.UnixTimestampMs).UtcDateTime; - } - if (substateData is CoreModel.IEntityOwner entityOwner) { foreach (var oe in entityOwner.GetOwnedEntities()) @@ -410,56 +361,62 @@ private async Task ProcessTransactions(ReadWriteDbContext db if (substateData is CoreModel.FungibleResourceManagerFieldDivisibilitySubstate fungibleResourceManager) { - referencedEntity.PostResolveConfigure((GlobalFungibleResourceEntity e) => - { - e.Divisibility = fungibleResourceManager.Value.Divisibility; - }); + referencedEntity.PostResolveConfigure( + (GlobalFungibleResourceEntity e) => + { + e.Divisibility = fungibleResourceManager.Value.Divisibility; + }); } if (substateData is CoreModel.NonFungibleResourceManagerFieldMutableFieldsSubstate mutableFields) { - referencedEntity.PostResolveConfigure((GlobalNonFungibleResourceEntity e) => - { - e.NonFungibleDataMutableFields = mutableFields.Value.MutableFields.Select(x => x.Name).ToList(); - }); + referencedEntity.PostResolveConfigure( + (GlobalNonFungibleResourceEntity e) => + { + e.NonFungibleDataMutableFields = mutableFields.Value.MutableFields.Select(x => x.Name).ToList(); + }); } if (substateData is CoreModel.NonFungibleResourceManagerFieldIdTypeSubstate nonFungibleResourceManagerFieldIdTypeSubstate) { - referencedEntity.PostResolveConfigure((GlobalNonFungibleResourceEntity e) => - { - e.NonFungibleIdType = nonFungibleResourceManagerFieldIdTypeSubstate.Value.NonFungibleIdType switch + referencedEntity.PostResolveConfigure( + (GlobalNonFungibleResourceEntity e) => { - CoreModel.NonFungibleIdType.String => NonFungibleIdType.String, - CoreModel.NonFungibleIdType.Integer => NonFungibleIdType.Integer, - CoreModel.NonFungibleIdType.Bytes => NonFungibleIdType.Bytes, - CoreModel.NonFungibleIdType.RUID => NonFungibleIdType.RUID, - _ => throw new ArgumentOutOfRangeException(nameof(e.NonFungibleIdType), e.NonFungibleIdType, "Unexpected value of NonFungibleIdType"), - }; - }); + e.NonFungibleIdType = nonFungibleResourceManagerFieldIdTypeSubstate.Value.NonFungibleIdType switch + { + CoreModel.NonFungibleIdType.String => NonFungibleIdType.String, + CoreModel.NonFungibleIdType.Integer => NonFungibleIdType.Integer, + CoreModel.NonFungibleIdType.Bytes => NonFungibleIdType.Bytes, + CoreModel.NonFungibleIdType.RUID => NonFungibleIdType.RUID, + _ => throw new ArgumentOutOfRangeException(nameof(e.NonFungibleIdType), e.NonFungibleIdType, "Unexpected value of NonFungibleIdType"), + }; + }); } if (substateData is CoreModel.TypeInfoModuleFieldTypeInfoSubstate typeInfoSubstate && typeInfoSubstate.Value.Details is CoreModel.ObjectTypeInfoDetails objectDetails) { - referencedEntity.PostResolveConfigure((ComponentEntity e) => - { - e.AssignedModuleIds = objectDetails.ModuleVersions - .Select(x => - { - return x.Module switch - { - CoreModel.AttachedModuleId.Metadata => ModuleId.Metadata, - CoreModel.AttachedModuleId.Royalty => ModuleId.Royalty, - CoreModel.AttachedModuleId.RoleAssignment => ModuleId.RoleAssignment, - _ => throw new ArgumentOutOfRangeException(nameof(x.Module), x.Module, "Unexpected value of AssignedModule"), - }; - }) - .OrderBy(x => x) - .ToList(); - - e.BlueprintName = objectDetails.BlueprintInfo.BlueprintName; - e.BlueprintVersion = objectDetails.BlueprintInfo.BlueprintVersion; - }); + referencedEntity.PostResolveConfigure( + (ComponentEntity e) => + { + e.AssignedModuleIds = objectDetails + .ModuleVersions + .Select( + x => + { + return x.Module switch + { + CoreModel.AttachedModuleId.Metadata => ModuleId.Metadata, + CoreModel.AttachedModuleId.Royalty => ModuleId.Royalty, + CoreModel.AttachedModuleId.RoleAssignment => ModuleId.RoleAssignment, + _ => throw new ArgumentOutOfRangeException(nameof(x.Module), x.Module, "Unexpected value of AssignedModule"), + }; + }) + .OrderBy(x => x) + .ToList(); + + e.BlueprintName = objectDetails.BlueprintInfo.BlueprintName; + e.BlueprintVersion = objectDetails.BlueprintInfo.BlueprintVersion; + }); } relationshipProcessor.ScanUpsert(substateData, referencedEntity, stateVersion); @@ -491,90 +448,6 @@ private async Task ProcessTransactions(ReadWriteDbContext db referencedEntities.MarkSeenAddress((EntityAddress)entityAddress); } } - - /* NB: - The Epoch Transition Transaction sort of fits between epochs, but it seems to fit slightly more naturally - as the _first_ transaction of a new epoch, as creates the next EpochData, and the RoundData to 0. - */ - - var isStartOfEpoch = epochUpdate.HasValue; - var isStartOfRound = roundInEpochUpdate.HasValue; - var roundTimestamp = roundTimestampUpdate ?? lastTransactionSummary.RoundTimestamp; - var createdTimestamp = _clock.UtcNow; - var normalizedRoundTimestamp = // Clamp between lastTransaction.NormalizedTimestamp and createdTimestamp - roundTimestamp < lastTransactionSummary.NormalizedRoundTimestamp ? lastTransactionSummary.NormalizedRoundTimestamp - : roundTimestamp > createdTimestamp ? createdTimestamp - : roundTimestamp; - - var summary = new TransactionSummary( - StateVersion: stateVersion, - RoundTimestamp: roundTimestamp, - NormalizedRoundTimestamp: normalizedRoundTimestamp, - TransactionTreeHash: lastTransactionSummary.TransactionTreeHash, - ReceiptTreeHash: lastTransactionSummary.ReceiptTreeHash, - StateTreeHash: lastTransactionSummary.StateTreeHash, - CreatedTimestamp: createdTimestamp, - Epoch: epochUpdate ?? lastTransactionSummary.Epoch, - RoundInEpoch: roundInEpochUpdate ?? lastTransactionSummary.RoundInEpoch, - IndexInEpoch: isStartOfEpoch ? 0 : lastTransactionSummary.IndexInEpoch + 1, - IndexInRound: isStartOfRound ? 0 : lastTransactionSummary.IndexInRound + 1); - - LedgerTransaction ledgerTransaction = committedTransaction.LedgerTransaction switch - { - CoreModel.GenesisLedgerTransaction => new GenesisLedgerTransaction(), - CoreModel.UserLedgerTransaction ult => new UserLedgerTransaction - { - PayloadHash = ult.NotarizedTransaction.HashBech32m, - IntentHash = ult.NotarizedTransaction.SignedIntent.Intent.HashBech32m, - SignedIntentHash = ult.NotarizedTransaction.SignedIntent.HashBech32m, - Message = ult.NotarizedTransaction.SignedIntent.Intent.Message?.ToJson(), - RawPayload = ult.NotarizedTransaction.GetPayloadBytes(), - ManifestInstructions = ult.NotarizedTransaction.SignedIntent.Intent.Instructions, - ManifestClasses = manifestClasses.TryGetValue(stateVersion, out var mc) ? mc.ToArray() : Array.Empty(), - }, - CoreModel.RoundUpdateLedgerTransaction => new RoundUpdateLedgerTransaction(), - CoreModel.FlashLedgerTransaction => new FlashLedgerTransaction(), - _ => throw new UnreachableException($"Unsupported transaction type: {committedTransaction.LedgerTransaction.GetType()}"), - }; - - ledgerTransaction.StateVersion = stateVersion; - ledgerTransaction.TransactionTreeHash = committedTransaction.ResultantStateIdentifiers.TransactionTreeHash; - ledgerTransaction.ReceiptTreeHash = committedTransaction.ResultantStateIdentifiers.ReceiptTreeHash; - ledgerTransaction.StateTreeHash = committedTransaction.ResultantStateIdentifiers.StateTreeHash; - ledgerTransaction.Epoch = summary.Epoch; - ledgerTransaction.RoundInEpoch = summary.RoundInEpoch; - ledgerTransaction.IndexInEpoch = summary.IndexInEpoch; - ledgerTransaction.IndexInRound = summary.IndexInRound; - ledgerTransaction.FeePaid = committedTransaction.Receipt.FeeSummary.TotalFee(); - ledgerTransaction.TipPaid = committedTransaction.Receipt.FeeSummary.TotalTip(); - ledgerTransaction.AffectedGlobalEntities = default!; // configured later on - ledgerTransaction.RoundTimestamp = summary.RoundTimestamp; - ledgerTransaction.CreatedTimestamp = summary.CreatedTimestamp; - ledgerTransaction.NormalizedRoundTimestamp = summary.NormalizedRoundTimestamp; - ledgerTransaction.ReceiptStateUpdates = committedTransaction.Receipt.StateUpdates.ToJson(); - ledgerTransaction.ReceiptStatus = committedTransaction.Receipt.Status.ToModel(); - ledgerTransaction.ReceiptFeeSummary = committedTransaction.Receipt.FeeSummary.ToJson(); - ledgerTransaction.ReceiptErrorMessage = committedTransaction.Receipt.ErrorMessage; - ledgerTransaction.ReceiptOutput = committedTransaction.Receipt.Output != null ? JsonConvert.SerializeObject(committedTransaction.Receipt.Output) : null; - ledgerTransaction.ReceiptNextEpoch = committedTransaction.Receipt.NextEpoch?.ToJson(); - ledgerTransaction.ReceiptCostingParameters = committedTransaction.Receipt.CostingParameters.ToJson(); - ledgerTransaction.ReceiptFeeSource = committedTransaction.Receipt.FeeSource?.ToJson(); - ledgerTransaction.ReceiptFeeDestination = committedTransaction.Receipt.FeeDestination?.ToJson(); - ledgerTransaction.BalanceChanges = committedTransaction.BalanceChanges?.ToJson(); - ledgerTransactionsToAdd.Add(ledgerTransaction); - - if (committedTransaction.Receipt.NextEpoch != null) - { - ledgerTransactionMarkersToAdd.Add( - new OriginLedgerTransactionMarker - { - Id = sequences.LedgerTransactionMarkerSequence++, - StateVersion = stateVersion, - OriginType = LedgerTransactionMarkerOriginType.EpochChange, - }); - } - - lastTransactionSummary = summary; } catch (Exception ex) { @@ -606,6 +479,7 @@ private async Task ProcessTransactions(ReadWriteDbContext db GlobalAccountLockerEntity => CoreModel.EntityType.GlobalAccountLocker, GlobalIdentityEntity => CoreModel.EntityType.GlobalIdentity, GlobalAccessControllerEntity => CoreModel.EntityType.GlobalAccessController, + // skipped GlobalVirtualSecp256k1Account, GlobalVirtualSecp256k1Identity, GlobalVirtualEd25519Account and GlobalVirtualEd25519Identity as they are virtual GlobalFungibleResourceEntity => CoreModel.EntityType.GlobalFungibleResource, InternalFungibleVaultEntity => CoreModel.EntityType.InternalFungibleVault, @@ -716,13 +590,14 @@ private async Task ProcessTransactions(ReadWriteDbContext db // we must rely on high priority PostResolveConfigure as other callbacks rely on the entity hierarchy tree referencedEntities .Get(childAddress) - .PostResolveConfigureHigh((Entity dbe) => - { - dbe.AncestorIds = allAncestors; - dbe.ParentAncestorId = parentId.Value; - dbe.OwnerAncestorId = ownerId.Value; - dbe.GlobalAncestorId = globalId.Value; - }); + .PostResolveConfigureHigh( + (Entity dbe) => + { + dbe.AncestorIds = allAncestors; + dbe.ParentAncestorId = parentId.Value; + dbe.OwnerAncestorId = ownerId.Value; + dbe.GlobalAncestorId = globalId.Value; + }); } referencedEntities.InvokePostResolveConfiguration(); @@ -730,29 +605,30 @@ private async Task ProcessTransactions(ReadWriteDbContext db await _observers.ForEachAsync(x => x.StageCompleted("resolve_and_create_entities", sw.Elapsed, null)); } - var resourceSupplyChanges = new List(); - var nonFungibleSchemaHistoryToAdd = new List(); - var keyValueStoreSchemaHistoryToAdd = new List(); - - var entityStateProcessor = new EntityStateProcessor(processorContext, referencedEntities); - var entityMetadataProcessor = new EntityMetadataProcessor(processorContext); - var entitySchemaProcessor = new EntitySchemaProcessor(processorContext); - var componentMethodRoyaltyProcessor = new ComponentMethodRoyaltyProcessor(processorContext); - var entityRoleAssignmentProcessor = new EntityRoleAssignmentProcessor(processorContext); - var packageCodeProcessor = new PackageCodeProcessor(processorContext); - var packageBlueprintProcessor = new PackageBlueprintProcessor(processorContext, referencedEntities); - var accountAuthorizedDepositorsProcessor = new AccountAuthorizedDepositorsProcessor(processorContext, referencedEntities); - var accountResourcePreferenceRulesProcessor = new AccountResourcePreferenceRulesProcessor(processorContext, referencedEntities); - var accountDefaultDepositRuleProcessor = new AccountDefaultDepositRuleProcessor(processorContext); - var keyValueStoreProcessor = new KeyValueStoreProcessor(processorContext); - var validatorProcessor = new ValidatorProcessor(processorContext, referencedEntities); - var validatorEmissionProcessor = new ValidatorEmissionProcessor(processorContext); - var accountLockerProcessor = new AccountLockerProcessor(processorContext, referencedEntities); - var globalEventEmitterProcessor = new GlobalEventEmitterProcessor(processorContext, referencedEntities, networkConfiguration); - var affectedGlobalEntitiesProcessor = new AffectedGlobalEntitiesProcessor(processorContext, referencedEntities, networkConfiguration); - var standardMetadataProcessor = new StandardMetadataProcessor(processorContext, referencedEntities); - var entityResourceProcessor = new EntityResourceProcessor(processorContext, dbContext, _observers); - var vaultProcessor = new VaultProcessor(processorContext); + var processors = new List + { + ledgerTransactionProcessor, + ledgerTransactionMarkersProcessor, + new ResourceSupplyProcessor(processorContext), + new EntityStateProcessor(processorContext, referencedEntities), + new EntityMetadataProcessor(processorContext), + new EntitySchemaProcessor(processorContext, referencedEntities), + new ComponentMethodRoyaltyProcessor(processorContext), + new EntityRoleAssignmentProcessor(processorContext), + new PackageCodeProcessor(processorContext), + new PackageBlueprintProcessor(processorContext, referencedEntities), + new PackageBlueprintProcessor(processorContext, referencedEntities), + new AccountAuthorizedDepositorsProcessor(processorContext, referencedEntities), + new AccountResourcePreferenceRulesProcessor(processorContext, referencedEntities), + new AccountDefaultDepositRuleProcessor(processorContext), + new KeyValueStoreProcessor(processorContext), + new ValidatorProcessor(processorContext, referencedEntities), + new ValidatorEmissionProcessor(processorContext), + new AccountLockerProcessor(processorContext, referencedEntities), + new StandardMetadataProcessor(processorContext, referencedEntities), + new EntityResourceProcessor(processorContext, dbContext, _observers), + new VaultProcessor(processorContext), + }; // step: scan all substates & events to figure out changes { @@ -763,92 +639,19 @@ private async Task ProcessTransactions(ReadWriteDbContext db var stateVersion = committedTransaction.ResultantStateIdentifiers.StateVersion; var stateUpdates = committedTransaction.Receipt.StateUpdates; var events = committedTransaction.Receipt.Events ?? new List(); - long? passingEpoch = null; + ledgerTransactionMarkersProcessor.VisitTransaction(committedTransaction, stateVersion); try { foreach (var substate in stateUpdates.UpsertedSubstates) { var substateId = substate.SubstateId; - var substateData = substate.Value.SubstateData; var referencedEntity = referencedEntities.Get((EntityAddress)substateId.EntityAddress); - if (substateData is CoreModel.ConsensusManagerFieldStateSubstate consensusManagerFieldStateSubstate) + foreach (var processor in processors.OfType()) { - if (consensusManagerFieldStateSubstate.Value.Round == 0) - { - passingEpoch = consensusManagerFieldStateSubstate.Value.Epoch - 1; - } - } - - if (substateData is CoreModel.TypeInfoModuleFieldTypeInfoSubstate typeInfoSubstate) - { - if (typeInfoSubstate.TryGetNonFungibleDataSchemaDetails(out var nonFungibleDataSchemaDetails)) - { - var schemaDefiningEntityId = !string.IsNullOrEmpty(nonFungibleDataSchemaDetails.Value.SchemaDefiningEntityAddress) - ? referencedEntities.Get((EntityAddress)nonFungibleDataSchemaDetails.Value.SchemaDefiningEntityAddress).DatabaseId - : referencedEntity.DatabaseId; - - nonFungibleSchemaHistoryToAdd.Add( - new NonFungibleSchemaHistory - { - Id = sequences.NonFungibleSchemaHistorySequence++, - ResourceEntityId = referencedEntity.DatabaseId, - SchemaHash = nonFungibleDataSchemaDetails.Value.SchemaHash.ConvertFromHex(), - SborTypeKind = nonFungibleDataSchemaDetails.Value.SborTypeKind.ToModel(), - TypeIndex = nonFungibleDataSchemaDetails.Value.TypeIndex, - SchemaDefiningEntityId = schemaDefiningEntityId, - FromStateVersion = stateVersion, - }); - } - - if (typeInfoSubstate.Value.Details is CoreModel.KeyValueStoreTypeInfoDetails - && typeInfoSubstate.TryGetKeyValueStoreKeySchemaDetails(out var keySchemaDetails) - && typeInfoSubstate.TryGetKeyValueStoreValueSchemaDetails(out var valueSchemaDetails)) - { - var keySchemaDefiningEntityId = !string.IsNullOrEmpty(keySchemaDetails.Value.SchemaDefiningEntityAddress) - ? referencedEntities.Get((EntityAddress)keySchemaDetails.Value.SchemaDefiningEntityAddress).DatabaseId - : referencedEntity.DatabaseId; - - var valueSchemaDefiningEntityId = !string.IsNullOrEmpty(valueSchemaDetails.Value.SchemaDefiningEntityAddress) - ? referencedEntities.Get((EntityAddress)valueSchemaDetails.Value.SchemaDefiningEntityAddress).DatabaseId - : referencedEntity.DatabaseId; - - keyValueStoreSchemaHistoryToAdd.Add( - new KeyValueStoreSchemaHistory - { - Id = sequences.KeyValueSchemaHistorySequence++, - KeyValueStoreEntityId = referencedEntity.DatabaseId, - KeySchemaDefiningEntityId = keySchemaDefiningEntityId, - KeySchemaHash = keySchemaDetails.Value.SchemaHash.ConvertFromHex(), - KeySborTypeKind = keySchemaDetails.Value.SborTypeKind.ToModel(), - KeyTypeIndex = keySchemaDetails.Value.TypeIndex, - ValueSchemaDefiningEntityId = valueSchemaDefiningEntityId, - ValueSchemaHash = valueSchemaDetails.Value.SchemaHash.ConvertFromHex(), - ValueSborTypeKind = valueSchemaDetails.Value.SborTypeKind.ToModel(), - ValueTypeIndex = valueSchemaDetails.Value.TypeIndex, - FromStateVersion = stateVersion, - }); - } + processor.VisitUpsert(substate, referencedEntity, stateVersion); } - - entityStateProcessor.VisitUpsert(substate, referencedEntity, stateVersion); - entityMetadataProcessor.VisitUpsert(substate, referencedEntity, stateVersion); - entitySchemaProcessor.VisitUpsert(substateData, referencedEntity, stateVersion); - componentMethodRoyaltyProcessor.VisitUpsert(substateData, referencedEntity, stateVersion); - entityRoleAssignmentProcessor.VisitUpsert(substateData, referencedEntity, stateVersion); - packageCodeProcessor.VisitUpsert(substateData, referencedEntity, stateVersion); - packageBlueprintProcessor.VisitUpsert(substateData, referencedEntity, stateVersion); - accountResourcePreferenceRulesProcessor.VisitUpsert(substateData, referencedEntity, stateVersion); - accountDefaultDepositRuleProcessor.VisitUpsert(substateData, referencedEntity, stateVersion); - accountAuthorizedDepositorsProcessor.VisitUpsert(substateData, referencedEntity, stateVersion); - keyValueStoreProcessor.VisitUpsert(substate, referencedEntity, stateVersion); - validatorProcessor.VisitUpsert(substateData, referencedEntity, stateVersion, passingEpoch); - accountLockerProcessor.VisitUpsert(substateData, referencedEntity, stateVersion); - affectedGlobalEntitiesProcessor.VisitUpsert(referencedEntity, stateVersion); - standardMetadataProcessor.VisitUpsert(substateData, referencedEntity, stateVersion); - vaultProcessor.VisitUpsert(substateData, referencedEntity, stateVersion, substate); - entityResourceProcessor.VisitUpsert(substateData, referencedEntity, stateVersion, substate); } foreach (var deletedSubstate in stateUpdates.DeletedSubstates) @@ -856,26 +659,20 @@ private async Task ProcessTransactions(ReadWriteDbContext db var substateId = deletedSubstate.SubstateId; var referencedEntity = referencedEntities.GetOrAdd((EntityAddress)substateId.EntityAddress, ea => new ReferencedEntity(ea, substateId.EntityType, stateVersion)); - affectedGlobalEntitiesProcessor.VisitDelete(referencedEntity, stateVersion); - packageCodeProcessor.VisitDelete(substateId, referencedEntity, stateVersion); - vaultProcessor.VisitDelete(substateId, referencedEntity, stateVersion); - entitySchemaProcessor.VisitDelete(substateId, referencedEntity, stateVersion); - } - - var transaction = ledgerTransactionsToAdd.Single(x => x.StateVersion == stateVersion); + ledgerTransactionMarkersProcessor.VisitDelete(substateId, referencedEntity, stateVersion); - transaction.AffectedGlobalEntities = affectedGlobalEntitiesProcessor.GetAllAffectedGlobalEntities(stateVersion).ToArray(); - transaction.ReceiptEventEmitters = events.Select(e => e.Type.Emitter.ToJson()).ToArray(); - transaction.ReceiptEventNames = events.Select(e => e.Type.Name).ToArray(); - transaction.ReceiptEventSbors = events.Select(e => e.Data.GetDataBytes()).ToArray(); - transaction.ReceiptEventSchemaEntityIds = events.Select(e => referencedEntities.Get((EntityAddress)e.Type.TypeReference.FullTypeId.EntityAddress).DatabaseId).ToArray(); - transaction.ReceiptEventSchemaHashes = events.Select(e => e.Type.TypeReference.FullTypeId.SchemaHash.ConvertFromHex()).ToArray(); - transaction.ReceiptEventTypeIndexes = events.Select(e => e.Type.TypeReference.FullTypeId.LocalTypeId.Id).ToArray(); - transaction.ReceiptEventSborTypeKinds = events.Select(e => e.Type.TypeReference.FullTypeId.LocalTypeId.Kind.ToModel()).ToArray(); + foreach (var processor in processors.OfType()) + { + processor.VisitDelete(substateId, referencedEntity, stateVersion); + } + } foreach (var @event in events) { - globalEventEmitterProcessor.VisitEvent(@event, stateVersion); + foreach (var processor in processors.OfType()) + { + processor.VisitEvent(@event, stateVersion); + } if (@event.Type.Emitter is not CoreModel.MethodEventEmitterIdentifier methodEventEmitter || methodEventEmitter.ObjectModuleId != CoreModel.ModuleId.Main @@ -886,160 +683,11 @@ private async Task ProcessTransactions(ReadWriteDbContext db } var eventEmitterEntity = referencedEntities.Get((EntityAddress)methodEventEmitter.Entity.EntityAddress); - using var decodedEvent = EventDecoder.DecodeEvent(@event, networkConfiguration.Id); - validatorEmissionProcessor.VisitEvent(decodedEvent, eventEmitterEntity, stateVersion); - - if (EventDecoder.TryGetFungibleVaultWithdrawalEvent(decodedEvent, out var fungibleVaultWithdrawalEvent)) - { - ledgerTransactionMarkersToAdd.Add( - new EventLedgerTransactionMarker - { - Id = sequences.LedgerTransactionMarkerSequence++, - StateVersion = stateVersion, - EventType = LedgerTransactionMarkerEventType.Withdrawal, - EntityId = eventEmitterEntity.DatabaseGlobalAncestorId, - ResourceEntityId = eventEmitterEntity.GetDatabaseEntity().GetResourceEntityId(), - Quantity = TokenAmount.FromDecimalString(fungibleVaultWithdrawalEvent.AsStr()), - }); - } - else if (EventDecoder.TryGetFungibleVaultDepositEvent(decodedEvent, out var fungibleVaultDepositEvent)) - { - ledgerTransactionMarkersToAdd.Add( - new EventLedgerTransactionMarker - { - Id = sequences.LedgerTransactionMarkerSequence++, - StateVersion = stateVersion, - EventType = LedgerTransactionMarkerEventType.Deposit, - EntityId = eventEmitterEntity.DatabaseGlobalAncestorId, - ResourceEntityId = eventEmitterEntity.GetDatabaseEntity().GetResourceEntityId(), - Quantity = TokenAmount.FromDecimalString(fungibleVaultDepositEvent.AsStr()), - }); - } - else if (EventDecoder.TryGetNonFungibleVaultWithdrawalEvent(decodedEvent, out var nonFungibleVaultWithdrawalEvent)) - { - ledgerTransactionMarkersToAdd.Add( - new EventLedgerTransactionMarker - { - Id = sequences.LedgerTransactionMarkerSequence++, - StateVersion = stateVersion, - EventType = LedgerTransactionMarkerEventType.Withdrawal, - EntityId = eventEmitterEntity.DatabaseGlobalAncestorId, - ResourceEntityId = eventEmitterEntity.GetDatabaseEntity().GetResourceEntityId(), - Quantity = TokenAmount.FromDecimalString(nonFungibleVaultWithdrawalEvent.Length.ToString()), - }); - } - else if (EventDecoder.TryGetNonFungibleVaultDepositEvent(decodedEvent, out var nonFungibleVaultDepositEvent)) - { - ledgerTransactionMarkersToAdd.Add( - new EventLedgerTransactionMarker - { - Id = sequences.LedgerTransactionMarkerSequence++, - StateVersion = stateVersion, - EventType = LedgerTransactionMarkerEventType.Deposit, - EntityId = eventEmitterEntity.DatabaseGlobalAncestorId, - ResourceEntityId = eventEmitterEntity.GetDatabaseEntity().GetResourceEntityId(), - Quantity = TokenAmount.FromDecimalString(nonFungibleVaultDepositEvent.Length.ToString()), - }); - } - else if (EventDecoder.TryGetFungibleResourceMintedEvent(decodedEvent, out var fungibleResourceMintedEvent)) - { - var mintedAmount = TokenAmount.FromDecimalString(fungibleResourceMintedEvent.amount.AsStr()); - resourceSupplyChanges.Add(new ResourceSupplyChange(eventEmitterEntity.DatabaseId, stateVersion, Minted: mintedAmount)); - } - else if (EventDecoder.TryGetFungibleResourceBurnedEvent(decodedEvent, out var fungibleResourceBurnedEvent)) - { - var burnedAmount = TokenAmount.FromDecimalString(fungibleResourceBurnedEvent.amount.AsStr()); - resourceSupplyChanges.Add(new ResourceSupplyChange(eventEmitterEntity.DatabaseId, stateVersion, Burned: burnedAmount)); - } - else if (EventDecoder.TryGetNonFungibleResourceMintedEvent(decodedEvent, out var nonFungibleResourceMintedEvent)) - { - var mintedCount = TokenAmount.FromDecimalString(nonFungibleResourceMintedEvent.ids.Length.ToString()); - resourceSupplyChanges.Add(new ResourceSupplyChange(eventEmitterEntity.DatabaseId, stateVersion, Minted: mintedCount)); - } - else if (EventDecoder.TryGetNonFungibleResourceBurnedEvent(decodedEvent, out var nonFungibleResourceBurnedEvent)) - { - var burnedCount = TokenAmount.FromDecimalString(nonFungibleResourceBurnedEvent.ids.Length.ToString()); - resourceSupplyChanges.Add(new ResourceSupplyChange(eventEmitterEntity.DatabaseId, stateVersion, Burned: burnedCount)); - } - } - - if (manifestExtractedAddresses.TryGetValue(stateVersion, out var extractedAddresses)) - { - foreach (var proofResourceAddress in extractedAddresses.PresentedProofs.Select(x => x.ResourceAddress).ToHashSet()) - { - if (referencedEntities.TryGet(proofResourceAddress, out var re)) - { - ledgerTransactionMarkersToAdd.Add( - new ManifestAddressLedgerTransactionMarker - { - Id = sequences.LedgerTransactionMarkerSequence++, - StateVersion = stateVersion, - OperationType = LedgerTransactionMarkerOperationType.BadgePresented, - EntityId = re.DatabaseId, - }); - } - } - - foreach (var address in extractedAddresses.ResourceAddresses) - { - if (referencedEntities.TryGet(address, out var re)) - { - ledgerTransactionMarkersToAdd.Add( - new ManifestAddressLedgerTransactionMarker - { - Id = sequences.LedgerTransactionMarkerSequence++, - StateVersion = stateVersion, - OperationType = LedgerTransactionMarkerOperationType.ResourceInUse, - EntityId = re.DatabaseId, - }); - } - } - - foreach (var address in extractedAddresses.AccountsRequiringAuth) - { - if (referencedEntities.TryGet(address, out var re)) - { - ledgerTransactionMarkersToAdd.Add( - new ManifestAddressLedgerTransactionMarker - { - Id = sequences.LedgerTransactionMarkerSequence++, - StateVersion = stateVersion, - OperationType = LedgerTransactionMarkerOperationType.AccountOwnerMethodCall, - EntityId = re.DatabaseId, - }); - } - } - - foreach (var address in extractedAddresses.AccountsDepositedInto) - { - if (referencedEntities.TryGet(address, out var re)) - { - ledgerTransactionMarkersToAdd.Add( - new ManifestAddressLedgerTransactionMarker - { - Id = sequences.LedgerTransactionMarkerSequence++, - StateVersion = stateVersion, - OperationType = LedgerTransactionMarkerOperationType.AccountDepositedInto, - EntityId = re.DatabaseId, - }); - } - } - - foreach (var address in extractedAddresses.AccountsWithdrawnFrom) + foreach (var processor in processors.OfType()) { - if (referencedEntities.TryGet(address, out var re)) - { - ledgerTransactionMarkersToAdd.Add( - new ManifestAddressLedgerTransactionMarker - { - Id = sequences.LedgerTransactionMarkerSequence++, - StateVersion = stateVersion, - OperationType = LedgerTransactionMarkerOperationType.AccountWithdrawnFrom, - EntityId = re.DatabaseId, - }); - } + processor.VisitDecodedEvent(decodedEvent, eventEmitterEntity, stateVersion); } } } @@ -1057,112 +705,28 @@ private async Task ProcessTransactions(ReadWriteDbContext db { var sw = Stopwatch.StartNew(); - var mostRecentResourceEntitySupplyHistory = await readHelper.MostRecentResourceEntitySupplyHistoryFor(resourceSupplyChanges, token); - - await entityMetadataProcessor.LoadDependencies(); - await entitySchemaProcessor.LoadDependencies(); - await componentMethodRoyaltyProcessor.LoadDependencies(); - await entityRoleAssignmentProcessor.LoadDependencies(); - await packageCodeProcessor.LoadDependencies(); - await packageBlueprintProcessor.LoadDependencies(); - await validatorProcessor.LoadDependencies(); - await validatorEmissionProcessor.LoadDependencies(); - await keyValueStoreProcessor.LoadDependencies(); - await accountAuthorizedDepositorsProcessor.LoadDependencies(); - await accountResourcePreferenceRulesProcessor.LoadDependencies(); - await accountLockerProcessor.LoadDependencies(); - await standardMetadataProcessor.LoadDependencies(); - await entityResourceProcessor.LoadDependencies(); - await vaultProcessor.LoadDependencies(); + foreach (var processor in processors) + { + await processor.LoadDependenciesAsync(); + } dbReadDuration += sw.Elapsed; - entityMetadataProcessor.ProcessChanges(); - entitySchemaProcessor.ProcessChanges(); - componentMethodRoyaltyProcessor.ProcessChanges(); - entityRoleAssignmentProcessor.ProcessChanges(); - packageCodeProcessor.ProcessChanges(); - packageBlueprintProcessor.ProcessChanges(); - accountDefaultDepositRuleProcessor.ProcessChanges(); - accountAuthorizedDepositorsProcessor.ProcessChanges(); - accountResourcePreferenceRulesProcessor.ProcessChanges(); - keyValueStoreProcessor.ProcessChanges(); - validatorProcessor.ProcessChanges(); - validatorEmissionProcessor.ProcessChanges(); - accountLockerProcessor.ProcessChanges(); - ledgerTransactionMarkersToAdd.AddRange(globalEventEmitterProcessor.CreateTransactionMarkers()); - ledgerTransactionMarkersToAdd.AddRange(affectedGlobalEntitiesProcessor.CreateTransactionMarkers()); - standardMetadataProcessor.ProcessChanges(); - entityResourceProcessor.ProcessChanges(); - vaultProcessor.ProcessChanges(); - - var resourceEntitySupplyHistoryToAdd = resourceSupplyChanges - .GroupBy(x => new { x.ResourceEntityId, x.StateVersion }) - .Select( - group => - { - var previous = mostRecentResourceEntitySupplyHistory.GetOrAdd( - group.Key.ResourceEntityId, - _ => new ResourceEntitySupplyHistory { TotalSupply = TokenAmount.Zero, TotalMinted = TokenAmount.Zero, TotalBurned = TokenAmount.Zero }); - - var minted = group - .Where(x => x.Minted.HasValue) - .Select(x => x.Minted) - .Aggregate(TokenAmount.Zero, (sum, x) => sum + x!.Value); - - var burned = group - .Where(x => x.Burned.HasValue) - .Select(x => x.Burned) - .Aggregate(TokenAmount.Zero, (sum, x) => sum + x!.Value); - - var totalSupply = previous.TotalSupply + minted - burned; - var totalMinted = previous.TotalMinted + minted; - var totalBurned = previous.TotalBurned + burned; - - previous.TotalSupply = totalSupply; - previous.TotalMinted = totalMinted; - previous.TotalBurned = totalBurned; - - return new ResourceEntitySupplyHistory - { - Id = sequences.ResourceEntitySupplyHistorySequence++, - FromStateVersion = group.Key.StateVersion, - ResourceEntityId = group.Key.ResourceEntityId, - TotalSupply = totalSupply, - TotalMinted = totalMinted, - TotalBurned = totalBurned, - }; - }) - .ToList(); + foreach (var processor in processors) + { + processor.ProcessChanges(); + } await _observers.ForEachAsync(x => x.StageCompleted("process_changes", sw.Elapsed, null)); sw = Stopwatch.StartNew(); rowsInserted += await writeHelper.CopyEntity(entitiesToAdd, token); - rowsInserted += await writeHelper.CopyLedgerTransaction(ledgerTransactionsToAdd, token); - rowsInserted += await writeHelper.CopyLedgerTransactionMarkers(ledgerTransactionMarkersToAdd, token); - rowsInserted += await writeHelper.CopyResourceEntitySupplyHistory(resourceEntitySupplyHistoryToAdd, token); - rowsInserted += await writeHelper.CopyNonFungibleDataSchemaHistory(nonFungibleSchemaHistoryToAdd, token); - rowsInserted += await writeHelper.CopyKeyValueStoreSchemaHistory(keyValueStoreSchemaHistoryToAdd, token); - - rowsInserted += await entityStateProcessor.SaveEntities(); - rowsInserted += await entityMetadataProcessor.SaveEntities(); - rowsInserted += await entitySchemaProcessor.SaveEntities(); - rowsInserted += await componentMethodRoyaltyProcessor.SaveEntities(); - rowsInserted += await entityRoleAssignmentProcessor.SaveEntities(); - rowsInserted += await packageCodeProcessor.SaveEntities(); - rowsInserted += await packageBlueprintProcessor.SaveEntities(); - rowsInserted += await accountDefaultDepositRuleProcessor.SaveEntities(); - rowsInserted += await accountAuthorizedDepositorsProcessor.SaveEntities(); - rowsInserted += await accountResourcePreferenceRulesProcessor.SaveEntities(); - rowsInserted += await keyValueStoreProcessor.SaveEntities(); - rowsInserted += await validatorProcessor.SaveEntities(); - rowsInserted += await validatorEmissionProcessor.SaveEntities(); - rowsInserted += await accountLockerProcessor.SaveEntities(); - rowsInserted += await standardMetadataProcessor.SaveEntities(); - rowsInserted += await entityResourceProcessor.SaveEntities(); - rowsInserted += await vaultProcessor.SaveEntities(); + + foreach (var processor in processors) + { + rowsInserted += await processor.SaveEntitiesAsync(); + } await writeHelper.UpdateSequences(sequences, token); @@ -1173,6 +737,11 @@ private async Task ProcessTransactions(ReadWriteDbContext db var contentHandlingDuration = outerStopwatch.Elapsed - dbReadDuration - dbWriteDuration; - return new ExtendLedgerReport(lastTransactionSummary, rowsInserted + rowsUpdated, dbReadDuration, dbWriteDuration, contentHandlingDuration); + return new ExtendLedgerReport( + ledgerTransactionProcessor.GetLastProcessedTransactionSummary(), + rowsInserted + rowsUpdated, + dbReadDuration, + dbWriteDuration, + contentHandlingDuration); } } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/AccountAuthorizedDepositorsProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountAuthorizedDepositorsProcessor.cs similarity index 90% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/AccountAuthorizedDepositorsProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountAuthorizedDepositorsProcessor.cs index f517aaefb..8a172a18c 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/AccountAuthorizedDepositorsProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountAuthorizedDepositorsProcessor.cs @@ -75,19 +75,19 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; -// It's using assumption that only possible badges are resource and non fungible resource badge. +internal class AccountAuthorizedDepositorsProcessor : IProcessorBase, ISubstateUpsertProcessor +{ + // It's using assumption that only possible badges are resource and non fungible resource badge. // If in future our model will be expanded and there will be more badge types it'll require rework. -internal record struct AccountAuthorizedDepositorsDbLookup(long AccountEntityId, long ResourceEntityId, string? NonFungibleId); + private record struct AccountAuthorizedDepositorsDbLookup(long AccountEntityId, long ResourceEntityId, string? NonFungibleId); -internal record struct AccountAuthorizedDepositorsChangePointerLookup(long AccountEntityId, long StateVersion); + private record struct AccountAuthorizedDepositorsChangePointerLookup(long AccountEntityId, long StateVersion); -internal record AccountAuthorizedDepositorsChangePointer -{ - public List AuthorizedDepositorEntries { get; } = new(); -} + private record AccountAuthorizedDepositorsChangePointer + { + public List AuthorizedDepositorEntries { get; } = new(); + } -internal class AccountAuthorizedDepositorsProcessor -{ private readonly ProcessorContext _context; private readonly ReferencedEntityDictionary _referencedEntityDictionary; @@ -107,8 +107,10 @@ public AccountAuthorizedDepositorsProcessor( _referencedEntityDictionary = referencedEntityDictionary; } - public void VisitUpsert(CoreModel.Substate substateData, ReferencedEntity referencedEntity, long stateVersion) + public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion) { + var substateData = substate.Value.SubstateData; + if (substateData is CoreModel.AccountAuthorizedDepositorEntrySubstate accountAuthorizedDepositorEntry) { _changeTracker @@ -207,13 +209,13 @@ public void ProcessChanges() } } - public async Task LoadDependencies() + public async Task LoadDependenciesAsync() { _mostRecentEntries.AddRange(await MostRecentAccountAuthorizedDepositorHistory()); _mostRecentAggregates.AddRange(await MostRecentAccountAuthorizedDepositorAggregateHistory()); } - public async Task SaveEntities() + public async Task SaveEntitiesAsync() { var rowsInserted = 0; @@ -233,28 +235,28 @@ AccountAuthorizedDepositorsChangePointerLookup lookup switch (accountAuthorizedDepositorEntrySubstate.Key.Badge) { case CoreModel.ResourceAuthorizedDepositorBadge resourceBadge: - { - var resourceEntityId = _referencedEntityDictionary.Get((EntityAddress)resourceBadge.ResourceAddress).DatabaseId; - - entryLookup = new AccountAuthorizedDepositorsDbLookup( - AccountEntityId: lookup.AccountEntityId, - ResourceEntityId: resourceEntityId, - NonFungibleId: null - ); - break; - } + { + var resourceEntityId = _referencedEntityDictionary.Get((EntityAddress)resourceBadge.ResourceAddress).DatabaseId; + + entryLookup = new AccountAuthorizedDepositorsDbLookup( + AccountEntityId: lookup.AccountEntityId, + ResourceEntityId: resourceEntityId, + NonFungibleId: null + ); + break; + } case CoreModel.NonFungibleAuthorizedDepositorBadge nonFungibleBadge: - { - var resourceEntityId = _referencedEntityDictionary.Get((EntityAddress)nonFungibleBadge.NonFungibleGlobalId.ResourceAddress).DatabaseId; - - entryLookup = new AccountAuthorizedDepositorsDbLookup( - AccountEntityId: lookup.AccountEntityId, - ResourceEntityId: resourceEntityId, - NonFungibleId: nonFungibleBadge.NonFungibleGlobalId.LocalId.SimpleRep - ); - break; - } + { + var resourceEntityId = _referencedEntityDictionary.Get((EntityAddress)nonFungibleBadge.NonFungibleGlobalId.ResourceAddress).DatabaseId; + + entryLookup = new AccountAuthorizedDepositorsDbLookup( + AccountEntityId: lookup.AccountEntityId, + ResourceEntityId: resourceEntityId, + NonFungibleId: nonFungibleBadge.NonFungibleGlobalId.LocalId.SimpleRep + ); + break; + } default: throw new UnreachableException( @@ -283,7 +285,7 @@ private async Task.Empty; } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/AccountDefaultDepositRuleProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountDefaultDepositRuleProcessor.cs similarity index 90% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/AccountDefaultDepositRuleProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountDefaultDepositRuleProcessor.cs index d4183550a..5290fa2d4 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/AccountDefaultDepositRuleProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountDefaultDepositRuleProcessor.cs @@ -71,15 +71,15 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; -internal record struct AccountDefaultDepositRuleChangePointerLookup(long AccountEntityId, long StateVersion); - -internal record AccountDefaultDepositRuleChangePointer +internal class AccountDefaultDepositRuleProcessor : IProcessorBase, ISubstateUpsertProcessor { - public List AccountFieldStateEntries { get; } = new(); -} + private record struct AccountDefaultDepositRuleChangePointerLookup(long AccountEntityId, long StateVersion); + + private record AccountDefaultDepositRuleChangePointer + { + public List AccountFieldStateEntries { get; } = new(); + } -internal class AccountDefaultDepositRuleProcessor -{ private readonly ProcessorContext _context; private readonly ChangeTracker _changeTracker = new(); private readonly List _rulesToAdd = new(); @@ -89,8 +89,10 @@ public AccountDefaultDepositRuleProcessor(ProcessorContext context) _context = context; } - public void VisitUpsert(CoreModel.Substate substateData, ReferencedEntity referencedEntity, long stateVersion) + public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion) { + var substateData = substate.Value.SubstateData; + if (substateData is CoreModel.AccountFieldStateSubstate accountFieldState) { _changeTracker @@ -102,6 +104,11 @@ public void VisitUpsert(CoreModel.Substate substateData, ReferencedEntity refere } } + public Task LoadDependenciesAsync() + { + return Task.CompletedTask; + } + public void ProcessChanges() { foreach (var (lookup, change) in _changeTracker.AsEnumerable()) @@ -120,7 +127,7 @@ public void ProcessChanges() } } - public async Task SaveEntities() + public async Task SaveEntitiesAsync() { var rowsInserted = 0; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/AccountLockerProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountLockerProcessor.cs similarity index 97% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/AccountLockerProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountLockerProcessor.cs index 7e3c25d14..a51f4e3b4 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/AccountLockerProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountLockerProcessor.cs @@ -74,11 +74,11 @@ using CoreModel = RadixDlt.CoreApiSdk.Model; using GatewayModel = RadixDlt.NetworkGateway.GatewayApiSdk.Model; -namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; +namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors; internal record struct AccountLockerEntryDbLookup(long LockerEntityId, long AccountEntityId); -internal class AccountLockerProcessor +internal class AccountLockerProcessor : IProcessorBase, ISubstateUpsertProcessor { private record ObservedTouch(long AccountLockerEntityId, long AccountEntityId, long StateVersion); @@ -101,8 +101,10 @@ public AccountLockerProcessor(ProcessorContext context, ReferencedEntityDictiona _referencedEntities = referencedEntities; } - public void VisitUpsert(CoreModel.Substate substateData, ReferencedEntity referencedEntity, long stateVersion) + public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion) { + var substateData = substate.Value.SubstateData; + if (substateData is CoreModel.AccountLockerAccountClaimsEntrySubstate accountLocker) { var account = _referencedEntities.Get((EntityAddress)accountLocker.Key.AccountAddress); @@ -156,7 +158,7 @@ public void VisitUpsert(CoreModel.Substate substateData, ReferencedEntity refere } } - public async Task LoadDependencies() + public async Task LoadDependenciesAsync() { _existingEntryDefinitions.AddRange(await ExistingAccountLockerEntryDefinitions()); } @@ -184,7 +186,7 @@ public void ProcessChanges() })); } - public async Task SaveEntities() + public async Task SaveEntitiesAsync() { var rowsInserted = 0; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/AccountResourcePreferenceRulesProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountResourcePreferenceRulesProcessor.cs similarity index 94% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/AccountResourcePreferenceRulesProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountResourcePreferenceRulesProcessor.cs index ee870dddd..2092f70ad 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/AccountResourcePreferenceRulesProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountResourcePreferenceRulesProcessor.cs @@ -74,17 +74,17 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; -internal record struct AccountResourcePreferenceRuleDbLookup(long AccountEntityId, long ResourceEntityId); +internal class AccountResourcePreferenceRulesProcessor : IProcessorBase, ISubstateUpsertProcessor +{ + private record struct AccountResourcePreferenceRuleDbLookup(long AccountEntityId, long ResourceEntityId); -internal record struct AccountResourcePreferenceRuleChangePointerLookup(long AccountEntityId, long ResourceEntityId, long StateVersion); + private record struct AccountResourcePreferenceRuleChangePointerLookup(long AccountEntityId, long ResourceEntityId, long StateVersion); -internal record AccountResourcePreferenceRuleChangePointer -{ - public List Entries { get; } = new(); -} + private record AccountResourcePreferenceRuleChangePointer + { + public List Entries { get; } = new(); + } -internal class AccountResourcePreferenceRulesProcessor -{ private readonly ProcessorContext _context; private readonly ReferencedEntityDictionary _referencedEntityDictionary; @@ -102,8 +102,10 @@ public AccountResourcePreferenceRulesProcessor(ProcessorContext context, Referen _referencedEntityDictionary = referencedEntityDictionary; } - public void VisitUpsert(CoreModel.Substate substateData, ReferencedEntity referencedEntity, long stateVersion) + public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion) { + var substateData = substate.Value.SubstateData; + if (substateData is CoreModel.AccountResourcePreferenceEntrySubstate accountResourcePreferenceEntry) { _changeTracker @@ -184,13 +186,13 @@ public void ProcessChanges() } } - public async Task LoadDependencies() + public async Task LoadDependenciesAsync() { _mostRecentEntries.AddRange(await MostRecentAccountResourcePreferenceRuleHistory()); _mostRecentAggregates.AddRange(await MostRecentAccountResourcePreferenceRuleAggregateHistory()); } - public async Task SaveEntities() + public async Task SaveEntitiesAsync() { var rowsInserted = 0; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ComponentMethodRoyaltyProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/ComponentMethodRoyaltyProcessor.cs similarity index 93% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ComponentMethodRoyaltyProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/ComponentMethodRoyaltyProcessor.cs index baced91ec..0d71c95f9 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ComponentMethodRoyaltyProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/ComponentMethodRoyaltyProcessor.cs @@ -72,17 +72,17 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; -internal record struct ComponentMethodRoyaltyEntryDbLookup(long EntityId, string MethodName); +internal class ComponentMethodRoyaltyProcessor : IProcessorBase, ISubstateUpsertProcessor +{ + private record struct ComponentMethodRoyaltyEntryDbLookup(long EntityId, string MethodName); -internal record struct ComponentMethodRoyaltyChangePointerLookup(long EntityId, long StateVersion); + private record struct ComponentMethodRoyaltyChangePointerLookup(long EntityId, long StateVersion); -internal record ComponentMethodRoyaltyChangePointer(ReferencedEntity ReferencedEntity) -{ - public List Entries { get; } = new(); -} + private record ComponentMethodRoyaltyChangePointer(ReferencedEntity ReferencedEntity) + { + public List Entries { get; } = new(); + } -internal class ComponentMethodRoyaltyProcessor -{ private readonly ProcessorContext _context; private ChangeTracker _changes = new(); @@ -98,17 +98,20 @@ public ComponentMethodRoyaltyProcessor(ProcessorContext context) _context = context; } - public void VisitUpsert(CoreModel.Substate substateData, ReferencedEntity referencedEntity, long stateVersion) + public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion) { + var substateData = substate.Value.SubstateData; + if (substateData is CoreModel.RoyaltyModuleMethodRoyaltyEntrySubstate methodRoyaltyEntry) { _changes .GetOrAdd(new ComponentMethodRoyaltyChangePointerLookup(referencedEntity.DatabaseId, stateVersion), _ => new ComponentMethodRoyaltyChangePointer(referencedEntity)) - .Entries.Add(methodRoyaltyEntry); + .Entries + .Add(methodRoyaltyEntry); } } - public async Task LoadDependencies() + public async Task LoadDependenciesAsync() { _mostRecentEntries.AddRange(await MostRecentComponentMethodRoyaltyEntryHistory()); _mostRecentAggregates.AddRange(await MostRecentComponentMethodRoyaltyAggregateHistory()); @@ -178,7 +181,7 @@ public void ProcessChanges() } } - public async Task SaveEntities() + public async Task SaveEntitiesAsync() { var rowsInserted = 0; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/EntityMetadataProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/EntityMetadataProcessor.cs similarity index 98% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/EntityMetadataProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/EntityMetadataProcessor.cs index 2f8b0aabd..55d73b190 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/EntityMetadataProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/EntityMetadataProcessor.cs @@ -84,7 +84,7 @@ internal record MetadataChangePointer public List Entries { get; } = new(); } -internal class EntityMetadataProcessor +internal class EntityMetadataProcessor : IProcessorBase, ISubstateUpsertProcessor { private readonly ProcessorContext _context; @@ -119,7 +119,7 @@ public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity r } } - public async Task LoadDependencies() + public async Task LoadDependenciesAsync() { _existingEntryDefinitions.AddRange(await ExistingMetadataEntryDefinitions()); _existingTotalsHistory.AddRange(await ExistingMetadataTotalsHistory()); @@ -194,7 +194,7 @@ public void ProcessChanges() } } - public async Task SaveEntities() + public async Task SaveEntitiesAsync() { var rowsInserted = 0; rowsInserted += await CopyEntityMetadataEntryHistory(); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/EntityResourceProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/EntityResourceProcessor.cs similarity index 98% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/EntityResourceProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/EntityResourceProcessor.cs index 6758280e2..88bc78902 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/EntityResourceProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/EntityResourceProcessor.cs @@ -80,7 +80,7 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; -internal class EntityResourceProcessor +internal class EntityResourceProcessor : IProcessorBase, ISubstateUpsertProcessor { private readonly record struct ByEntityDbLookup(long EntityId); @@ -128,8 +128,10 @@ public EntityResourceProcessor(ProcessorContext context, CommonDbContext commonD _observers = observers; } - public void VisitUpsert(CoreModel.Substate substateData, ReferencedEntity referencedEntity, long stateVersion, CoreModel.IUpsertedSubstate substate) + public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion) { + var substateData = substate.Value.SubstateData; + if (substateData is CoreModel.FungibleVaultFieldBalanceSubstate fungibleBalanceSubstate) { var vaultEntity = referencedEntity.GetDatabaseEntity(); @@ -156,7 +158,7 @@ public void VisitUpsert(CoreModel.Substate substateData, ReferencedEntity refere } } - public async Task LoadDependencies() + public async Task LoadDependenciesAsync() { _existingResourceDefinitions.AddRange(await ExistingEntityResourceEntryDefinitions()); _mostRecentResourceTotalsHistory.AddRange(await MostRecentEntityResourceTotalsHistory()); @@ -189,7 +191,7 @@ public void ProcessChanges() } } - public async Task SaveEntities() + public async Task SaveEntitiesAsync() { var rowsInserted = 0; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/EntityRoleAssignmentProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/EntityRoleAssignmentProcessor.cs similarity index 90% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/EntityRoleAssignmentProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/EntityRoleAssignmentProcessor.cs index 3da4d3eb4..d68a76810 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/EntityRoleAssignmentProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/EntityRoleAssignmentProcessor.cs @@ -75,37 +75,39 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; -internal record struct RoleAssignmentEntryDbLookup(long EntityId, string KeyRole, ModuleId KeyModule); +internal class EntityRoleAssignmentProcessor : IProcessorBase, ISubstateUpsertProcessor +{ + private record struct RoleAssignmentEntryDbLookup(long EntityId, string KeyRole, ModuleId KeyModule); -internal record struct RoleAssignmentsChangePointerLookup(long EntityId, long StateVersion); + private record struct RoleAssignmentsChangePointerLookup(long EntityId, long StateVersion); -internal record RoleAssignmentsChangePointer -{ - public CoreModel.RoleAssignmentModuleFieldOwnerRoleSubstate? OwnerRole { get; set; } + private record RoleAssignmentsChangePointer + { + public CoreModel.RoleAssignmentModuleFieldOwnerRoleSubstate? OwnerRole { get; set; } - public List Entries { get; } = new(); -} + public List Entries { get; } = new(); + } -internal class EntityRoleAssignmentProcessor -{ private readonly ProcessorContext _context; - private ChangeTracker _changes = new(); + private readonly ChangeTracker _changes = new(); - private Dictionary _mostRecentAggregates = new(); - private Dictionary _mostRecentEntries = new(); + private readonly Dictionary _mostRecentAggregates = new(); + private readonly Dictionary _mostRecentEntries = new(); - private List _aggregatesToAdd = new(); - private List _entriesToAdd = new(); - private List _ownersToAdd = new(); + private readonly List _aggregatesToAdd = new(); + private readonly List _entriesToAdd = new(); + private readonly List _ownersToAdd = new(); public EntityRoleAssignmentProcessor(ProcessorContext context) { _context = context; } - public void VisitUpsert(CoreModel.Substate substateData, ReferencedEntity referencedEntity, long stateVersion) + public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion) { + var substateData = substate.Value.SubstateData; + if (substateData is CoreModel.RoleAssignmentModuleFieldOwnerRoleSubstate accessRulesFieldOwnerRole) { _changes @@ -121,7 +123,7 @@ public void VisitUpsert(CoreModel.Substate substateData, ReferencedEntity refere } } - public async Task LoadDependencies() + public async Task LoadDependenciesAsync() { _mostRecentEntries.AddRange(await MostRecentEntityRoleAssignmentsEntryHistory()); _mostRecentAggregates.AddRange(await MostRecentEntityRoleAssignmentsAggregateHistory()); @@ -208,7 +210,7 @@ public void ProcessChanges() } } - public async Task SaveEntities() + public async Task SaveEntitiesAsync() { var rowsInserted = 0; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/EntitySchemaProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/EntitySchemaProcessor.cs similarity index 61% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/EntitySchemaProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/EntitySchemaProcessor.cs index 28ac0f3a9..168fc0801 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/EntitySchemaProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/EntitySchemaProcessor.cs @@ -76,20 +76,21 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; -internal record struct SchemaDefinitionEntryDbLookup(long EntityId, ValueBytes SchemaHash); +internal class EntitySchemaProcessor : IProcessorBase, ISubstateUpsertProcessor, ISubstateDeleteProcessor +{ + private record struct SchemaDefinitionEntryDbLookup(long EntityId, ValueBytes SchemaHash); -internal record struct SchemaChangePointerLookup(long EntityId, long StateVersion); + private record struct SchemaChangePointerLookup(long EntityId, long StateVersion); -internal record SchemaChangePointer -{ - public List Entries { get; } = new(); + private record SchemaChangePointer + { + public List Entries { get; } = new(); - public List DeletedSchemaHashes { get; } = new(); -} + public List DeletedSchemaHashes { get; } = new(); + } -internal class EntitySchemaProcessor -{ private readonly ProcessorContext _context; + private readonly ReferencedEntityDictionary _referencedEntities; private readonly ChangeTracker _changes = new(); @@ -98,14 +99,19 @@ internal class EntitySchemaProcessor private readonly List _aggregatesToAdd = new(); private readonly List _definitionsToAdd = new(); + private readonly List _nonFungibleSchemaHistoryToAdd = new(); + private readonly List _keyValueStoreSchemaHistoryToAdd = new(); - public EntitySchemaProcessor(ProcessorContext context) + public EntitySchemaProcessor(ProcessorContext context, ReferencedEntityDictionary referencedEntities) { _context = context; + _referencedEntities = referencedEntities; } - public void VisitUpsert(CoreModel.Substate substateData, ReferencedEntity referencedEntity, long stateVersion) + public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion) { + var substateData = substate.Value.SubstateData; + if (substateData is CoreModel.SchemaEntrySubstate schemaEntry) { _changes @@ -113,6 +119,57 @@ public void VisitUpsert(CoreModel.Substate substateData, ReferencedEntity refere .Entries .Add(schemaEntry); } + + if (substateData is CoreModel.TypeInfoModuleFieldTypeInfoSubstate typeInfoSubstate) + { + if (typeInfoSubstate.TryGetNonFungibleDataSchemaDetails(out var nonFungibleDataSchemaDetails)) + { + var schemaDefiningEntityId = !string.IsNullOrEmpty(nonFungibleDataSchemaDetails.Value.SchemaDefiningEntityAddress) + ? _referencedEntities.Get((EntityAddress)nonFungibleDataSchemaDetails.Value.SchemaDefiningEntityAddress).DatabaseId + : referencedEntity.DatabaseId; + + _nonFungibleSchemaHistoryToAdd.Add( + new NonFungibleSchemaHistory + { + Id = _context.Sequences.NonFungibleSchemaHistorySequence++, + ResourceEntityId = referencedEntity.DatabaseId, + SchemaHash = nonFungibleDataSchemaDetails.Value.SchemaHash.ConvertFromHex(), + SborTypeKind = nonFungibleDataSchemaDetails.Value.SborTypeKind.ToModel(), + TypeIndex = nonFungibleDataSchemaDetails.Value.TypeIndex, + SchemaDefiningEntityId = schemaDefiningEntityId, + FromStateVersion = stateVersion, + }); + } + + if (typeInfoSubstate.Value.Details is CoreModel.KeyValueStoreTypeInfoDetails + && typeInfoSubstate.TryGetKeyValueStoreKeySchemaDetails(out var keySchemaDetails) + && typeInfoSubstate.TryGetKeyValueStoreValueSchemaDetails(out var valueSchemaDetails)) + { + var keySchemaDefiningEntityId = !string.IsNullOrEmpty(keySchemaDetails.Value.SchemaDefiningEntityAddress) + ? _referencedEntities.Get((EntityAddress)keySchemaDetails.Value.SchemaDefiningEntityAddress).DatabaseId + : referencedEntity.DatabaseId; + + var valueSchemaDefiningEntityId = !string.IsNullOrEmpty(valueSchemaDetails.Value.SchemaDefiningEntityAddress) + ? _referencedEntities.Get((EntityAddress)valueSchemaDetails.Value.SchemaDefiningEntityAddress).DatabaseId + : referencedEntity.DatabaseId; + + _keyValueStoreSchemaHistoryToAdd.Add( + new KeyValueStoreSchemaHistory + { + Id = _context.Sequences.KeyValueSchemaHistorySequence++, + KeyValueStoreEntityId = referencedEntity.DatabaseId, + KeySchemaDefiningEntityId = keySchemaDefiningEntityId, + KeySchemaHash = keySchemaDetails.Value.SchemaHash.ConvertFromHex(), + KeySborTypeKind = keySchemaDetails.Value.SborTypeKind.ToModel(), + KeyTypeIndex = keySchemaDetails.Value.TypeIndex, + ValueSchemaDefiningEntityId = valueSchemaDefiningEntityId, + ValueSchemaHash = valueSchemaDetails.Value.SchemaHash.ConvertFromHex(), + ValueSborTypeKind = valueSchemaDetails.Value.SborTypeKind.ToModel(), + ValueTypeIndex = valueSchemaDetails.Value.TypeIndex, + FromStateVersion = stateVersion, + }); + } + } } public void VisitDelete(CoreModel.SubstateId substateId, ReferencedEntity referencedEntity, long stateVersion) @@ -129,7 +186,7 @@ public void VisitDelete(CoreModel.SubstateId substateId, ReferencedEntity refere } } - public async Task LoadDependencies() + public async Task LoadDependenciesAsync() { _mostRecentAggregates.AddRange(await MostRecentSchemaEntryAggregateHistory()); _existingSchemas.AddRange(await LoadExistingSchemas()); @@ -194,23 +251,27 @@ public void ProcessChanges() } else { - throw new UnreachableException($"Unexpected situation where SchemaEntryDefinition with EntityId:{entryLookup.EntityId}, SchemaHash:{deletedSchemaHash} got deleted but wasn't found in aggregate table."); + throw new UnreachableException( + $"Unexpected situation where SchemaEntryDefinition with EntityId:{entryLookup.EntityId}, SchemaHash:{deletedSchemaHash} got deleted but wasn't found in aggregate table."); } } else { - throw new UnreachableException($"Unexpected situation where SchemaEntryDefinition with EntityId:{entryLookup.EntityId}, SchemaHash:{deletedSchemaHash} got deleted but wasn't found in gateway database."); + throw new UnreachableException( + $"Unexpected situation where SchemaEntryDefinition with EntityId:{entryLookup.EntityId}, SchemaHash:{deletedSchemaHash} got deleted but wasn't found in gateway database."); } } } } - public async Task SaveEntities() + public async Task SaveEntitiesAsync() { var rowsInserted = 0; rowsInserted += await CopySchemaEntryDefinitions(); rowsInserted += await CopySchemaEntryAggregateHistory(); + rowsInserted += await CopyNonFungibleDataSchemaHistory(); + rowsInserted += await CopyKeyValueStoreSchemaHistory(); return rowsInserted; } @@ -288,4 +349,36 @@ private Task CopySchemaEntryAggregateHistory() => _context.WriteHelper.Copy await writer.WriteAsync(e.EntityId, NpgsqlDbType.Bigint, token); await writer.WriteAsync(e.EntryIds, NpgsqlDbType.Array | NpgsqlDbType.Bigint, token); }); + + private Task CopyNonFungibleDataSchemaHistory() => _context.WriteHelper.Copy( + _nonFungibleSchemaHistoryToAdd, + "COPY non_fungible_schema_history (id, from_state_version, resource_entity_id, schema_defining_entity_id, schema_hash, sbor_type_kind, type_index) FROM STDIN (FORMAT BINARY)", + async (writer, e, token) => + { + await writer.WriteAsync(e.Id, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(e.FromStateVersion, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(e.ResourceEntityId, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(e.SchemaDefiningEntityId, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(e.SchemaHash, NpgsqlDbType.Bytea, token); + await writer.WriteAsync(e.SborTypeKind, "sbor_type_kind", token); + await writer.WriteAsync(e.TypeIndex, NpgsqlDbType.Bigint, token); + }); + + private Task CopyKeyValueStoreSchemaHistory() => _context.WriteHelper.Copy( + _keyValueStoreSchemaHistoryToAdd, + "COPY key_value_store_schema_history (id, from_state_version, key_value_store_entity_id, key_schema_defining_entity_id, key_schema_hash, key_sbor_type_kind, key_type_index, value_schema_defining_entity_id, value_schema_hash, value_sbor_type_kind, value_type_index) FROM STDIN (FORMAT BINARY)", + async (writer, e, token) => + { + await writer.WriteAsync(e.Id, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(e.FromStateVersion, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(e.KeyValueStoreEntityId, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(e.KeySchemaDefiningEntityId, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(e.KeySchemaHash, NpgsqlDbType.Bytea, token); + await writer.WriteAsync(e.KeySborTypeKind, "sbor_type_kind", token); + await writer.WriteAsync(e.KeyTypeIndex, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(e.ValueSchemaDefiningEntityId, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(e.ValueSchemaHash, NpgsqlDbType.Bytea, token); + await writer.WriteAsync(e.ValueSborTypeKind, "sbor_type_kind", token); + await writer.WriteAsync(e.ValueTypeIndex, NpgsqlDbType.Bigint, token); + }); } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/EntityStateProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/EntityStateProcessor.cs similarity index 72% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/EntityStateProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/EntityStateProcessor.cs index 807c2ef16..20cffe8b3 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/EntityStateProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/EntityStateProcessor.cs @@ -75,7 +75,7 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; -internal class EntityStateProcessor +internal class EntityStateProcessor : IProcessorBase, ISubstateUpsertProcessor { private readonly ProcessorContext _context; private readonly ReferencedEntityDictionary _referencedEntities; @@ -101,87 +101,103 @@ public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity r var schemaDetails = objectFieldStructure.ValueSchema.GetSchemaDetails(); - _toAdd.Add(new SborStateHistory - { - Id = _context.Sequences.StateHistorySequence++, - FromStateVersion = stateVersion, - EntityId = referencedEntity.DatabaseId, - SborState = componentState.Value.DataStruct.StructData.GetDataBytes(), - SchemaHash = schemaDetails.SchemaHash.ConvertFromHex(), - SborTypeKind = schemaDetails.SborTypeKind.ToModel(), - TypeIndex = schemaDetails.TypeIndex, - SchemaDefiningEntityId = _referencedEntities.Get((EntityAddress)schemaDetails.SchemaDefiningEntityAddress).DatabaseId, - }); + _toAdd.Add( + new SborStateHistory + { + Id = _context.Sequences.StateHistorySequence++, + FromStateVersion = stateVersion, + EntityId = referencedEntity.DatabaseId, + SborState = componentState.Value.DataStruct.StructData.GetDataBytes(), + SchemaHash = schemaDetails.SchemaHash.ConvertFromHex(), + SborTypeKind = schemaDetails.SborTypeKind.ToModel(), + TypeIndex = schemaDetails.TypeIndex, + SchemaDefiningEntityId = _referencedEntities.Get((EntityAddress)schemaDetails.SchemaDefiningEntityAddress).DatabaseId, + }); } if (substateData is CoreModel.ValidatorFieldStateSubstate validator) { - _toAdd.Add(new JsonStateHistory - { - Id = _context.Sequences.StateHistorySequence++, - FromStateVersion = stateVersion, - EntityId = referencedEntity.DatabaseId, - JsonState = validator.Value.ToJson(), - }); + _toAdd.Add( + new JsonStateHistory + { + Id = _context.Sequences.StateHistorySequence++, + FromStateVersion = stateVersion, + EntityId = referencedEntity.DatabaseId, + JsonState = validator.Value.ToJson(), + }); } if (substateData is CoreModel.AccountFieldStateSubstate accountFieldState) { - _toAdd.Add(new JsonStateHistory - { - Id = _context.Sequences.StateHistorySequence++, - FromStateVersion = stateVersion, - EntityId = referencedEntity.DatabaseId, - JsonState = accountFieldState.Value.ToJson(), - }); + _toAdd.Add( + new JsonStateHistory + { + Id = _context.Sequences.StateHistorySequence++, + FromStateVersion = stateVersion, + EntityId = referencedEntity.DatabaseId, + JsonState = accountFieldState.Value.ToJson(), + }); } if (substateData is CoreModel.AccessControllerFieldStateSubstate accessControllerFieldState) { - _toAdd.Add(new JsonStateHistory - { - Id = _context.Sequences.StateHistorySequence++, - FromStateVersion = stateVersion, - EntityId = referencedEntity.DatabaseId, - JsonState = accessControllerFieldState.Value.ToJson(), - }); + _toAdd.Add( + new JsonStateHistory + { + Id = _context.Sequences.StateHistorySequence++, + FromStateVersion = stateVersion, + EntityId = referencedEntity.DatabaseId, + JsonState = accessControllerFieldState.Value.ToJson(), + }); } if (substateData is CoreModel.OneResourcePoolFieldStateSubstate oneResourcePoolFieldStateSubstate) { - _toAdd.Add(new JsonStateHistory - { - Id = _context.Sequences.StateHistorySequence++, - FromStateVersion = stateVersion, - EntityId = referencedEntity.DatabaseId, - JsonState = oneResourcePoolFieldStateSubstate.Value.ToJson(), - }); + _toAdd.Add( + new JsonStateHistory + { + Id = _context.Sequences.StateHistorySequence++, + FromStateVersion = stateVersion, + EntityId = referencedEntity.DatabaseId, + JsonState = oneResourcePoolFieldStateSubstate.Value.ToJson(), + }); } if (substateData is CoreModel.TwoResourcePoolFieldStateSubstate twoResourcePoolFieldStateSubstate) { - _toAdd.Add(new JsonStateHistory - { - Id = _context.Sequences.StateHistorySequence++, - FromStateVersion = stateVersion, - EntityId = referencedEntity.DatabaseId, - JsonState = twoResourcePoolFieldStateSubstate.Value.ToJson(), - }); + _toAdd.Add( + new JsonStateHistory + { + Id = _context.Sequences.StateHistorySequence++, + FromStateVersion = stateVersion, + EntityId = referencedEntity.DatabaseId, + JsonState = twoResourcePoolFieldStateSubstate.Value.ToJson(), + }); } if (substateData is CoreModel.MultiResourcePoolFieldStateSubstate multiResourcePoolFieldStateSubstate) { - _toAdd.Add(new JsonStateHistory - { - Id = _context.Sequences.StateHistorySequence++, - FromStateVersion = stateVersion, - EntityId = referencedEntity.DatabaseId, - JsonState = multiResourcePoolFieldStateSubstate.Value.ToJson(), - }); + _toAdd.Add( + new JsonStateHistory + { + Id = _context.Sequences.StateHistorySequence++, + FromStateVersion = stateVersion, + EntityId = referencedEntity.DatabaseId, + JsonState = multiResourcePoolFieldStateSubstate.Value.ToJson(), + }); } } - public async Task SaveEntities() + public Task LoadDependenciesAsync() + { + return Task.CompletedTask; + } + + public void ProcessChanges() + { + } + + public async Task SaveEntitiesAsync() { var rowsInserted = 0; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/KeyValueStoreProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/KeyValueStoreProcessor.cs similarity index 90% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/KeyValueStoreProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/KeyValueStoreProcessor.cs index b4fdec0eb..9a806c4e6 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/KeyValueStoreProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/KeyValueStoreProcessor.cs @@ -73,14 +73,14 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; -internal record struct KeyValueStoreEntryDbLookup(long KeyValueStoreEntityId, ValueBytes Key); +internal class KeyValueStoreProcessor : IProcessorBase, ISubstateUpsertProcessor +{ + private record struct KeyValueStoreEntryDbLookup(long KeyValueStoreEntityId, ValueBytes Key); -internal record struct KeyValueStoreChangePointerLookup(long KeyValueStoreEntityId, long StateVersion, ValueBytes Key); + private record struct KeyValueStoreChangePointerLookup(long KeyValueStoreEntityId, long StateVersion, ValueBytes Key); -internal record KeyValueStoreChangePointer(CoreModel.GenericKeyValueStoreEntrySubstate NewSubstate, CoreModel.GenericKeyValueStoreEntrySubstate? PreviousSubstate); + private record KeyValueStoreChangePointer(CoreModel.GenericKeyValueStoreEntrySubstate NewSubstate, CoreModel.GenericKeyValueStoreEntrySubstate? PreviousSubstate); -internal class KeyValueStoreProcessor -{ private readonly ProcessorContext _context; private readonly Dictionary _observedEntryDefinitions = new(); @@ -91,7 +91,7 @@ internal class KeyValueStoreProcessor private readonly List _entryHistoryToAdd = new(); private readonly List _totalsHistoryToAdd = new(); - private ChangeTracker _changes = new(); + private readonly ChangeTracker _changes = new(); public KeyValueStoreProcessor(ProcessorContext context) { @@ -112,7 +112,7 @@ public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity r } } - public async Task LoadDependencies() + public async Task LoadDependenciesAsync() { _existingEntryDefinitions.AddRange(await ExistingKeyValueStoreEntryDefinitions()); _existingTotalsHistory.AddRange(await ExistingKeyValueStoreTotalsHistory()); @@ -138,15 +138,16 @@ public void ProcessChanges() { var isDeleted = change.Value.NewSubstate.Value == null; - _entryHistoryToAdd.Add(new KeyValueStoreEntryHistory - { - Id = _context.Sequences.KeyValueStoreEntryHistorySequence++, - FromStateVersion = change.Key.StateVersion, - KeyValueStoreEntryDefinitionId = _existingEntryDefinitions[new KeyValueStoreEntryDbLookup(change.Key.KeyValueStoreEntityId, change.Key.Key)].Id, - Value = isDeleted ? null : change.Value.NewSubstate.Value!.Data.StructData.GetDataBytes(), - IsDeleted = isDeleted, - IsLocked = change.Value.NewSubstate.IsLocked, - }); + _entryHistoryToAdd.Add( + new KeyValueStoreEntryHistory + { + Id = _context.Sequences.KeyValueStoreEntryHistorySequence++, + FromStateVersion = change.Key.StateVersion, + KeyValueStoreEntryDefinitionId = _existingEntryDefinitions[new KeyValueStoreEntryDbLookup(change.Key.KeyValueStoreEntityId, change.Key.Key)].Id, + Value = isDeleted ? null : change.Value.NewSubstate.Value!.Data.StructData.GetDataBytes(), + IsDeleted = isDeleted, + IsLocked = change.Value.NewSubstate.IsLocked, + }); var totalsExists = _existingTotalsHistory.TryGetValue(change.Key.KeyValueStoreEntityId, out var previousTotals); @@ -181,7 +182,7 @@ public void ProcessChanges() } } - public async Task SaveEntities() + public async Task SaveEntitiesAsync() { var rowsInserted = 0; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/AffectedGlobalEntitiesProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/AffectedGlobalEntitiesProcessor.cs similarity index 79% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/AffectedGlobalEntitiesProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/AffectedGlobalEntitiesProcessor.cs index 14a1c1d3c..e78851c4b 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/AffectedGlobalEntitiesProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/AffectedGlobalEntitiesProcessor.cs @@ -67,33 +67,32 @@ using RadixDlt.NetworkGateway.PostgresIntegration.Models; using System.Collections.Generic; using System.Linq; +using CoreModel = RadixDlt.CoreApiSdk.Model; -namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; +namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors.LedgerTransactionMarkers; -internal class AffectedGlobalEntitiesProcessor +internal class AffectedGlobalEntitiesProcessor : ITransactionMarkerProcessor, ISubstateUpsertProcessor, ISubstateDeleteProcessor { - private readonly long[] _excludedEntityIds; private readonly ProcessorContext _context; + private readonly ReferencedEntityDictionary _referencedEntities; + private readonly NetworkConfiguration _networkConfiguration; private readonly Dictionary> _affectedGlobalEntities = new(); public AffectedGlobalEntitiesProcessor(ProcessorContext context, ReferencedEntityDictionary referencedEntities, NetworkConfiguration networkConfiguration) { _context = context; - _excludedEntityIds = new[] - { - referencedEntities.Get((EntityAddress)networkConfiguration.WellKnownAddresses.ConsensusManager).DatabaseId, - referencedEntities.Get((EntityAddress)networkConfiguration.WellKnownAddresses.TransactionTracker).DatabaseId, - }; + _referencedEntities = referencedEntities; + _networkConfiguration = networkConfiguration; } - public void VisitUpsert(ReferencedEntity referencedEntity, long stateVersion) + public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion) { _affectedGlobalEntities .GetOrAdd(stateVersion, _ => new HashSet()) .Add(referencedEntity.AffectedGlobalEntityId); } - public void VisitDelete(ReferencedEntity referencedEntity, long stateVersion) + public void VisitDelete(CoreApiSdk.Model.SubstateId substateId, ReferencedEntity referencedEntity, long stateVersion) { _affectedGlobalEntities .GetOrAdd(stateVersion, _ => new HashSet()) @@ -105,21 +104,24 @@ public HashSet GetAllAffectedGlobalEntities(long stateVersion) return _affectedGlobalEntities.TryGetValue(stateVersion, out var hashSet) ? hashSet : new HashSet(); } - public IEnumerable CreateTransactionMarkers() + public IEnumerable CreateTransactionMarkers() { + var excludedEntityIds = new[] + { + _referencedEntities.Get((EntityAddress)_networkConfiguration.WellKnownAddresses.ConsensusManager).DatabaseId, + _referencedEntities.Get((EntityAddress)_networkConfiguration.WellKnownAddresses.TransactionTracker).DatabaseId, + }; + foreach (var stateVersionAffectedEntities in _affectedGlobalEntities) { - foreach (var entityId in stateVersionAffectedEntities.Value) + foreach (var entityId in stateVersionAffectedEntities.Value.Where(entityId => !excludedEntityIds.Contains(entityId))) { - if (!_excludedEntityIds.Contains(entityId)) + yield return new AffectedGlobalEntityTransactionMarker { - yield return new AffectedGlobalEntityTransactionMarker - { - Id = _context.Sequences.LedgerTransactionMarkerSequence++, - EntityId = entityId, - StateVersion = stateVersionAffectedEntities.Key, - }; - } + Id = _context.Sequences.LedgerTransactionMarkerSequence++, + EntityId = entityId, + StateVersion = stateVersionAffectedEntities.Key, + }; } } } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/EventLedgerTransactionMarkerProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/EventLedgerTransactionMarkerProcessor.cs new file mode 100644 index 000000000..8cf6b68a1 --- /dev/null +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/EventLedgerTransactionMarkerProcessor.cs @@ -0,0 +1,80 @@ +using RadixDlt.NetworkGateway.Abstractions.Network; +using RadixDlt.NetworkGateway.Abstractions.Numerics; +using RadixDlt.NetworkGateway.PostgresIntegration.Models; +using RadixDlt.NetworkGateway.PostgresIntegration.Utils; +using RadixEngineToolkit; +using System.Collections.Generic; + +namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors.LedgerTransactionMarkers; + +internal class EventLedgerTransactionMarkerProcessor : ITransactionMarkerProcessor, IDecodedEventProcessor +{ + private readonly ProcessorContext _context; + private readonly List _ledgerTransactionMarkersToAdd = new(); + + public EventLedgerTransactionMarkerProcessor(ProcessorContext context, ReferencedEntityDictionary _, NetworkConfiguration __) + { + _context = context; + } + + public IEnumerable CreateTransactionMarkers() + { + return _ledgerTransactionMarkersToAdd; + } + + public void VisitDecodedEvent(TypedNativeEvent decodedEvent, ReferencedEntity eventEmitterEntity, long stateVersion) + { + if (EventDecoder.TryGetFungibleVaultWithdrawalEvent(decodedEvent, out var fungibleVaultWithdrawalEvent)) + { + _ledgerTransactionMarkersToAdd.Add( + new EventLedgerTransactionMarker + { + Id = _context.Sequences.LedgerTransactionMarkerSequence++, + StateVersion = stateVersion, + EventType = LedgerTransactionMarkerEventType.Withdrawal, + EntityId = eventEmitterEntity.DatabaseGlobalAncestorId, + ResourceEntityId = eventEmitterEntity.GetDatabaseEntity().GetResourceEntityId(), + Quantity = TokenAmount.FromDecimalString(fungibleVaultWithdrawalEvent.AsStr()), + }); + } + else if (EventDecoder.TryGetFungibleVaultDepositEvent(decodedEvent, out var fungibleVaultDepositEvent)) + { + _ledgerTransactionMarkersToAdd.Add( + new EventLedgerTransactionMarker + { + Id = _context.Sequences.LedgerTransactionMarkerSequence++, + StateVersion = stateVersion, + EventType = LedgerTransactionMarkerEventType.Deposit, + EntityId = eventEmitterEntity.DatabaseGlobalAncestorId, + ResourceEntityId = eventEmitterEntity.GetDatabaseEntity().GetResourceEntityId(), + Quantity = TokenAmount.FromDecimalString(fungibleVaultDepositEvent.AsStr()), + }); + } + else if (EventDecoder.TryGetNonFungibleVaultWithdrawalEvent(decodedEvent, out var nonFungibleVaultWithdrawalEvent)) + { + _ledgerTransactionMarkersToAdd.Add( + new EventLedgerTransactionMarker + { + Id = _context.Sequences.LedgerTransactionMarkerSequence++, + StateVersion = stateVersion, + EventType = LedgerTransactionMarkerEventType.Withdrawal, + EntityId = eventEmitterEntity.DatabaseGlobalAncestorId, + ResourceEntityId = eventEmitterEntity.GetDatabaseEntity().GetResourceEntityId(), + Quantity = TokenAmount.FromDecimalString(nonFungibleVaultWithdrawalEvent.Length.ToString()), + }); + } + else if (EventDecoder.TryGetNonFungibleVaultDepositEvent(decodedEvent, out var nonFungibleVaultDepositEvent)) + { + _ledgerTransactionMarkersToAdd.Add( + new EventLedgerTransactionMarker + { + Id = _context.Sequences.LedgerTransactionMarkerSequence++, + StateVersion = stateVersion, + EventType = LedgerTransactionMarkerEventType.Deposit, + EntityId = eventEmitterEntity.DatabaseGlobalAncestorId, + ResourceEntityId = eventEmitterEntity.GetDatabaseEntity().GetResourceEntityId(), + Quantity = TokenAmount.FromDecimalString(nonFungibleVaultDepositEvent.Length.ToString()), + }); + } + } +} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/GlobalEventEmitterProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/GlobalEventEmitterProcessor.cs similarity index 88% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/GlobalEventEmitterProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/GlobalEventEmitterProcessor.cs index 03c564bb6..c8a1a57a3 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/GlobalEventEmitterProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/GlobalEventEmitterProcessor.cs @@ -70,24 +70,20 @@ using System.Linq; using CoreModel = RadixDlt.CoreApiSdk.Model; -namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; +namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors.LedgerTransactionMarkers; -internal class GlobalEventEmitterProcessor +internal class GlobalEventEmitterProcessor : ITransactionMarkerProcessor, IEventProcessor { - private readonly long[] _excludedEntityIds; private readonly ProcessorContext _context; private readonly ReferencedEntityDictionary _referencedEntities; + private readonly NetworkConfiguration _networkConfiguration; private readonly Dictionary> _globalEventEmitters = new(); public GlobalEventEmitterProcessor(ProcessorContext context, ReferencedEntityDictionary referencedEntities, NetworkConfiguration networkConfiguration) { _context = context; _referencedEntities = referencedEntities; - _excludedEntityIds = new[] - { - referencedEntities.Get((EntityAddress)networkConfiguration.WellKnownAddresses.ConsensusManager).DatabaseId, - referencedEntities.Get((EntityAddress)networkConfiguration.WellKnownAddresses.Xrd).DatabaseId, - }; + _networkConfiguration = networkConfiguration; } public void VisitEvent(CoreModel.Event @event, long stateVersion) @@ -98,16 +94,14 @@ public void VisitEvent(CoreModel.Event @event, long stateVersion) var methodEventEmitterEntity = _referencedEntities.Get((EntityAddress)methodEventEmitterIdentifier.Entity.EntityAddress); var globalEventEmitterEntityId = methodEventEmitterEntity.GlobalEventEmitterEntityId; - if (!_excludedEntityIds.Contains(globalEventEmitterEntityId)) - { - _globalEventEmitters - .GetOrAdd(stateVersion, _ => new HashSet()) - .Add(globalEventEmitterEntityId); - } + _globalEventEmitters + .GetOrAdd(stateVersion, _ => new HashSet()) + .Add(globalEventEmitterEntityId); break; case CoreModel.FunctionEventEmitterIdentifier functionEventEmitterIdentifier: var functionEventEmitterEntity = _referencedEntities.Get((EntityAddress)functionEventEmitterIdentifier.PackageAddress); + _globalEventEmitters .GetOrAdd(stateVersion, _ => new HashSet()) .Add(functionEventEmitterEntity.GlobalEventEmitterEntityId); @@ -117,9 +111,15 @@ public void VisitEvent(CoreModel.Event @event, long stateVersion) } } - public IEnumerable CreateTransactionMarkers() + public IEnumerable CreateTransactionMarkers() { - foreach (var stateVersionAffectedEntities in _globalEventEmitters) + var excludedEntityIds = new[] + { + _referencedEntities.Get((EntityAddress)_networkConfiguration.WellKnownAddresses.ConsensusManager).DatabaseId, + _referencedEntities.Get((EntityAddress)_networkConfiguration.WellKnownAddresses.Xrd).DatabaseId, + }; + + foreach (var stateVersionAffectedEntities in _globalEventEmitters.Where(x => !excludedEntityIds.Contains(x.Key))) { foreach (var entityId in stateVersionAffectedEntities.Value) { diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/LedgerTransactionMarkersProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/LedgerTransactionMarkersProcessor.cs new file mode 100644 index 000000000..387c362bc --- /dev/null +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/LedgerTransactionMarkersProcessor.cs @@ -0,0 +1,168 @@ +using NpgsqlTypes; +using RadixDlt.NetworkGateway.Abstractions.Model; +using RadixDlt.NetworkGateway.Abstractions.Network; +using RadixDlt.NetworkGateway.PostgresIntegration.Models; +using RadixDlt.NetworkGateway.PostgresIntegration.Utils; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using CoreModel = RadixDlt.CoreApiSdk.Model; +using ToolkitModel = RadixEngineToolkit; + +namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors.LedgerTransactionMarkers; + +internal class LedgerTransactionMarkersProcessor : IProcessorBase, ISubstateUpsertProcessor, ISubstateDeleteProcessor, IDecodedEventProcessor, IEventProcessor, ITransactionProcessor +{ + private readonly ProcessorContext _context; + private readonly IWriteHelper _writeHelper; + private readonly List _ledgerTransactionMarkersToAdd = new(); + private readonly ManifestProcessor _manifestProcessor; + private readonly OriginLedgerTransactionMarkerProcessor _originLedgerTransactionMarkerProcessor; + private readonly GlobalEventEmitterProcessor _globalEventEmitterProcessor; + private readonly AffectedGlobalEntitiesProcessor _affectedGlobalEntitiesProcessor; + private readonly EventLedgerTransactionMarkerProcessor _eventLedgerTransactionMarkerProcessor; + + public LedgerTransactionMarkersProcessor( + ManifestProcessor manifestProcessor, + AffectedGlobalEntitiesProcessor affectedGlobalEntitiesProcessor, + ProcessorContext context, + ReferencedEntityDictionary referencedEntities, + IWriteHelper writeHelper, + NetworkConfiguration networkConfiguration) + { + _context = context; + _writeHelper = writeHelper; + _manifestProcessor = manifestProcessor; + _affectedGlobalEntitiesProcessor = affectedGlobalEntitiesProcessor; + _originLedgerTransactionMarkerProcessor = new OriginLedgerTransactionMarkerProcessor(context, referencedEntities, networkConfiguration); + _globalEventEmitterProcessor = new GlobalEventEmitterProcessor(context, referencedEntities, networkConfiguration); + _eventLedgerTransactionMarkerProcessor = new EventLedgerTransactionMarkerProcessor(context, referencedEntities, networkConfiguration); + } + + public Task LoadDependenciesAsync() + { + return Task.CompletedTask; + } + + public void VisitTransaction(CoreModel.CommittedTransaction transaction, long stateVersion) + { + _manifestProcessor.VisitTransaction(transaction, stateVersion); + _originLedgerTransactionMarkerProcessor.VisitTransaction(transaction, stateVersion); + } + + public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion) + { + _affectedGlobalEntitiesProcessor.VisitUpsert(substate, referencedEntity, stateVersion); + } + + public void VisitDelete(CoreModel.SubstateId substateId, ReferencedEntity referencedEntity, long stateVersion) + { + _affectedGlobalEntitiesProcessor.VisitDelete(substateId, referencedEntity, stateVersion); + } + + public void VisitDecodedEvent(ToolkitModel.TypedNativeEvent decodedEvent, ReferencedEntity eventEmitterEntity, long stateVersion) + { + _eventLedgerTransactionMarkerProcessor.VisitDecodedEvent(decodedEvent, eventEmitterEntity, stateVersion); + } + + public void VisitEvent(CoreModel.Event @event, long stateVersion) + { + _globalEventEmitterProcessor.VisitEvent(@event, stateVersion); + } + + public void ProcessChanges() + { + _ledgerTransactionMarkersToAdd.AddRange(_globalEventEmitterProcessor.CreateTransactionMarkers()); + _ledgerTransactionMarkersToAdd.AddRange(_affectedGlobalEntitiesProcessor.CreateTransactionMarkers()); + _ledgerTransactionMarkersToAdd.AddRange(_eventLedgerTransactionMarkerProcessor.CreateTransactionMarkers()); + _ledgerTransactionMarkersToAdd.AddRange(_manifestProcessor.CreateTransactionMarkers()); + _ledgerTransactionMarkersToAdd.AddRange(_originLedgerTransactionMarkerProcessor.CreateTransactionMarkers()); + } + + public async Task SaveEntitiesAsync() + { + var rowsInserted = 0; + + rowsInserted += await CopyLedgerTransactionMarkers(); + + return rowsInserted; + } + + private Task CopyLedgerTransactionMarkers() => _context.WriteHelper.Copy( + _ledgerTransactionMarkersToAdd, + "COPY ledger_transaction_markers (id, state_version, discriminator, event_type, entity_id, resource_entity_id, quantity, operation_type, origin_type, manifest_class, is_most_specific) FROM STDIN (FORMAT BINARY)", + async (writer, e, token) => + { + var discriminator = _writeHelper.GetDiscriminator(e.GetType()); + + await writer.WriteAsync(e.Id, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(e.StateVersion, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(discriminator, "ledger_transaction_marker_type", token); + + switch (e) + { + case EventLedgerTransactionMarker eltm: + await writer.WriteAsync(eltm.EventType, "ledger_transaction_marker_event_type", token); + await writer.WriteAsync(eltm.EntityId, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(eltm.ResourceEntityId, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(eltm.Quantity.GetSubUnitsSafeForPostgres(), NpgsqlDbType.Numeric, token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + break; + case ManifestAddressLedgerTransactionMarker maltm: + await writer.WriteNullAsync(token); + await writer.WriteAsync(maltm.EntityId, NpgsqlDbType.Bigint, token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteAsync(maltm.OperationType, "ledger_transaction_marker_operation_type", token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + break; + case OriginLedgerTransactionMarker oltm: + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteAsync(oltm.OriginType, "ledger_transaction_marker_origin_type", token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + break; + case AffectedGlobalEntityTransactionMarker oltm: + await writer.WriteNullAsync(token); + await writer.WriteAsync(oltm.EntityId, NpgsqlDbType.Bigint, token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + break; + case EventGlobalEmitterTransactionMarker egetm: + await writer.WriteNullAsync(token); + await writer.WriteAsync(egetm.EntityId, NpgsqlDbType.Bigint, token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + break; + case ManifestClassMarker ttm: + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteAsync(ttm.ManifestClass, "ledger_transaction_manifest_class", token); + await writer.WriteAsync(ttm.IsMostSpecific, NpgsqlDbType.Boolean, token); + break; + default: + throw new ArgumentOutOfRangeException(nameof(e), e, null); + } + }); +} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/LedgerTransactionProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/LedgerTransactionProcessor.cs new file mode 100644 index 000000000..674446b8d --- /dev/null +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/LedgerTransactionProcessor.cs @@ -0,0 +1,254 @@ +// +// Copyright (c) PlaceholderCompany. All rights reserved. +// + +using Newtonsoft.Json; +using NpgsqlTypes; +using RadixDlt.NetworkGateway.Abstractions; +using RadixDlt.NetworkGateway.Abstractions.Extensions; +using RadixDlt.NetworkGateway.Abstractions.Model; +using RadixDlt.NetworkGateway.DataAggregator.Services; +using RadixDlt.NetworkGateway.PostgresIntegration.Models; +using RadixDlt.NetworkGateway.PostgresIntegration.Utils; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using CoreModel = RadixDlt.CoreApiSdk.Model; + +namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors.LedgerTransactionMarkers; + +internal class LedgerTransactionProcessor : IProcessorBase, ITransactionProcessor, ISubstateUpsertProcessor +{ + private readonly ProcessorContext _context; + private readonly IClock _clock; + private readonly List _ledgerTransactionsToAdd = new(); + private readonly ReferencedEntityDictionary _referencedEntities; + private readonly ManifestProcessor _manifestProcessor; + private readonly AffectedGlobalEntitiesProcessor _affectedGlobalEntitiesProcessor; + private readonly IWriteHelper _writeHelper; + + private TransactionSummary _lastProcessedTransactionSummary; + + public LedgerTransactionProcessor( + ProcessorContext context, + IClock clock, + ReferencedEntityDictionary referencedEntities, + ManifestProcessor manifestProcessor, + AffectedGlobalEntitiesProcessor affectedGlobalEntitiesProcessor, + IWriteHelper writeHelper, + TransactionSummary lastTransactionSummary) + { + _context = context; + _clock = clock; + _referencedEntities = referencedEntities; + _manifestProcessor = manifestProcessor; + _affectedGlobalEntitiesProcessor = affectedGlobalEntitiesProcessor; + _writeHelper = writeHelper; + _lastProcessedTransactionSummary = lastTransactionSummary; + } + + public void VisitTransaction(CoreModel.CommittedTransaction committedTransaction, long stateVersion) + { + var events = committedTransaction.Receipt.Events ?? new List(); + + long? epochUpdate = null; + long? roundInEpochUpdate = null; + DateTime? roundTimestampUpdate = null; + + if (committedTransaction.LedgerTransaction is CoreModel.RoundUpdateLedgerTransaction roundUpdateTransaction) + { + epochUpdate = _lastProcessedTransactionSummary.Epoch != roundUpdateTransaction.RoundUpdateTransaction.Epoch ? roundUpdateTransaction.RoundUpdateTransaction.Epoch : null; + roundInEpochUpdate = roundUpdateTransaction.RoundUpdateTransaction.RoundInEpoch; + roundTimestampUpdate = DateTimeOffset.FromUnixTimeMilliseconds(roundUpdateTransaction.RoundUpdateTransaction.ProposerTimestamp.UnixTimestampMs).UtcDateTime; + } + + var isStartOfEpoch = epochUpdate.HasValue; + var isStartOfRound = roundInEpochUpdate.HasValue; + var createdTimestamp = _clock.UtcNow; + + var roundTimestamp = roundTimestampUpdate ?? _lastProcessedTransactionSummary.RoundTimestamp; + var normalizedRoundTimestamp = + roundTimestamp < _lastProcessedTransactionSummary.NormalizedRoundTimestamp ? _lastProcessedTransactionSummary.NormalizedRoundTimestamp + : roundTimestamp > createdTimestamp ? createdTimestamp + : roundTimestamp; + + var epoch = epochUpdate ?? _lastProcessedTransactionSummary.Epoch; + var roundInEpoch = roundInEpochUpdate ?? _lastProcessedTransactionSummary.RoundInEpoch; + var indexInEpoch = isStartOfEpoch ? 0 : _lastProcessedTransactionSummary.IndexInEpoch + 1; + var indexInRound = isStartOfRound ? 0 : _lastProcessedTransactionSummary.IndexInRound + 1; + + LedgerTransaction ledgerTransaction = committedTransaction.LedgerTransaction switch + { + CoreModel.GenesisLedgerTransaction => new GenesisLedgerTransaction(), + CoreModel.UserLedgerTransaction ult => new UserLedgerTransaction + { + PayloadHash = ult.NotarizedTransaction.HashBech32m, + IntentHash = ult.NotarizedTransaction.SignedIntent.Intent.HashBech32m, + SignedIntentHash = ult.NotarizedTransaction.SignedIntent.HashBech32m, + Message = ult.NotarizedTransaction.SignedIntent.Intent.Message?.ToJson(), + RawPayload = ult.NotarizedTransaction.GetPayloadBytes(), + ManifestInstructions = ult.NotarizedTransaction.SignedIntent.Intent.Instructions, + ManifestClasses = _manifestProcessor.GetManifestClasses(stateVersion), + }, + CoreModel.RoundUpdateLedgerTransaction => new RoundUpdateLedgerTransaction(), + CoreModel.FlashLedgerTransaction => new FlashLedgerTransaction(), + _ => throw new UnreachableException($"Unsupported transaction type: {committedTransaction.LedgerTransaction.GetType()}"), + }; + + + ledgerTransaction.StateVersion = stateVersion; + ledgerTransaction.TransactionTreeHash = committedTransaction.ResultantStateIdentifiers.TransactionTreeHash; + ledgerTransaction.ReceiptTreeHash = committedTransaction.ResultantStateIdentifiers.ReceiptTreeHash; + ledgerTransaction.StateTreeHash = committedTransaction.ResultantStateIdentifiers.StateTreeHash; + ledgerTransaction.Epoch = epoch; + ledgerTransaction.RoundInEpoch = roundInEpoch; + ledgerTransaction.IndexInEpoch = indexInEpoch; + ledgerTransaction.IndexInRound = indexInRound; + ledgerTransaction.FeePaid = committedTransaction.Receipt.FeeSummary.TotalFee(); + ledgerTransaction.TipPaid = committedTransaction.Receipt.FeeSummary.TotalTip(); + ledgerTransaction.RoundTimestamp = roundTimestamp; + ledgerTransaction.CreatedTimestamp = createdTimestamp; + ledgerTransaction.NormalizedRoundTimestamp = normalizedRoundTimestamp; + ledgerTransaction.ReceiptStateUpdates = committedTransaction.Receipt.StateUpdates.ToJson(); + ledgerTransaction.ReceiptStatus = committedTransaction.Receipt.Status.ToModel(); + ledgerTransaction.ReceiptFeeSummary = committedTransaction.Receipt.FeeSummary.ToJson(); + ledgerTransaction.ReceiptErrorMessage = committedTransaction.Receipt.ErrorMessage; + ledgerTransaction.ReceiptOutput = committedTransaction.Receipt.Output != null ? JsonConvert.SerializeObject(committedTransaction.Receipt.Output) : null; + ledgerTransaction.ReceiptNextEpoch = committedTransaction.Receipt.NextEpoch?.ToJson(); + ledgerTransaction.ReceiptCostingParameters = committedTransaction.Receipt.CostingParameters.ToJson(); + ledgerTransaction.ReceiptFeeSource = committedTransaction.Receipt.FeeSource?.ToJson(); + ledgerTransaction.ReceiptFeeDestination = committedTransaction.Receipt.FeeDestination?.ToJson(); + ledgerTransaction.BalanceChanges = committedTransaction.BalanceChanges?.ToJson(); + ledgerTransaction.AffectedGlobalEntities = _affectedGlobalEntitiesProcessor.GetAllAffectedGlobalEntities(stateVersion).ToArray(); + ledgerTransaction.ReceiptEventEmitters = events.Select(e => e.Type.Emitter.ToJson()).ToArray(); + ledgerTransaction.ReceiptEventNames = events.Select(e => e.Type.Name).ToArray(); + ledgerTransaction.ReceiptEventSbors = events.Select(e => e.Data.GetDataBytes()).ToArray(); + ledgerTransaction.ReceiptEventSchemaEntityIds = events.Select(e => _referencedEntities.Get((EntityAddress)e.Type.TypeReference.FullTypeId.EntityAddress).DatabaseId).ToArray(); + ledgerTransaction.ReceiptEventSchemaHashes = events.Select(e => e.Type.TypeReference.FullTypeId.SchemaHash.ConvertFromHex()).ToArray(); + ledgerTransaction.ReceiptEventTypeIndexes = events.Select(e => e.Type.TypeReference.FullTypeId.LocalTypeId.Id).ToArray(); + ledgerTransaction.ReceiptEventSborTypeKinds = events.Select(e => e.Type.TypeReference.FullTypeId.LocalTypeId.Kind.ToModel()).ToArray(); + + _ledgerTransactionsToAdd.Add(ledgerTransaction); + + _lastProcessedTransactionSummary = new TransactionSummary( + StateVersion: stateVersion, + RoundTimestamp: roundTimestamp, + NormalizedRoundTimestamp: normalizedRoundTimestamp, + // TODO PP: i'm not sure it's correct. + TransactionTreeHash: _lastProcessedTransactionSummary.TransactionTreeHash, + ReceiptTreeHash: _lastProcessedTransactionSummary.ReceiptTreeHash, + StateTreeHash: _lastProcessedTransactionSummary.StateTreeHash, + CreatedTimestamp: createdTimestamp, + Epoch: epoch, + RoundInEpoch: roundInEpoch, + IndexInEpoch: indexInEpoch, + IndexInRound: indexInRound); + } + + public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion) + { + var substateData = substate.Value.SubstateData; + + if (substateData is CoreModel.ConsensusManagerFieldCurrentTimeSubstate currentTime) + { + // TODO PP: we have to update it here. + // TODO PP: that never worked? + roundTimestampUpdate = DateTimeOffset.FromUnixTimeMilliseconds(currentTime.Value.ProposerTimestamp.UnixTimestampMs).UtcDateTime; + } + } + + public Task LoadDependenciesAsync() + { + return Task.CompletedTask; + } + + public void ProcessChanges() + { + } + + public TransactionSummary GetLastProcessedTransactionSummary() + { + return _lastProcessedTransactionSummary; + } + + public async Task SaveEntitiesAsync() + { + var rowsInserted = 0; + + rowsInserted += await CopyLedgerTransactionMarkers(); + + return rowsInserted; + } + + private Task CopyLedgerTransactionMarkers() => _context.WriteHelper.Copy( + _ledgerTransactionsToAdd, + "COPY ledger_transactions (state_version, transaction_tree_hash, receipt_tree_hash, state_tree_hash, epoch, round_in_epoch, index_in_epoch, index_in_round, fee_paid, tip_paid, affected_global_entities, round_timestamp, created_timestamp, normalized_round_timestamp, balance_changes, receipt_state_updates, receipt_status, receipt_fee_summary, receipt_fee_source, receipt_fee_destination, receipt_costing_parameters, receipt_error_message, receipt_output, receipt_next_epoch, receipt_event_emitters, receipt_event_names, receipt_event_sbors, receipt_event_schema_entity_ids, receipt_event_schema_hashes, receipt_event_type_indexes, receipt_event_sbor_type_kinds, discriminator, payload_hash, intent_hash, signed_intent_hash, message, raw_payload, manifest_instructions, manifest_classes) FROM STDIN (FORMAT BINARY)", + async (writer, lt, token) => + { + var discriminator = _writeHelper.GetDiscriminator(lt.GetType()); + + await _writeHelper.HandleMaxAggregateCounts(lt); + await writer.StartRowAsync(token); + await writer.WriteAsync(lt.StateVersion, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(lt.TransactionTreeHash, NpgsqlDbType.Text, token); + await writer.WriteAsync(lt.ReceiptTreeHash, NpgsqlDbType.Text, token); + await writer.WriteAsync(lt.StateTreeHash, NpgsqlDbType.Text, token); + await writer.WriteAsync(lt.Epoch, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(lt.RoundInEpoch, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(lt.IndexInEpoch, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(lt.IndexInRound, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(lt.FeePaid.GetSubUnitsSafeForPostgres(), NpgsqlDbType.Numeric, token); + await writer.WriteAsync(lt.TipPaid.GetSubUnitsSafeForPostgres(), NpgsqlDbType.Numeric, token); + await writer.WriteAsync(lt.AffectedGlobalEntities, NpgsqlDbType.Array | NpgsqlDbType.Bigint, token); + await writer.WriteAsync(lt.RoundTimestamp, NpgsqlDbType.TimestampTz, token); + await writer.WriteAsync(lt.CreatedTimestamp, NpgsqlDbType.TimestampTz, token); + await writer.WriteAsync(lt.NormalizedRoundTimestamp, NpgsqlDbType.TimestampTz, token); + await writer.WriteAsync(lt.BalanceChanges, NpgsqlDbType.Jsonb, token); + + await writer.WriteAsync(lt.EngineReceipt.StateUpdates, NpgsqlDbType.Jsonb, token); + await writer.WriteAsync(lt.EngineReceipt.Status, "ledger_transaction_status", token); + await writer.WriteAsync(lt.EngineReceipt.FeeSummary, NpgsqlDbType.Jsonb, token); + await writer.WriteAsync(lt.EngineReceipt.FeeSource, NpgsqlDbType.Jsonb, token); + await writer.WriteAsync(lt.EngineReceipt.FeeDestination, NpgsqlDbType.Jsonb, token); + await writer.WriteAsync(lt.EngineReceipt.CostingParameters, NpgsqlDbType.Jsonb, token); + await writer.WriteAsync(lt.EngineReceipt.ErrorMessage, NpgsqlDbType.Text, token); + await writer.WriteAsync(lt.EngineReceipt.Output, NpgsqlDbType.Jsonb, token); + await writer.WriteAsync(lt.EngineReceipt.NextEpoch, NpgsqlDbType.Jsonb, token); + await writer.WriteAsync(lt.EngineReceipt.Events.Emitters, NpgsqlDbType.Array | NpgsqlDbType.Jsonb, token); + await writer.WriteAsync(lt.EngineReceipt.Events.Names, NpgsqlDbType.Array | NpgsqlDbType.Text, token); + await writer.WriteAsync(lt.EngineReceipt.Events.Sbors, NpgsqlDbType.Array | NpgsqlDbType.Bytea, token); + await writer.WriteAsync(lt.EngineReceipt.Events.SchemaEntityIds, NpgsqlDbType.Array | NpgsqlDbType.Bigint, token); + await writer.WriteAsync(lt.EngineReceipt.Events.SchemaHashes, NpgsqlDbType.Array | NpgsqlDbType.Bytea, token); + await writer.WriteAsync(lt.EngineReceipt.Events.TypeIndexes, NpgsqlDbType.Array | NpgsqlDbType.Bigint, token); + await writer.WriteAsync(lt.EngineReceipt.Events.SborTypeKinds, "sbor_type_kind[]", token); + await writer.WriteAsync(discriminator, "ledger_transaction_type", token); + + switch (lt) + { + case GenesisLedgerTransaction: + case RoundUpdateLedgerTransaction: + case FlashLedgerTransaction: + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + break; + case UserLedgerTransaction ult: + await writer.WriteAsync(ult.PayloadHash, NpgsqlDbType.Text, token); + await writer.WriteAsync(ult.IntentHash, NpgsqlDbType.Text, token); + await writer.WriteAsync(ult.SignedIntentHash, NpgsqlDbType.Text, token); + await writer.WriteAsync(ult.Message, NpgsqlDbType.Jsonb, token); + await writer.WriteAsync(ult.RawPayload, NpgsqlDbType.Bytea, token); + await writer.WriteAsync(ult.ManifestInstructions, NpgsqlDbType.Text, token); + await writer.WriteAsync(ult.ManifestClasses, "ledger_transaction_manifest_class[]", token); + break; + default: + throw new ArgumentOutOfRangeException(nameof(lt), lt, null); + } + }); +} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/ManifestProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/ManifestProcessor.cs new file mode 100644 index 000000000..b12552c95 --- /dev/null +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/ManifestProcessor.cs @@ -0,0 +1,171 @@ +using RadixDlt.NetworkGateway.Abstractions.Extensions; +using RadixDlt.NetworkGateway.Abstractions.Model; +using RadixDlt.NetworkGateway.Abstractions.Network; +using RadixDlt.NetworkGateway.PostgresIntegration.Models; +using RadixDlt.NetworkGateway.PostgresIntegration.Utils; +using System; +using System.Collections.Generic; +using System.Linq; +using CoreModel = RadixDlt.CoreApiSdk.Model; +using ToolkitModel = RadixEngineToolkit; + +namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors.LedgerTransactionMarkers; + +internal class ManifestProcessor : ITransactionMarkerProcessor +{ + private readonly ProcessorContext _context; + private readonly ReferencedEntityDictionary _referencedEntities; + private readonly NetworkConfiguration _networkConfiguration; + private readonly List _ledgerTransactionMarkersToAdd = new(); + + private readonly Dictionary _manifestExtractedAddresses = new(); + private readonly Dictionary> _manifestClasses = new(); + + public ManifestProcessor(ProcessorContext context, ReferencedEntityDictionary referencedEntities, NetworkConfiguration networkConfiguration) + { + _context = context; + _referencedEntities = referencedEntities; + _networkConfiguration = networkConfiguration; + } + + public void VisitTransaction(CoreModel.CommittedTransaction transaction, long stateVersion) + { + if (transaction.LedgerTransaction is CoreModel.UserLedgerTransaction userLedgerTransaction) + { + var coreInstructions = userLedgerTransaction.NotarizedTransaction.SignedIntent.Intent.Instructions; + var coreBlobs = userLedgerTransaction.NotarizedTransaction.SignedIntent.Intent.BlobsHex; + using var manifestInstructions = ToolkitModel.Instructions.FromString(coreInstructions, _networkConfiguration.Id); + using var toolkitManifest = new ToolkitModel.TransactionManifest(manifestInstructions, coreBlobs.Values.Select(x => x.ConvertFromHex()).ToArray()); + + var extractedAddresses = ManifestAddressesExtractor.ExtractAddresses(toolkitManifest, _networkConfiguration.Id); + + foreach (var address in extractedAddresses.All()) + { + _referencedEntities.MarkSeenAddress(address); + } + + _manifestExtractedAddresses.Add(stateVersion, extractedAddresses); + + AnalyzeManifestClasses(toolkitManifest, stateVersion); + } + } + + public IEnumerable CreateTransactionMarkers() + { + foreach (var stateVersion in _manifestExtractedAddresses.Keys) + { + AnalyzeAddresses(stateVersion); + } + + return _ledgerTransactionMarkersToAdd; + } + + public LedgerTransactionManifestClass[] GetManifestClasses(long stateVersion) + { + return _manifestClasses.TryGetValue(stateVersion, out var mc) ? mc.ToArray() : Array.Empty(); + } + + public void AnalyzeManifestClasses(ToolkitModel.TransactionManifest toolkitManifest, long stateVersion) + { + var manifestSummary = toolkitManifest.Summary(_networkConfiguration.Id); + + for (var i = 0; i < manifestSummary.classification.Length; ++i) + { + var manifestClass = manifestSummary.classification[i].ToModel(); + + _manifestClasses + .GetOrAdd(stateVersion, _ => new List()) + .Add(manifestClass); + + _ledgerTransactionMarkersToAdd.Add( + new ManifestClassMarker + { + Id = _context.Sequences.LedgerTransactionMarkerSequence++, + StateVersion = stateVersion, + ManifestClass = manifestClass, + IsMostSpecific = i == 0, + }); + } + } + + private void AnalyzeAddresses(long stateVersion) + { + if (_manifestExtractedAddresses.TryGetValue(stateVersion, out var extractedAddresses)) + { + foreach (var proofResourceAddress in extractedAddresses.PresentedProofs.Select(x => x.ResourceAddress).ToHashSet()) + { + if (_referencedEntities.TryGet(proofResourceAddress, out var re)) + { + _ledgerTransactionMarkersToAdd.Add( + new ManifestAddressLedgerTransactionMarker + { + Id = _context.Sequences.LedgerTransactionMarkerSequence++, + StateVersion = stateVersion, + OperationType = LedgerTransactionMarkerOperationType.BadgePresented, + EntityId = re.DatabaseId, + }); + } + } + + foreach (var address in extractedAddresses.ResourceAddresses) + { + if (_referencedEntities.TryGet(address, out var re)) + { + _ledgerTransactionMarkersToAdd.Add( + new ManifestAddressLedgerTransactionMarker + { + Id = _context.Sequences.LedgerTransactionMarkerSequence++, + StateVersion = stateVersion, + OperationType = LedgerTransactionMarkerOperationType.ResourceInUse, + EntityId = re.DatabaseId, + }); + } + } + + foreach (var address in extractedAddresses.AccountsRequiringAuth) + { + if (_referencedEntities.TryGet(address, out var re)) + { + _ledgerTransactionMarkersToAdd.Add( + new ManifestAddressLedgerTransactionMarker + { + Id = _context.Sequences.LedgerTransactionMarkerSequence++, + StateVersion = stateVersion, + OperationType = LedgerTransactionMarkerOperationType.AccountOwnerMethodCall, + EntityId = re.DatabaseId, + }); + } + } + + foreach (var address in extractedAddresses.AccountsDepositedInto) + { + if (_referencedEntities.TryGet(address, out var re)) + { + _ledgerTransactionMarkersToAdd.Add( + new ManifestAddressLedgerTransactionMarker + { + Id = _context.Sequences.LedgerTransactionMarkerSequence++, + StateVersion = stateVersion, + OperationType = LedgerTransactionMarkerOperationType.AccountDepositedInto, + EntityId = re.DatabaseId, + }); + } + } + + foreach (var address in extractedAddresses.AccountsWithdrawnFrom) + { + if (_referencedEntities.TryGet(address, out var re)) + { + _ledgerTransactionMarkersToAdd.Add( + new ManifestAddressLedgerTransactionMarker + { + Id = _context.Sequences.LedgerTransactionMarkerSequence++, + StateVersion = stateVersion, + OperationType = LedgerTransactionMarkerOperationType.AccountWithdrawnFrom, + EntityId = re.DatabaseId, + }); + } + } + } + } +} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/OriginLedgerTransactionMarkerProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/OriginLedgerTransactionMarkerProcessor.cs new file mode 100644 index 000000000..88e5ce55e --- /dev/null +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/OriginLedgerTransactionMarkerProcessor.cs @@ -0,0 +1,46 @@ +using RadixDlt.NetworkGateway.Abstractions.Network; +using RadixDlt.NetworkGateway.PostgresIntegration.Models; +using System.Collections.Generic; + +namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors.LedgerTransactionMarkers; + +internal class OriginLedgerTransactionMarkerProcessor : ITransactionMarkerProcessor +{ + private readonly ProcessorContext _context; + private readonly List _ledgerTransactionMarkersToAdd = new(); + + public OriginLedgerTransactionMarkerProcessor(ProcessorContext context, ReferencedEntityDictionary _, NetworkConfiguration __) + { + _context = context; + } + + public void VisitTransaction(CoreApiSdk.Model.CommittedTransaction committedTransaction, long stateVersion) + { + if (committedTransaction.Receipt.NextEpoch != null) + { + _ledgerTransactionMarkersToAdd.Add( + new OriginLedgerTransactionMarker + { + Id = _context.Sequences.LedgerTransactionMarkerSequence++, + StateVersion = stateVersion, + OriginType = LedgerTransactionMarkerOriginType.EpochChange, + }); + } + + if (committedTransaction.LedgerTransaction is CoreApiSdk.Model.UserLedgerTransaction) + { + _ledgerTransactionMarkersToAdd.Add( + new OriginLedgerTransactionMarker + { + Id = _context.Sequences.LedgerTransactionMarkerSequence++, + StateVersion = stateVersion, + OriginType = LedgerTransactionMarkerOriginType.User, + }); + } + } + + public IEnumerable CreateTransactionMarkers() + { + return _ledgerTransactionMarkersToAdd; + } +} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PackageBlueprintProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/PackageBlueprintProcessor.cs similarity index 85% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PackageBlueprintProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/PackageBlueprintProcessor.cs index dc0c130dc..cdfc74902 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PackageBlueprintProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/PackageBlueprintProcessor.cs @@ -88,18 +88,18 @@ internal record PackageBlueprintChangePointer public CoreModel.PackageBlueprintAuthTemplateEntrySubstate? PackageBlueprintAuthTemplate { get; set; } } -internal class PackageBlueprintProcessor +internal class PackageBlueprintProcessor : IProcessorBase, ISubstateUpsertProcessor { private readonly ProcessorContext _context; private readonly ReferencedEntityDictionary _referencedEntities; - private ChangeTracker _changes = new(); + private readonly ChangeTracker _changes = new(); - private Dictionary _mostRecentAggregates = new(); - private Dictionary _mostRecentEntries = new(); + private readonly Dictionary _mostRecentAggregates = new(); + private readonly Dictionary _mostRecentEntries = new(); - private List _aggregatesToAdd = new(); - private List _entriesToAdd = new(); + private readonly List _aggregatesToAdd = new(); + private readonly List _entriesToAdd = new(); public PackageBlueprintProcessor(ProcessorContext context, ReferencedEntityDictionary referencedEntities) { @@ -107,38 +107,56 @@ public PackageBlueprintProcessor(ProcessorContext context, ReferencedEntityDicti _referencedEntities = referencedEntities; } - public void VisitUpsert(CoreModel.Substate substateData, ReferencedEntity referencedEntity, long stateVersion) + public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion) { + var substateData = substate.Value.SubstateData; + if (substateData is CoreModel.PackageBlueprintDefinitionEntrySubstate packageBlueprintDefinition) { _changes - .GetOrAdd(new PackageBlueprintChangePointerLookup(referencedEntity.DatabaseId, packageBlueprintDefinition.Key.BlueprintName, packageBlueprintDefinition.Key.BlueprintVersion, stateVersion), _ => new PackageBlueprintChangePointer()) + .GetOrAdd( + new PackageBlueprintChangePointerLookup(referencedEntity.DatabaseId, packageBlueprintDefinition.Key.BlueprintName, packageBlueprintDefinition.Key.BlueprintVersion, stateVersion), + _ => new PackageBlueprintChangePointer()) .PackageBlueprintDefinition = packageBlueprintDefinition; } if (substateData is CoreModel.PackageBlueprintDependenciesEntrySubstate packageBlueprintDependencies) { _changes - .GetOrAdd(new PackageBlueprintChangePointerLookup(referencedEntity.DatabaseId, packageBlueprintDependencies.Key.BlueprintName, packageBlueprintDependencies.Key.BlueprintVersion, stateVersion), _ => new PackageBlueprintChangePointer()) + .GetOrAdd( + new PackageBlueprintChangePointerLookup( + referencedEntity.DatabaseId, + packageBlueprintDependencies.Key.BlueprintName, + packageBlueprintDependencies.Key.BlueprintVersion, + stateVersion), + _ => new PackageBlueprintChangePointer()) .PackageBlueprintDependencies = packageBlueprintDependencies; } if (substateData is CoreModel.PackageBlueprintRoyaltyEntrySubstate packageBlueprintRoyalty) { _changes - .GetOrAdd(new PackageBlueprintChangePointerLookup(referencedEntity.DatabaseId, packageBlueprintRoyalty.Key.BlueprintName, packageBlueprintRoyalty.Key.BlueprintVersion, stateVersion), _ => new PackageBlueprintChangePointer()) + .GetOrAdd( + new PackageBlueprintChangePointerLookup(referencedEntity.DatabaseId, packageBlueprintRoyalty.Key.BlueprintName, packageBlueprintRoyalty.Key.BlueprintVersion, stateVersion), + _ => new PackageBlueprintChangePointer()) .PackageBlueprintRoyalty = packageBlueprintRoyalty; } if (substateData is CoreModel.PackageBlueprintAuthTemplateEntrySubstate packageBlueprintAuthTemplate) { _changes - .GetOrAdd(new PackageBlueprintChangePointerLookup(referencedEntity.DatabaseId, packageBlueprintAuthTemplate.Key.BlueprintName, packageBlueprintAuthTemplate.Key.BlueprintVersion, stateVersion), _ => new PackageBlueprintChangePointer()) + .GetOrAdd( + new PackageBlueprintChangePointerLookup( + referencedEntity.DatabaseId, + packageBlueprintAuthTemplate.Key.BlueprintName, + packageBlueprintAuthTemplate.Key.BlueprintVersion, + stateVersion), + _ => new PackageBlueprintChangePointer()) .PackageBlueprintAuthTemplate = packageBlueprintAuthTemplate; } } - public async Task LoadDependencies() + public async Task LoadDependenciesAsync() { _mostRecentEntries.AddRange(await MostRecentPackageCodeHistory()); _mostRecentAggregates.AddRange(await MostRecentPackageBlueprintAggregateHistory()); @@ -223,7 +241,8 @@ public void ProcessChanges() if (change.PackageBlueprintDependencies != null) { - entryHistory.DependantEntityIds = change.PackageBlueprintDependencies.Value.Dependencies.Dependencies.Select(address => _referencedEntities.Get((EntityAddress)address).DatabaseId).ToList(); + entryHistory.DependantEntityIds = + change.PackageBlueprintDependencies.Value.Dependencies.Dependencies.Select(address => _referencedEntities.Get((EntityAddress)address).DatabaseId).ToList(); } if (change.PackageBlueprintRoyalty != null) @@ -240,7 +259,7 @@ public void ProcessChanges() } } - public async Task SaveEntities() + public async Task SaveEntitiesAsync() { var rowsInserted = 0; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PackageCodeProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/PackageCodeProcessor.cs similarity index 86% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PackageCodeProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/PackageCodeProcessor.cs index 1d2211f03..8207cfa42 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PackageCodeProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/PackageCodeProcessor.cs @@ -76,51 +76,57 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; -internal record struct PackageCodeChangePointerLookup(long PackageEntityId, ValueBytes CodeHash, long StateVersion); +internal class PackageCodeProcessor : IProcessorBase, ISubstateUpsertProcessor, ISubstateDeleteProcessor +{ + private record struct PackageCodeChangePointerLookup(long PackageEntityId, ValueBytes CodeHash, long StateVersion); -internal record struct PackageCodeDbLookup(long PackageEntityId, ValueBytes CodeHash); + private record struct PackageCodeDbLookup(long PackageEntityId, ValueBytes CodeHash); -internal record PackageCodeChangePointer -{ - public CoreModel.PackageCodeOriginalCodeEntrySubstate? PackageCodeOriginalCode { get; set; } + private record PackageCodeChangePointer + { + public CoreModel.PackageCodeOriginalCodeEntrySubstate? PackageCodeOriginalCode { get; set; } - public CoreModel.PackageCodeVmTypeEntrySubstate? PackageCodeVmType { get; set; } + public CoreModel.PackageCodeVmTypeEntrySubstate? PackageCodeVmType { get; set; } - public bool CodeVmTypeIsDeleted { get; set; } + public bool CodeVmTypeIsDeleted { get; set; } - public bool PackageCodeIsDeleted { get; set; } -} + public bool PackageCodeIsDeleted { get; set; } + } -internal class PackageCodeProcessor -{ private readonly ProcessorContext _context; - private ChangeTracker _changes = new(); + private readonly ChangeTracker _changes = new(); - private Dictionary _mostRecentAggregates = new(); - private Dictionary _mostRecentEntries = new(); + private readonly Dictionary _mostRecentAggregates = new(); + private readonly Dictionary _mostRecentEntries = new(); - private List _aggregatesToAdd = new(); - private List _entriesToAdd = new(); + private readonly List _aggregatesToAdd = new(); + private readonly List _entriesToAdd = new(); public PackageCodeProcessor(ProcessorContext context) { _context = context; } - public void VisitUpsert(CoreModel.Substate substateData, ReferencedEntity referencedEntity, long stateVersion) + public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion) { + var substateData = substate.Value.SubstateData; + if (substateData is CoreModel.PackageCodeOriginalCodeEntrySubstate packageCodeOriginalCode) { _changes - .GetOrAdd(new PackageCodeChangePointerLookup(referencedEntity.DatabaseId, (ValueBytes)packageCodeOriginalCode.Key.CodeHash.ConvertFromHex(), stateVersion), _ => new PackageCodeChangePointer()) + .GetOrAdd( + new PackageCodeChangePointerLookup(referencedEntity.DatabaseId, (ValueBytes)packageCodeOriginalCode.Key.CodeHash.ConvertFromHex(), stateVersion), + _ => new PackageCodeChangePointer()) .PackageCodeOriginalCode = packageCodeOriginalCode; } if (substateData is CoreModel.PackageCodeVmTypeEntrySubstate packageCodeVmType) { _changes - .GetOrAdd(new PackageCodeChangePointerLookup(referencedEntity.DatabaseId, (ValueBytes)packageCodeVmType.Key.CodeHash.ConvertFromHex(), stateVersion), _ => new PackageCodeChangePointer()) + .GetOrAdd( + new PackageCodeChangePointerLookup(referencedEntity.DatabaseId, (ValueBytes)packageCodeVmType.Key.CodeHash.ConvertFromHex(), stateVersion), + _ => new PackageCodeChangePointer()) .PackageCodeVmType = packageCodeVmType; } } @@ -148,7 +154,7 @@ public void VisitDelete(CoreModel.SubstateId substateId, ReferencedEntity refere } } - public async Task LoadDependencies() + public async Task LoadDependenciesAsync() { _mostRecentEntries.AddRange(await MostRecentPackageCodeHistory()); _mostRecentAggregates.AddRange(await MostRecentPackageCodeAggregateHistory()); @@ -227,7 +233,8 @@ public void ProcessChanges() } else if (change.PackageCodeIsDeleted != change.CodeVmTypeIsDeleted) { - throw new UnreachableException($"Unexpected situation where PackageCode was deleted but VmType wasn't. PackageId: {lookup.PackageEntityId}, CodeHashHex: {lookup.CodeHash.ToHex()}, StateVersion: {lookup.StateVersion}"); + throw new UnreachableException( + $"Unexpected situation where PackageCode was deleted but VmType wasn't. PackageId: {lookup.PackageEntityId}, CodeHashHex: {lookup.CodeHash.ToHex()}, StateVersion: {lookup.StateVersion}"); } else { @@ -246,7 +253,7 @@ public void ProcessChanges() } } - public async Task SaveEntities() + public async Task SaveEntitiesAsync() { var rowsInserted = 0; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/ResourceSupplyProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/ResourceSupplyProcessor.cs new file mode 100644 index 000000000..64400eeab --- /dev/null +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/ResourceSupplyProcessor.cs @@ -0,0 +1,207 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +using NpgsqlTypes; +using RadixDlt.NetworkGateway.Abstractions.Numerics; +using RadixDlt.NetworkGateway.PostgresIntegration.Models; +using RadixDlt.NetworkGateway.PostgresIntegration.Utils; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Threading.Tasks; +using ToolkitModel = RadixEngineToolkit; + +namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; + +internal class ResourceSupplyProcessor : IProcessorBase, IDecodedEventProcessor +{ + private readonly ProcessorContext _context; + private readonly List _changes = new(); + private readonly Dictionary _mostRecent = new(); + private List _toAdd = new(); + + public ResourceSupplyProcessor(ProcessorContext context) + { + _context = context; + } + + public void VisitDecodedEvent(ToolkitModel.TypedNativeEvent decodedEvent, ReferencedEntity eventEmitterEntity, long stateVersion) + { + if (EventDecoder.TryGetFungibleResourceMintedEvent(decodedEvent, out var fungibleResourceMintedEvent)) + { + var mintedAmount = TokenAmount.FromDecimalString(fungibleResourceMintedEvent.amount.AsStr()); + _changes.Add(new ResourceSupplyChange(eventEmitterEntity.DatabaseId, stateVersion, Minted: mintedAmount)); + } + else if (EventDecoder.TryGetFungibleResourceBurnedEvent(decodedEvent, out var fungibleResourceBurnedEvent)) + { + var burnedAmount = TokenAmount.FromDecimalString(fungibleResourceBurnedEvent.amount.AsStr()); + _changes.Add(new ResourceSupplyChange(eventEmitterEntity.DatabaseId, stateVersion, Burned: burnedAmount)); + } + else if (EventDecoder.TryGetNonFungibleResourceMintedEvent(decodedEvent, out var nonFungibleResourceMintedEvent)) + { + var mintedCount = TokenAmount.FromDecimalString(nonFungibleResourceMintedEvent.ids.Length.ToString()); + _changes.Add(new ResourceSupplyChange(eventEmitterEntity.DatabaseId, stateVersion, Minted: mintedCount)); + } + else if (EventDecoder.TryGetNonFungibleResourceBurnedEvent(decodedEvent, out var nonFungibleResourceBurnedEvent)) + { + var burnedCount = TokenAmount.FromDecimalString(nonFungibleResourceBurnedEvent.ids.Length.ToString()); + _changes.Add(new ResourceSupplyChange(eventEmitterEntity.DatabaseId, stateVersion, Burned: burnedCount)); + } + } + + public async Task LoadDependenciesAsync() + { + _mostRecent.AddRange(await ExistingResourceEntitySupplyHistory()); + } + + public void ProcessChanges() + { + _toAdd = _changes + .GroupBy(x => new { x.ResourceEntityId, x.StateVersion }) + .Select( + group => + { + var previous = _mostRecent.GetOrAdd( + group.Key.ResourceEntityId, + _ => new ResourceEntitySupplyHistory { TotalSupply = TokenAmount.Zero, TotalMinted = TokenAmount.Zero, TotalBurned = TokenAmount.Zero }); + + var minted = group + .Where(x => x.Minted.HasValue) + .Select(x => x.Minted) + .Aggregate(TokenAmount.Zero, (sum, x) => sum + x!.Value); + + var burned = group + .Where(x => x.Burned.HasValue) + .Select(x => x.Burned) + .Aggregate(TokenAmount.Zero, (sum, x) => sum + x!.Value); + + var totalSupply = previous.TotalSupply + minted - burned; + var totalMinted = previous.TotalMinted + minted; + var totalBurned = previous.TotalBurned + burned; + + previous.TotalSupply = totalSupply; + previous.TotalMinted = totalMinted; + previous.TotalBurned = totalBurned; + + return new ResourceEntitySupplyHistory + { + Id = _context.Sequences.ResourceEntitySupplyHistorySequence++, + FromStateVersion = group.Key.StateVersion, + ResourceEntityId = group.Key.ResourceEntityId, + TotalSupply = totalSupply, + TotalMinted = totalMinted, + TotalBurned = totalBurned, + }; + }) + .ToList(); + } + + public async Task SaveEntitiesAsync() + { + var rowsInserted = 0; + + rowsInserted += await CopyResourceSupplyHistory(); + + return rowsInserted; + } + + private Task CopyResourceSupplyHistory() => _context.WriteHelper.Copy( + _toAdd, + "COPY resource_entity_supply_history (id, from_state_version, resource_entity_id, total_supply, total_minted, total_burned) FROM STDIN (FORMAT BINARY)", + async (writer, e, token) => + { + await writer.WriteAsync(e.Id, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(e.FromStateVersion, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(e.ResourceEntityId, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(e.TotalSupply.GetSubUnitsSafeForPostgres(), NpgsqlDbType.Numeric, token); + await writer.WriteAsync(e.TotalMinted.GetSubUnitsSafeForPostgres(), NpgsqlDbType.Numeric, token); + await writer.WriteAsync(e.TotalBurned.GetSubUnitsSafeForPostgres(), NpgsqlDbType.Numeric, token); + }); + + private async Task> ExistingResourceEntitySupplyHistory() + { + if (_changes.Count == 0) + { + return ImmutableDictionary.Empty; + } + + var ids = _changes.Select(x => x.ResourceEntityId).ToHashSet(); + + return await _context.ReadHelper.LoadDependencies( + @$" +WITH variables (resource_entity_id) AS ( + SELECT UNNEST({ids}) +) +SELECT rmesh.* +FROM variables +INNER JOIN LATERAL ( + SELECT * + FROM resource_entity_supply_history + WHERE resource_entity_id = variables.resource_entity_id + ORDER BY from_state_version DESC + LIMIT 1 +) rmesh ON true;", + e => e.ResourceEntityId); + } +} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/StandardMetadataProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/StandardMetadataProcessor.cs similarity index 72% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/StandardMetadataProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/StandardMetadataProcessor.cs index ca0904907..69bc468fc 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/StandardMetadataProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/StandardMetadataProcessor.cs @@ -78,18 +78,18 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; -internal record struct UnverifiedStandardMetadataEntryDbLookup(long EntityId, StandardMetadataKey Type); - -internal class StandardMetadataProcessor +internal class StandardMetadataProcessor : IProcessorBase, ISubstateUpsertProcessor { + private record struct UnverifiedStandardMetadataEntryDbLookup(long EntityId, StandardMetadataKey Type); + private readonly ProcessorContext _context; private readonly ReferencedEntityDictionary _referencedEntities; - private Dictionary _mostRecentAggregates = new(); - private Dictionary _mostRecentEntries = new(); + private readonly Dictionary _mostRecentAggregates = new(); + private readonly Dictionary _mostRecentEntries = new(); - private List _aggregatesToAdd = new(); - private List _entriesToAdd = new(); + private readonly List _aggregatesToAdd = new(); + private readonly List _entriesToAdd = new(); public StandardMetadataProcessor(ProcessorContext context, ReferencedEntityDictionary referencedEntities) { @@ -97,8 +97,10 @@ public StandardMetadataProcessor(ProcessorContext context, ReferencedEntityDicti _referencedEntities = referencedEntities; } - public void VisitUpsert(CoreModel.Substate substateData, ReferencedEntity referencedEntity, long stateVersion) + public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion) { + var substateData = substate.Value.SubstateData; + if (substateData is not CoreModel.MetadataModuleEntrySubstate metadataEntry) { return; @@ -127,98 +129,105 @@ bool TryParseValue([NotNullWhen(true)] out T? value) { if (key == StandardMetadataConstants.DappAccountType && TryParseValue(out var accountType)) { - _entriesToAdd.Add(new DappAccountTypeUnverifiedStandardMetadataEntryHistory - { - Id = _context.Sequences.UnverifiedStandardMetadataEntryHistorySequence++, - FromStateVersion = stateVersion, - EntityId = referencedEntity.DatabaseId, - IsDeleted = metadataEntry.Value == null, - IsLocked = substateData.IsLocked, - Value = accountType.Value, - }); + _entriesToAdd.Add( + new DappAccountTypeUnverifiedStandardMetadataEntryHistory + { + Id = _context.Sequences.UnverifiedStandardMetadataEntryHistorySequence++, + FromStateVersion = stateVersion, + EntityId = referencedEntity.DatabaseId, + IsDeleted = metadataEntry.Value == null, + IsLocked = substateData.IsLocked, + Value = accountType.Value, + }); } else if (key == StandardMetadataConstants.DappClaimedWebsites && TryParseValue(out var claimedWebsites)) { - _entriesToAdd.Add(new DappClaimedWebsitesUnverifiedStandardMetadataEntryHistory - { - Id = _context.Sequences.UnverifiedStandardMetadataEntryHistorySequence++, - FromStateVersion = stateVersion, - EntityId = referencedEntity.DatabaseId, - IsDeleted = metadataEntry.Value == null, - IsLocked = substateData.IsLocked, - ClaimedWebsites = claimedWebsites.Values.ToArray(), - }); + _entriesToAdd.Add( + new DappClaimedWebsitesUnverifiedStandardMetadataEntryHistory + { + Id = _context.Sequences.UnverifiedStandardMetadataEntryHistorySequence++, + FromStateVersion = stateVersion, + EntityId = referencedEntity.DatabaseId, + IsDeleted = metadataEntry.Value == null, + IsLocked = substateData.IsLocked, + ClaimedWebsites = claimedWebsites.Values.ToArray(), + }); } else if (key == StandardMetadataConstants.DappClaimedEntities && TryParseValue(out var claimedEntities)) { - _entriesToAdd.Add(new DappClaimedEntitiesUnverifiedStandardMetadataEntryHistory - { - Id = _context.Sequences.UnverifiedStandardMetadataEntryHistorySequence++, - FromStateVersion = stateVersion, - EntityId = referencedEntity.DatabaseId, - IsDeleted = metadataEntry.Value == null, - IsLocked = substateData.IsLocked, - ClaimedEntityIds = claimedEntities.Values.Select(address => _referencedEntities.Get((EntityAddress)address).DatabaseId).ToArray(), - }); + _entriesToAdd.Add( + new DappClaimedEntitiesUnverifiedStandardMetadataEntryHistory + { + Id = _context.Sequences.UnverifiedStandardMetadataEntryHistorySequence++, + FromStateVersion = stateVersion, + EntityId = referencedEntity.DatabaseId, + IsDeleted = metadataEntry.Value == null, + IsLocked = substateData.IsLocked, + ClaimedEntityIds = claimedEntities.Values.Select(address => _referencedEntities.Get((EntityAddress)address).DatabaseId).ToArray(), + }); } else if (key == StandardMetadataConstants.DappDefinitions && TryParseValue(out var dappDefinitions)) { - _entriesToAdd.Add(new DappDefinitionsUnverifiedStandardMetadataEntryHistory - { - Id = _context.Sequences.UnverifiedStandardMetadataEntryHistorySequence++, - FromStateVersion = stateVersion, - EntityId = referencedEntity.DatabaseId, - IsDeleted = metadataEntry.Value == null, - IsLocked = substateData.IsLocked, - DappDefinitionEntityIds = dappDefinitions.Values.Select(address => _referencedEntities.Get((EntityAddress)address).DatabaseId).ToArray(), - }); + _entriesToAdd.Add( + new DappDefinitionsUnverifiedStandardMetadataEntryHistory + { + Id = _context.Sequences.UnverifiedStandardMetadataEntryHistorySequence++, + FromStateVersion = stateVersion, + EntityId = referencedEntity.DatabaseId, + IsDeleted = metadataEntry.Value == null, + IsLocked = substateData.IsLocked, + DappDefinitionEntityIds = dappDefinitions.Values.Select(address => _referencedEntities.Get((EntityAddress)address).DatabaseId).ToArray(), + }); } else if (key == StandardMetadataConstants.DappAccountLocker && TryParseValue(out var dappAccountLocker)) { - _entriesToAdd.Add(new DappAccountLockerUnverifiedStandardMetadataEntryHistory - { - Id = _context.Sequences.UnverifiedStandardMetadataEntryHistorySequence++, - FromStateVersion = stateVersion, - EntityId = referencedEntity.DatabaseId, - IsDeleted = metadataEntry.Value == null, - IsLocked = substateData.IsLocked, - AccountLockerEntityId = _referencedEntities.Get((EntityAddress)dappAccountLocker.Value).DatabaseId, - }); + _entriesToAdd.Add( + new DappAccountLockerUnverifiedStandardMetadataEntryHistory + { + Id = _context.Sequences.UnverifiedStandardMetadataEntryHistorySequence++, + FromStateVersion = stateVersion, + EntityId = referencedEntity.DatabaseId, + IsDeleted = metadataEntry.Value == null, + IsLocked = substateData.IsLocked, + AccountLockerEntityId = _referencedEntities.Get((EntityAddress)dappAccountLocker.Value).DatabaseId, + }); } } else if (referencedEntity.Address.IsResource) { if (key == StandardMetadataConstants.DappDefinitions && TryParseValue(out var dappDefinitions)) { - _entriesToAdd.Add(new DappDefinitionsUnverifiedStandardMetadataEntryHistory - { - Id = _context.Sequences.UnverifiedStandardMetadataEntryHistorySequence++, - FromStateVersion = stateVersion, - EntityId = referencedEntity.DatabaseId, - IsDeleted = metadataEntry.Value == null, - IsLocked = substateData.IsLocked, - DappDefinitionEntityIds = dappDefinitions.Values.Select(address => _referencedEntities.Get((EntityAddress)address).DatabaseId).ToArray(), - }); + _entriesToAdd.Add( + new DappDefinitionsUnverifiedStandardMetadataEntryHistory + { + Id = _context.Sequences.UnverifiedStandardMetadataEntryHistorySequence++, + FromStateVersion = stateVersion, + EntityId = referencedEntity.DatabaseId, + IsDeleted = metadataEntry.Value == null, + IsLocked = substateData.IsLocked, + DappDefinitionEntityIds = dappDefinitions.Values.Select(address => _referencedEntities.Get((EntityAddress)address).DatabaseId).ToArray(), + }); } } else if (referencedEntity.Address.IsGlobal) { if (key == StandardMetadataConstants.DappDefinition && TryParseValue(out var dappDefinition)) { - _entriesToAdd.Add(new DappDefinitionUnverifiedStandardMetadataEntryHistory - { - Id = _context.Sequences.UnverifiedStandardMetadataEntryHistorySequence++, - FromStateVersion = stateVersion, - EntityId = referencedEntity.DatabaseId, - IsDeleted = metadataEntry.Value == null, - IsLocked = substateData.IsLocked, - DappDefinitionEntityId = _referencedEntities.Get((EntityAddress)dappDefinition.Value).DatabaseId, - }); + _entriesToAdd.Add( + new DappDefinitionUnverifiedStandardMetadataEntryHistory + { + Id = _context.Sequences.UnverifiedStandardMetadataEntryHistorySequence++, + FromStateVersion = stateVersion, + EntityId = referencedEntity.DatabaseId, + IsDeleted = metadataEntry.Value == null, + IsLocked = substateData.IsLocked, + DappDefinitionEntityId = _referencedEntities.Get((EntityAddress)dappDefinition.Value).DatabaseId, + }); } } } - public async Task LoadDependencies() + public async Task LoadDependenciesAsync() { _mostRecentEntries.AddRange(await MostRecentEntryHistory()); _mostRecentAggregates.AddRange(await MostRecentAggregateHistory()); @@ -274,7 +283,7 @@ public void ProcessChanges() } } - public async Task SaveEntities() + public async Task SaveEntitiesAsync() { var rowsInserted = 0; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ValidatorEmissionProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/ValidatorEmissionProcessor.cs similarity index 88% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ValidatorEmissionProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/ValidatorEmissionProcessor.cs index f71692aa0..fdd6ed5a6 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ValidatorEmissionProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/ValidatorEmissionProcessor.cs @@ -73,43 +73,42 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; -internal class ValidatorEmissionProcessor +internal class ValidatorEmissionProcessor : IProcessorBase, IDecodedEventProcessor { private record Emission(long FromStateVersion, long ValidatorEntityId, long EpochNumber, long ProposalsMade, long ProposalsMissed); private readonly ProcessorContext _context; - - private Dictionary _mostRecentCumulativeEmissions = new(); - - private List _emissions = new(); - private List _cumulativeEmissionsToAdd = new(); + private readonly Dictionary _mostRecentCumulativeEmissions = new(); + private readonly List _observedEmissions = new(); + private readonly List _cumulativeEmissionsToAdd = new(); public ValidatorEmissionProcessor(ProcessorContext context) { _context = context; } - public void VisitEvent(ToolkitModel.TypedNativeEvent decodedEvent, ReferencedEntity eventEmitterEntity, long stateVersion) + public void VisitDecodedEvent(ToolkitModel.TypedNativeEvent decodedEvent, ReferencedEntity eventEmitterEntity, long stateVersion) { if (EventDecoder.TryGetValidatorEmissionsAppliedEvent(decodedEvent, out var validatorUptimeEvent)) { - _emissions.Add(new Emission( - stateVersion, - eventEmitterEntity.DatabaseId, - (long)validatorUptimeEvent.epoch, - (long)validatorUptimeEvent.proposalsMade, - (long)validatorUptimeEvent.proposalsMissed)); + _observedEmissions.Add( + new Emission( + stateVersion, + eventEmitterEntity.DatabaseId, + (long)validatorUptimeEvent.epoch, + (long)validatorUptimeEvent.proposalsMade, + (long)validatorUptimeEvent.proposalsMissed)); } } - public async Task LoadDependencies() + public async Task LoadDependenciesAsync() { _mostRecentCumulativeEmissions.AddRange(await MostRecentCumulativeEmissions()); } public void ProcessChanges() { - foreach (var emission in _emissions) + foreach (var emission in _observedEmissions) { var proposalsMade = 0L; var proposalsMissed = 0L; @@ -142,7 +141,7 @@ public void ProcessChanges() } } - public async Task SaveEntities() + public async Task SaveEntitiesAsync() { var rowsInserted = 0; @@ -153,7 +152,7 @@ public async Task SaveEntities() private async Task> MostRecentCumulativeEmissions() { - var validatorEntityIds = _emissions.Select(e => e.ValidatorEntityId).ToHashSet().ToList(); + var validatorEntityIds = _observedEmissions.Select(e => e.ValidatorEntityId).ToHashSet().ToList(); if (!validatorEntityIds.Any()) { diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ValidatorProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/ValidatorProcessor.cs similarity index 77% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ValidatorProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/ValidatorProcessor.cs index 2879cef8b..5473b029a 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ValidatorProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/ValidatorProcessor.cs @@ -77,25 +77,20 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; -internal record struct ValidatorPublicKeyLookup(long ValidatorEntityId, PublicKeyType PublicKeyType, ValueBytes PublicKey); +internal class ValidatorProcessor : IProcessorBase, ISubstateUpsertProcessor +{ + private record struct ValidatorPublicKeyLookup(long ValidatorEntityId, PublicKeyType PublicKeyType, ValueBytes PublicKey); -internal record ValidatorActiveSet(long Epoch, IDictionary ValidatorStake, long StateVersion); + private record ValidatorActiveSet(long Epoch, IDictionary ValidatorStake, long StateVersion); -internal class ValidatorProcessor -{ private readonly ProcessorContext _context; private readonly ReferencedEntityDictionary _referencedEntities; - - private List _changes = new(); - - /// - /// A collection of validator public keys by the earliest state version they were observed on. - /// - private Dictionary _observedPublicKeys = new(); - private Dictionary _existingPublicKeys = new(); - - private List _publicKeysToAdd = new(); - private List _activeSetsToAdd = new(); + private readonly Dictionary _passingEpochDictionary = new(); + private readonly List _changes = new(); + private readonly Dictionary _publicKeyObservedAtStateVersion = new(); + private readonly Dictionary _existingPublicKeys = new(); + private readonly List _publicKeysToAdd = new(); + private readonly List _activeSetsToAdd = new(); public ValidatorProcessor(ProcessorContext context, ReferencedEntityDictionary referencedEntities) { @@ -103,26 +98,36 @@ public ValidatorProcessor(ProcessorContext context, ReferencedEntityDictionary r _referencedEntities = referencedEntities; } - public void VisitUpsert(CoreModel.Substate substateData, ReferencedEntity referencedEntity, long stateVersion, long? passingEpoch) + public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion) { + var substateData = substate.Value.SubstateData; + + if (substateData is CoreModel.ConsensusManagerFieldStateSubstate consensusManagerFieldStateSubstate) + { + if (consensusManagerFieldStateSubstate.Value.Round == 0) + { + _passingEpochDictionary.Add(stateVersion, consensusManagerFieldStateSubstate.Value.Epoch - 1); + } + } + if (substateData is CoreModel.ConsensusManagerRegisteredValidatorsByStakeIndexEntrySubstate entry) { var av = entry.Value.ActiveValidator; var lookup = new ValidatorPublicKeyLookup(_referencedEntities.Get((EntityAddress)av.Address).DatabaseId, av.Key.KeyType.ToModel(), av.Key.GetKeyBytes()); - _observedPublicKeys.TryAdd(lookup, stateVersion); + _publicKeyObservedAtStateVersion.TryAdd(lookup, stateVersion); } if (substateData is CoreModel.ValidatorFieldStateSubstate state) { var lookup = new ValidatorPublicKeyLookup(referencedEntity.DatabaseId, state.Value.PublicKey.KeyType.ToModel(), state.Value.PublicKey.GetKeyBytes()); - _observedPublicKeys.TryAdd(lookup, stateVersion); + _publicKeyObservedAtStateVersion.TryAdd(lookup, stateVersion); } if (substateData is CoreModel.ConsensusManagerFieldCurrentValidatorSetSubstate validatorSet) { - if (!passingEpoch.HasValue) + if (!_passingEpochDictionary.TryGetValue(stateVersion, out var passingEpoch)) { throw new InvalidOperationException("ConsensusManagerFieldCurrentValidatorSetSubstate can't be processed unless epoch change gets detected"); } @@ -133,27 +138,27 @@ public void VisitUpsert(CoreModel.Substate substateData, ReferencedEntity refere { var lookup = new ValidatorPublicKeyLookup(_referencedEntities.Get((EntityAddress)v.Address).DatabaseId, v.Key.KeyType.ToModel(), v.Key.GetKeyBytes()); - _observedPublicKeys.TryAdd(lookup, stateVersion); + _publicKeyObservedAtStateVersion.TryAdd(lookup, stateVersion); validatorStake[lookup] = TokenAmount.FromDecimalString(v.Stake); } - _changes.Add(new ValidatorActiveSet(passingEpoch.Value, validatorStake, stateVersion)); + _changes.Add(new ValidatorActiveSet(passingEpoch, validatorStake, stateVersion)); } } - public async Task LoadDependencies() + public async Task LoadDependenciesAsync() { _existingPublicKeys.AddRange(await ExistingValidatorPublicKeys()); } public void ProcessChanges() { - foreach (var lookup in _observedPublicKeys.Keys.Except(_existingPublicKeys.Keys)) + foreach (var lookup in _publicKeyObservedAtStateVersion.Keys.Except(_existingPublicKeys.Keys)) { var publicKey = new ValidatorPublicKeyHistory { Id = _context.Sequences.ValidatorPublicKeyHistorySequence++, - FromStateVersion = _observedPublicKeys[lookup], + FromStateVersion = _publicKeyObservedAtStateVersion[lookup], ValidatorEntityId = lookup.ValidatorEntityId, KeyType = lookup.PublicKeyType, Key = lookup.PublicKey, @@ -167,19 +172,20 @@ public void ProcessChanges() { foreach (var (lookup, stake) in change.ValidatorStake) { - _activeSetsToAdd.Add(new ValidatorActiveSetHistory - { - Id = _context.Sequences.ValidatorActiveSetHistorySequence++, - FromStateVersion = change.StateVersion, - Epoch = change.Epoch, - ValidatorPublicKeyHistoryId = _existingPublicKeys[lookup].Id, - Stake = stake, - }); + _activeSetsToAdd.Add( + new ValidatorActiveSetHistory + { + Id = _context.Sequences.ValidatorActiveSetHistorySequence++, + FromStateVersion = change.StateVersion, + Epoch = change.Epoch, + ValidatorPublicKeyHistoryId = _existingPublicKeys[lookup].Id, + Stake = stake, + }); } } } - public async Task SaveEntities() + public async Task SaveEntitiesAsync() { var rowsInserted = 0; @@ -191,7 +197,7 @@ public async Task SaveEntities() private async Task> ExistingValidatorPublicKeys() { - if (!_observedPublicKeys.Keys.ToHashSet().Unzip(x => x.ValidatorEntityId, x => x.PublicKeyType, x => (byte[])x.PublicKey, out var entityIds, out var keyTypes, out var keys)) + if (!_publicKeyObservedAtStateVersion.Keys.ToHashSet().Unzip(x => x.ValidatorEntityId, x => x.PublicKeyType, x => (byte[])x.PublicKey, out var entityIds, out var keyTypes, out var keys)) { return ImmutableDictionary.Empty; } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/VaultProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/VaultProcessor.cs similarity index 98% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/VaultProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/VaultProcessor.cs index 0dc6b351c..58abdf297 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/VaultProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/VaultProcessor.cs @@ -74,7 +74,7 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; -internal class VaultProcessor +internal class VaultProcessor : IProcessorBase, ISubstateUpsertProcessor { private readonly record struct NonFungibleIdDefinitionDbLookup(long NonFungibleResourceEntityId, string NonFungibleId); @@ -105,8 +105,10 @@ public VaultProcessor(ProcessorContext context) _context = context; } - public void VisitUpsert(CoreModel.Substate substateData, ReferencedEntity referencedEntity, long stateVersion, CoreModel.IUpsertedSubstate substate) + public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion) { + var substateData = substate.Value.SubstateData; + if (substateData is CoreModel.FungibleVaultFieldBalanceSubstate fungibleVaultFieldBalanceSubstate) { var vaultEntity = referencedEntity.GetDatabaseEntity(); @@ -156,7 +158,7 @@ public void VisitDelete(CoreModel.SubstateId substateId, ReferencedEntity refere } } - public async Task LoadDependencies() + public async Task LoadDependenciesAsync() { _existingNonFungibleIdDefinitions.AddRange(await ExistingNonFungibleIdDefinitions()); _existingNonFungibleVaultEntryDefinitions.AddRange(await ExistingNonFungibleVaultEntryDefinitions()); @@ -264,7 +266,7 @@ public void ProcessChanges() } } - public async Task SaveEntities() + public async Task SaveEntitiesAsync() { var rowsInserted = 0; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs index a73a52732..feb86f71c 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs @@ -113,40 +113,6 @@ public async Task> LoadDependencies([NotP return result; } - public async Task> MostRecentResourceEntitySupplyHistoryFor(List resourceSupplyChanges, CancellationToken token) - { - if (!resourceSupplyChanges.Any()) - { - return new Dictionary(); - } - - var sw = Stopwatch.GetTimestamp(); - var ids = resourceSupplyChanges.Select(c => c.ResourceEntityId).Distinct().ToList(); - - var result = await _dbContext - .ResourceEntitySupplyHistory - .FromSqlInterpolated(@$" -WITH variables (resource_entity_id) AS ( - SELECT UNNEST({ids}) -) -SELECT rmesh.* -FROM variables -INNER JOIN LATERAL ( - SELECT * - FROM resource_entity_supply_history - WHERE resource_entity_id = variables.resource_entity_id - ORDER BY from_state_version DESC - LIMIT 1 -) rmesh ON true;") - .AsNoTracking() - .AnnotateMetricName() - .ToDictionaryAsync(e => e.ResourceEntityId, token); - - await _observers.ForEachAsync(x => x.StageCompleted(nameof(MostRecentResourceEntitySupplyHistoryFor), Stopwatch.GetElapsedTime(sw), result.Count)); - - return result; - } - public async Task> ExistingEntitiesFor(ReferencedEntityDictionary referencedEntities, CancellationToken token) { var sw = Stopwatch.GetTimestamp(); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/WriteHelper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/WriteHelper.cs index 00e91f878..e88f5cd4d 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/WriteHelper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/WriteHelper.cs @@ -196,296 +196,6 @@ public async Task CopyEntity(ICollection entities, CancellationToke return entities.Count; } - public async Task CopyLedgerTransaction(ICollection entities, CancellationToken token) - { - if (!entities.Any()) - { - return 0; - } - - var sw = Stopwatch.GetTimestamp(); - - await using var writer = await _connection.BeginBinaryImportAsync( - "COPY ledger_transactions (state_version, transaction_tree_hash, receipt_tree_hash, state_tree_hash, epoch, round_in_epoch, index_in_epoch, index_in_round, fee_paid, tip_paid, affected_global_entities, round_timestamp, created_timestamp, normalized_round_timestamp, balance_changes, receipt_state_updates, receipt_status, receipt_fee_summary, receipt_fee_source, receipt_fee_destination, receipt_costing_parameters, receipt_error_message, receipt_output, receipt_next_epoch, receipt_event_emitters, receipt_event_names, receipt_event_sbors, receipt_event_schema_entity_ids, receipt_event_schema_hashes, receipt_event_type_indexes, receipt_event_sbor_type_kinds, discriminator, payload_hash, intent_hash, signed_intent_hash, message, raw_payload, manifest_instructions, manifest_classes) FROM STDIN (FORMAT BINARY)", - token); - - foreach (var lt in entities) - { - var discriminator = GetDiscriminator(lt.GetType()); - - await HandleMaxAggregateCounts(lt); - await writer.StartRowAsync(token); - await writer.WriteAsync(lt.StateVersion, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(lt.TransactionTreeHash, NpgsqlDbType.Text, token); - await writer.WriteAsync(lt.ReceiptTreeHash, NpgsqlDbType.Text, token); - await writer.WriteAsync(lt.StateTreeHash, NpgsqlDbType.Text, token); - await writer.WriteAsync(lt.Epoch, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(lt.RoundInEpoch, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(lt.IndexInEpoch, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(lt.IndexInRound, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(lt.FeePaid.GetSubUnitsSafeForPostgres(), NpgsqlDbType.Numeric, token); - await writer.WriteAsync(lt.TipPaid.GetSubUnitsSafeForPostgres(), NpgsqlDbType.Numeric, token); - await writer.WriteAsync(lt.AffectedGlobalEntities, NpgsqlDbType.Array | NpgsqlDbType.Bigint, token); - await writer.WriteAsync(lt.RoundTimestamp, NpgsqlDbType.TimestampTz, token); - await writer.WriteAsync(lt.CreatedTimestamp, NpgsqlDbType.TimestampTz, token); - await writer.WriteAsync(lt.NormalizedRoundTimestamp, NpgsqlDbType.TimestampTz, token); - await writer.WriteAsync(lt.BalanceChanges, NpgsqlDbType.Jsonb, token); - - await writer.WriteAsync(lt.EngineReceipt.StateUpdates, NpgsqlDbType.Jsonb, token); - await writer.WriteAsync(lt.EngineReceipt.Status, "ledger_transaction_status", token); - await writer.WriteAsync(lt.EngineReceipt.FeeSummary, NpgsqlDbType.Jsonb, token); - await writer.WriteAsync(lt.EngineReceipt.FeeSource, NpgsqlDbType.Jsonb, token); - await writer.WriteAsync(lt.EngineReceipt.FeeDestination, NpgsqlDbType.Jsonb, token); - await writer.WriteAsync(lt.EngineReceipt.CostingParameters, NpgsqlDbType.Jsonb, token); - await writer.WriteAsync(lt.EngineReceipt.ErrorMessage, NpgsqlDbType.Text, token); - await writer.WriteAsync(lt.EngineReceipt.Output, NpgsqlDbType.Jsonb, token); - await writer.WriteAsync(lt.EngineReceipt.NextEpoch, NpgsqlDbType.Jsonb, token); - await writer.WriteAsync(lt.EngineReceipt.Events.Emitters, NpgsqlDbType.Array | NpgsqlDbType.Jsonb, token); - await writer.WriteAsync(lt.EngineReceipt.Events.Names, NpgsqlDbType.Array | NpgsqlDbType.Text, token); - await writer.WriteAsync(lt.EngineReceipt.Events.Sbors, NpgsqlDbType.Array | NpgsqlDbType.Bytea, token); - await writer.WriteAsync(lt.EngineReceipt.Events.SchemaEntityIds, NpgsqlDbType.Array | NpgsqlDbType.Bigint, token); - await writer.WriteAsync(lt.EngineReceipt.Events.SchemaHashes, NpgsqlDbType.Array | NpgsqlDbType.Bytea, token); - await writer.WriteAsync(lt.EngineReceipt.Events.TypeIndexes, NpgsqlDbType.Array | NpgsqlDbType.Bigint, token); - await writer.WriteAsync(lt.EngineReceipt.Events.SborTypeKinds, "sbor_type_kind[]", token); - await writer.WriteAsync(discriminator, "ledger_transaction_type", token); - - switch (lt) - { - case GenesisLedgerTransaction: - case RoundUpdateLedgerTransaction: - case FlashLedgerTransaction: - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - break; - case UserLedgerTransaction ult: - await writer.WriteAsync(ult.PayloadHash, NpgsqlDbType.Text, token); - await writer.WriteAsync(ult.IntentHash, NpgsqlDbType.Text, token); - await writer.WriteAsync(ult.SignedIntentHash, NpgsqlDbType.Text, token); - await writer.WriteAsync(ult.Message, NpgsqlDbType.Jsonb, token); - await writer.WriteAsync(ult.RawPayload, NpgsqlDbType.Bytea, token); - await writer.WriteAsync(ult.ManifestInstructions, NpgsqlDbType.Text, token); - await writer.WriteAsync(ult.ManifestClasses, "ledger_transaction_manifest_class[]", token); - break; - default: - throw new ArgumentOutOfRangeException(nameof(lt), lt, null); - } - } - - await writer.CompleteAsync(token); - - await _observers.ForEachAsync(x => x.StageCompleted(nameof(CopyLedgerTransaction), Stopwatch.GetElapsedTime(sw), entities.Count)); - - return entities.Count; - } - - public async Task CopyLedgerTransactionMarkers(ICollection entities, CancellationToken token) - { - if (!entities.Any()) - { - return 0; - } - - var sw = Stopwatch.GetTimestamp(); - - await using var writer = - await _connection.BeginBinaryImportAsync( - "COPY ledger_transaction_markers (id, state_version, discriminator, event_type, entity_id, resource_entity_id, quantity, operation_type, origin_type, manifest_class, is_most_specific) FROM STDIN (FORMAT BINARY)", - token); - - foreach (var e in entities) - { - var discriminator = GetDiscriminator(e.GetType()); - - await HandleMaxAggregateCounts(e); - await writer.StartRowAsync(token); - await writer.WriteAsync(e.Id, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(e.StateVersion, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(discriminator, "ledger_transaction_marker_type", token); - - switch (e) - { - case EventLedgerTransactionMarker eltm: - await writer.WriteAsync(eltm.EventType, "ledger_transaction_marker_event_type", token); - await writer.WriteAsync(eltm.EntityId, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(eltm.ResourceEntityId, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(eltm.Quantity.GetSubUnitsSafeForPostgres(), NpgsqlDbType.Numeric, token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - break; - case ManifestAddressLedgerTransactionMarker maltm: - await writer.WriteNullAsync(token); - await writer.WriteAsync(maltm.EntityId, NpgsqlDbType.Bigint, token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteAsync(maltm.OperationType, "ledger_transaction_marker_operation_type", token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - break; - case OriginLedgerTransactionMarker oltm: - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteAsync(oltm.OriginType, "ledger_transaction_marker_origin_type", token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - break; - case AffectedGlobalEntityTransactionMarker oltm: - await writer.WriteNullAsync(token); - await writer.WriteAsync(oltm.EntityId, NpgsqlDbType.Bigint, token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - break; - case EventGlobalEmitterTransactionMarker egetm: - await writer.WriteNullAsync(token); - await writer.WriteAsync(egetm.EntityId, NpgsqlDbType.Bigint, token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - break; - case ManifestClassMarker ttm: - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteAsync(ttm.ManifestClass, "ledger_transaction_manifest_class", token); - await writer.WriteAsync(ttm.IsMostSpecific, NpgsqlDbType.Boolean, token); - break; - default: - throw new ArgumentOutOfRangeException(nameof(e), e, null); - } - } - - await writer.CompleteAsync(token); - - await _observers.ForEachAsync(x => x.StageCompleted(nameof(CopyLedgerTransactionMarkers), Stopwatch.GetElapsedTime(sw), entities.Count)); - - return entities.Count; - } - - public async Task CopyResourceEntitySupplyHistory(ICollection entities, CancellationToken token) - { - if (!entities.Any()) - { - return 0; - } - - var sw = Stopwatch.GetTimestamp(); - - await using var writer = - await _connection.BeginBinaryImportAsync( - "COPY resource_entity_supply_history (id, from_state_version, resource_entity_id, total_supply, total_minted, total_burned) FROM STDIN (FORMAT BINARY)", - token); - - foreach (var e in entities) - { - await HandleMaxAggregateCounts(e); - await writer.StartRowAsync(token); - await writer.WriteAsync(e.Id, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(e.FromStateVersion, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(e.ResourceEntityId, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(e.TotalSupply.GetSubUnitsSafeForPostgres(), NpgsqlDbType.Numeric, token); - await writer.WriteAsync(e.TotalMinted.GetSubUnitsSafeForPostgres(), NpgsqlDbType.Numeric, token); - await writer.WriteAsync(e.TotalBurned.GetSubUnitsSafeForPostgres(), NpgsqlDbType.Numeric, token); - } - - await writer.CompleteAsync(token); - - await _observers.ForEachAsync(x => x.StageCompleted(nameof(CopyResourceEntitySupplyHistory), Stopwatch.GetElapsedTime(sw), entities.Count)); - - return entities.Count; - } - - public async Task CopyNonFungibleDataSchemaHistory(ICollection entities, CancellationToken token) - { - if (!entities.Any()) - { - return 0; - } - - var sw = Stopwatch.GetTimestamp(); - - await using var writer = - await _connection.BeginBinaryImportAsync( - "COPY non_fungible_schema_history (id, from_state_version, resource_entity_id, schema_defining_entity_id, schema_hash, sbor_type_kind, type_index) FROM STDIN (FORMAT BINARY)", - token); - - foreach (var e in entities) - { - await HandleMaxAggregateCounts(e); - await writer.StartRowAsync(token); - await writer.WriteAsync(e.Id, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(e.FromStateVersion, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(e.ResourceEntityId, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(e.SchemaDefiningEntityId, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(e.SchemaHash, NpgsqlDbType.Bytea, token); - await writer.WriteAsync(e.SborTypeKind, "sbor_type_kind", token); - await writer.WriteAsync(e.TypeIndex, NpgsqlDbType.Bigint, token); - } - - await writer.CompleteAsync(token); - - await _observers.ForEachAsync(x => x.StageCompleted(nameof(CopyNonFungibleDataSchemaHistory), Stopwatch.GetElapsedTime(sw), entities.Count)); - - return entities.Count; - } - - public async Task CopyKeyValueStoreSchemaHistory(ICollection entities, CancellationToken token) - { - if (!entities.Any()) - { - return 0; - } - - var sw = Stopwatch.GetTimestamp(); - - await using var writer = - await _connection.BeginBinaryImportAsync( - "COPY key_value_store_schema_history (id, from_state_version, key_value_store_entity_id, key_schema_defining_entity_id, key_schema_hash, key_sbor_type_kind, key_type_index, value_schema_defining_entity_id, value_schema_hash, value_sbor_type_kind, value_type_index) FROM STDIN (FORMAT BINARY)", - token); - - foreach (var e in entities) - { - await HandleMaxAggregateCounts(e); - await writer.StartRowAsync(token); - await writer.WriteAsync(e.Id, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(e.FromStateVersion, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(e.KeyValueStoreEntityId, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(e.KeySchemaDefiningEntityId, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(e.KeySchemaHash, NpgsqlDbType.Bytea, token); - await writer.WriteAsync(e.KeySborTypeKind, "sbor_type_kind", token); - await writer.WriteAsync(e.KeyTypeIndex, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(e.ValueSchemaDefiningEntityId, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(e.ValueSchemaHash, NpgsqlDbType.Bytea, token); - await writer.WriteAsync(e.ValueSborTypeKind, "sbor_type_kind", token); - await writer.WriteAsync(e.ValueTypeIndex, NpgsqlDbType.Bigint, token); - } - - await writer.CompleteAsync(token); - - await _observers.ForEachAsync(x => x.StageCompleted(nameof(CopyKeyValueStoreSchemaHistory), Stopwatch.GetElapsedTime(sw), entities.Count)); - - return entities.Count; - } - public async Task UpdateSequences(SequencesHolder sequences, CancellationToken token) { var sw = Stopwatch.GetTimestamp(); @@ -608,7 +318,7 @@ public T GetDiscriminator(Type type) return discriminator; } - private async ValueTask HandleMaxAggregateCounts(object? entity) + public async ValueTask HandleMaxAggregateCounts(object? entity) { if (entity is not IAggregateHolder aggregateHolder) { diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/Entity.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/Entity.cs index 211a42490..3215f1d10 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/Entity.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/Entity.cs @@ -65,6 +65,7 @@ using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Model; using RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; +using RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors; using System; using System.Collections.Generic; using System.ComponentModel.DataAnnotations; From c56353a73b9de63a0ca9722f98e99e0ef16218d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Wed, 25 Sep 2024 16:53:30 +0200 Subject: [PATCH 2/5] add config parameters to allow reducing ledger transactions table size. --- CHANGELOG.md | 9 + docs/configuration.md | 4 + .../Configuration/StorageOptions.cs | 94 +++++ .../ServiceCollectionExtensions.cs | 1 + .../Services/ILedgerExtenderService.cs | 3 - .../CommonDbContext.cs | 9 +- .../CustomTypesRegistrator.cs | 2 + .../GatewayModelExtensions.cs | 69 ---- .../LedgerExtension/Extensions.cs | 16 + .../LedgerExtension/IProcessor.cs | 76 +++- .../PostgresLedgerExtenderService.cs | 39 +- .../LedgerExtension/ProcessorContext.cs | 9 +- .../AccountAuthorizedDepositorsProcessor.cs | 2 +- .../AccountDefaultDepositRuleProcessor.cs | 2 +- ...AccountResourcePreferenceRulesProcessor.cs | 2 +- .../EntityRelationshipProcessor.cs} | 8 +- .../EventLedgerTransactionMarkerProcessor.cs | 66 +++- .../GlobalEventEmitterProcessor.cs | 4 +- .../LedgerTransactionMarkersProcessor.cs | 74 +++- .../LedgerTransactionProcessor.cs | 254 ------------ .../ManifestProcessor.cs | 72 +++- .../OriginLedgerTransactionMarkerProcessor.cs | 66 +++- .../Processors/LedgerTransactionProcessor.cs | 365 ++++++++++++++++++ .../Processors/VaultProcessor.cs | 2 +- ... 20240925082442_InitialCreate.Designer.cs} | 86 +++-- ...ate.cs => 20240925082442_InitialCreate.cs} | 30 +- .../Migrations/IdempotentApplyMigrations.sql | 292 +++++++------- .../MigrationsDbContextModelSnapshot.cs | 82 ++-- .../LedgerTransactions/LedgerTransaction.cs | 119 ++---- .../Queries/EntityQuerier.cs | 30 +- .../AccountStateQuerier.cs | 4 +- .../DepositPreValidationQuerier.cs | 3 +- .../Services/DapperWrapperExtensions.cs | 4 +- .../Services/KeyValueStoreQuerier.cs | 4 +- .../Services/TopOfLedgerProvider.cs | 6 - .../Services/TransactionMapper.cs | 146 +++++++ .../Services/TransactionQuerier.cs | 285 +++++++++----- 37 files changed, 1537 insertions(+), 802 deletions(-) create mode 100644 src/RadixDlt.NetworkGateway.Abstractions/Configuration/StorageOptions.cs rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/{ => AccountSecurityRules}/AccountAuthorizedDepositorsProcessor.cs (99%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/{ => AccountSecurityRules}/AccountDefaultDepositRuleProcessor.cs (99%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/{ => AccountSecurityRules}/AccountResourcePreferenceRulesProcessor.cs (99%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/{RelationshipProcessor.cs => Processors/EntityRelationshipProcessor.cs} (97%) delete mode 100644 src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/LedgerTransactionProcessor.cs create mode 100644 src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionProcessor.cs rename src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/{20241001130214_InitialCreate.Designer.cs => 20240925082442_InitialCreate.Designer.cs} (99%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/{20241001130214_InitialCreate.cs => 20240925082442_InitialCreate.cs} (99%) create mode 100644 src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionMapper.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 139ea1566..9cb311e83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,13 @@ Release built: _not released yet_ > - `/state/entity/page/non-fungibles/` (when using `non_fungible_include_nfids` opt-in) > - `/state/entity/page/non-fungible-vaults/` (when using `non_fungible_include_nfids` opt-in) +### What’s new? +- New configuration options `DataAggregator__Storage__StoreTransactionReceiptEvents`, and `DataAggregator__Storage__StoreReceiptStateUpdates` for data aggregator to configure if transaction's receipt events and receipt state updates should be stored in the database. It is meant to be used by gateway runners who want to reduce their database size. Keep in mind that when disabled `/stream/transactions` and `/stream/transactions` will not return that data. + - Possible values: + - `StoreForAllTransactions` (default) - will store data for all transactions. + - `StoryOnlyForUserTransactionsAndEpochChanges` - will store data for user transactions and transactions that resulted in epoch change. + - `StoreOnlyForUserTransactions` - will store data only for user transactions. + - `DoNotStore` - will not store any data. ### Bug fixes - Added missing `total_count` property to `/state/validators/list` response. @@ -57,6 +64,8 @@ Release built: _not released yet_ - Renamed `entity_vault_history` to `vault_balance_history`. Holds information about vault content (amount of fungibles or count of non fungible ids inside vault) at a given state version. - Key value store - New `key_value_store_totals_history` table, which holds total count of all keys under a given store at a given state version. +- Changed `receipt_state_updates` in the `ledger_transactions` table to be nullable. +- Moved all `receipt_event_*` columns from `ledger_transactions` to separate `ledger_transaction_events` table. ## 1.7.3 Release built: 26.09.2024 diff --git a/docs/configuration.md b/docs/configuration.md index 1d6a12dae..450285f00 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -73,6 +73,10 @@ The Network Gateway services can be configured in line with the [configuration i - `DataAggregator__Network__MaxAllowedStateVersionLagToBeConsideredSynced` (type: `int`, default value: `100`) - maximum allowed state version lag for CoreAPI node to be considered synced. - `DataAggregator__Network__IgnoreNonSyncedNodes` (type: `bool`, default value: `true`) - controls if nodes with a different status than healthy and synced should be used by the Data Aggregator. +#### Storage +- `DataAggregator__Storage__StoreTransactionReceiptEvents` (type: `enum`, default value: `StoreForAllTransactions`) - controls if data aggregator should store transaction receipt events in database. +- `DataAggregator__Storage__StoreReceiptStateUpdates` (type: `enum`, default value: `StoreForAllTransactions`) - controls if data aggregator should store transaction receipt state updates in database. + #### Monitoring `DataAggregator__Monitoring__StartupGracePeriodSeconds` (type: `int`, default value: `10`) - duration (seconds) of start-up grace period for Data Aggregator. `DataAggregator__Monitoring__UnhealthyCommitmentGapSeconds` (type: `int`, default value: `20`) - time window since the last committed transaction (seconds) in which Data Aggregator is considered healthy if does not commit new transactions. diff --git a/src/RadixDlt.NetworkGateway.Abstractions/Configuration/StorageOptions.cs b/src/RadixDlt.NetworkGateway.Abstractions/Configuration/StorageOptions.cs new file mode 100644 index 000000000..07bb37fd5 --- /dev/null +++ b/src/RadixDlt.NetworkGateway.Abstractions/Configuration/StorageOptions.cs @@ -0,0 +1,94 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +using FluentValidation; +using Microsoft.Extensions.Configuration; + +namespace RadixDlt.NetworkGateway.Abstractions.Configuration; + +public sealed class StorageOptions +{ + [ConfigurationKeyName("StoreTransactionReceiptEvents")] + public LedgerTransactionStorageOption StoreTransactionReceiptEvents { get; set; } = LedgerTransactionStorageOption.StoreForAllTransactions; + + [ConfigurationKeyName("StoreReceiptStateUpdates")] + public LedgerTransactionStorageOption StoreReceiptStateUpdates { get; set; } = LedgerTransactionStorageOption.StoreForAllTransactions; +} + +public sealed class StorageOptionsValidator : AbstractOptionsValidator +{ + public StorageOptionsValidator() + { + RuleFor(x => x.StoreTransactionReceiptEvents).IsInEnum(); + RuleFor(x => x.StoreReceiptStateUpdates).IsInEnum(); + } +} + +public enum LedgerTransactionStorageOption +{ + StoreForAllTransactions, + StoryOnlyForUserTransactionsAndEpochChanges, + StoreOnlyForUserTransactions, + DoNotStore, +} diff --git a/src/RadixDlt.NetworkGateway.DataAggregator/ServiceCollectionExtensions.cs b/src/RadixDlt.NetworkGateway.DataAggregator/ServiceCollectionExtensions.cs index ca6039323..924d2194c 100644 --- a/src/RadixDlt.NetworkGateway.DataAggregator/ServiceCollectionExtensions.cs +++ b/src/RadixDlt.NetworkGateway.DataAggregator/ServiceCollectionExtensions.cs @@ -99,6 +99,7 @@ public static DataAggregatorBuilder AddNetworkGatewayDataAggregatorCore(this ISe services .AddValidatableOptionsAtSection("DataAggregator:Network") + .AddValidatableOptionsAtSection("DataAggregator:Storage") .AddValidatableOptionsAtSection("DataAggregator:SlowQueryLogging") .AddValidatableOptionsAtSection("DataAggregator:Monitoring") .AddValidatableOptionsAtSection("DataAggregator:Mempool") diff --git a/src/RadixDlt.NetworkGateway.DataAggregator/Services/ILedgerExtenderService.cs b/src/RadixDlt.NetworkGateway.DataAggregator/Services/ILedgerExtenderService.cs index decc875d0..ddb929afa 100644 --- a/src/RadixDlt.NetworkGateway.DataAggregator/Services/ILedgerExtenderService.cs +++ b/src/RadixDlt.NetworkGateway.DataAggregator/Services/ILedgerExtenderService.cs @@ -77,9 +77,6 @@ public interface ILedgerExtenderService public sealed record TransactionSummary( long StateVersion, - string TransactionTreeHash, - string ReceiptTreeHash, - string StateTreeHash, DateTime RoundTimestamp, DateTime NormalizedRoundTimestamp, DateTime CreatedTimestamp, diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs index bb5551abc..0f79fc06a 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/CommonDbContext.cs @@ -81,15 +81,10 @@ internal abstract class CommonDbContext : DbContext { internal const string DiscriminatorColumnName = "discriminator"; - /// - /// Gets LedgerTransactions. - /// - /// - /// A LedgerTransaction row contains large blobs, so you must SELECT the fields you need after using this, and not pull down the whole - /// ledger transaction row, to avoid possible performance issues. - /// public DbSet LedgerTransactions => Set(); + public DbSet LedgerTransactionEvents => Set(); + public DbSet LedgerTransactionMarkers => Set(); public DbSet PendingTransactions => Set(); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/CustomTypesRegistrator.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/CustomTypesRegistrator.cs index ec0ee233b..74e4ed65f 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/CustomTypesRegistrator.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/CustomTypesRegistrator.cs @@ -94,6 +94,8 @@ public static void EnsureConfigured() SqlMapper.AddTypeHandler(new GenericArrayHandler()); SqlMapper.AddTypeHandler(new GenericArrayHandler()); SqlMapper.AddTypeHandler(new GenericArrayHandler()); + SqlMapper.AddTypeHandler(new GenericArrayHandler()); + SqlMapper.AddTypeHandler(new GenericArrayHandler()); SqlMapper.AddTypeHandler(new GenericListHandler()); SqlMapper.AddTypeHandler(new GenericListHandler()); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayModelExtensions.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayModelExtensions.cs index ce27b0f36..9ff1c734c 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayModelExtensions.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayModelExtensions.cs @@ -180,75 +180,6 @@ public static GatewayModel.PublicKey ToGatewayPublicKey(this ValidatorPublicKeyH }; } - public static GatewayModel.CommittedTransactionInfo ToGatewayModel( - this LedgerTransaction lt, - GatewayModel.TransactionDetailsOptIns optIns, - Dictionary entityIdToAddressMap, - List? events, - GatewayModel.TransactionBalanceChanges? transactionBalanceChanges) - { - string? payloadHash = null; - string? intentHash = null; - string? rawHex = null; - JRaw? message = null; - string? manifestInstructions = null; - List? manifestClasses = null; - - if (lt is UserLedgerTransaction ult) - { - payloadHash = ult.PayloadHash; - intentHash = ult.IntentHash; - rawHex = optIns.RawHex ? ult.RawPayload.ToHex() : null; - message = ult.Message != null ? new JRaw(ult.Message) : null; - manifestInstructions = optIns.ManifestInstructions ? ult.ManifestInstructions : null; - manifestClasses = ult.ManifestClasses.Select(mc => mc.ToGatewayModel()).ToList(); - } - - var receipt = new GatewayModel.TransactionReceipt - { - ErrorMessage = lt.EngineReceipt.ErrorMessage, - Status = ToGatewayModel(lt.EngineReceipt.Status), - Output = optIns.ReceiptOutput && lt.EngineReceipt.Output != null ? new JRaw(lt.EngineReceipt.Output) : null, - FeeSummary = optIns.ReceiptFeeSummary ? new JRaw(lt.EngineReceipt.FeeSummary) : null, - FeeDestination = optIns.ReceiptFeeDestination && lt.EngineReceipt.FeeDestination != null ? new JRaw(lt.EngineReceipt.FeeDestination) : null, - FeeSource = optIns.ReceiptFeeSource && lt.EngineReceipt.FeeSource != null ? new JRaw(lt.EngineReceipt.FeeSource) : null, - CostingParameters = optIns.ReceiptCostingParameters ? new JRaw(lt.EngineReceipt.CostingParameters) : null, - NextEpoch = lt.EngineReceipt.NextEpoch != null ? new JRaw(lt.EngineReceipt.NextEpoch) : null, - StateUpdates = optIns.ReceiptStateChanges ? new JRaw(lt.EngineReceipt.StateUpdates) : null, - Events = optIns.ReceiptEvents ? events?.Select(x => new GatewayModel.EventsItem(x.Name, new JRaw(x.Emitter), x.Data)).ToList() : null, - }; - - return new GatewayModel.CommittedTransactionInfo( - stateVersion: lt.StateVersion, - epoch: lt.Epoch, - round: lt.RoundInEpoch, - roundTimestamp: lt.RoundTimestamp.AsUtcIsoDateWithMillisString(), - transactionStatus: lt.EngineReceipt.Status.ToGatewayModel(), - affectedGlobalEntities: optIns.AffectedGlobalEntities ? lt.AffectedGlobalEntities.Select(x => entityIdToAddressMap[x]).ToList() : null, - payloadHash: payloadHash, - intentHash: intentHash, - feePaid: lt.FeePaid.ToString(), - confirmedAt: lt.RoundTimestamp, - errorMessage: lt.EngineReceipt.ErrorMessage, - rawHex: rawHex, - receipt: receipt, - message: message, - balanceChanges: optIns.BalanceChanges ? transactionBalanceChanges : null, - manifestInstructions: manifestInstructions, - manifestClasses: manifestClasses - ); - } - - public static GatewayModel.TransactionStatus ToGatewayModel(this LedgerTransactionStatus status) - { - return status switch - { - LedgerTransactionStatus.Succeeded => GatewayModel.TransactionStatus.CommittedSuccess, - LedgerTransactionStatus.Failed => GatewayModel.TransactionStatus.CommittedFailure, - _ => throw new UnreachableException($"Didn't expect {status} value"), - }; - } - public static GatewayModel.PackageVmType ToGatewayModel(this PackageVmType vmType) { return vmType switch diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Extensions.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Extensions.cs index 6e9615109..0f67110cc 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Extensions.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Extensions.cs @@ -111,6 +111,22 @@ public static TVal AddOrUpdate(this IDictionary dictiona return value; } + public static TVal Update(this IDictionary dictionary, TKey key, Action updateFactory) + where TKey : notnull + { + ArgumentNullException.ThrowIfNull(dictionary, nameof(dictionary)); + ArgumentNullException.ThrowIfNull(key, nameof(key)); + ArgumentNullException.ThrowIfNull(updateFactory, nameof(updateFactory)); + + if (!dictionary.TryGetValue(key, out var existingValue)) + { + throw new ArgumentException($"Key with index: {key} not found in dictionary."); + } + + updateFactory(existingValue); + return existingValue; + } + public static void AddRange(this IDictionary dictionary, IDictionary other) { ArgumentNullException.ThrowIfNull(dictionary, nameof(dictionary)); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/IProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/IProcessor.cs index eb54b607d..373606f38 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/IProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/IProcessor.cs @@ -1,4 +1,68 @@ -using RadixDlt.NetworkGateway.PostgresIntegration.Models; +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +using RadixDlt.NetworkGateway.PostgresIntegration.Models; using System.Collections.Generic; using System.Threading.Tasks; using CoreModel = RadixDlt.CoreApiSdk.Model; @@ -35,6 +99,16 @@ internal interface ISubstateUpsertProcessor void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion); } +internal interface ISubstateScanUpsertProcessor +{ + void OnUpsertScan(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion); +} + +internal interface ITransactionScanProcessor +{ + void OnTransactionScan(CoreModel.CommittedTransaction transaction, long stateVersion); +} + internal interface ISubstateDeleteProcessor { void VisitDelete(CoreModel.SubstateId substateId, ReferencedEntity referencedEntity, long stateVersion); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs index 282cf8b3a..2222e4afb 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/PostgresLedgerExtenderService.cs @@ -64,14 +64,16 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; -using Newtonsoft.Json; +using Microsoft.Extensions.Options; using RadixDlt.NetworkGateway.Abstractions; +using RadixDlt.NetworkGateway.Abstractions.Configuration; using RadixDlt.NetworkGateway.Abstractions.Extensions; using RadixDlt.NetworkGateway.Abstractions.Model; using RadixDlt.NetworkGateway.Abstractions.Network; using RadixDlt.NetworkGateway.DataAggregator; using RadixDlt.NetworkGateway.DataAggregator.Services; using RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors; +using RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors.AccountSecurityRules; using RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors.LedgerTransactionMarkers; using RadixDlt.NetworkGateway.PostgresIntegration.Models; using RadixDlt.NetworkGateway.PostgresIntegration.Utils; @@ -95,6 +97,7 @@ private record ExtendLedgerReport(TransactionSummary FinalTransaction, int RowsT private readonly ITopOfLedgerProvider _topOfLedgerProvider; private readonly IEnumerable _observers; private readonly IClock _clock; + private readonly IOptionsMonitor _storageOptions; public PostgresLedgerExtenderService( ILogger logger, @@ -102,7 +105,8 @@ public PostgresLedgerExtenderService( INetworkConfigurationProvider networkConfigurationProvider, IEnumerable observers, IClock clock, - ITopOfLedgerProvider topOfLedgerProvider) + ITopOfLedgerProvider topOfLedgerProvider, + IOptionsMonitor storageOptions) { _logger = logger; _dbContextFactory = dbContextFactory; @@ -110,11 +114,11 @@ public PostgresLedgerExtenderService( _observers = observers; _clock = clock; _topOfLedgerProvider = topOfLedgerProvider; + _storageOptions = storageOptions; } public async Task CommitTransactions(ConsistentLedgerExtension ledgerExtension, CancellationToken token = default) { - // Create own context for ledger extension unit of work await using var dbContext = await _dbContextFactory.CreateDbContextAsync(token); await using var tx = await dbContext.Database.BeginTransactionAsync(token); @@ -248,8 +252,8 @@ private async Task ProcessTransactions(ReadWriteDbContext db await _observers.ForEachAsync(x => x.StageCompleted(nameof(UpdatePendingTransactions), sw.Elapsed, null)); } - var processorContext = new ProcessorContext(sequences, readHelper, writeHelper, networkConfiguration, token); - var relationshipProcessor = new RelationshipProcessor(referencedEntities); + var processorContext = new ProcessorContext(sequences, _storageOptions.CurrentValue, readHelper, writeHelper, networkConfiguration, token); + var relationshipProcessor = new EntityRelationshipProcessor(referencedEntities); var manifestProcessor = new ManifestProcessor(processorContext, referencedEntities, networkConfiguration); var affectedGlobalEntitiesProcessor = new AffectedGlobalEntitiesProcessor(processorContext, referencedEntities, networkConfiguration); @@ -261,15 +265,6 @@ private async Task ProcessTransactions(ReadWriteDbContext db writeHelper, networkConfiguration); - var ledgerTransactionProcessor = new LedgerTransactionProcessor( - processorContext, - _clock, - referencedEntities, - manifestProcessor, - affectedGlobalEntitiesProcessor, - writeHelper, - ledgerExtension.LatestTransactionSummary); - // step: scan for any referenced entities { var sw = Stopwatch.StartNew(); @@ -286,6 +281,8 @@ private async Task ProcessTransactions(ReadWriteDbContext db try { + ledgerTransactionMarkersProcessor.OnTransactionScan(committedTransaction, stateVersion); + foreach (var newGlobalEntity in stateUpdates.NewGlobalEntities) { var referencedEntity = referencedEntities.GetOrAdd((EntityAddress)newGlobalEntity.EntityAddress, ea => new ReferencedEntity(ea, newGlobalEntity.EntityType, stateVersion)); @@ -419,7 +416,7 @@ private async Task ProcessTransactions(ReadWriteDbContext db }); } - relationshipProcessor.ScanUpsert(substateData, referencedEntity, stateVersion); + relationshipProcessor.OnUpsertScan(substate, referencedEntity, stateVersion); } foreach (var deletedSubstate in stateUpdates.DeletedSubstates) @@ -605,6 +602,15 @@ private async Task ProcessTransactions(ReadWriteDbContext db await _observers.ForEachAsync(x => x.StageCompleted("resolve_and_create_entities", sw.Elapsed, null)); } + var ledgerTransactionProcessor = new LedgerTransactionProcessor( + processorContext, + _clock, + referencedEntities, + manifestProcessor, + affectedGlobalEntitiesProcessor, + writeHelper, + ledgerExtension.LatestTransactionSummary); + var processors = new List { ledgerTransactionProcessor, @@ -640,6 +646,7 @@ private async Task ProcessTransactions(ReadWriteDbContext db var stateUpdates = committedTransaction.Receipt.StateUpdates; var events = committedTransaction.Receipt.Events ?? new List(); ledgerTransactionMarkersProcessor.VisitTransaction(committedTransaction, stateVersion); + ledgerTransactionProcessor.VisitTransaction(committedTransaction, stateVersion); try { @@ -738,7 +745,7 @@ private async Task ProcessTransactions(ReadWriteDbContext db var contentHandlingDuration = outerStopwatch.Elapsed - dbReadDuration - dbWriteDuration; return new ExtendLedgerReport( - ledgerTransactionProcessor.GetLastProcessedTransactionSummary(), + ledgerTransactionProcessor.GetSummaryOfLastProcessedTransaction(), rowsInserted + rowsUpdated, dbReadDuration, dbWriteDuration, diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ProcessorContext.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ProcessorContext.cs index 303e98cf1..6c1594980 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ProcessorContext.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ProcessorContext.cs @@ -62,9 +62,16 @@ * permissions under this License. */ +using RadixDlt.NetworkGateway.Abstractions.Configuration; using RadixDlt.NetworkGateway.Abstractions.Network; using System.Threading; namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; -internal record ProcessorContext(SequencesHolder Sequences, IReadHelper ReadHelper, IWriteHelper WriteHelper, NetworkConfiguration NetworkConfiguration, CancellationToken Token); +internal record ProcessorContext( + SequencesHolder Sequences, + StorageOptions StorageOptions, + IReadHelper ReadHelper, + IWriteHelper WriteHelper, + NetworkConfiguration NetworkConfiguration, + CancellationToken Token); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountAuthorizedDepositorsProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountSecurityRules/AccountAuthorizedDepositorsProcessor.cs similarity index 99% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountAuthorizedDepositorsProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountSecurityRules/AccountAuthorizedDepositorsProcessor.cs index 8a172a18c..a349b60d0 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountAuthorizedDepositorsProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountSecurityRules/AccountAuthorizedDepositorsProcessor.cs @@ -73,7 +73,7 @@ using System.Threading.Tasks; using CoreModel = RadixDlt.CoreApiSdk.Model; -namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; +namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors.AccountSecurityRules; internal class AccountAuthorizedDepositorsProcessor : IProcessorBase, ISubstateUpsertProcessor { diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountDefaultDepositRuleProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountSecurityRules/AccountDefaultDepositRuleProcessor.cs similarity index 99% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountDefaultDepositRuleProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountSecurityRules/AccountDefaultDepositRuleProcessor.cs index 5290fa2d4..25d496bd5 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountDefaultDepositRuleProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountSecurityRules/AccountDefaultDepositRuleProcessor.cs @@ -69,7 +69,7 @@ using System.Threading.Tasks; using CoreModel = RadixDlt.CoreApiSdk.Model; -namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; +namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors.AccountSecurityRules; internal class AccountDefaultDepositRuleProcessor : IProcessorBase, ISubstateUpsertProcessor { diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountResourcePreferenceRulesProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountSecurityRules/AccountResourcePreferenceRulesProcessor.cs similarity index 99% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountResourcePreferenceRulesProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountSecurityRules/AccountResourcePreferenceRulesProcessor.cs index 2092f70ad..14a6fe7ef 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountResourcePreferenceRulesProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/AccountSecurityRules/AccountResourcePreferenceRulesProcessor.cs @@ -72,7 +72,7 @@ using System.Threading.Tasks; using CoreModel = RadixDlt.CoreApiSdk.Model; -namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; +namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors.AccountSecurityRules; internal class AccountResourcePreferenceRulesProcessor : IProcessorBase, ISubstateUpsertProcessor { diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/RelationshipProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/EntityRelationshipProcessor.cs similarity index 97% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/RelationshipProcessor.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/EntityRelationshipProcessor.cs index 85ce9e134..39e2665aa 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/RelationshipProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/EntityRelationshipProcessor.cs @@ -70,17 +70,19 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; -internal class RelationshipProcessor +internal class EntityRelationshipProcessor : ISubstateScanUpsertProcessor { private readonly ReferencedEntityDictionary _referencedEntities; - public RelationshipProcessor(ReferencedEntityDictionary referencedEntities) + public EntityRelationshipProcessor(ReferencedEntityDictionary referencedEntities) { _referencedEntities = referencedEntities; } - public void ScanUpsert(CoreModel.Substate substateData, ReferencedEntity referencedEntity, long stateVersion) + public void OnUpsertScan(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion) { + var substateData = substate.Value.SubstateData; + if (substateData is CoreModel.IRoyaltyVaultHolder royaltyVaultHolder && royaltyVaultHolder.TryGetRoyaltyVault(out var rv)) { referencedEntity.PostResolveConfigure((Entity e) => diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/EventLedgerTransactionMarkerProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/EventLedgerTransactionMarkerProcessor.cs index 8cf6b68a1..9d44f9f10 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/EventLedgerTransactionMarkerProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/EventLedgerTransactionMarkerProcessor.cs @@ -1,4 +1,68 @@ -using RadixDlt.NetworkGateway.Abstractions.Network; +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +using RadixDlt.NetworkGateway.Abstractions.Network; using RadixDlt.NetworkGateway.Abstractions.Numerics; using RadixDlt.NetworkGateway.PostgresIntegration.Models; using RadixDlt.NetworkGateway.PostgresIntegration.Utils; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/GlobalEventEmitterProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/GlobalEventEmitterProcessor.cs index c8a1a57a3..c476b8395 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/GlobalEventEmitterProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/GlobalEventEmitterProcessor.cs @@ -119,9 +119,9 @@ public IEnumerable CreateTransactionMarkers() _referencedEntities.Get((EntityAddress)_networkConfiguration.WellKnownAddresses.Xrd).DatabaseId, }; - foreach (var stateVersionAffectedEntities in _globalEventEmitters.Where(x => !excludedEntityIds.Contains(x.Key))) + foreach (var stateVersionAffectedEntities in _globalEventEmitters) { - foreach (var entityId in stateVersionAffectedEntities.Value) + foreach (var entityId in stateVersionAffectedEntities.Value.Where(x => !excludedEntityIds.Contains(x))) { yield return new EventGlobalEmitterTransactionMarker { diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/LedgerTransactionMarkersProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/LedgerTransactionMarkersProcessor.cs index 387c362bc..f18dc8022 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/LedgerTransactionMarkersProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/LedgerTransactionMarkersProcessor.cs @@ -1,4 +1,68 @@ -using NpgsqlTypes; +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +using NpgsqlTypes; using RadixDlt.NetworkGateway.Abstractions.Model; using RadixDlt.NetworkGateway.Abstractions.Network; using RadixDlt.NetworkGateway.PostgresIntegration.Models; @@ -11,7 +75,7 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors.LedgerTransactionMarkers; -internal class LedgerTransactionMarkersProcessor : IProcessorBase, ISubstateUpsertProcessor, ISubstateDeleteProcessor, IDecodedEventProcessor, IEventProcessor, ITransactionProcessor +internal class LedgerTransactionMarkersProcessor : IProcessorBase, ISubstateUpsertProcessor, ISubstateDeleteProcessor, IDecodedEventProcessor, IEventProcessor, ITransactionProcessor, ITransactionScanProcessor { private readonly ProcessorContext _context; private readonly IWriteHelper _writeHelper; @@ -44,9 +108,13 @@ public Task LoadDependenciesAsync() return Task.CompletedTask; } + public void OnTransactionScan(CoreModel.CommittedTransaction transaction, long stateVersion) + { + _manifestProcessor.OnTransactionScan(transaction, stateVersion); + } + public void VisitTransaction(CoreModel.CommittedTransaction transaction, long stateVersion) { - _manifestProcessor.VisitTransaction(transaction, stateVersion); _originLedgerTransactionMarkerProcessor.VisitTransaction(transaction, stateVersion); } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/LedgerTransactionProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/LedgerTransactionProcessor.cs deleted file mode 100644 index 674446b8d..000000000 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/LedgerTransactionProcessor.cs +++ /dev/null @@ -1,254 +0,0 @@ -// -// Copyright (c) PlaceholderCompany. All rights reserved. -// - -using Newtonsoft.Json; -using NpgsqlTypes; -using RadixDlt.NetworkGateway.Abstractions; -using RadixDlt.NetworkGateway.Abstractions.Extensions; -using RadixDlt.NetworkGateway.Abstractions.Model; -using RadixDlt.NetworkGateway.DataAggregator.Services; -using RadixDlt.NetworkGateway.PostgresIntegration.Models; -using RadixDlt.NetworkGateway.PostgresIntegration.Utils; -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Threading.Tasks; -using CoreModel = RadixDlt.CoreApiSdk.Model; - -namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors.LedgerTransactionMarkers; - -internal class LedgerTransactionProcessor : IProcessorBase, ITransactionProcessor, ISubstateUpsertProcessor -{ - private readonly ProcessorContext _context; - private readonly IClock _clock; - private readonly List _ledgerTransactionsToAdd = new(); - private readonly ReferencedEntityDictionary _referencedEntities; - private readonly ManifestProcessor _manifestProcessor; - private readonly AffectedGlobalEntitiesProcessor _affectedGlobalEntitiesProcessor; - private readonly IWriteHelper _writeHelper; - - private TransactionSummary _lastProcessedTransactionSummary; - - public LedgerTransactionProcessor( - ProcessorContext context, - IClock clock, - ReferencedEntityDictionary referencedEntities, - ManifestProcessor manifestProcessor, - AffectedGlobalEntitiesProcessor affectedGlobalEntitiesProcessor, - IWriteHelper writeHelper, - TransactionSummary lastTransactionSummary) - { - _context = context; - _clock = clock; - _referencedEntities = referencedEntities; - _manifestProcessor = manifestProcessor; - _affectedGlobalEntitiesProcessor = affectedGlobalEntitiesProcessor; - _writeHelper = writeHelper; - _lastProcessedTransactionSummary = lastTransactionSummary; - } - - public void VisitTransaction(CoreModel.CommittedTransaction committedTransaction, long stateVersion) - { - var events = committedTransaction.Receipt.Events ?? new List(); - - long? epochUpdate = null; - long? roundInEpochUpdate = null; - DateTime? roundTimestampUpdate = null; - - if (committedTransaction.LedgerTransaction is CoreModel.RoundUpdateLedgerTransaction roundUpdateTransaction) - { - epochUpdate = _lastProcessedTransactionSummary.Epoch != roundUpdateTransaction.RoundUpdateTransaction.Epoch ? roundUpdateTransaction.RoundUpdateTransaction.Epoch : null; - roundInEpochUpdate = roundUpdateTransaction.RoundUpdateTransaction.RoundInEpoch; - roundTimestampUpdate = DateTimeOffset.FromUnixTimeMilliseconds(roundUpdateTransaction.RoundUpdateTransaction.ProposerTimestamp.UnixTimestampMs).UtcDateTime; - } - - var isStartOfEpoch = epochUpdate.HasValue; - var isStartOfRound = roundInEpochUpdate.HasValue; - var createdTimestamp = _clock.UtcNow; - - var roundTimestamp = roundTimestampUpdate ?? _lastProcessedTransactionSummary.RoundTimestamp; - var normalizedRoundTimestamp = - roundTimestamp < _lastProcessedTransactionSummary.NormalizedRoundTimestamp ? _lastProcessedTransactionSummary.NormalizedRoundTimestamp - : roundTimestamp > createdTimestamp ? createdTimestamp - : roundTimestamp; - - var epoch = epochUpdate ?? _lastProcessedTransactionSummary.Epoch; - var roundInEpoch = roundInEpochUpdate ?? _lastProcessedTransactionSummary.RoundInEpoch; - var indexInEpoch = isStartOfEpoch ? 0 : _lastProcessedTransactionSummary.IndexInEpoch + 1; - var indexInRound = isStartOfRound ? 0 : _lastProcessedTransactionSummary.IndexInRound + 1; - - LedgerTransaction ledgerTransaction = committedTransaction.LedgerTransaction switch - { - CoreModel.GenesisLedgerTransaction => new GenesisLedgerTransaction(), - CoreModel.UserLedgerTransaction ult => new UserLedgerTransaction - { - PayloadHash = ult.NotarizedTransaction.HashBech32m, - IntentHash = ult.NotarizedTransaction.SignedIntent.Intent.HashBech32m, - SignedIntentHash = ult.NotarizedTransaction.SignedIntent.HashBech32m, - Message = ult.NotarizedTransaction.SignedIntent.Intent.Message?.ToJson(), - RawPayload = ult.NotarizedTransaction.GetPayloadBytes(), - ManifestInstructions = ult.NotarizedTransaction.SignedIntent.Intent.Instructions, - ManifestClasses = _manifestProcessor.GetManifestClasses(stateVersion), - }, - CoreModel.RoundUpdateLedgerTransaction => new RoundUpdateLedgerTransaction(), - CoreModel.FlashLedgerTransaction => new FlashLedgerTransaction(), - _ => throw new UnreachableException($"Unsupported transaction type: {committedTransaction.LedgerTransaction.GetType()}"), - }; - - - ledgerTransaction.StateVersion = stateVersion; - ledgerTransaction.TransactionTreeHash = committedTransaction.ResultantStateIdentifiers.TransactionTreeHash; - ledgerTransaction.ReceiptTreeHash = committedTransaction.ResultantStateIdentifiers.ReceiptTreeHash; - ledgerTransaction.StateTreeHash = committedTransaction.ResultantStateIdentifiers.StateTreeHash; - ledgerTransaction.Epoch = epoch; - ledgerTransaction.RoundInEpoch = roundInEpoch; - ledgerTransaction.IndexInEpoch = indexInEpoch; - ledgerTransaction.IndexInRound = indexInRound; - ledgerTransaction.FeePaid = committedTransaction.Receipt.FeeSummary.TotalFee(); - ledgerTransaction.TipPaid = committedTransaction.Receipt.FeeSummary.TotalTip(); - ledgerTransaction.RoundTimestamp = roundTimestamp; - ledgerTransaction.CreatedTimestamp = createdTimestamp; - ledgerTransaction.NormalizedRoundTimestamp = normalizedRoundTimestamp; - ledgerTransaction.ReceiptStateUpdates = committedTransaction.Receipt.StateUpdates.ToJson(); - ledgerTransaction.ReceiptStatus = committedTransaction.Receipt.Status.ToModel(); - ledgerTransaction.ReceiptFeeSummary = committedTransaction.Receipt.FeeSummary.ToJson(); - ledgerTransaction.ReceiptErrorMessage = committedTransaction.Receipt.ErrorMessage; - ledgerTransaction.ReceiptOutput = committedTransaction.Receipt.Output != null ? JsonConvert.SerializeObject(committedTransaction.Receipt.Output) : null; - ledgerTransaction.ReceiptNextEpoch = committedTransaction.Receipt.NextEpoch?.ToJson(); - ledgerTransaction.ReceiptCostingParameters = committedTransaction.Receipt.CostingParameters.ToJson(); - ledgerTransaction.ReceiptFeeSource = committedTransaction.Receipt.FeeSource?.ToJson(); - ledgerTransaction.ReceiptFeeDestination = committedTransaction.Receipt.FeeDestination?.ToJson(); - ledgerTransaction.BalanceChanges = committedTransaction.BalanceChanges?.ToJson(); - ledgerTransaction.AffectedGlobalEntities = _affectedGlobalEntitiesProcessor.GetAllAffectedGlobalEntities(stateVersion).ToArray(); - ledgerTransaction.ReceiptEventEmitters = events.Select(e => e.Type.Emitter.ToJson()).ToArray(); - ledgerTransaction.ReceiptEventNames = events.Select(e => e.Type.Name).ToArray(); - ledgerTransaction.ReceiptEventSbors = events.Select(e => e.Data.GetDataBytes()).ToArray(); - ledgerTransaction.ReceiptEventSchemaEntityIds = events.Select(e => _referencedEntities.Get((EntityAddress)e.Type.TypeReference.FullTypeId.EntityAddress).DatabaseId).ToArray(); - ledgerTransaction.ReceiptEventSchemaHashes = events.Select(e => e.Type.TypeReference.FullTypeId.SchemaHash.ConvertFromHex()).ToArray(); - ledgerTransaction.ReceiptEventTypeIndexes = events.Select(e => e.Type.TypeReference.FullTypeId.LocalTypeId.Id).ToArray(); - ledgerTransaction.ReceiptEventSborTypeKinds = events.Select(e => e.Type.TypeReference.FullTypeId.LocalTypeId.Kind.ToModel()).ToArray(); - - _ledgerTransactionsToAdd.Add(ledgerTransaction); - - _lastProcessedTransactionSummary = new TransactionSummary( - StateVersion: stateVersion, - RoundTimestamp: roundTimestamp, - NormalizedRoundTimestamp: normalizedRoundTimestamp, - // TODO PP: i'm not sure it's correct. - TransactionTreeHash: _lastProcessedTransactionSummary.TransactionTreeHash, - ReceiptTreeHash: _lastProcessedTransactionSummary.ReceiptTreeHash, - StateTreeHash: _lastProcessedTransactionSummary.StateTreeHash, - CreatedTimestamp: createdTimestamp, - Epoch: epoch, - RoundInEpoch: roundInEpoch, - IndexInEpoch: indexInEpoch, - IndexInRound: indexInRound); - } - - public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion) - { - var substateData = substate.Value.SubstateData; - - if (substateData is CoreModel.ConsensusManagerFieldCurrentTimeSubstate currentTime) - { - // TODO PP: we have to update it here. - // TODO PP: that never worked? - roundTimestampUpdate = DateTimeOffset.FromUnixTimeMilliseconds(currentTime.Value.ProposerTimestamp.UnixTimestampMs).UtcDateTime; - } - } - - public Task LoadDependenciesAsync() - { - return Task.CompletedTask; - } - - public void ProcessChanges() - { - } - - public TransactionSummary GetLastProcessedTransactionSummary() - { - return _lastProcessedTransactionSummary; - } - - public async Task SaveEntitiesAsync() - { - var rowsInserted = 0; - - rowsInserted += await CopyLedgerTransactionMarkers(); - - return rowsInserted; - } - - private Task CopyLedgerTransactionMarkers() => _context.WriteHelper.Copy( - _ledgerTransactionsToAdd, - "COPY ledger_transactions (state_version, transaction_tree_hash, receipt_tree_hash, state_tree_hash, epoch, round_in_epoch, index_in_epoch, index_in_round, fee_paid, tip_paid, affected_global_entities, round_timestamp, created_timestamp, normalized_round_timestamp, balance_changes, receipt_state_updates, receipt_status, receipt_fee_summary, receipt_fee_source, receipt_fee_destination, receipt_costing_parameters, receipt_error_message, receipt_output, receipt_next_epoch, receipt_event_emitters, receipt_event_names, receipt_event_sbors, receipt_event_schema_entity_ids, receipt_event_schema_hashes, receipt_event_type_indexes, receipt_event_sbor_type_kinds, discriminator, payload_hash, intent_hash, signed_intent_hash, message, raw_payload, manifest_instructions, manifest_classes) FROM STDIN (FORMAT BINARY)", - async (writer, lt, token) => - { - var discriminator = _writeHelper.GetDiscriminator(lt.GetType()); - - await _writeHelper.HandleMaxAggregateCounts(lt); - await writer.StartRowAsync(token); - await writer.WriteAsync(lt.StateVersion, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(lt.TransactionTreeHash, NpgsqlDbType.Text, token); - await writer.WriteAsync(lt.ReceiptTreeHash, NpgsqlDbType.Text, token); - await writer.WriteAsync(lt.StateTreeHash, NpgsqlDbType.Text, token); - await writer.WriteAsync(lt.Epoch, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(lt.RoundInEpoch, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(lt.IndexInEpoch, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(lt.IndexInRound, NpgsqlDbType.Bigint, token); - await writer.WriteAsync(lt.FeePaid.GetSubUnitsSafeForPostgres(), NpgsqlDbType.Numeric, token); - await writer.WriteAsync(lt.TipPaid.GetSubUnitsSafeForPostgres(), NpgsqlDbType.Numeric, token); - await writer.WriteAsync(lt.AffectedGlobalEntities, NpgsqlDbType.Array | NpgsqlDbType.Bigint, token); - await writer.WriteAsync(lt.RoundTimestamp, NpgsqlDbType.TimestampTz, token); - await writer.WriteAsync(lt.CreatedTimestamp, NpgsqlDbType.TimestampTz, token); - await writer.WriteAsync(lt.NormalizedRoundTimestamp, NpgsqlDbType.TimestampTz, token); - await writer.WriteAsync(lt.BalanceChanges, NpgsqlDbType.Jsonb, token); - - await writer.WriteAsync(lt.EngineReceipt.StateUpdates, NpgsqlDbType.Jsonb, token); - await writer.WriteAsync(lt.EngineReceipt.Status, "ledger_transaction_status", token); - await writer.WriteAsync(lt.EngineReceipt.FeeSummary, NpgsqlDbType.Jsonb, token); - await writer.WriteAsync(lt.EngineReceipt.FeeSource, NpgsqlDbType.Jsonb, token); - await writer.WriteAsync(lt.EngineReceipt.FeeDestination, NpgsqlDbType.Jsonb, token); - await writer.WriteAsync(lt.EngineReceipt.CostingParameters, NpgsqlDbType.Jsonb, token); - await writer.WriteAsync(lt.EngineReceipt.ErrorMessage, NpgsqlDbType.Text, token); - await writer.WriteAsync(lt.EngineReceipt.Output, NpgsqlDbType.Jsonb, token); - await writer.WriteAsync(lt.EngineReceipt.NextEpoch, NpgsqlDbType.Jsonb, token); - await writer.WriteAsync(lt.EngineReceipt.Events.Emitters, NpgsqlDbType.Array | NpgsqlDbType.Jsonb, token); - await writer.WriteAsync(lt.EngineReceipt.Events.Names, NpgsqlDbType.Array | NpgsqlDbType.Text, token); - await writer.WriteAsync(lt.EngineReceipt.Events.Sbors, NpgsqlDbType.Array | NpgsqlDbType.Bytea, token); - await writer.WriteAsync(lt.EngineReceipt.Events.SchemaEntityIds, NpgsqlDbType.Array | NpgsqlDbType.Bigint, token); - await writer.WriteAsync(lt.EngineReceipt.Events.SchemaHashes, NpgsqlDbType.Array | NpgsqlDbType.Bytea, token); - await writer.WriteAsync(lt.EngineReceipt.Events.TypeIndexes, NpgsqlDbType.Array | NpgsqlDbType.Bigint, token); - await writer.WriteAsync(lt.EngineReceipt.Events.SborTypeKinds, "sbor_type_kind[]", token); - await writer.WriteAsync(discriminator, "ledger_transaction_type", token); - - switch (lt) - { - case GenesisLedgerTransaction: - case RoundUpdateLedgerTransaction: - case FlashLedgerTransaction: - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - await writer.WriteNullAsync(token); - break; - case UserLedgerTransaction ult: - await writer.WriteAsync(ult.PayloadHash, NpgsqlDbType.Text, token); - await writer.WriteAsync(ult.IntentHash, NpgsqlDbType.Text, token); - await writer.WriteAsync(ult.SignedIntentHash, NpgsqlDbType.Text, token); - await writer.WriteAsync(ult.Message, NpgsqlDbType.Jsonb, token); - await writer.WriteAsync(ult.RawPayload, NpgsqlDbType.Bytea, token); - await writer.WriteAsync(ult.ManifestInstructions, NpgsqlDbType.Text, token); - await writer.WriteAsync(ult.ManifestClasses, "ledger_transaction_manifest_class[]", token); - break; - default: - throw new ArgumentOutOfRangeException(nameof(lt), lt, null); - } - }); -} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/ManifestProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/ManifestProcessor.cs index b12552c95..ffd596348 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/ManifestProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/ManifestProcessor.cs @@ -1,4 +1,68 @@ -using RadixDlt.NetworkGateway.Abstractions.Extensions; +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +using RadixDlt.NetworkGateway.Abstractions.Extensions; using RadixDlt.NetworkGateway.Abstractions.Model; using RadixDlt.NetworkGateway.Abstractions.Network; using RadixDlt.NetworkGateway.PostgresIntegration.Models; @@ -11,7 +75,7 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors.LedgerTransactionMarkers; -internal class ManifestProcessor : ITransactionMarkerProcessor +internal class ManifestProcessor : ITransactionMarkerProcessor, ITransactionScanProcessor { private readonly ProcessorContext _context; private readonly ReferencedEntityDictionary _referencedEntities; @@ -28,7 +92,7 @@ public ManifestProcessor(ProcessorContext context, ReferencedEntityDictionary re _networkConfiguration = networkConfiguration; } - public void VisitTransaction(CoreModel.CommittedTransaction transaction, long stateVersion) + public void OnTransactionScan(CoreModel.CommittedTransaction transaction, long stateVersion) { if (transaction.LedgerTransaction is CoreModel.UserLedgerTransaction userLedgerTransaction) { @@ -165,7 +229,7 @@ private void AnalyzeAddresses(long stateVersion) EntityId = re.DatabaseId, }); } + } } } } -} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/OriginLedgerTransactionMarkerProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/OriginLedgerTransactionMarkerProcessor.cs index 88e5ce55e..f8fee3156 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/OriginLedgerTransactionMarkerProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionMarkers/OriginLedgerTransactionMarkerProcessor.cs @@ -1,4 +1,68 @@ -using RadixDlt.NetworkGateway.Abstractions.Network; +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +using RadixDlt.NetworkGateway.Abstractions.Network; using RadixDlt.NetworkGateway.PostgresIntegration.Models; using System.Collections.Generic; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionProcessor.cs new file mode 100644 index 000000000..160542acc --- /dev/null +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/LedgerTransactionProcessor.cs @@ -0,0 +1,365 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +using Newtonsoft.Json; +using NpgsqlTypes; +using RadixDlt.NetworkGateway.Abstractions; +using RadixDlt.NetworkGateway.Abstractions.Configuration; +using RadixDlt.NetworkGateway.Abstractions.Extensions; +using RadixDlt.NetworkGateway.Abstractions.Model; +using RadixDlt.NetworkGateway.DataAggregator.Services; +using RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors.LedgerTransactionMarkers; +using RadixDlt.NetworkGateway.PostgresIntegration.Models; +using RadixDlt.NetworkGateway.PostgresIntegration.Utils; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using CoreModel = RadixDlt.CoreApiSdk.Model; + +namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors; + +internal class LedgerTransactionProcessor : IProcessorBase, ITransactionProcessor, ISubstateUpsertProcessor +{ + private record TransactionData( + long StateVersion, + DateTime CreatedAt, + CoreModel.CommittedTransaction RawCommittedTransaction + ) + { + public long? NewEpochIndex { get; set; } + + public long? NewRoundIndex { get; set; } + + public DateTime? RoundTimestampUpdate { get; set; } + } + + private readonly ProcessorContext _context; + private readonly IClock _clock; + private readonly List _ledgerTransactionsToAdd = new(); + private readonly List _ledgerTransactionEventsToAdd = new(); + private readonly ReferencedEntityDictionary _referencedEntities; + private readonly ManifestProcessor _manifestProcessor; + private readonly AffectedGlobalEntitiesProcessor _affectedGlobalEntitiesProcessor; + private readonly IWriteHelper _writeHelper; + + private readonly Dictionary _transactionData = new(); + private TransactionSummary _lastProcessedTransactionSummary; + + public LedgerTransactionProcessor( + ProcessorContext context, + IClock clock, + ReferencedEntityDictionary referencedEntities, + ManifestProcessor manifestProcessor, + AffectedGlobalEntitiesProcessor affectedGlobalEntitiesProcessor, + IWriteHelper writeHelper, + TransactionSummary lastTransactionSummary) + { + _context = context; + _clock = clock; + _referencedEntities = referencedEntities; + _manifestProcessor = manifestProcessor; + _affectedGlobalEntitiesProcessor = affectedGlobalEntitiesProcessor; + _writeHelper = writeHelper; + _lastProcessedTransactionSummary = lastTransactionSummary; + } + + public void VisitTransaction(CoreModel.CommittedTransaction committedTransaction, long stateVersion) + { + _transactionData.Add(stateVersion, new TransactionData(stateVersion, _clock.UtcNow, committedTransaction)); + + if (committedTransaction.LedgerTransaction is CoreModel.RoundUpdateLedgerTransaction roundUpdateTransaction) + { + var newEpochIndex = _lastProcessedTransactionSummary.Epoch != roundUpdateTransaction.RoundUpdateTransaction.Epoch ? roundUpdateTransaction.RoundUpdateTransaction.Epoch : (long?)null; + + _transactionData.Update( + stateVersion, + existing => + { + existing.NewEpochIndex = newEpochIndex; + existing.NewRoundIndex = roundUpdateTransaction.RoundUpdateTransaction.RoundInEpoch; + existing.RoundTimestampUpdate = DateTimeOffset.FromUnixTimeMilliseconds(roundUpdateTransaction.RoundUpdateTransaction.ProposerTimestamp.UnixTimestampMs).UtcDateTime; + }); + } + } + + public void VisitUpsert(CoreModel.IUpsertedSubstate substate, ReferencedEntity referencedEntity, long stateVersion) + { + var substateData = substate.Value.SubstateData; + + if (substateData is CoreModel.ConsensusManagerFieldCurrentTimeSubstate currentTime) + { + _transactionData.Update( + stateVersion, + existing => + { + existing.RoundTimestampUpdate = DateTimeOffset.FromUnixTimeMilliseconds(currentTime.Value.ProposerTimestamp.UnixTimestampMs).UtcDateTime; + }); + } + } + + public Task LoadDependenciesAsync() + { + return Task.CompletedTask; + } + + public void ProcessChanges() + { + foreach (var data in _transactionData.Values) + { + var stateVersion = data.StateVersion; + var committedTransaction = data.RawCommittedTransaction; + + var roundTimestamp = data.RoundTimestampUpdate ?? _lastProcessedTransactionSummary.RoundTimestamp; + + var normalizedRoundTimestamp = + roundTimestamp < _lastProcessedTransactionSummary.NormalizedRoundTimestamp ? _lastProcessedTransactionSummary.NormalizedRoundTimestamp + : roundTimestamp > data.CreatedAt ? data.CreatedAt + : roundTimestamp; + + var epoch = data.NewEpochIndex ?? _lastProcessedTransactionSummary.Epoch; + var roundInEpoch = data.NewRoundIndex ?? _lastProcessedTransactionSummary.RoundInEpoch; + var indexInEpoch = data.NewEpochIndex.HasValue ? 0 : _lastProcessedTransactionSummary.IndexInEpoch + 1; + var indexInRound = data.NewEpochIndex.HasValue ? 0 : _lastProcessedTransactionSummary.IndexInRound + 1; + + LedgerTransaction ledgerTransaction = committedTransaction.LedgerTransaction switch + { + CoreModel.GenesisLedgerTransaction => new GenesisLedgerTransaction(), + CoreModel.UserLedgerTransaction ult => new UserLedgerTransaction + { + PayloadHash = ult.NotarizedTransaction.HashBech32m, + IntentHash = ult.NotarizedTransaction.SignedIntent.Intent.HashBech32m, + SignedIntentHash = ult.NotarizedTransaction.SignedIntent.HashBech32m, + Message = ult.NotarizedTransaction.SignedIntent.Intent.Message?.ToJson(), + RawPayload = ult.NotarizedTransaction.GetPayloadBytes(), + ManifestInstructions = ult.NotarizedTransaction.SignedIntent.Intent.Instructions, + ManifestClasses = _manifestProcessor.GetManifestClasses(stateVersion), + }, + CoreModel.RoundUpdateLedgerTransaction => new RoundUpdateLedgerTransaction(), + CoreModel.FlashLedgerTransaction => new FlashLedgerTransaction(), + _ => throw new UnreachableException($"Unsupported transaction type: {committedTransaction.LedgerTransaction.GetType()}"), + }; + + ledgerTransaction.Epoch = epoch; + ledgerTransaction.RoundInEpoch = roundInEpoch; + ledgerTransaction.IndexInEpoch = indexInEpoch; + ledgerTransaction.IndexInRound = indexInRound; + ledgerTransaction.RoundTimestamp = roundTimestamp; + ledgerTransaction.CreatedTimestamp = data.CreatedAt; + ledgerTransaction.NormalizedRoundTimestamp = normalizedRoundTimestamp; + ledgerTransaction.StateVersion = stateVersion; + ledgerTransaction.TransactionTreeHash = committedTransaction.ResultantStateIdentifiers.TransactionTreeHash; + ledgerTransaction.ReceiptTreeHash = committedTransaction.ResultantStateIdentifiers.ReceiptTreeHash; + ledgerTransaction.StateTreeHash = committedTransaction.ResultantStateIdentifiers.StateTreeHash; + ledgerTransaction.FeePaid = committedTransaction.Receipt.FeeSummary.TotalFee(); + ledgerTransaction.TipPaid = committedTransaction.Receipt.FeeSummary.TotalTip(); + ledgerTransaction.ReceiptStatus = committedTransaction.Receipt.Status.ToModel(); + ledgerTransaction.ReceiptFeeSource = committedTransaction.Receipt.FeeSource?.ToJson(); + ledgerTransaction.ReceiptFeeDestination = committedTransaction.Receipt.FeeDestination?.ToJson(); + ledgerTransaction.ReceiptErrorMessage = committedTransaction.Receipt.ErrorMessage; + ledgerTransaction.ReceiptFeeSummary = committedTransaction.Receipt.FeeSummary.ToJson(); + ledgerTransaction.ReceiptOutput = committedTransaction.Receipt.Output != null ? JsonConvert.SerializeObject(committedTransaction.Receipt.Output) : null; + ledgerTransaction.ReceiptCostingParameters = committedTransaction.Receipt.CostingParameters.ToJson(); + ledgerTransaction.ReceiptNextEpoch = committedTransaction.Receipt.NextEpoch?.ToJson(); + ledgerTransaction.BalanceChanges = committedTransaction.BalanceChanges?.ToJson(); + ledgerTransaction.AffectedGlobalEntities = _affectedGlobalEntitiesProcessor.GetAllAffectedGlobalEntities(stateVersion).ToArray(); + + _ledgerTransactionsToAdd.Add(ledgerTransaction); + + var isUserTransaction = committedTransaction.LedgerTransaction is CoreModel.UserLedgerTransaction; + var isUserTransactionOrEpochChange = committedTransaction.LedgerTransaction is CoreModel.UserLedgerTransaction || data.NewEpochIndex.HasValue; + + if (_context.StorageOptions.StoreReceiptStateUpdates == LedgerTransactionStorageOption.StoreForAllTransactions || + (_context.StorageOptions.StoreReceiptStateUpdates == LedgerTransactionStorageOption.StoreOnlyForUserTransactions && isUserTransaction) || + (_context.StorageOptions.StoreReceiptStateUpdates == LedgerTransactionStorageOption.StoryOnlyForUserTransactionsAndEpochChanges && isUserTransactionOrEpochChange) + ) + { + ledgerTransaction.ReceiptStateUpdates = committedTransaction.Receipt.StateUpdates.ToJson(); + } + + if (_context.StorageOptions.StoreTransactionReceiptEvents == LedgerTransactionStorageOption.StoreForAllTransactions || + (_context.StorageOptions.StoreTransactionReceiptEvents == LedgerTransactionStorageOption.StoreOnlyForUserTransactions && isUserTransaction) || + (_context.StorageOptions.StoreTransactionReceiptEvents == LedgerTransactionStorageOption.StoryOnlyForUserTransactionsAndEpochChanges && isUserTransactionOrEpochChange) + ) + { + var events = committedTransaction.Receipt.Events ?? new List(); + + _ledgerTransactionEventsToAdd.Add( + new LedgerTransactionEvents + { + StateVersion = stateVersion, + ReceiptEventEmitters = events.Select(e => e.Type.Emitter.ToJson()).ToArray(), + ReceiptEventNames = events.Select(e => e.Type.Name).ToArray(), + ReceiptEventSbors = events.Select(e => e.Data.GetDataBytes()).ToArray(), + ReceiptEventSchemaEntityIds = events.Select(e => _referencedEntities.Get((EntityAddress)e.Type.TypeReference.FullTypeId.EntityAddress).DatabaseId).ToArray(), + ReceiptEventSchemaHashes = events.Select(e => e.Type.TypeReference.FullTypeId.SchemaHash.ConvertFromHex()).ToArray(), + ReceiptEventTypeIndexes = events.Select(e => e.Type.TypeReference.FullTypeId.LocalTypeId.Id).ToArray(), + ReceiptEventSborTypeKinds = events.Select(e => e.Type.TypeReference.FullTypeId.LocalTypeId.Kind.ToModel()).ToArray(), + }); + } + + _lastProcessedTransactionSummary = new TransactionSummary( + StateVersion: stateVersion, + RoundTimestamp: roundTimestamp, + NormalizedRoundTimestamp: normalizedRoundTimestamp, + CreatedTimestamp: data.CreatedAt, + Epoch: epoch, + RoundInEpoch: roundInEpoch, + IndexInEpoch: indexInEpoch, + IndexInRound: indexInRound); + } + } + + public TransactionSummary GetSummaryOfLastProcessedTransaction() + { + return _lastProcessedTransactionSummary; + } + + public async Task SaveEntitiesAsync() + { + var rowsInserted = 0; + + rowsInserted += await CopyLedgerTransactions(); + rowsInserted += await CopyLedgerTransactionEvents(); + + return rowsInserted; + } + + private Task CopyLedgerTransactions() => _context.WriteHelper.Copy( + _ledgerTransactionsToAdd, + "COPY ledger_transactions (state_version, transaction_tree_hash, receipt_tree_hash, state_tree_hash, epoch, round_in_epoch, index_in_epoch, index_in_round, fee_paid, tip_paid, affected_global_entities, round_timestamp, created_timestamp, normalized_round_timestamp, balance_changes, receipt_state_updates, receipt_status, receipt_fee_summary, receipt_fee_source, receipt_fee_destination, receipt_costing_parameters, receipt_error_message, receipt_output, receipt_next_epoch, discriminator, payload_hash, intent_hash, signed_intent_hash, message, raw_payload, manifest_instructions, manifest_classes) FROM STDIN (FORMAT BINARY)", + async (writer, lt, token) => + { + var discriminator = _writeHelper.GetDiscriminator(lt.GetType()); + + await writer.WriteAsync(lt.StateVersion, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(lt.TransactionTreeHash, NpgsqlDbType.Text, token); + await writer.WriteAsync(lt.ReceiptTreeHash, NpgsqlDbType.Text, token); + await writer.WriteAsync(lt.StateTreeHash, NpgsqlDbType.Text, token); + await writer.WriteAsync(lt.Epoch, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(lt.RoundInEpoch, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(lt.IndexInEpoch, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(lt.IndexInRound, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(lt.FeePaid.GetSubUnitsSafeForPostgres(), NpgsqlDbType.Numeric, token); + await writer.WriteAsync(lt.TipPaid.GetSubUnitsSafeForPostgres(), NpgsqlDbType.Numeric, token); + await writer.WriteAsync(lt.AffectedGlobalEntities, NpgsqlDbType.Array | NpgsqlDbType.Bigint, token); + await writer.WriteAsync(lt.RoundTimestamp, NpgsqlDbType.TimestampTz, token); + await writer.WriteAsync(lt.CreatedTimestamp, NpgsqlDbType.TimestampTz, token); + await writer.WriteAsync(lt.NormalizedRoundTimestamp, NpgsqlDbType.TimestampTz, token); + await writer.WriteAsync(lt.BalanceChanges, NpgsqlDbType.Jsonb, token); + + await writer.WriteAsync(lt.EngineReceipt.StateUpdates, NpgsqlDbType.Jsonb, token); + await writer.WriteAsync(lt.EngineReceipt.Status, "ledger_transaction_status", token); + await writer.WriteAsync(lt.EngineReceipt.FeeSummary, NpgsqlDbType.Jsonb, token); + await writer.WriteAsync(lt.EngineReceipt.FeeSource, NpgsqlDbType.Jsonb, token); + await writer.WriteAsync(lt.EngineReceipt.FeeDestination, NpgsqlDbType.Jsonb, token); + await writer.WriteAsync(lt.EngineReceipt.CostingParameters, NpgsqlDbType.Jsonb, token); + await writer.WriteAsync(lt.EngineReceipt.ErrorMessage, NpgsqlDbType.Text, token); + await writer.WriteAsync(lt.EngineReceipt.Output, NpgsqlDbType.Jsonb, token); + await writer.WriteAsync(lt.EngineReceipt.NextEpoch, NpgsqlDbType.Jsonb, token); + await writer.WriteAsync(discriminator, "ledger_transaction_type", token); + + switch (lt) + { + case GenesisLedgerTransaction: + case RoundUpdateLedgerTransaction: + case FlashLedgerTransaction: + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + await writer.WriteNullAsync(token); + break; + case UserLedgerTransaction ult: + await writer.WriteAsync(ult.PayloadHash, NpgsqlDbType.Text, token); + await writer.WriteAsync(ult.IntentHash, NpgsqlDbType.Text, token); + await writer.WriteAsync(ult.SignedIntentHash, NpgsqlDbType.Text, token); + await writer.WriteAsync(ult.Message, NpgsqlDbType.Jsonb, token); + await writer.WriteAsync(ult.RawPayload, NpgsqlDbType.Bytea, token); + await writer.WriteAsync(ult.ManifestInstructions, NpgsqlDbType.Text, token); + await writer.WriteAsync(ult.ManifestClasses, "ledger_transaction_manifest_class[]", token); + break; + default: + throw new ArgumentOutOfRangeException(nameof(lt), lt, null); + } + }); + + private Task CopyLedgerTransactionEvents() => _context.WriteHelper.Copy( + _ledgerTransactionEventsToAdd, + "COPY ledger_transaction_events (state_version, receipt_event_emitters, receipt_event_names, receipt_event_sbors, receipt_event_schema_entity_ids, receipt_event_schema_hashes, receipt_event_type_indexes, receipt_event_sbor_type_kinds) FROM STDIN (FORMAT BINARY)", + async (writer, lt, token) => + { + await writer.WriteAsync(lt.StateVersion, NpgsqlDbType.Bigint, token); + await writer.WriteAsync(lt.ReceiptEventEmitters, NpgsqlDbType.Array | NpgsqlDbType.Jsonb, token); + await writer.WriteAsync(lt.ReceiptEventNames, NpgsqlDbType.Array | NpgsqlDbType.Text, token); + await writer.WriteAsync(lt.ReceiptEventSbors, NpgsqlDbType.Array | NpgsqlDbType.Bytea, token); + await writer.WriteAsync(lt.ReceiptEventSchemaEntityIds, NpgsqlDbType.Array | NpgsqlDbType.Bigint, token); + await writer.WriteAsync(lt.ReceiptEventSchemaHashes, NpgsqlDbType.Array | NpgsqlDbType.Bytea, token); + await writer.WriteAsync(lt.ReceiptEventTypeIndexes, NpgsqlDbType.Array | NpgsqlDbType.Bigint, token); + await writer.WriteAsync(lt.ReceiptEventSborTypeKinds, "sbor_type_kind[]", token); + }); +} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/VaultProcessor.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/VaultProcessor.cs index 58abdf297..874cb8967 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/VaultProcessor.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/Processors/VaultProcessor.cs @@ -74,7 +74,7 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; -internal class VaultProcessor : IProcessorBase, ISubstateUpsertProcessor +internal class VaultProcessor : IProcessorBase, ISubstateUpsertProcessor, ISubstateDeleteProcessor { private readonly record struct NonFungibleIdDefinitionDbLookup(long NonFungibleResourceEntityId, string NonFungibleId); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20241001130214_InitialCreate.Designer.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240925082442_InitialCreate.Designer.cs similarity index 99% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20241001130214_InitialCreate.Designer.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240925082442_InitialCreate.Designer.cs index 4ec14acff..5561681ce 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20241001130214_InitialCreate.Designer.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240925082442_InitialCreate.Designer.cs @@ -81,7 +81,11 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.Migrations { [DbContext(typeof(MigrationsDbContext))] +<<<<<<<< HEAD:src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20241001130214_InitialCreate.Designer.cs [Migration("20241001130214_InitialCreate")] +======== + [Migration("20240925082442_InitialCreate")] +>>>>>>>> a54c65ae (add config parameters to allow reducing ledger transactions table size.):src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240925082442_InitialCreate.Designer.cs partial class InitialCreate { /// @@ -1076,41 +1080,6 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("text") .HasColumnName("receipt_error_message"); - b.Property("ReceiptEventEmitters") - .IsRequired() - .HasColumnType("jsonb[]") - .HasColumnName("receipt_event_emitters"); - - b.Property("ReceiptEventNames") - .IsRequired() - .HasColumnType("text[]") - .HasColumnName("receipt_event_names"); - - b.Property("ReceiptEventSborTypeKinds") - .IsRequired() - .HasColumnType("sbor_type_kind[]") - .HasColumnName("receipt_event_sbor_type_kinds"); - - b.Property("ReceiptEventSbors") - .IsRequired() - .HasColumnType("bytea[]") - .HasColumnName("receipt_event_sbors"); - - b.Property("ReceiptEventSchemaEntityIds") - .IsRequired() - .HasColumnType("bigint[]") - .HasColumnName("receipt_event_schema_entity_ids"); - - b.Property("ReceiptEventSchemaHashes") - .IsRequired() - .HasColumnType("bytea[]") - .HasColumnName("receipt_event_schema_hashes"); - - b.Property("ReceiptEventTypeIndexes") - .IsRequired() - .HasColumnType("bigint[]") - .HasColumnName("receipt_event_type_indexes"); - b.Property("ReceiptFeeDestination") .HasColumnType("jsonb") .HasColumnName("receipt_fee_destination"); @@ -1133,7 +1102,6 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnName("receipt_output"); b.Property("ReceiptStateUpdates") - .IsRequired() .HasColumnType("jsonb") .HasColumnName("receipt_state_updates"); @@ -1187,6 +1155,52 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.UseTphMappingStrategy(); }); + modelBuilder.Entity("RadixDlt.NetworkGateway.PostgresIntegration.Models.LedgerTransactionEvents", b => + { + b.Property("StateVersion") + .HasColumnType("bigint") + .HasColumnName("state_version"); + + b.Property("ReceiptEventEmitters") + .IsRequired() + .HasColumnType("jsonb[]") + .HasColumnName("receipt_event_emitters"); + + b.Property("ReceiptEventNames") + .IsRequired() + .HasColumnType("text[]") + .HasColumnName("receipt_event_names"); + + b.Property("ReceiptEventSborTypeKinds") + .IsRequired() + .HasColumnType("sbor_type_kind[]") + .HasColumnName("receipt_event_sbor_type_kinds"); + + b.Property("ReceiptEventSbors") + .IsRequired() + .HasColumnType("bytea[]") + .HasColumnName("receipt_event_sbors"); + + b.Property("ReceiptEventSchemaEntityIds") + .IsRequired() + .HasColumnType("bigint[]") + .HasColumnName("receipt_event_schema_entity_ids"); + + b.Property("ReceiptEventSchemaHashes") + .IsRequired() + .HasColumnType("bytea[]") + .HasColumnName("receipt_event_schema_hashes"); + + b.Property("ReceiptEventTypeIndexes") + .IsRequired() + .HasColumnType("bigint[]") + .HasColumnName("receipt_event_type_indexes"); + + b.HasKey("StateVersion"); + + b.ToTable("ledger_transaction_events"); + }); + modelBuilder.Entity("RadixDlt.NetworkGateway.PostgresIntegration.Models.LedgerTransactionMarker", b => { b.Property("Id") diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20241001130214_InitialCreate.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240925082442_InitialCreate.cs similarity index 99% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20241001130214_InitialCreate.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240925082442_InitialCreate.cs index e468d4452..e34fc368b 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20241001130214_InitialCreate.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240925082442_InitialCreate.cs @@ -539,6 +539,24 @@ protected override void Up(MigrationBuilder migrationBuilder) table.PrimaryKey("PK_key_value_store_totals_history", x => x.id); }); + migrationBuilder.CreateTable( + name: "ledger_transaction_events", + columns: table => new + { + state_version = table.Column(type: "bigint", nullable: false), + receipt_event_emitters = table.Column(type: "jsonb[]", nullable: false), + receipt_event_names = table.Column(type: "text[]", nullable: false), + receipt_event_sbors = table.Column(type: "bytea[]", nullable: false), + receipt_event_schema_entity_ids = table.Column(type: "bigint[]", nullable: false), + receipt_event_schema_hashes = table.Column(type: "bytea[]", nullable: false), + receipt_event_type_indexes = table.Column(type: "bigint[]", nullable: false), + receipt_event_sbor_type_kinds = table.Column(type: "sbor_type_kind[]", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_ledger_transaction_events", x => x.state_version); + }); + migrationBuilder.CreateTable( name: "ledger_transaction_markers", columns: table => new @@ -578,20 +596,13 @@ protected override void Up(MigrationBuilder migrationBuilder) normalized_round_timestamp = table.Column(type: "timestamp with time zone", nullable: false), receipt_status = table.Column(type: "ledger_transaction_status", nullable: false), receipt_fee_summary = table.Column(type: "jsonb", nullable: false), - receipt_state_updates = table.Column(type: "jsonb", nullable: false), + receipt_state_updates = table.Column(type: "jsonb", nullable: true), receipt_costing_parameters = table.Column(type: "jsonb", nullable: false), receipt_fee_source = table.Column(type: "jsonb", nullable: true), receipt_fee_destination = table.Column(type: "jsonb", nullable: true), receipt_next_epoch = table.Column(type: "jsonb", nullable: true), receipt_output = table.Column(type: "jsonb", nullable: true), receipt_error_message = table.Column(type: "text", nullable: true), - receipt_event_emitters = table.Column(type: "jsonb[]", nullable: false), - receipt_event_names = table.Column(type: "text[]", nullable: false), - receipt_event_sbors = table.Column(type: "bytea[]", nullable: false), - receipt_event_schema_entity_ids = table.Column(type: "bigint[]", nullable: false), - receipt_event_schema_hashes = table.Column(type: "bytea[]", nullable: false), - receipt_event_type_indexes = table.Column(type: "bigint[]", nullable: false), - receipt_event_sbor_type_kinds = table.Column(type: "sbor_type_kind[]", nullable: false), balance_changes = table.Column(type: "jsonb", nullable: true), transaction_tree_hash = table.Column(type: "text", nullable: false), receipt_tree_hash = table.Column(type: "text", nullable: false), @@ -1510,6 +1521,9 @@ protected override void Down(MigrationBuilder migrationBuilder) migrationBuilder.DropTable( name: "key_value_store_totals_history"); + migrationBuilder.DropTable( + name: "ledger_transaction_events"); + migrationBuilder.DropTable( name: "ledger_transaction_markers"); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/IdempotentApplyMigrations.sql b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/IdempotentApplyMigrations.sql index b1905128b..7a9160072 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/IdempotentApplyMigrations.sql +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/IdempotentApplyMigrations.sql @@ -9,7 +9,7 @@ START TRANSACTION; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TYPE account_default_deposit_rule AS ENUM ('accept', 'reject', 'allow_existing'); CREATE TYPE account_resource_preference_rule AS ENUM ('allowed', 'disallowed'); CREATE TYPE authorized_depositor_badge_type AS ENUM ('resource', 'non_fungible'); @@ -37,7 +37,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE account_authorized_depositor_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -50,7 +50,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE account_authorized_depositor_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -66,7 +66,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE account_default_deposit_rule_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -79,7 +79,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE account_locker_entry_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -93,7 +93,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE account_locker_entry_resource_vault_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -107,7 +107,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE account_locker_entry_touch_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -119,7 +119,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE account_resource_preference_rule_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -132,7 +132,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE account_resource_preference_rule_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -147,7 +147,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE component_method_royalty_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -160,7 +160,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE component_method_royalty_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -175,7 +175,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE entities ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -201,7 +201,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE entity_metadata_entry_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -214,7 +214,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE entity_metadata_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -229,7 +229,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE entity_metadata_totals_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -243,7 +243,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE entity_resource_balance_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -257,7 +257,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE entity_resource_entry_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -271,7 +271,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE entity_resource_totals_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -286,7 +286,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE entity_resource_vault_entry_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -300,7 +300,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE entity_resource_vault_totals_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -314,7 +314,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE entity_role_assignments_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -328,7 +328,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE entity_role_assignments_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -344,7 +344,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE entity_role_assignments_owner_role_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -357,7 +357,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE key_value_store_entry_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -370,7 +370,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE key_value_store_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -385,7 +385,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE key_value_store_schema_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -405,7 +405,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE key_value_store_totals_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -419,7 +419,24 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + CREATE TABLE ledger_transaction_events ( + state_version bigint NOT NULL, + receipt_event_emitters jsonb[] NOT NULL, + receipt_event_names text[] NOT NULL, + receipt_event_sbors bytea[] NOT NULL, + receipt_event_schema_entity_ids bigint[] NOT NULL, + receipt_event_schema_hashes bytea[] NOT NULL, + receipt_event_type_indexes bigint[] NOT NULL, + receipt_event_sbor_type_kinds sbor_type_kind[] NOT NULL, + CONSTRAINT "PK_ledger_transaction_events" PRIMARY KEY (state_version) + ); + END IF; +END $EF$; + +DO $EF$ +BEGIN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE ledger_transaction_markers ( id bigint GENERATED BY DEFAULT AS IDENTITY, state_version bigint NOT NULL, @@ -439,7 +456,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE ledger_transactions ( state_version bigint NOT NULL, epoch bigint NOT NULL, @@ -454,20 +471,13 @@ BEGIN normalized_round_timestamp timestamp with time zone NOT NULL, receipt_status ledger_transaction_status NOT NULL, receipt_fee_summary jsonb NOT NULL, - receipt_state_updates jsonb NOT NULL, + receipt_state_updates jsonb, receipt_costing_parameters jsonb NOT NULL, receipt_fee_source jsonb, receipt_fee_destination jsonb, receipt_next_epoch jsonb, receipt_output jsonb, receipt_error_message text, - receipt_event_emitters jsonb[] NOT NULL, - receipt_event_names text[] NOT NULL, - receipt_event_sbors bytea[] NOT NULL, - receipt_event_schema_entity_ids bigint[] NOT NULL, - receipt_event_schema_hashes bytea[] NOT NULL, - receipt_event_type_indexes bigint[] NOT NULL, - receipt_event_sbor_type_kinds sbor_type_kind[] NOT NULL, balance_changes jsonb, transaction_tree_hash text NOT NULL, receipt_tree_hash text NOT NULL, @@ -487,7 +497,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE non_fungible_id_data_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -502,7 +512,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE non_fungible_id_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -515,7 +525,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE non_fungible_id_location_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -528,7 +538,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE non_fungible_schema_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -544,7 +554,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE non_fungible_vault_entry_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -557,7 +567,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE non_fungible_vault_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -570,7 +580,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE package_blueprint_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -583,7 +593,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE package_blueprint_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -603,7 +613,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE package_code_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -616,7 +626,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE package_code_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -632,7 +642,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE pending_transactions ( id bigint GENERATED BY DEFAULT AS IDENTITY, payload_hash text NOT NULL, @@ -660,7 +670,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE resource_entity_supply_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -675,7 +685,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE resource_holders ( id bigint GENERATED BY DEFAULT AS IDENTITY, entity_id bigint NOT NULL, @@ -689,7 +699,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE schema_entry_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -702,7 +712,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE schema_entry_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -716,7 +726,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE state_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -735,7 +745,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE unverified_standard_metadata_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -748,7 +758,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE unverified_standard_metadata_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -765,7 +775,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE validator_cumulative_emission_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -781,7 +791,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE validator_public_key_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -795,7 +805,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE vault_balance_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -808,7 +818,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE pending_transaction_payloads ( id bigint GENERATED BY DEFAULT AS IDENTITY, pending_transaction_id bigint, @@ -821,7 +831,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE TABLE validator_active_set_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -836,555 +846,555 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_account_authorized_depositor_aggregate_history_account_enti~" ON account_authorized_depositor_aggregate_history (account_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_account_authorized_depositor_entry_history_account_entity_~1" ON account_authorized_depositor_entry_history (account_entity_id, resource_entity_id, non_fungible_id, from_state_version) WHERE discriminator = 'non_fungible'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_account_authorized_depositor_entry_history_account_entity_~2" ON account_authorized_depositor_entry_history (account_entity_id, resource_entity_id, from_state_version) WHERE discriminator = 'resource'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_account_authorized_depositor_entry_history_account_entity_i~" ON account_authorized_depositor_entry_history (account_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_account_default_deposit_rule_history_account_entity_id_from~" ON account_default_deposit_rule_history (account_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE UNIQUE INDEX "IX_account_locker_entry_definition_account_locker_entity_id_ac~" ON account_locker_entry_definition (account_locker_entity_id, account_entity_id); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN - CREATE INDEX "IX_account_locker_entry_resource_vault_definition_account_lock~" ON account_locker_entry_resource_vault_definition (account_locker_definition_id, from_state_version); + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + CREATE UNIQUE INDEX "IX_account_locker_entry_resource_vault_definition_account_lock~" ON account_locker_entry_resource_vault_definition (account_locker_definition_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_account_locker_entry_touch_history_account_locker_definitio~" ON account_locker_entry_touch_history (account_locker_definition_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_account_resource_preference_rule_aggregate_history_account_~" ON account_resource_preference_rule_aggregate_history (account_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_account_resource_preference_rule_entry_history_account_enti~" ON account_resource_preference_rule_entry_history (account_entity_id, resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_component_method_royalty_aggregate_history_entity_id_from_s~" ON component_method_royalty_aggregate_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_component_method_royalty_entry_history_entity_id_from_state~" ON component_method_royalty_entry_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_component_method_royalty_entry_history_entity_id_method_nam~" ON component_method_royalty_entry_history (entity_id, method_name, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE UNIQUE INDEX "IX_entities_address" ON entities (address); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_entities_from_state_version" ON entities (from_state_version) WHERE discriminator = 'global_validator'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_entity_metadata_entry_definition_entity_id_from_state_versi~" ON entity_metadata_entry_definition (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_entity_metadata_entry_definition_entity_id_key" ON entity_metadata_entry_definition (entity_id, key); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_entity_metadata_entry_history_entity_metadata_entry_definit~" ON entity_metadata_entry_history (entity_metadata_entry_definition_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_entity_metadata_totals_history_entity_id_from_state_version" ON entity_metadata_totals_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_entity_resource_balance_history_entity_id_resource_entity_i~" ON entity_resource_balance_history (entity_id, resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_entity_resource_entry_definition_entity_id_from_state_versi~" ON entity_resource_entry_definition (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_entity_resource_entry_definition_fungibles" ON entity_resource_entry_definition (entity_id, from_state_version) WHERE resource_type = 'fungible'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_entity_resource_entry_definition_non_fungibles" ON entity_resource_entry_definition (entity_id, from_state_version) WHERE resource_type = 'non_fungible'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_entity_resource_totals_history_entity_id_from_state_version" ON entity_resource_totals_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_entity_resource_vault_entry_definition_entity_id_resource_e~" ON entity_resource_vault_entry_definition (entity_id, resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_entity_resource_vault_totals_history_entity_id_resource_ent~" ON entity_resource_vault_totals_history (entity_id, resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_entity_role_assignments_aggregate_history_entity_id_from_st~" ON entity_role_assignments_aggregate_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_entity_role_assignments_entry_history_entity_id_key_role_ke~" ON entity_role_assignments_entry_history (entity_id, key_role, key_module, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_entity_role_assignments_owner_role_history_entity_id_from_s~" ON entity_role_assignments_owner_role_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_key_value_store_entry_definition_key_value_store_entity_id_~" ON key_value_store_entry_definition (key_value_store_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_key_value_store_entry_definition_key_value_store_entity_id~1" ON key_value_store_entry_definition (key_value_store_entity_id, key); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_key_value_store_entry_history_key_value_store_entry_definit~" ON key_value_store_entry_history (key_value_store_entry_definition_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_key_value_store_schema_history_key_value_store_entity_id_fr~" ON key_value_store_schema_history (key_value_store_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_key_value_store_totals_history_entity_id_from_state_version" ON key_value_store_totals_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_entity_id_state_version" ON ledger_transaction_markers (entity_id, state_version) WHERE discriminator = 'event_global_emitter'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_entity_id_state_version1" ON ledger_transaction_markers (entity_id, state_version) WHERE discriminator = 'affected_global_entity'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_event_type_entity_id_state_versi~" ON ledger_transaction_markers (event_type, entity_id, state_version) WHERE discriminator = 'event'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_manifest_class" ON ledger_transaction_markers (manifest_class, state_version) WHERE discriminator = 'manifest_class'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_manifest_class_is_most_specific" ON ledger_transaction_markers (manifest_class, state_version) WHERE discriminator = 'manifest_class' and is_most_specific = true; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_operation_type_entity_id_state_v~" ON ledger_transaction_markers (operation_type, entity_id, state_version) WHERE discriminator = 'manifest_address'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_origin_type_state_version" ON ledger_transaction_markers (origin_type, state_version) WHERE discriminator = 'origin'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_state_version" ON ledger_transaction_markers (state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE UNIQUE INDEX "IX_ledger_transactions_epoch_round_in_epoch" ON ledger_transactions (epoch, round_in_epoch) WHERE index_in_round = 0; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_ledger_transactions_intent_hash" ON ledger_transactions USING hash (intent_hash) WHERE intent_hash IS NOT NULL; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_ledger_transactions_round_timestamp" ON ledger_transactions (round_timestamp); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_non_fungible_id_data_history_non_fungible_id_definition_id_~" ON non_fungible_id_data_history (non_fungible_id_definition_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_non_fungible_id_definition_non_fungible_resource_entity_id_~" ON non_fungible_id_definition (non_fungible_resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE UNIQUE INDEX "IX_non_fungible_id_definition_non_fungible_resource_entity_id~1" ON non_fungible_id_definition (non_fungible_resource_entity_id, non_fungible_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_non_fungible_id_location_history_non_fungible_id_definition~" ON non_fungible_id_location_history (non_fungible_id_definition_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_non_fungible_schema_history_resource_entity_id_from_state_v~" ON non_fungible_schema_history (resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_non_fungible_vault_entry_definition_vault_entity_id_from_st~" ON non_fungible_vault_entry_definition (vault_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_non_fungible_vault_entry_history_non_fungible_vault_entry_d~" ON non_fungible_vault_entry_history (non_fungible_vault_entry_definition_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_package_blueprint_aggregate_history_package_entity_id_from_~" ON package_blueprint_aggregate_history (package_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_package_blueprint_history_package_entity_id_from_state_vers~" ON package_blueprint_history (package_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_package_blueprint_history_package_entity_id_name_version_fr~" ON package_blueprint_history (package_entity_id, name, version, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_package_code_aggregate_history_package_entity_id_from_state~" ON package_code_aggregate_history (package_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_package_code_history_package_entity_id_from_state_version" ON package_code_history (package_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE UNIQUE INDEX "IX_pending_transaction_payloads_pending_transaction_id" ON pending_transaction_payloads (pending_transaction_id); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_pending_transactions_first_submitted_to_gateway_timestamp" ON pending_transactions (first_submitted_to_gateway_timestamp); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_pending_transactions_intent_hash" ON pending_transactions (intent_hash); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE UNIQUE INDEX "IX_pending_transactions_payload_hash" ON pending_transactions (payload_hash); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_pending_transactions_resubmit_from_timestamp" ON pending_transactions (resubmit_from_timestamp); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_resource_entity_supply_history_resource_entity_id_from_stat~" ON resource_entity_supply_history (resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE UNIQUE INDEX "IX_resource_holders_entity_id_resource_entity_id" ON resource_holders (entity_id, resource_entity_id); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_resource_holders_entity_id_resource_entity_id_balance" ON resource_holders (entity_id, resource_entity_id, balance); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_schema_entry_aggregate_history_entity_id_from_state_version" ON schema_entry_aggregate_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_schema_entry_definition_entity_id_schema_hash" ON schema_entry_definition (entity_id, schema_hash); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_state_history_entity_id_from_state_version" ON state_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_unverified_standard_metadata_aggregate_history_entity_id_fr~" ON unverified_standard_metadata_aggregate_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_unverified_standard_metadata_entry_history_entity_id_discri~" ON unverified_standard_metadata_entry_history (entity_id, discriminator, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_validator_active_set_history_epoch" ON validator_active_set_history (epoch); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_validator_active_set_history_from_state_version" ON validator_active_set_history (from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_validator_active_set_history_validator_public_key_history_id" ON validator_active_set_history (validator_public_key_history_id); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_validator_cumulative_emission_history_validator_entity_id_e~" ON validator_cumulative_emission_history (validator_entity_id, epoch_number); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_validator_cumulative_emission_history_validator_entity_id_f~" ON validator_cumulative_emission_history (validator_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_validator_public_key_history_validator_entity_id_from_state~" ON validator_public_key_history (validator_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_validator_public_key_history_validator_entity_id_key_type_k~" ON validator_public_key_history (validator_entity_id, key_type, key); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN CREATE INDEX "IX_vault_balance_history_vault_entity_id_from_state_version" ON vault_balance_history (vault_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001130214_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN INSERT INTO "__EFMigrationsHistory" ("MigrationId", "ProductVersion") - VALUES ('20241001130214_InitialCreate', '8.0.2'); + VALUES ('20240925082442_InitialCreate', '8.0.2'); END IF; END $EF$; COMMIT; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs index 569fd91fb..18d4ba901 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs @@ -1073,41 +1073,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("text") .HasColumnName("receipt_error_message"); - b.Property("ReceiptEventEmitters") - .IsRequired() - .HasColumnType("jsonb[]") - .HasColumnName("receipt_event_emitters"); - - b.Property("ReceiptEventNames") - .IsRequired() - .HasColumnType("text[]") - .HasColumnName("receipt_event_names"); - - b.Property("ReceiptEventSborTypeKinds") - .IsRequired() - .HasColumnType("sbor_type_kind[]") - .HasColumnName("receipt_event_sbor_type_kinds"); - - b.Property("ReceiptEventSbors") - .IsRequired() - .HasColumnType("bytea[]") - .HasColumnName("receipt_event_sbors"); - - b.Property("ReceiptEventSchemaEntityIds") - .IsRequired() - .HasColumnType("bigint[]") - .HasColumnName("receipt_event_schema_entity_ids"); - - b.Property("ReceiptEventSchemaHashes") - .IsRequired() - .HasColumnType("bytea[]") - .HasColumnName("receipt_event_schema_hashes"); - - b.Property("ReceiptEventTypeIndexes") - .IsRequired() - .HasColumnType("bigint[]") - .HasColumnName("receipt_event_type_indexes"); - b.Property("ReceiptFeeDestination") .HasColumnType("jsonb") .HasColumnName("receipt_fee_destination"); @@ -1130,7 +1095,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnName("receipt_output"); b.Property("ReceiptStateUpdates") - .IsRequired() .HasColumnType("jsonb") .HasColumnName("receipt_state_updates"); @@ -1184,6 +1148,52 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.UseTphMappingStrategy(); }); + modelBuilder.Entity("RadixDlt.NetworkGateway.PostgresIntegration.Models.LedgerTransactionEvents", b => + { + b.Property("StateVersion") + .HasColumnType("bigint") + .HasColumnName("state_version"); + + b.Property("ReceiptEventEmitters") + .IsRequired() + .HasColumnType("jsonb[]") + .HasColumnName("receipt_event_emitters"); + + b.Property("ReceiptEventNames") + .IsRequired() + .HasColumnType("text[]") + .HasColumnName("receipt_event_names"); + + b.Property("ReceiptEventSborTypeKinds") + .IsRequired() + .HasColumnType("sbor_type_kind[]") + .HasColumnName("receipt_event_sbor_type_kinds"); + + b.Property("ReceiptEventSbors") + .IsRequired() + .HasColumnType("bytea[]") + .HasColumnName("receipt_event_sbors"); + + b.Property("ReceiptEventSchemaEntityIds") + .IsRequired() + .HasColumnType("bigint[]") + .HasColumnName("receipt_event_schema_entity_ids"); + + b.Property("ReceiptEventSchemaHashes") + .IsRequired() + .HasColumnType("bytea[]") + .HasColumnName("receipt_event_schema_hashes"); + + b.Property("ReceiptEventTypeIndexes") + .IsRequired() + .HasColumnType("bigint[]") + .HasColumnName("receipt_event_type_indexes"); + + b.HasKey("StateVersion"); + + b.ToTable("ledger_transaction_events"); + }); + modelBuilder.Entity("RadixDlt.NetworkGateway.PostgresIntegration.Models.LedgerTransactionMarker", b => { b.Property("Id") diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/LedgerTransactions/LedgerTransaction.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/LedgerTransactions/LedgerTransaction.cs index 93ad7f2ff..1e151c871 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/LedgerTransactions/LedgerTransaction.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/LedgerTransactions/LedgerTransaction.cs @@ -72,14 +72,36 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.Models; -internal record TransactionReceiptEventLookup(long EntityId, ValueBytes SchemaHash); +[Table("ledger_transaction_events")] +internal class LedgerTransactionEvents +{ + [Key] + [DatabaseGenerated(DatabaseGeneratedOption.None)] + [Column("state_version")] + public long StateVersion { get; set; } + + [Column("receipt_event_emitters", TypeName = "jsonb[]")] + public string[] ReceiptEventEmitters { get; set; } + + [Column("receipt_event_names", TypeName = "text[]")] + public string[] ReceiptEventNames { get; set; } + + [Column("receipt_event_sbors")] + public byte[][] ReceiptEventSbors { get; set; } + + [Column("receipt_event_schema_entity_ids")] + public long[] ReceiptEventSchemaEntityIds { get; set; } + + [Column("receipt_event_schema_hashes")] + public byte[][] ReceiptEventSchemaHashes { get; set; } -internal record TransactionReceiptEvent(string Name, string Emitter, byte[] Data, long EntityId, byte[] SchemaHash, long TypeIndex, SborTypeKind KeyTypeKind); + [Column("receipt_event_type_indexes")] + public long[] ReceiptEventTypeIndexes { get; set; } + + [Column("receipt_event_sbor_type_kinds")] + public SborTypeKind[] ReceiptEventSborTypeKinds { get; set; } +} -/// -/// A transaction committed onto the radix ledger. -/// This table forms a shell, to which other properties are connected. -/// [Table("ledger_transactions")] internal abstract class LedgerTransaction { @@ -140,7 +162,7 @@ internal abstract class LedgerTransaction public string ReceiptFeeSummary { get; set; } [Column("receipt_state_updates", TypeName = "jsonb")] - public string ReceiptStateUpdates { get; set; } + public string? ReceiptStateUpdates { get; set; } [Column("receipt_costing_parameters", TypeName = "jsonb")] public string ReceiptCostingParameters { get; set; } @@ -160,27 +182,6 @@ internal abstract class LedgerTransaction [Column("receipt_error_message")] public string? ReceiptErrorMessage { get; set; } - [Column("receipt_event_emitters", TypeName = "jsonb[]")] - public string[] ReceiptEventEmitters { get; set; } - - [Column("receipt_event_names", TypeName = "text[]")] - public string[] ReceiptEventNames { get; set; } - - [Column("receipt_event_sbors")] - public byte[][] ReceiptEventSbors { get; set; } - - [Column("receipt_event_schema_entity_ids")] - public long[] ReceiptEventSchemaEntityIds { get; set; } - - [Column("receipt_event_schema_hashes")] - public byte[][] ReceiptEventSchemaHashes { get; set; } - - [Column("receipt_event_type_indexes")] - public long[] ReceiptEventTypeIndexes { get; set; } - - [Column("receipt_event_sbor_type_kinds")] - public SborTypeKind[] ReceiptEventSborTypeKinds { get; set; } - [Column("balance_changes", TypeName = "jsonb")] public string? BalanceChanges { get; set; } @@ -256,8 +257,6 @@ internal class TransactionReceipt public TransactionReceipt(LedgerTransaction ledgerTransaction) { _ledgerTransaction = ledgerTransaction; - - Events = new ReceiptEvents(ledgerTransaction); } public string? ErrorMessage => _ledgerTransaction.ReceiptErrorMessage; @@ -276,63 +275,5 @@ public TransactionReceipt(LedgerTransaction ledgerTransaction) public string CostingParameters => _ledgerTransaction.ReceiptCostingParameters; - public string StateUpdates => _ledgerTransaction.ReceiptStateUpdates; - - public ReceiptEvents Events { get; } -} - -internal class ReceiptEvents -{ - private LedgerTransaction _ledgerTransaction; - - public ReceiptEvents(LedgerTransaction ledgerTransaction) - { - _ledgerTransaction = ledgerTransaction; - } - - public string[] Emitters => _ledgerTransaction.ReceiptEventEmitters; - - public string[] Names => _ledgerTransaction.ReceiptEventNames; - - public byte[][] Sbors => _ledgerTransaction.ReceiptEventSbors; - - public long[] SchemaEntityIds => _ledgerTransaction.ReceiptEventSchemaEntityIds; - - public byte[][] SchemaHashes => _ledgerTransaction.ReceiptEventSchemaHashes; - - public long[] TypeIndexes => _ledgerTransaction.ReceiptEventTypeIndexes; - - public SborTypeKind[] SborTypeKinds => _ledgerTransaction.ReceiptEventSborTypeKinds; - - public List GetEventLookups() - { - var result = new List(); - - for (var i = 0; i < _ledgerTransaction.ReceiptEventSbors.Length; ++i) - { - result.Add(new TransactionReceiptEventLookup(_ledgerTransaction.ReceiptEventSchemaEntityIds[i], _ledgerTransaction.ReceiptEventSchemaHashes[i])); - } - - return result; - } - - public List GetEvents() - { - var result = new List(); - - for (var i = 0; i < _ledgerTransaction.ReceiptEventSbors.Length; ++i) - { - var eventData = _ledgerTransaction.ReceiptEventSbors[i]; - var entityId = _ledgerTransaction.ReceiptEventSchemaEntityIds[i]; - var schemaHash = _ledgerTransaction.ReceiptEventSchemaHashes[i]; - var index = _ledgerTransaction.ReceiptEventTypeIndexes[i]; - var typeKind = _ledgerTransaction.ReceiptEventSborTypeKinds[i]; - var emitter = _ledgerTransaction.ReceiptEventEmitters[i]; - var name = _ledgerTransaction.ReceiptEventNames[i]; - - result.Add(new TransactionReceiptEvent(name, emitter, eventData, entityId, schemaHash, index, typeKind)); - } - - return result; - } + public string? StateUpdates => _ledgerTransaction.ReceiptStateUpdates; } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/EntityQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/EntityQuerier.cs index 5b0b9b2f2..76dbf06ab 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/EntityQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/EntityQuerier.cs @@ -68,6 +68,7 @@ using RadixDlt.NetworkGateway.PostgresIntegration.Models; using RadixDlt.NetworkGateway.PostgresIntegration.Services; using System.Collections.Generic; +using System.Collections.Immutable; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -80,10 +81,12 @@ internal interface IEntityQuerier Task GetEntity(EntityAddress address, GatewayApiSdk.Model.LedgerState ledgerState, CancellationToken token) where TEntity : Entity; - Task GetNonVirtualEntity(ReadOnlyDbContext dbContext, EntityAddress address, GatewayModel.LedgerState ledgerState, CancellationToken token) + Task GetNonVirtualEntity(EntityAddress address, GatewayModel.LedgerState ledgerState, CancellationToken token) where TEntity : Entity; - Task> ResolveEntityIds(ReadOnlyDbContext dbContext, List addresses, GatewayModel.LedgerState ledgerState, CancellationToken token); + Task> ResolveEntityIds(List addresses, GatewayModel.LedgerState ledgerState, CancellationToken token); + + Task> ResolveEntityAddresses(List entityIds, CancellationToken token = default); Task> GetEntities(List addresses, GatewayModel.LedgerState ledgerState, CancellationToken token); } @@ -127,10 +130,10 @@ public async Task GetEntity(EntityAddress address, GatewayApiS return typedEntity; } - public async Task GetNonVirtualEntity(ReadOnlyDbContext dbContext, EntityAddress address, GatewayModel.LedgerState ledgerState, CancellationToken token) + public async Task GetNonVirtualEntity(EntityAddress address, GatewayModel.LedgerState ledgerState, CancellationToken token) where TEntity : Entity { - var entity = await dbContext + var entity = await _dbContext .Entities .Where(e => e.FromStateVersion <= ledgerState.StateVersion) .AnnotateMetricName() @@ -149,9 +152,9 @@ public async Task GetNonVirtualEntity(ReadOnlyDbContext dbCont return typedEntity; } - public async Task> ResolveEntityIds(ReadOnlyDbContext dbContext, List addresses, GatewayModel.LedgerState ledgerState, CancellationToken token) + public async Task> ResolveEntityIds(List addresses, GatewayModel.LedgerState ledgerState, CancellationToken token) { - var entities = await dbContext + var entities = await _dbContext .Entities .Where(e => e.FromStateVersion <= ledgerState.StateVersion && addresses.Contains(e.Address)) .AnnotateMetricName() @@ -160,6 +163,21 @@ public async Task> ResolveEntityIds(ReadOnlyDbCo return entities; } + public async Task> ResolveEntityAddresses(List entityIds, CancellationToken token = default) + { + if (!entityIds.Any()) + { + return ImmutableDictionary.Empty; + } + + return await _dbContext + .Entities + .Where(e => entityIds.Contains(e.Id)) + .Select(e => new { e.Id, e.Address }) + .AnnotateMetricName() + .ToDictionaryAsync(e => e.Id, e => e.Address, token); + } + public async Task> GetEntities(List addresses, GatewayModel.LedgerState ledgerState, CancellationToken token) { var entities = await _dbContext diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountDepositSettings/AccountStateQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountDepositSettings/AccountStateQuerier.cs index a938a73fb..0a126fd9d 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountDepositSettings/AccountStateQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountDepositSettings/AccountStateQuerier.cs @@ -113,7 +113,7 @@ public AccountStateQuerier(IDapperWrapper dapperWrapper, ReadOnlyDbContext dbCon int limit, CancellationToken token = default) { - var accountEntity = await _entityQuerier.GetNonVirtualEntity(_dbContext, accountAddress, ledgerState, token); + var accountEntity = await _entityQuerier.GetNonVirtualEntity(accountAddress, ledgerState, token); var cd = new CommandDefinition( commandText: @" @@ -172,7 +172,7 @@ INNER JOIN LATERAL UNNEST(resource_preference_rules_slice) WITH ORDINALITY AS re int limit, CancellationToken token = default) { - var accountEntity = await _entityQuerier.GetNonVirtualEntity(_dbContext, accountAddress, ledgerState, token); + var accountEntity = await _entityQuerier.GetNonVirtualEntity(accountAddress, ledgerState, token); var cd = new CommandDefinition( commandText: @" diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountDepositSettings/DepositPreValidationQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountDepositSettings/DepositPreValidationQuerier.cs index c29e9c8b4..d9744a189 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountDepositSettings/DepositPreValidationQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountDepositSettings/DepositPreValidationQuerier.cs @@ -110,6 +110,7 @@ public DepositPreValidationQuerier( CancellationToken token = default ) { + var accountEntity = await _entityQuerier.GetNonVirtualEntity(accountAddress, ledgerState, token); var xrdResourceAddress = (await _networkConfigurationProvider.GetNetworkConfiguration(token)).WellKnownAddresses.Xrd; var accountEntity = await _entityQuerier.GetEntity(accountAddress, ledgerState, token); if (accountEntity is VirtualAccountComponentEntity) @@ -135,7 +136,7 @@ public DepositPreValidationQuerier( addressesToResolve.Add(badgeResourceAddress.Value); } - var entityAddressToIdDictionary = await _entityQuerier.ResolveEntityIds(_dbContext, addressesToResolve, ledgerState, token); + var entityAddressToIdDictionary = await _entityQuerier.ResolveEntityIds(addressesToResolve, ledgerState, token); var resourceEntityMap = entityAddressToIdDictionary .Where(x => resourceAddresses.Contains(x.Key)) .ToDictionary(); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapperExtensions.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapperExtensions.cs index d23303f62..d921993cb 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapperExtensions.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperWrapperExtensions.cs @@ -72,9 +72,9 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.Services; internal static class DapperWrapperExtensions { - public static async Task> ToListAsync(this IDapperWrapper dapperWrapper, DbConnection dbConnection, CommandDefinition cd) + public static async Task> ToListAsync(this IDapperWrapper dapperWrapper, DbConnection dbConnection, CommandDefinition cd, string operationName = "") { - var result = await dapperWrapper.QueryAsync(dbConnection, cd); + var result = await dapperWrapper.QueryAsync(dbConnection, cd, operationName); return result.ToList(); } diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/KeyValueStoreQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/KeyValueStoreQuerier.cs index 13bad5903..2d29f672f 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/KeyValueStoreQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/KeyValueStoreQuerier.cs @@ -106,7 +106,7 @@ public KeyValueStoreQuerier( CancellationToken token = default) { var networkId = (await _networkConfigurationProvider.GetNetworkConfiguration(token)).Id; - var keyValueStore = await _entityQuerier.GetNonVirtualEntity(_dbContext, keyValueStoreAddress, ledgerState, token); + var keyValueStore = await _entityQuerier.GetNonVirtualEntity(keyValueStoreAddress, ledgerState, token); var keyValueStoreSchema = await KeyValueStoreQueries.KeyValueStoreSchemaLookupQuery( _dbContext, @@ -138,7 +138,7 @@ public KeyValueStoreQuerier( CancellationToken token = default) { var networkId = (await _networkConfigurationProvider.GetNetworkConfiguration(token)).Id; - var keyValueStore = await _entityQuerier.GetNonVirtualEntity(_dbContext, keyValueStoreAddress, ledgerState, token); + var keyValueStore = await _entityQuerier.GetNonVirtualEntity(keyValueStoreAddress, ledgerState, token); var keyValueStoreSchema = await KeyValueStoreQueries.KeyValueStoreSchemaLookupQuery( _dbContext, diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TopOfLedgerProvider.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TopOfLedgerProvider.cs index a7f88f9e9..e6d64145e 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TopOfLedgerProvider.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TopOfLedgerProvider.cs @@ -121,9 +121,6 @@ public async Task GetTopOfLedger(CancellationToken token) ? await PreGenesisTransactionSummary() : new TransactionSummary( StateVersion: lastTransaction.StateVersion, - TransactionTreeHash: lastTransaction.TransactionTreeHash, - ReceiptTreeHash: lastTransaction.ReceiptTreeHash, - StateTreeHash: lastTransaction.StateTreeHash, RoundTimestamp: lastTransaction.RoundTimestamp, NormalizedRoundTimestamp: lastTransaction.NormalizedRoundTimestamp, CreatedTimestamp: lastTransaction.CreatedTimestamp, @@ -139,9 +136,6 @@ private async Task PreGenesisTransactionSummary() // Nearly all of theses turn out to be unused! return new TransactionSummary( StateVersion: 0, - TransactionTreeHash: string.Empty, - ReceiptTreeHash: string.Empty, - StateTreeHash: string.Empty, RoundTimestamp: DateTimeOffset.FromUnixTimeSeconds(0).UtcDateTime, NormalizedRoundTimestamp: DateTimeOffset.FromUnixTimeSeconds(0).UtcDateTime, CreatedTimestamp: _clock.UtcNow, diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionMapper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionMapper.cs new file mode 100644 index 000000000..5049d960c --- /dev/null +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionMapper.cs @@ -0,0 +1,146 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +using Newtonsoft.Json.Linq; +using RadixDlt.NetworkGateway.Abstractions; +using RadixDlt.NetworkGateway.Abstractions.Extensions; +using RadixDlt.NetworkGateway.Abstractions.Model; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; + +namespace RadixDlt.NetworkGateway.PostgresIntegration.Services; + +// TODO PP: do we want to move it back to query. +internal static class TransactionMapper +{ + internal static GatewayApiSdk.Model.CommittedTransactionInfo ToGatewayModel( + this LedgerTransactionQueryResult lt, + GatewayApiSdk.Model.TransactionDetailsOptIns optIns, + IDictionary entityIdToAddressMap, + List? events, + GatewayApiSdk.Model.TransactionBalanceChanges? transactionBalanceChanges) + { + string? payloadHash = null; + string? intentHash = null; + string? rawHex = null; + JRaw? message = null; + string? manifestInstructions = null; + List? manifestClasses = null; + + if (lt.Discriminator == LedgerTransactionType.User) + { + payloadHash = lt.PayloadHash; + intentHash = lt.IntentHash; + rawHex = optIns.RawHex ? lt.RawPayload!.ToHex() : null; + message = lt.Message != null ? new JRaw(lt.Message) : null; + manifestInstructions = optIns.ManifestInstructions ? lt.ManifestInstructions : null; + manifestClasses = lt.ManifestClasses.Select(mc => mc.ToGatewayModel()).ToList(); + } + + var receipt = new GatewayApiSdk.Model.TransactionReceipt + { + ErrorMessage = lt.ReceiptErrorMessage, + Status = MapTransactionStatus(lt.ReceiptStatus), + Output = optIns.ReceiptOutput && lt.ReceiptOutput != null ? new JRaw(lt.ReceiptOutput) : null, + FeeSummary = optIns.ReceiptFeeSummary ? new JRaw(lt.ReceiptFeeSummary) : null, + FeeDestination = optIns.ReceiptFeeDestination && lt.ReceiptFeeDestination != null ? new JRaw(lt.ReceiptFeeDestination) : null, + FeeSource = optIns.ReceiptFeeSource && lt.ReceiptFeeSource != null ? new JRaw(lt.ReceiptFeeSource) : null, + CostingParameters = optIns.ReceiptCostingParameters ? new JRaw(lt.ReceiptCostingParameters) : null, + NextEpoch = lt.ReceiptNextEpoch != null ? new JRaw(lt.ReceiptNextEpoch) : null, + StateUpdates = optIns.ReceiptStateChanges ? new JRaw(lt.ReceiptStateUpdates) : null, + Events = optIns.ReceiptEvents ? events?.Select(x => new GatewayApiSdk.Model.EventsItem(x.Name, new JRaw(x.Emitter), x.Data)).ToList() : null, + }; + + return new GatewayApiSdk.Model.CommittedTransactionInfo( + stateVersion: lt.StateVersion, + epoch: lt.Epoch, + round: lt.RoundInEpoch, + roundTimestamp: lt.RoundTimestamp.AsUtcIsoDateWithMillisString(), + transactionStatus: MapTransactionStatus(lt.ReceiptStatus), + affectedGlobalEntities: optIns.AffectedGlobalEntities ? lt.AffectedGlobalEntities.Select(x => entityIdToAddressMap[x].ToString()).ToList() : null, + payloadHash: payloadHash, + intentHash: intentHash, + feePaid: lt.FeePaid.ToString(), + confirmedAt: lt.RoundTimestamp, + errorMessage: lt.ReceiptErrorMessage, + rawHex: rawHex, + receipt: receipt, + message: message, + balanceChanges: optIns.BalanceChanges ? transactionBalanceChanges : null, + manifestInstructions: manifestInstructions, + manifestClasses: manifestClasses + ); + } + + private static GatewayApiSdk.Model.TransactionStatus MapTransactionStatus(this LedgerTransactionStatus status) + { + return status switch + { + LedgerTransactionStatus.Succeeded => GatewayApiSdk.Model.TransactionStatus.CommittedSuccess, + LedgerTransactionStatus.Failed => GatewayApiSdk.Model.TransactionStatus.CommittedFailure, + _ => throw new UnreachableException($"Didn't expect {status} value"), + }; + } +} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs index 6154db94a..da563daa1 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs @@ -62,15 +62,18 @@ * permissions under this License. */ +using Dapper; using Microsoft.EntityFrameworkCore; using Newtonsoft.Json; using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Extensions; using RadixDlt.NetworkGateway.Abstractions.Model; using RadixDlt.NetworkGateway.Abstractions.Network; +using RadixDlt.NetworkGateway.Abstractions.Numerics; using RadixDlt.NetworkGateway.GatewayApi.Services; using RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; using RadixDlt.NetworkGateway.PostgresIntegration.Models; +using RadixDlt.NetworkGateway.PostgresIntegration.Queries; using RadixDlt.NetworkGateway.PostgresIntegration.Services.PendingTransactions; using RadixDlt.NetworkGateway.PostgresIntegration.Utils; using System; @@ -84,24 +87,73 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.Services; +// TODO PP: all these can be made private when we combine mapping with query. +internal record ReceiptEvent(string Name, string Emitter, byte[] Data, long EntityId, byte[] SchemaHash, long TypeIndex, SborTypeKind KeyTypeKind); + +internal record LedgerTransactionQueryResult( + long StateVersion, + long Epoch, + long RoundInEpoch, + TokenAmount FeePaid, + long[] AffectedGlobalEntities, + DateTime RoundTimestamp, + LedgerTransactionStatus ReceiptStatus, + string? ReceiptFeeSource, + string? ReceiptFeeDestination, + string? ReceiptErrorMessage, + LedgerTransactionType Discriminator, + string PayloadHash, + string IntentHash, + string? Message, + LedgerTransactionManifestClass[] ManifestClasses, + byte[]? RawPayload, + string? ReceiptCostingParameters, + string? ReceiptFeeSummary, + string? ReceiptNextEpoch, + string? ReceiptOutput, + string? ReceiptStateUpdates, + string? BalanceChanges, + string? ManifestInstructions +) +{ + public List Events { get; set; } = new(); +} + +internal record ReceiptEvents( + string[] ReceiptEventEmitters, + string[] ReceiptEventNames, + byte[][] ReceiptEventSbors, + long[] ReceiptEventSchemaEntityIds, + byte[][] ReceiptEventSchemaHashes, + long[] ReceiptEventTypeIndexes, + SborTypeKind[] ReceiptEventSborTypeKinds); + +internal record Event(string Name, string Emitter, GatewayModel.ProgrammaticScryptoSborValue Data); + internal class TransactionQuerier : ITransactionQuerier { - internal record Event(string Name, string Emitter, GatewayModel.ProgrammaticScryptoSborValue Data); + private readonly record struct TransactionReceiptEventLookup(long EntityId, ValueBytes SchemaHash); - private record SchemaLookup(long EntityId, ValueBytes SchemaHash); + private readonly record struct SchemaLookup(long EntityId, ValueBytes SchemaHash); private readonly ReadOnlyDbContext _dbContext; private readonly ReadWriteDbContext _rwDbContext; private readonly INetworkConfigurationProvider _networkConfigurationProvider; + private readonly IDapperWrapper _dapperWrapper; + private readonly IEntityQuerier _entityQuerier; public TransactionQuerier( ReadOnlyDbContext dbContext, ReadWriteDbContext rwDbContext, - INetworkConfigurationProvider networkConfigurationProvider) + INetworkConfigurationProvider networkConfigurationProvider, + IDapperWrapper dapperWrapper, + IEntityQuerier entityQuerier) { _dbContext = dbContext; _rwDbContext = rwDbContext; _networkConfigurationProvider = networkConfigurationProvider; + _dapperWrapper = dapperWrapper; + _entityQuerier = entityQuerier; } public async Task GetTransactionStream(TransactionStreamPageRequest request, GatewayModel.LedgerState atLedgerState, CancellationToken token = default) @@ -116,26 +168,28 @@ public async Task GetTransactionStream(TransactionS .Concat(request.SearchCriteria.BadgesPresented) .Concat(request.SearchCriteria.AffectedGlobalEntities) .Concat(request.SearchCriteria.EventGlobalEmitters) - .Concat(request.SearchCriteria.Events.SelectMany(e => - { - var addresses = new List(); + .Concat( + request.SearchCriteria.Events.SelectMany( + e => + { + var addresses = new List(); - if (e.EmitterEntityAddress.HasValue) - { - addresses.Add(e.EmitterEntityAddress.Value); - } + if (e.EmitterEntityAddress.HasValue) + { + addresses.Add(e.EmitterEntityAddress.Value); + } - if (e.ResourceAddress.HasValue) - { - addresses.Add(e.ResourceAddress.Value); - } + if (e.ResourceAddress.HasValue) + { + addresses.Add(e.ResourceAddress.Value); + } - return addresses; - })) - .Select(a => (string)a) + return addresses; + })) + .Select(a => a) .ToList(); - var entityAddressToId = await GetEntityIds(referencedAddresses, token); + var entityAddressToId = await _entityQuerier.ResolveEntityIds(referencedAddresses, atLedgerState, token); var upperStateVersion = request.AscendingOrder ? atLedgerState.StateVersion @@ -454,11 +508,12 @@ internal record CommittedTransactionSummary( .LedgerTransactions .OfType() .Where(ult => ult.StateVersion <= ledgerState.StateVersion && ult.IntentHash == intentHash) - .Select(ult => new CommittedTransactionSummary( - ult.StateVersion, - ult.PayloadHash, - ult.EngineReceipt.Status, - ult.EngineReceipt.ErrorMessage)) + .Select( + ult => new CommittedTransactionSummary( + ult.StateVersion, + ult.PayloadHash, + ult.EngineReceipt.Status, + ult.EngineReceipt.ErrorMessage)) .AnnotateMetricName() .FirstOrDefaultAsync(token); @@ -479,75 +534,126 @@ private async Task> LookupPendingTransact return await _rwDbContext .PendingTransactions .Where(pt => pt.IntentHash == intentHash) - .Select(pt => - new PendingTransactionSummary( - pt.PayloadHash, - pt.EndEpochExclusive, - pt.GatewayHandling.ResubmitFromTimestamp, - pt.GatewayHandling.HandlingStatusReason, - pt.LedgerDetails.PayloadLedgerStatus, - pt.LedgerDetails.IntentLedgerStatus, - pt.LedgerDetails.InitialRejectionReason, - pt.LedgerDetails.LatestRejectionReason, - pt.NetworkDetails.LastSubmitErrorTitle - ) + .Select( + pt => + new PendingTransactionSummary( + pt.PayloadHash, + pt.EndEpochExclusive, + pt.GatewayHandling.ResubmitFromTimestamp, + pt.GatewayHandling.HandlingStatusReason, + pt.LedgerDetails.PayloadLedgerStatus, + pt.LedgerDetails.IntentLedgerStatus, + pt.LedgerDetails.InitialRejectionReason, + pt.LedgerDetails.LatestRejectionReason, + pt.NetworkDetails.LastSubmitErrorTitle + ) ) .AnnotateMetricName() .AsNoTracking() .ToListAsync(token); } + // TODO PP: DO we want to move just that query to separate file. I guess so. private async Task> GetTransactions( List transactionStateVersions, GatewayModel.TransactionDetailsOptIns optIns, CancellationToken token) { - var transactions = await _dbContext - .LedgerTransactions - .FromSqlInterpolated(@$" -WITH configuration AS ( + var cd = new CommandDefinition( + @"WITH vars AS ( SELECT - {optIns.RawHex} AS with_raw_payload, + @includeRawHex AS with_raw_payload, true AS with_receipt_costing_parameters, true AS with_receipt_fee_summary, true AS with_receipt_next_epoch, - {optIns.ReceiptOutput} AS with_receipt_output, - {optIns.ReceiptStateChanges} AS with_receipt_state_updates, - {optIns.ReceiptEvents} AS with_receipt_events, - {optIns.BalanceChanges} AS with_balance_changes, - {optIns.ManifestInstructions} AS with_manifest_instructions + @includeReceiptOutput AS with_receipt_output, + @includeReceiptStateChanges AS with_receipt_state_updates, + @includeReceiptEvents AS with_receipt_events, + @includeBalanceChanges AS with_balance_changes, + @includeManifestInstructions AS with_manifest_instructions, + @transactionStateVersions AS transaction_state_versions ) SELECT - state_version, epoch, round_in_epoch, index_in_epoch, - index_in_round, fee_paid, tip_paid, affected_global_entities, - round_timestamp, created_timestamp, normalized_round_timestamp, - receipt_status, receipt_fee_source, receipt_fee_destination, receipt_error_message, - transaction_tree_hash, receipt_tree_hash, state_tree_hash, - discriminator, payload_hash, intent_hash, signed_intent_hash, message, manifest_classes, - CASE WHEN configuration.with_raw_payload THEN raw_payload ELSE ''::bytea END AS raw_payload, - CASE WHEN configuration.with_receipt_costing_parameters THEN receipt_costing_parameters ELSE '{{}}'::jsonb END AS receipt_costing_parameters, - CASE WHEN configuration.with_receipt_fee_summary THEN receipt_fee_summary ELSE '{{}}'::jsonb END AS receipt_fee_summary, - CASE WHEN configuration.with_receipt_next_epoch THEN receipt_next_epoch ELSE '{{}}'::jsonb END AS receipt_next_epoch, - CASE WHEN configuration.with_receipt_output THEN receipt_output ELSE '{{}}'::jsonb END AS receipt_output, - CASE WHEN configuration.with_receipt_state_updates THEN receipt_state_updates ELSE '{{}}'::jsonb END AS receipt_state_updates, - CASE WHEN configuration.with_receipt_events THEN receipt_event_emitters ELSE '{{}}'::jsonb[] END AS receipt_event_emitters, - CASE WHEN configuration.with_receipt_events THEN receipt_event_names ELSE '{{}}'::text[] END AS receipt_event_names, - CASE WHEN configuration.with_receipt_events THEN receipt_event_sbors ELSE '{{}}'::bytea[] END AS receipt_event_sbors, - CASE WHEN configuration.with_receipt_events THEN receipt_event_schema_entity_ids ELSE '{{}}'::bigint[] END AS receipt_event_schema_entity_ids, - CASE WHEN configuration.with_receipt_events THEN receipt_event_schema_hashes ELSE '{{}}'::bytea[] END AS receipt_event_schema_hashes, - CASE WHEN configuration.with_receipt_events THEN receipt_event_type_indexes ELSE '{{}}'::bigint[] END AS receipt_event_type_indexes, - CASE WHEN configuration.with_receipt_events THEN receipt_event_sbor_type_kinds ELSE '{{}}'::sbor_type_kind[] END AS receipt_event_sbor_type_kinds, - CASE WHEN configuration.with_balance_changes THEN balance_changes ELSE '{{}}'::jsonb END AS balance_changes, - CASE WHEN configuration.with_manifest_instructions THEN manifest_instructions ELSE ''::text END AS manifest_instructions -FROM ledger_transactions, configuration -WHERE state_version = ANY({transactionStateVersions})") - .AnnotateMetricName("GetTransactions") - .ToListAsync(token); + lt.state_version, + epoch, + round_in_epoch, + CAST(fee_paid AS TEXT), + affected_global_entities, + round_timestamp, + receipt_status, + receipt_fee_source, + receipt_fee_destination, + receipt_error_message, + discriminator, + payload_hash, + intent_hash, + message, + manifest_classes, + CASE WHEN vars.with_raw_payload THEN raw_payload END AS raw_payload, + CASE WHEN vars.with_receipt_costing_parameters THEN receipt_costing_parameters END AS receipt_costing_parameters, + CASE WHEN vars.with_receipt_fee_summary THEN receipt_fee_summary END AS receipt_fee_summary, + CASE WHEN vars.with_receipt_next_epoch THEN receipt_next_epoch END AS receipt_next_epoch, + CASE WHEN vars.with_receipt_output THEN receipt_output END AS receipt_output, + CASE WHEN vars.with_receipt_state_updates THEN receipt_state_updates END AS receipt_state_updates, + CASE WHEN vars.with_balance_changes THEN balance_changes END AS balance_changes, + CASE WHEN vars.with_manifest_instructions THEN manifest_instructions END AS manifest_instructions, + CASE WHEN vars.with_receipt_events THEN lte.receipt_event_emitters END AS ReceiptEventEmitters, + CASE WHEN vars.with_receipt_events THEN lte.receipt_event_names END AS ReceiptEventNames, + CASE WHEN vars.with_receipt_events THEN lte.receipt_event_sbors END AS ReceiptEventSbors, + CASE WHEN vars.with_receipt_events THEN lte.receipt_event_schema_entity_ids END AS ReceiptEventSchemaEntityIds, + CASE WHEN vars.with_receipt_events THEN lte.receipt_event_schema_hashes END AS ReceiptEventSchemaHashes, + CASE WHEN vars.with_receipt_events THEN lte.receipt_event_type_indexes END AS ReceiptEventTypeIndexes, + CASE WHEN vars.with_receipt_events THEN lte.receipt_event_sbor_type_kinds END AS ReceiptEventSborTypeKinds +FROM vars +CROSS JOIN ledger_transactions lt +LEFT JOIN ledger_transaction_events lte ON vars.with_receipt_events AND lte.state_version = lt.state_version +WHERE lt.state_version = ANY(vars.transaction_state_versions)", + new + { + includeRawHex = optIns.RawHex, + includeReceiptOutput = optIns.ReceiptOutput, + includeReceiptStateChanges = optIns.ReceiptStateChanges, + includeReceiptEvents = optIns.ReceiptEvents, + includeBalanceChanges = optIns.BalanceChanges, + includeManifestInstructions = optIns.ManifestInstructions, + transactionStateVersions = transactionStateVersions, + }); + + var transactions = (await _dapperWrapper.QueryAsync( + _dbContext.Database.GetDbConnection(), + cd, + (transaction, events) => + { + if (events == null) + { + return transaction; + } - var entityIdToAddressMap = await GetEntityAddresses(transactions.SelectMany(x => x.AffectedGlobalEntities).ToHashSet().ToList(), token); + var mappedEvents = events + .ReceiptEventEmitters + .Select( + (_, i) => new ReceiptEvent( + events.ReceiptEventNames[i], + events.ReceiptEventEmitters[i], + events.ReceiptEventSbors[i], + events.ReceiptEventSchemaEntityIds[i], + events.ReceiptEventSchemaHashes[i], + events.ReceiptEventTypeIndexes[i], + events.ReceiptEventSborTypeKinds[i])) + .ToList(); + + transaction.Events = mappedEvents; + + return transaction; + }, + "ReceiptEventEmitters", + "GetTransactions")).ToList(); + + var entityIdToAddressMap = await _entityQuerier.ResolveEntityAddresses(transactions.SelectMany(x => x.AffectedGlobalEntities).ToHashSet().ToList(), token); var schemaLookups = transactions - .SelectMany(x => x.EngineReceipt.Events.GetEventLookups()) + .SelectMany(x => x.Events) + .Select(x => new TransactionReceiptEventLookup(x.EntityId, x.SchemaHash)) .ToHashSet(); Dictionary schemas = new Dictionary(); @@ -559,7 +665,8 @@ WITH configuration AS ( schemas = await _dbContext .SchemaEntryDefinition - .FromSqlInterpolated($@" + .FromSqlInterpolated( + $@" SELECT * FROM schema_entry_definition WHERE (entity_id, schema_hash) IN (SELECT UNNEST({entityIds}), UNNEST({schemaHashes}))") @@ -594,7 +701,7 @@ FROM schema_entry_definition { List events = new List(); - foreach (var @event in transaction.EngineReceipt.Events.GetEvents()) + foreach (var @event in transaction.Events) { if (!schemas.TryGetValue(new SchemaLookup(@event.EntityId, @event.SchemaHash), out var schema)) { @@ -611,34 +718,4 @@ FROM schema_entry_definition return mappedTransactions; } - - private async Task> GetEntityIds(List addresses, CancellationToken token = default) - { - if (!addresses.Any()) - { - return new Dictionary(); - } - - return await _dbContext - .Entities - .Where(e => addresses.Contains(e.Address)) - .Select(e => new { e.Id, e.Address }) - .AnnotateMetricName() - .ToDictionaryAsync(e => e.Address.ToString(), e => e.Id, token); - } - - private async Task> GetEntityAddresses(List entityIds, CancellationToken token = default) - { - if (!entityIds.Any()) - { - return new Dictionary(); - } - - return await _dbContext - .Entities - .Where(e => entityIds.Contains(e.Id)) - .Select(e => new { e.Id, e.Address }) - .AnnotateMetricName() - .ToDictionaryAsync(e => e.Id, e => e.Address.ToString(), token); - } } From 46f3ef7c05c1dcd4f38b6468af0153a69806d570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Tue, 1 Oct 2024 15:23:00 +0200 Subject: [PATCH 3/5] regenerate migration. --- ... 20241001131441_InitialCreate.Designer.cs} | 6 +- ...ate.cs => 20241001131441_InitialCreate.cs} | 0 .../Migrations/IdempotentApplyMigrations.sql | 268 +++++++++--------- 3 files changed, 135 insertions(+), 139 deletions(-) rename src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/{20240925082442_InitialCreate.Designer.cs => 20241001131441_InitialCreate.Designer.cs} (99%) rename src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/{20240925082442_InitialCreate.cs => 20241001131441_InitialCreate.cs} (100%) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240925082442_InitialCreate.Designer.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20241001131441_InitialCreate.Designer.cs similarity index 99% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240925082442_InitialCreate.Designer.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20241001131441_InitialCreate.Designer.cs index 5561681ce..cc823201d 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240925082442_InitialCreate.Designer.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20241001131441_InitialCreate.Designer.cs @@ -81,11 +81,7 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.Migrations { [DbContext(typeof(MigrationsDbContext))] -<<<<<<<< HEAD:src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20241001130214_InitialCreate.Designer.cs - [Migration("20241001130214_InitialCreate")] -======== - [Migration("20240925082442_InitialCreate")] ->>>>>>>> a54c65ae (add config parameters to allow reducing ledger transactions table size.):src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240925082442_InitialCreate.Designer.cs + [Migration("20241001131441_InitialCreate")] partial class InitialCreate { /// diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240925082442_InitialCreate.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20241001131441_InitialCreate.cs similarity index 100% rename from src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20240925082442_InitialCreate.cs rename to src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/20241001131441_InitialCreate.cs diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/IdempotentApplyMigrations.sql b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/IdempotentApplyMigrations.sql index 7a9160072..d6cc92b94 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/IdempotentApplyMigrations.sql +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/IdempotentApplyMigrations.sql @@ -9,7 +9,7 @@ START TRANSACTION; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TYPE account_default_deposit_rule AS ENUM ('accept', 'reject', 'allow_existing'); CREATE TYPE account_resource_preference_rule AS ENUM ('allowed', 'disallowed'); CREATE TYPE authorized_depositor_badge_type AS ENUM ('resource', 'non_fungible'); @@ -37,7 +37,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE account_authorized_depositor_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -50,7 +50,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE account_authorized_depositor_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -66,7 +66,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE account_default_deposit_rule_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -79,7 +79,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE account_locker_entry_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -93,7 +93,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE account_locker_entry_resource_vault_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -107,7 +107,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE account_locker_entry_touch_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -119,7 +119,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE account_resource_preference_rule_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -132,7 +132,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE account_resource_preference_rule_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -147,7 +147,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE component_method_royalty_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -160,7 +160,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE component_method_royalty_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -175,7 +175,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE entities ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -201,7 +201,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE entity_metadata_entry_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -214,7 +214,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE entity_metadata_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -229,7 +229,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE entity_metadata_totals_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -243,7 +243,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE entity_resource_balance_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -257,7 +257,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE entity_resource_entry_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -271,7 +271,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE entity_resource_totals_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -286,7 +286,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE entity_resource_vault_entry_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -300,7 +300,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE entity_resource_vault_totals_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -314,7 +314,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE entity_role_assignments_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -328,7 +328,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE entity_role_assignments_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -344,7 +344,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE entity_role_assignments_owner_role_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -357,7 +357,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE key_value_store_entry_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -370,7 +370,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE key_value_store_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -385,7 +385,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE key_value_store_schema_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -405,7 +405,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE key_value_store_totals_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -419,7 +419,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE ledger_transaction_events ( state_version bigint NOT NULL, receipt_event_emitters jsonb[] NOT NULL, @@ -436,7 +436,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE ledger_transaction_markers ( id bigint GENERATED BY DEFAULT AS IDENTITY, state_version bigint NOT NULL, @@ -456,7 +456,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE ledger_transactions ( state_version bigint NOT NULL, epoch bigint NOT NULL, @@ -497,7 +497,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE non_fungible_id_data_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -512,7 +512,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE non_fungible_id_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -525,7 +525,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE non_fungible_id_location_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -538,7 +538,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE non_fungible_schema_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -554,7 +554,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE non_fungible_vault_entry_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -567,7 +567,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE non_fungible_vault_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -580,7 +580,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE package_blueprint_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -593,7 +593,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE package_blueprint_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -613,7 +613,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE package_code_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -626,7 +626,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE package_code_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -642,7 +642,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE pending_transactions ( id bigint GENERATED BY DEFAULT AS IDENTITY, payload_hash text NOT NULL, @@ -670,7 +670,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE resource_entity_supply_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -685,7 +685,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE resource_holders ( id bigint GENERATED BY DEFAULT AS IDENTITY, entity_id bigint NOT NULL, @@ -699,7 +699,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE schema_entry_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -712,7 +712,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE schema_entry_definition ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -726,7 +726,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE state_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -745,7 +745,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE unverified_standard_metadata_aggregate_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -758,7 +758,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE unverified_standard_metadata_entry_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -775,7 +775,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE validator_cumulative_emission_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -791,7 +791,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE validator_public_key_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -805,7 +805,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE vault_balance_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -818,7 +818,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE pending_transaction_payloads ( id bigint GENERATED BY DEFAULT AS IDENTITY, pending_transaction_id bigint, @@ -831,7 +831,7 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE TABLE validator_active_set_history ( id bigint GENERATED BY DEFAULT AS IDENTITY, from_state_version bigint NOT NULL, @@ -846,555 +846,555 @@ END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_account_authorized_depositor_aggregate_history_account_enti~" ON account_authorized_depositor_aggregate_history (account_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_account_authorized_depositor_entry_history_account_entity_~1" ON account_authorized_depositor_entry_history (account_entity_id, resource_entity_id, non_fungible_id, from_state_version) WHERE discriminator = 'non_fungible'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_account_authorized_depositor_entry_history_account_entity_~2" ON account_authorized_depositor_entry_history (account_entity_id, resource_entity_id, from_state_version) WHERE discriminator = 'resource'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_account_authorized_depositor_entry_history_account_entity_i~" ON account_authorized_depositor_entry_history (account_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_account_default_deposit_rule_history_account_entity_id_from~" ON account_default_deposit_rule_history (account_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE UNIQUE INDEX "IX_account_locker_entry_definition_account_locker_entity_id_ac~" ON account_locker_entry_definition (account_locker_entity_id, account_entity_id); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN - CREATE UNIQUE INDEX "IX_account_locker_entry_resource_vault_definition_account_lock~" ON account_locker_entry_resource_vault_definition (account_locker_definition_id, from_state_version); + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN + CREATE INDEX "IX_account_locker_entry_resource_vault_definition_account_lock~" ON account_locker_entry_resource_vault_definition (account_locker_definition_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_account_locker_entry_touch_history_account_locker_definitio~" ON account_locker_entry_touch_history (account_locker_definition_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_account_resource_preference_rule_aggregate_history_account_~" ON account_resource_preference_rule_aggregate_history (account_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_account_resource_preference_rule_entry_history_account_enti~" ON account_resource_preference_rule_entry_history (account_entity_id, resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_component_method_royalty_aggregate_history_entity_id_from_s~" ON component_method_royalty_aggregate_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_component_method_royalty_entry_history_entity_id_from_state~" ON component_method_royalty_entry_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_component_method_royalty_entry_history_entity_id_method_nam~" ON component_method_royalty_entry_history (entity_id, method_name, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE UNIQUE INDEX "IX_entities_address" ON entities (address); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_entities_from_state_version" ON entities (from_state_version) WHERE discriminator = 'global_validator'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_entity_metadata_entry_definition_entity_id_from_state_versi~" ON entity_metadata_entry_definition (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_entity_metadata_entry_definition_entity_id_key" ON entity_metadata_entry_definition (entity_id, key); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_entity_metadata_entry_history_entity_metadata_entry_definit~" ON entity_metadata_entry_history (entity_metadata_entry_definition_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_entity_metadata_totals_history_entity_id_from_state_version" ON entity_metadata_totals_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_entity_resource_balance_history_entity_id_resource_entity_i~" ON entity_resource_balance_history (entity_id, resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_entity_resource_entry_definition_entity_id_from_state_versi~" ON entity_resource_entry_definition (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_entity_resource_entry_definition_fungibles" ON entity_resource_entry_definition (entity_id, from_state_version) WHERE resource_type = 'fungible'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_entity_resource_entry_definition_non_fungibles" ON entity_resource_entry_definition (entity_id, from_state_version) WHERE resource_type = 'non_fungible'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_entity_resource_totals_history_entity_id_from_state_version" ON entity_resource_totals_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_entity_resource_vault_entry_definition_entity_id_resource_e~" ON entity_resource_vault_entry_definition (entity_id, resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_entity_resource_vault_totals_history_entity_id_resource_ent~" ON entity_resource_vault_totals_history (entity_id, resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_entity_role_assignments_aggregate_history_entity_id_from_st~" ON entity_role_assignments_aggregate_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_entity_role_assignments_entry_history_entity_id_key_role_ke~" ON entity_role_assignments_entry_history (entity_id, key_role, key_module, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_entity_role_assignments_owner_role_history_entity_id_from_s~" ON entity_role_assignments_owner_role_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_key_value_store_entry_definition_key_value_store_entity_id_~" ON key_value_store_entry_definition (key_value_store_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_key_value_store_entry_definition_key_value_store_entity_id~1" ON key_value_store_entry_definition (key_value_store_entity_id, key); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_key_value_store_entry_history_key_value_store_entry_definit~" ON key_value_store_entry_history (key_value_store_entry_definition_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_key_value_store_schema_history_key_value_store_entity_id_fr~" ON key_value_store_schema_history (key_value_store_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_key_value_store_totals_history_entity_id_from_state_version" ON key_value_store_totals_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_entity_id_state_version" ON ledger_transaction_markers (entity_id, state_version) WHERE discriminator = 'event_global_emitter'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_entity_id_state_version1" ON ledger_transaction_markers (entity_id, state_version) WHERE discriminator = 'affected_global_entity'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_event_type_entity_id_state_versi~" ON ledger_transaction_markers (event_type, entity_id, state_version) WHERE discriminator = 'event'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_manifest_class" ON ledger_transaction_markers (manifest_class, state_version) WHERE discriminator = 'manifest_class'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_manifest_class_is_most_specific" ON ledger_transaction_markers (manifest_class, state_version) WHERE discriminator = 'manifest_class' and is_most_specific = true; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_operation_type_entity_id_state_v~" ON ledger_transaction_markers (operation_type, entity_id, state_version) WHERE discriminator = 'manifest_address'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_origin_type_state_version" ON ledger_transaction_markers (origin_type, state_version) WHERE discriminator = 'origin'; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_ledger_transaction_markers_state_version" ON ledger_transaction_markers (state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE UNIQUE INDEX "IX_ledger_transactions_epoch_round_in_epoch" ON ledger_transactions (epoch, round_in_epoch) WHERE index_in_round = 0; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_ledger_transactions_intent_hash" ON ledger_transactions USING hash (intent_hash) WHERE intent_hash IS NOT NULL; END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_ledger_transactions_round_timestamp" ON ledger_transactions (round_timestamp); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_non_fungible_id_data_history_non_fungible_id_definition_id_~" ON non_fungible_id_data_history (non_fungible_id_definition_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_non_fungible_id_definition_non_fungible_resource_entity_id_~" ON non_fungible_id_definition (non_fungible_resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE UNIQUE INDEX "IX_non_fungible_id_definition_non_fungible_resource_entity_id~1" ON non_fungible_id_definition (non_fungible_resource_entity_id, non_fungible_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_non_fungible_id_location_history_non_fungible_id_definition~" ON non_fungible_id_location_history (non_fungible_id_definition_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_non_fungible_schema_history_resource_entity_id_from_state_v~" ON non_fungible_schema_history (resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_non_fungible_vault_entry_definition_vault_entity_id_from_st~" ON non_fungible_vault_entry_definition (vault_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_non_fungible_vault_entry_history_non_fungible_vault_entry_d~" ON non_fungible_vault_entry_history (non_fungible_vault_entry_definition_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_package_blueprint_aggregate_history_package_entity_id_from_~" ON package_blueprint_aggregate_history (package_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_package_blueprint_history_package_entity_id_from_state_vers~" ON package_blueprint_history (package_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_package_blueprint_history_package_entity_id_name_version_fr~" ON package_blueprint_history (package_entity_id, name, version, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_package_code_aggregate_history_package_entity_id_from_state~" ON package_code_aggregate_history (package_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_package_code_history_package_entity_id_from_state_version" ON package_code_history (package_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE UNIQUE INDEX "IX_pending_transaction_payloads_pending_transaction_id" ON pending_transaction_payloads (pending_transaction_id); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_pending_transactions_first_submitted_to_gateway_timestamp" ON pending_transactions (first_submitted_to_gateway_timestamp); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_pending_transactions_intent_hash" ON pending_transactions (intent_hash); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE UNIQUE INDEX "IX_pending_transactions_payload_hash" ON pending_transactions (payload_hash); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_pending_transactions_resubmit_from_timestamp" ON pending_transactions (resubmit_from_timestamp); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_resource_entity_supply_history_resource_entity_id_from_stat~" ON resource_entity_supply_history (resource_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE UNIQUE INDEX "IX_resource_holders_entity_id_resource_entity_id" ON resource_holders (entity_id, resource_entity_id); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_resource_holders_entity_id_resource_entity_id_balance" ON resource_holders (entity_id, resource_entity_id, balance); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_schema_entry_aggregate_history_entity_id_from_state_version" ON schema_entry_aggregate_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_schema_entry_definition_entity_id_schema_hash" ON schema_entry_definition (entity_id, schema_hash); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_state_history_entity_id_from_state_version" ON state_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_unverified_standard_metadata_aggregate_history_entity_id_fr~" ON unverified_standard_metadata_aggregate_history (entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_unverified_standard_metadata_entry_history_entity_id_discri~" ON unverified_standard_metadata_entry_history (entity_id, discriminator, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_validator_active_set_history_epoch" ON validator_active_set_history (epoch); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_validator_active_set_history_from_state_version" ON validator_active_set_history (from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_validator_active_set_history_validator_public_key_history_id" ON validator_active_set_history (validator_public_key_history_id); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_validator_cumulative_emission_history_validator_entity_id_e~" ON validator_cumulative_emission_history (validator_entity_id, epoch_number); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_validator_cumulative_emission_history_validator_entity_id_f~" ON validator_cumulative_emission_history (validator_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_validator_public_key_history_validator_entity_id_from_state~" ON validator_public_key_history (validator_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_validator_public_key_history_validator_entity_id_key_type_k~" ON validator_public_key_history (validator_entity_id, key_type, key); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN CREATE INDEX "IX_vault_balance_history_vault_entity_id_from_state_version" ON vault_balance_history (vault_entity_id, from_state_version); END IF; END $EF$; DO $EF$ BEGIN - IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20240925082442_InitialCreate') THEN + IF NOT EXISTS(SELECT 1 FROM "__EFMigrationsHistory" WHERE "MigrationId" = '20241001131441_InitialCreate') THEN INSERT INTO "__EFMigrationsHistory" ("MigrationId", "ProductVersion") - VALUES ('20240925082442_InitialCreate', '8.0.2'); + VALUES ('20241001131441_InitialCreate', '8.0.2'); END IF; END $EF$; COMMIT; From 8f2bfe33d6601a1a31eb1cfa74f0aa0e435aec12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Thu, 3 Oct 2024 15:39:43 +0200 Subject: [PATCH 4/5] cleanup. --- CHANGELOG.md | 12 ++++++------ docs/configuration.md | 10 ++++++++++ .../DepositPreValidationQuerier.cs | 1 - .../Services/TransactionMapper.cs | 2 +- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cb311e83..149d3a2c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,12 +28,12 @@ Release built: _not released yet_ > - `/state/entity/page/non-fungible-vaults/` (when using `non_fungible_include_nfids` opt-in) ### What’s new? -- New configuration options `DataAggregator__Storage__StoreTransactionReceiptEvents`, and `DataAggregator__Storage__StoreReceiptStateUpdates` for data aggregator to configure if transaction's receipt events and receipt state updates should be stored in the database. It is meant to be used by gateway runners who want to reduce their database size. Keep in mind that when disabled `/stream/transactions` and `/stream/transactions` will not return that data. - - Possible values: - - `StoreForAllTransactions` (default) - will store data for all transactions. - - `StoryOnlyForUserTransactionsAndEpochChanges` - will store data for user transactions and transactions that resulted in epoch change. - - `StoreOnlyForUserTransactions` - will store data only for user transactions. - - `DoNotStore` - will not store any data. +- New configuration options `DataAggregator__Storage__StoreTransactionReceiptEvents`, and `DataAggregator__Storage__StoreReceiptStateUpdates` for the data aggregator to configure if a transaction's receipt events and receipt state updates should be stored in the database. It is meant to be used by gateway runners who want to reduce their database size. Keep in mind that when disabled, the corresponding properties will be missing on a response from both the `/stream/transactions` and the `/transaction/committed-details` endpoints. You can save significant space by using `StoryOnlyForUserTransactionsAndEpochChanges` and only excluding round change transactions, which aren't typically read from the `/stream/transactions` endpoint. + - Possible values: + - `StoreForAllTransactions` (default) - will store data for all transactions. + - `StoryOnlyForUserTransactionsAndEpochChanges` - will store data for user transactions and transactions that resulted in epoch change. + - `StoreOnlyForUserTransactions` - will store data only for user transactions. + - `DoNotStore` - will not store any data. ### Bug fixes - Added missing `total_count` property to `/state/validators/list` response. diff --git a/docs/configuration.md b/docs/configuration.md index 450285f00..05787e10a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -75,7 +75,17 @@ The Network Gateway services can be configured in line with the [configuration i #### Storage - `DataAggregator__Storage__StoreTransactionReceiptEvents` (type: `enum`, default value: `StoreForAllTransactions`) - controls if data aggregator should store transaction receipt events in database. + - Possible values: + - `StoreForAllTransactions` (default) - will store data for all transactions. + - `StoryOnlyForUserTransactionsAndEpochChanges` - will store data for user transactions and transactions that resulted in epoch change. + - `StoreOnlyForUserTransactions` - will store data only for user transactions. + - `DoNotStore` - will not store any data. - `DataAggregator__Storage__StoreReceiptStateUpdates` (type: `enum`, default value: `StoreForAllTransactions`) - controls if data aggregator should store transaction receipt state updates in database. + - Possible values: + - `StoreForAllTransactions` (default) - will store data for all transactions. + - `StoryOnlyForUserTransactionsAndEpochChanges` - will store data for user transactions and transactions that resulted in epoch change. + - `StoreOnlyForUserTransactions` - will store data only for user transactions. + - `DoNotStore` - will not store any data. #### Monitoring `DataAggregator__Monitoring__StartupGracePeriodSeconds` (type: `int`, default value: `10`) - duration (seconds) of start-up grace period for Data Aggregator. diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountDepositSettings/DepositPreValidationQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountDepositSettings/DepositPreValidationQuerier.cs index d9744a189..353df9e15 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountDepositSettings/DepositPreValidationQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountDepositSettings/DepositPreValidationQuerier.cs @@ -110,7 +110,6 @@ public DepositPreValidationQuerier( CancellationToken token = default ) { - var accountEntity = await _entityQuerier.GetNonVirtualEntity(accountAddress, ledgerState, token); var xrdResourceAddress = (await _networkConfigurationProvider.GetNetworkConfiguration(token)).WellKnownAddresses.Xrd; var accountEntity = await _entityQuerier.GetEntity(accountAddress, ledgerState, token); if (accountEntity is VirtualAccountComponentEntity) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionMapper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionMapper.cs index 5049d960c..6ceab01d5 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionMapper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionMapper.cs @@ -109,7 +109,7 @@ internal static GatewayApiSdk.Model.CommittedTransactionInfo ToGatewayModel( FeeSource = optIns.ReceiptFeeSource && lt.ReceiptFeeSource != null ? new JRaw(lt.ReceiptFeeSource) : null, CostingParameters = optIns.ReceiptCostingParameters ? new JRaw(lt.ReceiptCostingParameters) : null, NextEpoch = lt.ReceiptNextEpoch != null ? new JRaw(lt.ReceiptNextEpoch) : null, - StateUpdates = optIns.ReceiptStateChanges ? new JRaw(lt.ReceiptStateUpdates) : null, + StateUpdates = optIns.ReceiptStateChanges && lt.ReceiptStateUpdates != null ? new JRaw(lt.ReceiptStateUpdates) : null, Events = optIns.ReceiptEvents ? events?.Select(x => new GatewayApiSdk.Model.EventsItem(x.Name, new JRaw(x.Emitter), x.Data)).ToList() : null, }; From 8cbaa733609f264db0dec4a8675589cbf4c0df59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Pawelec?= Date: Mon, 7 Oct 2024 11:26:41 +0200 Subject: [PATCH 5/5] move parameters before sql query. Make sure sql string is properly detected as SQL in IDE. --- .../GatewayModelExtensions.cs | 4 - .../LedgerExtension/ReadHelper.cs | 3 +- .../LedgerExtension/WriteHelper.cs | 106 +++---- .../MigrationsDbContextModelSnapshot.cs | 3 - .../Models/Entity.cs | 1 - .../LedgerTransactions/LedgerTransaction.cs | 2 - ...ityFungibleResourcesAndVaultsPagedQuery.cs | 35 +-- ...NonFungibleResourcesAndVaultsPagedQuery.cs | 35 +-- .../Queries/KeyValueStoreQueries.cs | 55 ++-- .../Queries/MetadataMultiLookupQuery.cs | 17 +- .../Queries/MetadataPagedQuery.cs | 25 +- .../Queries/NonFungibleResourceQueries.cs | 86 +++--- .../NonFungibleVaultContentsPagedQuery.cs | 25 +- .../Queries/PackageQueries.cs | 37 ++- .../AccountStateQuerier.cs | 37 +-- .../DepositPreValidationQuerier.cs | 35 ++- .../Services/AccountLockerQuerier.cs | 51 ++-- .../Services/DapperExtensions.cs | 85 ++++++ .../Services/EntityStateQuerier.cs | 84 ++++-- .../Services/NativeResourceDetailsResolver.cs | 101 ++++--- .../Services/ResourceHoldersQuerier.cs | 19 +- .../Services/StandardMetadataResolver.cs | 31 +- .../Services/TransactionDetailsQuery.cs | 224 ++++++++++++++ .../Services/TransactionMapper.cs | 146 --------- .../Services/TransactionQuerier.cs | 276 +++++++----------- 25 files changed, 859 insertions(+), 664 deletions(-) create mode 100644 src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperExtensions.cs create mode 100644 src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionDetailsQuery.cs delete mode 100644 src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionMapper.cs diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayModelExtensions.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayModelExtensions.cs index 9ff1c734c..7e28c8d35 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayModelExtensions.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/GatewayModelExtensions.cs @@ -69,20 +69,16 @@ using RadixDlt.NetworkGateway.Abstractions.Model; using RadixDlt.NetworkGateway.Abstractions.StandardMetadata; using RadixDlt.NetworkGateway.PostgresIntegration.Models; -using RadixDlt.NetworkGateway.PostgresIntegration.Queries; using RadixDlt.NetworkGateway.PostgresIntegration.Queries.CustomTypes; -using RadixDlt.NetworkGateway.PostgresIntegration.Services; using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using CoreModel = RadixDlt.CoreApiSdk.Model; using GatewayModel = RadixDlt.NetworkGateway.GatewayApiSdk.Model; -using LedgerTransaction = RadixDlt.NetworkGateway.PostgresIntegration.Models.LedgerTransaction; using NonFungibleIdType = RadixDlt.NetworkGateway.Abstractions.Model.NonFungibleIdType; using PublicKeyType = RadixDlt.NetworkGateway.Abstractions.Model.PublicKeyType; using ToolkitModel = RadixEngineToolkit; -using UserLedgerTransaction = RadixDlt.NetworkGateway.PostgresIntegration.Models.UserLedgerTransaction; namespace RadixDlt.NetworkGateway.PostgresIntegration; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs index feb86f71c..be75f3c64 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/ReadHelper.cs @@ -70,6 +70,7 @@ using RadixDlt.NetworkGateway.Abstractions.Extensions; using RadixDlt.NetworkGateway.DataAggregator.Services; using RadixDlt.NetworkGateway.PostgresIntegration.Models; +using RadixDlt.NetworkGateway.PostgresIntegration.Services; using System; using System.Collections.Generic; using System.Diagnostics; @@ -142,7 +143,7 @@ FROM entities public async Task LoadSequences(CancellationToken token) { var sw = Stopwatch.GetTimestamp(); - var cd = new CommandDefinition( + var cd = DapperExtensions.CreateCommandDefinition( commandText: @" SELECT nextval('account_locker_entry_definition_id_seq') AS AccountLockerEntryDefinitionSequence, diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/WriteHelper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/WriteHelper.cs index e88f5cd4d..16c3d8158 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/WriteHelper.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/LedgerExtension/WriteHelper.cs @@ -71,7 +71,7 @@ using RadixDlt.NetworkGateway.Abstractions.Model; using RadixDlt.NetworkGateway.DataAggregator.Services; using RadixDlt.NetworkGateway.PostgresIntegration.Models; -using RadixDlt.NetworkGateway.PostgresIntegration.Utils; +using RadixDlt.NetworkGateway.PostgresIntegration.Services; using System; using System.Collections.Generic; using System.Diagnostics; @@ -200,7 +200,58 @@ public async Task UpdateSequences(SequencesHolder sequences, CancellationToken t { var sw = Stopwatch.GetTimestamp(); - var cd = new CommandDefinition( + var parameters = new + { + accountLockerEntryDefinitionSequence = sequences.AccountLockerEntryDefinitionSequence, + accountLockerEntryResourceVaultDefinitionSequence = sequences.AccountLockerEntryResourceVaultDefinitionSequence, + accountLockerEntryTouchHistorySequence = sequences.AccountLockerEntryTouchHistorySequence, + accountDefaultDepositRuleHistorySequence = sequences.AccountDefaultDepositRuleHistorySequence, + accountResourcePreferenceRuleEntryHistorySequence = sequences.AccountResourcePreferenceRuleEntryHistorySequence, + accountResourcePreferenceRuleAggregateHistorySequence = sequences.AccountResourcePreferenceRuleAggregateHistorySequence, + stateHistorySequence = sequences.StateHistorySequence, + entitySequence = sequences.EntitySequence, + entityMetadataEntryHistorySequence = sequences.EntityMetadataEntryHistorySequence, + entityMetadataEntryDefinitionSequence = sequences.EntityMetadataEntryDefinitionSequence, + entityMetadataTotalsHistorySequence = sequences.EntityMetadataTotalsHistorySequence, + entityRoleAssignmentsAggregateHistorySequence = sequences.EntityRoleAssignmentsAggregateHistorySequence, + entityRoleAssignmentsEntryHistorySequence = sequences.EntityRoleAssignmentsEntryHistorySequence, + entityRoleAssignmentsOwnerRoleHistorySequence = sequences.EntityRoleAssignmentsOwnerRoleHistorySequence, + componentMethodRoyaltyEntryHistorySequence = sequences.ComponentMethodRoyaltyEntryHistorySequence, + componentMethodRoyaltyAggregateHistorySequence = sequences.ComponentMethodRoyaltyAggregateHistorySequence, + resourceEntitySupplyHistorySequence = sequences.ResourceEntitySupplyHistorySequence, + nonFungibleIdDataSequence = sequences.NonFungibleIdDefinitionSequence, + nonFungibleIdDataHistorySequence = sequences.NonFungibleIdDataHistorySequence, + nonFungibleIdLocationHistorySequence = sequences.NonFungibleIdLocationHistorySequence, + validatorPublicKeyHistorySequence = sequences.ValidatorPublicKeyHistorySequence, + validatorActiveSetHistorySequence = sequences.ValidatorActiveSetHistorySequence, + ledgerTransactionMarkerSequence = sequences.LedgerTransactionMarkerSequence, + packageBlueprintHistorySequence = sequences.PackageBlueprintHistorySequence, + packageCodeHistorySequence = sequences.PackageCodeHistorySequence, + schemaEntryDefinitionSequence = sequences.SchemaEntryDefinitionSequence, + schemaEntryAggregateHistorySequence = sequences.SchemaEntryAggregateHistorySequence, + keyValueStoreEntryDefinitionSequence = sequences.KeyValueStoreEntryDefinitionSequence, + keyValueStoreEntryHistorySequence = sequences.KeyValueStoreEntryHistorySequence, + validatorCumulativeEmissionHistorySequence = sequences.ValidatorCumulativeEmissionHistorySequence, + nonFungibleSchemaHistorySequence = sequences.NonFungibleSchemaHistorySequence, + keyValueSchemaHistorySequence = sequences.KeyValueSchemaHistorySequence, + packageBlueprintAggregateHistorySequence = sequences.PackageBlueprintAggregateHistorySequence, + packageCodeAggregateHistorySequence = sequences.PackageCodeAggregateHistorySequence, + accountAuthorizedDepositorEntryHistorySequence = sequences.AccountAuthorizedDepositorEntryHistorySequence, + accountAuthorizedDepositorAggregateHistorySequence = sequences.AccountAuthorizedDepositorAggregateHistorySequence, + unverifiedStandardMetadataAggregateHistorySequence = sequences.UnverifiedStandardMetadataAggregateHistorySequence, + unverifiedStandardMetadataEntryHistorySequence = sequences.UnverifiedStandardMetadataEntryHistorySequence, + resourceHoldersSequence = sequences.ResourceHoldersSequence, + entityResourceEntryDefinitionSequence = sequences.EntityResourceEntryDefinitionSequence, + entityResourceVaultEntryDefinitionSequence = sequences.EntityResourceVaultEntryDefinitionSequence, + entityResourceTotalsHistorySequence = sequences.EntityResourceTotalsHistorySequence, + entityResourceVaultTotalsHistorySequence = sequences.EntityResourceVaultTotalsHistorySequence, + entityResourceBalanceHistorySequence = sequences.EntityResourceBalanceHistorySequence, + vaultBalanceHistorySequence = sequences.VaultBalanceHistorySequence, + nonFungibleVaultEntryDefinitionSequence = sequences.NonFungibleVaultEntryDefinitionSequence, + nonFungibleVaultEntryHistorySequence = sequences.NonFungibleVaultEntryHistorySequence, + }; + + var cd = DapperExtensions.CreateCommandDefinition( commandText: @" SELECT setval('account_locker_entry_definition_id_seq', @accountLockerEntryDefinitionSequence), @@ -251,56 +302,7 @@ public async Task UpdateSequences(SequencesHolder sequences, CancellationToken t setval('non_fungible_vault_entry_definition_id_seq', @nonFungibleVaultEntryDefinitionSequence), setval('non_fungible_vault_entry_history_id_seq', @nonFungibleVaultEntryHistorySequence) ", - parameters: new - { - accountLockerEntryDefinitionSequence = sequences.AccountLockerEntryDefinitionSequence, - accountLockerEntryResourceVaultDefinitionSequence = sequences.AccountLockerEntryResourceVaultDefinitionSequence, - accountLockerEntryTouchHistorySequence = sequences.AccountLockerEntryTouchHistorySequence, - accountDefaultDepositRuleHistorySequence = sequences.AccountDefaultDepositRuleHistorySequence, - accountResourcePreferenceRuleEntryHistorySequence = sequences.AccountResourcePreferenceRuleEntryHistorySequence, - accountResourcePreferenceRuleAggregateHistorySequence = sequences.AccountResourcePreferenceRuleAggregateHistorySequence, - stateHistorySequence = sequences.StateHistorySequence, - entitySequence = sequences.EntitySequence, - entityMetadataEntryHistorySequence = sequences.EntityMetadataEntryHistorySequence, - entityMetadataEntryDefinitionSequence = sequences.EntityMetadataEntryDefinitionSequence, - entityMetadataTotalsHistorySequence = sequences.EntityMetadataTotalsHistorySequence, - entityRoleAssignmentsAggregateHistorySequence = sequences.EntityRoleAssignmentsAggregateHistorySequence, - entityRoleAssignmentsEntryHistorySequence = sequences.EntityRoleAssignmentsEntryHistorySequence, - entityRoleAssignmentsOwnerRoleHistorySequence = sequences.EntityRoleAssignmentsOwnerRoleHistorySequence, - componentMethodRoyaltyEntryHistorySequence = sequences.ComponentMethodRoyaltyEntryHistorySequence, - componentMethodRoyaltyAggregateHistorySequence = sequences.ComponentMethodRoyaltyAggregateHistorySequence, - resourceEntitySupplyHistorySequence = sequences.ResourceEntitySupplyHistorySequence, - nonFungibleIdDataSequence = sequences.NonFungibleIdDefinitionSequence, - nonFungibleIdDataHistorySequence = sequences.NonFungibleIdDataHistorySequence, - nonFungibleIdLocationHistorySequence = sequences.NonFungibleIdLocationHistorySequence, - validatorPublicKeyHistorySequence = sequences.ValidatorPublicKeyHistorySequence, - validatorActiveSetHistorySequence = sequences.ValidatorActiveSetHistorySequence, - ledgerTransactionMarkerSequence = sequences.LedgerTransactionMarkerSequence, - packageBlueprintHistorySequence = sequences.PackageBlueprintHistorySequence, - packageCodeHistorySequence = sequences.PackageCodeHistorySequence, - schemaEntryDefinitionSequence = sequences.SchemaEntryDefinitionSequence, - schemaEntryAggregateHistorySequence = sequences.SchemaEntryAggregateHistorySequence, - keyValueStoreEntryDefinitionSequence = sequences.KeyValueStoreEntryDefinitionSequence, - keyValueStoreEntryHistorySequence = sequences.KeyValueStoreEntryHistorySequence, - validatorCumulativeEmissionHistorySequence = sequences.ValidatorCumulativeEmissionHistorySequence, - nonFungibleSchemaHistorySequence = sequences.NonFungibleSchemaHistorySequence, - keyValueSchemaHistorySequence = sequences.KeyValueSchemaHistorySequence, - packageBlueprintAggregateHistorySequence = sequences.PackageBlueprintAggregateHistorySequence, - packageCodeAggregateHistorySequence = sequences.PackageCodeAggregateHistorySequence, - accountAuthorizedDepositorEntryHistorySequence = sequences.AccountAuthorizedDepositorEntryHistorySequence, - accountAuthorizedDepositorAggregateHistorySequence = sequences.AccountAuthorizedDepositorAggregateHistorySequence, - unverifiedStandardMetadataAggregateHistorySequence = sequences.UnverifiedStandardMetadataAggregateHistorySequence, - unverifiedStandardMetadataEntryHistorySequence = sequences.UnverifiedStandardMetadataEntryHistorySequence, - resourceHoldersSequence = sequences.ResourceHoldersSequence, - entityResourceEntryDefinitionSequence = sequences.EntityResourceEntryDefinitionSequence, - entityResourceVaultEntryDefinitionSequence = sequences.EntityResourceVaultEntryDefinitionSequence, - entityResourceTotalsHistorySequence = sequences.EntityResourceTotalsHistorySequence, - entityResourceVaultTotalsHistorySequence = sequences.EntityResourceVaultTotalsHistorySequence, - entityResourceBalanceHistorySequence = sequences.EntityResourceBalanceHistorySequence, - vaultBalanceHistorySequence = sequences.VaultBalanceHistorySequence, - nonFungibleVaultEntryDefinitionSequence = sequences.NonFungibleVaultEntryDefinitionSequence, - nonFungibleVaultEntryHistorySequence = sequences.NonFungibleVaultEntryHistorySequence, - }, + parameters, cancellationToken: token); await _connection.ExecuteAsync(cd); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs index 18d4ba901..b3f9f3ff7 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Migrations/MigrationsDbContextModelSnapshot.cs @@ -68,11 +68,8 @@ using System.Numerics; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; using RadixDlt.NetworkGateway.Abstractions.Model; using RadixDlt.NetworkGateway.Abstractions.StandardMetadata; -using RadixDlt.NetworkGateway.PostgresIntegration; using RadixDlt.NetworkGateway.PostgresIntegration.Models; #nullable disable diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/Entity.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/Entity.cs index 3215f1d10..a1778ec9e 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/Entity.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/Entity.cs @@ -64,7 +64,6 @@ using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Model; -using RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension; using RadixDlt.NetworkGateway.PostgresIntegration.LedgerExtension.Processors; using System; using System.Collections.Generic; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/LedgerTransactions/LedgerTransaction.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/LedgerTransactions/LedgerTransaction.cs index 1e151c871..cea172f90 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/LedgerTransactions/LedgerTransaction.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Models/LedgerTransactions/LedgerTransaction.cs @@ -62,11 +62,9 @@ * permissions under this License. */ -using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Model; using RadixDlt.NetworkGateway.Abstractions.Numerics; using System; -using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations.Schema; diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/EntityFungibleResourcesAndVaultsPagedQuery.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/EntityFungibleResourcesAndVaultsPagedQuery.cs index 30a2410bb..42bc68a31 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/EntityFungibleResourcesAndVaultsPagedQuery.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/EntityFungibleResourcesAndVaultsPagedQuery.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using Dapper; using Microsoft.EntityFrameworkCore; using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Model; @@ -269,7 +268,23 @@ private static async Task> Execute( throw new InvalidOperationException("Neither resource filter nor cursors can be used if executing against multiple entities."); } - var cd = new CommandDefinition( + var parameters = new + { + entityIds = entityIds.ToList(), + resourceEntityId = resourceEntityId, + fungibleResourcesPerEntity = configuration.FungibleResourcesPerEntity == 0 ? 0 : configuration.FungibleResourcesPerEntity + 1, + vaultsPerResource = configuration.VaultsPerResource == 0 ? 0 : configuration.VaultsPerResource + 1, + useVaultAggregation = configuration.VaultsPerResource > 0, + useResourceCursor = configuration.ResourceCursor is not null, + resourceCursorStateVersion = configuration.ResourceCursor?.StateVersion, + resourceCursorId = configuration.ResourceCursor?.Id, + useVaultCursor = configuration.VaultCursor is not null, + vaultCursorStateVersion = configuration.VaultCursor?.StateVersion, + vaultCursorId = configuration.VaultCursor?.Id, + atLedgerState = configuration.AtLedgerState, + }; + + var cd = DapperExtensions.CreateCommandDefinition( @" WITH variables AS ( SELECT @@ -394,21 +409,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) vault_balance_history ON var.use_vault_aggregation ORDER BY resource_definitions.cursor DESC, vault_entry_definitions.cursor DESC", - new - { - entityIds = entityIds.ToList(), - resourceEntityId = resourceEntityId, - fungibleResourcesPerEntity = configuration.FungibleResourcesPerEntity == 0 ? 0 : configuration.FungibleResourcesPerEntity + 1, - vaultsPerResource = configuration.VaultsPerResource == 0 ? 0 : configuration.VaultsPerResource + 1, - useVaultAggregation = configuration.VaultsPerResource > 0, - useResourceCursor = configuration.ResourceCursor is not null, - resourceCursorStateVersion = configuration.ResourceCursor?.StateVersion, - resourceCursorId = configuration.ResourceCursor?.Id, - useVaultCursor = configuration.VaultCursor is not null, - vaultCursorStateVersion = configuration.VaultCursor?.StateVersion, - vaultCursorId = configuration.VaultCursor?.Id, - atLedgerState = configuration.AtLedgerState, - }, + parameters, cancellationToken: token); var result = new Dictionary(); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/EntityNonFungibleResourcesAndVaultsPagedQuery.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/EntityNonFungibleResourcesAndVaultsPagedQuery.cs index 420e4f17e..3cffd2a27 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/EntityNonFungibleResourcesAndVaultsPagedQuery.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/EntityNonFungibleResourcesAndVaultsPagedQuery.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using Dapper; using Microsoft.EntityFrameworkCore; using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Model; @@ -283,7 +282,23 @@ private static async Task> Execute( throw new InvalidOperationException("Neither resource filter nor cursors can be used if executing against multiple entities."); } - var cd = new CommandDefinition( + var parameters = new + { + entityIds = entityIds.ToList(), + resourceEntityId = resourceEntityId, + nonFungibleResourcesPerEntity = configuration.NonFungibleResourcesPerEntity == 0 ? 0 : configuration.NonFungibleResourcesPerEntity + 1, + vaultsPerResource = configuration.VaultsPerResource == 0 ? 0 : configuration.VaultsPerResource + 1, + useVaultAggregation = configuration.VaultsPerResource > 0, + useResourceCursor = configuration.ResourceCursor is not null, + resourceCursorStateVersion = configuration.ResourceCursor?.StateVersion, + resourceCursorId = configuration.ResourceCursor?.Id, + useVaultCursor = configuration.VaultCursor is not null, + vaultCursorStateVersion = configuration.VaultCursor?.StateVersion, + vaultCursorId = configuration.VaultCursor?.Id, + atLedgerState = configuration.AtLedgerState, + }; + + var cd = DapperExtensions.CreateCommandDefinition( @" WITH variables AS ( SELECT @@ -409,21 +424,7 @@ ORDER BY from_state_version DESC LIMIT 1 ) vault_balance_history ON var.use_vault_aggregation ORDER BY definitions.cursor DESC, vault_entry_definition.cursor DESC", - new - { - entityIds = entityIds.ToList(), - resourceEntityId = resourceEntityId, - nonFungibleResourcesPerEntity = configuration.NonFungibleResourcesPerEntity == 0 ? 0 : configuration.NonFungibleResourcesPerEntity + 1, - vaultsPerResource = configuration.VaultsPerResource == 0 ? 0 : configuration.VaultsPerResource + 1, - useVaultAggregation = configuration.VaultsPerResource > 0, - useResourceCursor = configuration.ResourceCursor is not null, - resourceCursorStateVersion = configuration.ResourceCursor?.StateVersion, - resourceCursorId = configuration.ResourceCursor?.Id, - useVaultCursor = configuration.VaultCursor is not null, - vaultCursorStateVersion = configuration.VaultCursor?.StateVersion, - vaultCursorId = configuration.VaultCursor?.Id, - atLedgerState = configuration.AtLedgerState, - }, + parameters, cancellationToken: token); var result = new Dictionary(); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/KeyValueStoreQueries.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/KeyValueStoreQueries.cs index 45238077e..b508744dd 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/KeyValueStoreQueries.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/KeyValueStoreQueries.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using Dapper; using Microsoft.EntityFrameworkCore; using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Extensions; @@ -132,7 +131,18 @@ private readonly record struct KeyValueStoreDataResultRow( KeyValueStoreKeysQueryConfiguration queryConfiguration, CancellationToken token = default) { - var cd = new CommandDefinition( + var parameters = new + { + keyValueStoreEntityId = keyValueStoreEntity.Id, + atLedgerState = ledgerState.StateVersion, + useCursor = queryConfiguration.Cursor is not null, + cursorStateVersion = queryConfiguration.Cursor?.StateVersionBoundary ?? 0, + cursorDefinitionId = queryConfiguration.Cursor?.IdBoundary ?? 0, + definitionReadLimit = queryConfiguration.MaxDefinitionsLookupLimit, + pageLimit = queryConfiguration.PageSize + 1, + }; + + var cd = DapperExtensions.CreateCommandDefinition( commandText: @" WITH vars AS ( SELECT @@ -233,16 +243,7 @@ LIMIT vars.page_limit ) entries_per_entity ON TRUE ORDER BY entries_per_entity.cursor DESC ;", - parameters: new - { - keyValueStoreEntityId = keyValueStoreEntity.Id, - atLedgerState = ledgerState.StateVersion, - useCursor = queryConfiguration.Cursor is not null, - cursorStateVersion = queryConfiguration.Cursor?.StateVersionBoundary ?? 0, - cursorDefinitionId = queryConfiguration.Cursor?.IdBoundary ?? 0, - definitionReadLimit = queryConfiguration.MaxDefinitionsLookupLimit, - pageLimit = queryConfiguration.PageSize + 1, - }, + parameters: parameters, cancellationToken: token); var queryResult = (await dapperWrapper.QueryAsync(dbContext.Database.GetDbConnection(), cd)).ToList(); @@ -295,7 +296,14 @@ ORDER BY entries_per_entity.cursor DESC GatewayModel.LedgerState ledgerState, CancellationToken token = default) { - var cd = new CommandDefinition( + var parameters = new + { + entityId = keyValueStoreEntity.Id, + atLedgerState = ledgerState.StateVersion, + keys = keys.Distinct().Select(k => (byte[])k).ToList(), + }; + + var cd = DapperExtensions.CreateCommandDefinition( commandText: @" WITH vars AS ( SELECT @@ -339,12 +347,7 @@ LIMIT 1 ) history on true WHERE definition.key_value_store_entity_id = vars.entity_id AND definition.key = vars.key ) entries_with_definitions on TRUE", - parameters: new - { - entityId = keyValueStoreEntity.Id, - atLedgerState = ledgerState.StateVersion, - keys = keys.Distinct().Select(k => (byte[])k).ToList(), - }, + parameters: parameters, cancellationToken: token); var queryResult = await dapperWrapper.QueryAsync(dbContext.Database.GetDbConnection(), cd); @@ -385,7 +388,13 @@ internal static async Task KeyValueStoreSchemaLook GatewayModel.LedgerState ledgerState, CancellationToken token = default) { - var keyValueStoreSchemaQuery = new CommandDefinition( + var parameters = new + { + stateVersion = ledgerState.StateVersion, + entityId = keyValueStoreId, + }; + + var keyValueStoreSchemaQuery = DapperExtensions.CreateCommandDefinition( commandText: @" SELECT ksh.schema AS KeySchema, @@ -400,11 +409,7 @@ FROM key_value_store_schema_history kvssh WHERE kvssh.key_value_store_entity_id = @entityId AND kvssh.from_state_version <= @stateVersion ORDER BY kvssh.from_state_version DESC ", - parameters: new - { - stateVersion = ledgerState.StateVersion, - entityId = keyValueStoreId, - }, + parameters: parameters, cancellationToken: token); var keyValueStoreSchema = await dapperWrapper.QueryFirstOrDefaultAsync( diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/MetadataMultiLookupQuery.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/MetadataMultiLookupQuery.cs index 17078a3d8..4ebbe13b7 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/MetadataMultiLookupQuery.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/MetadataMultiLookupQuery.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using Dapper; using RadixDlt.NetworkGateway.Abstractions.Extensions; using RadixDlt.NetworkGateway.PostgresIntegration.Services; using RadixDlt.NetworkGateway.PostgresIntegration.Utils; @@ -116,7 +115,14 @@ bool FilterOut metadataKeysParameter.Add(metadataKey); } - var commandDefinition = new CommandDefinition( + var parameters = new + { + entityIds = entityIdsParameter, + metadataKeys = metadataKeysParameter, + atLedgerState = ledgerState.StateVersion, + }; + + var commandDefinition = DapperExtensions.CreateCommandDefinition( @" WITH vars AS ( SELECT @@ -162,12 +168,7 @@ LIMIT 1 WHERE definition.entity_id = vars.entity_id AND definition.key = vars.metadata_key ) entries_with_definitions on TRUE ;", - new - { - entityIds = entityIdsParameter, - metadataKeys = metadataKeysParameter, - atLedgerState = ledgerState.StateVersion, - }, + parameters, cancellationToken: token ); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/MetadataPagedQuery.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/MetadataPagedQuery.cs index c47992725..e4c83eac0 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/MetadataPagedQuery.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/MetadataPagedQuery.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using Dapper; using RadixDlt.NetworkGateway.Abstractions.Extensions; using RadixDlt.NetworkGateway.PostgresIntegration.Queries.CustomTypes; using RadixDlt.NetworkGateway.PostgresIntegration.Services; @@ -116,7 +115,18 @@ private readonly record struct QueryResultRow( return ImmutableDictionary.Empty; } - var cd = new CommandDefinition( + var parameters = new + { + entityIds = entityIds, + useCursor = queryConfiguration.Cursor is not null, + atLedgerState = ledgerState.StateVersion, + cursorStateVersion = queryConfiguration.Cursor?.StateVersionBoundary ?? 0, + cursorDefinitionId = queryConfiguration.Cursor?.IdBoundary ?? 0, + perEntityPageLimit = queryConfiguration.PageSize + 1, + perEntityDefinitionReadLimit = Math.Floor(queryConfiguration.MaxDefinitionsLookupLimit / (decimal)entityIds.Count), + }; + + var cd = DapperExtensions.CreateCommandDefinition( @" WITH vars AS ( SELECT @@ -218,16 +228,7 @@ LIMIT vars.per_entity_page_limit ) entries_per_entity ON TRUE ORDER BY entries_per_entity.cursor DESC ;", - new - { - entityIds = entityIds, - useCursor = queryConfiguration.Cursor is not null, - atLedgerState = ledgerState.StateVersion, - cursorStateVersion = queryConfiguration.Cursor?.StateVersionBoundary ?? 0, - cursorDefinitionId = queryConfiguration.Cursor?.IdBoundary ?? 0, - perEntityPageLimit = queryConfiguration.PageSize + 1, - perEntityDefinitionReadLimit = Math.Floor(queryConfiguration.MaxDefinitionsLookupLimit / (decimal)entityIds.Count), - }, + parameters, cancellationToken: token); var queryResult = (await dapperWrapper.ToListAsync(dbConnection, cd)).ToList(); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/NonFungibleResourceQueries.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/NonFungibleResourceQueries.cs index 62ebbd46d..3122fa77a 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/NonFungibleResourceQueries.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/NonFungibleResourceQueries.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using Dapper; using Microsoft.EntityFrameworkCore; using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Extensions; @@ -88,7 +87,12 @@ private record NonFungibleIdDataViewModel(string NonFungibleId, bool IsDeleted, private record NonFungibleIdLocationViewModel(string NonFungibleId, bool IsDeleted, long OwnerVaultId, EntityAddress OwnerVaultAddress, long FromStateVersion); - private record NonFungibleIdLocationVaultOwnerViewModel(long VaultId, long VaultParentAncestorId, EntityAddress VaultParentAncestorAddress, long VaultGlobalAncestorId, EntityAddress VaultGlobalAncestorAddress); + private record NonFungibleIdLocationVaultOwnerViewModel( + long VaultId, + long VaultParentAncestorId, + EntityAddress VaultParentAncestorAddress, + long VaultGlobalAncestorId, + EntityAddress VaultGlobalAncestorAddress); private record NonFungibleIdsQueryResult(long Id, long FromStateVersion, string NonFungibleId, TokenAmount TotalMinted); @@ -101,7 +105,16 @@ private record NonFungibleIdsQueryResult(long Id, long FromStateVersion, string int pageSize, CancellationToken token = default) { - var cd = new CommandDefinition( + var parameters = new + { + nonFungibleResourceEntityId = resourceEntity.Id, + stateVersion = ledgerState.StateVersion, + cursorStateVersion = cursor?.StateVersionBoundary ?? 1, + cursorId = cursor?.IdBoundary ?? 1, + limit = pageSize + 1, + }; + + var cd = DapperExtensions.CreateCommandDefinition( commandText: $@" SELECT d.id AS Id, @@ -130,14 +143,7 @@ LIMIT 1 ORDER BY (d.from_state_version, d.id) ASC LIMIT @limit ;", - parameters: new - { - nonFungibleResourceEntityId = resourceEntity.Id, - stateVersion = ledgerState.StateVersion, - cursorStateVersion = cursor?.StateVersionBoundary ?? 1, - cursorId = cursor?.IdBoundary ?? 1, - limit = pageSize + 1, - }, + parameters: parameters, cancellationToken: token); var entriesAndOneMore = (await dapperWrapper.QueryAsync(dbContext.Database.GetDbConnection(), cd)) @@ -172,7 +178,13 @@ LIMIT @limit GatewayModel.LedgerState ledgerState, CancellationToken token = default) { - var nonFungibleDataSchemaQuery = new CommandDefinition( + var parameters = new + { + stateVersion = ledgerState.StateVersion, + entityId = resourceEntity.Id, + }; + + var nonFungibleDataSchemaQuery = DapperExtensions.CreateCommandDefinition( commandText: @" SELECT sh.schema, @@ -182,11 +194,7 @@ FROM non_fungible_schema_history nfsh INNER JOIN schema_entry_definition sh ON sh.schema_hash = nfsh.schema_hash AND sh.entity_id = nfsh.schema_defining_entity_id WHERE nfsh.resource_entity_id = @entityId AND nfsh.from_state_version <= @stateVersion ORDER BY nfsh.from_state_version DESC", - parameters: new - { - stateVersion = ledgerState.StateVersion, - entityId = resourceEntity.Id, - }, + parameters: parameters, cancellationToken: token); var nonFungibleDataSchema = await dapperWrapper.QueryFirstOrDefaultAsync( @@ -200,7 +208,14 @@ FROM non_fungible_schema_history nfsh throw new UnreachableException($"No schema found for nonfungible resource: {resourceEntity.Address}"); } - var cd = new CommandDefinition( + var o = new + { + stateVersion = ledgerState.StateVersion, + entityId = resourceEntity.Id, + nonFungibleIds = nonFungibleIds, + }; + + var cd = DapperExtensions.CreateCommandDefinition( commandText: @" SELECT nfid.non_fungible_id AS NonFungibleId, md.is_deleted AS IsDeleted, md.data AS Data, md.from_state_version AS DataLastUpdatedAtStateVersion FROM non_fungible_id_definition nfid @@ -214,12 +229,7 @@ LIMIT 1 WHERE nfid.from_state_version <= @stateVersion AND nfid.non_fungible_resource_entity_id = @entityId AND nfid.non_fungible_id = ANY(@nonFungibleIds) ORDER BY nfid.from_state_version DESC ", - parameters: new - { - stateVersion = ledgerState.StateVersion, - entityId = resourceEntity.Id, - nonFungibleIds = nonFungibleIds, - }, + parameters: o, cancellationToken: token); var items = new List(); @@ -263,7 +273,14 @@ ORDER BY nfid.from_state_version DESC GatewayModel.LedgerState ledgerState, CancellationToken token = default) { - var vaultLocationsCd = new CommandDefinition( + var vaultLocationsParameters = new + { + stateVersion = ledgerState.StateVersion, + resourceEntityId = resourceEntity.Id, + nonFungibleIds = nonFungibleIds, + }; + + var vaultLocationsCd = DapperExtensions.CreateCommandDefinition( commandText: @" WITH variables (non_fungible_id) AS ( SELECT UNNEST(@nonFungibleIds) @@ -297,18 +314,18 @@ ORDER BY from_state_version DESC LIMIT 1 ) lh ON TRUE INNER JOIN entities e ON e.id = lh.vault_entity_id AND e.from_state_version <= @stateVersion", - parameters: new - { - stateVersion = ledgerState.StateVersion, - resourceEntityId = resourceEntity.Id, - nonFungibleIds = nonFungibleIds, - }, + parameters: vaultLocationsParameters, cancellationToken: token); var vaultLocationResults = (await dapperWrapper.QueryAsync(dbContext.Database.GetDbConnection(), vaultLocationsCd)) .ToList(); - var vaultAncestorsCd = new CommandDefinition( + var vaultAncestorsParameters = new + { + vaultIds = vaultLocationResults.Select(x => x.OwnerVaultId).Distinct().ToList(), + }; + + var vaultAncestorsCd = DapperExtensions.CreateCommandDefinition( commandText: @" SELECT e.id AS VaultId, @@ -320,10 +337,7 @@ FROM entities e INNER JOIN entities pae ON e.parent_ancestor_id = pae.id INNER JOIN entities gae ON e.global_ancestor_id = gae.id WHERE e.id = ANY(@vaultIds)", - parameters: new - { - vaultIds = vaultLocationResults.Select(x => x.OwnerVaultId).Distinct().ToList(), - }, + parameters: vaultAncestorsParameters, cancellationToken: token); var vaultAncestorResults = (await dapperWrapper.QueryAsync(dbContext.Database.GetDbConnection(), vaultAncestorsCd)) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/NonFungibleVaultContentsPagedQuery.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/NonFungibleVaultContentsPagedQuery.cs index d2a419b76..36e2e0faa 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/NonFungibleVaultContentsPagedQuery.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/NonFungibleVaultContentsPagedQuery.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using Dapper; using Microsoft.EntityFrameworkCore; using RadixDlt.NetworkGateway.Abstractions.Numerics; using RadixDlt.NetworkGateway.PostgresIntegration.Queries.CustomTypes; @@ -118,7 +117,18 @@ private readonly record struct QueryResultRow( throw new InvalidOperationException("Can't use cursor if executing against multiple vault entities."); } - var cd = new CommandDefinition( + var parameters = new + { + vaultEntityIds = vaultEntityIds.ToList(), + useCursor = configuration.Cursor is not null, + cursorStateVersion = configuration.Cursor?.StateVersion, + cursorId = configuration.Cursor?.Id, + atLedgerState = ledgerState.StateVersion, + perEntityDefinitionReadLimit = Math.Floor(configuration.MaxDefinitionsLookupLimit / (decimal)vaultEntityIds.Count), + perEntityPageLimit = configuration.PageSize + 1, + }; + + var cd = DapperExtensions.CreateCommandDefinition( @" WITH vars AS ( SELECT @@ -218,16 +228,7 @@ LIMIT 1 ) vault_balance ON TRUE ORDER BY entries.cursor DESC ;", - new - { - vaultEntityIds = vaultEntityIds.ToList(), - useCursor = configuration.Cursor is not null, - cursorStateVersion = configuration.Cursor?.StateVersion, - cursorId = configuration.Cursor?.Id, - atLedgerState = ledgerState.StateVersion, - perEntityDefinitionReadLimit = Math.Floor(configuration.MaxDefinitionsLookupLimit / (decimal)vaultEntityIds.Count), - perEntityPageLimit = configuration.PageSize + 1, - }, + parameters, cancellationToken: token); var queryResult = (await dapperWrapper.QueryAsync(dbContext.Database.GetDbConnection(), cd)).ToList(); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/PackageQueries.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/PackageQueries.cs index bc84b076c..855063dfd 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/PackageQueries.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Queries/PackageQueries.cs @@ -67,6 +67,7 @@ using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Model; using RadixDlt.NetworkGateway.PostgresIntegration.Models; +using RadixDlt.NetworkGateway.PostgresIntegration.Services; using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; @@ -100,7 +101,15 @@ public static async Task> Package return ImmutableDictionary.Empty; } - var cd = new CommandDefinition( + var parameters = new + { + packageEntityIds = packageEntityIds.ToList(), + startIndex = offset + 1, + endIndex = offset + limit, + stateVersion = ledgerState.StateVersion, + }; + + var cd = DapperExtensions.CreateCommandDefinition( commandText: @" WITH variables (package_entity_id) AS (SELECT UNNEST(@packageEntityIds)), blueprint_slices AS @@ -120,13 +129,7 @@ FROM blueprint_slices AS bs INNER JOIN LATERAL UNNEST(blueprint_slice) WITH ORDINALITY AS blueprint_join(id, ordinality) ON TRUE INNER JOIN package_blueprint_history pbh ON pbh.id = blueprint_join.id ORDER BY blueprint_join.ordinality ASC;", - parameters: new - { - packageEntityIds = packageEntityIds.ToList(), - startIndex = offset + 1, - endIndex = offset + limit, - stateVersion = ledgerState.StateVersion, - }, + parameters: parameters, cancellationToken: token); return (await dbContext.Database.GetDbConnection().QueryAsync(cd)) @@ -148,7 +151,15 @@ public static async Task> PackageCodeH return ImmutableDictionary.Empty; } - var cd = new CommandDefinition( + var parameters = new + { + packageEntityIds = packageEntityIds.ToList(), + startIndex = offset + 1, + endIndex = offset + limit, + stateVersion = ledgerState.StateVersion, + }; + + var cd = DapperExtensions.CreateCommandDefinition( commandText: @" WITH variables (package_entity_id) AS (SELECT UNNEST(@packageEntityIds)), code_slices AS @@ -168,13 +179,7 @@ FROM code_slices AS cs INNER JOIN LATERAL UNNEST(code_slice) WITH ORDINALITY AS code_join(id, ordinality) ON TRUE INNER JOIN package_code_history pch ON pch.id = code_join.id ORDER BY code_join.ordinality ASC;", - parameters: new - { - packageEntityIds = packageEntityIds.ToList(), - startIndex = offset + 1, - endIndex = offset + limit, - stateVersion = ledgerState.StateVersion, - }, + parameters: parameters, cancellationToken: token); return (await dbContext.Database.GetDbConnection().QueryAsync(cd)) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountDepositSettings/AccountStateQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountDepositSettings/AccountStateQuerier.cs index 0a126fd9d..e38b5ce74 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountDepositSettings/AccountStateQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountDepositSettings/AccountStateQuerier.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using Dapper; using Microsoft.EntityFrameworkCore; using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Model; @@ -115,7 +114,15 @@ public AccountStateQuerier(IDapperWrapper dapperWrapper, ReadOnlyDbContext dbCon { var accountEntity = await _entityQuerier.GetNonVirtualEntity(accountAddress, ledgerState, token); - var cd = new CommandDefinition( + var parameters = new + { + accountEntityId = accountEntity.Id, + stateVersion = ledgerState.StateVersion, + startIndex = offset + 1, + endIndex = offset + limit, + }; + + var cd = DapperExtensions.CreateCommandDefinition( commandText: @" WITH slices AS ( SELECT @@ -135,13 +142,7 @@ INNER JOIN LATERAL UNNEST(resource_preference_rules_slice) WITH ORDINALITY AS re INNER JOIN account_resource_preference_rule_entry_history arpreh ON arpreh.id = resource_preference_join.id INNER JOIN entities resource_entity on arpreh.resource_entity_id = resource_entity.id ORDER BY resource_preference_join.ordinality ASC;", - parameters: new - { - accountEntityId = accountEntity.Id, - stateVersion = ledgerState.StateVersion, - startIndex = offset + 1, - endIndex = offset + limit, - }, + parameters: parameters, cancellationToken: token); var queryResult = (await _dapperWrapper.QueryAsync(_dbContext.Database.GetDbConnection(), cd)).ToList(); @@ -174,7 +175,15 @@ INNER JOIN LATERAL UNNEST(resource_preference_rules_slice) WITH ORDINALITY AS re { var accountEntity = await _entityQuerier.GetNonVirtualEntity(accountAddress, ledgerState, token); - var cd = new CommandDefinition( + var parameters = new + { + accountEntityId = accountEntity.Id, + stateVersion = ledgerState.StateVersion, + startIndex = offset + 1, + endIndex = offset + limit, + }; + + var cd = DapperExtensions.CreateCommandDefinition( commandText: @" WITH slices AS ( SELECT @@ -197,13 +206,7 @@ INNER JOIN LATERAL UNNEST(resource_authorized_depositors_slice) WITH ORDINALITY LEFT JOIN entities resource_entity on aadeh.resource_entity_id = resource_entity.id LEFT JOIN entities non_fungible_resource_entity on aadeh.resource_entity_id = non_fungible_resource_entity.id ORDER BY resource_authorized_depositors_join.ordinality ASC;", - parameters: new - { - accountEntityId = accountEntity.Id, - stateVersion = ledgerState.StateVersion, - startIndex = offset + 1, - endIndex = offset + limit, - }, + parameters: parameters, cancellationToken: token); var queryResult = (await _dapperWrapper.QueryAsync(_dbContext.Database.GetDbConnection(), cd)).ToList(); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountDepositSettings/DepositPreValidationQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountDepositSettings/DepositPreValidationQuerier.cs index 353df9e15..27078e729 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountDepositSettings/DepositPreValidationQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountDepositSettings/DepositPreValidationQuerier.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using Dapper; using Microsoft.EntityFrameworkCore; using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Model; @@ -197,7 +196,14 @@ private async Task x.Value).ToList(), + }; + + var resourcePreferencesQuery = DapperExtensions.CreateCommandDefinition( commandText: @" SELECT arpreh.resource_entity_id AS ResourceEntityId, arpreh.account_resource_preference_rule AS AccountResourcePreferenceRule FROM account_resource_preference_rule_entry_history arpreh @@ -210,12 +216,7 @@ limit 1 ) AS arprah ON arpreh.id = ANY(arprah.entry_ids) AND arpreh.resource_entity_id = ANY(@resourceEntityIds) WHERE arpreh.account_entity_id = @accountEntityId AND arpreh.from_state_version <= @stateVersion", - parameters: new - { - accountEntityId = accountEntityId, - stateVersion = ledgerState.StateVersion, - resourceEntityIds = resourceMap.Select(x => x.Value).ToList(), - }, + parameters: parameters, cancellationToken: token); var resourcePreferencesQueryResult = @@ -234,7 +235,15 @@ private async Task IsBadgeAuthorizedDepositor( GatewayModel.LedgerState ledgerState, CancellationToken token = default) { - var authorizedDepositorQuery = new CommandDefinition( + var parameters = new + { + accountEntityId = accountEntityId, + stateVersion = ledgerState.StateVersion, + badgeResourceEntityId = badgeResourceEntityId, + nonFungibleId = nonFungibleBadgeNfid, + }; + + var authorizedDepositorQuery = DapperExtensions.CreateCommandDefinition( commandText: @" SELECT 1 FROM account_authorized_depositor_entry_history aadeh @@ -251,13 +260,7 @@ limit 1 OR (aadeh.resource_entity_id = @badgeResourceEntityId AND aadeh.non_fungible_id = @nonFungibleId AND aadeh.discriminator = 'non_fungible') ) WHERE aadeh.account_entity_id = @accountEntityId AND aadeh.from_state_version <= @stateVersion;", - parameters: new - { - accountEntityId = accountEntityId, - stateVersion = ledgerState.StateVersion, - badgeResourceEntityId = badgeResourceEntityId, - nonFungibleId = nonFungibleBadgeNfid, - }, + parameters: parameters, cancellationToken: token); var authorizedDepositorEntry = await _dapperWrapper.QueryFirstOrDefaultAsync( diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountLockerQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountLockerQuerier.cs index 0169a6b36..032dc6cb9 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountLockerQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/AccountLockerQuerier.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using Dapper; using Microsoft.EntityFrameworkCore; using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Model; @@ -87,7 +86,14 @@ internal class AccountLockerQuerier : IAccountLockerQuerier private readonly ReadOnlyDbContext _dbContext; private readonly IDapperWrapper _dapperWrapper; - private record AccountLockerVaultsResultRow(long Id, long FromStateVersion, EntityType ResourceDiscriminator, string ResourceAddress, string VaultAddress, long LastUpdatedAtStateVersion, TokenAmount? Balance); + private record AccountLockerVaultsResultRow( + long Id, + long FromStateVersion, + EntityType ResourceDiscriminator, + string ResourceAddress, + string VaultAddress, + long LastUpdatedAtStateVersion, + TokenAmount? Balance); private record AccountLockerTouchedAtResultRow(long AccountLockerEntityId, long AccountEntityId, long LastUpdatedAt); @@ -116,7 +122,16 @@ public AccountLockerQuerier(IEntityQuerier entityQuerier, ReadOnlyDbContext read throw new AccountLockerNotFoundException(pageRequest.AccountLockerAddress.LockerAddress, pageRequest.AccountLockerAddress.AccountAddress); } - var cd = new CommandDefinition( + var parameters = new + { + accountLockerDefinitionId = accountLockerEntryDefinition.Id, + stateVersion = ledgerState.StateVersion, + cursorStateVersion = pageRequest.Cursor?.StateVersionBoundary ?? long.MaxValue, + cursorId = pageRequest.Cursor?.IdBoundary ?? long.MaxValue, + limit = pageRequest.Limit + 1, + }; + + var cd = DapperExtensions.CreateCommandDefinition( commandText: @" SELECT d.id AS Id, @@ -142,14 +157,7 @@ LIMIT 1 AND (d.from_state_version, d.id) <= (@cursorStateVersion, @cursorId) ORDER BY d.from_state_version DESC, d.id DESC LIMIT @limit;", - parameters: new - { - accountLockerDefinitionId = accountLockerEntryDefinition.Id, - stateVersion = ledgerState.StateVersion, - cursorStateVersion = pageRequest.Cursor?.StateVersionBoundary ?? long.MaxValue, - cursorId = pageRequest.Cursor?.IdBoundary ?? long.MaxValue, - limit = pageRequest.Limit + 1, - }, + parameters: parameters, cancellationToken: token); var vaultsAndOneMore = (await _dapperWrapper.QueryAsync(_dbContext.Database.GetDbConnection(), cd, "vaultsAndOneMore")).ToList(); @@ -198,7 +206,10 @@ LIMIT 1 ); } - public async Task AccountLockersTouchedAt(IList accountLockers, GatewayModel.LedgerState atLedgerState, CancellationToken token = default) + public async Task AccountLockersTouchedAt( + IList accountLockers, + GatewayModel.LedgerState atLedgerState, + CancellationToken token = default) { var entityAddresses = accountLockers.SelectMany(l => new[] { l.LockerAddress, l.AccountAddress }).ToHashSet().ToList(); var entities = await _entityQuerier.GetEntities(entityAddresses, atLedgerState, token); @@ -217,7 +228,14 @@ long GetAddressOrThrow(EntityAddress input) accountLockers.Unzip(l => GetAddressOrThrow(l.LockerAddress), l => GetAddressOrThrow(l.AccountAddress), out var accountLockerEntityIds, out var accountEntityIds); - var cd = new CommandDefinition( + var parameters = new + { + accountLockerEntityIds = accountLockerEntityIds, + accountEntityIds = accountEntityIds, + stateVersion = atLedgerState.StateVersion, + }; + + var cd = DapperExtensions.CreateCommandDefinition( commandText: @" WITH variables AS ( SELECT UNNEST(@accountLockerEntityIds) AS account_locker_entity_id, UNNEST(@accountEntityIds) AS account_entity_id @@ -236,12 +254,7 @@ FROM account_locker_entry_touch_history ORDER BY from_state_version DESC LIMIT 1 ) th ON TRUE;", - parameters: new - { - accountLockerEntityIds = accountLockerEntityIds, - accountEntityIds = accountEntityIds, - stateVersion = atLedgerState.StateVersion, - }, + parameters: parameters, cancellationToken: token); var items = new List(); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperExtensions.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperExtensions.cs new file mode 100644 index 000000000..d98d6a24f --- /dev/null +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/DapperExtensions.cs @@ -0,0 +1,85 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +using Dapper; +using System.Data; +using System.Diagnostics.CodeAnalysis; +using System.Threading; + +namespace RadixDlt.NetworkGateway.PostgresIntegration.Services; + +internal static class DapperExtensions +{ + public static CommandDefinition CreateCommandDefinition( + [StringSyntax("PostgreSQL")]string commandText, + object? parameters = null, + IDbTransaction? transaction = null, + int? commandTimeout = null, + CommandType? commandType = null, + CommandFlags flags = CommandFlags.Buffered, + CancellationToken cancellationToken = default) + { + return new CommandDefinition(commandText, parameters, transaction, commandTimeout, commandType, flags, cancellationToken); + } +} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs index f084187d3..92764d7cc 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/EntityStateQuerier.cs @@ -170,26 +170,35 @@ public EntityStateQuerier( var correlatedAddresses = await PackageQueries.GetCorrelatedEntityAddresses(_dbContext, entities, packageBlueprintHistory, ledgerState, token); // top-level & royalty vaults (if opted-in) - var vaultIds = entities.OfType() + var vaultIds = entities + .OfType() .Select(x => x.Id) .ToHashSet(); if (optIns.ComponentRoyaltyVaultBalance || optIns.PackageRoyaltyVaultBalance) { - vaultIds.UnionWith(globalPersistedComponentEntities.Select(x => x.TryGetCorrelation(EntityRelationship.EntityToRoyaltyVault, out var royaltyVaultCorrelation) ? royaltyVaultCorrelation.EntityId : -1).Where(x => x > 0)); + vaultIds.UnionWith(globalPersistedComponentEntities + .Select(x => x.TryGetCorrelation(EntityRelationship.EntityToRoyaltyVault, out var royaltyVaultCorrelation) ? royaltyVaultCorrelation.EntityId : -1) + .Where(x => x > 0)); } var vaultBalances = await GetVaultBalances(vaultIds.ToArray(), ledgerState, token); - var fungibleResourcesConfiguration = new EntityFungibleResourcesAndVaultsPagedQuery.DetailsQueryConfiguration(defaultPageSize, aggregatePerVault ? defaultPageSize : 0, ledgerState.StateVersion); - var entityFungibleResources = await EntityFungibleResourcesAndVaultsPagedQuery.Details(_dbContext, _dapperWrapper, componentEntities.Select(e => e.Id).ToArray(), fungibleResourcesConfiguration, token); - var nonFungibleResourcesConfiguration = new EntityNonFungibleResourcesAndVaultsPagedQuery.DetailsQueryConfiguration(defaultPageSize, aggregatePerVault ? defaultPageSize : 0, ledgerState.StateVersion); - var entityNonFungibleResources = await EntityNonFungibleResourcesAndVaultsPagedQuery.Details(_dbContext, _dapperWrapper, componentEntities.Select(e => e.Id).ToArray(), nonFungibleResourcesConfiguration, token); - - var allEntitiesToQueryForMetadata = entityFungibleResources.Values + var fungibleResourcesConfiguration = + new EntityFungibleResourcesAndVaultsPagedQuery.DetailsQueryConfiguration(defaultPageSize, aggregatePerVault ? defaultPageSize : 0, ledgerState.StateVersion); + var entityFungibleResources = + await EntityFungibleResourcesAndVaultsPagedQuery.Details(_dbContext, _dapperWrapper, componentEntities.Select(e => e.Id).ToArray(), fungibleResourcesConfiguration, token); + var nonFungibleResourcesConfiguration = + new EntityNonFungibleResourcesAndVaultsPagedQuery.DetailsQueryConfiguration(defaultPageSize, aggregatePerVault ? defaultPageSize : 0, ledgerState.StateVersion); + var entityNonFungibleResources = + await EntityNonFungibleResourcesAndVaultsPagedQuery.Details(_dbContext, _dapperWrapper, componentEntities.Select(e => e.Id).ToArray(), nonFungibleResourcesConfiguration, token); + + var allEntitiesToQueryForMetadata = entityFungibleResources + .Values .SelectMany(x => x.FungibleResources.Values) .Select(r => r.ResourceEntityId) .Union( - entityNonFungibleResources.Values + entityNonFungibleResources + .Values .SelectMany(x => x.NonFungibleResources.Values) .Select(x => x.ResourceEntityId)) .Union(entities.Select(e => e.Id)) @@ -201,7 +210,8 @@ public EntityStateQuerier( .Select(x => x.Id) .ToArray() .Union( - entityNonFungibleResources.Values + entityNonFungibleResources + .Values .SelectMany(x => x.NonFungibleResources.Values) .SelectMany(x => x.Vaults.Values) .Select(x => x.VaultEntityId) @@ -284,7 +294,8 @@ public EntityStateQuerier( case GlobalPackageEntity pe: string? packageRoyaltyVaultBalance = null; - if (optIns.PackageRoyaltyVaultBalance && pe.TryGetCorrelation(EntityRelationship.EntityToRoyaltyVault, out var packageRoyaltyRelation) && vaultBalances.TryGetValue(packageRoyaltyRelation.EntityId, out var packageRoyaltyVault)) + if (optIns.PackageRoyaltyVaultBalance && pe.TryGetCorrelation(EntityRelationship.EntityToRoyaltyVault, out var packageRoyaltyRelation) && + vaultBalances.TryGetValue(packageRoyaltyRelation.EntityId, out var packageRoyaltyVault)) { packageRoyaltyVaultBalance = packageRoyaltyVault.Balance.ToString(); } @@ -393,7 +404,8 @@ public EntityStateQuerier( roleAssignmentsHistory.TryGetValue(ce.Id, out var roleAssignments); componentRoyaltyConfigs?.TryGetValue(ce.Id, out componentRoyaltyConfig); - if (optIns.ComponentRoyaltyVaultBalance && ce.TryGetCorrelation(EntityRelationship.EntityToRoyaltyVault, out var componentRoyaltyRelation) && vaultBalances.TryGetValue(componentRoyaltyRelation.EntityId, out var componentRoyaltyVault)) + if (optIns.ComponentRoyaltyVaultBalance && ce.TryGetCorrelation(EntityRelationship.EntityToRoyaltyVault, out var componentRoyaltyRelation) && + vaultBalances.TryGetValue(componentRoyaltyRelation.EntityId, out var componentRoyaltyVault)) { componentRoyaltyVaultBalance = componentRoyaltyVault.Balance.ToString(); } @@ -455,12 +467,16 @@ public EntityStateQuerier( var defaultPageSize = _endpointConfiguration.Value.DefaultPageSize; var entity = await _entityQuerier.GetEntity(pageRequest.Address, ledgerState, token); - var entityResourcesConfiguration = new EntityFungibleResourcesAndVaultsPagedQuery.ResourcesPageQueryConfiguration(defaultPageSize, aggregatePerVault ? defaultPageSize : 0, pageRequest.Cursor.FromGatewayModel(), ledgerState.StateVersion); + var entityResourcesConfiguration = new EntityFungibleResourcesAndVaultsPagedQuery.ResourcesPageQueryConfiguration(defaultPageSize, aggregatePerVault ? defaultPageSize : 0, + pageRequest.Cursor.FromGatewayModel(), ledgerState.StateVersion); var fungibleResourcesPage = await EntityFungibleResourcesAndVaultsPagedQuery.FungibleResourcesPage(_dbContext, _dapperWrapper, entity.Id, entityResourcesConfiguration, token); if (fungibleResourcesPage != null) { - var entityIds = fungibleResourcesPage.FungibleResources.Values.Select(x => x.ResourceEntityId) + var entityIds = fungibleResourcesPage + .FungibleResources + .Values + .Select(x => x.ResourceEntityId) .Union(new[] { entity.Id }) .ToHashSet() .ToArray(); @@ -494,7 +510,8 @@ public EntityStateQuerier( var entity = await _entityQuerier.GetEntity(request.Address, ledgerState, token); var resourceEntity = await _entityQuerier.GetEntity(request.ResourceAddress, ledgerState, token); var entityResourcesConfiguration = new EntityFungibleResourcesAndVaultsPagedQuery.VaultsPageQueryConfiguration(defaultPageSize, request.Cursor.FromGatewayModel(), ledgerState.StateVersion); - var fungibleResourceVaultsPage = await EntityFungibleResourcesAndVaultsPagedQuery.FungibleResourceVaultsPage(_dbContext, _dapperWrapper, entity.Id, resourceEntity.Id, entityResourcesConfiguration, token); + var fungibleResourceVaultsPage = + await EntityFungibleResourcesAndVaultsPagedQuery.FungibleResourceVaultsPage(_dbContext, _dapperWrapper, entity.Id, resourceEntity.Id, entityResourcesConfiguration, token); if (fungibleResourceVaultsPage != null) { @@ -508,7 +525,8 @@ public EntityStateQuerier( } } - return new GatewayModel.StateEntityFungibleResourceVaultsPageResponse(ledgerState, 0, null, new List(), entity.Address, resourceEntity.Address); + return new GatewayModel.StateEntityFungibleResourceVaultsPageResponse(ledgerState, 0, null, new List(), entity.Address, + resourceEntity.Address); } public async Task EntityNonFungibleResourcesPage( @@ -521,12 +539,16 @@ public EntityStateQuerier( var defaultPageSize = _endpointConfiguration.Value.DefaultPageSize; var entity = await _entityQuerier.GetEntity(pageRequest.Address, ledgerState, token); - var entityResourcesConfiguration = new EntityNonFungibleResourcesAndVaultsPagedQuery.ResourcesPageQueryConfiguration(defaultPageSize, aggregatePerVault ? defaultPageSize : 0, pageRequest.Cursor.FromGatewayModel(), ledgerState.StateVersion); + var entityResourcesConfiguration = new EntityNonFungibleResourcesAndVaultsPagedQuery.ResourcesPageQueryConfiguration(defaultPageSize, aggregatePerVault ? defaultPageSize : 0, + pageRequest.Cursor.FromGatewayModel(), ledgerState.StateVersion); var entityResources = await EntityNonFungibleResourcesAndVaultsPagedQuery.NonFungibleResourcesPage(_dbContext, _dapperWrapper, entity.Id, entityResourcesConfiguration, token); if (entityResources != null) { - var entityIds = entityResources.NonFungibleResources.Values.Select(x => x.ResourceEntityId) + var entityIds = entityResources + .NonFungibleResources + .Values + .Select(x => x.ResourceEntityId) .Union(new[] { entity.Id }) .ToHashSet() .ToArray(); @@ -570,7 +592,8 @@ public EntityStateQuerier( var entity = await _entityQuerier.GetEntity(request.Address, ledgerState, token); var resourceEntity = await _entityQuerier.GetEntity(request.ResourceAddress, ledgerState, token); var entityResourcesConfiguration = new EntityNonFungibleResourcesAndVaultsPagedQuery.VaultsPageQueryConfiguration(defaultPageSize, request.Cursor.FromGatewayModel(), ledgerState.StateVersion); - var entityResources = await EntityNonFungibleResourcesAndVaultsPagedQuery.NonFungibleResourceVaultsPage(_dbContext, _dapperWrapper, entity.Id, resourceEntity.Id, entityResourcesConfiguration, token); + var entityResources = + await EntityNonFungibleResourcesAndVaultsPagedQuery.NonFungibleResourceVaultsPage(_dbContext, _dapperWrapper, entity.Id, resourceEntity.Id, entityResourcesConfiguration, token); if (entityResources != null) { @@ -594,7 +617,8 @@ public EntityStateQuerier( } } - return new GatewayModel.StateEntityNonFungibleResourceVaultsPageResponse(ledgerState, 0, null, new List(), entity.Address, resourceEntity.Address); + return new GatewayModel.StateEntityNonFungibleResourceVaultsPageResponse(ledgerState, 0, null, new List(), + entity.Address, resourceEntity.Address); } public async Task EntityMetadata( @@ -668,7 +692,7 @@ public EntityStateQuerier( request.Cursor.FromGatewayModel(), request.Limit, _endpointConfiguration.Value.MaxDefinitionsLookupLimit - ); + ); var nonFungibleIdsPerVault = await NonFungibleVaultContentsPagedQuery.Execute( _dbContext, @@ -724,7 +748,15 @@ private async Task> GetEntitySchemaHis return ImmutableDictionary.Empty; } - var cd = new CommandDefinition( + var parameters = new + { + entityIds = entityIds.ToList(), + startIndex = offset + 1, + endIndex = offset + limit, + stateVersion = ledgerState.StateVersion, + }; + + var cd = DapperExtensions.CreateCommandDefinition( commandText: @" WITH variables (entity_id) AS (SELECT UNNEST(@entityIds)), schema_slices AS @@ -744,13 +776,7 @@ FROM schema_slices AS ss INNER JOIN LATERAL UNNEST(schema_slice) WITH ORDINALITY AS schema_join(id, ordinality) ON TRUE INNER JOIN schema_entry_definition sed ON sed.id = schema_join.id ORDER BY schema_join.ordinality ASC;", - parameters: new - { - entityIds = entityIds.ToList(), - startIndex = offset + 1, - endIndex = offset + limit, - stateVersion = ledgerState.StateVersion, - }, + parameters: parameters, cancellationToken: token); return (await _dbContext.Database.GetDbConnection().QueryAsync(cd)) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/NativeResourceDetailsResolver.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/NativeResourceDetailsResolver.cs index 68abb75aa..b6c57a303 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/NativeResourceDetailsResolver.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/NativeResourceDetailsResolver.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using Dapper; using Microsoft.EntityFrameworkCore; using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Model; @@ -95,7 +94,10 @@ public NativeResourceDetailsResolver(ReadOnlyDbContext dbContext, IDapperWrapper _networkConfiguration = networkConfiguration; } - public async Task> GetNativeResourceDetails(ICollection entities, GatewayModel.LedgerState ledgerState, CancellationToken token) + public async Task> GetNativeResourceDetails( + ICollection entities, + GatewayModel.LedgerState ledgerState, + CancellationToken token) { var result = new Dictionary(entities.Count); var missingEntities = new List(); @@ -120,11 +122,24 @@ public NativeResourceDetailsResolver(ReadOnlyDbContext dbContext, IDapperWrapper return result; } - private record struct DbRow(EntityRelationship BaseRelationship, string BaseEntityAddress, TokenAmount? BaseTotalSupply, EntityType RootEntityType, string RootEntityAddress, string? ResourceEntityAddress, string? ResourceBalance); + private record struct DbRow( + EntityRelationship BaseRelationship, + string BaseEntityAddress, + TokenAmount? BaseTotalSupply, + EntityType RootEntityType, + string RootEntityAddress, + string? ResourceEntityAddress, + string? ResourceBalance); private async Task> GetFromDatabase(List addresses, GatewayModel.LedgerState ledgerState, CancellationToken token) { - var cd = new CommandDefinition( + var parameters = new + { + addresses = addresses.Select(e => (string)e).ToList(), + stateVersion = ledgerState.StateVersion, + }; + + var cd = DapperExtensions.CreateCommandDefinition( @"WITH variables AS ( SELECT @@ -180,60 +195,58 @@ LIMIT 1 LEFT JOIN entities ve ON ve.id = vault.vault_entity_id LEFT JOIN entities re ON re.id = ve.correlated_entity_ids[array_position(ve.correlated_entity_relationships, 'vault_to_resource')] WHERE bwr.root_correlated_entity_relationship = ANY('{resource_pool_to_resource_vault, validator_to_stake_vault, access_controller_to_recovery_badge}'::entity_relationship[]);", - new - { - addresses = addresses.Select(e => (string)e).ToList(), - stateVersion = ledgerState.StateVersion, - }, + parameters, cancellationToken: token ); var rows = await _dapperWrapper.ToListAsync(_dbContext.Database.GetDbConnection(), cd); - return rows.GroupBy(r => (EntityAddress)r.BaseEntityAddress).ToDictionary(g => g.Key, grouping => - { - var rootEntityType = grouping.First().RootEntityType; - var rootEntityAddress = grouping.First().RootEntityAddress; - var baseRelationship = grouping.First().BaseRelationship; - - if (rootEntityType == EntityType.GlobalAccessController) + return rows + .GroupBy(r => (EntityAddress)r.BaseEntityAddress) + .ToDictionary(g => g.Key, grouping => { - return new GatewayModel.NativeResourceAccessControllerRecoveryBadgeValue(rootEntityAddress); - } + var rootEntityType = grouping.First().RootEntityType; + var rootEntityAddress = grouping.First().RootEntityAddress; + var baseRelationship = grouping.First().BaseRelationship; - if (rootEntityType == EntityType.GlobalValidator && baseRelationship == EntityRelationship.ClaimTokenOfValidator) - { - return new GatewayModel.NativeResourceValidatorClaimNftValue(rootEntityAddress); - } + if (rootEntityType == EntityType.GlobalAccessController) + { + return new GatewayModel.NativeResourceAccessControllerRecoveryBadgeValue(rootEntityAddress); + } - var baseTotalSupply = grouping.First().BaseTotalSupply ?? throw new InvalidOperationException($"BaseTotalSupply cannot be empty on {grouping.Key}"); - var redemptionValues = new List(); + if (rootEntityType == EntityType.GlobalValidator && baseRelationship == EntityRelationship.ClaimTokenOfValidator) + { + return new GatewayModel.NativeResourceValidatorClaimNftValue(rootEntityAddress); + } - foreach (var entry in grouping) - { - var resourceAddress = entry.ResourceEntityAddress ?? throw new InvalidOperationException($"ResourceEntityAddress cannot be empty on {grouping.Key}"); - TokenAmount? amount = null; + var baseTotalSupply = grouping.First().BaseTotalSupply ?? throw new InvalidOperationException($"BaseTotalSupply cannot be empty on {grouping.Key}"); + var redemptionValues = new List(); - if (baseTotalSupply > TokenAmount.Zero) + foreach (var entry in grouping) { - var resourceBalance = entry.ResourceBalance ?? throw new InvalidOperationException($"ResourceBalance cannot be empty on {grouping.Key}"); - amount = TokenAmount.FromSubUnitsString(resourceBalance) / baseTotalSupply; - } + var resourceAddress = entry.ResourceEntityAddress ?? throw new InvalidOperationException($"ResourceEntityAddress cannot be empty on {grouping.Key}"); + TokenAmount? amount = null; - redemptionValues.Add(new GatewayModel.NativeResourceRedemptionValueItem(resourceAddress, amount?.ToString())); - } + if (baseTotalSupply > TokenAmount.Zero) + { + var resourceBalance = entry.ResourceBalance ?? throw new InvalidOperationException($"ResourceBalance cannot be empty on {grouping.Key}"); + amount = TokenAmount.FromSubUnitsString(resourceBalance) / baseTotalSupply; + } - GatewayModel.NativeResourceDetails result = rootEntityType switch - { - EntityType.GlobalValidator => new GatewayModel.NativeResourceValidatorLiquidStakeUnitValue(rootEntityAddress, redemptionValues.Count, redemptionValues), - EntityType.GlobalOneResourcePool => new GatewayModel.NativeResourceOneResourcePoolUnitValue(rootEntityAddress, redemptionValues.Count, redemptionValues), - EntityType.GlobalTwoResourcePool => new GatewayModel.NativeResourceTwoResourcePoolUnitValue(rootEntityAddress, redemptionValues.Count, redemptionValues), - EntityType.GlobalMultiResourcePool => new GatewayModel.NativeResourceMultiResourcePoolUnitValue(rootEntityAddress, redemptionValues.Count, redemptionValues), - _ => throw new ArgumentOutOfRangeException(nameof(rootEntityType), rootEntityType, null), - }; + redemptionValues.Add(new GatewayModel.NativeResourceRedemptionValueItem(resourceAddress, amount?.ToString())); + } + + GatewayModel.NativeResourceDetails result = rootEntityType switch + { + EntityType.GlobalValidator => new GatewayModel.NativeResourceValidatorLiquidStakeUnitValue(rootEntityAddress, redemptionValues.Count, redemptionValues), + EntityType.GlobalOneResourcePool => new GatewayModel.NativeResourceOneResourcePoolUnitValue(rootEntityAddress, redemptionValues.Count, redemptionValues), + EntityType.GlobalTwoResourcePool => new GatewayModel.NativeResourceTwoResourcePoolUnitValue(rootEntityAddress, redemptionValues.Count, redemptionValues), + EntityType.GlobalMultiResourcePool => new GatewayModel.NativeResourceMultiResourcePoolUnitValue(rootEntityAddress, redemptionValues.Count, redemptionValues), + _ => throw new ArgumentOutOfRangeException(nameof(rootEntityType), rootEntityType, null), + }; - return result; - }); + return result; + }); } private bool TryGetWellKnown(EntityAddress entityAddress, [NotNullWhen(true)] out GatewayModel.NativeResourceDetails? result) diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/ResourceHoldersQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/ResourceHoldersQuerier.cs index b259a3adf..363cc22e8 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/ResourceHoldersQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/ResourceHoldersQuerier.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using Dapper; using Microsoft.EntityFrameworkCore; using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Numerics; @@ -113,7 +112,15 @@ public ResourceHoldersQuerier(ReadOnlyDbContext dbContext, IDapperWrapper dapper var totalCount = await _dbContext.ResourceHolders.CountAsync(x => x.ResourceEntityId == resourceEntity.Id, token); - var cd = new CommandDefinition( + var parameters = new + { + resourceEntityId = resourceEntity.Id, + balanceBoundary = cursor?.BalanceBoundary ?? TokenAmount.MaxValue.ToString(), + idBoundary = cursor?.IdBoundary ?? long.MaxValue, + limit = limit + 1, + }; + + var cd = DapperExtensions.CreateCommandDefinition( @" SELECT ro.id as Id, @@ -127,13 +134,7 @@ INNER JOIN entities e AND (ro.balance, ro.id) <= (Cast(@balanceBoundary AS numeric(1000,0)), @idBoundary) ORDER BY (ro.balance, ro.entity_id) DESC LIMIT @limit", - new - { - resourceEntityId = resourceEntity.Id, - balanceBoundary = cursor?.BalanceBoundary ?? TokenAmount.MaxValue.ToString(), - idBoundary = cursor?.IdBoundary ?? long.MaxValue, - limit = limit + 1, - }, + parameters, cancellationToken: token ); diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/StandardMetadataResolver.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/StandardMetadataResolver.cs index bf0135376..a0606adde 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/StandardMetadataResolver.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/StandardMetadataResolver.cs @@ -62,7 +62,6 @@ * permissions under this License. */ -using Dapper; using Microsoft.EntityFrameworkCore; using Newtonsoft.Json; using RadixDlt.NetworkGateway.Abstractions; @@ -123,7 +122,18 @@ public async Task>> R return ImmutableDictionary>.Empty; } - var cd = new CommandDefinition( + var parameters = new + { + dappAccountTypeDappDefinition = StandardMetadataConstants.DappAccountTypeDappDefinition, + validationUnknown = StandardMetadataConstants.ValidationUnknown, + validationOnLedgerSucceeded = StandardMetadataConstants.ValidationOnLedgerSucceeded, + validationOnLedgerAppCheck = StandardMetadataConstants.ValidationOnLedgerAppCheck, + validationOffLedgerAppCheck = StandardMetadataConstants.ValidationOffLedgerAppCheck, + stateVersion = ledgerState.StateVersion, + entityIds = entityIds, + }; + + var cd = DapperExtensions.CreateCommandDefinition( @" WITH variables (entity_id) AS (SELECT UNNEST(@entityIds)), @@ -292,16 +302,7 @@ FROM candidates c coalesce(target_value, target_entity_address) AS TargetValue, validation_result AS ValidationResult FROM resolved", - new - { - dappAccountTypeDappDefinition = StandardMetadataConstants.DappAccountTypeDappDefinition, - validationUnknown = StandardMetadataConstants.ValidationUnknown, - validationOnLedgerSucceeded = StandardMetadataConstants.ValidationOnLedgerSucceeded, - validationOnLedgerAppCheck = StandardMetadataConstants.ValidationOnLedgerAppCheck, - validationOffLedgerAppCheck = StandardMetadataConstants.ValidationOffLedgerAppCheck, - stateVersion = ledgerState.StateVersion, - entityIds = entityIds, - }, + parameters, cancellationToken: token); var partiallyValidatedEntries = await _dapperWrapper.ToListAsync(_dbContext.Database.GetDbConnection(), cd); @@ -326,7 +327,11 @@ await Parallel.ForEachAsync(partiallyValidatedEntries, options, async (pv, inner return result.ToDictionary(e => e.Key, e => (ICollection)e.Value.ToList()); } - private async ValueTask ResolveTwoWayLink(PartiallyValidatedTwoWayLink link, bool validateOnLedgerOnly, ICollection allEntries, CancellationToken token) + private async ValueTask ResolveTwoWayLink( + PartiallyValidatedTwoWayLink link, + bool validateOnLedgerOnly, + ICollection allEntries, + CancellationToken token) { if (link.ValidationResult == StandardMetadataConstants.ValidationUnknown) { diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionDetailsQuery.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionDetailsQuery.cs new file mode 100644 index 000000000..6815fe38f --- /dev/null +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionDetailsQuery.cs @@ -0,0 +1,224 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +// +// Copyright (c) PlaceholderCompany. All rights reserved. +// + +using Microsoft.EntityFrameworkCore; +using RadixDlt.NetworkGateway.Abstractions.Model; +using RadixDlt.NetworkGateway.Abstractions.Numerics; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace RadixDlt.NetworkGateway.PostgresIntegration.Services; + +internal static class TransactionDetailsQuery +{ + internal record ReceiptEvent(string Name, string Emitter, byte[] Data, long EntityId, byte[] SchemaHash, long TypeIndex, SborTypeKind KeyTypeKind); + + internal record LedgerTransactionQueryResult( + long StateVersion, + long Epoch, + long RoundInEpoch, + TokenAmount FeePaid, + long[] AffectedGlobalEntities, + DateTime RoundTimestamp, + LedgerTransactionStatus ReceiptStatus, + string? ReceiptFeeSource, + string? ReceiptFeeDestination, + string? ReceiptErrorMessage, + LedgerTransactionType Discriminator, + string PayloadHash, + string IntentHash, + string? Message, + LedgerTransactionManifestClass[] ManifestClasses, + byte[]? RawPayload, + string? ReceiptCostingParameters, + string? ReceiptFeeSummary, + string? ReceiptNextEpoch, + string? ReceiptOutput, + string? ReceiptStateUpdates, + string? BalanceChanges, + string? ManifestInstructions + ) + { + public List Events { get; set; } = new(); + } + + private record ReceiptEvents( + string[] ReceiptEventEmitters, + string[] ReceiptEventNames, + byte[][] ReceiptEventSbors, + long[] ReceiptEventSchemaEntityIds, + byte[][] ReceiptEventSchemaHashes, + long[] ReceiptEventTypeIndexes, + SborTypeKind[] ReceiptEventSborTypeKinds); + + internal static async Task> Execute( + IDapperWrapper dapperWrapper, + CommonDbContext dbContext, + List transactionStateVersions, + GatewayApiSdk.Model.TransactionDetailsOptIns optIns, + CancellationToken token) + { + var parameters = new + { + includeRawHex = optIns.RawHex, + includeReceiptOutput = optIns.ReceiptOutput, + includeReceiptStateChanges = optIns.ReceiptStateChanges, + includeReceiptEvents = optIns.ReceiptEvents, + includeBalanceChanges = optIns.BalanceChanges, + includeManifestInstructions = optIns.ManifestInstructions, + transactionStateVersions = transactionStateVersions, + }; + + var cd = DapperExtensions.CreateCommandDefinition( + @" +WITH vars AS ( + SELECT + @includeRawHex AS with_raw_payload, + true AS with_receipt_costing_parameters, + true AS with_receipt_fee_summary, + true AS with_receipt_next_epoch, + @includeReceiptOutput AS with_receipt_output, + @includeReceiptStateChanges AS with_receipt_state_updates, + @includeReceiptEvents AS with_receipt_events, + @includeBalanceChanges AS with_balance_changes, + @includeManifestInstructions AS with_manifest_instructions, + @transactionStateVersions AS transaction_state_versions +) +SELECT + lt.state_version, + epoch, + round_in_epoch, + CAST(fee_paid AS TEXT), + affected_global_entities, + round_timestamp, + receipt_status, + receipt_fee_source, + receipt_fee_destination, + receipt_error_message, + discriminator, + payload_hash, + intent_hash, + message, + manifest_classes, + CASE WHEN vars.with_raw_payload THEN raw_payload END AS raw_payload, + CASE WHEN vars.with_receipt_costing_parameters THEN receipt_costing_parameters END AS receipt_costing_parameters, + CASE WHEN vars.with_receipt_fee_summary THEN receipt_fee_summary END AS receipt_fee_summary, + CASE WHEN vars.with_receipt_next_epoch THEN receipt_next_epoch END AS receipt_next_epoch, + CASE WHEN vars.with_receipt_output THEN receipt_output END AS receipt_output, + CASE WHEN vars.with_receipt_state_updates THEN receipt_state_updates END AS receipt_state_updates, + CASE WHEN vars.with_balance_changes THEN balance_changes END AS balance_changes, + CASE WHEN vars.with_manifest_instructions THEN manifest_instructions END AS manifest_instructions, + CASE WHEN vars.with_receipt_events THEN lte.receipt_event_emitters END AS ReceiptEventEmitters, + CASE WHEN vars.with_receipt_events THEN lte.receipt_event_names END AS ReceiptEventNames, + CASE WHEN vars.with_receipt_events THEN lte.receipt_event_sbors END AS ReceiptEventSbors, + CASE WHEN vars.with_receipt_events THEN lte.receipt_event_schema_entity_ids END AS ReceiptEventSchemaEntityIds, + CASE WHEN vars.with_receipt_events THEN lte.receipt_event_schema_hashes END AS ReceiptEventSchemaHashes, + CASE WHEN vars.with_receipt_events THEN lte.receipt_event_type_indexes END AS ReceiptEventTypeIndexes, + CASE WHEN vars.with_receipt_events THEN lte.receipt_event_sbor_type_kinds END AS ReceiptEventSborTypeKinds +FROM vars +CROSS JOIN ledger_transactions lt +LEFT JOIN ledger_transaction_events lte ON vars.with_receipt_events AND lte.state_version = lt.state_version +WHERE lt.state_version = ANY(vars.transaction_state_versions)", + parameters, + cancellationToken: token); + + var transactions = (await dapperWrapper.QueryAsync( + dbContext.Database.GetDbConnection(), + cd, + (transaction, events) => + { + if (events == null) + { + return transaction; + } + + var mappedEvents = events + .ReceiptEventEmitters + .Select( + (_, i) => new ReceiptEvent( + events.ReceiptEventNames[i], + events.ReceiptEventEmitters[i], + events.ReceiptEventSbors[i], + events.ReceiptEventSchemaEntityIds[i], + events.ReceiptEventSchemaHashes[i], + events.ReceiptEventTypeIndexes[i], + events.ReceiptEventSborTypeKinds[i])) + .ToList(); + + transaction.Events = mappedEvents; + + return transaction; + }, + "ReceiptEventEmitters")).ToList(); + + return transactions; + } +} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionMapper.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionMapper.cs deleted file mode 100644 index 6ceab01d5..000000000 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionMapper.cs +++ /dev/null @@ -1,146 +0,0 @@ -/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). - * - * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this - * file except in compliance with the License. You may obtain a copy of the License at: - * - * radixfoundation.org/licenses/LICENSE-v1 - * - * The Licensor hereby grants permission for the Canonical version of the Work to be - * published, distributed and used under or by reference to the Licensor’s trademark - * Radix ® and use of any unregistered trade names, logos or get-up. - * - * The Licensor provides the Work (and each Contributor provides its Contributions) on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, - * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, - * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. - * - * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create - * a distributed ledger it is your responsibility to test and validate the code, together - * with all logic and performance of that code under all foreseeable scenarios. - * - * The Licensor does not make or purport to make and hereby excludes liability for all - * and any representation, warranty or undertaking in any form whatsoever, whether express - * or implied, to any entity or person, including any representation, warranty or - * undertaking, as to the functionality security use, value or other characteristics of - * any distributed ledger nor in respect the functioning or value of any tokens which may - * be created stored or transferred using the Work. The Licensor does not warrant that the - * Work or any use of the Work complies with any law or regulation in any territory where - * it may be implemented or used or that it will be appropriate for any specific purpose. - * - * Neither the licensor nor any current or former employees, officers, directors, partners, - * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor - * shall be liable for any direct or indirect, special, incidental, consequential or other - * losses of any kind, in tort, contract or otherwise (including but not limited to loss - * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss - * of any economic or other opportunity of whatsoever nature or howsoever arising), arising - * out of or in connection with (without limitation of any use, misuse, of any ledger system - * or use made or its functionality or any performance or operation of any code or protocol - * caused by bugs or programming or logic errors or otherwise); - * - * A. any offer, purchase, holding, use, sale, exchange or transmission of any - * cryptographic keys, tokens or assets created, exchanged, stored or arising from any - * interaction with the Work; - * - * B. any failure in a transmission or loss of any token or assets keys or other digital - * artefacts due to errors in transmission; - * - * C. bugs, hacks, logic errors or faults in the Work or any communication; - * - * D. system software or apparatus including but not limited to losses caused by errors - * in holding or transmitting tokens by any third-party; - * - * E. breaches or failure of security including hacker attacks, loss or disclosure of - * password, loss of private key, unauthorised use or misuse of such passwords or keys; - * - * F. any losses including loss of anticipated savings or other benefits resulting from - * use of the Work or any changes to the Work (however implemented). - * - * You are solely responsible for; testing, validating and evaluation of all operation - * logic, functionality, security and appropriateness of using the Work for any commercial - * or non-commercial purpose and for any reproduction or redistribution by You of the - * Work. You assume all risks associated with Your use of the Work and the exercise of - * permissions under this License. - */ - -using Newtonsoft.Json.Linq; -using RadixDlt.NetworkGateway.Abstractions; -using RadixDlt.NetworkGateway.Abstractions.Extensions; -using RadixDlt.NetworkGateway.Abstractions.Model; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; - -namespace RadixDlt.NetworkGateway.PostgresIntegration.Services; - -// TODO PP: do we want to move it back to query. -internal static class TransactionMapper -{ - internal static GatewayApiSdk.Model.CommittedTransactionInfo ToGatewayModel( - this LedgerTransactionQueryResult lt, - GatewayApiSdk.Model.TransactionDetailsOptIns optIns, - IDictionary entityIdToAddressMap, - List? events, - GatewayApiSdk.Model.TransactionBalanceChanges? transactionBalanceChanges) - { - string? payloadHash = null; - string? intentHash = null; - string? rawHex = null; - JRaw? message = null; - string? manifestInstructions = null; - List? manifestClasses = null; - - if (lt.Discriminator == LedgerTransactionType.User) - { - payloadHash = lt.PayloadHash; - intentHash = lt.IntentHash; - rawHex = optIns.RawHex ? lt.RawPayload!.ToHex() : null; - message = lt.Message != null ? new JRaw(lt.Message) : null; - manifestInstructions = optIns.ManifestInstructions ? lt.ManifestInstructions : null; - manifestClasses = lt.ManifestClasses.Select(mc => mc.ToGatewayModel()).ToList(); - } - - var receipt = new GatewayApiSdk.Model.TransactionReceipt - { - ErrorMessage = lt.ReceiptErrorMessage, - Status = MapTransactionStatus(lt.ReceiptStatus), - Output = optIns.ReceiptOutput && lt.ReceiptOutput != null ? new JRaw(lt.ReceiptOutput) : null, - FeeSummary = optIns.ReceiptFeeSummary ? new JRaw(lt.ReceiptFeeSummary) : null, - FeeDestination = optIns.ReceiptFeeDestination && lt.ReceiptFeeDestination != null ? new JRaw(lt.ReceiptFeeDestination) : null, - FeeSource = optIns.ReceiptFeeSource && lt.ReceiptFeeSource != null ? new JRaw(lt.ReceiptFeeSource) : null, - CostingParameters = optIns.ReceiptCostingParameters ? new JRaw(lt.ReceiptCostingParameters) : null, - NextEpoch = lt.ReceiptNextEpoch != null ? new JRaw(lt.ReceiptNextEpoch) : null, - StateUpdates = optIns.ReceiptStateChanges && lt.ReceiptStateUpdates != null ? new JRaw(lt.ReceiptStateUpdates) : null, - Events = optIns.ReceiptEvents ? events?.Select(x => new GatewayApiSdk.Model.EventsItem(x.Name, new JRaw(x.Emitter), x.Data)).ToList() : null, - }; - - return new GatewayApiSdk.Model.CommittedTransactionInfo( - stateVersion: lt.StateVersion, - epoch: lt.Epoch, - round: lt.RoundInEpoch, - roundTimestamp: lt.RoundTimestamp.AsUtcIsoDateWithMillisString(), - transactionStatus: MapTransactionStatus(lt.ReceiptStatus), - affectedGlobalEntities: optIns.AffectedGlobalEntities ? lt.AffectedGlobalEntities.Select(x => entityIdToAddressMap[x].ToString()).ToList() : null, - payloadHash: payloadHash, - intentHash: intentHash, - feePaid: lt.FeePaid.ToString(), - confirmedAt: lt.RoundTimestamp, - errorMessage: lt.ReceiptErrorMessage, - rawHex: rawHex, - receipt: receipt, - message: message, - balanceChanges: optIns.BalanceChanges ? transactionBalanceChanges : null, - manifestInstructions: manifestInstructions, - manifestClasses: manifestClasses - ); - } - - private static GatewayApiSdk.Model.TransactionStatus MapTransactionStatus(this LedgerTransactionStatus status) - { - return status switch - { - LedgerTransactionStatus.Succeeded => GatewayApiSdk.Model.TransactionStatus.CommittedSuccess, - LedgerTransactionStatus.Failed => GatewayApiSdk.Model.TransactionStatus.CommittedFailure, - _ => throw new UnreachableException($"Didn't expect {status} value"), - }; - } -} diff --git a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs index da563daa1..2fc8af94a 100644 --- a/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs +++ b/src/RadixDlt.NetworkGateway.PostgresIntegration/Services/TransactionQuerier.cs @@ -62,14 +62,13 @@ * permissions under this License. */ -using Dapper; using Microsoft.EntityFrameworkCore; using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using RadixDlt.NetworkGateway.Abstractions; using RadixDlt.NetworkGateway.Abstractions.Extensions; using RadixDlt.NetworkGateway.Abstractions.Model; using RadixDlt.NetworkGateway.Abstractions.Network; -using RadixDlt.NetworkGateway.Abstractions.Numerics; using RadixDlt.NetworkGateway.GatewayApi.Services; using RadixDlt.NetworkGateway.PostgresIntegration.Interceptors; using RadixDlt.NetworkGateway.PostgresIntegration.Models; @@ -87,47 +86,6 @@ namespace RadixDlt.NetworkGateway.PostgresIntegration.Services; -// TODO PP: all these can be made private when we combine mapping with query. -internal record ReceiptEvent(string Name, string Emitter, byte[] Data, long EntityId, byte[] SchemaHash, long TypeIndex, SborTypeKind KeyTypeKind); - -internal record LedgerTransactionQueryResult( - long StateVersion, - long Epoch, - long RoundInEpoch, - TokenAmount FeePaid, - long[] AffectedGlobalEntities, - DateTime RoundTimestamp, - LedgerTransactionStatus ReceiptStatus, - string? ReceiptFeeSource, - string? ReceiptFeeDestination, - string? ReceiptErrorMessage, - LedgerTransactionType Discriminator, - string PayloadHash, - string IntentHash, - string? Message, - LedgerTransactionManifestClass[] ManifestClasses, - byte[]? RawPayload, - string? ReceiptCostingParameters, - string? ReceiptFeeSummary, - string? ReceiptNextEpoch, - string? ReceiptOutput, - string? ReceiptStateUpdates, - string? BalanceChanges, - string? ManifestInstructions -) -{ - public List Events { get; set; } = new(); -} - -internal record ReceiptEvents( - string[] ReceiptEventEmitters, - string[] ReceiptEventNames, - byte[][] ReceiptEventSbors, - long[] ReceiptEventSchemaEntityIds, - byte[][] ReceiptEventSchemaHashes, - long[] ReceiptEventTypeIndexes, - SborTypeKind[] ReceiptEventSborTypeKinds); - internal record Event(string Name, string Emitter, GatewayModel.ProgrammaticScryptoSborValue Data); internal class TransactionQuerier : ITransactionQuerier @@ -529,126 +487,12 @@ internal record CommittedTransactionSummary( return aggregator.IntoResponse(); } - private async Task> LookupPendingTransactionsByIntentHash(string intentHash, CancellationToken token = default) - { - return await _rwDbContext - .PendingTransactions - .Where(pt => pt.IntentHash == intentHash) - .Select( - pt => - new PendingTransactionSummary( - pt.PayloadHash, - pt.EndEpochExclusive, - pt.GatewayHandling.ResubmitFromTimestamp, - pt.GatewayHandling.HandlingStatusReason, - pt.LedgerDetails.PayloadLedgerStatus, - pt.LedgerDetails.IntentLedgerStatus, - pt.LedgerDetails.InitialRejectionReason, - pt.LedgerDetails.LatestRejectionReason, - pt.NetworkDetails.LastSubmitErrorTitle - ) - ) - .AnnotateMetricName() - .AsNoTracking() - .ToListAsync(token); - } - - // TODO PP: DO we want to move just that query to separate file. I guess so. private async Task> GetTransactions( List transactionStateVersions, GatewayModel.TransactionDetailsOptIns optIns, CancellationToken token) { - var cd = new CommandDefinition( - @"WITH vars AS ( - SELECT - @includeRawHex AS with_raw_payload, - true AS with_receipt_costing_parameters, - true AS with_receipt_fee_summary, - true AS with_receipt_next_epoch, - @includeReceiptOutput AS with_receipt_output, - @includeReceiptStateChanges AS with_receipt_state_updates, - @includeReceiptEvents AS with_receipt_events, - @includeBalanceChanges AS with_balance_changes, - @includeManifestInstructions AS with_manifest_instructions, - @transactionStateVersions AS transaction_state_versions -) -SELECT - lt.state_version, - epoch, - round_in_epoch, - CAST(fee_paid AS TEXT), - affected_global_entities, - round_timestamp, - receipt_status, - receipt_fee_source, - receipt_fee_destination, - receipt_error_message, - discriminator, - payload_hash, - intent_hash, - message, - manifest_classes, - CASE WHEN vars.with_raw_payload THEN raw_payload END AS raw_payload, - CASE WHEN vars.with_receipt_costing_parameters THEN receipt_costing_parameters END AS receipt_costing_parameters, - CASE WHEN vars.with_receipt_fee_summary THEN receipt_fee_summary END AS receipt_fee_summary, - CASE WHEN vars.with_receipt_next_epoch THEN receipt_next_epoch END AS receipt_next_epoch, - CASE WHEN vars.with_receipt_output THEN receipt_output END AS receipt_output, - CASE WHEN vars.with_receipt_state_updates THEN receipt_state_updates END AS receipt_state_updates, - CASE WHEN vars.with_balance_changes THEN balance_changes END AS balance_changes, - CASE WHEN vars.with_manifest_instructions THEN manifest_instructions END AS manifest_instructions, - CASE WHEN vars.with_receipt_events THEN lte.receipt_event_emitters END AS ReceiptEventEmitters, - CASE WHEN vars.with_receipt_events THEN lte.receipt_event_names END AS ReceiptEventNames, - CASE WHEN vars.with_receipt_events THEN lte.receipt_event_sbors END AS ReceiptEventSbors, - CASE WHEN vars.with_receipt_events THEN lte.receipt_event_schema_entity_ids END AS ReceiptEventSchemaEntityIds, - CASE WHEN vars.with_receipt_events THEN lte.receipt_event_schema_hashes END AS ReceiptEventSchemaHashes, - CASE WHEN vars.with_receipt_events THEN lte.receipt_event_type_indexes END AS ReceiptEventTypeIndexes, - CASE WHEN vars.with_receipt_events THEN lte.receipt_event_sbor_type_kinds END AS ReceiptEventSborTypeKinds -FROM vars -CROSS JOIN ledger_transactions lt -LEFT JOIN ledger_transaction_events lte ON vars.with_receipt_events AND lte.state_version = lt.state_version -WHERE lt.state_version = ANY(vars.transaction_state_versions)", - new - { - includeRawHex = optIns.RawHex, - includeReceiptOutput = optIns.ReceiptOutput, - includeReceiptStateChanges = optIns.ReceiptStateChanges, - includeReceiptEvents = optIns.ReceiptEvents, - includeBalanceChanges = optIns.BalanceChanges, - includeManifestInstructions = optIns.ManifestInstructions, - transactionStateVersions = transactionStateVersions, - }); - - var transactions = (await _dapperWrapper.QueryAsync( - _dbContext.Database.GetDbConnection(), - cd, - (transaction, events) => - { - if (events == null) - { - return transaction; - } - - var mappedEvents = events - .ReceiptEventEmitters - .Select( - (_, i) => new ReceiptEvent( - events.ReceiptEventNames[i], - events.ReceiptEventEmitters[i], - events.ReceiptEventSbors[i], - events.ReceiptEventSchemaEntityIds[i], - events.ReceiptEventSchemaHashes[i], - events.ReceiptEventTypeIndexes[i], - events.ReceiptEventSborTypeKinds[i])) - .ToList(); - - transaction.Events = mappedEvents; - - return transaction; - }, - "ReceiptEventEmitters", - "GetTransactions")).ToList(); - + var transactions = await TransactionDetailsQuery.Execute(_dapperWrapper, _dbContext, transactionStateVersions, optIns, token); var entityIdToAddressMap = await _entityQuerier.ResolveEntityAddresses(transactions.SelectMany(x => x.AffectedGlobalEntities).ToHashSet().ToList(), token); var schemaLookups = transactions @@ -674,8 +518,44 @@ FROM schema_entry_definition .ToDictionaryAsync(x => new SchemaLookup(x.EntityId, x.SchemaHash), x => x.Schema, token); } - List mappedTransactions = new List(); var networkId = (await _networkConfigurationProvider.GetNetworkConfiguration(token)).Id; + var mappedTransactions = MapTransactions(transactions, transactionStateVersions, optIns, entityIdToAddressMap, schemas, networkId); + return mappedTransactions; + } + + private async Task> LookupPendingTransactionsByIntentHash(string intentHash, CancellationToken token = default) + { + return await _rwDbContext + .PendingTransactions + .Where(pt => pt.IntentHash == intentHash) + .Select( + pt => + new PendingTransactionSummary( + pt.PayloadHash, + pt.EndEpochExclusive, + pt.GatewayHandling.ResubmitFromTimestamp, + pt.GatewayHandling.HandlingStatusReason, + pt.LedgerDetails.PayloadLedgerStatus, + pt.LedgerDetails.IntentLedgerStatus, + pt.LedgerDetails.InitialRejectionReason, + pt.LedgerDetails.LatestRejectionReason, + pt.NetworkDetails.LastSubmitErrorTitle + ) + ) + .AnnotateMetricName() + .AsNoTracking() + .ToListAsync(token); + } + + private List MapTransactions( + IList transactions, + IList transactionStateVersions, + GatewayModel.TransactionDetailsOptIns optIns, + IDictionary entityIdToAddressMap, + IDictionary schemas, + byte networkId) + { + var mappedTransactions = new List(); foreach (var transaction in transactions.OrderBy(lt => transactionStateVersions.IndexOf(lt.StateVersion)).ToList()) { @@ -685,17 +565,19 @@ FROM schema_entry_definition { var storedBalanceChanges = JsonConvert.DeserializeObject(transaction.BalanceChanges); - if (storedBalanceChanges == null) + if (storedBalanceChanges != null) + { + balanceChanges = storedBalanceChanges.ToGatewayModel(); + } + else { throw new InvalidOperationException("Unable to deserialize stored balance changes into CoreModel.CommittedTransactionBalanceChanges"); } - - balanceChanges = storedBalanceChanges.ToGatewayModel(); } - if (!optIns.ReceiptEvents || schemaLookups?.Any() == false) + if (!optIns.ReceiptEvents) { - mappedTransactions.Add(transaction.ToGatewayModel(optIns, entityIdToAddressMap, null, balanceChanges)); + mappedTransactions.Add(MapSingleTransaction(transaction, optIns, entityIdToAddressMap, null, balanceChanges)); } else { @@ -712,10 +594,74 @@ FROM schema_entry_definition events.Add(new Event(@event.Name, @event.Emitter, eventData)); } - mappedTransactions.Add(transaction.ToGatewayModel(optIns, entityIdToAddressMap, events, balanceChanges)); + mappedTransactions.Add(MapSingleTransaction(transaction, optIns, entityIdToAddressMap, events, balanceChanges)); } } return mappedTransactions; } + + private GatewayModel.CommittedTransactionInfo MapSingleTransaction( + TransactionDetailsQuery.LedgerTransactionQueryResult lt, + GatewayModel.TransactionDetailsOptIns optIns, + IDictionary entityIdToAddressMap, + IList? events, + GatewayModel.TransactionBalanceChanges? transactionBalanceChanges) + { + string? payloadHash = null; + string? intentHash = null; + string? rawHex = null; + JRaw? message = null; + string? manifestInstructions = null; + List? manifestClasses = null; + + if (lt.Discriminator == LedgerTransactionType.User) + { + payloadHash = lt.PayloadHash; + intentHash = lt.IntentHash; + rawHex = optIns.RawHex ? lt.RawPayload!.ToHex() : null; + message = lt.Message != null ? new JRaw(lt.Message) : null; + manifestInstructions = optIns.ManifestInstructions ? lt.ManifestInstructions : null; + manifestClasses = lt.ManifestClasses.Select(mc => mc.ToGatewayModel()).ToList(); + } + + var status = lt.ReceiptStatus switch + { + LedgerTransactionStatus.Succeeded => GatewayModel.TransactionStatus.CommittedSuccess, + LedgerTransactionStatus.Failed => GatewayModel.TransactionStatus.CommittedFailure, + _ => throw new UnreachableException($"Didn't expect {lt.ReceiptStatus} value"), + }; + + return new GatewayModel.CommittedTransactionInfo( + stateVersion: lt.StateVersion, + epoch: lt.Epoch, + round: lt.RoundInEpoch, + roundTimestamp: lt.RoundTimestamp.AsUtcIsoDateWithMillisString(), + transactionStatus: status, + affectedGlobalEntities: optIns.AffectedGlobalEntities ? lt.AffectedGlobalEntities.Select(x => entityIdToAddressMap[x].ToString()).ToList() : null, + payloadHash: payloadHash, + intentHash: intentHash, + feePaid: lt.FeePaid.ToString(), + confirmedAt: lt.RoundTimestamp, + errorMessage: lt.ReceiptErrorMessage, + rawHex: rawHex, + receipt: new GatewayModel.TransactionReceipt + { + ErrorMessage = lt.ReceiptErrorMessage, + Status = status, + Output = optIns.ReceiptOutput && lt.ReceiptOutput != null ? new JRaw(lt.ReceiptOutput) : null, + FeeSummary = optIns.ReceiptFeeSummary ? new JRaw(lt.ReceiptFeeSummary) : null, + FeeDestination = optIns.ReceiptFeeDestination && lt.ReceiptFeeDestination != null ? new JRaw(lt.ReceiptFeeDestination) : null, + FeeSource = optIns.ReceiptFeeSource && lt.ReceiptFeeSource != null ? new JRaw(lt.ReceiptFeeSource) : null, + CostingParameters = optIns.ReceiptCostingParameters ? new JRaw(lt.ReceiptCostingParameters) : null, + NextEpoch = lt.ReceiptNextEpoch != null ? new JRaw(lt.ReceiptNextEpoch) : null, + StateUpdates = optIns.ReceiptStateChanges && lt.ReceiptStateUpdates != null ? new JRaw(lt.ReceiptStateUpdates) : null, + Events = optIns.ReceiptEvents ? events?.Select(x => new GatewayModel.EventsItem(x.Name, new JRaw(x.Emitter), x.Data)).ToList() : null, + }, + message: message, + balanceChanges: optIns.BalanceChanges ? transactionBalanceChanges : null, + manifestInstructions: manifestInstructions, + manifestClasses: manifestClasses + ); + } }