Feat/collapse trie based on size

cmd/node/config/config.toml

@@ -887,9 +887,11 @@
     SnapshotsEnabled = true
     AccountsStatePruningEnabled = false
     PeerStatePruningEnabled = true
-    MaxStateTrieLevelInMemory = 5
-    MaxPeerTrieLevelInMemory = 5
     StateStatisticsEnabled = false
+    MaxUserTrieSizeInMemory = 524288000 #500MB
+    MaxPeerTrieSizeInMemory = 104857600 #100MB
+    DataTriesSizeInMemory = 524288000 #500MB
+    NumLeavesToCollapseSingleRun = 100
 
 [TrieLeavesRetrieverConfig]
     Enabled = false

common/common.go

@@ -28,6 +28,14 @@ const (
 	nonceIndex     = 0
 )
 
+const (
+	// TenMbSize defines the size of 10 megabytes in bytes, used as a constant for memory limits or buffer sizes
+	TenMbSize = uint64(10485760)
+
+	// NumLeavesToCollapseSingleRun defines the number of leaves to collapse in a single run, used in trie collapsing operations to manage performance and resource usage
+	NumLeavesToCollapseSingleRun = 100
+)
+
 type executionResultHandler interface {
 	GetMiniBlockHeadersHandlers() []data.MiniBlockHeaderHandler
 }

common/interface.go

@@ -59,6 +59,7 @@ type Trie interface {
 	VerifyProof(rootHash []byte, key []byte, proof [][]byte) (bool, error)
 	GetStorageManager() StorageManager
 	IsMigratedToLatestVersion() (bool, error)
+	SizeInMemory() int
 	Close() error
 	IsInterfaceNil() bool
 }
@@ -151,13 +152,20 @@ type SnapshotDbHandler interface {
 // TriesHolder is used to store multiple tries
 type TriesHolder interface {
 	Put([]byte, Trie)
-	Replace(key []byte, tr Trie)
 	Get([]byte) Trie
 	GetAll() []Trie
+	Remove([]byte)
+	MarkAsDirty([]byte)
 	Reset()
 	IsInterfaceNil() bool
 }
 
+// DataTrieCreator is an adapter for the Trie interface used only for recreating tries
+type DataTrieCreator interface {
+	Recreate(options RootHashHolder) (Trie, error)
+	IsInterfaceNil() bool
+}
+
 // Locker defines the operations used to lock different critical areas. Implemented by the RWMutex.
 type Locker interface {
 	Lock()
@@ -527,3 +535,15 @@ type AOTSelectionPreempter interface {
 	CancelOngoingSelection()
 	IsInterfaceNil() bool
 }
+
+// TrieCollapseManager defines the behavior of a trie collapse manager
+type TrieCollapseManager interface {
+	MarkKeyAsAccessed(key []byte, sizeLoadedInMemory int)
+	RemoveKey(key []byte, sizeLoadedInMemory int)
+	ShouldCollapseTrie() bool
+	GetCollapsibleLeaves() ([][]byte, error)
+	AddSizeInMemory(size int)
+	GetSizeInMemory() int
+	CloneWithoutState() TrieCollapseManager
+	IsInterfaceNil() bool
+}

config/config.go

@@ -455,12 +455,14 @@ type FacadeConfig struct {
 
 // StateTriesConfig will hold information about state tries
 type StateTriesConfig struct {
-	SnapshotsEnabled            bool
-	AccountsStatePruningEnabled bool
-	PeerStatePruningEnabled     bool
-	MaxStateTrieLevelInMemory   uint
-	MaxPeerTrieLevelInMemory    uint
-	StateStatisticsEnabled      bool
+	SnapshotsEnabled             bool
+	AccountsStatePruningEnabled  bool
+	PeerStatePruningEnabled      bool
+	StateStatisticsEnabled       bool
+	MaxUserTrieSizeInMemory      uint64
+	MaxPeerTrieSizeInMemory      uint64
+	DataTriesSizeInMemory        uint64
+	NumLeavesToCollapseSingleRun uint32
 }
 
 // StateAccessesCollectorConfig will hold information about state accesses collector

config/tomlConfig_test.go

@@ -169,11 +169,10 @@ func TestTomlParser(t *testing.T) {
 			},
 		},
 		StateTriesConfig: StateTriesConfig{
-			SnapshotsEnabled:            true,
-			AccountsStatePruningEnabled: true,
-			PeerStatePruningEnabled:     true,
-			MaxStateTrieLevelInMemory:   38,
-			MaxPeerTrieLevelInMemory:    39,
+			SnapshotsEnabled:             true,
+			AccountsStatePruningEnabled:  true,
+			PeerStatePruningEnabled:      true,
+			NumLeavesToCollapseSingleRun: 100,
 		},
 		TxCacheBounds: TxCacheBoundsConfig{
 			MaxNumBytesPerSenderUpperBound: 33_554_432,
@@ -632,8 +631,7 @@ func TestTomlParser(t *testing.T) {
     SnapshotsEnabled = true
     AccountsStatePruningEnabled = true
     PeerStatePruningEnabled = true
-    MaxStateTrieLevelInMemory = 38
-    MaxPeerTrieLevelInMemory = 39
+	NumLeavesToCollapseSingleRun = 100
 `
 	cfg := Config{}
 

dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/multiversx/mx-chain-go/dataRetriever/mock"
 	"github.com/multiversx/mx-chain-go/p2p"
 	"github.com/multiversx/mx-chain-go/process/factory"
-	"github.com/multiversx/mx-chain-go/state"
+	"github.com/multiversx/mx-chain-go/state/triesHolder"
 	"github.com/multiversx/mx-chain-go/storage"
 	"github.com/multiversx/mx-chain-go/testscommon"
 	"github.com/multiversx/mx-chain-go/testscommon/cache"
@@ -86,10 +86,10 @@ func createStoreForMeta() dataRetriever.StorageService {
 }
 
 func createTriesHolderForMeta() common.TriesHolder {
-	triesHolder := state.NewDataTriesHolder()
-	triesHolder.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{})
-	triesHolder.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{})
-	return triesHolder
+	triesContainer := triesHolder.NewTriesHolder()
+	triesContainer.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{})
+	triesContainer.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{})
+	return triesContainer
 }
 
 // ------- NewResolversContainerFactory

dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/multiversx/mx-chain-go/dataRetriever/mock"
 	"github.com/multiversx/mx-chain-go/p2p"
 	"github.com/multiversx/mx-chain-go/process/factory"
-	"github.com/multiversx/mx-chain-go/state"
+	"github.com/multiversx/mx-chain-go/state/triesHolder"
 	"github.com/multiversx/mx-chain-go/storage"
 	"github.com/multiversx/mx-chain-go/testscommon"
 	"github.com/multiversx/mx-chain-go/testscommon/cache"
@@ -92,10 +92,10 @@ func createStoreForShard() dataRetriever.StorageService {
 }
 
 func createTriesHolderForShard() common.TriesHolder {
-	triesHolder := state.NewDataTriesHolder()
-	triesHolder.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{})
-	triesHolder.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{})
-	return triesHolder
+	triesContainer := triesHolder.NewTriesHolder()
+	triesContainer.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{})
+	triesContainer.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{})
+	return triesContainer
 }
 
 // ------- NewResolversContainerFactory

dataRetriever/factory/storageRequestersContainer/metaRequestersContainerFactory_test.go

@@ -221,8 +221,6 @@ func getArgumentsMeta() storagerequesterscontainer.FactoryArgs {
 			StateTriesConfig: config.StateTriesConfig{
 				AccountsStatePruningEnabled: false,
 				PeerStatePruningEnabled:     false,
-				MaxStateTrieLevelInMemory:   5,
-				MaxPeerTrieLevelInMemory:    5,
 			},
 		},
 		ShardIDForTries:          0,

dataRetriever/factory/storageRequestersContainer/shardRequestersContainerFactory_test.go

@@ -206,8 +206,6 @@ func getArgumentsShard() storagerequesterscontainer.FactoryArgs {
 			StateTriesConfig: config.StateTriesConfig{
 				AccountsStatePruningEnabled: false,
 				PeerStatePruningEnabled:     false,
-				MaxStateTrieLevelInMemory:   5,
-				MaxPeerTrieLevelInMemory:    5,
 			},
 		},
 		ShardIDForTries:          0,

epochStart/bootstrap/process.go

@@ -15,10 +15,9 @@ import (
 	"github.com/multiversx/mx-chain-core-go/data"
 	"github.com/multiversx/mx-chain-core-go/data/block"
 	"github.com/multiversx/mx-chain-core-go/data/typeConverters/uint64ByteSlice"
+	"github.com/multiversx/mx-chain-go/state/triesHolder"
 	logger "github.com/multiversx/mx-chain-logger-go"
 
-	"github.com/multiversx/mx-chain-go/process/interceptors/processor"
-
 	"github.com/multiversx/mx-chain-go/common"
 	disabledCommon "github.com/multiversx/mx-chain-go/common/disabled"
 	"github.com/multiversx/mx-chain-go/common/ordering"
@@ -42,11 +41,11 @@ import (
 	"github.com/multiversx/mx-chain-go/process/heartbeat/validator"
 	"github.com/multiversx/mx-chain-go/process/interceptors"
 	disabledInterceptors "github.com/multiversx/mx-chain-go/process/interceptors/disabled"
+	"github.com/multiversx/mx-chain-go/process/interceptors/processor"
 	"github.com/multiversx/mx-chain-go/process/peer"
 	"github.com/multiversx/mx-chain-go/redundancy"
 	"github.com/multiversx/mx-chain-go/sharding"
 	"github.com/multiversx/mx-chain-go/sharding/nodesCoordinator"
-	"github.com/multiversx/mx-chain-go/state"
 	"github.com/multiversx/mx-chain-go/state/syncer"
 	"github.com/multiversx/mx-chain-go/storage"
 	"github.com/multiversx/mx-chain-go/storage/cache"
@@ -283,7 +282,7 @@ func NewEpochStartBootstrap(args ArgsEpochStartBootstrap) (*epochStartBootstrap,
 		return nil, err
 	}
 
-	epochStartProvider.trieContainer = state.NewDataTriesHolder()
+	epochStartProvider.trieContainer = triesHolder.NewTriesHolder()
 	epochStartProvider.trieStorageManagers = make(map[string]common.StorageManager)
 
 	if epochStartProvider.generalConfig.Hardfork.AfterHardFork {
@@ -1573,7 +1572,6 @@ func (e *epochStartBootstrap) syncUserAccountsState(rootHash []byte) error {
 			RequestHandler:                    e.requestHandler,
 			Timeout:                           common.TimeoutGettingTrieNodes,
 			Cacher:                            e.dataPool.TrieNodes(),
-			MaxTrieLevelInMemory:              e.generalConfig.StateTriesConfig.MaxStateTrieLevelInMemory,
 			MaxHardCapForMissingNodes:         e.maxHardCapForMissingNodes,
 			TrieSyncerVersion:                 e.trieSyncerVersion,
 			CheckNodesOnDisk:                  e.checkNodesOnDisk,
@@ -1647,7 +1645,6 @@ func (e *epochStartBootstrap) syncValidatorAccountsState(rootHash []byte) error
 			RequestHandler:                    e.requestHandler,
 			Timeout:                           common.TimeoutGettingTrieNodes,
 			Cacher:                            e.dataPool.TrieNodes(),
-			MaxTrieLevelInMemory:              e.generalConfig.StateTriesConfig.MaxPeerTrieLevelInMemory,
 			MaxHardCapForMissingNodes:         e.maxHardCapForMissingNodes,
 			TrieSyncerVersion:                 e.trieSyncerVersion,
 			CheckNodesOnDisk:                  e.checkNodesOnDisk,

epochStart/bootstrap/process_test.go

@@ -182,8 +182,9 @@ func createMockEpochStartBootstrapArgs(
 				AccountsStatePruningEnabled: true,
 				SnapshotsEnabled:            true,
 				PeerStatePruningEnabled:     true,
-				MaxStateTrieLevelInMemory:   5,
-				MaxPeerTrieLevelInMemory:    5,
+				MaxUserTrieSizeInMemory:     generalCfg.StateTriesConfig.MaxUserTrieSizeInMemory,
+				MaxPeerTrieSizeInMemory:     generalCfg.StateTriesConfig.MaxPeerTrieSizeInMemory,
+				DataTriesSizeInMemory:       generalCfg.StateTriesConfig.DataTriesSizeInMemory,
 			},
 			TrieStorageManagerConfig: config.TrieStorageManagerConfig{
 				PruningBufferLen:      1000,

epochStart/metachain/baseRewards_test.go

@@ -14,22 +14,21 @@ import (
 	"github.com/multiversx/mx-chain-core-go/data/rewardTx"
 	"github.com/multiversx/mx-chain-core-go/hashing/sha256"
 	"github.com/multiversx/mx-chain-core-go/marshal"
+	"github.com/multiversx/mx-chain-go/state"
 	vmcommon "github.com/multiversx/mx-chain-vm-common-go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/multiversx/mx-chain-go/epochStart"
 	"github.com/multiversx/mx-chain-go/epochStart/mock"
 	"github.com/multiversx/mx-chain-go/process"
-	"github.com/multiversx/mx-chain-go/state/factory"
 	"github.com/multiversx/mx-chain-go/testscommon"
 	txExecOrderStub "github.com/multiversx/mx-chain-go/testscommon/common"
 	dataRetrieverMock "github.com/multiversx/mx-chain-go/testscommon/dataRetriever"
 	"github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock"
 	"github.com/multiversx/mx-chain-go/testscommon/hashingMocks"
 	"github.com/multiversx/mx-chain-go/testscommon/shardingMocks"
 	stateMock "github.com/multiversx/mx-chain-go/testscommon/state"
-	"github.com/multiversx/mx-chain-go/testscommon/storage"
 	"github.com/multiversx/mx-chain-go/trie"
 )
 
@@ -1140,20 +1139,14 @@ func getBaseRewardsArguments() BaseRewardsCreatorArgs {
 	hasher := sha256.NewSha256()
 	marshalizer := &marshal.GogoProtoMarshalizer{}
 
-	storageManagerArgs := storage.GetStorageManagerArgs()
+	storageManagerArgs := txExecOrderStub.GetStorageManagerArgs()
 	storageManagerArgs.Marshalizer = marshalizer
 	storageManagerArgs.Hasher = hasher
 
-	trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, storage.GetStorageManagerOptions())
-	argsAccCreator := factory.ArgsAccountCreator{
-		Hasher:                 hasher,
-		Marshaller:             marshalizer,
-		EnableEpochsHandler:    &enableEpochsHandlerMock.EnableEpochsHandlerStub{},
-		StateAccessesCollector: &stateMock.StateAccessesCollectorStub{},
-	}
-	accCreator, _ := factory.NewAccountCreator(argsAccCreator)
+	trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, txExecOrderStub.GetStorageManagerOptions())
 	enableEpochsHandler := &enableEpochsHandlerMock.EnableEpochsHandlerStub{}
-	userAccountsDB := createAccountsDB(hasher, marshalizer, accCreator, trieFactoryManager, enableEpochsHandler)
+	userAccountsDBArgs := createAccountsDBArgs(hasher, marshalizer, trieFactoryManager, enableEpochsHandler)
+	userAccountsDB, _ := state.NewAccountsDB(userAccountsDBArgs)
 	shardCoordinator := mock.NewMultiShardsCoordinatorMock(2)
 	shardCoordinator.CurrentShard = core.MetachainShardId
 	shardCoordinator.ComputeIdCalled = func(address []byte) uint32 {