diff --git a/epochStart/metachain/trigger.go b/epochStart/metachain/trigger.go index 9d4855bb11e..e1009d011f3 100644 --- a/epochStart/metachain/trigger.go +++ b/epochStart/metachain/trigger.go @@ -15,13 +15,13 @@ import ( "github.com/multiversx/mx-chain-core-go/display" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" - "github.com/multiversx/mx-chain-logger-go" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/epochStart" "github.com/multiversx/mx-chain-go/process" "github.com/multiversx/mx-chain-go/storage" + "github.com/multiversx/mx-chain-logger-go" ) var log = logger.GetOrCreate("epochStart/metachain") @@ -209,9 +209,11 @@ func (t *trigger) Update(round uint64, nonce uint64) { } isZeroEpochEdgeCase := nonce < minimumNonceToStartEpoch + epochStartNonce := t.epochStartMeta.GetNonce() + hasMinBlocksInEpoch := nonce >= epochStartNonce+minimumNonceToStartEpoch isNormalEpochStart := t.currentRound > t.currEpochStartRound+t.roundsPerEpoch isWithEarlyEndOfEpoch := t.currentRound >= t.nextEpochStartRound - shouldTriggerEpochStart := (isNormalEpochStart || isWithEarlyEndOfEpoch) && !isZeroEpochEdgeCase + shouldTriggerEpochStart := (isNormalEpochStart || isWithEarlyEndOfEpoch) && !isZeroEpochEdgeCase && hasMinBlocksInEpoch if shouldTriggerEpochStart { t.epoch += 1 t.isEpochStart = true diff --git a/epochStart/metachain/trigger_test.go b/epochStart/metachain/trigger_test.go index c30a9cf4bd6..4538199c424 100644 --- a/epochStart/metachain/trigger_test.go +++ b/epochStart/metachain/trigger_test.go @@ -462,3 +462,82 @@ func TestTrigger_RevertBehindEpochStartBlock(t *testing.T) { ret = epochStartTrigger.IsEpochStart() assert.False(t, ret) } + +func TestTrigger_UpdateShouldNotStartEpochWithLessThanMinimumBlocks(t *testing.T) { + t.Parallel() + + epoch := uint32(0) + arguments := createMockEpochStartTriggerArguments() + arguments.Settings.RoundsPerEpoch = 2 + arguments.Settings.MinRoundsBetweenEpochs = 1 + arguments.Epoch = epoch + epochStartTrigger, err := NewEpochStartTrigger(arguments) + require.Nil(t, err) + + epochStartNonce := uint64(100) + epochStartRound := uint64(50) + + // simulate an epoch start block already processed + epochStartTrigger.SetProcessed(&block.MetaBlock{ + Nonce: epochStartNonce, + Round: epochStartRound, + Epoch: epoch, + EpochStart: block.EpochStart{ + LastFinalizedHeaders: []block.EpochStartShardData{{RootHash: []byte("root")}}, + }, + }, nil) + require.False(t, epochStartTrigger.IsEpochStart()) + require.Equal(t, epoch, epochStartTrigger.Epoch()) + + // round condition is met but nonce is only epochStartNonce+1 (1 block in epoch) + // this should NOT trigger a new epoch because minimumBlocksPerEpoch = 2 + nextRound := epochStartRound + uint64(arguments.Settings.RoundsPerEpoch) + 1 + epochStartTrigger.Update(nextRound, epochStartNonce+1) + assert.False(t, epochStartTrigger.IsEpochStart(), + "epoch should not start with only 1 block in the current epoch") + assert.Equal(t, epoch, epochStartTrigger.Epoch()) + + // now with nonce = epochStartNonce+4 (4 blocks in epoch), it should trigger + epochStartTrigger.Update(nextRound+1, epochStartNonce+4) + assert.True(t, epochStartTrigger.IsEpochStart(), + "epoch should start once minimum blocks per epoch is reached") + assert.Equal(t, epoch+1, epochStartTrigger.Epoch()) +} + +func TestTrigger_ForceEpochStartShouldRespectMinimumBlocks(t *testing.T) { + t.Parallel() + + epoch := uint32(0) + arguments := createMockEpochStartTriggerArguments() + arguments.Settings.RoundsPerEpoch = 200 + arguments.Settings.MinRoundsBetweenEpochs = 20 + arguments.Epoch = epoch + epochStartTrigger, err := NewEpochStartTrigger(arguments) + require.Nil(t, err) + + epochStartNonce := uint64(500) + epochStartRound := uint64(1000) + + // simulate an epoch start block already processed + epochStartTrigger.SetProcessed(&block.MetaBlock{ + Nonce: epochStartNonce, + Round: epochStartRound, + Epoch: epoch, + EpochStart: block.EpochStart{ + LastFinalizedHeaders: []block.EpochStartShardData{{RootHash: []byte("root")}}, + }, + }, nil) + + // force epoch start at round 1025 + epochStartTrigger.ForceEpochStart(epochStartRound + 25) + + // round condition met via force, but only 1 block since epoch start + epochStartTrigger.Update(epochStartRound+25, epochStartNonce+1) + assert.False(t, epochStartTrigger.IsEpochStart(), + "forced epoch should not start with only 1 block in the current epoch") + + // with 4 blocks, the forced epoch start should proceed + epochStartTrigger.Update(epochStartRound+26, epochStartNonce+4) + assert.True(t, epochStartTrigger.IsEpochStart(), + "forced epoch should start once minimum blocks per epoch is reached") +} diff --git a/process/block/export_test.go b/process/block/export_test.go index a5392de6d38..d7818ece09a 100644 --- a/process/block/export_test.go +++ b/process/block/export_test.go @@ -832,22 +832,6 @@ func DisplayHeader( return displayHeader(headerHandler, headerProof) } -// DetectStaleSelfNotarizedHeaders - -func (mp *metaProcessor) DetectStaleSelfNotarizedHeaders() bool { - return mp.detectStaleSelfNotarizedHeaders() -} - -// SetSelfNotarizedHeadersStale - -func (mp *metaProcessor) SetSelfNotarizedHeadersStale(stale bool) { - mp.selfNotarizedHeadersStale = stale - mp.selfNotarizedHeadersStaleOnce.Do(func() {}) -} - -// GetSelfNotarizedHeadersStale - -func (mp *metaProcessor) GetSelfNotarizedHeadersStale() bool { - return mp.selfNotarizedHeadersStale -} - // VerifyShardDataAgainstHeaders - func (mp *metaProcessor) VerifyShardDataAgainstHeaders(metaHdr *block.MetaBlock) error { return mp.verifyShardDataAgainstHeaders(metaHdr) diff --git a/process/block/metablock.go b/process/block/metablock.go index d0ef040abc0..6089b3cd697 100644 --- a/process/block/metablock.go +++ b/process/block/metablock.go @@ -34,19 +34,17 @@ var _ process.BlockProcessor = (*metaProcessor)(nil) // metaProcessor implements metaProcessor interface, and actually it tries to execute block type metaProcessor struct { *baseProcessor - scToProtocol process.SmartContractToProtocolHandler - epochStartDataCreator process.EpochStartDataCreator - epochEconomics process.EndOfEpochEconomics - epochRewardsCreator process.RewardsCreator - validatorInfoCreator process.EpochStartValidatorInfoCreator - epochSystemSCProcessor process.EpochStartSystemSCProcessor - pendingMiniBlocksHandler process.PendingMiniBlocksHandler - validatorStatisticsProcessor process.ValidatorStatisticsProcessor - shardsHeadersNonce *sync.Map - shardBlockFinality uint32 - headersCounter *headersCounter - selfNotarizedHeadersStale bool - selfNotarizedHeadersStaleOnce sync.Once + scToProtocol process.SmartContractToProtocolHandler + epochStartDataCreator process.EpochStartDataCreator + epochEconomics process.EndOfEpochEconomics + epochRewardsCreator process.RewardsCreator + validatorInfoCreator process.EpochStartValidatorInfoCreator + epochSystemSCProcessor process.EpochStartSystemSCProcessor + pendingMiniBlocksHandler process.PendingMiniBlocksHandler + validatorStatisticsProcessor process.ValidatorStatisticsProcessor + shardsHeadersNonce *sync.Map + shardBlockFinality uint32 + headersCounter *headersCounter } // NewMetaProcessor creates a new metaProcessor object @@ -190,27 +188,6 @@ func NewMetaProcessor(arguments ArgMetaProcessor) (*metaProcessor, error) { return &mp, nil } -// detectStaleSelfNotarizedHeaders returns true when bootstrap data has stale self-notarized -// headers (nonce 0) while cross-notarized headers have progressed past genesis. -func (mp *metaProcessor) detectStaleSelfNotarizedHeaders() bool { - for shardID := uint32(0); shardID < mp.shardCoordinator.NumberOfShards(); shardID++ { - crossNotarized, _, err := mp.blockTracker.GetLastCrossNotarizedHeader(shardID) - if err != nil || check.IfNil(crossNotarized) || crossNotarized.GetNonce() == 0 { - continue - } - - selfNotarized, _, err := mp.blockTracker.GetLastSelfNotarizedHeader(shardID) - if err != nil || check.IfNil(selfNotarized) || selfNotarized.GetNonce() == 0 { - log.Debug("detected stale self-notarized headers after bootstrap", - "shardID", shardID, - "crossNotarizedNonce", crossNotarized.GetNonce()) - return true - } - } - - return false -} - func (mp *metaProcessor) isRewardsV2Enabled(headerHandler data.HeaderHandler) bool { return mp.enableEpochsHandler.IsFlagEnabledInEpoch(common.StakingV2Flag, headerHandler.GetEpoch()) } @@ -1371,6 +1348,8 @@ func (mp *metaProcessor) CommitBlock( mp.blockTracker.AddSelfNotarizedHeader(shardID, lastSelfNotarizedHeader, lastSelfNotarizedHeaderHash) } + mp.completeMissingSelfNotarizedHeaders(header) + go mp.historyRepo.OnNotarizedBlocks(mp.shardCoordinator.SelfId(), []data.HeaderHandler{currentHeader}, [][]byte{currentHeaderHash}) log.Debug("highest final meta block", @@ -1458,10 +1437,6 @@ func (mp *metaProcessor) CommitBlock( mp.blockProcessingCutoffHandler.HandlePauseCutoff(header) - if mp.selfNotarizedHeadersStale { - mp.selfNotarizedHeadersStale = mp.detectStaleSelfNotarizedHeaders() - } - return nil } @@ -1668,7 +1643,10 @@ func (mp *metaProcessor) getLastSelfNotarizedHeaderByShard( mp.store, ) if errGet != nil { - log.Trace("getLastSelfNotarizedHeaderByShard.GetMetaHeader", "error", errGet.Error()) + log.Warn("getLastSelfNotarizedHeaderByShard: could not get referenced meta header, self notarized may not be updated", + "shardID", shardID, + "metaHash", metaHash, + "error", errGet.Error()) continue } @@ -1693,6 +1671,16 @@ func (mp *metaProcessor) getLastSelfNotarizedHeaderByShard( return lastNotarizedMetaHeader, lastNotarizedMetaHeaderHash } +func (mp *metaProcessor) completeMissingSelfNotarizedHeaders(currentHeader *block.MetaBlock) { + process.CompleteMissingSelfNotarizedHeaders( + currentHeader.GetPrevHash(), + mp.shardCoordinator.NumberOfShards(), + mp.blockTracker, + mp.marshalizer, + mp.store, + ) +} + // getRewardsTxs must be called before method commitEpoch start because when commit is done rewards txs are removed from pool and saved in storage func (mp *metaProcessor) getRewardsTxs(header *block.MetaBlock, body *block.Body) (rewardsTx map[string]data.TransactionHandler) { if !mp.outportHandler.HasDrivers() { @@ -1910,10 +1898,6 @@ func (mp *metaProcessor) checkShardHeadersValidity(metaHdr *block.MetaBlock) (ma } func (mp *metaProcessor) verifyShardDataAgainstHeaders(metaHdr *block.MetaBlock) error { - mp.selfNotarizedHeadersStaleOnce.Do(func() { - mp.selfNotarizedHeadersStale = mp.detectStaleSelfNotarizedHeaders() - }) - mp.hdrsForCurrBlock.mutHdrsForBlock.Lock() defer mp.hdrsForCurrBlock.mutHdrsForBlock.Unlock() @@ -1943,15 +1927,11 @@ func (mp *metaProcessor) verifyShardDataAgainstHeaders(metaHdr *block.MetaBlock) expected := mp.buildShardDataFromHeader(shardHdr, shardData.HeaderHash) expected.NumPendingMiniBlocks = uint32(len(mp.pendingMiniBlocksHandler.GetPendingMiniBlocks(expected.ShardID))) - if mp.selfNotarizedHeadersStale { - expected.LastIncludedMetaNonce = shardData.LastIncludedMetaNonce - } else { - lastSelfNotarizedHeader, _, err := mp.blockTracker.GetLastSelfNotarizedHeader(shardHdr.GetShardID()) - if err != nil { - return err - } - expected.LastIncludedMetaNonce = lastSelfNotarizedHeader.GetNonce() + lastSelfNotarizedHeader, _, err := mp.blockTracker.GetLastSelfNotarizedHeader(shardHdr.GetShardID()) + if err != nil { + return err } + expected.LastIncludedMetaNonce = lastSelfNotarizedHeader.GetNonce() if !expected.Equal(&shardData) { log.Debug("shard data mismatch", @@ -2270,14 +2250,14 @@ func (mp *metaProcessor) computeExistingAndRequestMissingShardHeaders(metaBlock mp.requestProofIfNeeded(shardData.HeaderHash, hdr) - if common.IsEpochChangeBlockForFlagActivation(hdr, mp.enableEpochsHandler, common.AndromedaFlag) { - continue - } - if hdr.GetNonce() > mp.hdrsForCurrBlock.highestHdrNonce[shardData.ShardID] { mp.hdrsForCurrBlock.highestHdrNonce[shardData.ShardID] = hdr.GetNonce() } + if common.IsEpochChangeBlockForFlagActivation(hdr, mp.enableEpochsHandler, common.AndromedaFlag) { + continue + } + mp.updateLastNotarizedBlockForShard(hdr, shardData.HeaderHash) } diff --git a/process/common.go b/process/common.go index 198645ea6ab..940c69b994e 100644 --- a/process/common.go +++ b/process/common.go @@ -27,6 +27,7 @@ import ( var log = logger.GetOrCreate("process") +const maxSelfNotarizedLookback = 50 const VMStoragePrefix = "VM@" // ShardedCacheSearchMethod defines the algorithm for searching through a sharded cache @@ -963,3 +964,102 @@ func CheckIfIndexesAreOutOfBound( return nil } + +// CompleteMissingSelfNotarizedHeaders walks backward through metablocks from startHash +// and derives self-notarized headers for shards that are still at genesis (nonce 0). +func CompleteMissingSelfNotarizedHeaders( + startHash []byte, + numShards uint32, + blockTracker BlockTracker, + marshalizer marshal.Marshalizer, + store dataRetriever.StorageService, +) { + missingShards := make(map[uint32]bool) + for shardID := uint32(0); shardID < numShards; shardID++ { + lastSelfNotarized, _, err := blockTracker.GetLastSelfNotarizedHeader(shardID) + if err != nil || check.IfNil(lastSelfNotarized) || lastSelfNotarized.GetNonce() == 0 { + missingShards[shardID] = true + } + } + + if len(missingShards) == 0 { + return + } + + log.Debug("CompleteMissingSelfNotarizedHeaders", + "numMissing", len(missingShards)) + + currentHash := startHash + for i := 0; i < maxSelfNotarizedLookback && len(missingShards) > 0 && len(currentHash) > 0; i++ { + metaBlock, err := GetMetaHeaderFromStorage(currentHash, marshalizer, store) + if err != nil { + break + } + + for shardID := range missingShards { + bestNonce, bestHeader, bestHash := findSelfNotarizedMetaHeaderInBlock(metaBlock, shardID, marshalizer, store) + if bestHeader != nil { + log.Debug("CompleteMissingSelfNotarizedHeaders: derived self-notarized header", + "shardID", shardID, + "metaNonce", bestNonce, + "metaHash", bestHash) + blockTracker.AddSelfNotarizedHeader(shardID, bestHeader, bestHash) + delete(missingShards, shardID) + } + } + + currentHash = metaBlock.GetPrevHash() + } + + if len(missingShards) > 0 { + log.Warn("CompleteMissingSelfNotarizedHeaders: could not derive all self-notarized headers", + "numStillMissing", len(missingShards)) + } +} + +func findSelfNotarizedMetaHeaderInBlock( + metaBlock *block.MetaBlock, + shardID uint32, + marshalizer marshal.Marshalizer, + store dataRetriever.StorageService, +) (uint64, data.HeaderHandler, []byte) { + var bestNonce uint64 + var bestHeader data.HeaderHandler + var bestHash []byte + hadLoadErrors := false + + for i := range metaBlock.ShardInfo { + if metaBlock.ShardInfo[i].ShardID != shardID { + continue + } + + shardHeader, err := GetShardHeaderFromStorage(metaBlock.ShardInfo[i].HeaderHash, marshalizer, store) + if err != nil { + log.Warn("findSelfNotarizedMetaHeaderInBlock: could not load shard header", + "shardID", shardID, + "headerHash", metaBlock.ShardInfo[i].HeaderHash, + "error", err.Error()) + hadLoadErrors = true + continue + } + + for _, metaHash := range shardHeader.GetMetaBlockHashes() { + metaHeader, errGet := GetMetaHeaderFromStorage(metaHash, marshalizer, store) + if errGet != nil { + continue + } + + if metaHeader.GetNonce() > bestNonce { + bestNonce = metaHeader.GetNonce() + bestHeader = metaHeader + bestHash = metaHash + } + } + } + + if hadLoadErrors { + return 0, nil, nil + } + + return bestNonce, bestHeader, bestHash +} diff --git a/process/sync/storageBootstrap/baseStorageBootstrapper.go b/process/sync/storageBootstrap/baseStorageBootstrapper.go index d42a9456f3d..3d5043ba542 100644 --- a/process/sync/storageBootstrap/baseStorageBootstrapper.go +++ b/process/sync/storageBootstrap/baseStorageBootstrapper.go @@ -409,6 +409,11 @@ func (st *storageBootstrapper) applyBootInfos(bootInfos []bootstrapStorage.Boots st.blockTracker.AddTrackedHeader(header, bootInfos[i].LastHeader.Hash) } + errComplete := st.bootstrapper.completeSelfNotarizedHeaders(bootInfos[0].LastHeader.Hash) + if errComplete != nil { + log.Warn("could not complete self notarized headers", "error", errComplete.Error()) + } + if len(bootInfos) == 1 { st.forkDetector.SetFinalToLastCheckpoint() } diff --git a/process/sync/storageBootstrap/interface.go b/process/sync/storageBootstrap/interface.go index c7e06cc6717..04bafb59d08 100644 --- a/process/sync/storageBootstrap/interface.go +++ b/process/sync/storageBootstrap/interface.go @@ -2,6 +2,7 @@ package storageBootstrap import ( "github.com/multiversx/mx-chain-core-go/data" + "github.com/multiversx/mx-chain-go/process/block/bootstrapStorage" ) @@ -12,6 +13,7 @@ type storageBootstrapperHandler interface { applyCrossNotarizedHeaders(crossNotarizedHeaders []bootstrapStorage.BootstrapHeaderInfo) error applyNumPendingMiniBlocks(pendingMiniBlocks []bootstrapStorage.PendingMiniBlocksInfo) applySelfNotarizedHeaders(selfNotarizedHeaders []bootstrapStorage.BootstrapHeaderInfo) ([]data.HeaderHandler, [][]byte, error) + completeSelfNotarizedHeaders(lastMetaBlockHash []byte) error cleanupNotarizedStorage(hash []byte) cleanupNotarizedStorageForHigherNoncesIfExist(crossNotarizedHeaders []bootstrapStorage.BootstrapHeaderInfo) getRootHash(hash []byte) []byte diff --git a/process/sync/storageBootstrap/metaStorageBootstrapper.go b/process/sync/storageBootstrap/metaStorageBootstrapper.go index c236018229f..69d8cd392d9 100644 --- a/process/sync/storageBootstrap/metaStorageBootstrapper.go +++ b/process/sync/storageBootstrap/metaStorageBootstrapper.go @@ -183,6 +183,17 @@ func (msb *metaStorageBootstrapper) applySelfNotarizedHeaders( return make([]data.HeaderHandler, 0), make([][]byte, 0), nil } +func (msb *metaStorageBootstrapper) completeSelfNotarizedHeaders(lastMetaBlockHash []byte) error { + process.CompleteMissingSelfNotarizedHeaders( + lastMetaBlockHash, + msb.shardCoordinator.NumberOfShards(), + msb.blockTracker, + msb.marshalizer, + msb.store, + ) + return nil +} + func (msb *metaStorageBootstrapper) applyNumPendingMiniBlocks(pendingMiniBlocksInfo []bootstrapStorage.PendingMiniBlocksInfo) { for _, pendingMiniBlockInfo := range pendingMiniBlocksInfo { msb.pendingMiniBlocksHandler.SetPendingMiniBlocks(pendingMiniBlockInfo.ShardID, pendingMiniBlockInfo.MiniBlocksHashes) diff --git a/process/sync/storageBootstrap/shardStorageBootstrapper.go b/process/sync/storageBootstrap/shardStorageBootstrapper.go index ebc8992df05..57ede4b0332 100644 --- a/process/sync/storageBootstrap/shardStorageBootstrapper.go +++ b/process/sync/storageBootstrap/shardStorageBootstrapper.go @@ -282,6 +282,10 @@ func (ssb *shardStorageBootstrapper) applySelfNotarizedHeaders( func (ssb *shardStorageBootstrapper) applyNumPendingMiniBlocks(_ []bootstrapStorage.PendingMiniBlocksInfo) { } +func (ssb *shardStorageBootstrapper) completeSelfNotarizedHeaders(_ []byte) error { + return nil +} + func (ssb *shardStorageBootstrapper) getRootHash(shardHeaderHash []byte) []byte { shardHeader, err := process.GetShardHeaderFromStorage(shardHeaderHash, ssb.marshalizer, ssb.store) if err != nil {