add retry logic

Author: PatStiles

core/chainio/retryable.go

@@ -299,3 +299,22 @@ func SubscribeToNewTasksV3Retryable(
 ) (event.Subscription, error) {
 	return retry.RetryWithData(SubscribeToNewTasksV3(opts, serviceManager, newTaskCreatedChan, batchMerkleRoot), retry.DefaultRetryConfig())
 }
+
+// |---AVS Reader---|
+
+func DisabledVerifiers(r *AvsReader, opts *bind.CallOpts) func() (*big.Int, error) {
+	disabled_ver_func := func() (*big.Int, error) {
+		return r.AvsContractBindings.ServiceManager.ContractAlignedLayerServiceManagerCaller.DisabledVerifiers(opts)
+	}
+	return disabled_ver_func
+}
+
+/*
+Retrieves the Verifiers Blacklist Bitmap from the `AlignedServiceManager` Contract.
+- All errors are considered Transient Errors
+- Retry times (3 retries): 12 sec (1 Blocks), 24 sec (2 Blocks), 48 sec (4 Blocks)
+- NOTE: Contract call reverts are not considered `PermanentError`'s as block reorg's may lead to contract call revert in which case the aggregator should retry.
+*/
+func (r *AvsReader) DisabledVerifiersRetryable(opts *bind.CallOpts) (*big.Int, error) {
+	return retry.RetryWithData(DisabledVerifiers(r, opts), retry.ChainRetryConfig())
+}

core/retry_test.go

@@ -928,3 +928,51 @@ func TestBatchersBalances(t *testing.T) {
 		return
 	}
 }
+
+func TestDisabledVerifiers(t *testing.T) {
+	cmd, _, err := SetupAnvil(8545)
+	if err != nil {
+		t.Errorf("Error setting up Anvil: %s\n", err)
+	}
+
+	aggregatorConfig := config.NewAggregatorConfig("../config-files/config-aggregator-test.yaml")
+	avsReader, err := chainio.NewAvsReaderFromConfig(aggregatorConfig.BaseConfig, aggregatorConfig.EcdsaConfig)
+	if err != nil {
+		log.Fatalf("Could not create AVS reader")
+	}
+
+	dis_ver_func := chainio.DisabledVerifiers(avsReader, &bind.CallOpts{})
+	_, err = dis_ver_func()
+	assert.Nil(t, err)
+
+	if err := cmd.Process.Kill(); err != nil {
+		t.Errorf("Error killing process: %v\n", err)
+		return
+	}
+
+	dis_ver_func = chainio.DisabledVerifiers(avsReader, &bind.CallOpts{})
+	_, err = dis_ver_func()
+	assert.NotNil(t, err)
+	if _, ok := err.(retry.PermanentError); ok {
+		t.Errorf("DisabledVerifiers Emitted non-Transient error: %s\n", err)
+		return
+	}
+	if !strings.Contains(err.Error(), "connection reset") {
+		t.Errorf(" did not return expected error: %s\n", err)
+		return
+	}
+
+	cmd, _, err = SetupAnvil(8545)
+	if err != nil {
+		t.Errorf("Error setting up Anvil: %s\n", err)
+	}
+
+	dis_ver_func = chainio.DisabledVerifiers(avsReader, &bind.CallOpts{})
+	_, err = dis_ver_func()
+	assert.Nil(t, err)
+
+	if err := cmd.Process.Kill(); err != nil {
+		t.Errorf("Error killing process: %v\n", err)
+		return
+	}
+}

operator/pkg/operator.go

@@ -15,6 +15,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/urfave/cli/v2"
 	"github.com/yetanotherco/aligned_layer/operator/risc_zero"
@@ -372,7 +373,7 @@ func (o *Operator) ProcessNewBatchLogV2(newBatchLog *servicemanager.ContractAlig
 	var wg sync.WaitGroup
 	wg.Add(verificationDataBatchLen)
 
-	disabledVerifiersBitmap, err := o.avsReader.DisabledVerifiers()
+	disabledVerifiersBitmap, err := o.avsReader.DisabledVerifiersRetryable(&bind.CallOpts{})
 	if err != nil {
 		o.Logger.Errorf("Could not check verifiers status: %s", err)
 		results <- false
@@ -452,7 +453,7 @@ func (o *Operator) ProcessNewBatchLogV3(newBatchLog *servicemanager.ContractAlig
 	results := make(chan bool, verificationDataBatchLen)
 	var wg sync.WaitGroup
 	wg.Add(verificationDataBatchLen)
-	disabledVerifiersBitmap, err := o.avsReader.DisabledVerifiers()
+	disabledVerifiersBitmap, err := o.avsReader.DisabledVerifiersRetryable(&bind.CallOpts{})
 	if err != nil {
 		o.Logger.Errorf("Could not check verifiers status: %s", err)
 		results <- false