fix: bid is not closed after the manifest timeout

gateway/rest/router.go

@@ -110,7 +110,7 @@ func newRouter(log log.Logger, addr sdk.Address, pclient provider.Client, ctxCon
 		authorizeProviderMiddleware,
 		requireOwner,
 	)
-	
+
 	hostnameRouter := authedRouter.PathPrefix(apclient.HostnamePrefix).Subrouter()
 	hostnameRouter.HandleFunc(apclient.MigratePathPrefix,
 		migrateHandler(log, pclient.Hostname(), pclient.ClusterService())).

manifest/watchdog.go

@@ -75,17 +75,20 @@ func (wd *watchdog) run() {
 				ID:     mtypes.MakeBidID(wd.leaseID.OrderID(), wd.sess.Provider().Address()),
 				Reason: mtypes.LeaseClosedReasonManifestTimeout,
 			}
-
 			return runner.NewResult(wd.sess.Client().Tx().BroadcastMsgs(wd.ctx, []sdk.Msg{msg}, aclient.WithResultCodeAsError()))
 		})
 	case err = <-wd.lc.ShutdownRequest():
 	}
 
-	wd.lc.ShutdownInitiated(err)
 	if runch != nil {
-		result := <-runch
-		if err := result.Error(); err != nil {
-			wd.log.Error("failed closing bid", "err", err)
+		select {
+		case result := <-runch:
+			if err := result.Error(); err != nil {
+				wd.log.Error("failed closing bid", "err", err)
+			}
+		case err = <-wd.lc.ShutdownRequest():
+			// we need this case to skip waiting on runch when stop() is called
 		}
 	}
+	wd.lc.ShutdownInitiated(err)
 }

manifest/watchdog_test.go

@@ -29,6 +29,10 @@ type watchdogTestScaffold struct {
 }
 
 func makeWatchdogTestScaffold(t *testing.T, timeout time.Duration) (*watchdog, *watchdogTestScaffold) {
+	return makeWatchdogTestScaffoldWithBlocking(t, timeout, nil)
+}
+
+func makeWatchdogTestScaffoldWithBlocking(t *testing.T, timeout time.Duration, blockUntilRelease <-chan struct{}) (*watchdog, *watchdogTestScaffold) {
 	scaffold := &watchdogTestScaffold{}
 	scaffold.parentCh = make(chan struct{})
 	scaffold.doneCh = make(chan dtypes.DeploymentID, 1)
@@ -39,6 +43,9 @@ func makeWatchdogTestScaffold(t *testing.T, timeout time.Duration) (*watchdog, *
 
 	txClientMock := &clientmocks.TxClient{}
 	txClientMock.On("BroadcastMsgs", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
+		if blockUntilRelease != nil {
+			<-blockUntilRelease
+		}
 		scaffold.broadcasts <- args.Get(1).([]sdk.Msg)
 	}).Return(&sdk.Result{}, nil)
 
@@ -72,6 +79,7 @@ func TestWatchdogTimeout(t *testing.T) {
 
 	msg := msgs[0].(*mvbeta.MsgCloseBid)
 	require.Equal(t, scaffold.leaseID, msg.ID.LeaseID())
+	require.Equal(t, mtypes.LeaseClosedReasonManifestTimeout, msg.Reason)
 
 	deploymentID := testutil.ChannelWaitForValue(t, scaffold.doneCh)
 	require.Equal(t, deploymentID, scaffold.leaseID.DeploymentID())
@@ -121,3 +129,22 @@ func TestWatchdogStopsOnParent(t *testing.T) {
 	deploymentID := testutil.ChannelWaitForValue(t, scaffold.doneCh)
 	require.Equal(t, deploymentID, scaffold.leaseID.DeploymentID())
 }
+
+func TestWatchdogStopWhileWaitingForBroadcast(t *testing.T) {
+	releaseCh := make(chan struct{})
+	wd, scaffold := makeWatchdogTestScaffoldWithBlocking(t, 100*time.Millisecond, releaseCh)
+
+	<-time.After(200 * time.Millisecond)
+	wd.stop()
+
+	select {
+	case <-wd.lc.Done():
+	case <-time.After(5 * time.Second):
+		t.Fatal("deadlock: stop() blocked while watchdog was waiting on broadcast")
+	}
+
+	close(releaseCh)
+
+	deploymentID := testutil.ChannelWaitForValue(t, scaffold.doneCh)
+	require.Equal(t, deploymentID, scaffold.leaseID.DeploymentID())
+}