feat(turnkey): Integrate Turnkey support for remote signer functionality

.github/workflows/build.yaml

@@ -56,9 +56,7 @@ jobs:
       - name: Setup ubuntu container
         run: |
           apt update
-          apt install -yqq build-essential make software-properties-common
-          add-apt-repository -y ppa:git-core/candidate
-          apt update && apt install -yqq git zip unzip zlib1g-dev zlib1g libzlcore-dev yasm
+          apt install -yqq build-essential make git zip unzip zlib1g-dev zlib1g libzlcore-dev yasm
 
       - name: Check out code
         uses: actions/checkout@v4
@@ -97,15 +95,24 @@ jobs:
 
       - name: Install dependencies
         run: |
-          apt update \
-            && apt install -yqq software-properties-common curl apt-transport-https lsb-release \
-            && curl -fsSl https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \
-            && add-apt-repository "deb https://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-14 main" \
-            && apt update \
-            && apt -yqq install \
-              nasm clang-14 clang-tools-14 lld-14 build-essential pkg-config autoconf git python3 \
-              gcc-mingw-w64 libgcc-9-dev-arm64-cross mingw-w64-tools gcc-mingw-w64-x86-64 mingw-w64-x86-64-dev \
-              golang-goprotobuf-dev protobuf-compiler-grpc
+          apt update
+          apt install -yqq ca-certificates curl lsb-release
+
+          if ! apt-cache show clang-14 >/dev/null 2>&1; then
+            install -d -m 0755 /usr/share/keyrings
+            curl -fsSL --retry 5 --retry-delay 2 --connect-timeout 15 \
+              -o /usr/share/keyrings/apt.llvm.org.asc \
+              https://apt.llvm.org/llvm-snapshot.gpg.key
+            printf '%s\n' \
+              "deb [signed-by=/usr/share/keyrings/apt.llvm.org.asc] https://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-14 main" \
+              > /etc/apt/sources.list.d/llvm-toolchain-14.list
+            apt update
+          fi
+
+          apt install -yqq \
+            nasm clang-14 clang-tools-14 lld-14 build-essential pkg-config autoconf git python3 \
+            gcc-mingw-w64 libgcc-9-dev-arm64-cross mingw-w64-tools gcc-mingw-w64-x86-64 mingw-w64-x86-64-dev \
+            golang-goprotobuf-dev protobuf-compiler-grpc
 
           update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-14 30 \
             && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 30 \
@@ -124,11 +131,13 @@ jobs:
         if: matrix.target.GOOS == 'windows'
         env:
           CC: x86_64-w64-mingw32-gcc
+          ZLIB_VERSION: 1.2.11
         run: |
-          apt install -yqq wget
-          wget https://zlib.net/fossils/zlib-1.2.11.tar.gz
-          tar -xzf zlib-1.2.11.tar.gz
-          cd zlib-1.2.11
+          curl -fsSL --retry 5 --retry-delay 2 --connect-timeout 15 \
+            -o "zlib-${ZLIB_VERSION}.tar.gz" \
+            "https://github.com/madler/zlib/archive/refs/tags/v${ZLIB_VERSION}.tar.gz"
+          tar -xzf "zlib-${ZLIB_VERSION}.tar.gz"
+          cd "zlib-${ZLIB_VERSION}"
           ./configure --prefix=/usr/x86_64-w64-mingw32 --static
           make
           make install

CHANGELOG_PENDING.md

@@ -4,10 +4,14 @@
 
 ### Breaking Changes 🚨🚨
 
+* [#3959](https://github.com/livepeer/go-livepeer/pull/3959) remote-signer: Refuse to start when `/generate-live-payment` would be unauthenticated (`-remoteSignerWebhookUrl` unset) on a publicly-accessible `-httpAddr`; pass `-remoteSignerAllowNoAuth` to override (@rickstaa)
+
 ### Features ⚒
 
 #### General
 
+- [#3944](https://github.com/livepeer/go-livepeer/pull/3944) Bridge slog level to the glog `-v` flag so `-v` controls newer subsystem logging (@rickstaa)
+
 #### Broadcaster
 
 #### Orchestrator
@@ -18,6 +22,8 @@
 
 #### General
 
+* [#3962](https://github.com/livepeer/go-livepeer/pull/3962) remote-signer: Default `-cliAddr` to a loopback address in remote signer mode so the node no longer fails to start by binding the CLI server to `:80` (@rickstaa)
+
 #### Broadcaster
 
 * [a6c4b1e](https://github.com/livepeer/go-livepeer/commit/a6c4b1ef70d8f4d3da0e7d8164ac8d1faf80ad0e) pm: Reject ticket params with a zero expiration block so they are subjected to the economic caps (@rickstaa)

README.md

@@ -38,7 +38,7 @@ This project requires `go` and a unix shell.
 
 ## Getting Started
 
-To get started, clone the repo and follow the [installation guide](https://docs.livepeer.org/guides/orchestrating/install-go-livepeer).
+To get started, clone the repo and follow the [installation guide](https://docs.livepeer.org/v2/orchestrators/setup/install).
 
 Next, follow [the guide to set up a private ETH network with the Livepeer protocol deployed](cmd/devtool/README.md).
 

cmd/devtool/devtool/devtool_utils.go

@@ -289,7 +289,15 @@ func (d *Devtool) InitializeRound() error {
 	} else {
 		err = d.Client.CheckTx(tx)
 		if err != nil {
-			glog.Errorf("Error initializng round: %v", err)
+			initialized, initErr := d.Client.CurrentRoundInitialized()
+			if initErr == nil && initialized {
+				glog.Infof("Round initialized despite failed transaction: %v", err)
+				return nil
+			}
+			if initErr != nil {
+				glog.Errorf("Error checking initialized round after failed transaction: %v", initErr)
+			}
+			glog.Errorf("Error initializing round: %v", err)
 			return err
 		}
 	}

cmd/livepeer/livepeer.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"log/slog"
 	"os"
 	"os/signal"
 	"runtime"
@@ -53,6 +54,12 @@ func main() {
 
 	vFlag.Value.Set(*verbosity)
 
+	// Bridge slog (used by newer subsystems) to the glog -v level until we
+	// migrate off glog; without this, slog.Debug is dropped regardless of -v.
+	if glog.V(6) {
+		slog.SetLogLoggerLevel(slog.LevelDebug)
+	}
+
 	if *mistJSON {
 		mistconnector.PrintMistConfigJson(
 			"livepeer",

cmd/livepeer/starter/flags.go

@@ -140,10 +140,14 @@ func NewLivepeerConfig(fs *flag.FlagSet) LivepeerConfig {
 	cfg.TestOrchAvail = fs.Bool("startupAvailabilityCheck", *cfg.TestOrchAvail, "Set to false to disable the startup Orchestrator availability check on the configured serviceAddr")
 	cfg.RemoteSigner = fs.Bool("remoteSigner", *cfg.RemoteSigner, "Set to true to run remote signer service")
 	cfg.RemoteSignerUrl = fs.String("remoteSignerUrl", *cfg.RemoteSignerUrl, "URL of remote signer service to use (e.g., http://localhost:8935). Gateway only.")
+	cfg.RemoteSignerAddress = fs.String("remoteSignerAddress", *cfg.RemoteSignerAddress, "Gateway only. Optional Ethereum address to pin remote signer identity when the signer uses Turnkey multi-address mode")
 	cfg.RemoteSignerHeaders = fs.String("remoteSignerHeaders", *cfg.RemoteSignerHeaders, "Map of headers to use for gateway remote signer requests. e.g. 'header:val,header2:val2'")
 	cfg.RemoteSignerWebhookURL = fs.String("remoteSignerWebhookUrl", *cfg.RemoteSignerWebhookURL, "Authentication webhook URL called by remote signer during GenerateLivePayment")
 	cfg.RemoteSignerWebhookHeaders = fs.String("remoteSignerWebhookHeaders", *cfg.RemoteSignerWebhookHeaders, "Map of headers to use for remote signer webhook requests. e.g. 'header:val,header2:val2'")
+	cfg.RemoteSignerAllowNoAuth = fs.Bool("remoteSignerAllowNoAuth", *cfg.RemoteSignerAllowNoAuth, "Allow an unauthenticated remote signer on a public -httpAddr (no webhook). UNSAFE: signs payments from this node's deposit for any reachable caller; restrict access externally (proxy/private network).")
 	cfg.RemoteDiscovery = fs.Bool("remoteDiscovery", *cfg.RemoteDiscovery, "Enable orchestrator discovery on remote signers")
+	cfg.TurnkeyOrg = fs.String("turnkeyOrg", *cfg.TurnkeyOrg, "Remote signer only. Turnkey organization id; when set, Ethereum keys are managed in Turnkey instead of a local keystore")
+	cfg.TurnkeyApiKeyName = fs.String("turnkeyApiKeyName", *cfg.TurnkeyApiKeyName, "Name of the Turnkey API key in ~/.turnkey/keys/<name>/ (used with -turnkeyOrg)")
 
 	// Gateway metrics
 	cfg.KafkaBootstrapServers = fs.String("kafkaBootstrapServers", *cfg.KafkaBootstrapServers, "URL of Kafka Bootstrap Servers")

cmd/livepeer/starter/starter.go

@@ -44,6 +44,7 @@ import (
 	"github.com/livepeer/livepeer-data/pkg/event"
 	"github.com/livepeer/lpms/ffmpeg"
 	"github.com/olekukonko/tablewriter"
+	sdk "github.com/tkhq/go-sdk"
 )
 
 var (
@@ -75,6 +76,7 @@ const (
 	OrchestratorCliPort = "7935"
 	TranscoderCliPort   = "6935"
 	AIWorkerCliPort     = "4935"
+	RemoteSignerCliPort = "3935"
 
 	RefreshPerfScoreInterval = 10 * time.Minute
 )
@@ -169,10 +171,14 @@ type LivepeerConfig struct {
 	TestOrchAvail              *bool
 	RemoteSigner               *bool
 	RemoteSignerUrl            *string
+	RemoteSignerAddress        *string
 	RemoteSignerHeaders        *string
 	RemoteSignerWebhookURL     *string
 	RemoteSignerWebhookHeaders *string
+	RemoteSignerAllowNoAuth    *bool
 	RemoteDiscovery            *bool
+	TurnkeyOrg                 *string
+	TurnkeyApiKeyName          *string
 	AIRunnerImage              *string
 	AIRunnerImageOverrides     *string
 	AIVerboseLogs              *bool
@@ -310,10 +316,14 @@ func DefaultLivepeerConfig() LivepeerConfig {
 	defaultTestOrchAvail := true
 	defaultRemoteSigner := false
 	defaultRemoteSignerUrl := ""
+	defaultRemoteSignerAddress := ""
 	defaultRemoteSignerHeaders := ""
 	defaultRemoteSignerWebhookURL := ""
 	defaultRemoteSignerWebhookHeaders := ""
+	defaultRemoteSignerAllowNoAuth := false
 	defaultRemoteDiscovery := false
+	defaultTurnkeyOrg := ""
+	defaultTurnkeyApiKeyName := "default"
 
 	// Gateway logs
 	defaultKafkaBootstrapServers := ""
@@ -437,10 +447,14 @@ func DefaultLivepeerConfig() LivepeerConfig {
 		TestOrchAvail:              &defaultTestOrchAvail,
 		RemoteSigner:               &defaultRemoteSigner,
 		RemoteSignerUrl:            &defaultRemoteSignerUrl,
+		RemoteSignerAddress:        &defaultRemoteSignerAddress,
 		RemoteSignerHeaders:        &defaultRemoteSignerHeaders,
 		RemoteSignerWebhookURL:     &defaultRemoteSignerWebhookURL,
 		RemoteSignerWebhookHeaders: &defaultRemoteSignerWebhookHeaders,
+		RemoteSignerAllowNoAuth:    &defaultRemoteSignerAllowNoAuth,
 		RemoteDiscovery:            &defaultRemoteDiscovery,
+		TurnkeyOrg:                 &defaultTurnkeyOrg,
+		TurnkeyApiKeyName:          &defaultTurnkeyApiKeyName,
 
 		// Gateway logs
 		KafkaBootstrapServers: &defaultKafkaBootstrapServers,
@@ -486,6 +500,8 @@ func (cfg LivepeerConfig) PrintConfig(w io.Writer) {
 }
 
 func StartLivepeer(ctx context.Context, cfg LivepeerConfig) {
+	var turnkeyAdminClient *sdk.Client
+
 	if *cfg.MaxSessions == "auto" && *cfg.Orchestrator {
 		if *cfg.Transcoder {
 			glog.Exit("-maxSessions 'auto' cannot be used when both -orchestrator and -transcoder are specified")
@@ -705,6 +721,9 @@ func StartLivepeer(ctx context.Context, cfg LivepeerConfig) {
 			exit("Remote signer mode requires on-chain network")
 		}
 	}
+	if *cfg.TurnkeyOrg != "" && !*cfg.RemoteSigner {
+		exit("-turnkeyOrg requires -remoteSigner")
+	}
 
 	if *cfg.Redeemer {
 		n.NodeType = core.RedeemerNode
@@ -846,15 +865,72 @@ func StartLivepeer(ctx context.Context, cfg LivepeerConfig) {
 		}
 		defer gpm.Stop()
 
-		am, err := eth.NewAccountManager(ethcommon.HexToAddress(*cfg.EthAcctAddr), keystoreDir, chainID, *cfg.EthPassword)
-		if err != nil {
-			glog.Errorf("Error creating Ethereum account manager: %v", err)
-			return
-		}
+		var am eth.AccountManager
+		if *cfg.TurnkeyOrg != "" {
+			if n.NodeType != core.RemoteSignerNode {
+				glog.Exit("-turnkeyOrg is only supported when running as a remote signer (-remoteSigner)")
+			}
+			tkClient, err := sdk.New(sdk.WithAPIKeyName(*cfg.TurnkeyApiKeyName))
+			if err != nil {
+				glog.Exit("Failed to create Turnkey client: ", err)
+			}
+			turnkeyAdminClient = tkClient
+			orgID := *cfg.TurnkeyOrg
+			accts, err := eth.ListTurnkeyEthereumAccounts(tkClient, orgID)
+			if err != nil {
+				glog.Exit("Failed to list Turnkey Ethereum accounts: ", err)
+			}
+			var signAddr ethcommon.Address
+			if *cfg.EthAcctAddr != "" {
+				signAddr = ethcommon.HexToAddress(*cfg.EthAcctAddr)
+				found := false
+				for _, a := range accts {
+					if a.Address == signAddr {
+						found = true
+						break
+					}
+				}
+				if !found {
+					glog.Exit("-ethAcctAddr does not match any Turnkey Ethereum account in the organization")
+				}
+			} else if len(accts) > 0 {
+				signAddr = accts[0].Address
+			} else {
+				wname := fmt.Sprintf("livepeer-remote-signer-%d", time.Now().Unix())
+				_, addr, err := eth.TurnkeyCreateWallet(tkClient, orgID, wname)
+				if err != nil {
+					glog.Exit("No Turnkey wallets in org and failed to create one: ", err)
+				}
+				glog.Infof("Created Turnkey wallet with default Ethereum address %s", addr.Hex())
+				signAddr = addr
+				accts, err = eth.ListTurnkeyEthereumAccounts(tkClient, orgID)
+				if err != nil {
+					glog.Errorf("Warning: failed to refresh Turnkey account list: %v", err)
+					accts = []eth.TurnkeyWalletAccount{{OrganizationID: orgID, Address: signAddr}}
+				}
+			}
+			tkAm := eth.NewTurnkeyAccountManager(tkClient, orgID, chainID, signAddr)
+			am = tkAm
+			n.TurnkeyMode = true
+			n.TurnkeyOrgID = orgID
+			n.TurnkeyAccount = tkAm
+			addrList := make([]ethcommon.Address, 0, len(accts))
+			for _, a := range accts {
+				addrList = append(addrList, a.Address)
+			}
+			n.ReplaceTurnkeyAddressBook(addrList)
+		} else {
+			var err error
+			am, err = eth.NewAccountManager(ethcommon.HexToAddress(*cfg.EthAcctAddr), keystoreDir, chainID, *cfg.EthPassword)
+			if err != nil {
+				glog.Errorf("Error creating Ethereum account manager: %v", err)
+				return
+			}
 
-		if err := am.Unlock(*cfg.EthPassword); err != nil {
-			glog.Errorf("Error unlocking Ethereum account: %v", err)
-			return
+			if err := am.Unlock(*cfg.EthPassword); err != nil {
+				glog.Errorf("Error unlocking Ethereum account: %v", err)
+				return
+			}
 		}
 
 		tm := eth.NewTransactionManager(backend, gpm, am, *cfg.TxTimeout, *cfg.MaxTxReplacements)
@@ -1628,7 +1704,12 @@ func StartLivepeer(ctx context.Context, cfg LivepeerConfig) {
 			}
 
 			glog.Info("Retrieving OrchestratorInfo fields from remote signer: ", url)
-			fields, err := server.GetOrchInfoSig(url, n.RemoteSignerHeaders)
+			pinAddr := ""
+			if *cfg.RemoteSignerAddress != "" {
+				pinAddr = *cfg.RemoteSignerAddress
+				n.GatewayRemoteSignerAddress = ethcommon.HexToAddress(pinAddr)
+			}
+			fields, err := server.GetOrchInfoSig(url, pinAddr, n.RemoteSignerHeaders)
 			if err != nil {
 				glog.Exit("Unable to query remote signer: ", err)
 			}
@@ -1778,6 +1859,8 @@ func StartLivepeer(ctx context.Context, cfg LivepeerConfig) {
 		*cfg.CliAddr = defaultAddr(*cfg.CliAddr, "127.0.0.1", TranscoderCliPort)
 	} else if n.NodeType == core.AIWorkerNode {
 		*cfg.CliAddr = defaultAddr(*cfg.CliAddr, "127.0.0.1", AIWorkerCliPort)
+	} else if n.NodeType == core.RemoteSignerNode {
+		*cfg.CliAddr = defaultAddr(*cfg.CliAddr, "127.0.0.1", RemoteSignerCliPort)
 	}
 
 	// Apply default capabilities if not running as a transcoder.
@@ -1860,6 +1943,7 @@ func StartLivepeer(ctx context.Context, cfg LivepeerConfig) {
 	if err != nil {
 		exit("Error creating Livepeer server: err=%q", err)
 	}
+	s.TurnkeyAdmin = turnkeyAdminClient
 
 	ec := make(chan error)
 	tc := make(chan struct{})
@@ -1947,9 +2031,25 @@ func StartLivepeer(ctx context.Context, cfg LivepeerConfig) {
 		// Start remote signer server
 		go func() {
 			*cfg.HttpAddr = defaultAddr(*cfg.HttpAddr, "127.0.0.1", OrchestratorRpcPort)
+
+			// Refuse to start a public signer with no webhook auth. It would sign payments
+			// from this node's deposit for any caller (override with -remoteSignerAllowNoAuth).
+			if n.RemoteSignerWebhookURL == nil {
+				isLocalHTTP, err := isLocalURL("https://" + *cfg.HttpAddr)
+				if err != nil {
+					exit("Error checking for local -httpAddr: %v", err)
+				}
+				if !isLocalHTTP && !*cfg.RemoteSignerAllowNoAuth {
+					exit("Refusing to start: remote signer on public -httpAddr %s with no "+
+						"-remoteSignerWebhookUrl signs payments from this node's deposit for any caller. "+
+						"Set the webhook, or pass -remoteSignerAllowNoAuth to override.", *cfg.HttpAddr)
+				}
+				glog.Warning("WARNING: remote signer has no webhook auth. /generate-live-payment is " +
+					"UNAUTHENTICATED and signs payments from this node's deposit. Set -remoteSignerWebhookUrl.")
+			}
+
 			glog.Info("Starting remote signer server on ", *cfg.HttpAddr)
-			err := server.StartRemoteSignerServer(s, *cfg.HttpAddr)
-			if err != nil {
+			if err := server.StartRemoteSignerServer(s, *cfg.HttpAddr); err != nil {
 				exit("Error starting remote signer server: err=%q", err)
 			}
 		}()