From a334ecd243651840c8561cf263e49bbfeaee1cac Mon Sep 17 00:00:00 2001 From: logan Date: Thu, 16 Jan 2025 12:28:15 -0600 Subject: [PATCH 01/27] feat: barebones --- docker/preflight/Dockerfile | 56 ++++++++++++++ .../preflight/docker-compose-pre-flight.yml | 44 +++++++++++ docker/preflight/docker-compose.orbit.yml | 65 ++++++++++++++++ docker/preflight/preflight.go | 47 ++++++++++++ pkg/resourceprovider/preflight/gpu.go | 76 +++++++++++++++++++ pkg/resourceprovider/preflight/types.go | 38 ++++++++++ 6 files changed, 326 insertions(+) create mode 100644 docker/preflight/Dockerfile create mode 100644 docker/preflight/docker-compose-pre-flight.yml create mode 100644 docker/preflight/docker-compose.orbit.yml create mode 100644 docker/preflight/preflight.go create mode 100644 pkg/resourceprovider/preflight/gpu.go create mode 100644 pkg/resourceprovider/preflight/types.go diff --git a/docker/preflight/Dockerfile b/docker/preflight/Dockerfile new file mode 100644 index 000000000..21b47d8cc --- /dev/null +++ b/docker/preflight/Dockerfile @@ -0,0 +1,56 @@ +# FROM alpine:latest AS base +# WORKDIR /usr/src/app + +# ARG network=dev + +# COPY . . +# RUN go mod download && go mod verify +# RUN go build -v . +# RUN go install + +# RUN (curl -Ls --tlsv1.2 --proto "=https" --retry 3 https://cli.doppler.com/install.sh || wget -t 3 -qO- https://cli.doppler.com/install.sh) | sh + +# RUN touch run +# RUN echo "#!/bin/bash" >> run +# RUN if [ "${network}" = "dev" ]; then \ +# echo "lilypad jobcreator --network $network" >> run; \ +# else \ +# echo "doppler run -- lilypad jobcreator --network $network" >> run; \ +# fi +# RUN chmod +x run +# WORKDIR /app +# COPY ./preflight /app +# RUN chmod +x /app/preflight +# # CMD ["/app/preflight"] +# # CMD ["/bin/sh"] +# CMD tail -f /dev/null +FROM golang:1.23 AS builder +WORKDIR /app +# RUN pwd +COPY ./preflight.go /app +# COPY . . +# RUN cd preflight +RUN go mod init preflight +RUN go mod tidy +RUN go build -o ./preflight preflight.go + +FROM nvidia/cuda:12.0.1-cudnn8-devel-ubuntu22.04 AS base + +RUN apt-get update && \ + apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg \ + lsb-release && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ + echo "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \ + apt-get update && \ + apt-get install -y docker-ce docker-ce-cli containerd.io && \ + rm -rf /var/lib/apt/lists/* + + +WORKDIR /app +COPY --from=builder /app . +RUN chmod +x /app/preflight +# CMD ["/app/preflight","arsenum"] \ No newline at end of file diff --git a/docker/preflight/docker-compose-pre-flight.yml b/docker/preflight/docker-compose-pre-flight.yml new file mode 100644 index 000000000..f74038aa2 --- /dev/null +++ b/docker/preflight/docker-compose-pre-flight.yml @@ -0,0 +1,44 @@ +version: '3.8' + +services: + preflight: + build: + context: . + dockerfile: ./Dockerfile + image: preflight + container_name: ${ORG:-default_org}_preflight + volumes: + - /var/run/docker.sock:/var/run/docker.sock + runtime: nvidia # Ensure nvidia runtime is available + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: ["/app/preflight", "check-gpu"] # Assuming your binary has a check-gpu command + environment: + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=compute,utility + + resource-provider: + image: ghcr.io/lilypad-tech/resource-provider:latest + container_name: ${ORG:-default_org}_resource-provider + runtime: nvidia + depends_on: + preflight: + condition: service_completed_successfully + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=compute,utility + - OFFER_GPU=1s \ No newline at end of file diff --git a/docker/preflight/docker-compose.orbit.yml b/docker/preflight/docker-compose.orbit.yml new file mode 100644 index 000000000..5b7d6a742 --- /dev/null +++ b/docker/preflight/docker-compose.orbit.yml @@ -0,0 +1,65 @@ +name: lilypad_orbit +services: + preflight: + build: + context: . + dockerfile: ./Dockerfile + image: preflight + container_name: orbit_preflight + runtime: nvidia + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=compute,utility + networks: + - localnet + command: ["/app/preflight", "check-gpu"] + + resource-provider: + image: ghcr.io/lilypad-tech/resource-provider:latest + container_name: orbit_resource-provider + runtime: nvidia + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - lilypad-data:/tmp/lilypad/data + environment: + - BACALHAU_API_HOST=orbit_bacalhau + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=compute,utility + - OFFER_GPU=1 + # Keeping original orbit environment variables + - WEB3_RPC_URL=wss://wss.orbit.arsenum.com + - WEB3_CHAIN_ID=68283778764 + - WEB3_PRIVATE_KEY=0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a + - WEB3_CONTROLLER_ADDRESS=0xa85233C63b9Ee964Add6F2cffe00Fd84eb32338f + - WEB3_PAYMENTS_ADDRESS=0x2279B7A0a67DB372996a5FaB50D91eAA73d2eBe6 + - WEB3_POW_ADDRESS=0x4ed7c70F96B99c776995fB64377f0d4aB3B0e1C1 + - WEB3_STORAGE_ADDRESS=0xB7f8BC63BbcaD18155201308C8f3540b07f84F5e + - WEB3_TOKEN_ADDRESS=0xa513E6E4b8f2a923D98304ec87F64353C4D5C853 + - WEB3_USERS_ADDRESS=0x0DCd1Bf9A1b36cE34237eEaFef220932846BCD82 + - LOG_LEVEL=debug + networks: + - localnet + +volumes: + bacalhau-data: + lilypad-data: + +networks: + localnet: + external: true \ No newline at end of file diff --git a/docker/preflight/preflight.go b/docker/preflight/preflight.go new file mode 100644 index 000000000..97f06e247 --- /dev/null +++ b/docker/preflight/preflight.go @@ -0,0 +1,47 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "strings" +) + +func main() { + if len(os.Args) < 2 || os.Args[1] != "check-gpu" { + fmt.Println("Usage: preflight check-gpu") + os.Exit(1) + } + + // Check nvidia-smi is available + if _, err := exec.LookPath("nvidia-smi"); err != nil { + fmt.Println("Error: nvidia-smi not found. Please ensure NVIDIA drivers are installed") + os.Exit(1) + } + + // Run nvidia-smi to check GPU availability + cmd := exec.Command("nvidia-smi") + output, err := cmd.CombinedOutput() + if err != nil { + fmt.Printf("Error running nvidia-smi: %v\n", err) + os.Exit(1) + } + + // Check docker runtime + cmd = exec.Command("docker", "info", "--format", "'{{.Runtimes}}'") + runtimeOutput, err := cmd.CombinedOutput() + if err != nil { + fmt.Printf("Error checking Docker runtimes: %v\n", err) + os.Exit(1) + } + + if !strings.Contains(string(runtimeOutput), "nvidia") { + fmt.Println("Error: NVIDIA runtime not found in Docker") + os.Exit(1) + } + + fmt.Println("GPU Check Output:") + fmt.Println(string(output)) + fmt.Println("NVIDIA runtime is available") + fmt.Println("All preflight checks passed successfully") +} diff --git a/pkg/resourceprovider/preflight/gpu.go b/pkg/resourceprovider/preflight/gpu.go new file mode 100644 index 000000000..ec76ffafc --- /dev/null +++ b/pkg/resourceprovider/preflight/gpu.go @@ -0,0 +1,76 @@ +package preflight + +import ( + "context" + "encoding/json" + "fmt" + "os/exec" +) + +type nvidiaSmiResponse struct { + GPU []struct { + UUID string `json:"uuid"` + ProductName string `json:"product_name"` + Memory int64 `json:"memory_total"` + DriverVersion string `json:"driver_version"` + } `json:"gpu"` +} + +func checkNvidiaSMI() error { + _, err := exec.LookPath("nvidia-smi") + return err +} + +func (v *preflight) GetAvailableGPUs(ctx context.Context) ([]GPUInfo, error) { + if err := checkNvidiaSMI(); err != nil { + return nil, fmt.Errorf("nvidia-smi not found: %w", err) + } + + cmd := exec.CommandContext(ctx, "nvidia-smi", "--query-gpu=gpu_uuid,gpu_name,memory.total,driver_version", "--format=json") + output, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("error running nvidia-smi: %w", err) + } + + var response nvidiaSmiResponse + if err := json.Unmarshal(output, &response); err != nil { + return nil, fmt.Errorf("error parsing nvidia-smi output: %w", err) + } + + gpus := make([]GPUInfo, len(response.GPU)) + for i, gpu := range response.GPU { + gpus[i] = GPUInfo{ + UUID: gpu.UUID, + Name: gpu.ProductName, + MemoryTotal: gpu.Memory, + DriverVersion: gpu.DriverVersion, + } + } + + return gpus, nil +} + +func (v *preflight) PreflightGPU(ctx context.Context, req *GPURequirements) PreflightResult { + gpus, err := v.GetAvailableGPUs(ctx) + if err != nil { + return PreflightResult{ + Success: false, + Error: err, + Details: "Failed to get GPU information", + } + } + + if len(gpus) < req.MinGPUs { + return PreflightResult{ + Success: false, + Details: fmt.Sprintf("Insufficient GPUs. Required: %d, Available: %d", req.MinGPUs, len(gpus)), + } + } + + // Additional validation logic for memory and capabilities can be added here + + return PreflightResult{ + Success: true, + Details: fmt.Sprintf("Found %d suitable GPUs", len(gpus)), + } +} diff --git a/pkg/resourceprovider/preflight/types.go b/pkg/resourceprovider/preflight/types.go new file mode 100644 index 000000000..842de67d7 --- /dev/null +++ b/pkg/resourceprovider/preflight/types.go @@ -0,0 +1,38 @@ +package preflight + +import "context" + +// ValidationResult represents the outcome of a validation check +type ValidationResult struct { + Success bool + Details string + Error error +} + +// GPUInfo represents information about an available GPU +type GPUInfo struct { + UUID string + Name string + MemoryTotal int64 + DriverVersion string + Capabilities []string +} + +// GPURequirements defines what GPU capabilities are required +type GPURequirements struct { + MinMemory int64 + MinGPUs int + Capabilities []string +} + +// Validator defines the interface for performing validations +type Validator interface { + // ValidateGPU checks if the required GPU capabilities are available + ValidateGPU(ctx context.Context, req *GPURequirements) ValidationResult + + // ValidateDockerRuntime checks if the nvidia runtime is available + ValidateDockerRuntime(ctx context.Context) ValidationResult + + // GetAvailableGPUs returns information about all available GPUs + GetAvailableGPUs(ctx context.Context) ([]GPUInfo, error) +} From ced7238ee25c279e7976615a0ae7be076fa90e4c Mon Sep 17 00:00:00 2001 From: logan Date: Thu, 23 Jan 2025 16:30:04 -0600 Subject: [PATCH 02/27] feat: Implement preflight checks for GPU and Docker runtime --- pkg/resourceprovider/preflight/docker.go | 57 +++++++++++++++++ pkg/resourceprovider/preflight/gpu.go | 38 ++++++----- pkg/resourceprovider/preflight/preflight.go | 47 ++++++++++++++ pkg/resourceprovider/preflight/types.go | 31 +++++++-- pkg/resourceprovider/resourceprovider.go | 70 +++++++++++---------- 5 files changed, 190 insertions(+), 53 deletions(-) create mode 100644 pkg/resourceprovider/preflight/docker.go create mode 100644 pkg/resourceprovider/preflight/preflight.go diff --git a/pkg/resourceprovider/preflight/docker.go b/pkg/resourceprovider/preflight/docker.go new file mode 100644 index 000000000..ba49a11c5 --- /dev/null +++ b/pkg/resourceprovider/preflight/docker.go @@ -0,0 +1,57 @@ +package preflight + +import ( + "context" + "encoding/json" + "fmt" + "os/exec" +) + +type dockerInfo struct { + Runtimes map[string]interface{} `json:"Runtimes"` +} + +func (p *preflightChecker) CheckDockerRuntime(ctx context.Context) CheckResult { + cmd := exec.CommandContext(ctx, "docker", "info", "--format", "{{json .}}") + output, err := cmd.Output() + if err != nil { + return CheckResult{ + Passed: false, + Error: fmt.Errorf("failed to get Docker info: %w", err), + Message: "Docker check failed", + } + } + + var info dockerInfo + if err := json.Unmarshal(output, &info); err != nil { + return CheckResult{ + Passed: false, + Error: fmt.Errorf("failed to parse Docker info: %w", err), + Message: "Docker info parsing failed", + } + } + + // Check for nvidia runtime + _, hasNvidia := info.Runtimes["nvidia"] + if !hasNvidia { + return CheckResult{ + Passed: false, + Message: "NVIDIA runtime not found in Docker", + } + } + + // Test nvidia runtime + testCmd := exec.CommandContext(ctx, "docker", "run", "--rm", "--runtime=nvidia", "nvidia/cuda:11.8.0-base", "nvidia-smi") + if err := testCmd.Run(); err != nil { + return CheckResult{ + Passed: false, + Message: "NVIDIA runtime test failed", + Error: err, + } + } + + return CheckResult{ + Passed: true, + Message: "NVIDIA runtime is available and functional", + } +} diff --git a/pkg/resourceprovider/preflight/gpu.go b/pkg/resourceprovider/preflight/gpu.go index ec76ffafc..16441393c 100644 --- a/pkg/resourceprovider/preflight/gpu.go +++ b/pkg/resourceprovider/preflight/gpu.go @@ -16,12 +16,22 @@ type nvidiaSmiResponse struct { } `json:"gpu"` } +type preflightChecker struct { + gpuInfo []GPUInfo +} + +type GPUCheckConfig struct { + MinGPUs int + MinMemory int64 + Capabilities []string +} + func checkNvidiaSMI() error { _, err := exec.LookPath("nvidia-smi") return err } -func (v *preflight) GetAvailableGPUs(ctx context.Context) ([]GPUInfo, error) { +func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { if err := checkNvidiaSMI(); err != nil { return nil, fmt.Errorf("nvidia-smi not found: %w", err) } @@ -50,27 +60,25 @@ func (v *preflight) GetAvailableGPUs(ctx context.Context) ([]GPUInfo, error) { return gpus, nil } -func (v *preflight) PreflightGPU(ctx context.Context, req *GPURequirements) PreflightResult { - gpus, err := v.GetAvailableGPUs(ctx) +func (p *preflightChecker) CheckGPU(ctx context.Context, config *GPUCheckConfig) CheckResult { + gpus, err := p.GetGPUInfo(ctx) if err != nil { - return PreflightResult{ - Success: false, + return CheckResult{ + Passed: false, Error: err, - Details: "Failed to get GPU information", + Message: "Failed to get GPU information", } } - if len(gpus) < req.MinGPUs { - return PreflightResult{ - Success: false, - Details: fmt.Sprintf("Insufficient GPUs. Required: %d, Available: %d", req.MinGPUs, len(gpus)), + if len(gpus) < config.MinGPUs { + return CheckResult{ + Passed: false, + Message: fmt.Sprintf("Insufficient GPUs. Required: %d, Available: %d", config.MinGPUs, len(gpus)), } } - // Additional validation logic for memory and capabilities can be added here - - return PreflightResult{ - Success: true, - Details: fmt.Sprintf("Found %d suitable GPUs", len(gpus)), + return CheckResult{ + Passed: true, + Message: fmt.Sprintf("Found %d suitable GPUs", len(gpus)), } } diff --git a/pkg/resourceprovider/preflight/preflight.go b/pkg/resourceprovider/preflight/preflight.go new file mode 100644 index 000000000..efbaeb819 --- /dev/null +++ b/pkg/resourceprovider/preflight/preflight.go @@ -0,0 +1,47 @@ +package preflight + +import ( + "context" + "fmt" +) + +// NewPreflightChecker creates a new instance of PreflightChecker +func NewPreflightChecker() PreflightChecker { + return &preflightChecker{} +} + +// PreflightConfig holds all configuration for preflight checks +type PreflightConfig struct { + GPU struct { + Enabled bool + MinGPUs int + MinMemoryGB int64 + Capabilities []string + } + Docker struct { + CheckRuntime bool + } +} + +// RunAllChecks performs all configured preflight checks +func (p *preflightChecker) RunAllChecks(ctx context.Context, config PreflightConfig) error { + if config.GPU.Enabled { + gpuResult := p.CheckGPU(ctx, &GPUCheckConfig{ + MinGPUs: config.GPU.MinGPUs, + MinMemory: config.GPU.MinMemoryGB * 1024 * 1024 * 1024, + Capabilities: config.GPU.Capabilities, + }) + if !gpuResult.Passed { + return fmt.Errorf("GPU check failed: %s", gpuResult.Message) + } + } + + if config.Docker.CheckRuntime { + runtimeResult := p.CheckDockerRuntime(ctx) + if !runtimeResult.Passed { + return fmt.Errorf("Docker runtime check failed: %s", runtimeResult.Message) + } + } + + return nil +} diff --git a/pkg/resourceprovider/preflight/types.go b/pkg/resourceprovider/preflight/types.go index 842de67d7..12836465d 100644 --- a/pkg/resourceprovider/preflight/types.go +++ b/pkg/resourceprovider/preflight/types.go @@ -1,6 +1,8 @@ package preflight -import "context" +import ( + "context" +) // ValidationResult represents the outcome of a validation check type ValidationResult struct { @@ -11,11 +13,17 @@ type ValidationResult struct { // GPUInfo represents information about an available GPU type GPUInfo struct { - UUID string - Name string - MemoryTotal int64 - DriverVersion string - Capabilities []string + UUID string + Name string + MemoryTotal int64 + DriverVersion string + Capabilities []string +} + +type CheckResult struct { + Passed bool + Message string + Error error } // GPURequirements defines what GPU capabilities are required @@ -25,6 +33,17 @@ type GPURequirements struct { Capabilities []string } +type PreflightChecker interface { + // CheckGPU verifies GPU requirements + CheckGPU(ctx context.Context, config *GPUCheckConfig) CheckResult + // CheckDockerRuntime verifies Docker runtime configuration + CheckDockerRuntime(ctx context.Context) CheckResult + // GetGPUInfo retrieves information about available GPUs + GetGPUInfo(ctx context.Context) ([]GPUInfo, error) + // RunAllChecks performs all configured checks + RunAllChecks(ctx context.Context, config PreflightConfig) error +} + // Validator defines the interface for performing validations type Validator interface { // ValidateGPU checks if the required GPU capabilities are available diff --git a/pkg/resourceprovider/resourceprovider.go b/pkg/resourceprovider/resourceprovider.go index cb3a25d6e..ee21ba70b 100644 --- a/pkg/resourceprovider/resourceprovider.go +++ b/pkg/resourceprovider/resourceprovider.go @@ -17,6 +17,7 @@ import ( "github.com/lilypad-tech/lilypad/pkg/executor/bacalhau" "github.com/lilypad-tech/lilypad/pkg/ipfs" "github.com/lilypad-tech/lilypad/pkg/powLogs" + "github.com/lilypad-tech/lilypad/pkg/resourceprovider/preflight" "github.com/lilypad-tech/lilypad/pkg/system" "github.com/lilypad-tech/lilypad/pkg/web3" "github.com/lilypad-tech/lilypad/pkg/web3/bindings/pow" @@ -24,44 +25,24 @@ import ( "go.opentelemetry.io/otel/trace" ) -// this configures the resource offers we will keep track of type ResourceProviderOfferOptions struct { - // if we are configuring a single machine then - // these values are populated by the flags - OfferSpec data.MachineSpec - // we can dupliate the single spec to create a list of specs - OfferCount int - // this represents how many machines we will keep - // offering to the network - // we can configure this with a config file - // to start with we will just add --cpu --gpu and --ram flags - // to the resource provider CLI which constrains them to a single machine - Specs []data.MachineSpec - // the list of modules we are willing to run - // an empty list means anything - Modules []string - - // this will normally be FixedPrice for RP's - Mode data.PricingMode - - // the default pricing for this resource provider - // for all modules that don't have a specific price + // ... [keep existing fields] + OfferSpec data.MachineSpec + OfferCount int + Specs []data.MachineSpec + Modules []string + Mode data.PricingMode DefaultPricing data.DealPricing DefaultTimeouts data.DealTimeouts - - // allow different pricing for different modules - ModulePricing map[string]data.DealPricing - ModuleTimeouts map[string]data.DealTimeouts - - // which mediators and directories this RP will trust - Services data.ServiceConfig + ModulePricing map[string]data.DealPricing + ModuleTimeouts map[string]data.DealTimeouts + Services data.ServiceConfig } -// this configures the pow we will keep track of type ResourceProviderPowOptions struct { - DisablePow bool - NumWorkers int - + // ... [keep existing fields] + DisablePow bool + NumWorkers int CudaGridSize int CudaBlockSize int CudaHashsPerThread int @@ -74,12 +55,14 @@ type ResourceProviderOptions struct { Pow ResourceProviderPowOptions IPFS ipfs.IPFSOptions Telemetry system.TelemetryOptions + Preflight preflight.PreflightConfig // Add preflight config } type ResourceProvider struct { web3SDK *web3.Web3SDK options ResourceProviderOptions controller *ResourceProviderController + gpuInfo []preflight.GPUInfo } func NewResourceProvider( @@ -88,10 +71,15 @@ func NewResourceProvider( executor executor.Executor, tracer trace.Tracer, ) (*ResourceProvider, error) { + if err := runPreflightChecks(context.Background(), options.Preflight); err != nil { + return nil, fmt.Errorf("preflight checks failed: %w", err) + } + controller, err := NewResourceProviderController(options, web3SDK, executor, tracer) if err != nil { return nil, err } + solver := &ResourceProvider{ controller: controller, options: options, @@ -101,7 +89,25 @@ func NewResourceProvider( return solver, nil } +func runPreflightChecks(ctx context.Context, config preflight.PreflightConfig) error { + checker := preflight.NewPreflightChecker() + return checker.RunAllChecks(ctx, config) +} + func (resourceProvider *ResourceProvider) Start(ctx context.Context, cm *system.CleanupManager) chan error { + errorChan := make(chan error, 1) + + // Update GPU info if enabled + if resourceProvider.options.Preflight.GPU.Enabled { + checker := preflight.NewPreflightChecker() + gpuInfo, err := checker.GetGPUInfo(ctx) + if err != nil { + errorChan <- fmt.Errorf("failed to get GPU information: %w", err) + return errorChan + } + resourceProvider.gpuInfo = gpuInfo + } + if !resourceProvider.options.Pow.DisablePow { if errCh := resourceProvider.StartMineLoop(ctx); errCh != nil { return errCh From a7ee2ae655e92fd3b8793ee39fb7271e1daf1294 Mon Sep 17 00:00:00 2001 From: logan Date: Thu, 23 Jan 2025 17:49:30 -0600 Subject: [PATCH 03/27] feat: Preflight logging and success messages --- pkg/resourceprovider/resourceprovider.go | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/pkg/resourceprovider/resourceprovider.go b/pkg/resourceprovider/resourceprovider.go index ee21ba70b..b87d90221 100644 --- a/pkg/resourceprovider/resourceprovider.go +++ b/pkg/resourceprovider/resourceprovider.go @@ -26,7 +26,6 @@ import ( ) type ResourceProviderOfferOptions struct { - // ... [keep existing fields] OfferSpec data.MachineSpec OfferCount int Specs []data.MachineSpec @@ -40,7 +39,6 @@ type ResourceProviderOfferOptions struct { } type ResourceProviderPowOptions struct { - // ... [keep existing fields] DisablePow bool NumWorkers int CudaGridSize int @@ -55,7 +53,7 @@ type ResourceProviderOptions struct { Pow ResourceProviderPowOptions IPFS ipfs.IPFSOptions Telemetry system.TelemetryOptions - Preflight preflight.PreflightConfig // Add preflight config + Preflight preflight.PreflightConfig } type ResourceProvider struct { @@ -90,8 +88,25 @@ func NewResourceProvider( } func runPreflightChecks(ctx context.Context, config preflight.PreflightConfig) error { + log.Info().Msg("Starting preflight checks...") checker := preflight.NewPreflightChecker() - return checker.RunAllChecks(ctx, config) + + // Logging GPU requirements + if config.GPU.Enabled { + log.Info(). + Int("min_gpus", config.GPU.MinGPUs). + Int64("min_memory_gb", config.GPU.MinMemoryGB). + Msg("GPU requirements") + } + + err := checker.RunAllChecks(ctx, config) + if err != nil { + log.Error().Err(err).Msg("Preflight checks failed") + return err + } + + log.Info().Msg("All preflight checks passed successfully") + return nil } func (resourceProvider *ResourceProvider) Start(ctx context.Context, cm *system.CleanupManager) chan error { From 6f6ed686dee5b846207d0de7d1f8ad6079bd7c83 Mon Sep 17 00:00:00 2001 From: logan Date: Thu, 23 Jan 2025 17:56:02 -0600 Subject: [PATCH 04/27] chore: restore comments in [pkg/resourceprovider/resourceprovider.go] --- pkg/resourceprovider/resourceprovider.go | 36 ++++++++++++++++++------ 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/pkg/resourceprovider/resourceprovider.go b/pkg/resourceprovider/resourceprovider.go index b87d90221..cdb4dd37d 100644 --- a/pkg/resourceprovider/resourceprovider.go +++ b/pkg/resourceprovider/resourceprovider.go @@ -26,18 +26,38 @@ import ( ) type ResourceProviderOfferOptions struct { - OfferSpec data.MachineSpec - OfferCount int - Specs []data.MachineSpec - Modules []string - Mode data.PricingMode + // if we are configuring a single machine then + // these values are populated by the flags + OfferSpec data.MachineSpec + // we can dupliate the single spec to create a list of specs + OfferCount int + // this represents how many machines we will keep + // offering to the network + // we can configure this with a config file + // to start with we will just add --cpu --gpu and --ram flags + // to the resource provider CLI which constrains them to a single machine + Specs []data.MachineSpec + // the list of modules we are willing to run + // an empty list means anything + Modules []string + + // this will normally be FixedPrice for RP's + Mode data.PricingMode + + // the default pricing for this resource provider + // for all modules that don't have a specific price DefaultPricing data.DealPricing DefaultTimeouts data.DealTimeouts - ModulePricing map[string]data.DealPricing - ModuleTimeouts map[string]data.DealTimeouts - Services data.ServiceConfig + + // allow different pricing for different modules + ModulePricing map[string]data.DealPricing + ModuleTimeouts map[string]data.DealTimeouts + + // which mediators and directories this RP will trust + Services data.ServiceConfig } +// this configures the pow we will keep track of type ResourceProviderPowOptions struct { DisablePow bool NumWorkers int From f201b4d80f8947bbb23c11ac98231770625ed653 Mon Sep 17 00:00:00 2001 From: logan Date: Fri, 24 Jan 2025 13:19:10 -0600 Subject: [PATCH 05/27] feat: gpu check + cleanup --- pkg/resourceprovider/preflight/gpu.go | 47 ++++++++++++++- pkg/resourceprovider/preflight/preflight.go | 13 ----- pkg/resourceprovider/preflight/types.go | 64 ++++++++------------- 3 files changed, 67 insertions(+), 57 deletions(-) diff --git a/pkg/resourceprovider/preflight/gpu.go b/pkg/resourceprovider/preflight/gpu.go index 16441393c..c9ddfb81e 100644 --- a/pkg/resourceprovider/preflight/gpu.go +++ b/pkg/resourceprovider/preflight/gpu.go @@ -5,6 +5,8 @@ import ( "encoding/json" "fmt" "os/exec" + + "github.com/rs/zerolog/log" ) type nvidiaSmiResponse struct { @@ -21,6 +23,7 @@ type preflightChecker struct { } type GPUCheckConfig struct { + Required bool MinGPUs int MinMemory int64 Capabilities []string @@ -61,22 +64,60 @@ func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { } func (p *preflightChecker) CheckGPU(ctx context.Context, config *GPUCheckConfig) CheckResult { + if !config.Required { + log.Info().Msg("GPU checks skipped - not required") + return CheckResult{ + Passed: true, + Message: "GPU not required for this configuration", + } + } + + log.Info().Msg("Starting GPU preflight check") + gpus, err := p.GetGPUInfo(ctx) if err != nil { + log.Warn().Err(err).Msg("Failed to get GPU information") + // If GPUs are required but not found, return failure + if config.Required { + return CheckResult{ + Passed: false, + Error: err, + Message: "Failed to get GPU information", + } + } + // If GPUs aren't required, pass even if we can't find them return CheckResult{ - Passed: false, - Error: err, - Message: "Failed to get GPU information", + Passed: true, + Message: "No GPUs found but none required", } } + log.Info(). + Int("gpu_count", len(gpus)). + Int("required_gpus", config.MinGPUs). + Msg("Found GPUs") + + for i, gpu := range gpus { + log.Info(). + Str("uuid", gpu.UUID). + Str("name", gpu.Name). + Int64("memory", gpu.MemoryTotal). + Int("index", i). + Msg("GPU details") + } + if len(gpus) < config.MinGPUs { + log.Warn(). + Int("available", len(gpus)). + Int("required", config.MinGPUs). + Msg("Insufficient GPUs") return CheckResult{ Passed: false, Message: fmt.Sprintf("Insufficient GPUs. Required: %d, Available: %d", config.MinGPUs, len(gpus)), } } + log.Info().Msg("GPU check passed successfully") return CheckResult{ Passed: true, Message: fmt.Sprintf("Found %d suitable GPUs", len(gpus)), diff --git a/pkg/resourceprovider/preflight/preflight.go b/pkg/resourceprovider/preflight/preflight.go index efbaeb819..f7d73fbe6 100644 --- a/pkg/resourceprovider/preflight/preflight.go +++ b/pkg/resourceprovider/preflight/preflight.go @@ -10,19 +10,6 @@ func NewPreflightChecker() PreflightChecker { return &preflightChecker{} } -// PreflightConfig holds all configuration for preflight checks -type PreflightConfig struct { - GPU struct { - Enabled bool - MinGPUs int - MinMemoryGB int64 - Capabilities []string - } - Docker struct { - CheckRuntime bool - } -} - // RunAllChecks performs all configured preflight checks func (p *preflightChecker) RunAllChecks(ctx context.Context, config PreflightConfig) error { if config.GPU.Enabled { diff --git a/pkg/resourceprovider/preflight/types.go b/pkg/resourceprovider/preflight/types.go index 12836465d..b22597b98 100644 --- a/pkg/resourceprovider/preflight/types.go +++ b/pkg/resourceprovider/preflight/types.go @@ -4,54 +4,36 @@ import ( "context" ) -// ValidationResult represents the outcome of a validation check -type ValidationResult struct { - Success bool - Details string - Error error -} - -// GPUInfo represents information about an available GPU type GPUInfo struct { - UUID string - Name string - MemoryTotal int64 - DriverVersion string - Capabilities []string + UUID string + Name string + MemoryTotal int64 + DriverVersion string + Capabilities []string } type CheckResult struct { - Passed bool - Message string - Error error + Passed bool + Message string + Error error } -// GPURequirements defines what GPU capabilities are required -type GPURequirements struct { - MinMemory int64 - MinGPUs int - Capabilities []string +type PreflightConfig struct { + GPU struct { + Required bool + Enabled bool + MinGPUs int + MinMemoryGB int64 + Capabilities []string + } + Docker struct { + CheckRuntime bool + } } type PreflightChecker interface { - // CheckGPU verifies GPU requirements - CheckGPU(ctx context.Context, config *GPUCheckConfig) CheckResult - // CheckDockerRuntime verifies Docker runtime configuration - CheckDockerRuntime(ctx context.Context) CheckResult - // GetGPUInfo retrieves information about available GPUs - GetGPUInfo(ctx context.Context) ([]GPUInfo, error) - // RunAllChecks performs all configured checks - RunAllChecks(ctx context.Context, config PreflightConfig) error -} - -// Validator defines the interface for performing validations -type Validator interface { - // ValidateGPU checks if the required GPU capabilities are available - ValidateGPU(ctx context.Context, req *GPURequirements) ValidationResult - - // ValidateDockerRuntime checks if the nvidia runtime is available - ValidateDockerRuntime(ctx context.Context) ValidationResult - - // GetAvailableGPUs returns information about all available GPUs - GetAvailableGPUs(ctx context.Context) ([]GPUInfo, error) + CheckGPU(ctx context.Context, config *GPUCheckConfig) CheckResult + CheckDockerRuntime(ctx context.Context) CheckResult + GetGPUInfo(ctx context.Context) ([]GPUInfo, error) + RunAllChecks(ctx context.Context, config PreflightConfig) error } From f4360846a8a4815df6538740f93cfc400f448411 Mon Sep 17 00:00:00 2001 From: logan Date: Fri, 24 Jan 2025 14:50:23 -0600 Subject: [PATCH 06/27] fix: gpu nvidia-smi check --- pkg/options/resource-provider.go | 14 +++++ pkg/resourceprovider/preflight/gpu.go | 68 +++++++++++++++--------- pkg/resourceprovider/resourceprovider.go | 4 +- 3 files changed, 61 insertions(+), 25 deletions(-) diff --git a/pkg/options/resource-provider.go b/pkg/options/resource-provider.go index 06e7a1cc5..dbc1ef316 100644 --- a/pkg/options/resource-provider.go +++ b/pkg/options/resource-provider.go @@ -6,6 +6,7 @@ import ( "github.com/lilypad-tech/lilypad/pkg/data" "github.com/lilypad-tech/lilypad/pkg/resourceprovider" + "github.com/lilypad-tech/lilypad/pkg/resourceprovider/preflight" "github.com/lilypad-tech/lilypad/pkg/system" "github.com/spf13/cobra" ) @@ -18,6 +19,19 @@ func NewResourceProviderOptions() resourceprovider.ResourceProviderOptions { Pow: GetDefaultResourceProviderPowOptions(), IPFS: GetDefaultIPFSOptions(), Telemetry: GetDefaultTelemetryOptions(), + Preflight: preflight.PreflightConfig{ + GPU: struct { + Required bool + Enabled bool + MinGPUs int + MinMemoryGB int64 + Capabilities []string + }{ + Required: false, // Don't require GPU + Enabled: true, // Enable checks to detect if GPU exists + MinGPUs: 0, // Don't require minimum GPUs + }, + }, } options.Web3.Service = system.ResourceProviderService return options diff --git a/pkg/resourceprovider/preflight/gpu.go b/pkg/resourceprovider/preflight/gpu.go index c9ddfb81e..37cef38c5 100644 --- a/pkg/resourceprovider/preflight/gpu.go +++ b/pkg/resourceprovider/preflight/gpu.go @@ -2,22 +2,14 @@ package preflight import ( "context" - "encoding/json" "fmt" "os/exec" + "strconv" + "strings" "github.com/rs/zerolog/log" ) -type nvidiaSmiResponse struct { - GPU []struct { - UUID string `json:"uuid"` - ProductName string `json:"product_name"` - Memory int64 `json:"memory_total"` - DriverVersion string `json:"driver_version"` - } `json:"gpu"` -} - type preflightChecker struct { gpuInfo []GPUInfo } @@ -34,30 +26,58 @@ func checkNvidiaSMI() error { return err } +type nvidiaSmiResponse struct { + UUID string + Name string + MemoryTotal string // Set to a string since CSV doesn't parse numbers + DriverVersion string +} + +// GetGPUInfo runs nvidia-smi to get information about available GPUs func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { + log.Info().Msg("Attempting to run nvidia-smi check") + if err := checkNvidiaSMI(); err != nil { - return nil, fmt.Errorf("nvidia-smi not found: %w", err) + return nil, err } - cmd := exec.CommandContext(ctx, "nvidia-smi", "--query-gpu=gpu_uuid,gpu_name,memory.total,driver_version", "--format=json") - output, err := cmd.Output() + log.Info().Msg("Running nvidia-smi command") + cmd := exec.CommandContext(ctx, "nvidia-smi", + "--query-gpu=gpu_uuid,gpu_name,memory.total,driver_version", + "--format=csv,noheader") + output, err := cmd.CombinedOutput() if err != nil { + log.Error().Str("output", string(output)).Err(err).Msg("nvidia-smi command failed") return nil, fmt.Errorf("error running nvidia-smi: %w", err) } - var response nvidiaSmiResponse - if err := json.Unmarshal(output, &response); err != nil { - return nil, fmt.Errorf("error parsing nvidia-smi output: %w", err) - } + // Parse CSV output + records := strings.Split(strings.TrimSpace(string(output)), "\n") + gpus := make([]GPUInfo, 0) - gpus := make([]GPUInfo, len(response.GPU)) - for i, gpu := range response.GPU { - gpus[i] = GPUInfo{ - UUID: gpu.UUID, - Name: gpu.ProductName, - MemoryTotal: gpu.Memory, - DriverVersion: gpu.DriverVersion, + for _, record := range records { + fields := strings.Split(record, ", ") + if len(fields) != 4 { + continue } + + // Parse memory string (e.g. "12282 MiB" -> 12282) + memoryStr := strings.Split(fields[2], " ")[0] + memoryMiB, _ := strconv.ParseInt(memoryStr, 10, 64) + + gpu := GPUInfo{ + UUID: strings.TrimSpace(fields[0]), + Name: strings.TrimSpace(fields[1]), + MemoryTotal: memoryMiB, + DriverVersion: strings.TrimSpace(fields[3]), + } + gpus = append(gpus, gpu) + + log.Info(). + Str("name", gpu.Name). + Str("uuid", gpu.UUID). + Int64("memory_mb", gpu.MemoryTotal). + Msgf("🎮 GPU %d details", len(gpus)) } return gpus, nil diff --git a/pkg/resourceprovider/resourceprovider.go b/pkg/resourceprovider/resourceprovider.go index cdb4dd37d..ad0773c1f 100644 --- a/pkg/resourceprovider/resourceprovider.go +++ b/pkg/resourceprovider/resourceprovider.go @@ -107,6 +107,9 @@ func NewResourceProvider( return solver, nil } +// This is where we run the preflight checks! +// I put it before the startMineLoop because if the preflight checks fail, we don't want to start the miner + func runPreflightChecks(ctx context.Context, config preflight.PreflightConfig) error { log.Info().Msg("Starting preflight checks...") checker := preflight.NewPreflightChecker() @@ -132,7 +135,6 @@ func runPreflightChecks(ctx context.Context, config preflight.PreflightConfig) e func (resourceProvider *ResourceProvider) Start(ctx context.Context, cm *system.CleanupManager) chan error { errorChan := make(chan error, 1) - // Update GPU info if enabled if resourceProvider.options.Preflight.GPU.Enabled { checker := preflight.NewPreflightChecker() gpuInfo, err := checker.GetGPUInfo(ctx) From d6099e08a38c1d2f75faddb8d0d916b963b55e4b Mon Sep 17 00:00:00 2001 From: logan Date: Sun, 26 Jan 2025 15:53:08 -0600 Subject: [PATCH 07/27] fix: handle no-GPU case gracefully --- pkg/resourceprovider/preflight/gpu.go | 59 ++++++++++++--------------- 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/pkg/resourceprovider/preflight/gpu.go b/pkg/resourceprovider/preflight/gpu.go index 37cef38c5..671a15b23 100644 --- a/pkg/resourceprovider/preflight/gpu.go +++ b/pkg/resourceprovider/preflight/gpu.go @@ -29,16 +29,16 @@ func checkNvidiaSMI() error { type nvidiaSmiResponse struct { UUID string Name string - MemoryTotal string // Set to a string since CSV doesn't parse numbers + MemoryTotal string DriverVersion string } -// GetGPUInfo runs nvidia-smi to get information about available GPUs func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { - log.Info().Msg("Attempting to run nvidia-smi check") + log.Debug().Msg("Checking for nvidia-smi") if err := checkNvidiaSMI(); err != nil { - return nil, err + log.Warn().Msg("⚠️ nvidia-smi not found - system appears to have no NVIDIA GPU") + return nil, fmt.Errorf("nvidia-smi not available: %w", err) } log.Info().Msg("Running nvidia-smi command") @@ -51,7 +51,6 @@ func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { return nil, fmt.Errorf("error running nvidia-smi: %w", err) } - // Parse CSV output records := strings.Split(strings.TrimSpace(string(output)), "\n") gpus := make([]GPUInfo, 0) @@ -61,7 +60,6 @@ func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { continue } - // Parse memory string (e.g. "12282 MiB" -> 12282) memoryStr := strings.Split(fields[2], " ")[0] memoryMiB, _ := strconv.ParseInt(memoryStr, 10, 64) @@ -85,59 +83,52 @@ func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { func (p *preflightChecker) CheckGPU(ctx context.Context, config *GPUCheckConfig) CheckResult { if !config.Required { - log.Info().Msg("GPU checks skipped - not required") + // Try to get GPU info + gpus, err := p.GetGPUInfo(ctx) + if err != nil { + log.Warn().Msg("⚠️ Running without GPU support - Resource Provider will operate in CPU-only mode") + return CheckResult{ + Passed: true, + Message: "Operating in CPU-only mode", + } + } + + // If we found GPUs, log them but still continue + log.Info().Msgf("🎮 Found %d optional GPUs available for use", len(gpus)) return CheckResult{ Passed: true, - Message: "GPU not required for this configuration", + Message: fmt.Sprintf("Found %d GPUs (optional)", len(gpus)), } } - log.Info().Msg("Starting GPU preflight check") - + // Required GPU checks + log.Info().Msg("Starting required GPU checks") gpus, err := p.GetGPUInfo(ctx) if err != nil { - log.Warn().Err(err).Msg("Failed to get GPU information") - // If GPUs are required but not found, return failure - if config.Required { - return CheckResult{ - Passed: false, - Error: err, - Message: "Failed to get GPU information", - } - } - // If GPUs aren't required, pass even if we can't find them return CheckResult{ - Passed: true, - Message: "No GPUs found but none required", + Passed: false, + Error: err, + Message: "Required GPU check failed - no NVIDIA GPUs detected", } } log.Info(). Int("gpu_count", len(gpus)). Int("required_gpus", config.MinGPUs). - Msg("Found GPUs") - - for i, gpu := range gpus { - log.Info(). - Str("uuid", gpu.UUID). - Str("name", gpu.Name). - Int64("memory", gpu.MemoryTotal). - Int("index", i). - Msg("GPU details") - } + Msg("Checking GPU requirements") if len(gpus) < config.MinGPUs { log.Warn(). Int("available", len(gpus)). Int("required", config.MinGPUs). - Msg("Insufficient GPUs") + Msg("Insufficient GPUs for requirements") return CheckResult{ Passed: false, Message: fmt.Sprintf("Insufficient GPUs. Required: %d, Available: %d", config.MinGPUs, len(gpus)), } } - log.Info().Msg("GPU check passed successfully") + log.Info().Msg("✅ GPU requirements satisfied") return CheckResult{ Passed: true, Message: fmt.Sprintf("Found %d suitable GPUs", len(gpus)), From 834cd16cad4d73012ff28a470bf2281f023f6ddf Mon Sep 17 00:00:00 2001 From: logan Date: Sun, 26 Jan 2025 16:03:34 -0600 Subject: [PATCH 08/27] fix: update start method in RP --- pkg/resourceprovider/resourceprovider.go | 28 +++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/pkg/resourceprovider/resourceprovider.go b/pkg/resourceprovider/resourceprovider.go index ad0773c1f..1b31a22f7 100644 --- a/pkg/resourceprovider/resourceprovider.go +++ b/pkg/resourceprovider/resourceprovider.go @@ -139,18 +139,36 @@ func (resourceProvider *ResourceProvider) Start(ctx context.Context, cm *system. checker := preflight.NewPreflightChecker() gpuInfo, err := checker.GetGPUInfo(ctx) if err != nil { - errorChan <- fmt.Errorf("failed to get GPU information: %w", err) - return errorChan + // Instead of returning error, just log warning and continue + log.Warn().Err(err).Msg("⚠️ GPU capabilities will not be available - continuing in CPU-only mode") + // Set empty GPU info + resourceProvider.gpuInfo = []preflight.GPUInfo{} + } else { + resourceProvider.gpuInfo = gpuInfo + log.Info().Msgf("🎮 Successfully initialized with %d GPUs", len(gpuInfo)) } - resourceProvider.gpuInfo = gpuInfo } if !resourceProvider.options.Pow.DisablePow { if errCh := resourceProvider.StartMineLoop(ctx); errCh != nil { - return errCh + // Forward any mining errors to the main error channel + go func() { + if err := <-errCh; err != nil { + errorChan <- fmt.Errorf("mining error: %w", err) + } + }() } } - return resourceProvider.controller.Start(ctx, cm) + + // Start the controller and forward any errors + controllerErrChan := resourceProvider.controller.Start(ctx, cm) + go func() { + if err := <-controllerErrChan; err != nil { + errorChan <- err + } + }() + + return errorChan } func (resourceProvider *ResourceProvider) StartMineLoop(ctx context.Context) chan error { From 95c6ca0bdc633e0d015d4ffe35087334dd41d861 Mon Sep 17 00:00:00 2001 From: logan Date: Mon, 27 Jan 2025 14:08:33 -0600 Subject: [PATCH 09/27] chore: remove unused dockerfiles --- docker/preflight/Dockerfile | 56 ---------------- .../preflight/docker-compose-pre-flight.yml | 44 ------------- docker/preflight/docker-compose.orbit.yml | 65 ------------------- docker/preflight/preflight.go | 47 -------------- 4 files changed, 212 deletions(-) delete mode 100644 docker/preflight/Dockerfile delete mode 100644 docker/preflight/docker-compose-pre-flight.yml delete mode 100644 docker/preflight/docker-compose.orbit.yml delete mode 100644 docker/preflight/preflight.go diff --git a/docker/preflight/Dockerfile b/docker/preflight/Dockerfile deleted file mode 100644 index 21b47d8cc..000000000 --- a/docker/preflight/Dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -# FROM alpine:latest AS base -# WORKDIR /usr/src/app - -# ARG network=dev - -# COPY . . -# RUN go mod download && go mod verify -# RUN go build -v . -# RUN go install - -# RUN (curl -Ls --tlsv1.2 --proto "=https" --retry 3 https://cli.doppler.com/install.sh || wget -t 3 -qO- https://cli.doppler.com/install.sh) | sh - -# RUN touch run -# RUN echo "#!/bin/bash" >> run -# RUN if [ "${network}" = "dev" ]; then \ -# echo "lilypad jobcreator --network $network" >> run; \ -# else \ -# echo "doppler run -- lilypad jobcreator --network $network" >> run; \ -# fi -# RUN chmod +x run -# WORKDIR /app -# COPY ./preflight /app -# RUN chmod +x /app/preflight -# # CMD ["/app/preflight"] -# # CMD ["/bin/sh"] -# CMD tail -f /dev/null -FROM golang:1.23 AS builder -WORKDIR /app -# RUN pwd -COPY ./preflight.go /app -# COPY . . -# RUN cd preflight -RUN go mod init preflight -RUN go mod tidy -RUN go build -o ./preflight preflight.go - -FROM nvidia/cuda:12.0.1-cudnn8-devel-ubuntu22.04 AS base - -RUN apt-get update && \ - apt-get install -y \ - apt-transport-https \ - ca-certificates \ - curl \ - gnupg \ - lsb-release && \ - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ - echo "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \ - apt-get update && \ - apt-get install -y docker-ce docker-ce-cli containerd.io && \ - rm -rf /var/lib/apt/lists/* - - -WORKDIR /app -COPY --from=builder /app . -RUN chmod +x /app/preflight -# CMD ["/app/preflight","arsenum"] \ No newline at end of file diff --git a/docker/preflight/docker-compose-pre-flight.yml b/docker/preflight/docker-compose-pre-flight.yml deleted file mode 100644 index f74038aa2..000000000 --- a/docker/preflight/docker-compose-pre-flight.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: '3.8' - -services: - preflight: - build: - context: . - dockerfile: ./Dockerfile - image: preflight - container_name: ${ORG:-default_org}_preflight - volumes: - - /var/run/docker.sock:/var/run/docker.sock - runtime: nvidia # Ensure nvidia runtime is available - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: all - capabilities: [gpu] - command: ["/app/preflight", "check-gpu"] # Assuming your binary has a check-gpu command - environment: - - NVIDIA_VISIBLE_DEVICES=all - - NVIDIA_DRIVER_CAPABILITIES=compute,utility - - resource-provider: - image: ghcr.io/lilypad-tech/resource-provider:latest - container_name: ${ORG:-default_org}_resource-provider - runtime: nvidia - depends_on: - preflight: - condition: service_completed_successfully - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: all - capabilities: [gpu] - volumes: - - /var/run/docker.sock:/var/run/docker.sock - environment: - - NVIDIA_VISIBLE_DEVICES=all - - NVIDIA_DRIVER_CAPABILITIES=compute,utility - - OFFER_GPU=1s \ No newline at end of file diff --git a/docker/preflight/docker-compose.orbit.yml b/docker/preflight/docker-compose.orbit.yml deleted file mode 100644 index 5b7d6a742..000000000 --- a/docker/preflight/docker-compose.orbit.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: lilypad_orbit -services: - preflight: - build: - context: . - dockerfile: ./Dockerfile - image: preflight - container_name: orbit_preflight - runtime: nvidia - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: all - capabilities: [gpu] - volumes: - - /var/run/docker.sock:/var/run/docker.sock - environment: - - NVIDIA_VISIBLE_DEVICES=all - - NVIDIA_DRIVER_CAPABILITIES=compute,utility - networks: - - localnet - command: ["/app/preflight", "check-gpu"] - - resource-provider: - image: ghcr.io/lilypad-tech/resource-provider:latest - container_name: orbit_resource-provider - runtime: nvidia - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: all - capabilities: [gpu] - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - lilypad-data:/tmp/lilypad/data - environment: - - BACALHAU_API_HOST=orbit_bacalhau - - NVIDIA_VISIBLE_DEVICES=all - - NVIDIA_DRIVER_CAPABILITIES=compute,utility - - OFFER_GPU=1 - # Keeping original orbit environment variables - - WEB3_RPC_URL=wss://wss.orbit.arsenum.com - - WEB3_CHAIN_ID=68283778764 - - WEB3_PRIVATE_KEY=0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a - - WEB3_CONTROLLER_ADDRESS=0xa85233C63b9Ee964Add6F2cffe00Fd84eb32338f - - WEB3_PAYMENTS_ADDRESS=0x2279B7A0a67DB372996a5FaB50D91eAA73d2eBe6 - - WEB3_POW_ADDRESS=0x4ed7c70F96B99c776995fB64377f0d4aB3B0e1C1 - - WEB3_STORAGE_ADDRESS=0xB7f8BC63BbcaD18155201308C8f3540b07f84F5e - - WEB3_TOKEN_ADDRESS=0xa513E6E4b8f2a923D98304ec87F64353C4D5C853 - - WEB3_USERS_ADDRESS=0x0DCd1Bf9A1b36cE34237eEaFef220932846BCD82 - - LOG_LEVEL=debug - networks: - - localnet - -volumes: - bacalhau-data: - lilypad-data: - -networks: - localnet: - external: true \ No newline at end of file diff --git a/docker/preflight/preflight.go b/docker/preflight/preflight.go deleted file mode 100644 index 97f06e247..000000000 --- a/docker/preflight/preflight.go +++ /dev/null @@ -1,47 +0,0 @@ -package main - -import ( - "fmt" - "os" - "os/exec" - "strings" -) - -func main() { - if len(os.Args) < 2 || os.Args[1] != "check-gpu" { - fmt.Println("Usage: preflight check-gpu") - os.Exit(1) - } - - // Check nvidia-smi is available - if _, err := exec.LookPath("nvidia-smi"); err != nil { - fmt.Println("Error: nvidia-smi not found. Please ensure NVIDIA drivers are installed") - os.Exit(1) - } - - // Run nvidia-smi to check GPU availability - cmd := exec.Command("nvidia-smi") - output, err := cmd.CombinedOutput() - if err != nil { - fmt.Printf("Error running nvidia-smi: %v\n", err) - os.Exit(1) - } - - // Check docker runtime - cmd = exec.Command("docker", "info", "--format", "'{{.Runtimes}}'") - runtimeOutput, err := cmd.CombinedOutput() - if err != nil { - fmt.Printf("Error checking Docker runtimes: %v\n", err) - os.Exit(1) - } - - if !strings.Contains(string(runtimeOutput), "nvidia") { - fmt.Println("Error: NVIDIA runtime not found in Docker") - os.Exit(1) - } - - fmt.Println("GPU Check Output:") - fmt.Println(string(output)) - fmt.Println("NVIDIA runtime is available") - fmt.Println("All preflight checks passed successfully") -} From 7db0a3f1ffe5ae3028b050a0a4bfeb93e4cc7d12 Mon Sep 17 00:00:00 2001 From: logan Date: Mon, 27 Jan 2025 14:19:29 -0600 Subject: [PATCH 10/27] chore: restore comments in [pkg/resourceprovider/resourceprovider.go] --- pkg/resourceprovider/resourceprovider.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/resourceprovider/resourceprovider.go b/pkg/resourceprovider/resourceprovider.go index 1b31a22f7..a43edff37 100644 --- a/pkg/resourceprovider/resourceprovider.go +++ b/pkg/resourceprovider/resourceprovider.go @@ -25,6 +25,7 @@ import ( "go.opentelemetry.io/otel/trace" ) +// this configures the resource offers we will keep track of type ResourceProviderOfferOptions struct { // if we are configuring a single machine then // these values are populated by the flags From 4db086281915ad5f44972e4ca5426f7543b03867 Mon Sep 17 00:00:00 2001 From: logan Date: Mon, 27 Jan 2025 15:03:18 -0600 Subject: [PATCH 11/27] chore: remove comments within [pkg/resourceprovider/resourceprovider.go] --- pkg/resourceprovider/resourceprovider.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/resourceprovider/resourceprovider.go b/pkg/resourceprovider/resourceprovider.go index a43edff37..cebcb3fb4 100644 --- a/pkg/resourceprovider/resourceprovider.go +++ b/pkg/resourceprovider/resourceprovider.go @@ -108,9 +108,6 @@ func NewResourceProvider( return solver, nil } -// This is where we run the preflight checks! -// I put it before the startMineLoop because if the preflight checks fail, we don't want to start the miner - func runPreflightChecks(ctx context.Context, config preflight.PreflightConfig) error { log.Info().Msg("Starting preflight checks...") checker := preflight.NewPreflightChecker() From 8e1e504669f27ff00374a2ccdf7d725792b4c424 Mon Sep 17 00:00:00 2001 From: logan Date: Mon, 27 Jan 2025 16:05:00 -0600 Subject: [PATCH 12/27] refactor: removed minimum GPU parameter --- pkg/options/resource-provider.go | 2 -- pkg/resourceprovider/preflight/preflight.go | 1 - pkg/resourceprovider/preflight/types.go | 1 - pkg/resourceprovider/resourceprovider.go | 1 - 4 files changed, 5 deletions(-) diff --git a/pkg/options/resource-provider.go b/pkg/options/resource-provider.go index dbc1ef316..96b01db27 100644 --- a/pkg/options/resource-provider.go +++ b/pkg/options/resource-provider.go @@ -23,13 +23,11 @@ func NewResourceProviderOptions() resourceprovider.ResourceProviderOptions { GPU: struct { Required bool Enabled bool - MinGPUs int MinMemoryGB int64 Capabilities []string }{ Required: false, // Don't require GPU Enabled: true, // Enable checks to detect if GPU exists - MinGPUs: 0, // Don't require minimum GPUs }, }, } diff --git a/pkg/resourceprovider/preflight/preflight.go b/pkg/resourceprovider/preflight/preflight.go index f7d73fbe6..e1ffad90b 100644 --- a/pkg/resourceprovider/preflight/preflight.go +++ b/pkg/resourceprovider/preflight/preflight.go @@ -14,7 +14,6 @@ func NewPreflightChecker() PreflightChecker { func (p *preflightChecker) RunAllChecks(ctx context.Context, config PreflightConfig) error { if config.GPU.Enabled { gpuResult := p.CheckGPU(ctx, &GPUCheckConfig{ - MinGPUs: config.GPU.MinGPUs, MinMemory: config.GPU.MinMemoryGB * 1024 * 1024 * 1024, Capabilities: config.GPU.Capabilities, }) diff --git a/pkg/resourceprovider/preflight/types.go b/pkg/resourceprovider/preflight/types.go index b22597b98..f83a22ae5 100644 --- a/pkg/resourceprovider/preflight/types.go +++ b/pkg/resourceprovider/preflight/types.go @@ -22,7 +22,6 @@ type PreflightConfig struct { GPU struct { Required bool Enabled bool - MinGPUs int MinMemoryGB int64 Capabilities []string } diff --git a/pkg/resourceprovider/resourceprovider.go b/pkg/resourceprovider/resourceprovider.go index cebcb3fb4..a817e6bf6 100644 --- a/pkg/resourceprovider/resourceprovider.go +++ b/pkg/resourceprovider/resourceprovider.go @@ -115,7 +115,6 @@ func runPreflightChecks(ctx context.Context, config preflight.PreflightConfig) e // Logging GPU requirements if config.GPU.Enabled { log.Info(). - Int("min_gpus", config.GPU.MinGPUs). Int64("min_memory_gb", config.GPU.MinMemoryGB). Msg("GPU requirements") } From 06329f06ec0338803072fa6c669c3b02a2d0cd6e Mon Sep 17 00:00:00 2001 From: logan Date: Tue, 28 Jan 2025 17:50:27 -0600 Subject: [PATCH 13/27] feat: 1gb ram requirement --- pkg/options/resource-provider.go | 5 +++-- pkg/resourceprovider/preflight/gpu.go | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/options/resource-provider.go b/pkg/options/resource-provider.go index 96b01db27..61585c9d5 100644 --- a/pkg/options/resource-provider.go +++ b/pkg/options/resource-provider.go @@ -26,8 +26,9 @@ func NewResourceProviderOptions() resourceprovider.ResourceProviderOptions { MinMemoryGB int64 Capabilities []string }{ - Required: false, // Don't require GPU - Enabled: true, // Enable checks to detect if GPU exists + Required: false, // Enable to require GPU + Enabled: true, // Enable checks to detect if GPU exists + MinMemoryGB: 1, // Minimum memory required for GPU (we can match this with the resourceOffer) }, }, } diff --git a/pkg/resourceprovider/preflight/gpu.go b/pkg/resourceprovider/preflight/gpu.go index 671a15b23..e1fb43b89 100644 --- a/pkg/resourceprovider/preflight/gpu.go +++ b/pkg/resourceprovider/preflight/gpu.go @@ -83,7 +83,7 @@ func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { func (p *preflightChecker) CheckGPU(ctx context.Context, config *GPUCheckConfig) CheckResult { if !config.Required { - // Try to get GPU info + // Attempt to retrieve GPU info gpus, err := p.GetGPUInfo(ctx) if err != nil { log.Warn().Msg("⚠️ Running without GPU support - Resource Provider will operate in CPU-only mode") @@ -97,7 +97,7 @@ func (p *preflightChecker) CheckGPU(ctx context.Context, config *GPUCheckConfig) log.Info().Msgf("🎮 Found %d optional GPUs available for use", len(gpus)) return CheckResult{ Passed: true, - Message: fmt.Sprintf("Found %d GPUs (optional)", len(gpus)), + Message: fmt.Sprintf("Found %d NVIDIA GPUs (optional)", len(gpus)), } } From a77b80d5f55d586975235e7cfe5c01be41733d41 Mon Sep 17 00:00:00 2001 From: logan Date: Fri, 31 Jan 2025 14:42:31 -0600 Subject: [PATCH 14/27] refactor: simplify GPU configuration by removing unnecessary parameters --- pkg/options/resource-provider.go | 12 ++++----- pkg/resourceprovider/preflight/docker.go | 3 ++- pkg/resourceprovider/preflight/preflight.go | 15 +++++------ pkg/resourceprovider/preflight/types.go | 6 +---- pkg/resourceprovider/resourceprovider.go | 30 +++++++++------------ 5 files changed, 27 insertions(+), 39 deletions(-) diff --git a/pkg/options/resource-provider.go b/pkg/options/resource-provider.go index 61585c9d5..7dec29bfc 100644 --- a/pkg/options/resource-provider.go +++ b/pkg/options/resource-provider.go @@ -21,14 +21,12 @@ func NewResourceProviderOptions() resourceprovider.ResourceProviderOptions { Telemetry: GetDefaultTelemetryOptions(), Preflight: preflight.PreflightConfig{ GPU: struct { - Required bool - Enabled bool - MinMemoryGB int64 - Capabilities []string + MinMemoryGB int64 }{ - Required: false, // Enable to require GPU - Enabled: true, // Enable checks to detect if GPU exists - MinMemoryGB: 1, // Minimum memory required for GPU (we can match this with the resourceOffer) + MinMemoryGB: 1, // Minimum memory(we can match this with the resourceOffer) + + // Remove the "Required" and "Enabled" toggles above + // teh place where you do the check, dont just just put the number, assign to variable so it has a name - gives you iform }, }, } diff --git a/pkg/resourceprovider/preflight/docker.go b/pkg/resourceprovider/preflight/docker.go index ba49a11c5..a6bbe8672 100644 --- a/pkg/resourceprovider/preflight/docker.go +++ b/pkg/resourceprovider/preflight/docker.go @@ -36,6 +36,7 @@ func (p *preflightChecker) CheckDockerRuntime(ctx context.Context) CheckResult { if !hasNvidia { return CheckResult{ Passed: false, + Error: fmt.Errorf("nvidia runtime not found in Docker configuration"), Message: "NVIDIA runtime not found in Docker", } } @@ -45,8 +46,8 @@ func (p *preflightChecker) CheckDockerRuntime(ctx context.Context) CheckResult { if err := testCmd.Run(); err != nil { return CheckResult{ Passed: false, + Error: fmt.Errorf("failed to run NVIDIA runtime test: %w", err), Message: "NVIDIA runtime test failed", - Error: err, } } diff --git a/pkg/resourceprovider/preflight/preflight.go b/pkg/resourceprovider/preflight/preflight.go index e1ffad90b..58566206d 100644 --- a/pkg/resourceprovider/preflight/preflight.go +++ b/pkg/resourceprovider/preflight/preflight.go @@ -10,16 +10,13 @@ func NewPreflightChecker() PreflightChecker { return &preflightChecker{} } -// RunAllChecks performs all configured preflight checks func (p *preflightChecker) RunAllChecks(ctx context.Context, config PreflightConfig) error { - if config.GPU.Enabled { - gpuResult := p.CheckGPU(ctx, &GPUCheckConfig{ - MinMemory: config.GPU.MinMemoryGB * 1024 * 1024 * 1024, - Capabilities: config.GPU.Capabilities, - }) - if !gpuResult.Passed { - return fmt.Errorf("GPU check failed: %s", gpuResult.Message) - } + + gpuResult := p.CheckGPU(ctx, &GPUCheckConfig{ + MinMemory: config.GPU.MinMemoryGB * 1024 * 1024 * 1024, + }) + if !gpuResult.Passed { + return fmt.Errorf("GPU check failed: %s", gpuResult.Message) } if config.Docker.CheckRuntime { diff --git a/pkg/resourceprovider/preflight/types.go b/pkg/resourceprovider/preflight/types.go index f83a22ae5..1d7842eb9 100644 --- a/pkg/resourceprovider/preflight/types.go +++ b/pkg/resourceprovider/preflight/types.go @@ -9,7 +9,6 @@ type GPUInfo struct { Name string MemoryTotal int64 DriverVersion string - Capabilities []string } type CheckResult struct { @@ -20,10 +19,7 @@ type CheckResult struct { type PreflightConfig struct { GPU struct { - Required bool - Enabled bool - MinMemoryGB int64 - Capabilities []string + MinMemoryGB int64 } Docker struct { CheckRuntime bool diff --git a/pkg/resourceprovider/resourceprovider.go b/pkg/resourceprovider/resourceprovider.go index a817e6bf6..58e98d4d2 100644 --- a/pkg/resourceprovider/resourceprovider.go +++ b/pkg/resourceprovider/resourceprovider.go @@ -113,11 +113,9 @@ func runPreflightChecks(ctx context.Context, config preflight.PreflightConfig) e checker := preflight.NewPreflightChecker() // Logging GPU requirements - if config.GPU.Enabled { - log.Info(). - Int64("min_memory_gb", config.GPU.MinMemoryGB). - Msg("GPU requirements") - } + log.Info(). + Int64("min_memory_gb", config.GPU.MinMemoryGB). + Msg("GPU requirements") err := checker.RunAllChecks(ctx, config) if err != nil { @@ -132,18 +130,16 @@ func runPreflightChecks(ctx context.Context, config preflight.PreflightConfig) e func (resourceProvider *ResourceProvider) Start(ctx context.Context, cm *system.CleanupManager) chan error { errorChan := make(chan error, 1) - if resourceProvider.options.Preflight.GPU.Enabled { - checker := preflight.NewPreflightChecker() - gpuInfo, err := checker.GetGPUInfo(ctx) - if err != nil { - // Instead of returning error, just log warning and continue - log.Warn().Err(err).Msg("⚠️ GPU capabilities will not be available - continuing in CPU-only mode") - // Set empty GPU info - resourceProvider.gpuInfo = []preflight.GPUInfo{} - } else { - resourceProvider.gpuInfo = gpuInfo - log.Info().Msgf("🎮 Successfully initialized with %d GPUs", len(gpuInfo)) - } + checker := preflight.NewPreflightChecker() + gpuInfo, err := checker.GetGPUInfo(ctx) + if err != nil { + // Instead of returning error, just log warning and continue + log.Warn().Err(err).Msg("⚠️ GPU capabilities will not be available - continuing in CPU-only mode") + // Set empty GPU info + resourceProvider.gpuInfo = []preflight.GPUInfo{} + } else { + resourceProvider.gpuInfo = gpuInfo + log.Info().Msgf("🎮 Successfully initialized with %d GPUs", len(gpuInfo)) } if !resourceProvider.options.Pow.DisablePow { From 952898e13568bfe3cdca2000205b21636f86a648 Mon Sep 17 00:00:00 2001 From: logan Date: Fri, 31 Jan 2025 15:17:18 -0600 Subject: [PATCH 15/27] refactor: enhance GPU info logging and remove types file --- pkg/resourceprovider/preflight/gpu.go | 19 ------------ pkg/resourceprovider/preflight/preflight.go | 29 ++++++++++++++++++ pkg/resourceprovider/preflight/types.go | 34 --------------------- pkg/resourceprovider/resourceprovider.go | 19 +++++++----- 4 files changed, 40 insertions(+), 61 deletions(-) delete mode 100644 pkg/resourceprovider/preflight/types.go diff --git a/pkg/resourceprovider/preflight/gpu.go b/pkg/resourceprovider/preflight/gpu.go index e1fb43b89..923fdce69 100644 --- a/pkg/resourceprovider/preflight/gpu.go +++ b/pkg/resourceprovider/preflight/gpu.go @@ -34,14 +34,11 @@ type nvidiaSmiResponse struct { } func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { - log.Debug().Msg("Checking for nvidia-smi") if err := checkNvidiaSMI(); err != nil { - log.Warn().Msg("⚠️ nvidia-smi not found - system appears to have no NVIDIA GPU") return nil, fmt.Errorf("nvidia-smi not available: %w", err) } - log.Info().Msg("Running nvidia-smi command") cmd := exec.CommandContext(ctx, "nvidia-smi", "--query-gpu=gpu_uuid,gpu_name,memory.total,driver_version", "--format=csv,noheader") @@ -112,22 +109,6 @@ func (p *preflightChecker) CheckGPU(ctx context.Context, config *GPUCheckConfig) } } - log.Info(). - Int("gpu_count", len(gpus)). - Int("required_gpus", config.MinGPUs). - Msg("Checking GPU requirements") - - if len(gpus) < config.MinGPUs { - log.Warn(). - Int("available", len(gpus)). - Int("required", config.MinGPUs). - Msg("Insufficient GPUs for requirements") - return CheckResult{ - Passed: false, - Message: fmt.Sprintf("Insufficient GPUs. Required: %d, Available: %d", config.MinGPUs, len(gpus)), - } - } - log.Info().Msg("✅ GPU requirements satisfied") return CheckResult{ Passed: true, diff --git a/pkg/resourceprovider/preflight/preflight.go b/pkg/resourceprovider/preflight/preflight.go index 58566206d..5213df6e2 100644 --- a/pkg/resourceprovider/preflight/preflight.go +++ b/pkg/resourceprovider/preflight/preflight.go @@ -5,6 +5,35 @@ import ( "fmt" ) +type GPUInfo struct { + UUID string + Name string + MemoryTotal int64 + DriverVersion string +} + +type CheckResult struct { + Passed bool + Message string + Error error +} + +type PreflightConfig struct { + GPU struct { + MinMemoryGB int64 + } + Docker struct { + CheckRuntime bool + } +} + +type PreflightChecker interface { + CheckGPU(ctx context.Context, config *GPUCheckConfig) CheckResult + CheckDockerRuntime(ctx context.Context) CheckResult + GetGPUInfo(ctx context.Context) ([]GPUInfo, error) + RunAllChecks(ctx context.Context, config PreflightConfig) error +} + // NewPreflightChecker creates a new instance of PreflightChecker func NewPreflightChecker() PreflightChecker { return &preflightChecker{} diff --git a/pkg/resourceprovider/preflight/types.go b/pkg/resourceprovider/preflight/types.go deleted file mode 100644 index 1d7842eb9..000000000 --- a/pkg/resourceprovider/preflight/types.go +++ /dev/null @@ -1,34 +0,0 @@ -package preflight - -import ( - "context" -) - -type GPUInfo struct { - UUID string - Name string - MemoryTotal int64 - DriverVersion string -} - -type CheckResult struct { - Passed bool - Message string - Error error -} - -type PreflightConfig struct { - GPU struct { - MinMemoryGB int64 - } - Docker struct { - CheckRuntime bool - } -} - -type PreflightChecker interface { - CheckGPU(ctx context.Context, config *GPUCheckConfig) CheckResult - CheckDockerRuntime(ctx context.Context) CheckResult - GetGPUInfo(ctx context.Context) ([]GPUInfo, error) - RunAllChecks(ctx context.Context, config PreflightConfig) error -} diff --git a/pkg/resourceprovider/resourceprovider.go b/pkg/resourceprovider/resourceprovider.go index 58e98d4d2..6f0f5bd92 100644 --- a/pkg/resourceprovider/resourceprovider.go +++ b/pkg/resourceprovider/resourceprovider.go @@ -113,17 +113,21 @@ func runPreflightChecks(ctx context.Context, config preflight.PreflightConfig) e checker := preflight.NewPreflightChecker() // Logging GPU requirements - log.Info(). - Int64("min_memory_gb", config.GPU.MinMemoryGB). - Msg("GPU requirements") + gpuInfo, err := checker.GetGPUInfo(ctx) + if err != nil { + log.Warn().Err(err).Msg("⚠️ No GPU detected - will operate in CPU-only mode") + } else { + log.Info(). + Int("gpu_count", len(gpuInfo)). + Int64("min_memory_gb", config.GPU.MinMemoryGB). + Msg("🎮 GPU requirements") + } - err := checker.RunAllChecks(ctx, config) + err = checker.RunAllChecks(ctx, config) if err != nil { - log.Error().Err(err).Msg("Preflight checks failed") + log.Error().Err(err).Msg("❌ Preflight checks failed") return err } - - log.Info().Msg("All preflight checks passed successfully") return nil } @@ -135,7 +139,6 @@ func (resourceProvider *ResourceProvider) Start(ctx context.Context, cm *system. if err != nil { // Instead of returning error, just log warning and continue log.Warn().Err(err).Msg("⚠️ GPU capabilities will not be available - continuing in CPU-only mode") - // Set empty GPU info resourceProvider.gpuInfo = []preflight.GPUInfo{} } else { resourceProvider.gpuInfo = gpuInfo From 00e7467c165398b2d86502dfa97b1b7442b57feb Mon Sep 17 00:00:00 2001 From: logan Date: Fri, 31 Jan 2025 15:40:11 -0600 Subject: [PATCH 16/27] refactor: replace hardcoded GPU memory with default constant --- pkg/options/resource-provider.go | 5 +---- pkg/resourceprovider/preflight/preflight.go | 2 ++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/options/resource-provider.go b/pkg/options/resource-provider.go index 7dec29bfc..be4a3e3e3 100644 --- a/pkg/options/resource-provider.go +++ b/pkg/options/resource-provider.go @@ -23,10 +23,7 @@ func NewResourceProviderOptions() resourceprovider.ResourceProviderOptions { GPU: struct { MinMemoryGB int64 }{ - MinMemoryGB: 1, // Minimum memory(we can match this with the resourceOffer) - - // Remove the "Required" and "Enabled" toggles above - // teh place where you do the check, dont just just put the number, assign to variable so it has a name - gives you iform + MinMemoryGB: preflight.RequiredGPUMemoryGB, }, }, } diff --git a/pkg/resourceprovider/preflight/preflight.go b/pkg/resourceprovider/preflight/preflight.go index 5213df6e2..aba156d79 100644 --- a/pkg/resourceprovider/preflight/preflight.go +++ b/pkg/resourceprovider/preflight/preflight.go @@ -5,6 +5,8 @@ import ( "fmt" ) +const RequiredGPUMemoryGB = 1 + type GPUInfo struct { UUID string Name string From 7e746d54249c21482912f1414781d891afaf188c Mon Sep 17 00:00:00 2001 From: logan Date: Mon, 3 Feb 2025 11:59:12 -0600 Subject: [PATCH 17/27] refactor: improve GPU info parsing and validation in GetGPUInfo --- pkg/resourceprovider/preflight/gpu.go | 70 +++++++++++++++++++++------ 1 file changed, 54 insertions(+), 16 deletions(-) diff --git a/pkg/resourceprovider/preflight/gpu.go b/pkg/resourceprovider/preflight/gpu.go index 923fdce69..dd6947a33 100644 --- a/pkg/resourceprovider/preflight/gpu.go +++ b/pkg/resourceprovider/preflight/gpu.go @@ -33,8 +33,51 @@ type nvidiaSmiResponse struct { DriverVersion string } -func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { +func parseGPURecord(record string) (*GPUInfo, error) { + fields := strings.Split(record, ", ") + if len(fields) != 4 { + return nil, fmt.Errorf("invalid record format: expected 4 fields, got %d", len(fields)) + } + + // Parse memory, handling potential empty fields + memoryParts := strings.Split(strings.TrimSpace(fields[2]), " ") + if len(memoryParts) != 2 { + return nil, fmt.Errorf("invalid memory format: %s", fields[2]) + } + + memoryStr := memoryParts[0] + if memoryStr == "" { + return nil, fmt.Errorf("empty memory value") + } + + memoryMiB, err := strconv.ParseInt(memoryStr, 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse memory value '%s': %w", memoryStr, err) + } + + // Create GPU info with trimmed fields and validated memory + gpu := &GPUInfo{ + UUID: strings.TrimSpace(fields[0]), + Name: strings.TrimSpace(fields[1]), + MemoryTotal: memoryMiB, + DriverVersion: strings.TrimSpace(fields[3]), + } + + // Validate required fields + if gpu.UUID == "" { + return nil, fmt.Errorf("empty UUID") + } + if gpu.Name == "" { + return nil, fmt.Errorf("empty Name") + } + if gpu.DriverVersion == "" { + return nil, fmt.Errorf("empty DriverVersion") + } + return gpu, nil +} + +func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { if err := checkNvidiaSMI(); err != nil { return nil, fmt.Errorf("nvidia-smi not available: %w", err) } @@ -49,25 +92,16 @@ func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { } records := strings.Split(strings.TrimSpace(string(output)), "\n") - gpus := make([]GPUInfo, 0) + gpus := make([]GPUInfo, 0, len(records)) - for _, record := range records { - fields := strings.Split(record, ", ") - if len(fields) != 4 { + for i, record := range records { + gpu, err := parseGPURecord(record) + if err != nil { + log.Warn().Err(err).Int("index", i).Msg("Failed to parse GPU record") continue } - memoryStr := strings.Split(fields[2], " ")[0] - memoryMiB, _ := strconv.ParseInt(memoryStr, 10, 64) - - gpu := GPUInfo{ - UUID: strings.TrimSpace(fields[0]), - Name: strings.TrimSpace(fields[1]), - MemoryTotal: memoryMiB, - DriverVersion: strings.TrimSpace(fields[3]), - } - gpus = append(gpus, gpu) - + gpus = append(gpus, *gpu) log.Info(). Str("name", gpu.Name). Str("uuid", gpu.UUID). @@ -75,6 +109,10 @@ func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { Msgf("🎮 GPU %d details", len(gpus)) } + if len(gpus) == 0 { + return nil, fmt.Errorf("no valid GPUs found in nvidia-smi output") + } + return gpus, nil } From 2e5699375cfda33a242eb4e0b0a5d0e440d74c06 Mon Sep 17 00:00:00 2001 From: logan Date: Mon, 3 Feb 2025 12:21:14 -0600 Subject: [PATCH 18/27] chore: comments for required GPU VRAM --- pkg/resourceprovider/preflight/preflight.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/resourceprovider/preflight/preflight.go b/pkg/resourceprovider/preflight/preflight.go index aba156d79..22cd4c1cf 100644 --- a/pkg/resourceprovider/preflight/preflight.go +++ b/pkg/resourceprovider/preflight/preflight.go @@ -5,7 +5,7 @@ import ( "fmt" ) -const RequiredGPUMemoryGB = 1 +const RequiredGPUMemoryGB = 1 // 1GB of VRAM is required to startup if GPU is enabled type GPUInfo struct { UUID string From c2165041cd39b21bef4962fb9aa086ee081332ea Mon Sep 17 00:00:00 2001 From: Brian Ginsburg Date: Mon, 3 Feb 2025 15:17:02 -0800 Subject: [PATCH 19/27] refactor: Move preflight checker from interface to struct Co-authored-by: logan --- pkg/resourceprovider/preflight/docker.go | 2 +- pkg/resourceprovider/preflight/gpu.go | 8 ++------ pkg/resourceprovider/preflight/preflight.go | 14 +++----------- pkg/resourceprovider/resourceprovider.go | 4 ++-- 4 files changed, 8 insertions(+), 20 deletions(-) diff --git a/pkg/resourceprovider/preflight/docker.go b/pkg/resourceprovider/preflight/docker.go index a6bbe8672..0d65a1db5 100644 --- a/pkg/resourceprovider/preflight/docker.go +++ b/pkg/resourceprovider/preflight/docker.go @@ -11,7 +11,7 @@ type dockerInfo struct { Runtimes map[string]interface{} `json:"Runtimes"` } -func (p *preflightChecker) CheckDockerRuntime(ctx context.Context) CheckResult { +func (p *PreflightChecker) CheckDockerRuntime(ctx context.Context) CheckResult { cmd := exec.CommandContext(ctx, "docker", "info", "--format", "{{json .}}") output, err := cmd.Output() if err != nil { diff --git a/pkg/resourceprovider/preflight/gpu.go b/pkg/resourceprovider/preflight/gpu.go index dd6947a33..d461b6ca9 100644 --- a/pkg/resourceprovider/preflight/gpu.go +++ b/pkg/resourceprovider/preflight/gpu.go @@ -10,10 +10,6 @@ import ( "github.com/rs/zerolog/log" ) -type preflightChecker struct { - gpuInfo []GPUInfo -} - type GPUCheckConfig struct { Required bool MinGPUs int @@ -77,7 +73,7 @@ func parseGPURecord(record string) (*GPUInfo, error) { return gpu, nil } -func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { +func (p *PreflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { if err := checkNvidiaSMI(); err != nil { return nil, fmt.Errorf("nvidia-smi not available: %w", err) } @@ -116,7 +112,7 @@ func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { return gpus, nil } -func (p *preflightChecker) CheckGPU(ctx context.Context, config *GPUCheckConfig) CheckResult { +func (p *PreflightChecker) CheckGPU(ctx context.Context, config *GPUCheckConfig) CheckResult { if !config.Required { // Attempt to retrieve GPU info gpus, err := p.GetGPUInfo(ctx) diff --git a/pkg/resourceprovider/preflight/preflight.go b/pkg/resourceprovider/preflight/preflight.go index 22cd4c1cf..b8e0cb9ce 100644 --- a/pkg/resourceprovider/preflight/preflight.go +++ b/pkg/resourceprovider/preflight/preflight.go @@ -29,19 +29,11 @@ type PreflightConfig struct { } } -type PreflightChecker interface { - CheckGPU(ctx context.Context, config *GPUCheckConfig) CheckResult - CheckDockerRuntime(ctx context.Context) CheckResult - GetGPUInfo(ctx context.Context) ([]GPUInfo, error) - RunAllChecks(ctx context.Context, config PreflightConfig) error +type PreflightChecker struct { + gpuInfo []GPUInfo } -// NewPreflightChecker creates a new instance of PreflightChecker -func NewPreflightChecker() PreflightChecker { - return &preflightChecker{} -} - -func (p *preflightChecker) RunAllChecks(ctx context.Context, config PreflightConfig) error { +func (p *PreflightChecker) RunAllChecks(ctx context.Context, config PreflightConfig) error { gpuResult := p.CheckGPU(ctx, &GPUCheckConfig{ MinMemory: config.GPU.MinMemoryGB * 1024 * 1024 * 1024, diff --git a/pkg/resourceprovider/resourceprovider.go b/pkg/resourceprovider/resourceprovider.go index 6f0f5bd92..ebafca856 100644 --- a/pkg/resourceprovider/resourceprovider.go +++ b/pkg/resourceprovider/resourceprovider.go @@ -110,7 +110,7 @@ func NewResourceProvider( func runPreflightChecks(ctx context.Context, config preflight.PreflightConfig) error { log.Info().Msg("Starting preflight checks...") - checker := preflight.NewPreflightChecker() + checker := &preflight.PreflightChecker{} // Logging GPU requirements gpuInfo, err := checker.GetGPUInfo(ctx) @@ -134,7 +134,7 @@ func runPreflightChecks(ctx context.Context, config preflight.PreflightConfig) e func (resourceProvider *ResourceProvider) Start(ctx context.Context, cm *system.CleanupManager) chan error { errorChan := make(chan error, 1) - checker := preflight.NewPreflightChecker() + checker := &preflight.PreflightChecker{} gpuInfo, err := checker.GetGPUInfo(ctx) if err != nil { // Instead of returning error, just log warning and continue From b6342c6dc2eee6a6260b89ecdc709222573de5cc Mon Sep 17 00:00:00 2001 From: Brian Ginsburg Date: Mon, 3 Feb 2025 15:28:58 -0800 Subject: [PATCH 20/27] chore: Remove preflight check from start function Co-authored-by: logan --- pkg/resourceprovider/resourceprovider.go | 31 ++---------------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/pkg/resourceprovider/resourceprovider.go b/pkg/resourceprovider/resourceprovider.go index ebafca856..9c40db3ee 100644 --- a/pkg/resourceprovider/resourceprovider.go +++ b/pkg/resourceprovider/resourceprovider.go @@ -132,39 +132,12 @@ func runPreflightChecks(ctx context.Context, config preflight.PreflightConfig) e } func (resourceProvider *ResourceProvider) Start(ctx context.Context, cm *system.CleanupManager) chan error { - errorChan := make(chan error, 1) - - checker := &preflight.PreflightChecker{} - gpuInfo, err := checker.GetGPUInfo(ctx) - if err != nil { - // Instead of returning error, just log warning and continue - log.Warn().Err(err).Msg("⚠️ GPU capabilities will not be available - continuing in CPU-only mode") - resourceProvider.gpuInfo = []preflight.GPUInfo{} - } else { - resourceProvider.gpuInfo = gpuInfo - log.Info().Msgf("🎮 Successfully initialized with %d GPUs", len(gpuInfo)) - } - if !resourceProvider.options.Pow.DisablePow { if errCh := resourceProvider.StartMineLoop(ctx); errCh != nil { - // Forward any mining errors to the main error channel - go func() { - if err := <-errCh; err != nil { - errorChan <- fmt.Errorf("mining error: %w", err) - } - }() + return errCh } } - - // Start the controller and forward any errors - controllerErrChan := resourceProvider.controller.Start(ctx, cm) - go func() { - if err := <-controllerErrChan; err != nil { - errorChan <- err - } - }() - - return errorChan + return resourceProvider.controller.Start(ctx, cm) } func (resourceProvider *ResourceProvider) StartMineLoop(ctx context.Context) chan error { From 4582da5074777c79f62aeb7b4b1bac8ec417670c Mon Sep 17 00:00:00 2001 From: Brian Ginsburg Date: Mon, 3 Feb 2025 15:40:45 -0800 Subject: [PATCH 21/27] refactor: Move RunPreflightChecks function to preflight package Co-authored-by: logan --- pkg/resourceprovider/preflight/docker.go | 2 +- pkg/resourceprovider/preflight/gpu.go | 4 +-- pkg/resourceprovider/preflight/preflight.go | 29 +++++++++++++++++++-- pkg/resourceprovider/resourceprovider.go | 25 +----------------- 4 files changed, 31 insertions(+), 29 deletions(-) diff --git a/pkg/resourceprovider/preflight/docker.go b/pkg/resourceprovider/preflight/docker.go index 0d65a1db5..a6bbe8672 100644 --- a/pkg/resourceprovider/preflight/docker.go +++ b/pkg/resourceprovider/preflight/docker.go @@ -11,7 +11,7 @@ type dockerInfo struct { Runtimes map[string]interface{} `json:"Runtimes"` } -func (p *PreflightChecker) CheckDockerRuntime(ctx context.Context) CheckResult { +func (p *preflightChecker) CheckDockerRuntime(ctx context.Context) CheckResult { cmd := exec.CommandContext(ctx, "docker", "info", "--format", "{{json .}}") output, err := cmd.Output() if err != nil { diff --git a/pkg/resourceprovider/preflight/gpu.go b/pkg/resourceprovider/preflight/gpu.go index d461b6ca9..69fce8f92 100644 --- a/pkg/resourceprovider/preflight/gpu.go +++ b/pkg/resourceprovider/preflight/gpu.go @@ -73,7 +73,7 @@ func parseGPURecord(record string) (*GPUInfo, error) { return gpu, nil } -func (p *PreflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { +func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { if err := checkNvidiaSMI(); err != nil { return nil, fmt.Errorf("nvidia-smi not available: %w", err) } @@ -112,7 +112,7 @@ func (p *PreflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { return gpus, nil } -func (p *PreflightChecker) CheckGPU(ctx context.Context, config *GPUCheckConfig) CheckResult { +func (p *preflightChecker) CheckGPU(ctx context.Context, config *GPUCheckConfig) CheckResult { if !config.Required { // Attempt to retrieve GPU info gpus, err := p.GetGPUInfo(ctx) diff --git a/pkg/resourceprovider/preflight/preflight.go b/pkg/resourceprovider/preflight/preflight.go index b8e0cb9ce..ea6565a48 100644 --- a/pkg/resourceprovider/preflight/preflight.go +++ b/pkg/resourceprovider/preflight/preflight.go @@ -3,6 +3,8 @@ package preflight import ( "context" "fmt" + + "github.com/rs/zerolog/log" ) const RequiredGPUMemoryGB = 1 // 1GB of VRAM is required to startup if GPU is enabled @@ -29,11 +31,34 @@ type PreflightConfig struct { } } -type PreflightChecker struct { +type preflightChecker struct { gpuInfo []GPUInfo } -func (p *PreflightChecker) RunAllChecks(ctx context.Context, config PreflightConfig) error { +func RunPreflightChecks(ctx context.Context, config PreflightConfig) error { + log.Info().Msg("Starting preflight checks...") + checker := &preflightChecker{} + + // Logging GPU requirements + gpuInfo, err := checker.GetGPUInfo(ctx) + if err != nil { + log.Warn().Err(err).Msg("⚠️ No GPU detected - will operate in CPU-only mode") + } else { + log.Info(). + Int("gpu_count", len(gpuInfo)). + Int64("min_memory_gb", config.GPU.MinMemoryGB). + Msg("🎮 GPU requirements") + } + + err = checker.RunAllChecks(ctx, config) + if err != nil { + log.Error().Err(err).Msg("❌ Preflight checks failed") + return err + } + return nil +} + +func (p *preflightChecker) RunAllChecks(ctx context.Context, config PreflightConfig) error { gpuResult := p.CheckGPU(ctx, &GPUCheckConfig{ MinMemory: config.GPU.MinMemoryGB * 1024 * 1024 * 1024, diff --git a/pkg/resourceprovider/resourceprovider.go b/pkg/resourceprovider/resourceprovider.go index 9c40db3ee..fec7b5664 100644 --- a/pkg/resourceprovider/resourceprovider.go +++ b/pkg/resourceprovider/resourceprovider.go @@ -90,7 +90,7 @@ func NewResourceProvider( executor executor.Executor, tracer trace.Tracer, ) (*ResourceProvider, error) { - if err := runPreflightChecks(context.Background(), options.Preflight); err != nil { + if err := preflight.RunPreflightChecks(context.Background(), options.Preflight); err != nil { return nil, fmt.Errorf("preflight checks failed: %w", err) } @@ -108,29 +108,6 @@ func NewResourceProvider( return solver, nil } -func runPreflightChecks(ctx context.Context, config preflight.PreflightConfig) error { - log.Info().Msg("Starting preflight checks...") - checker := &preflight.PreflightChecker{} - - // Logging GPU requirements - gpuInfo, err := checker.GetGPUInfo(ctx) - if err != nil { - log.Warn().Err(err).Msg("⚠️ No GPU detected - will operate in CPU-only mode") - } else { - log.Info(). - Int("gpu_count", len(gpuInfo)). - Int64("min_memory_gb", config.GPU.MinMemoryGB). - Msg("🎮 GPU requirements") - } - - err = checker.RunAllChecks(ctx, config) - if err != nil { - log.Error().Err(err).Msg("❌ Preflight checks failed") - return err - } - return nil -} - func (resourceProvider *ResourceProvider) Start(ctx context.Context, cm *system.CleanupManager) chan error { if !resourceProvider.options.Pow.DisablePow { if errCh := resourceProvider.StartMineLoop(ctx); errCh != nil { From f01160d58a2c567e2ee5fa2502526c67fbe80fe7 Mon Sep 17 00:00:00 2001 From: Brian Ginsburg Date: Mon, 3 Feb 2025 15:55:31 -0800 Subject: [PATCH 22/27] refactor: Move preflight config to preflight package Co-authored-by: logan --- pkg/options/resource-provider.go | 8 -------- pkg/resourceprovider/preflight/preflight.go | 13 ++++++++++--- pkg/resourceprovider/resourceprovider.go | 3 +-- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/pkg/options/resource-provider.go b/pkg/options/resource-provider.go index be4a3e3e3..06e7a1cc5 100644 --- a/pkg/options/resource-provider.go +++ b/pkg/options/resource-provider.go @@ -6,7 +6,6 @@ import ( "github.com/lilypad-tech/lilypad/pkg/data" "github.com/lilypad-tech/lilypad/pkg/resourceprovider" - "github.com/lilypad-tech/lilypad/pkg/resourceprovider/preflight" "github.com/lilypad-tech/lilypad/pkg/system" "github.com/spf13/cobra" ) @@ -19,13 +18,6 @@ func NewResourceProviderOptions() resourceprovider.ResourceProviderOptions { Pow: GetDefaultResourceProviderPowOptions(), IPFS: GetDefaultIPFSOptions(), Telemetry: GetDefaultTelemetryOptions(), - Preflight: preflight.PreflightConfig{ - GPU: struct { - MinMemoryGB int64 - }{ - MinMemoryGB: preflight.RequiredGPUMemoryGB, - }, - }, } options.Web3.Service = system.ResourceProviderService return options diff --git a/pkg/resourceprovider/preflight/preflight.go b/pkg/resourceprovider/preflight/preflight.go index ea6565a48..7a269134a 100644 --- a/pkg/resourceprovider/preflight/preflight.go +++ b/pkg/resourceprovider/preflight/preflight.go @@ -22,7 +22,7 @@ type CheckResult struct { Error error } -type PreflightConfig struct { +type preflightConfig struct { GPU struct { MinMemoryGB int64 } @@ -35,9 +35,16 @@ type preflightChecker struct { gpuInfo []GPUInfo } -func RunPreflightChecks(ctx context.Context, config PreflightConfig) error { +func RunPreflightChecks(ctx context.Context) error { log.Info().Msg("Starting preflight checks...") checker := &preflightChecker{} + config := preflightConfig{ + GPU: struct { + MinMemoryGB int64 + }{ + MinMemoryGB: RequiredGPUMemoryGB, + }, + } // Logging GPU requirements gpuInfo, err := checker.GetGPUInfo(ctx) @@ -58,7 +65,7 @@ func RunPreflightChecks(ctx context.Context, config PreflightConfig) error { return nil } -func (p *preflightChecker) RunAllChecks(ctx context.Context, config PreflightConfig) error { +func (p *preflightChecker) RunAllChecks(ctx context.Context, config preflightConfig) error { gpuResult := p.CheckGPU(ctx, &GPUCheckConfig{ MinMemory: config.GPU.MinMemoryGB * 1024 * 1024 * 1024, diff --git a/pkg/resourceprovider/resourceprovider.go b/pkg/resourceprovider/resourceprovider.go index fec7b5664..b60765558 100644 --- a/pkg/resourceprovider/resourceprovider.go +++ b/pkg/resourceprovider/resourceprovider.go @@ -74,7 +74,6 @@ type ResourceProviderOptions struct { Pow ResourceProviderPowOptions IPFS ipfs.IPFSOptions Telemetry system.TelemetryOptions - Preflight preflight.PreflightConfig } type ResourceProvider struct { @@ -90,7 +89,7 @@ func NewResourceProvider( executor executor.Executor, tracer trace.Tracer, ) (*ResourceProvider, error) { - if err := preflight.RunPreflightChecks(context.Background(), options.Preflight); err != nil { + if err := preflight.RunPreflightChecks(context.Background()); err != nil { return nil, fmt.Errorf("preflight checks failed: %w", err) } From 65330adea100380528107e3ca715a6a7c0fb33b1 Mon Sep 17 00:00:00 2001 From: logan Date: Mon, 3 Feb 2025 19:26:45 -0600 Subject: [PATCH 23/27] chore: refactor context within resource provider --- pkg/resourceprovider/preflight/preflight.go | 3 ++- pkg/resourceprovider/resourceprovider.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/resourceprovider/preflight/preflight.go b/pkg/resourceprovider/preflight/preflight.go index 7a269134a..833329b81 100644 --- a/pkg/resourceprovider/preflight/preflight.go +++ b/pkg/resourceprovider/preflight/preflight.go @@ -35,7 +35,8 @@ type preflightChecker struct { gpuInfo []GPUInfo } -func RunPreflightChecks(ctx context.Context) error { +func RunPreflightChecks() error { + ctx := context.Background() log.Info().Msg("Starting preflight checks...") checker := &preflightChecker{} config := preflightConfig{ diff --git a/pkg/resourceprovider/resourceprovider.go b/pkg/resourceprovider/resourceprovider.go index b60765558..4940998d9 100644 --- a/pkg/resourceprovider/resourceprovider.go +++ b/pkg/resourceprovider/resourceprovider.go @@ -89,7 +89,8 @@ func NewResourceProvider( executor executor.Executor, tracer trace.Tracer, ) (*ResourceProvider, error) { - if err := preflight.RunPreflightChecks(context.Background()); err != nil { + + if err := preflight.RunPreflightChecks(); err != nil { return nil, fmt.Errorf("preflight checks failed: %w", err) } From 12d42386d9b6b5037d36dd983450e4f8f04077b2 Mon Sep 17 00:00:00 2001 From: logan Date: Tue, 4 Feb 2025 11:35:17 -0600 Subject: [PATCH 24/27] chore: remove unused gpuInfo field --- pkg/resourceprovider/resourceprovider.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/resourceprovider/resourceprovider.go b/pkg/resourceprovider/resourceprovider.go index 4940998d9..fa5c0827b 100644 --- a/pkg/resourceprovider/resourceprovider.go +++ b/pkg/resourceprovider/resourceprovider.go @@ -80,7 +80,6 @@ type ResourceProvider struct { web3SDK *web3.Web3SDK options ResourceProviderOptions controller *ResourceProviderController - gpuInfo []preflight.GPUInfo } func NewResourceProvider( From f8c43acebe6a8b1b9ba7632b2d52a5638e1844e8 Mon Sep 17 00:00:00 2001 From: Brian Ginsburg Date: Tue, 4 Feb 2025 10:08:04 -0800 Subject: [PATCH 25/27] refactor: Make functions and structs private where possible Co-authored-by: logan --- pkg/resourceprovider/preflight/docker.go | 40 +++++------ pkg/resourceprovider/preflight/gpu.go | 80 ++++++++++----------- pkg/resourceprovider/preflight/preflight.go | 41 ++++++----- pkg/resourceprovider/resourceprovider.go | 1 - 4 files changed, 80 insertions(+), 82 deletions(-) diff --git a/pkg/resourceprovider/preflight/docker.go b/pkg/resourceprovider/preflight/docker.go index a6bbe8672..f4adba830 100644 --- a/pkg/resourceprovider/preflight/docker.go +++ b/pkg/resourceprovider/preflight/docker.go @@ -11,48 +11,48 @@ type dockerInfo struct { Runtimes map[string]interface{} `json:"Runtimes"` } -func (p *preflightChecker) CheckDockerRuntime(ctx context.Context) CheckResult { +func (p *preflightChecker) checkDockerRuntime(ctx context.Context) checkResult { cmd := exec.CommandContext(ctx, "docker", "info", "--format", "{{json .}}") output, err := cmd.Output() if err != nil { - return CheckResult{ - Passed: false, - Error: fmt.Errorf("failed to get Docker info: %w", err), - Message: "Docker check failed", + return checkResult{ + passed: false, + error: fmt.Errorf("failed to get Docker info: %w", err), + message: "Docker check failed", } } var info dockerInfo if err := json.Unmarshal(output, &info); err != nil { - return CheckResult{ - Passed: false, - Error: fmt.Errorf("failed to parse Docker info: %w", err), - Message: "Docker info parsing failed", + return checkResult{ + passed: false, + error: fmt.Errorf("failed to parse Docker info: %w", err), + message: "Docker info parsing failed", } } // Check for nvidia runtime _, hasNvidia := info.Runtimes["nvidia"] if !hasNvidia { - return CheckResult{ - Passed: false, - Error: fmt.Errorf("nvidia runtime not found in Docker configuration"), - Message: "NVIDIA runtime not found in Docker", + return checkResult{ + passed: false, + error: fmt.Errorf("nvidia runtime not found in Docker configuration"), + message: "NVIDIA runtime not found in Docker", } } // Test nvidia runtime testCmd := exec.CommandContext(ctx, "docker", "run", "--rm", "--runtime=nvidia", "nvidia/cuda:11.8.0-base", "nvidia-smi") if err := testCmd.Run(); err != nil { - return CheckResult{ - Passed: false, - Error: fmt.Errorf("failed to run NVIDIA runtime test: %w", err), - Message: "NVIDIA runtime test failed", + return checkResult{ + passed: false, + error: fmt.Errorf("failed to run NVIDIA runtime test: %w", err), + message: "NVIDIA runtime test failed", } } - return CheckResult{ - Passed: true, - Message: "NVIDIA runtime is available and functional", + return checkResult{ + passed: true, + message: "NVIDIA runtime is available and functional", } } diff --git a/pkg/resourceprovider/preflight/gpu.go b/pkg/resourceprovider/preflight/gpu.go index 69fce8f92..b9d8152bc 100644 --- a/pkg/resourceprovider/preflight/gpu.go +++ b/pkg/resourceprovider/preflight/gpu.go @@ -10,11 +10,11 @@ import ( "github.com/rs/zerolog/log" ) -type GPUCheckConfig struct { - Required bool - MinGPUs int - MinMemory int64 - Capabilities []string +type gpuCheckConfig struct { + required bool + minGPUs int + minMemory int64 + capabilities []string } func checkNvidiaSMI() error { @@ -23,13 +23,13 @@ func checkNvidiaSMI() error { } type nvidiaSmiResponse struct { - UUID string - Name string - MemoryTotal string - DriverVersion string + uuid string + name string + memoryTotal string + driverVersion string } -func parseGPURecord(record string) (*GPUInfo, error) { +func parseGPURecord(record string) (*gpuInfo, error) { fields := strings.Split(record, ", ") if len(fields) != 4 { return nil, fmt.Errorf("invalid record format: expected 4 fields, got %d", len(fields)) @@ -52,28 +52,28 @@ func parseGPURecord(record string) (*GPUInfo, error) { } // Create GPU info with trimmed fields and validated memory - gpu := &GPUInfo{ - UUID: strings.TrimSpace(fields[0]), - Name: strings.TrimSpace(fields[1]), - MemoryTotal: memoryMiB, - DriverVersion: strings.TrimSpace(fields[3]), + gpu := &gpuInfo{ + uuid: strings.TrimSpace(fields[0]), + name: strings.TrimSpace(fields[1]), + memoryTotal: memoryMiB, + driverVersion: strings.TrimSpace(fields[3]), } // Validate required fields - if gpu.UUID == "" { + if gpu.uuid == "" { return nil, fmt.Errorf("empty UUID") } - if gpu.Name == "" { + if gpu.name == "" { return nil, fmt.Errorf("empty Name") } - if gpu.DriverVersion == "" { + if gpu.driverVersion == "" { return nil, fmt.Errorf("empty DriverVersion") } return gpu, nil } -func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { +func (p *preflightChecker) getGPUInfo(ctx context.Context) ([]gpuInfo, error) { if err := checkNvidiaSMI(); err != nil { return nil, fmt.Errorf("nvidia-smi not available: %w", err) } @@ -88,7 +88,7 @@ func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { } records := strings.Split(strings.TrimSpace(string(output)), "\n") - gpus := make([]GPUInfo, 0, len(records)) + gpus := make([]gpuInfo, 0, len(records)) for i, record := range records { gpu, err := parseGPURecord(record) @@ -99,9 +99,9 @@ func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { gpus = append(gpus, *gpu) log.Info(). - Str("name", gpu.Name). - Str("uuid", gpu.UUID). - Int64("memory_mb", gpu.MemoryTotal). + Str("name", gpu.name). + Str("uuid", gpu.uuid). + Int64("memory_mb", gpu.memoryTotal). Msgf("🎮 GPU %d details", len(gpus)) } @@ -112,40 +112,40 @@ func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { return gpus, nil } -func (p *preflightChecker) CheckGPU(ctx context.Context, config *GPUCheckConfig) CheckResult { - if !config.Required { +func (p *preflightChecker) checkGPU(ctx context.Context, config *gpuCheckConfig) checkResult { + if !config.required { // Attempt to retrieve GPU info - gpus, err := p.GetGPUInfo(ctx) + gpus, err := p.getGPUInfo(ctx) if err != nil { log.Warn().Msg("⚠️ Running without GPU support - Resource Provider will operate in CPU-only mode") - return CheckResult{ - Passed: true, - Message: "Operating in CPU-only mode", + return checkResult{ + passed: true, + message: "Operating in CPU-only mode", } } // If we found GPUs, log them but still continue log.Info().Msgf("🎮 Found %d optional GPUs available for use", len(gpus)) - return CheckResult{ - Passed: true, - Message: fmt.Sprintf("Found %d NVIDIA GPUs (optional)", len(gpus)), + return checkResult{ + passed: true, + message: fmt.Sprintf("Found %d NVIDIA GPUs (optional)", len(gpus)), } } // Required GPU checks log.Info().Msg("Starting required GPU checks") - gpus, err := p.GetGPUInfo(ctx) + gpus, err := p.getGPUInfo(ctx) if err != nil { - return CheckResult{ - Passed: false, - Error: err, - Message: "Required GPU check failed - no NVIDIA GPUs detected", + return checkResult{ + passed: false, + error: err, + message: "Required GPU check failed - no NVIDIA GPUs detected", } } log.Info().Msg("✅ GPU requirements satisfied") - return CheckResult{ - Passed: true, - Message: fmt.Sprintf("Found %d suitable GPUs", len(gpus)), + return checkResult{ + passed: true, + message: fmt.Sprintf("Found %d suitable GPUs", len(gpus)), } } diff --git a/pkg/resourceprovider/preflight/preflight.go b/pkg/resourceprovider/preflight/preflight.go index 833329b81..26e152e5e 100644 --- a/pkg/resourceprovider/preflight/preflight.go +++ b/pkg/resourceprovider/preflight/preflight.go @@ -9,17 +9,17 @@ import ( const RequiredGPUMemoryGB = 1 // 1GB of VRAM is required to startup if GPU is enabled -type GPUInfo struct { - UUID string - Name string - MemoryTotal int64 - DriverVersion string +type gpuInfo struct { + uuid string + name string + memoryTotal int64 + driverVersion string } -type CheckResult struct { - Passed bool - Message string - Error error +type checkResult struct { + passed bool + message string + error error } type preflightConfig struct { @@ -32,7 +32,7 @@ type preflightConfig struct { } type preflightChecker struct { - gpuInfo []GPUInfo + gpuInfo []gpuInfo } func RunPreflightChecks() error { @@ -48,7 +48,7 @@ func RunPreflightChecks() error { } // Logging GPU requirements - gpuInfo, err := checker.GetGPUInfo(ctx) + gpuInfo, err := checker.getGPUInfo(ctx) if err != nil { log.Warn().Err(err).Msg("⚠️ No GPU detected - will operate in CPU-only mode") } else { @@ -58,7 +58,7 @@ func RunPreflightChecks() error { Msg("🎮 GPU requirements") } - err = checker.RunAllChecks(ctx, config) + err = checker.runAllChecks(ctx, config) if err != nil { log.Error().Err(err).Msg("❌ Preflight checks failed") return err @@ -66,19 +66,18 @@ func RunPreflightChecks() error { return nil } -func (p *preflightChecker) RunAllChecks(ctx context.Context, config preflightConfig) error { - - gpuResult := p.CheckGPU(ctx, &GPUCheckConfig{ - MinMemory: config.GPU.MinMemoryGB * 1024 * 1024 * 1024, +func (p *preflightChecker) runAllChecks(ctx context.Context, config preflightConfig) error { + gpuResult := p.checkGPU(ctx, &gpuCheckConfig{ + minMemory: config.GPU.MinMemoryGB * 1024 * 1024 * 1024, }) - if !gpuResult.Passed { - return fmt.Errorf("GPU check failed: %s", gpuResult.Message) + if !gpuResult.passed { + return fmt.Errorf("GPU check failed: %s", gpuResult.message) } if config.Docker.CheckRuntime { - runtimeResult := p.CheckDockerRuntime(ctx) - if !runtimeResult.Passed { - return fmt.Errorf("Docker runtime check failed: %s", runtimeResult.Message) + runtimeResult := p.checkDockerRuntime(ctx) + if !runtimeResult.passed { + return fmt.Errorf("Docker runtime check failed: %s", runtimeResult.message) } } diff --git a/pkg/resourceprovider/resourceprovider.go b/pkg/resourceprovider/resourceprovider.go index fa5c0827b..1cbcab1e6 100644 --- a/pkg/resourceprovider/resourceprovider.go +++ b/pkg/resourceprovider/resourceprovider.go @@ -88,7 +88,6 @@ func NewResourceProvider( executor executor.Executor, tracer trace.Tracer, ) (*ResourceProvider, error) { - if err := preflight.RunPreflightChecks(); err != nil { return nil, fmt.Errorf("preflight checks failed: %w", err) } From 5ac7ee590c06400a4a268eb04a06a0b8b7ea4738 Mon Sep 17 00:00:00 2001 From: Brian Ginsburg Date: Tue, 4 Feb 2025 10:09:04 -0800 Subject: [PATCH 26/27] chore: Exit early when no GPU detected Co-authored-by: logan --- pkg/resourceprovider/preflight/preflight.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/resourceprovider/preflight/preflight.go b/pkg/resourceprovider/preflight/preflight.go index 26e152e5e..c047229c6 100644 --- a/pkg/resourceprovider/preflight/preflight.go +++ b/pkg/resourceprovider/preflight/preflight.go @@ -51,6 +51,7 @@ func RunPreflightChecks() error { gpuInfo, err := checker.getGPUInfo(ctx) if err != nil { log.Warn().Err(err).Msg("⚠️ No GPU detected - will operate in CPU-only mode") + return nil } else { log.Info(). Int("gpu_count", len(gpuInfo)). From 0150090ceadf9f33ff750d7d754f319ade210c73 Mon Sep 17 00:00:00 2001 From: Brian Ginsburg Date: Tue, 4 Feb 2025 10:25:53 -0800 Subject: [PATCH 27/27] chore: Improve failed to parse GPU string error Co-authored-by: logan --- pkg/resourceprovider/preflight/gpu.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/resourceprovider/preflight/gpu.go b/pkg/resourceprovider/preflight/gpu.go index b9d8152bc..160943a30 100644 --- a/pkg/resourceprovider/preflight/gpu.go +++ b/pkg/resourceprovider/preflight/gpu.go @@ -90,10 +90,10 @@ func (p *preflightChecker) getGPUInfo(ctx context.Context) ([]gpuInfo, error) { records := strings.Split(strings.TrimSpace(string(output)), "\n") gpus := make([]gpuInfo, 0, len(records)) - for i, record := range records { + for _, record := range records { gpu, err := parseGPURecord(record) if err != nil { - log.Warn().Err(err).Int("index", i).Msg("Failed to parse GPU record") + log.Warn().Err(err).Msgf("Failed to parse GPU record: %s", record) continue }