diff --git a/pkg/resourceprovider/preflight/docker.go b/pkg/resourceprovider/preflight/docker.go new file mode 100644 index 000000000..f4adba830 --- /dev/null +++ b/pkg/resourceprovider/preflight/docker.go @@ -0,0 +1,58 @@ +package preflight + +import ( + "context" + "encoding/json" + "fmt" + "os/exec" +) + +type dockerInfo struct { + Runtimes map[string]interface{} `json:"Runtimes"` +} + +func (p *preflightChecker) checkDockerRuntime(ctx context.Context) checkResult { + cmd := exec.CommandContext(ctx, "docker", "info", "--format", "{{json .}}") + output, err := cmd.Output() + if err != nil { + return checkResult{ + passed: false, + error: fmt.Errorf("failed to get Docker info: %w", err), + message: "Docker check failed", + } + } + + var info dockerInfo + if err := json.Unmarshal(output, &info); err != nil { + return checkResult{ + passed: false, + error: fmt.Errorf("failed to parse Docker info: %w", err), + message: "Docker info parsing failed", + } + } + + // Check for nvidia runtime + _, hasNvidia := info.Runtimes["nvidia"] + if !hasNvidia { + return checkResult{ + passed: false, + error: fmt.Errorf("nvidia runtime not found in Docker configuration"), + message: "NVIDIA runtime not found in Docker", + } + } + + // Test nvidia runtime + testCmd := exec.CommandContext(ctx, "docker", "run", "--rm", "--runtime=nvidia", "nvidia/cuda:11.8.0-base", "nvidia-smi") + if err := testCmd.Run(); err != nil { + return checkResult{ + passed: false, + error: fmt.Errorf("failed to run NVIDIA runtime test: %w", err), + message: "NVIDIA runtime test failed", + } + } + + return checkResult{ + passed: true, + message: "NVIDIA runtime is available and functional", + } +} diff --git a/pkg/resourceprovider/preflight/gpu.go b/pkg/resourceprovider/preflight/gpu.go new file mode 100644 index 000000000..160943a30 --- /dev/null +++ b/pkg/resourceprovider/preflight/gpu.go @@ -0,0 +1,151 @@ +package preflight + +import ( + "context" + "fmt" + "os/exec" + "strconv" + "strings" + + "github.com/rs/zerolog/log" +) + +type gpuCheckConfig struct { + required bool + minGPUs int + minMemory int64 + capabilities []string +} + +func checkNvidiaSMI() error { + _, err := exec.LookPath("nvidia-smi") + return err +} + +type nvidiaSmiResponse struct { + uuid string + name string + memoryTotal string + driverVersion string +} + +func parseGPURecord(record string) (*gpuInfo, error) { + fields := strings.Split(record, ", ") + if len(fields) != 4 { + return nil, fmt.Errorf("invalid record format: expected 4 fields, got %d", len(fields)) + } + + // Parse memory, handling potential empty fields + memoryParts := strings.Split(strings.TrimSpace(fields[2]), " ") + if len(memoryParts) != 2 { + return nil, fmt.Errorf("invalid memory format: %s", fields[2]) + } + + memoryStr := memoryParts[0] + if memoryStr == "" { + return nil, fmt.Errorf("empty memory value") + } + + memoryMiB, err := strconv.ParseInt(memoryStr, 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse memory value '%s': %w", memoryStr, err) + } + + // Create GPU info with trimmed fields and validated memory + gpu := &gpuInfo{ + uuid: strings.TrimSpace(fields[0]), + name: strings.TrimSpace(fields[1]), + memoryTotal: memoryMiB, + driverVersion: strings.TrimSpace(fields[3]), + } + + // Validate required fields + if gpu.uuid == "" { + return nil, fmt.Errorf("empty UUID") + } + if gpu.name == "" { + return nil, fmt.Errorf("empty Name") + } + if gpu.driverVersion == "" { + return nil, fmt.Errorf("empty DriverVersion") + } + + return gpu, nil +} + +func (p *preflightChecker) getGPUInfo(ctx context.Context) ([]gpuInfo, error) { + if err := checkNvidiaSMI(); err != nil { + return nil, fmt.Errorf("nvidia-smi not available: %w", err) + } + + cmd := exec.CommandContext(ctx, "nvidia-smi", + "--query-gpu=gpu_uuid,gpu_name,memory.total,driver_version", + "--format=csv,noheader") + output, err := cmd.CombinedOutput() + if err != nil { + log.Error().Str("output", string(output)).Err(err).Msg("nvidia-smi command failed") + return nil, fmt.Errorf("error running nvidia-smi: %w", err) + } + + records := strings.Split(strings.TrimSpace(string(output)), "\n") + gpus := make([]gpuInfo, 0, len(records)) + + for _, record := range records { + gpu, err := parseGPURecord(record) + if err != nil { + log.Warn().Err(err).Msgf("Failed to parse GPU record: %s", record) + continue + } + + gpus = append(gpus, *gpu) + log.Info(). + Str("name", gpu.name). + Str("uuid", gpu.uuid). + Int64("memory_mb", gpu.memoryTotal). + Msgf("🎮 GPU %d details", len(gpus)) + } + + if len(gpus) == 0 { + return nil, fmt.Errorf("no valid GPUs found in nvidia-smi output") + } + + return gpus, nil +} + +func (p *preflightChecker) checkGPU(ctx context.Context, config *gpuCheckConfig) checkResult { + if !config.required { + // Attempt to retrieve GPU info + gpus, err := p.getGPUInfo(ctx) + if err != nil { + log.Warn().Msg("⚠️ Running without GPU support - Resource Provider will operate in CPU-only mode") + return checkResult{ + passed: true, + message: "Operating in CPU-only mode", + } + } + + // If we found GPUs, log them but still continue + log.Info().Msgf("🎮 Found %d optional GPUs available for use", len(gpus)) + return checkResult{ + passed: true, + message: fmt.Sprintf("Found %d NVIDIA GPUs (optional)", len(gpus)), + } + } + + // Required GPU checks + log.Info().Msg("Starting required GPU checks") + gpus, err := p.getGPUInfo(ctx) + if err != nil { + return checkResult{ + passed: false, + error: err, + message: "Required GPU check failed - no NVIDIA GPUs detected", + } + } + + log.Info().Msg("✅ GPU requirements satisfied") + return checkResult{ + passed: true, + message: fmt.Sprintf("Found %d suitable GPUs", len(gpus)), + } +} diff --git a/pkg/resourceprovider/preflight/preflight.go b/pkg/resourceprovider/preflight/preflight.go new file mode 100644 index 000000000..c047229c6 --- /dev/null +++ b/pkg/resourceprovider/preflight/preflight.go @@ -0,0 +1,86 @@ +package preflight + +import ( + "context" + "fmt" + + "github.com/rs/zerolog/log" +) + +const RequiredGPUMemoryGB = 1 // 1GB of VRAM is required to startup if GPU is enabled + +type gpuInfo struct { + uuid string + name string + memoryTotal int64 + driverVersion string +} + +type checkResult struct { + passed bool + message string + error error +} + +type preflightConfig struct { + GPU struct { + MinMemoryGB int64 + } + Docker struct { + CheckRuntime bool + } +} + +type preflightChecker struct { + gpuInfo []gpuInfo +} + +func RunPreflightChecks() error { + ctx := context.Background() + log.Info().Msg("Starting preflight checks...") + checker := &preflightChecker{} + config := preflightConfig{ + GPU: struct { + MinMemoryGB int64 + }{ + MinMemoryGB: RequiredGPUMemoryGB, + }, + } + + // Logging GPU requirements + gpuInfo, err := checker.getGPUInfo(ctx) + if err != nil { + log.Warn().Err(err).Msg("⚠️ No GPU detected - will operate in CPU-only mode") + return nil + } else { + log.Info(). + Int("gpu_count", len(gpuInfo)). + Int64("min_memory_gb", config.GPU.MinMemoryGB). + Msg("🎮 GPU requirements") + } + + err = checker.runAllChecks(ctx, config) + if err != nil { + log.Error().Err(err).Msg("❌ Preflight checks failed") + return err + } + return nil +} + +func (p *preflightChecker) runAllChecks(ctx context.Context, config preflightConfig) error { + gpuResult := p.checkGPU(ctx, &gpuCheckConfig{ + minMemory: config.GPU.MinMemoryGB * 1024 * 1024 * 1024, + }) + if !gpuResult.passed { + return fmt.Errorf("GPU check failed: %s", gpuResult.message) + } + + if config.Docker.CheckRuntime { + runtimeResult := p.checkDockerRuntime(ctx) + if !runtimeResult.passed { + return fmt.Errorf("Docker runtime check failed: %s", runtimeResult.message) + } + } + + return nil +} diff --git a/pkg/resourceprovider/resourceprovider.go b/pkg/resourceprovider/resourceprovider.go index cb3a25d6e..1cbcab1e6 100644 --- a/pkg/resourceprovider/resourceprovider.go +++ b/pkg/resourceprovider/resourceprovider.go @@ -17,6 +17,7 @@ import ( "github.com/lilypad-tech/lilypad/pkg/executor/bacalhau" "github.com/lilypad-tech/lilypad/pkg/ipfs" "github.com/lilypad-tech/lilypad/pkg/powLogs" + "github.com/lilypad-tech/lilypad/pkg/resourceprovider/preflight" "github.com/lilypad-tech/lilypad/pkg/system" "github.com/lilypad-tech/lilypad/pkg/web3" "github.com/lilypad-tech/lilypad/pkg/web3/bindings/pow" @@ -59,9 +60,8 @@ type ResourceProviderOfferOptions struct { // this configures the pow we will keep track of type ResourceProviderPowOptions struct { - DisablePow bool - NumWorkers int - + DisablePow bool + NumWorkers int CudaGridSize int CudaBlockSize int CudaHashsPerThread int @@ -88,10 +88,15 @@ func NewResourceProvider( executor executor.Executor, tracer trace.Tracer, ) (*ResourceProvider, error) { + if err := preflight.RunPreflightChecks(); err != nil { + return nil, fmt.Errorf("preflight checks failed: %w", err) + } + controller, err := NewResourceProviderController(options, web3SDK, executor, tracer) if err != nil { return nil, err } + solver := &ResourceProvider{ controller: controller, options: options,