-
Notifications
You must be signed in to change notification settings - Fork 28
feat: implement preflight #502
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from 23 commits
Commits
Show all changes
27 commits
Select commit
Hold shift + click to select a range
a334ecd
feat: barebones
logan-wrld ced7238
feat: Implement preflight checks for GPU and Docker runtime
logan-wrld a7ee2ae
feat: Preflight logging and success messages
logan-wrld 6f6ed68
chore: restore comments in [pkg/resourceprovider/resourceprovider.go]
logan-wrld f201b4d
feat: gpu check + cleanup
logan-wrld f436084
fix: gpu nvidia-smi check
logan-wrld d6099e0
fix: handle no-GPU case gracefully
logan-wrld 834cd16
fix: update start method in RP
logan-wrld 95c6ca0
chore: remove unused dockerfiles
logan-wrld 7db0a3f
chore: restore comments in [pkg/resourceprovider/resourceprovider.go]
logan-wrld 4db0862
chore: remove comments within [pkg/resourceprovider/resourceprovider.go]
logan-wrld 8e1e504
refactor: removed minimum GPU parameter
logan-wrld 06329f0
feat: 1gb ram requirement
logan-wrld a77b80d
refactor: simplify GPU configuration by removing unnecessary parameters
logan-wrld 952898e
refactor: enhance GPU info logging and remove types file
logan-wrld 00e7467
refactor: replace hardcoded GPU memory with default constant
logan-wrld 7e746d5
refactor: improve GPU info parsing and validation in GetGPUInfo
logan-wrld 2e56993
chore: comments for required GPU VRAM
logan-wrld c216504
refactor: Move preflight checker from interface to struct
bgins b6342c6
chore: Remove preflight check from start function
bgins 4582da5
refactor: Move RunPreflightChecks function to preflight package
bgins f01160d
refactor: Move preflight config to preflight package
bgins 65330ad
chore: refactor context within resource provider
logan-wrld 12d4238
chore: remove unused gpuInfo field
logan-wrld f8c43ac
refactor: Make functions and structs private where possible
bgins 5ac7ee5
chore: Exit early when no GPU detected
bgins 0150090
chore: Improve failed to parse GPU string error
bgins File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| package preflight | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "fmt" | ||
| "os/exec" | ||
| ) | ||
|
|
||
| type dockerInfo struct { | ||
| Runtimes map[string]interface{} `json:"Runtimes"` | ||
| } | ||
|
|
||
| func (p *preflightChecker) CheckDockerRuntime(ctx context.Context) CheckResult { | ||
| cmd := exec.CommandContext(ctx, "docker", "info", "--format", "{{json .}}") | ||
| output, err := cmd.Output() | ||
| if err != nil { | ||
| return CheckResult{ | ||
| Passed: false, | ||
| Error: fmt.Errorf("failed to get Docker info: %w", err), | ||
| Message: "Docker check failed", | ||
| } | ||
| } | ||
|
|
||
| var info dockerInfo | ||
| if err := json.Unmarshal(output, &info); err != nil { | ||
| return CheckResult{ | ||
| Passed: false, | ||
| Error: fmt.Errorf("failed to parse Docker info: %w", err), | ||
| Message: "Docker info parsing failed", | ||
| } | ||
| } | ||
|
|
||
| // Check for nvidia runtime | ||
| _, hasNvidia := info.Runtimes["nvidia"] | ||
| if !hasNvidia { | ||
| return CheckResult{ | ||
| Passed: false, | ||
| Error: fmt.Errorf("nvidia runtime not found in Docker configuration"), | ||
| Message: "NVIDIA runtime not found in Docker", | ||
| } | ||
| } | ||
|
|
||
| // Test nvidia runtime | ||
| testCmd := exec.CommandContext(ctx, "docker", "run", "--rm", "--runtime=nvidia", "nvidia/cuda:11.8.0-base", "nvidia-smi") | ||
| if err := testCmd.Run(); err != nil { | ||
| return CheckResult{ | ||
| Passed: false, | ||
| Error: fmt.Errorf("failed to run NVIDIA runtime test: %w", err), | ||
| Message: "NVIDIA runtime test failed", | ||
| } | ||
| } | ||
|
|
||
| return CheckResult{ | ||
| Passed: true, | ||
| Message: "NVIDIA runtime is available and functional", | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,151 @@ | ||
| package preflight | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "os/exec" | ||
| "strconv" | ||
| "strings" | ||
|
|
||
| "github.com/rs/zerolog/log" | ||
| ) | ||
|
|
||
| type GPUCheckConfig struct { | ||
| Required bool | ||
| MinGPUs int | ||
| MinMemory int64 | ||
| Capabilities []string | ||
| } | ||
|
|
||
| func checkNvidiaSMI() error { | ||
| _, err := exec.LookPath("nvidia-smi") | ||
| return err | ||
| } | ||
|
|
||
| type nvidiaSmiResponse struct { | ||
| UUID string | ||
| Name string | ||
| MemoryTotal string | ||
| DriverVersion string | ||
| } | ||
|
|
||
| func parseGPURecord(record string) (*GPUInfo, error) { | ||
| fields := strings.Split(record, ", ") | ||
| if len(fields) != 4 { | ||
| return nil, fmt.Errorf("invalid record format: expected 4 fields, got %d", len(fields)) | ||
| } | ||
|
|
||
| // Parse memory, handling potential empty fields | ||
| memoryParts := strings.Split(strings.TrimSpace(fields[2]), " ") | ||
| if len(memoryParts) != 2 { | ||
| return nil, fmt.Errorf("invalid memory format: %s", fields[2]) | ||
| } | ||
|
|
||
| memoryStr := memoryParts[0] | ||
| if memoryStr == "" { | ||
| return nil, fmt.Errorf("empty memory value") | ||
| } | ||
|
|
||
| memoryMiB, err := strconv.ParseInt(memoryStr, 10, 64) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to parse memory value '%s': %w", memoryStr, err) | ||
| } | ||
|
|
||
| // Create GPU info with trimmed fields and validated memory | ||
| gpu := &GPUInfo{ | ||
| UUID: strings.TrimSpace(fields[0]), | ||
| Name: strings.TrimSpace(fields[1]), | ||
| MemoryTotal: memoryMiB, | ||
| DriverVersion: strings.TrimSpace(fields[3]), | ||
| } | ||
|
|
||
| // Validate required fields | ||
| if gpu.UUID == "" { | ||
| return nil, fmt.Errorf("empty UUID") | ||
| } | ||
| if gpu.Name == "" { | ||
| return nil, fmt.Errorf("empty Name") | ||
| } | ||
| if gpu.DriverVersion == "" { | ||
| return nil, fmt.Errorf("empty DriverVersion") | ||
| } | ||
|
|
||
| return gpu, nil | ||
| } | ||
|
|
||
| func (p *preflightChecker) GetGPUInfo(ctx context.Context) ([]GPUInfo, error) { | ||
| if err := checkNvidiaSMI(); err != nil { | ||
| return nil, fmt.Errorf("nvidia-smi not available: %w", err) | ||
| } | ||
|
|
||
| cmd := exec.CommandContext(ctx, "nvidia-smi", | ||
| "--query-gpu=gpu_uuid,gpu_name,memory.total,driver_version", | ||
| "--format=csv,noheader") | ||
| output, err := cmd.CombinedOutput() | ||
| if err != nil { | ||
| log.Error().Str("output", string(output)).Err(err).Msg("nvidia-smi command failed") | ||
| return nil, fmt.Errorf("error running nvidia-smi: %w", err) | ||
| } | ||
|
|
||
| records := strings.Split(strings.TrimSpace(string(output)), "\n") | ||
| gpus := make([]GPUInfo, 0, len(records)) | ||
|
|
||
| for i, record := range records { | ||
| gpu, err := parseGPURecord(record) | ||
| if err != nil { | ||
| log.Warn().Err(err).Int("index", i).Msg("Failed to parse GPU record") | ||
| continue | ||
| } | ||
|
|
||
| gpus = append(gpus, *gpu) | ||
| log.Info(). | ||
| Str("name", gpu.Name). | ||
| Str("uuid", gpu.UUID). | ||
| Int64("memory_mb", gpu.MemoryTotal). | ||
| Msgf("🎮 GPU %d details", len(gpus)) | ||
| } | ||
|
|
||
| if len(gpus) == 0 { | ||
| return nil, fmt.Errorf("no valid GPUs found in nvidia-smi output") | ||
| } | ||
|
|
||
| return gpus, nil | ||
| } | ||
|
|
||
| func (p *preflightChecker) CheckGPU(ctx context.Context, config *GPUCheckConfig) CheckResult { | ||
| if !config.Required { | ||
| // Attempt to retrieve GPU info | ||
| gpus, err := p.GetGPUInfo(ctx) | ||
| if err != nil { | ||
| log.Warn().Msg("⚠️ Running without GPU support - Resource Provider will operate in CPU-only mode") | ||
| return CheckResult{ | ||
| Passed: true, | ||
| Message: "Operating in CPU-only mode", | ||
| } | ||
| } | ||
|
|
||
| // If we found GPUs, log them but still continue | ||
| log.Info().Msgf("🎮 Found %d optional GPUs available for use", len(gpus)) | ||
| return CheckResult{ | ||
| Passed: true, | ||
| Message: fmt.Sprintf("Found %d NVIDIA GPUs (optional)", len(gpus)), | ||
| } | ||
| } | ||
|
|
||
| // Required GPU checks | ||
| log.Info().Msg("Starting required GPU checks") | ||
| gpus, err := p.GetGPUInfo(ctx) | ||
| if err != nil { | ||
| return CheckResult{ | ||
| Passed: false, | ||
| Error: err, | ||
| Message: "Required GPU check failed - no NVIDIA GPUs detected", | ||
| } | ||
| } | ||
|
|
||
| log.Info().Msg("✅ GPU requirements satisfied") | ||
| return CheckResult{ | ||
| Passed: true, | ||
| Message: fmt.Sprintf("Found %d suitable GPUs", len(gpus)), | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| package preflight | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
|
|
||
| "github.com/rs/zerolog/log" | ||
| ) | ||
|
|
||
| const RequiredGPUMemoryGB = 1 // 1GB of VRAM is required to startup if GPU is enabled | ||
|
|
||
| type GPUInfo struct { | ||
| UUID string | ||
| Name string | ||
| MemoryTotal int64 | ||
| DriverVersion string | ||
| } | ||
|
|
||
| type CheckResult struct { | ||
| Passed bool | ||
| Message string | ||
| Error error | ||
| } | ||
|
|
||
| type preflightConfig struct { | ||
| GPU struct { | ||
| MinMemoryGB int64 | ||
| } | ||
| Docker struct { | ||
| CheckRuntime bool | ||
| } | ||
| } | ||
|
|
||
| type preflightChecker struct { | ||
| gpuInfo []GPUInfo | ||
| } | ||
|
|
||
| func RunPreflightChecks() error { | ||
| ctx := context.Background() | ||
| log.Info().Msg("Starting preflight checks...") | ||
| checker := &preflightChecker{} | ||
| config := preflightConfig{ | ||
| GPU: struct { | ||
| MinMemoryGB int64 | ||
| }{ | ||
| MinMemoryGB: RequiredGPUMemoryGB, | ||
| }, | ||
| } | ||
|
|
||
| // Logging GPU requirements | ||
| gpuInfo, err := checker.GetGPUInfo(ctx) | ||
| if err != nil { | ||
| log.Warn().Err(err).Msg("⚠️ No GPU detected - will operate in CPU-only mode") | ||
| } else { | ||
| log.Info(). | ||
| Int("gpu_count", len(gpuInfo)). | ||
| Int64("min_memory_gb", config.GPU.MinMemoryGB). | ||
| Msg("🎮 GPU requirements") | ||
| } | ||
|
|
||
| err = checker.RunAllChecks(ctx, config) | ||
| if err != nil { | ||
| log.Error().Err(err).Msg("❌ Preflight checks failed") | ||
| return err | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func (p *preflightChecker) RunAllChecks(ctx context.Context, config preflightConfig) error { | ||
|
|
||
| gpuResult := p.CheckGPU(ctx, &GPUCheckConfig{ | ||
| MinMemory: config.GPU.MinMemoryGB * 1024 * 1024 * 1024, | ||
| }) | ||
| if !gpuResult.Passed { | ||
| return fmt.Errorf("GPU check failed: %s", gpuResult.Message) | ||
| } | ||
|
|
||
| if config.Docker.CheckRuntime { | ||
| runtimeResult := p.CheckDockerRuntime(ctx) | ||
| if !runtimeResult.Passed { | ||
| return fmt.Errorf("Docker runtime check failed: %s", runtimeResult.Message) | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Are we using this?