diff --git a/api/v1alpha1/bmc_types.go b/api/v1alpha1/bmc_types.go index 5c2d04d60..d6b7d5a5c 100644 --- a/api/v1alpha1/bmc_types.go +++ b/api/v1alpha1/bmc_types.go @@ -206,6 +206,10 @@ type BMCStatus struct { // +optional LastResetTime *metav1.Time `json:"lastResetTime,omitempty"` + // Tasks tracks ongoing and recent BMC operations. + // +optional + Tasks []BMCTask `json:"tasks,omitempty"` + // Conditions represents the latest available observations of the BMC's current state. // +patchStrategy=merge // +patchMergeKey=type @@ -227,6 +231,67 @@ const ( BMCStatePending BMCState = "Pending" ) +// BMCTask represents a single BMC operation task. +type BMCTask struct { + // TaskURI is the URI to monitor the task on the BMC. + // +required + TaskURI string `json:"taskURI"` + + // TaskType indicates the type of operation. + // +required + // +kubebuilder:validation:Enum=DiskErase;BIOSReset;BMCReset;NetworkClear;FirmwareUpdate;ConfigurationChange;AccountManagement;Other + TaskType BMCTaskType `json:"taskType"` + + // TargetID identifies what the task is operating on (e.g., "BIOS", "BMC", "Drive-1"). + // +optional + TargetID string `json:"targetID,omitempty"` + + // State is the current state of the task. + // +optional + State string `json:"state,omitempty"` + + // PercentComplete indicates completion percentage (0-100). + // +optional + PercentComplete int32 `json:"percentComplete,omitempty"` + + // Message provides additional information about the task. + // +optional + Message string `json:"message,omitempty"` + + // LastUpdateTime is when this task status was last updated. + // +optional + LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"` +} + +// BMCTaskType defines the type of BMC task. +type BMCTaskType string + +const ( + // BMCTaskTypeDiskErase indicates a disk erasing task. + BMCTaskTypeDiskErase BMCTaskType = "DiskErase" + + // BMCTaskTypeBIOSReset indicates a BIOS reset task. + BMCTaskTypeBIOSReset BMCTaskType = "BIOSReset" + + // BMCTaskTypeBMCReset indicates a BMC reset task. + BMCTaskTypeBMCReset BMCTaskType = "BMCReset" + + // BMCTaskTypeNetworkClear indicates a network configuration clear task. + BMCTaskTypeNetworkClear BMCTaskType = "NetworkClear" + + // BMCTaskTypeFirmwareUpdate indicates a firmware update task (BIOS or BMC). + BMCTaskTypeFirmwareUpdate BMCTaskType = "FirmwareUpdate" + + // BMCTaskTypeConfigurationChange indicates a configuration change task. + BMCTaskTypeConfigurationChange BMCTaskType = "ConfigurationChange" + + // BMCTaskTypeAccountManagement indicates an account management task. + BMCTaskTypeAccountManagement BMCTaskType = "AccountManagement" + + // BMCTaskTypeOther indicates a task type not covered by the specific types. + BMCTaskTypeOther BMCTaskType = "Other" +) + // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Cluster diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 8b96f5e64..753947691 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -927,6 +927,13 @@ func (in *BMCStatus) DeepCopyInto(out *BMCStatus) { in, out := &in.LastResetTime, &out.LastResetTime *out = (*in).DeepCopy() } + if in.Tasks != nil { + in, out := &in.Tasks, &out.Tasks + *out = make([]BMCTask, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]metav1.Condition, len(*in)) @@ -946,6 +953,22 @@ func (in *BMCStatus) DeepCopy() *BMCStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BMCTask) DeepCopyInto(out *BMCTask) { + *out = *in + in.LastUpdateTime.DeepCopyInto(&out.LastUpdateTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BMCTask. +func (in *BMCTask) DeepCopy() *BMCTask { + if in == nil { + return nil + } + out := new(BMCTask) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BMCUser) DeepCopyInto(out *BMCUser) { *out = *in diff --git a/bmc/bmc.go b/bmc/bmc.go index d9f6634b7..87cdb1594 100644 --- a/bmc/bmc.go +++ b/bmc/bmc.go @@ -109,6 +109,9 @@ type BMC interface { // GetBMCUpgradeTask retrieves the task for the BMC upgrade. GetBMCUpgradeTask(ctx context.Context, manufacturer string, taskURI string) (*schemas.Task, error) + // GetTaskStatus retrieves the status of a task by its URI. + GetTaskStatus(ctx context.Context, taskURI string) (*schemas.Task, error) + // CreateOrUpdateAccount creates or updates a BMC user account. CreateOrUpdateAccount(ctx context.Context, userName, role, password string, enabled bool) error diff --git a/bmc/redfish.go b/bmc/redfish.go index 08d3d195e..ceefe9950 100644 --- a/bmc/redfish.go +++ b/bmc/redfish.go @@ -12,6 +12,7 @@ import ( "io" "maps" "math/big" + "net/http" "slices" "strings" "time" @@ -873,6 +874,28 @@ func (r *RedfishBaseBMC) GetBMCUpgradeTask(_ context.Context, _ string, _ string return nil, fmt.Errorf("firmware upgrade task not supported for manufacturer %q", r.manufacturer) } +// GetTaskStatus retrieves the status of a task by its URI. +func (r *RedfishBaseBMC) GetTaskStatus(ctx context.Context, taskURI string) (*schemas.Task, error) { + client := r.client.GetService().GetClient() + + resp, err := client.Get(taskURI) + if err != nil { + return nil, fmt.Errorf("failed to get task status: %w", err) + } + defer resp.Body.Close() // nolint: errcheck + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code %d when getting task status", resp.StatusCode) + } + + var task schemas.Task + if err := json.NewDecoder(resp.Body).Decode(&task); err != nil { + return nil, fmt.Errorf("failed to decode task response: %w", err) + } + + return &task, nil +} + const ( charLower = "abcdefghijklmnopqrstuvwxyz" charUpper = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" diff --git a/bmc/redfish_kube.go b/bmc/redfish_kube.go index 7e31cbda0..6db51e833 100644 --- a/bmc/redfish_kube.go +++ b/bmc/redfish_kube.go @@ -367,6 +367,12 @@ func (r *RedfishKubeBMC) GetBMCUpgradeTask(ctx context.Context, manufacturer, ta return task, nil } +// GetTaskStatus retrieves the status of a task by its URI. +func (r *RedfishKubeBMC) GetTaskStatus(ctx context.Context, taskURI string) (*schemas.Task, error) { + // Delegate to the underlying RedfishBaseBMC implementation + return r.RedfishBaseBMC.GetTaskStatus(ctx, taskURI) +} + // SetPXEBootOnce sets the boot device for the next system boot using Redfish. func (r *RedfishKubeBMC) SetPXEBootOnce(ctx context.Context, systemURI string) error { system, err := r.getSystemFromUri(ctx, systemURI) diff --git a/cmd/main.go b/cmd/main.go index 5610fd63f..2343e3952 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -93,6 +93,7 @@ func main() { // nolint: gocyclo serverMaxConcurrentReconciles int serverClaimMaxConcurrentReconciles int dnsRecordTemplatePath string + taskPollInterval time.Duration ) flag.IntVar(&serverMaxConcurrentReconciles, "server-max-concurrent-reconciles", 5, @@ -153,6 +154,8 @@ func main() { // nolint: gocyclo "Timeout for BIOS Settings Controller") flag.StringVar(&dnsRecordTemplatePath, "dns-record-template-path", "", "Path to the DNS record template file used for creating DNS records for Servers.") + flag.DurationVar(&taskPollInterval, "task-poll-interval", 30*time.Second, + "Interval for polling BMC task status.") opts := zap.Options{ Development: true, @@ -527,6 +530,18 @@ func main() { // nolint: gocyclo setupLog.Error(err, "Failed to create controller", "controller", "BMCUser") os.Exit(1) } + if err = (&controller.BMCTaskReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Insecure: insecure, + PollInterval: taskPollInterval, + BMCOptions: bmc.Options{ + BasicAuth: true, + }, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "Failed to create controller", "controller", "BMCTask") + os.Exit(1) + } // nolint:goconst if os.Getenv("ENABLE_WEBHOOKS") != "false" { diff --git a/config/crd/bases/metal.ironcore.dev_bmcs.yaml b/config/crd/bases/metal.ironcore.dev_bmcs.yaml index efdefb8fa..2404d3fcb 100644 --- a/config/crd/bases/metal.ironcore.dev_bmcs.yaml +++ b/config/crd/bases/metal.ironcore.dev_bmcs.yaml @@ -284,6 +284,52 @@ spec: State represents the current state of the BMC. kubebuilder:validation:Enum=Enabled;Error;Pending type: string + tasks: + description: Tasks tracks ongoing and recent BMC operations. + items: + description: BMCTask represents a single BMC operation task. + properties: + lastUpdateTime: + description: LastUpdateTime is when this task status was last + updated. + format: date-time + type: string + message: + description: Message provides additional information about the + task. + type: string + percentComplete: + description: PercentComplete indicates completion percentage + (0-100). + format: int32 + type: integer + state: + description: State is the current state of the task. + type: string + targetID: + description: TargetID identifies what the task is operating + on (e.g., "BIOS", "BMC", "Drive-1"). + type: string + taskType: + description: TaskType indicates the type of operation. + enum: + - DiskErase + - BIOSReset + - BMCReset + - NetworkClear + - FirmwareUpdate + - ConfigurationChange + - AccountManagement + - Other + type: string + taskURI: + description: TaskURI is the URI to monitor the task on the BMC. + type: string + required: + - taskType + - taskURI + type: object + type: array type: object type: object served: true diff --git a/docs/api-reference/api.md b/docs/api-reference/api.md index 0a09802c8..04ed11e81 100644 --- a/docs/api-reference/api.md +++ b/docs/api-reference/api.md @@ -670,9 +670,55 @@ _Appears in:_ | `state` _[BMCState](#bmcstate)_ | State represents the current state of the BMC.
kubebuilder:validation:Enum=Enabled;Error;Pending | Pending | | | `powerState` _[BMCPowerState](#bmcpowerstate)_ | PowerState represents the current power state of the BMC. | | | | `lastResetTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#time-v1-meta)_ | LastResetTime is the timestamp of the last reset operation performed on the BMC. | | | +| `tasks` _[BMCTask](#bmctask) array_ | Tasks tracks ongoing and recent BMC operations. | | | | `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#condition-v1-meta) array_ | Conditions represents the latest available observations of the BMC's current state. | | | +#### BMCTask + + + +BMCTask represents a single BMC operation task. + + + +_Appears in:_ +- [BMCStatus](#bmcstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `taskURI` _string_ | TaskURI is the URI to monitor the task on the BMC. | | | +| `taskType` _[BMCTaskType](#bmctasktype)_ | TaskType indicates the type of operation. | | Enum: [DiskErase BIOSReset BMCReset NetworkClear FirmwareUpdate ConfigurationChange AccountManagement Other]
| +| `targetID` _string_ | TargetID identifies what the task is operating on (e.g., "BIOS", "BMC", "Drive-1"). | | | +| `state` _string_ | State is the current state of the task. | | | +| `percentComplete` _integer_ | PercentComplete indicates completion percentage (0-100). | | | +| `message` _string_ | Message provides additional information about the task. | | | +| `lastUpdateTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#time-v1-meta)_ | LastUpdateTime is when this task status was last updated. | | | + + +#### BMCTaskType + +_Underlying type:_ _string_ + +BMCTaskType defines the type of BMC task. + + + +_Appears in:_ +- [BMCTask](#bmctask) + +| Field | Description | +| --- | --- | +| `DiskErase` | BMCTaskTypeDiskErase indicates a disk erasing task.
| +| `BIOSReset` | BMCTaskTypeBIOSReset indicates a BIOS reset task.
| +| `BMCReset` | BMCTaskTypeBMCReset indicates a BMC reset task.
| +| `NetworkClear` | BMCTaskTypeNetworkClear indicates a network configuration clear task.
| +| `FirmwareUpdate` | BMCTaskTypeFirmwareUpdate indicates a firmware update task (BIOS or BMC).
| +| `ConfigurationChange` | BMCTaskTypeConfigurationChange indicates a configuration change task.
| +| `AccountManagement` | BMCTaskTypeAccountManagement indicates an account management task.
| +| `Other` | BMCTaskTypeOther indicates a task type not covered by the specific types.
| + + #### BMCUser diff --git a/docs/bmc-task-tracking.md b/docs/bmc-task-tracking.md new file mode 100644 index 000000000..7f0930c79 --- /dev/null +++ b/docs/bmc-task-tracking.md @@ -0,0 +1,553 @@ +# BMC Task Tracking + +## Overview + +All BMC operations are tracked centrally in `BMC.Status.Tasks[]`. This provides a single source of truth for all BMC operations across multiple controllers. + +## Architecture + +### Dedicated Task Controller (New in v0.x.x) - Initial Rollout for ServerCleaning + +The **BMCTask controller** is a dedicated controller responsible for monitoring BMC task progress. This separation of concerns provides: + +- ✅ **Consistent polling** - All tasks polled at configurable intervals (default 30s) +- ✅ **Automatic monitoring** - Tasks update even when parent resources don't change +- ✅ **Better performance** - No task polling overhead on cleaning operations +- ✅ **Simplified controllers** - Controllers only create tasks, don't poll + +**Current Implementation Status:** +- ✅ **ServerCleaning Controller** - Uses BMCTask controller for task monitoring +- 🔄 **Other Controllers** - Still use their own polling mechanisms (future enhancement) + +``` +┌─────────────────────────────────────────────────────────────┐ +│ BMC Resource │ +│ ┌────────────────────────────────────────────────────────┐ │ +│ │ Status: │ │ +│ │ Tasks: []BMCTask ← Single source of truth │ │ +│ │ - TaskURI, Type, State, Progress, Message │ │ +│ └────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ + ▲ ▲ + │ Creates tasks │ Polls & updates + │ │ + ┌────┴─────┐ ┌──────┴────────┐ + │SrvClean │ ◄─────watches───────│ BMCTask │ + │ │ task updates │ Controller │ + │ │ │ │ + └──────────┘ │ • Watches BMC │ + │ • Polls tasks │ + │ • Updates │ + │ progress │ + │ • Requeues │ + └───────────────┘ +``` + +### Controller Responsibilities + +**BMCTask Controller (Dedicated Task Monitor):** +- Watches BMC resources that have tasks +- Polls BMC API for task status every 30s (configurable via `--task-poll-interval`) +- Updates `BMC.Status.Tasks` with latest State, PercentComplete, Message +- Automatically requeues when active tasks exist +- Stops polling when all tasks reach terminal states +- **Currently used by**: ServerCleaning controller + +**Controllers Using BMCTask Controller:** +- **ServerCleaning Controller**: Creates tasks for cleaning operations, watches BMC for updates + +**Controllers Using Own Polling (Future Migration):** +- **BMC Controller**: Still polls tasks during reconciliation (uses `updateBMCTaskStatus()`) +- **BMCVersion Controller**: Still has 2-minute polling via `ResyncInterval` +- **BMCSettings Controller**: Synchronous operations (no polling needed) + +**Interaction Pattern (ServerCleaning):** +1. **Task Creation**: ServerCleaning adds task entry to `BMC.Status.Tasks` with initial state +2. **Automatic Monitoring**: BMCTask controller automatically detects new task and begins polling +3. **Progress Updates**: BMCTask controller updates task status every 30s +4. **Completion Detection**: BMCTask controller stops polling when task reaches terminal state +5. **Watch for Updates**: ServerCleaning controller watches BMC resources and reacts to task status changes + +### Task Structure + +Each `BMCTask` contains: + +```go +type BMCTask struct { + TaskURI string // Unique identifier for the task + TaskType BMCTaskType // Type of operation + TargetID string // What the task operates on (e.g., "BMC", "BIOS", "Drive-1") + State string // Current state (e.g., "New", "Running", "Completed", "Failed") + PercentComplete int32 // Progress (0-100) + Message string // Additional information + LastUpdateTime metav1.Time // When task was last updated +} +``` + +### Task Types + +- **FirmwareUpdate**: BMC/BIOS firmware upgrades +- **ConfigurationChange**: BMC/BIOS attribute changes +- **DiskErase**: Disk wiping operations +- **BMCReset**: BMC reset operations +- **BIOSReset**: BIOS reset to defaults +- **NetworkClear**: Network configuration cleanup +- **AccountManagement**: User account operations +- **Other**: Other operations + +## Task Lifecycle + +### Automatic Task Monitoring (BMCTask Controller) + +The **BMCTask controller** is a dedicated controller that automatically monitors all in-progress tasks: + +**How it works:** +1. **Watches BMC resources** that have non-empty `Status.Tasks` arrays +2. **Runs every 30 seconds** (configurable via `--task-poll-interval` flag) +3. **Iterates through tasks** in `BMC.Status.Tasks` +4. **Skips terminal states**: `Completed`, `Failed`, `Killed`, `Exception`, `Cancelled` +5. **Polls the BMC** via `bmcClient.GetTaskStatus(taskURI)` for active tasks +6. **Updates task status** with latest `State`, `PercentComplete`, `Message`, and `LastUpdateTime` +7. **Persists changes** via `Status().Update()` if any tasks were updated +8. **Automatic requeue**: Continues polling as long as active tasks exist + +**Key Benefits:** +- ✅ **Automatic monitoring** - Tasks update even if BMC resource doesn't change +- ✅ **Consistent frequency** - All tasks polled at same interval regardless of source +- ✅ **No event dependency** - Doesn't rely on BMC reconciliation to trigger updates +- ✅ **Works across restarts** - Tasks persisted in BMC status survive controller restarts +- ✅ **Simplified controllers** - BMCVersion/BMCSettings/ServerCleaning don't need polling logic + +**Terminal States** (tasks that are no longer polled): +- `Completed` - Task finished successfully +- `Failed` - Task encountered an error +- `Killed` - Task was terminated +- `Exception` - Task threw an exception +- `Cancelled` - Task was cancelled + +**Configuration:** +```bash +# Default 30 second polling interval +./manager + +# Custom interval (e.g., 15 seconds) +./manager --task-poll-interval=15s + +# Longer interval for less frequent updates (e.g., 1 minute) +./manager --task-poll-interval=1m +``` + +### 1. Synchronous Operations + +For operations that complete immediately (e.g., BMC settings changes): + +```go +task := metalv1alpha1.BMCTask{ + TaskURI: fmt.Sprintf("config-change-%s-%s", name, time.Now().Format("20060102-150405")), + TaskType: metalv1alpha1.BMCTaskTypeConfigurationChange, + TargetID: "BMC", + State: "Completed", + PercentComplete: 100, + Message: fmt.Sprintf("Applied %d BMC attributes", len(attributes)), + LastUpdateTime: metav1.Now(), +} +``` + +### 2. Asynchronous Operations + +For long-running operations (e.g., firmware updates): + +**Initial Creation:** +```go +task := metalv1alpha1.BMCTask{ + TaskURI: taskMonitorURI, // From BMC client + TaskType: metalv1alpha1.BMCTaskTypeFirmwareUpdate, + TargetID: "BMC", + State: "New", + PercentComplete: 0, + Message: fmt.Sprintf("Upgrading BMC firmware to %s", version), + LastUpdateTime: metav1.Now(), +} +``` + +**Progress Updates:** +```go +// Poll task status from BMC client +taskStatus, err := bmcClient.GetBMCUpgradeTask(ctx, manufacturer, taskURI) + +// Update task in BMC status +updateBMCTask(ctx, bmcName, namespace, taskURI, func(bmcTask *metalv1alpha1.BMCTask) { + bmcTask.State = string(taskStatus.TaskState) + bmcTask.PercentComplete = int32(*taskStatus.PercentComplete) + bmcTask.Message = fmt.Sprintf("Status: %s", taskStatus.TaskStatus) +}) +``` + +## Controller-Specific Implementations + +### BMCTask Controller (Dedicated Task Monitor) + +**Responsibility:** +- Automatic monitoring of all BMC tasks across all controllers + +**Operations:** +- Polls task status from BMC API +- Updates `BMC.Status.Tasks` with progress +- Manages requeue for active tasks + +**Implementation Details:** +```go +// Only reconciles BMCs with tasks (via event filter) +func hasTasksPredicate() predicate.Predicate { + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + bmc := e.Object.(*metalv1alpha1.BMC) + return len(bmc.Status.Tasks) > 0 + }, + UpdateFunc: func(e event.UpdateEvent) bool { + bmc := e.ObjectNew.(*metalv1alpha1.BMC) + return len(bmc.Status.Tasks) > 0 + }, + } +} + +// Polls tasks and updates status +func (r *BMCTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // Fetch BMC, skip if no tasks + // Get BMC client + // Iterate through tasks, poll non-terminal ones + // Update BMC.Status.Tasks if changed + // Requeue if active tasks exist + return ctrl.Result{RequeueAfter: r.PollInterval}, nil +} +``` + +**Configuration:** +- `--task-poll-interval` flag controls polling frequency (default 30s) + +### BMC Controller + +**Operations Tracked:** +- BMC reset operations + +**Helper Functions:** +- `addBMCTask(bmcObj, task)` - Add new task to BMC status +- `updateBMCTask(bmcObj, taskURI, updateFn)` - Update existing task +- `getBMCTask(bmcObj, taskURI)` - Retrieve task by URI + +**Important:** The BMC controller **no longer polls tasks**. It only creates tasks for its operations. The BMCTask controller handles all polling automatically. + +**Example Usage:** +```go +func (r *BMCReconciler) resetBMC(ctx context.Context, bmcObj *metalv1alpha1.BMC) error { + // ... perform reset ... + + task := metalv1alpha1.BMCTask{ + TaskURI: fmt.Sprintf("bmc-reset-%s", time.Now().Format("20060102-150405")), + TaskType: metalv1alpha1.BMCTaskTypeBMCReset, + TargetID: "BMC", + State: "Completed", + PercentComplete: 100, + Message: "BMC reset initiated", + LastUpdateTime: metav1.Now(), + } + r.addBMCTask(bmcObj, task) + + return r.updateBMCState(ctx, bmcObj, metalv1alpha1.BMCStatePending) +} +``` + +### BMCVersion Controller + +**Operations Tracked:** +- Firmware upgrade operations + +**Helper Functions:** +- `addTaskToBMC(ctx, bmcName, namespace, task)` - Add task to referenced BMC + +**Important:** The BMCVersion controller **no longer polls** for task progress. The BMCTask controller automatically monitors all in-progress tasks. The BMCVersion controller only needs to: +1. Create the task when starting a firmware upgrade +2. Watch the BMC resource for task status updates +3. React to task completion/failure + +**Example Usage:** +```go +// When issuing upgrade +taskMonitor, _, err := bmcClient.UpgradeBMCVersion(ctx, manufacturer, params) +if taskMonitor != "" { + r.addTaskToBMC(ctx, bmcVersion.Spec.BMCRef.Name, bmcVersion.Namespace, metalv1alpha1.BMCTask{ + TaskURI: taskMonitor, + TaskType: metalv1alpha1.BMCTaskTypeFirmwareUpdate, + TargetID: "BMC", + State: "New", + PercentComplete: 0, + Message: fmt.Sprintf("Upgrading BMC firmware to %s", bmcVersion.Spec.Version), + LastUpdateTime: metav1.Now(), + }) +} + +// To check progress - read from BMC.Status.Tasks (BMCTask controller updates it automatically) +bmc := &metalv1alpha1.BMC{} +if err := r.Get(ctx, types.NamespacedName{Name: bmcName, Namespace: namespace}, bmc); err != nil { + return err +} +for _, task := range bmc.Status.Tasks { + if task.TaskURI == taskMonitor { + // Task is automatically updated by BMCTask controller + if task.State == "Completed" { + // Firmware upgrade complete + } else if task.State == "Failed" { + // Firmware upgrade failed + } + break + } +} +``` + +### BMCSettings Controller + +**Operations Tracked:** +- BMC attribute configuration changes + +**Helper Functions:** +- `addTaskToBMC(ctx, bmcName, namespace, task)` - Add task to referenced BMC + +**Important:** For synchronous operations (immediate configuration changes), tasks are created with `State: "Completed"`. The BMCTask controller will not poll these since they're already in a terminal state. + +**Example Usage:** +```go +err = bmcClient.SetBMCAttributesImmediately(ctx, BMC.Spec.BMCUUID, attributes) +if err != nil { + return fmt.Errorf("failed to set BMC settings: %w", err) +} + +// Record configuration change (synchronous operation - already completed) +taskURI := fmt.Sprintf("config-change-%s-%s", bmcSetting.Name, time.Now().Format("20060102-150405")) +r.addTaskToBMC(ctx, bmcSetting.Spec.BMCRef.Name, bmcSetting.Namespace, metalv1alpha1.BMCTask{ + TaskURI: taskURI, + TaskType: metalv1alpha1.BMCTaskTypeConfigurationChange, + TargetID: "BMC", + State: "Completed", + PercentComplete: 100, + Message: fmt.Sprintf("Applied %d BMC attributes", len(attributes)), + LastUpdateTime: metav1.Now(), +}) +``` + +### ServerCleaning Controller + +**Operations Tracked:** +- Disk erase operations +- BIOS reset operations +- Network configuration cleanup +- Account management operations + +**Helper Functions:** +- `addTaskToBMC(ctx, bmcName, namespace, task)` - Add task to referenced BMC + +**Important:** The ServerCleaning controller **no longer polls** for task progress. The BMCTask controller automatically monitors all in-progress tasks. The ServerCleaning controller only needs to: +1. Create tasks when starting cleaning operations +2. Watch the BMC resource for task status updates +3. React to task completion/failure to proceed with next cleaning steps + +**Example Usage:** +```go +// Start disk erase operation +taskURI, err := bmcClient.ErasePhysicalDrive(ctx, driveURI) +if err != nil { + return err +} + +// Create task in BMC status +r.addTaskToBMC(ctx, bmcName, namespace, metalv1alpha1.BMCTask{ + TaskURI: taskURI, + TaskType: metalv1alpha1.BMCTaskTypeDiskErase, + TargetID: driveURI, + State: "New", + PercentComplete: 0, + Message: fmt.Sprintf("Erasing drive %s", driveURI), + LastUpdateTime: metav1.Now(), +}) + +// BMCTask controller will automatically poll and update this task +// ServerCleaning controller watches BMC and reacts to task completion +``` + +## Task Cleanup + +Tasks are automatically pruned to prevent unbounded growth: +- Only the **last 10 tasks** are retained per BMC +- Older tasks are automatically removed when new tasks are added +- This happens transparently in `addBMCTask()` helper functions + +## Querying Tasks + +### From CLI + +```bash +# List all tasks for a BMC +kubectl get bmc -o jsonpath='{.status.tasks[*]}' | jq + +# Get specific task type +kubectl get bmc -o jsonpath='{.status.tasks[?(@.taskType=="FirmwareUpdate")]}' | jq + +# Watch task progress +watch 'kubectl get bmc -o jsonpath="{.status.tasks[0]}" | jq' + +# Get tasks with specific state +kubectl get bmc -o jsonpath='{.status.tasks[?(@.state=="Running")]}' | jq +``` + +### From Code + +```go +// Get BMC object +bmc := &metalv1alpha1.BMC{} +err := client.Get(ctx, types.NamespacedName{Name: bmcName}, bmc) + +// List all tasks +for _, task := range bmc.Status.Tasks { + fmt.Printf("Task: %s, Type: %s, State: %s, Progress: %d%%\n", + task.TaskURI, task.TaskType, task.State, task.PercentComplete) +} + +// Find specific task +for _, task := range bmc.Status.Tasks { + if task.TaskURI == targetURI { + fmt.Printf("Found task: %s at %d%% complete\n", task.Message, task.PercentComplete) + break + } +} +``` + +## Benefits + +### Single Source of Truth +- All BMC operations tracked in one place +- Eliminates duplication across controller status fields +- Simplifies operational monitoring + +### Cross-Controller Awareness +- See all operations affecting a BMC regardless of source +- Better understanding of BMC state and activity +- Prevents conflicting operations + +### Operational Transparency +- Complete audit trail of BMC operations +- Task history preserved (last 10 tasks) +- Clear progress indicators for async operations + +### Better Failure Recovery +- Tasks persist in BMC status across controller restarts +- Can resume monitoring of long-running operations +- Clear indication of failed operations + +## Migration Notes + +### Backward Compatibility + +**BMCVersion Controller:** +- Still maintains `Status.UpgradeTask` field (deprecated but updated) +- This allows existing monitoring/tooling to continue working +- Plan to remove in future version once consumers migrate + +**BMCSettings Controller:** +- No previous task tracking existed +- Pure addition of functionality + +**BMC Controller:** +- Tasks field was previously unused +- Now actively populated + +### Architecture Changes (v0.x.x) + +**What Changed:** + +**Before (Old Architecture):** +- BMC controller polled tasks during every reconciliation (event-driven, inconsistent) +- BMCVersion controller had its own 2-minute polling loop +- ServerCleaning controller had its own 30-second polling loop +- Tasks only updated when reconciliation triggered +- Redundant BMC API calls from multiple controllers + +**After (New Architecture):** +- Dedicated BMCTask controller handles ALL task polling +- Consistent 30-second polling interval (configurable) +- Tasks update automatically even without reconciliation events +- Single BMC API call per task per interval +- Other controllers only create tasks and watch for updates + +**Migration Impact:** + +✅ **No API changes** - `BMC.Status.Tasks` structure unchanged +✅ **No configuration changes** - Works with existing BMC resources +✅ **New flag available** - `--task-poll-interval` (default 30s maintains similar behavior) +✅ **Better consistency** - Tasks now update predictably every 30s +✅ **Improved performance** - Eliminates redundant polling overhead + +**Deployment:** + +1. Deploy new controller version with BMCTask controller +2. Verify task polling works as expected +3. Monitor logs for any issues +4. Roll back if needed (old architecture code preserved in git history) + +**Testing:** + +```bash +# Verify BMCTask controller is running +kubectl get pods -n metal-operator-system +kubectl logs -n metal-operator-system deployment/controller-manager | grep BMCTaskReconciler + +# Test task polling +kubectl apply -f test-bmcversion.yaml + +# Watch task progress (should update every 30s) +watch 'kubectl get bmc -o jsonpath="{.status.tasks[0]}" | jq' + +# Verify consistent updates +kubectl get bmc -o jsonpath='{.status.tasks[0].lastUpdateTime}' +# Should update every ~30 seconds for active tasks +``` + +**Rollback Plan:** + +If issues are found: +1. Revert to previous version +2. BMC controller will resume event-driven polling +3. No data loss - tasks persisted in BMC.Status.Tasks +4. Report issue with logs and reproduction steps + +### Migrating Consumers + +If you're consuming BMC operation status: + +**Before:** +```go +// Old way - check specific controller status +bmcVersion := &metalv1alpha1.BMCVersion{} +client.Get(ctx, key, bmcVersion) +progress := bmcVersion.Status.UpgradeTask.PercentComplete +``` + +**After:** +```go +// New way - check BMC tasks +bmc := &metalv1alpha1.BMC{} +client.Get(ctx, key, bmc) +for _, task := range bmc.Status.Tasks { + if task.TaskType == metalv1alpha1.BMCTaskTypeFirmwareUpdate { + progress := task.PercentComplete + break + } +} +``` + +## Future Enhancements + +Potential improvements: +- Task filtering by date range +- Task persistence to external storage for long-term audit +- Webhooks/events when tasks complete +- Task cancellation support +- Task priority/scheduling diff --git a/internal/controller/bmctask_controller.go b/internal/controller/bmctask_controller.go new file mode 100644 index 000000000..a7cb3ab0a --- /dev/null +++ b/internal/controller/bmctask_controller.go @@ -0,0 +1,199 @@ +// SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "time" + + metalv1alpha1 "github.com/ironcore-dev/metal-operator/api/v1alpha1" + "github.com/ironcore-dev/metal-operator/bmc" + "github.com/ironcore-dev/metal-operator/internal/bmcutils" + "github.com/stmcginnis/gofish/schemas" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// BMCTaskReconciler reconciles BMC tasks by polling task status from the BMC. +// This controller is responsible for monitoring all in-progress BMC operations +// and updating task status in BMC.Status.Tasks. +type BMCTaskReconciler struct { + client.Client + Scheme *runtime.Scheme + // Insecure allows insecure connections to the BMC. + Insecure bool + // BMCOptions contains additional options for BMC clients. + BMCOptions bmc.Options + // PollInterval defines how often to poll task status from the BMC. + PollInterval time.Duration +} + +// +kubebuilder:rbac:groups=metal.ironcore.dev,resources=bmcs,verbs=get;list;watch +// +kubebuilder:rbac:groups=metal.ironcore.dev,resources=bmcs/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=metal.ironcore.dev,resources=bmcsecrets,verbs=get;list;watch +// +kubebuilder:rbac:groups=metal.ironcore.dev,resources=endpoints,verbs=get;list;watch + +// Reconcile monitors BMC tasks and updates their status by polling the BMC. +func (r *BMCTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := ctrl.LoggerFrom(ctx) + log.V(1).Info("Reconciling BMC tasks") + + // Check if context is canceled (during shutdown) + select { + case <-ctx.Done(): + return ctrl.Result{}, nil + default: + // Continue with reconciliation + } + + // Fetch the BMC object + bmcObj := &metalv1alpha1.BMC{} + if err := r.Get(ctx, req.NamespacedName, bmcObj); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // Skip reconciliation if the BMC is being deleted + if !bmcObj.DeletionTimestamp.IsZero() { + return ctrl.Result{}, nil + } + + // Skip if there are no tasks to monitor + if len(bmcObj.Status.Tasks) == 0 { + log.V(1).Info("No tasks to monitor") + return ctrl.Result{}, nil + } + + // Check if there are any non-terminal tasks + hasActiveTasks := false + for i := range bmcObj.Status.Tasks { + task := &bmcObj.Status.Tasks[i] + if !isTerminalState(task.State) { + hasActiveTasks = true + break + } + } + + if !hasActiveTasks { + log.V(1).Info("All tasks are in terminal state") + return ctrl.Result{}, nil + } + + // Get BMC client + bmcClient, err := bmcutils.GetBMCClientFromBMC(ctx, r.Client, bmcObj, r.Insecure, r.BMCOptions) + if err != nil { + log.V(1).Info("Failed to get BMC client, will retry", "error", err) + // Don't fail the reconciliation, just requeue + return ctrl.Result{RequeueAfter: r.PollInterval}, nil + } + defer bmcClient.Logout() + + // Poll and update task statuses + needsUpdate := false + for i := range bmcObj.Status.Tasks { + task := &bmcObj.Status.Tasks[i] + + // Skip tasks in terminal states + if isTerminalState(task.State) { + continue + } + + // Poll task status from BMC + taskStatus, err := bmcClient.GetTaskStatus(ctx, task.TaskURI) + if err != nil { + log.V(1).Info("Failed to get task status", "taskURI", task.TaskURI, "error", err) + continue + } + + // Update task if status changed + if taskStatus != nil { + oldState := task.State + oldPercent := task.PercentComplete + + task.State = string(taskStatus.TaskState) + if taskStatus.PercentComplete != nil { + task.PercentComplete = int32(*taskStatus.PercentComplete) + } + if taskStatus.TaskStatus != "" { + task.Message = string(taskStatus.TaskStatus) + } + task.LastUpdateTime = metav1.Now() + + // Log if status changed + if oldState != task.State || oldPercent != task.PercentComplete { + log.V(1).Info("Updated task status", + "taskURI", task.TaskURI, + "taskType", task.TaskType, + "state", task.State, + "percentComplete", task.PercentComplete) + needsUpdate = true + } + } + } + + // Persist changes if any tasks were updated + if needsUpdate { + bmcBase := bmcObj.DeepCopy() + if err := r.Status().Patch(ctx, bmcObj, client.MergeFrom(bmcBase)); err != nil { + log.Error(err, "Failed to update BMC task status") + return ctrl.Result{}, err + } + log.V(1).Info("Successfully updated BMC task status") + } + + // Requeue to continue monitoring active tasks + return ctrl.Result{RequeueAfter: r.PollInterval}, nil +} + +// isTerminalState checks if a task state is terminal (no further updates expected). +func isTerminalState(state string) bool { + return state == "Completed" || + state == "Failed" || + state == string(schemas.CompletedTaskState) || + state == string(schemas.KilledTaskState) || + state == string(schemas.ExceptionTaskState) || + state == string(schemas.CancelledTaskState) +} + +// SetupWithManager sets up the controller with the Manager. +func (r *BMCTaskReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&metalv1alpha1.BMC{}). + WithEventFilter(hasTasksPredicate()). + Complete(r) +} + +// hasTasksPredicate filters BMC events to only reconcile BMCs that have tasks. +func hasTasksPredicate() predicate.Predicate { + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + bmc, ok := e.Object.(*metalv1alpha1.BMC) + if !ok { + return false + } + return len(bmc.Status.Tasks) > 0 + }, + UpdateFunc: func(e event.UpdateEvent) bool { + bmcNew, ok := e.ObjectNew.(*metalv1alpha1.BMC) + if !ok { + return false + } + return len(bmcNew.Status.Tasks) > 0 + }, + DeleteFunc: func(e event.DeleteEvent) bool { + // Don't reconcile on delete + return false + }, + GenericFunc: func(e event.GenericEvent) bool { + bmc, ok := e.Object.(*metalv1alpha1.BMC) + if !ok { + return false + } + return len(bmc.Status.Tasks) > 0 + }, + } +} diff --git a/internal/controller/bmctask_controller_test.go b/internal/controller/bmctask_controller_test.go new file mode 100644 index 000000000..19f2af7ee --- /dev/null +++ b/internal/controller/bmctask_controller_test.go @@ -0,0 +1,908 @@ +// SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "time" + + metalv1alpha1 "github.com/ironcore-dev/metal-operator/api/v1alpha1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + . "sigs.k8s.io/controller-runtime/pkg/envtest/komega" + "sigs.k8s.io/controller-runtime/pkg/event" +) + +var _ = Describe("BMCTask Controller", func() { + _ = SetupTest(nil) + + AfterEach(func(ctx SpecContext) { + EnsureCleanState() + }) + + It("Should update BMC.Status.Tasks when polling active tasks", func(ctx SpecContext) { + By("Creating a BMCSecret") + bmcSecret := &metalv1alpha1.BMCSecret{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-", + }, + Data: map[string][]byte{ + metalv1alpha1.BMCSecretUsernameKeyName: []byte("foo"), + metalv1alpha1.BMCSecretPasswordKeyName: []byte("bar"), + }, + } + Expect(k8sClient.Create(ctx, bmcSecret)).To(Succeed()) + + By("Creating a BMC resource with active tasks") + bmc := &metalv1alpha1.BMC{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-bmc-", + }, + Spec: metalv1alpha1.BMCSpec{ + Endpoint: &metalv1alpha1.InlineEndpoint{ + IP: metalv1alpha1.MustParseIP(MockServerIP), + MACAddress: "aa:bb:cc:dd:ee:ff", + }, + Protocol: metalv1alpha1.Protocol{ + Name: metalv1alpha1.ProtocolRedfishLocal, + Port: MockServerPort, + }, + BMCSecretRef: v1.LocalObjectReference{ + Name: bmcSecret.Name, + }, + }, + Status: metalv1alpha1.BMCStatus{ + Tasks: []metalv1alpha1.BMCTask{ + { + TaskURI: "/redfish/v1/TaskService/Tasks/1", + TaskType: metalv1alpha1.BMCTaskTypeDiskErase, + TargetID: "Drive-1", + State: "Running", + PercentComplete: 0, + Message: "Erasing disk", + LastUpdateTime: metav1.Now(), + }, + }, + }, + } + Expect(k8sClient.Create(ctx, bmc)).To(Succeed()) + Expect(k8sClient.Status().Update(ctx, bmc)).To(Succeed()) + + By("Ensuring that the task status is updated by the controller") + // The mock BMC will return Completed status + Eventually(Object(bmc)).Should(SatisfyAll( + HaveField("Status.Tasks", HaveLen(1)), + HaveField("Status.Tasks[0].State", "Completed"), + HaveField("Status.Tasks[0].PercentComplete", BeNumerically(">=", 0)), + )) + + // cleanup + Expect(k8sClient.Delete(ctx, bmc)).To(Succeed()) + Expect(k8sClient.Delete(ctx, bmcSecret)).To(Succeed()) + }) + + It("Should only reconcile BMCs with tasks due to event filter", func(ctx SpecContext) { + By("Creating a BMCSecret") + bmcSecret := &metalv1alpha1.BMCSecret{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-", + }, + Data: map[string][]byte{ + metalv1alpha1.BMCSecretUsernameKeyName: []byte("foo"), + metalv1alpha1.BMCSecretPasswordKeyName: []byte("bar"), + }, + } + Expect(k8sClient.Create(ctx, bmcSecret)).To(Succeed()) + + By("Creating a BMC resource without tasks") + bmcWithoutTasks := &metalv1alpha1.BMC{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-bmc-notasks-", + }, + Spec: metalv1alpha1.BMCSpec{ + Endpoint: &metalv1alpha1.InlineEndpoint{ + IP: metalv1alpha1.MustParseIP(MockServerIP), + MACAddress: "aa:bb:cc:dd:ee:11", + }, + Protocol: metalv1alpha1.Protocol{ + Name: metalv1alpha1.ProtocolRedfishLocal, + Port: MockServerPort, + }, + BMCSecretRef: v1.LocalObjectReference{ + Name: bmcSecret.Name, + }, + }, + } + Expect(k8sClient.Create(ctx, bmcWithoutTasks)).To(Succeed()) + + By("Ensuring BMC without tasks remains unchanged") + Consistently(Object(bmcWithoutTasks)).Should(HaveField("Status.Tasks", BeEmpty())) + + By("Creating a BMC resource with tasks") + bmcWithTasks := &metalv1alpha1.BMC{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-bmc-withtasks-", + }, + Spec: metalv1alpha1.BMCSpec{ + Endpoint: &metalv1alpha1.InlineEndpoint{ + IP: metalv1alpha1.MustParseIP(MockServerIP), + MACAddress: "aa:bb:cc:dd:ee:22", + }, + Protocol: metalv1alpha1.Protocol{ + Name: metalv1alpha1.ProtocolRedfishLocal, + Port: MockServerPort, + }, + BMCSecretRef: v1.LocalObjectReference{ + Name: bmcSecret.Name, + }, + }, + Status: metalv1alpha1.BMCStatus{ + Tasks: []metalv1alpha1.BMCTask{ + { + TaskURI: "/redfish/v1/TaskService/Tasks/1", + TaskType: metalv1alpha1.BMCTaskTypeDiskErase, + State: "Running", + PercentComplete: 0, + LastUpdateTime: metav1.Now(), + }, + }, + }, + } + Expect(k8sClient.Create(ctx, bmcWithTasks)).To(Succeed()) + Expect(k8sClient.Status().Update(ctx, bmcWithTasks)).To(Succeed()) + + By("Ensuring BMC with tasks is reconciled") + Eventually(Object(bmcWithTasks)).Should(SatisfyAll( + HaveField("Status.Tasks", HaveLen(1)), + HaveField("Status.Tasks[0].State", "Completed"), + )) + + // cleanup + Expect(k8sClient.Delete(ctx, bmcWithoutTasks)).To(Succeed()) + Expect(k8sClient.Delete(ctx, bmcWithTasks)).To(Succeed()) + Expect(k8sClient.Delete(ctx, bmcSecret)).To(Succeed()) + }) + + It("Should automatically requeue when active tasks exist", func(ctx SpecContext) { + By("Creating a BMCSecret") + bmcSecret := &metalv1alpha1.BMCSecret{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-", + }, + Data: map[string][]byte{ + metalv1alpha1.BMCSecretUsernameKeyName: []byte("foo"), + metalv1alpha1.BMCSecretPasswordKeyName: []byte("bar"), + }, + } + Expect(k8sClient.Create(ctx, bmcSecret)).To(Succeed()) + + By("Creating a BMC resource with an active task") + bmc := &metalv1alpha1.BMC{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-bmc-", + }, + Spec: metalv1alpha1.BMCSpec{ + Endpoint: &metalv1alpha1.InlineEndpoint{ + IP: metalv1alpha1.MustParseIP(MockServerIP), + MACAddress: "aa:bb:cc:dd:ee:33", + }, + Protocol: metalv1alpha1.Protocol{ + Name: metalv1alpha1.ProtocolRedfishLocal, + Port: MockServerPort, + }, + BMCSecretRef: v1.LocalObjectReference{ + Name: bmcSecret.Name, + }, + }, + Status: metalv1alpha1.BMCStatus{ + Tasks: []metalv1alpha1.BMCTask{ + { + TaskURI: "/redfish/v1/TaskService/Tasks/active", + TaskType: metalv1alpha1.BMCTaskTypeFirmwareUpdate, + State: "Running", + PercentComplete: 25, + LastUpdateTime: metav1.Now(), + }, + }, + }, + } + Expect(k8sClient.Create(ctx, bmc)).To(Succeed()) + Expect(k8sClient.Status().Update(ctx, bmc)).To(Succeed()) + + By("Ensuring the task is polled multiple times due to requeue") + initialUpdateTime := metav1.Now() + + // Since the mock returns completed, we verify the task was updated + Eventually(Object(bmc)).Should(SatisfyAll( + HaveField("Status.Tasks", HaveLen(1)), + HaveField("Status.Tasks[0].State", "Completed"), + HaveField("Status.Tasks[0].LastUpdateTime", Not(Equal(initialUpdateTime))), + )) + + // cleanup + Expect(k8sClient.Delete(ctx, bmc)).To(Succeed()) + Expect(k8sClient.Delete(ctx, bmcSecret)).To(Succeed()) + }) + + It("Should not requeue when all tasks are in terminal state", func(ctx SpecContext) { + By("Creating a BMCSecret") + bmcSecret := &metalv1alpha1.BMCSecret{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-", + }, + Data: map[string][]byte{ + metalv1alpha1.BMCSecretUsernameKeyName: []byte("foo"), + metalv1alpha1.BMCSecretPasswordKeyName: []byte("bar"), + }, + } + Expect(k8sClient.Create(ctx, bmcSecret)).To(Succeed()) + + By("Creating a BMC resource with only terminal tasks") + bmc := &metalv1alpha1.BMC{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-bmc-", + }, + Spec: metalv1alpha1.BMCSpec{ + Endpoint: &metalv1alpha1.InlineEndpoint{ + IP: metalv1alpha1.MustParseIP(MockServerIP), + MACAddress: "aa:bb:cc:dd:ee:44", + }, + Protocol: metalv1alpha1.Protocol{ + Name: metalv1alpha1.ProtocolRedfishLocal, + Port: MockServerPort, + }, + BMCSecretRef: v1.LocalObjectReference{ + Name: bmcSecret.Name, + }, + }, + Status: metalv1alpha1.BMCStatus{ + Tasks: []metalv1alpha1.BMCTask{ + { + TaskURI: "/redfish/v1/TaskService/Tasks/completed", + TaskType: metalv1alpha1.BMCTaskTypeDiskErase, + State: "Completed", + PercentComplete: 100, + LastUpdateTime: metav1.Now(), + }, + { + TaskURI: "/redfish/v1/TaskService/Tasks/failed", + TaskType: metalv1alpha1.BMCTaskTypeBIOSReset, + State: "Failed", + PercentComplete: 50, + Message: "Operation failed", + LastUpdateTime: metav1.Now(), + }, + }, + }, + } + Expect(k8sClient.Create(ctx, bmc)).To(Succeed()) + Expect(k8sClient.Status().Update(ctx, bmc)).To(Succeed()) + + By("Ensuring terminal tasks are not updated") + // Store the initial last update time + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(bmc), bmc)).To(Succeed()) + initialUpdateTime1 := bmc.Status.Tasks[0].LastUpdateTime + initialUpdateTime2 := bmc.Status.Tasks[1].LastUpdateTime + + // Wait a bit and verify the tasks haven't changed + time.Sleep(200 * time.Millisecond) + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(bmc), bmc)).To(Succeed()) + + Expect(bmc.Status.Tasks).To(HaveLen(2)) + Expect(bmc.Status.Tasks[0].LastUpdateTime).To(Equal(initialUpdateTime1)) + Expect(bmc.Status.Tasks[1].LastUpdateTime).To(Equal(initialUpdateTime2)) + + // cleanup + Expect(k8sClient.Delete(ctx, bmc)).To(Succeed()) + Expect(k8sClient.Delete(ctx, bmcSecret)).To(Succeed()) + }) + + It("Should handle BMC client errors gracefully", func(ctx SpecContext) { + By("Creating a BMCSecret with invalid credentials") + bmcSecret := &metalv1alpha1.BMCSecret{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-", + }, + Data: map[string][]byte{ + metalv1alpha1.BMCSecretUsernameKeyName: []byte("invalid"), + metalv1alpha1.BMCSecretPasswordKeyName: []byte("invalid"), + }, + } + Expect(k8sClient.Create(ctx, bmcSecret)).To(Succeed()) + + By("Creating a BMC resource with active tasks") + bmc := &metalv1alpha1.BMC{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-bmc-", + }, + Spec: metalv1alpha1.BMCSpec{ + Endpoint: &metalv1alpha1.InlineEndpoint{ + IP: metalv1alpha1.MustParseIP("192.0.2.1"), // TEST-NET-1 (unreachable) + MACAddress: "aa:bb:cc:dd:ee:55", + }, + Protocol: metalv1alpha1.Protocol{ + Name: metalv1alpha1.ProtocolRedfish, + Port: 8000, + }, + BMCSecretRef: v1.LocalObjectReference{ + Name: bmcSecret.Name, + }, + }, + Status: metalv1alpha1.BMCStatus{ + Tasks: []metalv1alpha1.BMCTask{ + { + TaskURI: "/redfish/v1/TaskService/Tasks/1", + TaskType: metalv1alpha1.BMCTaskTypeDiskErase, + State: "Running", + PercentComplete: 0, + LastUpdateTime: metav1.Now(), + }, + }, + }, + } + Expect(k8sClient.Create(ctx, bmc)).To(Succeed()) + Expect(k8sClient.Status().Update(ctx, bmc)).To(Succeed()) + + By("Ensuring the controller handles the error gracefully") + // The controller should not crash and should keep retrying + Consistently(Object(bmc), "2s", "100ms").Should(SatisfyAll( + HaveField("Status.Tasks", HaveLen(1)), + HaveField("Status.Tasks[0].State", "Running"), + )) + + // cleanup + Expect(k8sClient.Delete(ctx, bmc)).To(Succeed()) + Expect(k8sClient.Delete(ctx, bmcSecret)).To(Succeed()) + }) + + It("Should only update changed tasks", func(ctx SpecContext) { + By("Creating a BMCSecret") + bmcSecret := &metalv1alpha1.BMCSecret{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-", + }, + Data: map[string][]byte{ + metalv1alpha1.BMCSecretUsernameKeyName: []byte("foo"), + metalv1alpha1.BMCSecretPasswordKeyName: []byte("bar"), + }, + } + Expect(k8sClient.Create(ctx, bmcSecret)).To(Succeed()) + + By("Creating a BMC resource with mixed terminal and active tasks") + bmc := &metalv1alpha1.BMC{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-bmc-", + }, + Spec: metalv1alpha1.BMCSpec{ + Endpoint: &metalv1alpha1.InlineEndpoint{ + IP: metalv1alpha1.MustParseIP(MockServerIP), + MACAddress: "aa:bb:cc:dd:ee:66", + }, + Protocol: metalv1alpha1.Protocol{ + Name: metalv1alpha1.ProtocolRedfishLocal, + Port: MockServerPort, + }, + BMCSecretRef: v1.LocalObjectReference{ + Name: bmcSecret.Name, + }, + }, + Status: metalv1alpha1.BMCStatus{ + Tasks: []metalv1alpha1.BMCTask{ + { + TaskURI: "/redfish/v1/TaskService/Tasks/completed", + TaskType: metalv1alpha1.BMCTaskTypeDiskErase, + State: "Completed", + PercentComplete: 100, + Message: "Disk erased successfully", + LastUpdateTime: metav1.Now(), + }, + { + TaskURI: "/redfish/v1/TaskService/Tasks/active", + TaskType: metalv1alpha1.BMCTaskTypeBIOSReset, + State: "Running", + PercentComplete: 50, + LastUpdateTime: metav1.Now(), + }, + }, + }, + } + Expect(k8sClient.Create(ctx, bmc)).To(Succeed()) + Expect(k8sClient.Status().Update(ctx, bmc)).To(Succeed()) + + By("Getting the initial state") + Eventually(Get(bmc)).Should(Succeed()) + initialTask1UpdateTime := bmc.Status.Tasks[0].LastUpdateTime + initialTask2UpdateTime := bmc.Status.Tasks[1].LastUpdateTime + + By("Ensuring only active task is updated") + Eventually(Object(bmc)).Should(SatisfyAll( + HaveField("Status.Tasks", HaveLen(2)), + // First task (completed) should remain unchanged + HaveField("Status.Tasks[0].State", "Completed"), + HaveField("Status.Tasks[0].PercentComplete", BeNumerically("==", 100)), + // Second task (active) should be updated by the mock BMC + HaveField("Status.Tasks[1].State", "Completed"), + HaveField("Status.Tasks[1].LastUpdateTime", Not(Equal(initialTask2UpdateTime))), + )) + + // Verify first task was not updated + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(bmc), bmc)).To(Succeed()) + Expect(bmc.Status.Tasks[0].LastUpdateTime).To(Equal(initialTask1UpdateTime)) + + // cleanup + Expect(k8sClient.Delete(ctx, bmc)).To(Succeed()) + Expect(k8sClient.Delete(ctx, bmcSecret)).To(Succeed()) + }) + + It("Should handle multiple tasks with mixed states correctly", func(ctx SpecContext) { + By("Creating a BMCSecret") + bmcSecret := &metalv1alpha1.BMCSecret{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-", + }, + Data: map[string][]byte{ + metalv1alpha1.BMCSecretUsernameKeyName: []byte("foo"), + metalv1alpha1.BMCSecretPasswordKeyName: []byte("bar"), + }, + } + Expect(k8sClient.Create(ctx, bmcSecret)).To(Succeed()) + + By("Creating a BMC resource with multiple tasks in various states") + bmc := &metalv1alpha1.BMC{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-bmc-", + }, + Spec: metalv1alpha1.BMCSpec{ + Endpoint: &metalv1alpha1.InlineEndpoint{ + IP: metalv1alpha1.MustParseIP(MockServerIP), + MACAddress: "aa:bb:cc:dd:ee:77", + }, + Protocol: metalv1alpha1.Protocol{ + Name: metalv1alpha1.ProtocolRedfishLocal, + Port: MockServerPort, + }, + BMCSecretRef: v1.LocalObjectReference{ + Name: bmcSecret.Name, + }, + }, + Status: metalv1alpha1.BMCStatus{ + Tasks: []metalv1alpha1.BMCTask{ + { + TaskURI: "/redfish/v1/TaskService/Tasks/task1", + TaskType: metalv1alpha1.BMCTaskTypeDiskErase, + TargetID: "Drive-1", + State: "Running", + PercentComplete: 10, + Message: "Erasing drive 1", + LastUpdateTime: metav1.Now(), + }, + { + TaskURI: "/redfish/v1/TaskService/Tasks/task2", + TaskType: metalv1alpha1.BMCTaskTypeBMCReset, + State: "Completed", + PercentComplete: 100, + Message: "BMC reset completed", + LastUpdateTime: metav1.Now(), + }, + { + TaskURI: "/redfish/v1/TaskService/Tasks/task3", + TaskType: metalv1alpha1.BMCTaskTypeFirmwareUpdate, + State: "Running", + PercentComplete: 75, + Message: "Updating firmware", + LastUpdateTime: metav1.Now(), + }, + { + TaskURI: "/redfish/v1/TaskService/Tasks/task4", + TaskType: metalv1alpha1.BMCTaskTypeNetworkClear, + State: "Failed", + PercentComplete: 0, + Message: "Network clear failed", + LastUpdateTime: metav1.Now(), + }, + }, + }, + } + Expect(k8sClient.Create(ctx, bmc)).To(Succeed()) + Expect(k8sClient.Status().Update(ctx, bmc)).To(Succeed()) + + By("Ensuring only non-terminal tasks are updated") + Eventually(Object(bmc)).Should(SatisfyAll( + HaveField("Status.Tasks", HaveLen(4)), + // Task 1: was Running, should be updated to Completed by mock + HaveField("Status.Tasks[0].State", "Completed"), + // Task 2: was Completed, should remain Completed + HaveField("Status.Tasks[1].State", "Completed"), + HaveField("Status.Tasks[1].PercentComplete", BeNumerically("==", 100)), + // Task 3: was Running, should be updated to Completed by mock + HaveField("Status.Tasks[2].State", "Completed"), + // Task 4: was Failed, should remain Failed + HaveField("Status.Tasks[3].State", "Failed"), + HaveField("Status.Tasks[3].Message", "Network clear failed"), + )) + + // cleanup + Expect(k8sClient.Delete(ctx, bmc)).To(Succeed()) + Expect(k8sClient.Delete(ctx, bmcSecret)).To(Succeed()) + }) + + It("Should skip reconciliation if BMC is being deleted", func(ctx SpecContext) { + By("Creating a BMCSecret") + bmcSecret := &metalv1alpha1.BMCSecret{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-", + }, + Data: map[string][]byte{ + metalv1alpha1.BMCSecretUsernameKeyName: []byte("foo"), + metalv1alpha1.BMCSecretPasswordKeyName: []byte("bar"), + }, + } + Expect(k8sClient.Create(ctx, bmcSecret)).To(Succeed()) + + By("Creating a BMC resource with tasks and a finalizer") + bmc := &metalv1alpha1.BMC{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-bmc-", + Finalizers: []string{"test.finalizer"}, + }, + Spec: metalv1alpha1.BMCSpec{ + Endpoint: &metalv1alpha1.InlineEndpoint{ + IP: metalv1alpha1.MustParseIP(MockServerIP), + MACAddress: "aa:bb:cc:dd:ee:88", + }, + Protocol: metalv1alpha1.Protocol{ + Name: metalv1alpha1.ProtocolRedfishLocal, + Port: MockServerPort, + }, + BMCSecretRef: v1.LocalObjectReference{ + Name: bmcSecret.Name, + }, + }, + Status: metalv1alpha1.BMCStatus{ + Tasks: []metalv1alpha1.BMCTask{ + { + TaskURI: "/redfish/v1/TaskService/Tasks/1", + TaskType: metalv1alpha1.BMCTaskTypeDiskErase, + State: "Running", + PercentComplete: 0, + LastUpdateTime: metav1.Now(), + }, + }, + }, + } + Expect(k8sClient.Create(ctx, bmc)).To(Succeed()) + Expect(k8sClient.Status().Update(ctx, bmc)).To(Succeed()) + + By("Deleting the BMC") + Expect(k8sClient.Delete(ctx, bmc)).To(Succeed()) + + By("Ensuring tasks are not updated during deletion") + Eventually(Get(bmc)).Should(Succeed()) + Expect(bmc.DeletionTimestamp).NotTo(BeNil()) + + // Store the task state when deletion started + initialTaskState := bmc.Status.Tasks[0].State + initialUpdateTime := bmc.Status.Tasks[0].LastUpdateTime + + // Wait a bit and verify the task hasn't been updated + time.Sleep(200 * time.Millisecond) + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(bmc), bmc)).To(Succeed()) + Expect(bmc.Status.Tasks[0].State).To(Equal(initialTaskState)) + Expect(bmc.Status.Tasks[0].LastUpdateTime).To(Equal(initialUpdateTime)) + + By("Removing finalizer to allow deletion") + Eventually(Update(bmc, func() { + bmc.Finalizers = []string{} + })).Should(Succeed()) + + // cleanup + Expect(k8sClient.Delete(ctx, bmcSecret)).To(Succeed()) + }) + + It("Should handle BMCs with empty task list", func(ctx SpecContext) { + By("Creating a BMCSecret") + bmcSecret := &metalv1alpha1.BMCSecret{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-", + }, + Data: map[string][]byte{ + metalv1alpha1.BMCSecretUsernameKeyName: []byte("foo"), + metalv1alpha1.BMCSecretPasswordKeyName: []byte("bar"), + }, + } + Expect(k8sClient.Create(ctx, bmcSecret)).To(Succeed()) + + By("Creating a BMC resource") + bmc := &metalv1alpha1.BMC{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-bmc-", + }, + Spec: metalv1alpha1.BMCSpec{ + Endpoint: &metalv1alpha1.InlineEndpoint{ + IP: metalv1alpha1.MustParseIP(MockServerIP), + MACAddress: "aa:bb:cc:dd:ee:99", + }, + Protocol: metalv1alpha1.Protocol{ + Name: metalv1alpha1.ProtocolRedfishLocal, + Port: MockServerPort, + }, + BMCSecretRef: v1.LocalObjectReference{ + Name: bmcSecret.Name, + }, + }, + Status: metalv1alpha1.BMCStatus{ + Tasks: []metalv1alpha1.BMCTask{}, + }, + } + Expect(k8sClient.Create(ctx, bmc)).To(Succeed()) + Expect(k8sClient.Status().Update(ctx, bmc)).To(Succeed()) + + By("Ensuring the controller doesn't fail with empty task list") + Consistently(Object(bmc), "1s", "100ms").Should(HaveField("Status.Tasks", BeEmpty())) + + // cleanup + Expect(k8sClient.Delete(ctx, bmc)).To(Succeed()) + Expect(k8sClient.Delete(ctx, bmcSecret)).To(Succeed()) + }) + + It("Should register BMCTask controller in the test setup", func(ctx SpecContext) { + By("Verifying the BMCTask controller is registered") + // This test verifies that the controller is properly set up in suite_test.go + // The fact that other tests pass indicates the controller is working + // This is a placeholder to ensure we remember to register it in suite_test.go + + By("Creating a BMCSecret") + bmcSecret := &metalv1alpha1.BMCSecret{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-", + }, + Data: map[string][]byte{ + metalv1alpha1.BMCSecretUsernameKeyName: []byte("foo"), + metalv1alpha1.BMCSecretPasswordKeyName: []byte("bar"), + }, + } + Expect(k8sClient.Create(ctx, bmcSecret)).To(Succeed()) + + By("Creating a BMC with tasks to trigger reconciliation") + bmc := &metalv1alpha1.BMC{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-bmc-controller-", + }, + Spec: metalv1alpha1.BMCSpec{ + Endpoint: &metalv1alpha1.InlineEndpoint{ + IP: metalv1alpha1.MustParseIP(MockServerIP), + MACAddress: "aa:bb:cc:dd:ee:00", + }, + Protocol: metalv1alpha1.Protocol{ + Name: metalv1alpha1.ProtocolRedfishLocal, + Port: MockServerPort, + }, + BMCSecretRef: v1.LocalObjectReference{ + Name: bmcSecret.Name, + }, + }, + Status: metalv1alpha1.BMCStatus{ + Tasks: []metalv1alpha1.BMCTask{ + { + TaskURI: "/redfish/v1/TaskService/Tasks/1", + TaskType: metalv1alpha1.BMCTaskTypeDiskErase, + State: "Running", + PercentComplete: 0, + LastUpdateTime: metav1.Now(), + }, + }, + }, + } + Expect(k8sClient.Create(ctx, bmc)).To(Succeed()) + Expect(k8sClient.Status().Update(ctx, bmc)).To(Succeed()) + + By("Ensuring controller processes the BMC task") + Eventually(Object(bmc), "5s", "100ms").Should(SatisfyAll( + HaveField("Status.Tasks", HaveLen(1)), + HaveField("Status.Tasks[0].State", "Completed"), + )) + + // cleanup + Expect(k8sClient.Delete(ctx, bmc)).To(Succeed()) + Expect(k8sClient.Delete(ctx, bmcSecret)).To(Succeed()) + }) +}) + +var _ = Describe("BMCTask Event Filter", func() { + It("Should filter BMCs without tasks on create event", func() { + predicate := hasTasksPredicate() + + By("Testing with BMC without tasks") + bmcWithoutTasks := &metalv1alpha1.BMC{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-bmc-no-tasks", + }, + Status: metalv1alpha1.BMCStatus{ + Tasks: []metalv1alpha1.BMCTask{}, + }, + } + + // Create event should be filtered (return false) + Expect(predicate.Create(MockCreateEvent(bmcWithoutTasks))).To(BeFalse()) + + By("Testing with BMC with tasks") + bmcWithTasks := &metalv1alpha1.BMC{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-bmc-with-tasks", + }, + Status: metalv1alpha1.BMCStatus{ + Tasks: []metalv1alpha1.BMCTask{ + { + TaskURI: "/redfish/v1/TaskService/Tasks/1", + TaskType: metalv1alpha1.BMCTaskTypeDiskErase, + State: "Running", + }, + }, + }, + } + + // Create event should pass (return true) + Expect(predicate.Create(MockCreateEvent(bmcWithTasks))).To(BeTrue()) + }) + + It("Should filter BMCs without tasks on update event", func() { + predicate := hasTasksPredicate() + + By("Testing update with old BMC having tasks, new BMC without tasks") + oldBMC := &metalv1alpha1.BMC{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-bmc", + }, + Status: metalv1alpha1.BMCStatus{ + Tasks: []metalv1alpha1.BMCTask{ + { + TaskURI: "/redfish/v1/TaskService/Tasks/1", + TaskType: metalv1alpha1.BMCTaskTypeDiskErase, + State: "Running", + }, + }, + }, + } + newBMC := &metalv1alpha1.BMC{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-bmc", + }, + Status: metalv1alpha1.BMCStatus{ + Tasks: []metalv1alpha1.BMCTask{}, + }, + } + + // Update event should be filtered when new BMC has no tasks + Expect(predicate.Update(MockUpdateEvent(oldBMC, newBMC))).To(BeFalse()) + + By("Testing update with both BMCs having tasks") + newBMCWithTasks := &metalv1alpha1.BMC{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-bmc", + }, + Status: metalv1alpha1.BMCStatus{ + Tasks: []metalv1alpha1.BMCTask{ + { + TaskURI: "/redfish/v1/TaskService/Tasks/1", + TaskType: metalv1alpha1.BMCTaskTypeDiskErase, + State: "Completed", + }, + }, + }, + } + + // Update event should pass when new BMC has tasks + Expect(predicate.Update(MockUpdateEvent(oldBMC, newBMCWithTasks))).To(BeTrue()) + }) + + It("Should always filter delete events", func() { + predicate := hasTasksPredicate() + + bmc := &metalv1alpha1.BMC{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-bmc", + }, + Status: metalv1alpha1.BMCStatus{ + Tasks: []metalv1alpha1.BMCTask{ + { + TaskURI: "/redfish/v1/TaskService/Tasks/1", + TaskType: metalv1alpha1.BMCTaskTypeDiskErase, + State: "Running", + }, + }, + }, + } + + // Delete events should always be filtered regardless of tasks + Expect(predicate.Delete(MockDeleteEvent(bmc))).To(BeFalse()) + }) + + It("Should filter generic events based on task presence", func() { + predicate := hasTasksPredicate() + + By("Testing generic event without tasks") + bmcWithoutTasks := &metalv1alpha1.BMC{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-bmc", + }, + Status: metalv1alpha1.BMCStatus{ + Tasks: []metalv1alpha1.BMCTask{}, + }, + } + + Expect(predicate.Generic(MockGenericEvent(bmcWithoutTasks))).To(BeFalse()) + + By("Testing generic event with tasks") + bmcWithTasks := &metalv1alpha1.BMC{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-bmc", + }, + Status: metalv1alpha1.BMCStatus{ + Tasks: []metalv1alpha1.BMCTask{ + { + TaskURI: "/redfish/v1/TaskService/Tasks/1", + TaskType: metalv1alpha1.BMCTaskTypeDiskErase, + State: "Running", + }, + }, + }, + } + + Expect(predicate.Generic(MockGenericEvent(bmcWithTasks))).To(BeTrue()) + }) +}) + +var _ = Describe("isTerminalState", func() { + It("Should identify terminal states correctly", func() { + By("Testing completed state") + Expect(isTerminalState("Completed")).To(BeTrue()) + + By("Testing failed state") + Expect(isTerminalState("Failed")).To(BeTrue()) + + By("Testing Redfish terminal states") + Expect(isTerminalState("Killed")).To(BeTrue()) + Expect(isTerminalState("Exception")).To(BeTrue()) + Expect(isTerminalState("Cancelled")).To(BeTrue()) + + By("Testing non-terminal states") + Expect(isTerminalState("Running")).To(BeFalse()) + Expect(isTerminalState("Pending")).To(BeFalse()) + Expect(isTerminalState("Starting")).To(BeFalse()) + Expect(isTerminalState("")).To(BeFalse()) + }) +}) + +// Helper functions for creating mock events for predicate testing + +// MockCreateEvent creates a mock CreateEvent for testing predicates. +func MockCreateEvent(obj client.Object) event.CreateEvent { + return event.CreateEvent{ + Object: obj, + } +} + +// MockUpdateEvent creates a mock UpdateEvent for testing predicates. +func MockUpdateEvent(oldObj, newObj client.Object) event.UpdateEvent { + return event.UpdateEvent{ + ObjectOld: oldObj, + ObjectNew: newObj, + } +} + +// MockDeleteEvent creates a mock DeleteEvent for testing predicates. +func MockDeleteEvent(obj client.Object) event.DeleteEvent { + return event.DeleteEvent{ + Object: obj, + } +} + +// MockGenericEvent creates a mock GenericEvent for testing predicates. +func MockGenericEvent(obj client.Object) event.GenericEvent { + return event.GenericEvent{ + Object: obj, + } +} diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 9db3050f7..e8f74877a 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -110,7 +110,11 @@ func SetupTest(redfishMockServers []netip.AddrPort) *corev1.Namespace { BeforeEach(func(ctx SpecContext) { var mgrCtx context.Context mgrCtx, cancel := context.WithCancel(context.Background()) - DeferCleanup(cancel) + DeferCleanup(func() { + cancel() + // Give in-flight reconciliations time to complete + time.Sleep(200 * time.Millisecond) + }) *ns = corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -311,6 +315,16 @@ func SetupTest(redfishMockServers []netip.AddrPort) *corev1.Namespace { }, }).SetupWithManager(k8sManager)).To(Succeed()) + Expect((&BMCTaskReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + Insecure: true, + PollInterval: 50 * time.Millisecond, + BMCOptions: bmc.Options{ + BasicAuth: true, + }, + }).SetupWithManager(k8sManager)).To(Succeed()) + By("Starting the registry server") Expect(k8sManager.Add(manager.RunnableFunc(func(ctx context.Context) error { registryServer := registry.NewServer(GinkgoLogr, ":30000", k8sManager.GetClient())