diff --git a/backend/api/hack/generator.sh b/backend/api/hack/generator.sh index 21ddea34afb..1fb0ac4d5ee 100755 --- a/backend/api/hack/generator.sh +++ b/backend/api/hack/generator.sh @@ -167,6 +167,15 @@ if ! grep -Eq '^[[:space:]]*Tasks \[\]\*V2PipelineTask `json:"tasks,omitempty"`$ exit 1 fi +# Unset log history must remain compatible with servers predating this field. +sed -i -- 's/LogHistory \[\]\*PipelineTaskLogHistory `json:"log_history"`/LogHistory []*PipelineTaskLogHistory `json:"log_history,omitempty"`/g' \ + backend/api/${API_VERSION}/go_http_client/run_model/${API_VERSION}_pipeline_task.go +if ! grep -Eq '^[[:space:]]*LogHistory \[\]\*PipelineTaskLogHistory `json:"log_history,omitempty"`$' \ + backend/api/${API_VERSION}/go_http_client/run_model/${API_VERSION}_pipeline_task.go; then + echo 'Generated PipelineTask.log_history must have json:"log_history,omitempty"; update the go-swagger compatibility substitution.' >&2 + exit 1 +fi + # grpc-gateway 2.32 renders Any.additionalProperties as true rather than {}. # Preserve the exported Go map field name used by existing HTTP client callers. for model in artifact experiment healthz pipeline recurring_run run; do diff --git a/backend/api/v2/go_client/run.pb.go b/backend/api/v2/go_client/run.pb.go index 752d5422429..bcddebcda06 100644 --- a/backend/api/v2/go_client/run.pb.go +++ b/backend/api/v2/go_client/run.pb.go @@ -1225,8 +1225,11 @@ type PipelineTask struct { // One of image-pull, scheduling, runtime, admission, or unknown. // Empty when lifecycle_message is empty. LifecycleCategory *string `protobuf:"bytes,20,opt,name=lifecycle_category,json=lifecycleCategory,proto3,oneof" json:"lifecycle_category,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Output only. Prior executions archived by manual retries, oldest first. + // History begins when preservation is enabled; older retries are not backfilled. + LogHistory []*PipelineTask_LogHistory `protobuf:"bytes,21,rep,name=log_history,json=logHistory,proto3" json:"log_history,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *PipelineTask) Reset() { @@ -1399,6 +1402,13 @@ func (x *PipelineTask) GetLifecycleCategory() string { return "" } +func (x *PipelineTask) GetLogHistory() []*PipelineTask_LogHistory { + if x != nil { + return x.LogHistory + } + return nil +} + type CreateRunRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // The ID of the parent experiment. @@ -2889,6 +2899,112 @@ func (x *PipelineTask_InputOutputs) GetArtifacts() []*PipelineTask_InputOutputs_ return nil } +// Log references from an execution archived before a manual run retry. +// References can outlive pods or archived objects and do not guarantee availability. +type PipelineTask_LogHistory struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Together with task_id, identifies this snapshot. This is the manual retry + // generation that archived it, not an automatic pod retry attempt number. + BeforeRetryGeneration int64 `protobuf:"varint,1,opt,name=before_retry_generation,json=beforeRetryGeneration,proto3" json:"before_retry_generation,omitempty"` + ArchivedAt *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=archived_at,json=archivedAt,proto3" json:"archived_at,omitempty"` + StartTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"` + EndTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=end_time,json=endTime,proto3" json:"end_time,omitempty"` + State PipelineTask_TaskState `protobuf:"varint,5,opt,name=state,proto3,enum=kubeflow.pipelines.backend.api.v2.PipelineTask_TaskState" json:"state,omitempty"` + Pods []*PipelineTask_TaskPod `protobuf:"bytes,6,rep,name=pods,proto3" json:"pods,omitempty"` + // Log artifacts only; never includes task outputs used by downstream tasks. + LogArtifacts []*PipelineTask_InputOutputs_IOArtifact `protobuf:"bytes,7,rep,name=log_artifacts,json=logArtifacts,proto3" json:"log_artifacts,omitempty"` + // Legacy driver log reference, when not represented by a log artifact. + DriverLogsUri string `protobuf:"bytes,8,opt,name=driver_logs_uri,json=driverLogsUri,proto3" json:"driver_logs_uri,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PipelineTask_LogHistory) Reset() { + *x = PipelineTask_LogHistory{} + mi := &file_backend_api_v2_run_proto_msgTypes[34] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PipelineTask_LogHistory) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PipelineTask_LogHistory) ProtoMessage() {} + +func (x *PipelineTask_LogHistory) ProtoReflect() protoreflect.Message { + mi := &file_backend_api_v2_run_proto_msgTypes[34] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PipelineTask_LogHistory.ProtoReflect.Descriptor instead. +func (*PipelineTask_LogHistory) Descriptor() ([]byte, []int) { + return file_backend_api_v2_run_proto_rawDescGZIP(), []int{6, 6} +} + +func (x *PipelineTask_LogHistory) GetBeforeRetryGeneration() int64 { + if x != nil { + return x.BeforeRetryGeneration + } + return 0 +} + +func (x *PipelineTask_LogHistory) GetArchivedAt() *timestamppb.Timestamp { + if x != nil { + return x.ArchivedAt + } + return nil +} + +func (x *PipelineTask_LogHistory) GetStartTime() *timestamppb.Timestamp { + if x != nil { + return x.StartTime + } + return nil +} + +func (x *PipelineTask_LogHistory) GetEndTime() *timestamppb.Timestamp { + if x != nil { + return x.EndTime + } + return nil +} + +func (x *PipelineTask_LogHistory) GetState() PipelineTask_TaskState { + if x != nil { + return x.State + } + return PipelineTask_RUNTIME_STATE_UNSPECIFIED +} + +func (x *PipelineTask_LogHistory) GetPods() []*PipelineTask_TaskPod { + if x != nil { + return x.Pods + } + return nil +} + +func (x *PipelineTask_LogHistory) GetLogArtifacts() []*PipelineTask_InputOutputs_IOArtifact { + if x != nil { + return x.LogArtifacts + } + return nil +} + +func (x *PipelineTask_LogHistory) GetDriverLogsUri() string { + if x != nil { + return x.DriverLogsUri + } + return "" +} + type PipelineTask_InputOutputs_IOParameter struct { state protoimpl.MessageState `protogen:"open.v1"` Value *structpb.Value `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` @@ -2904,7 +3020,7 @@ type PipelineTask_InputOutputs_IOParameter struct { func (x *PipelineTask_InputOutputs_IOParameter) Reset() { *x = PipelineTask_InputOutputs_IOParameter{} - mi := &file_backend_api_v2_run_proto_msgTypes[35] + mi := &file_backend_api_v2_run_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2916,7 +3032,7 @@ func (x *PipelineTask_InputOutputs_IOParameter) String() string { func (*PipelineTask_InputOutputs_IOParameter) ProtoMessage() {} func (x *PipelineTask_InputOutputs_IOParameter) ProtoReflect() protoreflect.Message { - mi := &file_backend_api_v2_run_proto_msgTypes[35] + mi := &file_backend_api_v2_run_proto_msgTypes[36] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2973,7 +3089,7 @@ type PipelineTask_InputOutputs_IOArtifact struct { func (x *PipelineTask_InputOutputs_IOArtifact) Reset() { *x = PipelineTask_InputOutputs_IOArtifact{} - mi := &file_backend_api_v2_run_proto_msgTypes[36] + mi := &file_backend_api_v2_run_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2985,7 +3101,7 @@ func (x *PipelineTask_InputOutputs_IOArtifact) String() string { func (*PipelineTask_InputOutputs_IOArtifact) ProtoMessage() {} func (x *PipelineTask_InputOutputs_IOArtifact) ProtoReflect() protoreflect.Message { - mi := &file_backend_api_v2_run_proto_msgTypes[36] + mi := &file_backend_api_v2_run_proto_msgTypes[37] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3103,7 +3219,7 @@ const file_backend_api_v2_run_proto_rawDesc = "" + "\n" + "RunDetails\x12.\n" + "\x13pipeline_context_id\x18\x01 \x01(\x03R\x11pipelineContextId\x125\n" + - "\x17pipeline_run_context_id\x18\x02 \x01(\x03R\x14pipelineRunContextIdJ\x04\b\x03\x10\x04R\ftask_details\"\xd1\x19\n" + + "\x17pipeline_run_context_id\x18\x02 \x01(\x03R\x14pipelineRunContextIdJ\x04\b\x03\x10\x04R\ftask_details\"\xd8\x1e\n" + "\fPipelineTask\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12!\n" + "\fdisplay_name\x18\x02 \x01(\tR\vdisplayName\x12\x17\n" + @@ -3128,7 +3244,9 @@ const file_backend_api_v2_run_proto_rawDesc = "" + "\n" + "scope_path\x18\x12 \x01(\tR\tscopePath\x120\n" + "\x11lifecycle_message\x18\x13 \x01(\tH\x01R\x10lifecycleMessage\x88\x01\x01\x122\n" + - "\x12lifecycle_category\x18\x14 \x01(\tH\x02R\x11lifecycleCategory\x88\x01\x01\x1a\x80\x01\n" + + "\x12lifecycle_category\x18\x14 \x01(\tH\x02R\x11lifecycleCategory\x88\x01\x01\x12[\n" + + "\vlog_history\x18\x15 \x03(\v2:.kubeflow.pipelines.backend.api.v2.PipelineTask.LogHistoryR\n" + + "logHistory\x1a\x80\x01\n" + "\aTaskPod\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x10\n" + "\x03uid\x18\x02 \x01(\tR\x03uid\x12O\n" + @@ -3171,7 +3289,19 @@ const file_backend_api_v2_run_proto_rawDesc = "" + "\tartifacts\x18\x01 \x03(\v2+.kubeflow.pipelines.backend.api.v2.ArtifactR\tartifacts\x12=\n" + "\x04type\x18\x02 \x01(\x0e2).kubeflow.pipelines.backend.api.v2.IOTypeR\x04type\x12!\n" + "\fartifact_key\x18\x03 \x01(\tR\vartifactKey\x12I\n" + - "\bproducer\x18\x04 \x01(\v2-.kubeflow.pipelines.backend.api.v2.IOProducerR\bproducer\"8\n" + + "\bproducer\x18\x04 \x01(\v2-.kubeflow.pipelines.backend.api.v2.IOProducerR\bproducer\x1a\xa7\x04\n" + + "\n" + + "LogHistory\x126\n" + + "\x17before_retry_generation\x18\x01 \x01(\x03R\x15beforeRetryGeneration\x12;\n" + + "\varchived_at\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\n" + + "archivedAt\x129\n" + + "\n" + + "start_time\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\tstartTime\x125\n" + + "\bend_time\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\aendTime\x12O\n" + + "\x05state\x18\x05 \x01(\x0e29.kubeflow.pipelines.backend.api.v2.PipelineTask.TaskStateR\x05state\x12K\n" + + "\x04pods\x18\x06 \x03(\v27.kubeflow.pipelines.backend.api.v2.PipelineTask.TaskPodR\x04pods\x12l\n" + + "\rlog_artifacts\x18\a \x03(\v2G.kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOArtifactR\flogArtifacts\x12&\n" + + "\x0fdriver_logs_uri\x18\b \x01(\tR\rdriverLogsUri\"8\n" + "\vTaskPodType\x12\x0f\n" + "\vUNSPECIFIED\x10\x00\x12\n" + "\n" + @@ -3358,7 +3488,7 @@ func file_backend_api_v2_run_proto_rawDescGZIP() []byte { } var file_backend_api_v2_run_proto_enumTypes = make([]protoimpl.EnumInfo, 9) -var file_backend_api_v2_run_proto_msgTypes = make([]protoimpl.MessageInfo, 39) +var file_backend_api_v2_run_proto_msgTypes = make([]protoimpl.MessageInfo, 40) var file_backend_api_v2_run_proto_goTypes = []any{ (RuntimeState)(0), // 0: kubeflow.pipelines.backend.api.v2.RuntimeState (PluginState)(0), // 1: kubeflow.pipelines.backend.api.v2.PluginState @@ -3403,47 +3533,48 @@ var file_backend_api_v2_run_proto_goTypes = []any{ (*PipelineTask_TypeAttributes)(nil), // 40: kubeflow.pipelines.backend.api.v2.PipelineTask.TypeAttributes (*PipelineTask_ChildTask)(nil), // 41: kubeflow.pipelines.backend.api.v2.PipelineTask.ChildTask (*PipelineTask_InputOutputs)(nil), // 42: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs - nil, // 43: kubeflow.pipelines.backend.api.v2.PipelineTask.StatusMetadata.CustomPropertiesEntry - (*PipelineTask_InputOutputs_IOParameter)(nil), // 44: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOParameter - (*PipelineTask_InputOutputs_IOArtifact)(nil), // 45: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOArtifact - nil, // 46: kubeflow.pipelines.backend.api.v2.UpdateTasksBulkRequest.TasksEntry - nil, // 47: kubeflow.pipelines.backend.api.v2.UpdateTasksBulkResponse.TasksEntry - (*structpb.Struct)(nil), // 48: google.protobuf.Struct - (*RuntimeConfig)(nil), // 49: kubeflow.pipelines.backend.api.v2.RuntimeConfig - (*timestamppb.Timestamp)(nil), // 50: google.protobuf.Timestamp - (*status.Status)(nil), // 51: google.rpc.Status - (*structpb.Value)(nil), // 52: google.protobuf.Value - (IOType)(0), // 53: kubeflow.pipelines.backend.api.v2.IOType - (*IOProducer)(nil), // 54: kubeflow.pipelines.backend.api.v2.IOProducer - (*Artifact)(nil), // 55: kubeflow.pipelines.backend.api.v2.Artifact - (*emptypb.Empty)(nil), // 56: google.protobuf.Empty + (*PipelineTask_LogHistory)(nil), // 43: kubeflow.pipelines.backend.api.v2.PipelineTask.LogHistory + nil, // 44: kubeflow.pipelines.backend.api.v2.PipelineTask.StatusMetadata.CustomPropertiesEntry + (*PipelineTask_InputOutputs_IOParameter)(nil), // 45: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOParameter + (*PipelineTask_InputOutputs_IOArtifact)(nil), // 46: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOArtifact + nil, // 47: kubeflow.pipelines.backend.api.v2.UpdateTasksBulkRequest.TasksEntry + nil, // 48: kubeflow.pipelines.backend.api.v2.UpdateTasksBulkResponse.TasksEntry + (*structpb.Struct)(nil), // 49: google.protobuf.Struct + (*RuntimeConfig)(nil), // 50: kubeflow.pipelines.backend.api.v2.RuntimeConfig + (*timestamppb.Timestamp)(nil), // 51: google.protobuf.Timestamp + (*status.Status)(nil), // 52: google.rpc.Status + (*structpb.Value)(nil), // 53: google.protobuf.Value + (IOType)(0), // 54: kubeflow.pipelines.backend.api.v2.IOType + (*IOProducer)(nil), // 55: kubeflow.pipelines.backend.api.v2.IOProducer + (*Artifact)(nil), // 56: kubeflow.pipelines.backend.api.v2.Artifact + (*emptypb.Empty)(nil), // 57: google.protobuf.Empty } var file_backend_api_v2_run_proto_depIdxs = []int32{ 2, // 0: kubeflow.pipelines.backend.api.v2.Run.storage_state:type_name -> kubeflow.pipelines.backend.api.v2.Run.StorageState - 48, // 1: kubeflow.pipelines.backend.api.v2.Run.pipeline_spec:type_name -> google.protobuf.Struct + 49, // 1: kubeflow.pipelines.backend.api.v2.Run.pipeline_spec:type_name -> google.protobuf.Struct 10, // 2: kubeflow.pipelines.backend.api.v2.Run.pipeline_version_reference:type_name -> kubeflow.pipelines.backend.api.v2.PipelineVersionReference - 49, // 3: kubeflow.pipelines.backend.api.v2.Run.runtime_config:type_name -> kubeflow.pipelines.backend.api.v2.RuntimeConfig - 50, // 4: kubeflow.pipelines.backend.api.v2.Run.created_at:type_name -> google.protobuf.Timestamp - 50, // 5: kubeflow.pipelines.backend.api.v2.Run.scheduled_at:type_name -> google.protobuf.Timestamp - 50, // 6: kubeflow.pipelines.backend.api.v2.Run.finished_at:type_name -> google.protobuf.Timestamp + 50, // 3: kubeflow.pipelines.backend.api.v2.Run.runtime_config:type_name -> kubeflow.pipelines.backend.api.v2.RuntimeConfig + 51, // 4: kubeflow.pipelines.backend.api.v2.Run.created_at:type_name -> google.protobuf.Timestamp + 51, // 5: kubeflow.pipelines.backend.api.v2.Run.scheduled_at:type_name -> google.protobuf.Timestamp + 51, // 6: kubeflow.pipelines.backend.api.v2.Run.finished_at:type_name -> google.protobuf.Timestamp 0, // 7: kubeflow.pipelines.backend.api.v2.Run.state:type_name -> kubeflow.pipelines.backend.api.v2.RuntimeState - 51, // 8: kubeflow.pipelines.backend.api.v2.Run.error:type_name -> google.rpc.Status + 52, // 8: kubeflow.pipelines.backend.api.v2.Run.error:type_name -> google.rpc.Status 14, // 9: kubeflow.pipelines.backend.api.v2.Run.run_details:type_name -> kubeflow.pipelines.backend.api.v2.RunDetails 11, // 10: kubeflow.pipelines.backend.api.v2.Run.state_history:type_name -> kubeflow.pipelines.backend.api.v2.RuntimeStatus 34, // 11: kubeflow.pipelines.backend.api.v2.Run.plugins_input:type_name -> kubeflow.pipelines.backend.api.v2.Run.PluginsInputEntry 35, // 12: kubeflow.pipelines.backend.api.v2.Run.plugins_output:type_name -> kubeflow.pipelines.backend.api.v2.Run.PluginsOutputEntry 10, // 13: kubeflow.pipelines.backend.api.v2.Run.pipeline_reference:type_name -> kubeflow.pipelines.backend.api.v2.PipelineVersionReference 15, // 14: kubeflow.pipelines.backend.api.v2.Run.tasks:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask - 50, // 15: kubeflow.pipelines.backend.api.v2.RuntimeStatus.update_time:type_name -> google.protobuf.Timestamp + 51, // 15: kubeflow.pipelines.backend.api.v2.RuntimeStatus.update_time:type_name -> google.protobuf.Timestamp 0, // 16: kubeflow.pipelines.backend.api.v2.RuntimeStatus.state:type_name -> kubeflow.pipelines.backend.api.v2.RuntimeState - 51, // 17: kubeflow.pipelines.backend.api.v2.RuntimeStatus.error:type_name -> google.rpc.Status - 52, // 18: kubeflow.pipelines.backend.api.v2.MetadataValue.value:type_name -> google.protobuf.Value + 52, // 17: kubeflow.pipelines.backend.api.v2.RuntimeStatus.error:type_name -> google.rpc.Status + 53, // 18: kubeflow.pipelines.backend.api.v2.MetadataValue.value:type_name -> google.protobuf.Value 3, // 19: kubeflow.pipelines.backend.api.v2.MetadataValue.render_type:type_name -> kubeflow.pipelines.backend.api.v2.MetadataValue.RenderType 36, // 20: kubeflow.pipelines.backend.api.v2.PluginOutput.entries:type_name -> kubeflow.pipelines.backend.api.v2.PluginOutput.EntriesEntry 1, // 21: kubeflow.pipelines.backend.api.v2.PluginOutput.state:type_name -> kubeflow.pipelines.backend.api.v2.PluginState 37, // 22: kubeflow.pipelines.backend.api.v2.PipelineTask.pods:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.TaskPod - 50, // 23: kubeflow.pipelines.backend.api.v2.PipelineTask.create_time:type_name -> google.protobuf.Timestamp - 50, // 24: kubeflow.pipelines.backend.api.v2.PipelineTask.end_time:type_name -> google.protobuf.Timestamp + 51, // 23: kubeflow.pipelines.backend.api.v2.PipelineTask.create_time:type_name -> google.protobuf.Timestamp + 51, // 24: kubeflow.pipelines.backend.api.v2.PipelineTask.end_time:type_name -> google.protobuf.Timestamp 5, // 25: kubeflow.pipelines.backend.api.v2.PipelineTask.state:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.TaskState 38, // 26: kubeflow.pipelines.backend.api.v2.PipelineTask.status_metadata:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.StatusMetadata 39, // 27: kubeflow.pipelines.backend.api.v2.PipelineTask.state_history:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.TaskStatus @@ -3452,68 +3583,75 @@ var file_backend_api_v2_run_proto_depIdxs = []int32{ 41, // 30: kubeflow.pipelines.backend.api.v2.PipelineTask.child_tasks:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.ChildTask 42, // 31: kubeflow.pipelines.backend.api.v2.PipelineTask.inputs:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs 42, // 32: kubeflow.pipelines.backend.api.v2.PipelineTask.outputs:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs - 9, // 33: kubeflow.pipelines.backend.api.v2.CreateRunRequest.run:type_name -> kubeflow.pipelines.backend.api.v2.Run - 7, // 34: kubeflow.pipelines.backend.api.v2.GetRunRequest.view:type_name -> kubeflow.pipelines.backend.api.v2.GetRunRequest.ViewMode - 8, // 35: kubeflow.pipelines.backend.api.v2.ListRunsRequest.view:type_name -> kubeflow.pipelines.backend.api.v2.ListRunsRequest.ViewMode - 9, // 36: kubeflow.pipelines.backend.api.v2.ListRunsResponse.runs:type_name -> kubeflow.pipelines.backend.api.v2.Run - 15, // 37: kubeflow.pipelines.backend.api.v2.CreateTaskRequest.task:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask - 15, // 38: kubeflow.pipelines.backend.api.v2.UpdateTaskRequest.task:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask - 46, // 39: kubeflow.pipelines.backend.api.v2.UpdateTasksBulkRequest.tasks:type_name -> kubeflow.pipelines.backend.api.v2.UpdateTasksBulkRequest.TasksEntry - 47, // 40: kubeflow.pipelines.backend.api.v2.UpdateTasksBulkResponse.tasks:type_name -> kubeflow.pipelines.backend.api.v2.UpdateTasksBulkResponse.TasksEntry - 15, // 41: kubeflow.pipelines.backend.api.v2.ListTasksResponse.tasks:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask - 15, // 42: kubeflow.pipelines.backend.api.v2.FindCachedTaskResponse.task:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask - 48, // 43: kubeflow.pipelines.backend.api.v2.Run.PluginsInputEntry.value:type_name -> google.protobuf.Struct - 13, // 44: kubeflow.pipelines.backend.api.v2.Run.PluginsOutputEntry.value:type_name -> kubeflow.pipelines.backend.api.v2.PluginOutput - 12, // 45: kubeflow.pipelines.backend.api.v2.PluginOutput.EntriesEntry.value:type_name -> kubeflow.pipelines.backend.api.v2.MetadataValue - 4, // 46: kubeflow.pipelines.backend.api.v2.PipelineTask.TaskPod.type:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.TaskPodType - 43, // 47: kubeflow.pipelines.backend.api.v2.PipelineTask.StatusMetadata.custom_properties:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.StatusMetadata.CustomPropertiesEntry - 50, // 48: kubeflow.pipelines.backend.api.v2.PipelineTask.TaskStatus.update_time:type_name -> google.protobuf.Timestamp - 5, // 49: kubeflow.pipelines.backend.api.v2.PipelineTask.TaskStatus.state:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.TaskState - 51, // 50: kubeflow.pipelines.backend.api.v2.PipelineTask.TaskStatus.error:type_name -> google.rpc.Status - 44, // 51: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.parameters:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOParameter - 45, // 52: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.artifacts:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOArtifact - 52, // 53: kubeflow.pipelines.backend.api.v2.PipelineTask.StatusMetadata.CustomPropertiesEntry.value:type_name -> google.protobuf.Value - 52, // 54: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOParameter.value:type_name -> google.protobuf.Value - 53, // 55: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOParameter.type:type_name -> kubeflow.pipelines.backend.api.v2.IOType - 54, // 56: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOParameter.producer:type_name -> kubeflow.pipelines.backend.api.v2.IOProducer - 55, // 57: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOArtifact.artifacts:type_name -> kubeflow.pipelines.backend.api.v2.Artifact - 53, // 58: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOArtifact.type:type_name -> kubeflow.pipelines.backend.api.v2.IOType - 54, // 59: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOArtifact.producer:type_name -> kubeflow.pipelines.backend.api.v2.IOProducer - 15, // 60: kubeflow.pipelines.backend.api.v2.UpdateTasksBulkRequest.TasksEntry.value:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask - 15, // 61: kubeflow.pipelines.backend.api.v2.UpdateTasksBulkResponse.TasksEntry.value:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask - 16, // 62: kubeflow.pipelines.backend.api.v2.RunService.CreateRun:input_type -> kubeflow.pipelines.backend.api.v2.CreateRunRequest - 17, // 63: kubeflow.pipelines.backend.api.v2.RunService.GetRun:input_type -> kubeflow.pipelines.backend.api.v2.GetRunRequest - 18, // 64: kubeflow.pipelines.backend.api.v2.RunService.ListRuns:input_type -> kubeflow.pipelines.backend.api.v2.ListRunsRequest - 21, // 65: kubeflow.pipelines.backend.api.v2.RunService.ArchiveRun:input_type -> kubeflow.pipelines.backend.api.v2.ArchiveRunRequest - 22, // 66: kubeflow.pipelines.backend.api.v2.RunService.UnarchiveRun:input_type -> kubeflow.pipelines.backend.api.v2.UnarchiveRunRequest - 23, // 67: kubeflow.pipelines.backend.api.v2.RunService.DeleteRun:input_type -> kubeflow.pipelines.backend.api.v2.DeleteRunRequest - 19, // 68: kubeflow.pipelines.backend.api.v2.RunService.TerminateRun:input_type -> kubeflow.pipelines.backend.api.v2.TerminateRunRequest - 24, // 69: kubeflow.pipelines.backend.api.v2.RunService.RetryRun:input_type -> kubeflow.pipelines.backend.api.v2.RetryRunRequest - 25, // 70: kubeflow.pipelines.backend.api.v2.RunService.CreateTask:input_type -> kubeflow.pipelines.backend.api.v2.CreateTaskRequest - 26, // 71: kubeflow.pipelines.backend.api.v2.RunService.UpdateTask:input_type -> kubeflow.pipelines.backend.api.v2.UpdateTaskRequest - 27, // 72: kubeflow.pipelines.backend.api.v2.RunService.UpdateTasksBulk:input_type -> kubeflow.pipelines.backend.api.v2.UpdateTasksBulkRequest - 29, // 73: kubeflow.pipelines.backend.api.v2.RunService.GetTask:input_type -> kubeflow.pipelines.backend.api.v2.GetTaskRequest - 30, // 74: kubeflow.pipelines.backend.api.v2.RunService.ListTasks:input_type -> kubeflow.pipelines.backend.api.v2.ListTasksRequest - 32, // 75: kubeflow.pipelines.backend.api.v2.RunService.FindCachedTask:input_type -> kubeflow.pipelines.backend.api.v2.FindCachedTaskRequest - 9, // 76: kubeflow.pipelines.backend.api.v2.RunService.CreateRun:output_type -> kubeflow.pipelines.backend.api.v2.Run - 9, // 77: kubeflow.pipelines.backend.api.v2.RunService.GetRun:output_type -> kubeflow.pipelines.backend.api.v2.Run - 20, // 78: kubeflow.pipelines.backend.api.v2.RunService.ListRuns:output_type -> kubeflow.pipelines.backend.api.v2.ListRunsResponse - 56, // 79: kubeflow.pipelines.backend.api.v2.RunService.ArchiveRun:output_type -> google.protobuf.Empty - 56, // 80: kubeflow.pipelines.backend.api.v2.RunService.UnarchiveRun:output_type -> google.protobuf.Empty - 56, // 81: kubeflow.pipelines.backend.api.v2.RunService.DeleteRun:output_type -> google.protobuf.Empty - 56, // 82: kubeflow.pipelines.backend.api.v2.RunService.TerminateRun:output_type -> google.protobuf.Empty - 56, // 83: kubeflow.pipelines.backend.api.v2.RunService.RetryRun:output_type -> google.protobuf.Empty - 15, // 84: kubeflow.pipelines.backend.api.v2.RunService.CreateTask:output_type -> kubeflow.pipelines.backend.api.v2.PipelineTask - 15, // 85: kubeflow.pipelines.backend.api.v2.RunService.UpdateTask:output_type -> kubeflow.pipelines.backend.api.v2.PipelineTask - 28, // 86: kubeflow.pipelines.backend.api.v2.RunService.UpdateTasksBulk:output_type -> kubeflow.pipelines.backend.api.v2.UpdateTasksBulkResponse - 15, // 87: kubeflow.pipelines.backend.api.v2.RunService.GetTask:output_type -> kubeflow.pipelines.backend.api.v2.PipelineTask - 31, // 88: kubeflow.pipelines.backend.api.v2.RunService.ListTasks:output_type -> kubeflow.pipelines.backend.api.v2.ListTasksResponse - 33, // 89: kubeflow.pipelines.backend.api.v2.RunService.FindCachedTask:output_type -> kubeflow.pipelines.backend.api.v2.FindCachedTaskResponse - 76, // [76:90] is the sub-list for method output_type - 62, // [62:76] is the sub-list for method input_type - 62, // [62:62] is the sub-list for extension type_name - 62, // [62:62] is the sub-list for extension extendee - 0, // [0:62] is the sub-list for field type_name + 43, // 33: kubeflow.pipelines.backend.api.v2.PipelineTask.log_history:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.LogHistory + 9, // 34: kubeflow.pipelines.backend.api.v2.CreateRunRequest.run:type_name -> kubeflow.pipelines.backend.api.v2.Run + 7, // 35: kubeflow.pipelines.backend.api.v2.GetRunRequest.view:type_name -> kubeflow.pipelines.backend.api.v2.GetRunRequest.ViewMode + 8, // 36: kubeflow.pipelines.backend.api.v2.ListRunsRequest.view:type_name -> kubeflow.pipelines.backend.api.v2.ListRunsRequest.ViewMode + 9, // 37: kubeflow.pipelines.backend.api.v2.ListRunsResponse.runs:type_name -> kubeflow.pipelines.backend.api.v2.Run + 15, // 38: kubeflow.pipelines.backend.api.v2.CreateTaskRequest.task:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask + 15, // 39: kubeflow.pipelines.backend.api.v2.UpdateTaskRequest.task:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask + 47, // 40: kubeflow.pipelines.backend.api.v2.UpdateTasksBulkRequest.tasks:type_name -> kubeflow.pipelines.backend.api.v2.UpdateTasksBulkRequest.TasksEntry + 48, // 41: kubeflow.pipelines.backend.api.v2.UpdateTasksBulkResponse.tasks:type_name -> kubeflow.pipelines.backend.api.v2.UpdateTasksBulkResponse.TasksEntry + 15, // 42: kubeflow.pipelines.backend.api.v2.ListTasksResponse.tasks:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask + 15, // 43: kubeflow.pipelines.backend.api.v2.FindCachedTaskResponse.task:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask + 49, // 44: kubeflow.pipelines.backend.api.v2.Run.PluginsInputEntry.value:type_name -> google.protobuf.Struct + 13, // 45: kubeflow.pipelines.backend.api.v2.Run.PluginsOutputEntry.value:type_name -> kubeflow.pipelines.backend.api.v2.PluginOutput + 12, // 46: kubeflow.pipelines.backend.api.v2.PluginOutput.EntriesEntry.value:type_name -> kubeflow.pipelines.backend.api.v2.MetadataValue + 4, // 47: kubeflow.pipelines.backend.api.v2.PipelineTask.TaskPod.type:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.TaskPodType + 44, // 48: kubeflow.pipelines.backend.api.v2.PipelineTask.StatusMetadata.custom_properties:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.StatusMetadata.CustomPropertiesEntry + 51, // 49: kubeflow.pipelines.backend.api.v2.PipelineTask.TaskStatus.update_time:type_name -> google.protobuf.Timestamp + 5, // 50: kubeflow.pipelines.backend.api.v2.PipelineTask.TaskStatus.state:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.TaskState + 52, // 51: kubeflow.pipelines.backend.api.v2.PipelineTask.TaskStatus.error:type_name -> google.rpc.Status + 45, // 52: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.parameters:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOParameter + 46, // 53: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.artifacts:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOArtifact + 51, // 54: kubeflow.pipelines.backend.api.v2.PipelineTask.LogHistory.archived_at:type_name -> google.protobuf.Timestamp + 51, // 55: kubeflow.pipelines.backend.api.v2.PipelineTask.LogHistory.start_time:type_name -> google.protobuf.Timestamp + 51, // 56: kubeflow.pipelines.backend.api.v2.PipelineTask.LogHistory.end_time:type_name -> google.protobuf.Timestamp + 5, // 57: kubeflow.pipelines.backend.api.v2.PipelineTask.LogHistory.state:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.TaskState + 37, // 58: kubeflow.pipelines.backend.api.v2.PipelineTask.LogHistory.pods:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.TaskPod + 46, // 59: kubeflow.pipelines.backend.api.v2.PipelineTask.LogHistory.log_artifacts:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOArtifact + 53, // 60: kubeflow.pipelines.backend.api.v2.PipelineTask.StatusMetadata.CustomPropertiesEntry.value:type_name -> google.protobuf.Value + 53, // 61: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOParameter.value:type_name -> google.protobuf.Value + 54, // 62: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOParameter.type:type_name -> kubeflow.pipelines.backend.api.v2.IOType + 55, // 63: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOParameter.producer:type_name -> kubeflow.pipelines.backend.api.v2.IOProducer + 56, // 64: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOArtifact.artifacts:type_name -> kubeflow.pipelines.backend.api.v2.Artifact + 54, // 65: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOArtifact.type:type_name -> kubeflow.pipelines.backend.api.v2.IOType + 55, // 66: kubeflow.pipelines.backend.api.v2.PipelineTask.InputOutputs.IOArtifact.producer:type_name -> kubeflow.pipelines.backend.api.v2.IOProducer + 15, // 67: kubeflow.pipelines.backend.api.v2.UpdateTasksBulkRequest.TasksEntry.value:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask + 15, // 68: kubeflow.pipelines.backend.api.v2.UpdateTasksBulkResponse.TasksEntry.value:type_name -> kubeflow.pipelines.backend.api.v2.PipelineTask + 16, // 69: kubeflow.pipelines.backend.api.v2.RunService.CreateRun:input_type -> kubeflow.pipelines.backend.api.v2.CreateRunRequest + 17, // 70: kubeflow.pipelines.backend.api.v2.RunService.GetRun:input_type -> kubeflow.pipelines.backend.api.v2.GetRunRequest + 18, // 71: kubeflow.pipelines.backend.api.v2.RunService.ListRuns:input_type -> kubeflow.pipelines.backend.api.v2.ListRunsRequest + 21, // 72: kubeflow.pipelines.backend.api.v2.RunService.ArchiveRun:input_type -> kubeflow.pipelines.backend.api.v2.ArchiveRunRequest + 22, // 73: kubeflow.pipelines.backend.api.v2.RunService.UnarchiveRun:input_type -> kubeflow.pipelines.backend.api.v2.UnarchiveRunRequest + 23, // 74: kubeflow.pipelines.backend.api.v2.RunService.DeleteRun:input_type -> kubeflow.pipelines.backend.api.v2.DeleteRunRequest + 19, // 75: kubeflow.pipelines.backend.api.v2.RunService.TerminateRun:input_type -> kubeflow.pipelines.backend.api.v2.TerminateRunRequest + 24, // 76: kubeflow.pipelines.backend.api.v2.RunService.RetryRun:input_type -> kubeflow.pipelines.backend.api.v2.RetryRunRequest + 25, // 77: kubeflow.pipelines.backend.api.v2.RunService.CreateTask:input_type -> kubeflow.pipelines.backend.api.v2.CreateTaskRequest + 26, // 78: kubeflow.pipelines.backend.api.v2.RunService.UpdateTask:input_type -> kubeflow.pipelines.backend.api.v2.UpdateTaskRequest + 27, // 79: kubeflow.pipelines.backend.api.v2.RunService.UpdateTasksBulk:input_type -> kubeflow.pipelines.backend.api.v2.UpdateTasksBulkRequest + 29, // 80: kubeflow.pipelines.backend.api.v2.RunService.GetTask:input_type -> kubeflow.pipelines.backend.api.v2.GetTaskRequest + 30, // 81: kubeflow.pipelines.backend.api.v2.RunService.ListTasks:input_type -> kubeflow.pipelines.backend.api.v2.ListTasksRequest + 32, // 82: kubeflow.pipelines.backend.api.v2.RunService.FindCachedTask:input_type -> kubeflow.pipelines.backend.api.v2.FindCachedTaskRequest + 9, // 83: kubeflow.pipelines.backend.api.v2.RunService.CreateRun:output_type -> kubeflow.pipelines.backend.api.v2.Run + 9, // 84: kubeflow.pipelines.backend.api.v2.RunService.GetRun:output_type -> kubeflow.pipelines.backend.api.v2.Run + 20, // 85: kubeflow.pipelines.backend.api.v2.RunService.ListRuns:output_type -> kubeflow.pipelines.backend.api.v2.ListRunsResponse + 57, // 86: kubeflow.pipelines.backend.api.v2.RunService.ArchiveRun:output_type -> google.protobuf.Empty + 57, // 87: kubeflow.pipelines.backend.api.v2.RunService.UnarchiveRun:output_type -> google.protobuf.Empty + 57, // 88: kubeflow.pipelines.backend.api.v2.RunService.DeleteRun:output_type -> google.protobuf.Empty + 57, // 89: kubeflow.pipelines.backend.api.v2.RunService.TerminateRun:output_type -> google.protobuf.Empty + 57, // 90: kubeflow.pipelines.backend.api.v2.RunService.RetryRun:output_type -> google.protobuf.Empty + 15, // 91: kubeflow.pipelines.backend.api.v2.RunService.CreateTask:output_type -> kubeflow.pipelines.backend.api.v2.PipelineTask + 15, // 92: kubeflow.pipelines.backend.api.v2.RunService.UpdateTask:output_type -> kubeflow.pipelines.backend.api.v2.PipelineTask + 28, // 93: kubeflow.pipelines.backend.api.v2.RunService.UpdateTasksBulk:output_type -> kubeflow.pipelines.backend.api.v2.UpdateTasksBulkResponse + 15, // 94: kubeflow.pipelines.backend.api.v2.RunService.GetTask:output_type -> kubeflow.pipelines.backend.api.v2.PipelineTask + 31, // 95: kubeflow.pipelines.backend.api.v2.RunService.ListTasks:output_type -> kubeflow.pipelines.backend.api.v2.ListTasksResponse + 33, // 96: kubeflow.pipelines.backend.api.v2.RunService.FindCachedTask:output_type -> kubeflow.pipelines.backend.api.v2.FindCachedTaskResponse + 83, // [83:97] is the sub-list for method output_type + 69, // [69:83] is the sub-list for method input_type + 69, // [69:69] is the sub-list for extension type_name + 69, // [69:69] is the sub-list for extension extendee + 0, // [0:69] is the sub-list for field type_name } func init() { file_backend_api_v2_run_proto_init() } @@ -3536,14 +3674,14 @@ func file_backend_api_v2_run_proto_init() { (*ListTasksRequest_ParentId)(nil), } file_backend_api_v2_run_proto_msgTypes[31].OneofWrappers = []any{} - file_backend_api_v2_run_proto_msgTypes[35].OneofWrappers = []any{} + file_backend_api_v2_run_proto_msgTypes[36].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_backend_api_v2_run_proto_rawDesc), len(file_backend_api_v2_run_proto_rawDesc)), NumEnums: 9, - NumMessages: 39, + NumMessages: 40, NumExtensions: 0, NumServices: 1, }, diff --git a/backend/api/v2/go_http_client/run_model/pipeline_task_log_history.go b/backend/api/v2/go_http_client/run_model/pipeline_task_log_history.go new file mode 100644 index 00000000000..fcb510dee48 --- /dev/null +++ b/backend/api/v2/go_http_client/run_model/pipeline_task_log_history.go @@ -0,0 +1,326 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package run_model + +import ( + "context" + stderrors "errors" + "strconv" + + "github.com/go-openapi/errors" + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag/jsonutils" + "github.com/go-openapi/swag/typeutils" + "github.com/go-openapi/validate" +) + +// PipelineTaskLogHistory Log references from an execution archived before a manual run retry. +// References can outlive pods or archived objects and do not guarantee availability. +// +// swagger:model PipelineTaskLogHistory +type PipelineTaskLogHistory struct { + + // archived at + // Format: date-time + ArchivedAt strfmt.DateTime `json:"archived_at,omitempty"` + + // Together with task_id, identifies this snapshot. This is the manual retry + // generation that archived it, not an automatic pod retry attempt number. + BeforeRetryGeneration string `json:"before_retry_generation,omitempty"` + + // Legacy driver log reference, when not represented by a log artifact. + DriverLogsURI string `json:"driver_logs_uri,omitempty"` + + // end time + // Format: date-time + EndTime strfmt.DateTime `json:"end_time,omitempty"` + + // Log artifacts only; never includes task outputs used by downstream tasks. + LogArtifacts []*InputOutputsIOArtifact `json:"log_artifacts"` + + // pods + Pods []*PipelineTaskTaskPod `json:"pods"` + + // start time + // Format: date-time + StartTime strfmt.DateTime `json:"start_time,omitempty"` + + // state + State *PipelineTaskTaskState `json:"state,omitempty"` +} + +// Validate validates this pipeline task log history +func (m *PipelineTaskLogHistory) Validate(formats strfmt.Registry) error { + var res []error + + if err := m.validateArchivedAt(formats); err != nil { + res = append(res, err) + } + + if err := m.validateEndTime(formats); err != nil { + res = append(res, err) + } + + if err := m.validateLogArtifacts(formats); err != nil { + res = append(res, err) + } + + if err := m.validatePods(formats); err != nil { + res = append(res, err) + } + + if err := m.validateStartTime(formats); err != nil { + res = append(res, err) + } + + if err := m.validateState(formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (m *PipelineTaskLogHistory) validateArchivedAt(formats strfmt.Registry) error { + if typeutils.IsZero(m.ArchivedAt) { // not required + return nil + } + + if err := validate.FormatOf("archived_at", "body", "date-time", m.ArchivedAt.String(), formats); err != nil { + return err + } + + return nil +} + +func (m *PipelineTaskLogHistory) validateEndTime(formats strfmt.Registry) error { + if typeutils.IsZero(m.EndTime) { // not required + return nil + } + + if err := validate.FormatOf("end_time", "body", "date-time", m.EndTime.String(), formats); err != nil { + return err + } + + return nil +} + +func (m *PipelineTaskLogHistory) validateLogArtifacts(formats strfmt.Registry) error { + if typeutils.IsZero(m.LogArtifacts) { // not required + return nil + } + + for i := 0; i < len(m.LogArtifacts); i++ { + if typeutils.IsZero(m.LogArtifacts[i]) { // not required + continue + } + + if m.LogArtifacts[i] != nil { + if err := m.LogArtifacts[i].Validate(formats); err != nil { + ve := new(errors.Validation) + if stderrors.As(err, &ve) { + return ve.ValidateName("log_artifacts" + "." + strconv.Itoa(i)) + } + ce := new(errors.CompositeError) + if stderrors.As(err, &ce) { + return ce.ValidateName("log_artifacts" + "." + strconv.Itoa(i)) + } + + return err + } + } + + } + + return nil +} + +func (m *PipelineTaskLogHistory) validatePods(formats strfmt.Registry) error { + if typeutils.IsZero(m.Pods) { // not required + return nil + } + + for i := 0; i < len(m.Pods); i++ { + if typeutils.IsZero(m.Pods[i]) { // not required + continue + } + + if m.Pods[i] != nil { + if err := m.Pods[i].Validate(formats); err != nil { + ve := new(errors.Validation) + if stderrors.As(err, &ve) { + return ve.ValidateName("pods" + "." + strconv.Itoa(i)) + } + ce := new(errors.CompositeError) + if stderrors.As(err, &ce) { + return ce.ValidateName("pods" + "." + strconv.Itoa(i)) + } + + return err + } + } + + } + + return nil +} + +func (m *PipelineTaskLogHistory) validateStartTime(formats strfmt.Registry) error { + if typeutils.IsZero(m.StartTime) { // not required + return nil + } + + if err := validate.FormatOf("start_time", "body", "date-time", m.StartTime.String(), formats); err != nil { + return err + } + + return nil +} + +func (m *PipelineTaskLogHistory) validateState(formats strfmt.Registry) error { + if typeutils.IsZero(m.State) { // not required + return nil + } + + if m.State != nil { + if err := m.State.Validate(formats); err != nil { + ve := new(errors.Validation) + if stderrors.As(err, &ve) { + return ve.ValidateName("state") + } + ce := new(errors.CompositeError) + if stderrors.As(err, &ce) { + return ce.ValidateName("state") + } + + return err + } + } + + return nil +} + +// ContextValidate validate this pipeline task log history based on the context it is used +func (m *PipelineTaskLogHistory) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + var res []error + + if err := m.contextValidateLogArtifacts(ctx, formats); err != nil { + res = append(res, err) + } + + if err := m.contextValidatePods(ctx, formats); err != nil { + res = append(res, err) + } + + if err := m.contextValidateState(ctx, formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +func (m *PipelineTaskLogHistory) contextValidateLogArtifacts(ctx context.Context, formats strfmt.Registry) error { + + for i := 0; i < len(m.LogArtifacts); i++ { + + if m.LogArtifacts[i] != nil { + + if typeutils.IsZero(m.LogArtifacts[i]) { // not required + return nil + } + + if err := m.LogArtifacts[i].ContextValidate(ctx, formats); err != nil { + ve := new(errors.Validation) + if stderrors.As(err, &ve) { + return ve.ValidateName("log_artifacts" + "." + strconv.Itoa(i)) + } + ce := new(errors.CompositeError) + if stderrors.As(err, &ce) { + return ce.ValidateName("log_artifacts" + "." + strconv.Itoa(i)) + } + + return err + } + } + + } + + return nil +} + +func (m *PipelineTaskLogHistory) contextValidatePods(ctx context.Context, formats strfmt.Registry) error { + + for i := 0; i < len(m.Pods); i++ { + + if m.Pods[i] != nil { + + if typeutils.IsZero(m.Pods[i]) { // not required + return nil + } + + if err := m.Pods[i].ContextValidate(ctx, formats); err != nil { + ve := new(errors.Validation) + if stderrors.As(err, &ve) { + return ve.ValidateName("pods" + "." + strconv.Itoa(i)) + } + ce := new(errors.CompositeError) + if stderrors.As(err, &ce) { + return ce.ValidateName("pods" + "." + strconv.Itoa(i)) + } + + return err + } + } + + } + + return nil +} + +func (m *PipelineTaskLogHistory) contextValidateState(ctx context.Context, formats strfmt.Registry) error { + + if m.State != nil { + + if typeutils.IsZero(m.State) { // not required + return nil + } + + if err := m.State.ContextValidate(ctx, formats); err != nil { + ve := new(errors.Validation) + if stderrors.As(err, &ve) { + return ve.ValidateName("state") + } + ce := new(errors.CompositeError) + if stderrors.As(err, &ce) { + return ce.ValidateName("state") + } + + return err + } + } + + return nil +} + +// MarshalBinary interface implementation +func (m *PipelineTaskLogHistory) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return jsonutils.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *PipelineTaskLogHistory) UnmarshalBinary(b []byte) error { + var res PipelineTaskLogHistory + if err := jsonutils.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/backend/api/v2/go_http_client/run_model/v2_pipeline_task.go b/backend/api/v2/go_http_client/run_model/v2_pipeline_task.go index 37eebef6c3c..aa7a6ce4f06 100644 --- a/backend/api/v2/go_http_client/run_model/v2_pipeline_task.go +++ b/backend/api/v2/go_http_client/run_model/v2_pipeline_task.go @@ -49,6 +49,11 @@ type V2PipelineTask struct { // Read-only via REST; the persistence agent clears this field automatically on recovery. LifecycleMessage string `json:"lifecycle_message,omitempty"` + // Output only. Prior executions archived by manual retries, oldest first. + // History begins when preservation is enabled; older retries are not backfilled. + // Read Only: true + LogHistory []*PipelineTaskLogHistory `json:"log_history,omitempty"` + // name Name string `json:"name,omitempty"` @@ -115,6 +120,10 @@ func (m *V2PipelineTask) Validate(formats strfmt.Registry) error { res = append(res, err) } + if err := m.validateLogHistory(formats); err != nil { + res = append(res, err) + } + if err := m.validateOutputs(formats); err != nil { res = append(res, err) } @@ -226,6 +235,36 @@ func (m *V2PipelineTask) validateInputs(formats strfmt.Registry) error { return nil } +func (m *V2PipelineTask) validateLogHistory(formats strfmt.Registry) error { + if typeutils.IsZero(m.LogHistory) { // not required + return nil + } + + for i := 0; i < len(m.LogHistory); i++ { + if typeutils.IsZero(m.LogHistory[i]) { // not required + continue + } + + if m.LogHistory[i] != nil { + if err := m.LogHistory[i].Validate(formats); err != nil { + ve := new(errors.Validation) + if stderrors.As(err, &ve) { + return ve.ValidateName("log_history" + "." + strconv.Itoa(i)) + } + ce := new(errors.CompositeError) + if stderrors.As(err, &ce) { + return ce.ValidateName("log_history" + "." + strconv.Itoa(i)) + } + + return err + } + } + + } + + return nil +} + func (m *V2PipelineTask) validateOutputs(formats strfmt.Registry) error { if typeutils.IsZero(m.Outputs) { // not required return nil @@ -413,6 +452,10 @@ func (m *V2PipelineTask) ContextValidate(ctx context.Context, formats strfmt.Reg res = append(res, err) } + if err := m.contextValidateLogHistory(ctx, formats); err != nil { + res = append(res, err) + } + if err := m.contextValidateOutputs(ctx, formats); err != nil { res = append(res, err) } @@ -501,6 +544,39 @@ func (m *V2PipelineTask) contextValidateInputs(ctx context.Context, formats strf return nil } +func (m *V2PipelineTask) contextValidateLogHistory(ctx context.Context, formats strfmt.Registry) error { + + if err := validate.ReadOnly(ctx, "log_history", "body", m.LogHistory); err != nil { + return err + } + + for i := 0; i < len(m.LogHistory); i++ { + + if m.LogHistory[i] != nil { + + if typeutils.IsZero(m.LogHistory[i]) { // not required + return nil + } + + if err := m.LogHistory[i].ContextValidate(ctx, formats); err != nil { + ve := new(errors.Validation) + if stderrors.As(err, &ve) { + return ve.ValidateName("log_history" + "." + strconv.Itoa(i)) + } + ce := new(errors.CompositeError) + if stderrors.As(err, &ce) { + return ce.ValidateName("log_history" + "." + strconv.Itoa(i)) + } + + return err + } + } + + } + + return nil +} + func (m *V2PipelineTask) contextValidateOutputs(ctx context.Context, formats strfmt.Registry) error { if m.Outputs != nil { diff --git a/backend/api/v2/python_http_client/README.md b/backend/api/v2/python_http_client/README.md index abd379335bb..a16a0396d73 100644 --- a/backend/api/v2/python_http_client/README.md +++ b/backend/api/v2/python_http_client/README.md @@ -147,6 +147,7 @@ Class | Method | HTTP request | Description - [MetadataValueRenderType](docs/MetadataValueRenderType.md) - [PipelineTaskChildTask](docs/PipelineTaskChildTask.md) - [PipelineTaskInputOutputs](docs/PipelineTaskInputOutputs.md) + - [PipelineTaskLogHistory](docs/PipelineTaskLogHistory.md) - [PipelineTaskStatusMetadata](docs/PipelineTaskStatusMetadata.md) - [PipelineTaskTaskPod](docs/PipelineTaskTaskPod.md) - [PipelineTaskTaskPodType](docs/PipelineTaskTaskPodType.md) diff --git a/backend/api/v2/python_http_client/docs/PipelineTaskLogHistory.md b/backend/api/v2/python_http_client/docs/PipelineTaskLogHistory.md new file mode 100644 index 00000000000..11bb53ffee8 --- /dev/null +++ b/backend/api/v2/python_http_client/docs/PipelineTaskLogHistory.md @@ -0,0 +1,18 @@ +# PipelineTaskLogHistory + +Log references from an execution archived before a manual run retry. References can outlive pods or archived objects and do not guarantee availability. +## Properties +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**before_retry_generation** | **str** | Together with task_id, identifies this snapshot. This is the manual retry generation that archived it, not an automatic pod retry attempt number. | [optional] +**archived_at** | **datetime** | | [optional] +**start_time** | **datetime** | | [optional] +**end_time** | **datetime** | | [optional] +**state** | [**PipelineTaskTaskState**](PipelineTaskTaskState.md) | | [optional] +**pods** | [**list[PipelineTaskTaskPod]**](PipelineTaskTaskPod.md) | | [optional] +**log_artifacts** | [**list[InputOutputsIOArtifact]**](InputOutputsIOArtifact.md) | Log artifacts only; never includes task outputs used by downstream tasks. | [optional] +**driver_logs_uri** | **str** | Legacy driver log reference, when not represented by a log artifact. | [optional] + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/backend/api/v2/python_http_client/docs/V2PipelineTask.md b/backend/api/v2/python_http_client/docs/V2PipelineTask.md index 9873aea8111..6a225f0b90f 100644 --- a/backend/api/v2/python_http_client/docs/V2PipelineTask.md +++ b/backend/api/v2/python_http_client/docs/V2PipelineTask.md @@ -24,6 +24,7 @@ Name | Type | Description | Notes **scope_path** | **str** | | [optional] **lifecycle_message** | **str** | Latest pod lifecycle diagnostic from the execution engine. Read-only via REST; the persistence agent clears this field automatically on recovery. | [optional] **lifecycle_category** | **str** | Classified category of lifecycle_message. One of image-pull, scheduling, runtime, admission, or unknown. Empty when lifecycle_message is empty. | [optional] +**log_history** | [**list[PipelineTaskLogHistory]**](PipelineTaskLogHistory.md) | Output only. Prior executions archived by manual retries, oldest first. History begins when preservation is enabled; older retries are not backfilled. | [optional] [readonly] [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/backend/api/v2/run.proto b/backend/api/v2/run.proto index 8c11c08dd7c..04dd6959fc7 100644 --- a/backend/api/v2/run.proto +++ b/backend/api/v2/run.proto @@ -585,6 +585,27 @@ message PipelineTask { // One of image-pull, scheduling, runtime, admission, or unknown. // Empty when lifecycle_message is empty. optional string lifecycle_category = 20; + // Log references from an execution archived before a manual run retry. + // References can outlive pods or archived objects and do not guarantee availability. + message LogHistory { + // Together with task_id, identifies this snapshot. This is the manual retry + // generation that archived it, not an automatic pod retry attempt number. + int64 before_retry_generation = 1; + google.protobuf.Timestamp archived_at = 2; + google.protobuf.Timestamp start_time = 3; + google.protobuf.Timestamp end_time = 4; + TaskState state = 5; + repeated TaskPod pods = 6; + // Log artifacts only; never includes task outputs used by downstream tasks. + repeated InputOutputs.IOArtifact log_artifacts = 7; + // Legacy driver log reference, when not represented by a log artifact. + string driver_logs_uri = 8; + } + + // Output only. Prior executions archived by manual retries, oldest first. + // History begins when preservation is enabled; older retries are not backfilled. + repeated LogHistory log_history = 21; + } message CreateRunRequest { diff --git a/backend/api/v2/swagger/kfp_api_single_file.swagger.json b/backend/api/v2/swagger/kfp_api_single_file.swagger.json index cda52188343..c9209ca5d13 100644 --- a/backend/api/v2/swagger/kfp_api_single_file.swagger.json +++ b/backend/api/v2/swagger/kfp_api_single_file.swagger.json @@ -3347,6 +3347,51 @@ } } }, + "PipelineTaskLogHistory": { + "type": "object", + "properties": { + "before_retry_generation": { + "type": "string", + "format": "int64", + "description": "Together with task_id, identifies this snapshot. This is the manual retry\ngeneration that archived it, not an automatic pod retry attempt number." + }, + "archived_at": { + "type": "string", + "format": "date-time" + }, + "start_time": { + "type": "string", + "format": "date-time" + }, + "end_time": { + "type": "string", + "format": "date-time" + }, + "state": { + "$ref": "#/definitions/PipelineTaskTaskState" + }, + "pods": { + "type": "array", + "items": { + "type": "object", + "$ref": "#/definitions/PipelineTaskTaskPod" + } + }, + "log_artifacts": { + "type": "array", + "items": { + "type": "object", + "$ref": "#/definitions/InputOutputsIOArtifact" + }, + "description": "Log artifacts only; never includes task outputs used by downstream tasks." + }, + "driver_logs_uri": { + "type": "string", + "description": "Legacy driver log reference, when not represented by a log artifact." + } + }, + "description": "Log references from an execution archived before a manual run retry.\nReferences can outlive pods or archived objects and do not guarantee availability." + }, "PipelineTaskStatusMetadata": { "type": "object", "properties": { @@ -3636,6 +3681,15 @@ "lifecycle_category": { "type": "string", "description": "Classified category of lifecycle_message.\nOne of image-pull, scheduling, runtime, admission, or unknown.\nEmpty when lifecycle_message is empty." + }, + "log_history": { + "type": "array", + "items": { + "type": "object", + "$ref": "#/definitions/PipelineTaskLogHistory" + }, + "description": "Output only. Prior executions archived by manual retries, oldest first.\nHistory begins when preservation is enabled; older retries are not backfilled.", + "readOnly": true } }, "description": "Runtime information of a task execution used by task APIs and Run.tasks." diff --git a/backend/api/v2/swagger/run.swagger.json b/backend/api/v2/swagger/run.swagger.json index f7847fe8f7d..2003b4a4bce 100644 --- a/backend/api/v2/swagger/run.swagger.json +++ b/backend/api/v2/swagger/run.swagger.json @@ -758,6 +758,51 @@ } } }, + "PipelineTaskLogHistory": { + "type": "object", + "properties": { + "before_retry_generation": { + "type": "string", + "format": "int64", + "description": "Together with task_id, identifies this snapshot. This is the manual retry\ngeneration that archived it, not an automatic pod retry attempt number." + }, + "archived_at": { + "type": "string", + "format": "date-time" + }, + "start_time": { + "type": "string", + "format": "date-time" + }, + "end_time": { + "type": "string", + "format": "date-time" + }, + "state": { + "$ref": "#/definitions/PipelineTaskTaskState" + }, + "pods": { + "type": "array", + "items": { + "type": "object", + "$ref": "#/definitions/PipelineTaskTaskPod" + } + }, + "log_artifacts": { + "type": "array", + "items": { + "type": "object", + "$ref": "#/definitions/InputOutputsIOArtifact" + }, + "description": "Log artifacts only; never includes task outputs used by downstream tasks." + }, + "driver_logs_uri": { + "type": "string", + "description": "Legacy driver log reference, when not represented by a log artifact." + } + }, + "description": "Log references from an execution archived before a manual run retry.\nReferences can outlive pods or archived objects and do not guarantee availability." + }, "PipelineTaskStatusMetadata": { "type": "object", "properties": { @@ -1166,6 +1211,15 @@ "lifecycle_category": { "type": "string", "description": "Classified category of lifecycle_message.\nOne of image-pull, scheduling, runtime, admission, or unknown.\nEmpty when lifecycle_message is empty." + }, + "log_history": { + "type": "array", + "items": { + "type": "object", + "$ref": "#/definitions/PipelineTaskLogHistory" + }, + "description": "Output only. Prior executions archived by manual retries, oldest first.\nHistory begins when preservation is enabled; older retries are not backfilled.", + "readOnly": true } }, "description": "Runtime information of a task execution used by task APIs and Run.tasks." diff --git a/backend/api/v2beta1/go_client/aliases.go b/backend/api/v2beta1/go_client/aliases.go index 16f3db3a675..b2ce6498548 100644 --- a/backend/api/v2beta1/go_client/aliases.go +++ b/backend/api/v2beta1/go_client/aliases.go @@ -95,6 +95,7 @@ type ( PipelineTask_InputOutputs = canonical.PipelineTask_InputOutputs PipelineTask_InputOutputs_IOArtifact = canonical.PipelineTask_InputOutputs_IOArtifact PipelineTask_InputOutputs_IOParameter = canonical.PipelineTask_InputOutputs_IOParameter + PipelineTask_LogHistory = canonical.PipelineTask_LogHistory PipelineTask_StatusMetadata = canonical.PipelineTask_StatusMetadata PipelineTask_TaskPod = canonical.PipelineTask_TaskPod PipelineTask_TaskPodType = canonical.PipelineTask_TaskPodType diff --git a/backend/api/v2beta1/go_http_client/run_model/aliases.go b/backend/api/v2beta1/go_http_client/run_model/aliases.go index b4008c43bee..7b9058ae38a 100644 --- a/backend/api/v2beta1/go_http_client/run_model/aliases.go +++ b/backend/api/v2beta1/go_http_client/run_model/aliases.go @@ -16,6 +16,7 @@ type ( MetadataValueRenderType = canonical.MetadataValueRenderType PipelineTaskChildTask = canonical.PipelineTaskChildTask PipelineTaskInputOutputs = canonical.PipelineTaskInputOutputs + PipelineTaskLogHistory = canonical.PipelineTaskLogHistory PipelineTaskStatusMetadata = canonical.PipelineTaskStatusMetadata PipelineTaskTaskPod = canonical.PipelineTaskTaskPod PipelineTaskTaskPodType = canonical.PipelineTaskTaskPodType diff --git a/backend/src/apiserver/client_manager/client_manager.go b/backend/src/apiserver/client_manager/client_manager.go index f867866c91d..7379831e1ed 100644 --- a/backend/src/apiserver/client_manager/client_manager.go +++ b/backend/src/apiserver/client_manager/client_manager.go @@ -910,7 +910,18 @@ func runLegacyUpgradeFlow(db *gorm.DB, dialect sqldrv.DBDialect) error { func autoMigrate(db *gorm.DB) error { glog.Infof("Running AutoMigrate.") - if err := db.AutoMigrate(model.AllModels()...); err != nil { + if err := createTaskLogHistory(db); err != nil { + return err + } + // The immutable-history table has its own coordinated create-only migration. + // Never let broad reconciliation alter it on subsequent startups. + models := make([]any, 0, len(model.AllModels())) + for _, entry := range model.AllModels() { + if _, history := entry.(*model.TaskLogHistory); !history { + models = append(models, entry) + } + } + if err := db.AutoMigrate(models...); err != nil { return fmt.Errorf("AutoMigrate failed: %w", err) } diff --git a/backend/src/apiserver/client_manager/task_log_history_migration.go b/backend/src/apiserver/client_manager/task_log_history_migration.go new file mode 100644 index 00000000000..cbe2a28a1c7 --- /dev/null +++ b/backend/src/apiserver/client_manager/task_log_history_migration.go @@ -0,0 +1,136 @@ +// Copyright 2026 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clientmanager + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "time" + + "github.com/kubeflow/pipelines/backend/src/apiserver/model" + "gorm.io/gorm" +) + +const taskLogHistoryMigrationLock = "kfp_task_log_history_v1" + +// createTaskLogHistory creates only the new standalone table and its index. It +// never reconciles an existing table, adds foreign keys, or backfills old rows. +// Startup fails on contention rather than waiting behind an unbounded DDL lock. +func createTaskLogHistory(db *gorm.DB) (result error) { + ctx, cancel := context.WithTimeout(db.Statement.Context, 10*time.Second) + defer cancel() + if db.Name() == "sqlite" { + if db.Migrator().HasTable(&model.TaskLogHistory{}) { + return nil + } + return db.WithContext(ctx).Migrator().CreateTable(&model.TaskLogHistory{}) + } + pool, err := db.DB() + if err != nil { + return err + } + conn, err := pool.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + session := db.Session(&gorm.Session{NewDB: true}).WithContext(ctx) + session.Statement.ConnPool = conn + + // Cleanup uses a fresh bounded context even if startup was canceled. If + // session settings/ownership cannot be restored, discard this connection. + var cleanup func(context.Context) error + defer func() { + if cleanup == nil { + return + } + cleanupCtx, stop := context.WithTimeout(context.Background(), 2*time.Second) + defer stop() + if err := cleanup(cleanupCtx); err != nil { + _ = conn.Raw(func(any) error { return driver.ErrBadConn }) + result = errors.Join(result, fmt.Errorf("release retry log migration session: %w", err)) + } + }() + switch db.Name() { + case "mysql": + cleanup = func(ctx context.Context) error { + var released sql.NullInt64 + return conn.QueryRowContext(ctx, "SELECT RELEASE_LOCK(SHA2(CONCAT(DATABASE(), ?), 256))", taskLogHistoryMigrationLock).Scan(&released) + } + var acquired int + if err := conn.QueryRowContext(ctx, "SELECT GET_LOCK(SHA2(CONCAT(DATABASE(), ?), 256), 0)", taskLogHistoryMigrationLock).Scan(&acquired); err != nil { + return err + } + if acquired != 1 { + return fmt.Errorf("retry log history migration is owned by another API server; retry startup") + } + var previous int + if err := conn.QueryRowContext(ctx, "SELECT @@SESSION.lock_wait_timeout").Scan(&previous); err != nil { + return err + } + cleanup = func(ctx context.Context) error { + _, restore := conn.ExecContext(ctx, fmt.Sprintf("SET SESSION lock_wait_timeout = %d", previous)) + var released int + release := conn.QueryRowContext(ctx, "SELECT RELEASE_LOCK(SHA2(CONCAT(DATABASE(), ?), 256))", taskLogHistoryMigrationLock).Scan(&released) + return errors.Join(restore, release) + } + if _, err := conn.ExecContext(ctx, "SET SESSION lock_wait_timeout = 1"); err != nil { + return err + } + case "postgres": + cleanup = func(ctx context.Context) error { + var released bool + return conn.QueryRowContext(ctx, "SELECT pg_advisory_unlock(hashtext($1))", taskLogHistoryMigrationLock).Scan(&released) + } + var acquired bool + if err := conn.QueryRowContext(ctx, "SELECT pg_try_advisory_lock(hashtext($1))", taskLogHistoryMigrationLock).Scan(&acquired); err != nil { + return err + } + if !acquired { + return fmt.Errorf("retry log history migration is owned by another API server; retry startup") + } + var previous string + if err := conn.QueryRowContext(ctx, "SHOW lock_timeout").Scan(&previous); err != nil { + return err + } + cleanup = func(ctx context.Context) error { + _, restore := conn.ExecContext(ctx, "SELECT set_config('lock_timeout', $1, false)", previous) + var released bool + release := conn.QueryRowContext(ctx, "SELECT pg_advisory_unlock(hashtext($1))", taskLogHistoryMigrationLock).Scan(&released) + return errors.Join(restore, release) + } + if _, err := conn.ExecContext(ctx, "SET lock_timeout = '1s'"); err != nil { + return err + } + default: + return fmt.Errorf("retry log history migration does not support database %q", db.Name()) + } + if session.Migrator().HasTable(&model.TaskLogHistory{}) { + return nil + } + create := func(tx *gorm.DB) error { return tx.Migrator().CreateTable(&model.TaskLogHistory{}) } + if db.Name() == "postgres" { + err = session.Transaction(create) + } else { + err = create(session) + } + if err != nil { + return fmt.Errorf("create retry log history table (bounded lock wait; retry startup after resolving contention): %w", err) + } + return nil +} diff --git a/backend/src/apiserver/client_manager/task_log_history_migration_test.go b/backend/src/apiserver/client_manager/task_log_history_migration_test.go new file mode 100644 index 00000000000..ab4bddc727d --- /dev/null +++ b/backend/src/apiserver/client_manager/task_log_history_migration_test.go @@ -0,0 +1,281 @@ +// Copyright 2026 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clientmanager + +import ( + "context" + "database/sql" + "fmt" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + mysqldriver "github.com/go-sql-driver/mysql" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/stdlib" + "github.com/kubeflow/pipelines/backend/src/apiserver/model" + "github.com/stretchr/testify/require" + "gorm.io/driver/mysql" + "gorm.io/driver/postgres" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +type retryMigrationDDL struct { + logger.Interface + mu sync.Mutex + statements []string +} + +func (l *retryMigrationDDL) Trace(ctx context.Context, started time.Time, fc func() (string, int64), err error) { + sql, _ := fc() + if strings.Contains(sql, "task_log_history") && (strings.HasPrefix(sql, "CREATE ") || strings.HasPrefix(sql, "ALTER ")) { + l.mu.Lock() + l.statements = append(l.statements, sql) + l.mu.Unlock() + } + l.Interface.Trace(ctx, started, fc, err) +} + +func retryHistoryDatabase(t *testing.T, engine string) *gorm.DB { + t.Helper() + env := "KFP_RECURRING_" + strings.ToUpper(engine) + "_TEST_DSN" + dsn := os.Getenv(env) + if dsn == "" { + t.Skip("set " + env + " for production-engine migration coverage") + } + name := fmt.Sprintf("retry_history_%d", time.Now().UnixNano()) + var admin, pool *sql.DB + var dialector gorm.Dialector + if engine == "mysql" { + config, err := mysqldriver.ParseDSN(dsn) + require.NoError(t, err) + config.DBName = "" + admin, err = sql.Open("mysql", config.FormatDSN()) + require.NoError(t, err) + _, err = admin.Exec("CREATE DATABASE " + name) + require.NoError(t, err) + t.Cleanup(func() { _, err := admin.Exec("DROP DATABASE " + name); require.NoError(t, err); admin.Close() }) + config.DBName = name + pool, err = sql.Open("mysql", config.FormatDSN()) + require.NoError(t, err) + dialector = mysql.New(mysql.Config{Conn: pool}) + } else { + config, err := pgx.ParseConfig(dsn) + require.NoError(t, err) + admin = stdlib.OpenDB(*config) + _, err = admin.Exec("CREATE SCHEMA " + name) + require.NoError(t, err) + t.Cleanup(func() { + _, err := admin.Exec("DROP SCHEMA " + name + " CASCADE") + require.NoError(t, err) + admin.Close() + }) + config.RuntimeParams["search_path"] = name + pool = stdlib.OpenDB(*config) + dialector = postgres.New(postgres.Config{Conn: pool}) + } + pool.SetMaxOpenConns(8) + t.Cleanup(func() { require.NoError(t, pool.Close()) }) + db, err := gorm.Open(dialector, &gorm.Config{}) + require.NoError(t, err) + return db +} + +func TestRetryLogHistoryMigrationProductionDatabases(t *testing.T) { + for _, engine := range []string{"mysql", "postgres"} { + t.Run(engine, func(t *testing.T) { + db := retryHistoryDatabase(t, engine) + // Construct the immediately preceding schema, without the new model. + prior := []any{} + for _, entry := range model.AllModels() { + if _, history := entry.(*model.TaskLogHistory); !history { + prior = append(prior, entry) + } + } + require.NoError(t, db.AutoMigrate(prior...)) + require.NoError(t, db.Create(&model.Experiment{UUID: "prior-experiment", Name: "existing experiment", Namespace: "test"}).Error) + require.NoError(t, db.Create(&model.Run{UUID: "prior-run", ExperimentId: "prior-experiment", Namespace: "test", DisplayName: "existing run"}).Error) + longScope := strings.Repeat("long-existing-task-scope/", 64) + for i := 0; i < 20; i++ { + require.NoError(t, db.Table("tasks").Create(map[string]any{"UUID": fmt.Sprintf("prior-task-%d", i), "RunUUID": "prior-run", "Namespace": "test", "pods": "[]", "TypeAttrs": "{}", "ScopePath": longScope, "CreatedAtInSec": int64(0), "Fingerprint": "", "State": 0, "Type": 0}).Error) + } + + require.NoError(t, db.Exec("CREATE TABLE migration_live_requests (id INTEGER PRIMARY KEY, payload TEXT NOT NULL, writes INTEGER NOT NULL)").Error) + tableIdentity := func() string { + var identity string + if engine == "mysql" { + require.NoError(t, db.Raw("SELECT TABLE_ID FROM information_schema.INNODB_TABLES WHERE NAME=CONCAT(DATABASE(), '/tasks')").Scan(&identity).Error) + } else { + require.NoError(t, db.Raw("SELECT 'tasks'::regclass::oid::text").Scan(&identity).Error) + } + return identity + } + beforeIdentity := tableIdentity() + require.NotEmpty(t, beforeIdentity) + payload := strings.Repeat("historical-value-", 64) + for i := 0; i < 100; i++ { + require.NoError(t, db.Exec("INSERT INTO migration_live_requests VALUES (?, ?, 0)", i, payload).Error) + } + var active atomic.Bool + var completed atomic.Int64 + var maxLatency atomic.Int64 + stop := make(chan struct{}) + workerErrors := make(chan error, 1) + var worker sync.WaitGroup + active.Store(true) + worker.Add(1) + go func() { + defer worker.Done() + for { + select { + case <-stop: + return + default: + } + started := time.Now() + if err := db.Model(&model.Task{}).Where(map[string]any{"UUID": "prior-task-0"}).UpdateColumn("FinishedInSec", gorm.Expr("?", int64(42))).Error; err != nil { + workerErrors <- err + return + } + if err := db.Exec("UPDATE migration_live_requests SET writes=writes+1 WHERE id=0").Error; err != nil { + workerErrors <- err + return + } + var value string + if err := db.Raw("SELECT payload FROM migration_live_requests WHERE id=0").Scan(&value).Error; err != nil { + workerErrors <- err + return + } + if value != payload { + workerErrors <- fmt.Errorf("prior data changed") + return + } + if active.Load() { + completed.Add(1) + } + latency := time.Since(started).Nanoseconds() + for old := maxLatency.Load(); latency > old; old = maxLatency.Load() { + if maxLatency.CompareAndSwap(old, latency) { + break + } + } + } + }() + ddl := &retryMigrationDDL{Interface: db.Logger} + migrationErr := autoMigrate(db.Session(&gorm.Session{Logger: ddl})) + active.Store(false) + close(stop) + worker.Wait() + require.NoError(t, migrationErr) + require.NotEmpty(t, ddl.statements) + for _, statement := range ddl.statements { + require.NotContains(t, strings.ToUpper(statement), "ALTER TABLE") + require.NotContains(t, strings.ToUpper(statement), "FOREIGN KEY") + t.Logf("new-table DDL: %s", statement) + } + select { + case err := <-workerErrors: + require.NoError(t, err) + default: + } + require.Positive(t, completed.Load(), "requests must finish during the actual startup migration") + require.Equal(t, beforeIdentity, tableIdentity(), "existing tasks table must not be rebuilt") + t.Logf("concurrent read/write pairs=%d maximum pair latency=%s; prior fixture rows=100 payload bytes=%d", completed.Load(), time.Duration(maxLatency.Load()), len(payload)) + var priorTask model.Task + require.NoError(t, db.Where(map[string]any{"UUID": "prior-task-0"}).First(&priorTask).Error) + require.Equal(t, longScope, priorTask.ScopePath) + require.True(t, db.Migrator().HasTable(&model.TaskLogHistory{})) + require.True(t, db.Migrator().HasIndex(&model.TaskLogHistory{}, "idx_task_log_history_run")) + require.NoError(t, autoMigrate(db)) + require.NoError(t, db.Create(&model.TaskLogHistory{TaskID: "task", BeforeRetryGeneration: 1, RunUUID: "run", Snapshot: model.LargeText("retained")}).Error) + require.NoError(t, autoMigrate(db)) + var retained model.TaskLogHistory + require.NoError(t, db.First(&retained).Error) + require.Equal(t, model.LargeText("retained"), retained.Snapshot) + t.Run("competing startup creation", func(t *testing.T) { + require.NoError(t, db.Migrator().DropTable(&model.TaskLogHistory{})) + start := make(chan struct{}) + results := make(chan error, 2) + for range 2 { + go func() { <-start; results <- createTaskLogHistory(db) }() + } + close(start) + successes := 0 + for range 2 { + err := <-results + if err == nil { + successes++ + } else { + require.ErrorContains(t, err, "owned by another") + } + } + require.Positive(t, successes) + require.NoError(t, createTaskLogHistory(db)) + require.True(t, db.Migrator().HasIndex(&model.TaskLogHistory{}, "idx_task_log_history_run")) + }) + if engine == "postgres" { + t.Run("blocked DDL timeout and recovery", func(t *testing.T) { + require.NoError(t, db.Migrator().DropTable(&model.TaskLogHistory{})) + pool, err := db.DB() + require.NoError(t, err) + owner, err := pool.Begin() + require.NoError(t, err) + _, err = owner.Exec("CREATE TABLE task_log_history (blocked INTEGER)") + require.NoError(t, err) + started := time.Now() + migrationErr := createTaskLogHistory(db) + require.Error(t, migrationErr) + require.Less(t, time.Since(started), 4*time.Second) + require.NoError(t, owner.Rollback()) + require.NoError(t, createTaskLogHistory(db)) + require.True(t, db.Migrator().HasIndex(&model.TaskLogHistory{}, "idx_task_log_history_run")) + }) + } + t.Run("competing owner and cancellation", func(t *testing.T) { + pool, err := db.DB() + require.NoError(t, err) + owner, err := pool.Conn(context.Background()) + require.NoError(t, err) + defer owner.Close() + if engine == "mysql" { + var got int + require.NoError(t, owner.QueryRowContext(context.Background(), "SELECT GET_LOCK(SHA2(CONCAT(DATABASE(), ?), 256),0)", taskLogHistoryMigrationLock).Scan(&got)) + require.Equal(t, 1, got) + } else { + var got bool + require.NoError(t, owner.QueryRowContext(context.Background(), "SELECT pg_try_advisory_lock(hashtext($1))", taskLogHistoryMigrationLock).Scan(&got)) + require.True(t, got) + } + started := time.Now() + require.ErrorContains(t, createTaskLogHistory(db), "owned by another") + require.Less(t, time.Since(started), 3*time.Second) + if engine == "mysql" { + _, err = owner.ExecContext(context.Background(), "SELECT RELEASE_LOCK(SHA2(CONCAT(DATABASE(), ?), 256))", taskLogHistoryMigrationLock) + } else { + _, err = owner.ExecContext(context.Background(), "SELECT pg_advisory_unlock(hashtext($1))", taskLogHistoryMigrationLock) + } + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.Error(t, createTaskLogHistory(db.WithContext(ctx))) + require.NoError(t, createTaskLogHistory(db)) + }) + }) + } +} diff --git a/backend/src/apiserver/model/common.go b/backend/src/apiserver/model/common.go index 19d31718ba3..3e6d634b3fe 100644 --- a/backend/src/apiserver/model/common.go +++ b/backend/src/apiserver/model/common.go @@ -87,6 +87,7 @@ func AllModels() []any { &Run{}, &RunMetricV1{}, &Task{}, + &TaskLogHistory{}, &Artifact{}, &ArtifactTask{}, &ResourceReference{}, diff --git a/backend/src/apiserver/model/task.go b/backend/src/apiserver/model/task.go index 7385da822c8..e259fbd3e30 100644 --- a/backend/src/apiserver/model/task.go +++ b/backend/src/apiserver/model/task.go @@ -163,8 +163,9 @@ type Task struct { LifecycleCategory *string `gorm:"column:LifecycleCategory; type:varchar(32); default:null;"` // Transient fields populated during hydration (not stored in DB) - InputArtifactsHydrated []TaskArtifactHydrated `gorm:"-"` - OutputArtifactsHydrated []TaskArtifactHydrated `gorm:"-"` + LogHistory []*apiv2.PipelineTask_LogHistory `gorm:"-"` + InputArtifactsHydrated []TaskArtifactHydrated `gorm:"-"` + OutputArtifactsHydrated []TaskArtifactHydrated `gorm:"-"` } func (t Task) ToString() string { diff --git a/backend/src/apiserver/model/task_log_history.go b/backend/src/apiserver/model/task_log_history.go new file mode 100644 index 00000000000..18a93126f68 --- /dev/null +++ b/backend/src/apiserver/model/task_log_history.go @@ -0,0 +1,28 @@ +// Copyright 2026 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +// TaskLogHistory retains only troubleshooting references from a task before a +// manual retry resets it. Deliberately no foreign keys: introducing this table +// must not alter or lock existing task/run tables during schema migration. +// Run deletion removes these rows in the same transaction as the tasks. +type TaskLogHistory struct { + TaskID string `gorm:"column:TaskID;type:varchar(191);primaryKey;not null"` + BeforeRetryGeneration int64 `gorm:"column:BeforeRetryGeneration;primaryKey;autoIncrement:false;not null"` + RunUUID string `gorm:"column:RunUUID;type:varchar(191);not null;index:idx_task_log_history_run"` + Snapshot LargeText `gorm:"column:Snapshot;not null"` +} + +func (TaskLogHistory) TableName() string { return "task_log_history" } diff --git a/backend/src/apiserver/resource/resource_manager.go b/backend/src/apiserver/resource/resource_manager.go index edcdaa4834e..16d090e7cfc 100644 --- a/backend/src/apiserver/resource/resource_manager.go +++ b/backend/src/apiserver/resource/resource_manager.go @@ -1644,24 +1644,9 @@ func (r *ResourceManager) resetRetriedTaskState(run *model.Run) error { return nil } - // Logical task identity stays stable within a run so duplicate CreateTask - // delivery still resolves to the existing row. Clear only the previous - // attempt's transient task state before Argo resumes so those rows can - // safely represent the new attempt. - if err := r.artifactTaskStore.DeleteOutputArtifactTasksByTaskIDs(taskIDsToReset); err != nil { - return err - } - // Input links are also attempt-local: leaving them in place causes - // CreateArtifactTasks UniqueLink conflicts when the retried driver recreates - // the same (artifact, task, key, type) input rows. - if err := r.artifactTaskStore.DeleteInputArtifactTasksByTaskIDs(taskIDsToReset); err != nil { - return err - } - - // Output parameters and output artifact links are attempt-local. Resetting - // them here prevents a retried task from exposing stale failed-attempt - // outputs while leaving successful sibling results intact. - return r.taskStore.ResetTasksForRetry(taskIDsToReset) + // Preserve log references, unlink attempt-local artifacts, and reset state in + // one generation-fenced transaction before the workflow resumes. + return r.taskStore.ResetTasksForRetry(run.UUID, run.RetryGeneration, taskIDsToReset) } func shouldPreserveTaskAcrossRetry(task *model.Task) bool { diff --git a/backend/src/apiserver/server/api_converter.go b/backend/src/apiserver/server/api_converter.go index cee2a5b6fae..1e8e4ffd021 100644 --- a/backend/src/apiserver/server/api_converter.go +++ b/backend/src/apiserver/server/api_converter.go @@ -1872,6 +1872,7 @@ func toAPITask(modelTask *model.Task, childTasks []*model.Task) (*apiv2.Pipeline Name: modelTask.Name, DisplayName: modelTask.DisplayName, CacheFingerprint: modelTask.Fingerprint, + LogHistory: modelTask.LogHistory, Inputs: &apiv2.PipelineTask_InputOutputs{}, Outputs: &apiv2.PipelineTask_InputOutputs{}, } diff --git a/backend/src/apiserver/server/api_converter_test.go b/backend/src/apiserver/server/api_converter_test.go index 13c4b554d66..41a8fc62c37 100644 --- a/backend/src/apiserver/server/api_converter_test.go +++ b/backend/src/apiserver/server/api_converter_test.go @@ -1252,6 +1252,34 @@ func Test_toAPIRuntimeStatuses(t *testing.T) { assert.Equal(t, expected, got) } +func TestToModelAndAPITask_LogHistory(t *testing.T) { + history := []*apiv2.PipelineTask_LogHistory{{ + BeforeRetryGeneration: 2, + ArchivedAt: ×tamppb.Timestamp{Seconds: 200}, + StartTime: ×tamppb.Timestamp{Seconds: 100}, + EndTime: ×tamppb.Timestamp{Seconds: 150}, + State: apiv2.PipelineTask_FAILED, + Pods: []*apiv2.PipelineTask_TaskPod{{Name: "old-executor", Uid: "old-uid", Type: apiv2.PipelineTask_EXECUTOR}}, + LogArtifacts: []*apiv2.PipelineTask_InputOutputs_IOArtifact{{ArtifactKey: "executor-logs"}}, + DriverLogsUri: "gs://logs/old-driver.log", + }} + task := &model.Task{UUID: "task-1", RunUUID: "run-1", LogHistory: history} + exported, err := toAPITask(task, nil) + require.NoError(t, err) + assert.Equal(t, history, exported.GetLogHistory()) + assert.Empty(t, exported.GetPods()) + assert.Empty(t, exported.GetOutputs().GetArtifacts(), "historical logs must not become current outputs") + + // Clients cannot inject server-owned history through task create/update input. + input, err := toModelTask(exported) + require.NoError(t, err) + assert.Empty(t, input.LogHistory) + + empty, err := toAPITask(&model.Task{UUID: "legacy-task"}, nil) + require.NoError(t, err) + assert.Empty(t, empty.GetLogHistory(), "pre-upgrade tasks have no synthetic history") +} + func TestToModelAndAPITask_LifecycleMessage(t *testing.T) { msg := "ImagePullBackOff" apiTask := &apiv2.PipelineTask{ diff --git a/backend/src/apiserver/server/run_server_retry_history_test.go b/backend/src/apiserver/server/run_server_retry_history_test.go new file mode 100644 index 00000000000..2c2c41a67c3 --- /dev/null +++ b/backend/src/apiserver/server/run_server_retry_history_test.go @@ -0,0 +1,67 @@ +// Copyright 2026 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "testing" + + apiv2 "github.com/kubeflow/pipelines/backend/api/v2/go_client" + "github.com/kubeflow/pipelines/backend/src/apiserver/client" + "github.com/kubeflow/pipelines/backend/src/apiserver/common" + "github.com/kubeflow/pipelines/backend/src/apiserver/model" + "github.com/kubeflow/pipelines/backend/src/apiserver/resource" + "github.com/kubeflow/pipelines/backend/src/common/util" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" +) + +func TestRetryLogHistoryUsesRunAuthorization(t *testing.T) { + viper.Set(common.MultiUserMode, "true") + t.Cleanup(func() { viper.Set(common.MultiUserMode, "false") }) + clients, manager, run := initWithOneTimeRunV2(t) + defer clients.Close() + task, err := clients.TaskStore().CreateTask(&model.Task{ + RunUUID: run.UUID, Namespace: run.Namespace, Name: "failed-task", + State: model.TaskStatus(apiv2.PipelineTask_FAILED), + StatusMetadata: model.JSONData{"customProperties": map[string]interface{}{"driver_logs_uri": "gs://private/run/driver-logs"}}, + }) + require.NoError(t, err) + _, err = clients.DB().Exec(`UPDATE run_details SET RetryGeneration = 1 WHERE UUID = ?`, run.UUID) + require.NoError(t, err) + require.NoError(t, clients.TaskStore().ResetTasksForRetry(run.UUID, 1, []string{task.UUID})) + ctx := ctxWithUser() + server := createRunServer(manager) + got, err := server.GetTask(ctx, &apiv2.GetTaskRequest{RunId: run.UUID, TaskId: task.UUID}) + require.NoError(t, err) + require.Len(t, got.GetLogHistory(), 1) + require.Equal(t, "gs://private/run/driver-logs", got.GetLogHistory()[0].GetDriverLogsUri()) + require.Empty(t, got.GetStatusMetadata()) + + // Knowing a task ID must not allow fetching its history through another run. + got, err = server.GetTask(ctx, &apiv2.GetTaskRequest{RunId: "another-run", TaskId: task.UUID}) + require.Error(t, err) + require.Nil(t, got) + + clients.SubjectAccessReviewClientFake = client.NewFakeSubjectAccessReviewClientUnauthorized() + manager = resource.NewResourceManager(clients, &resource.ResourceManagerOptions{CollectMetrics: false}) + server = createRunServer(manager) + got, err = server.GetTask(ctx, &apiv2.GetTaskRequest{RunId: run.UUID, TaskId: task.UUID}) + require.True(t, util.IsUserErrorCodeMatch(err, codes.PermissionDenied), "%v", err) + require.Nil(t, got) + listed, err := server.ListTasks(ctx, &apiv2.ListTasksRequest{RunId: run.UUID}) + require.True(t, util.IsUserErrorCodeMatch(err, codes.PermissionDenied), "%v", err) + require.Nil(t, listed) +} diff --git a/backend/src/apiserver/storage/native_postgres_integration_test.go b/backend/src/apiserver/storage/native_postgres_integration_test.go index 4c74c6515dd..9dd3706c7b4 100644 --- a/backend/src/apiserver/storage/native_postgres_integration_test.go +++ b/backend/src/apiserver/storage/native_postgres_integration_test.go @@ -131,9 +131,9 @@ func TestNativeStoresPostgreSQL(t *testing.T) { require.Len(t, linkPage, 1) require.Equal(t, 1, total) - _, err = tasks.UpdateTask(&model.Task{UUID: task.UUID, DisplayName: "Updated"}) + _, err = tasks.UpdateTask(&model.Task{UUID: task.UUID, DisplayName: "Updated", State: model.TaskStatus(apiv2.PipelineTask_FAILED)}) require.NoError(t, err) - require.NoError(t, tasks.ResetTasksForRetry([]string{task.UUID})) + require.NoError(t, tasks.ResetTasksForRetry("pg-run", 0, []string{task.UUID})) loaded, err = tasks.GetTask(task.UUID) require.NoError(t, err) require.Equal(t, model.TaskStatus(apiv2.PipelineTask_RUNNING), loaded.State) diff --git a/backend/src/apiserver/storage/task_store.go b/backend/src/apiserver/storage/task_store.go index fd74249e998..61f3b4a4b27 100644 --- a/backend/src/apiserver/storage/task_store.go +++ b/backend/src/apiserver/storage/task_store.go @@ -32,6 +32,7 @@ import ( "github.com/kubeflow/pipelines/backend/src/apiserver/list" "github.com/kubeflow/pipelines/backend/src/apiserver/model" "github.com/kubeflow/pipelines/backend/src/common/util" + "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/structpb" "google.golang.org/protobuf/types/known/timestamppb" @@ -89,7 +90,7 @@ type TaskStoreInterface interface { // ResetTasksForRetry clears attempt-local state so retried tasks can resume // from a clean RUNNING attempt while preserving task history. - ResetTasksForRetry(taskIDs []string) error + ResetTasksForRetry(runUUID string, retryGeneration int64, taskIDs []string) error // GetChildTasks Fetches all child tasks for a given task UUID. GetChildTasks(taskID string) ([]*model.Task, error) @@ -260,7 +261,11 @@ func scanTaskRow(rowscanner interface{ Scan(dest ...any) error }) (*model.Task, // hydrateArtifactsForTasks fills InputArtifactsHydrated and OutputArtifactsHydrated for provided tasks by // querying artifact_tasks joined with artifacts. It uses TaskID IN (...) to limit scope. // The helper clears previously hydrated transient slices first so repeated hydration is idempotent. -func hydrateArtifactsForTasks(db *sql.DB, tasks []*model.Task, d dialect.DBDialect) error { +type taskQueryer interface { + Query(string, ...any) (*sql.Rows, error) +} + +func hydrateArtifactsForTasks(db taskQueryer, tasks []*model.Task, d dialect.DBDialect) error { q := d.QuoteIdentifier qb := d.QueryBuilder() if len(tasks) == 0 { @@ -424,7 +429,13 @@ func hydrateArtifactsForTasks(db *sql.DB, tasks []*model.Task, d dialect.DBDiale task.InputArtifactsHydrated = append(task.InputArtifactsHydrated, h) } } - return rows.Err() + if err := rows.Err(); err != nil { + return err + } + if err := rows.Close(); err != nil { + return err + } + return hydrateTaskLogHistory(db, tasks, d) } func iOTypeIsOutput(ioType apiv2.IOType) (bool, error) { @@ -1287,7 +1298,7 @@ func (s *TaskStore) UpdateTask(new *model.Task) (*model.Task, error) { return s.GetTask(new.UUID) } -func (s *TaskStore) ResetTasksForRetry(taskIDs []string) error { +func (s *TaskStore) ResetTasksForRetry(runUUID string, retryGeneration int64, taskIDs []string) error { q := s.dbDialect.QuoteIdentifier qb := s.dbDialect.QueryBuilder() if len(taskIDs) == 0 { @@ -1304,10 +1315,23 @@ func (s *TaskStore) ResetTasksForRetry(taskIDs []string) error { } }() + // Lock the run first so an older retry claimant cannot reset a newer attempt. + runSQL, runArgs, err := qb.Select(q("RetryGeneration")).From(q("run_details")).Where(sq.Eq{q("UUID"): runUUID}).ToSql() + if err != nil { + return err + } + var generation int64 + if err := tx.QueryRow(s.dbDialect.SelectForUpdate(runSQL), runArgs...).Scan(&generation); err != nil { + return err + } + if generation != retryGeneration { + return util.NewFailedPreconditionError(fmt.Errorf("retry generation changed"), "Retry task reset is stale") + } + sqlStr, args, err := qb. Select(dialect.QuoteAll(q, taskColumns)...). From(q(tableName)). - Where(sq.Eq{q("UUID"): taskIDs}). + Where(sq.Eq{q("UUID"): taskIDs, q("RunUUID"): runUUID}). OrderBy(q("CreatedAtInSec")+" ASC", q("UUID")+" ASC"). ToSql() if err != nil { @@ -1332,9 +1356,73 @@ func (s *TaskStore) ResetTasksForRetry(taskIDs []string) error { return util.NewInternalServerError(err, "Failed while reading retry tasks for reset: %v", err.Error()) } + if err := rows.Close(); err != nil { + return err + } + if len(tasks) != len(taskIDs) { + return fmt.Errorf("retry task does not belong to run %s", runUUID) + } + if err := hydrateArtifactsForTasks(tx, tasks, s.dbDialect); err != nil { + return err + } + // Read idempotency markers separately: history without log references is + // intentionally omitted from API hydration, but must still fence redelivery. + markerSQL, markerArgs, err := qb.Select(q("TaskID")).From(q("task_log_history")).Where(sq.Eq{q("RunUUID"): runUUID, q("BeforeRetryGeneration"): retryGeneration}).ToSql() + if err != nil { + return err + } + markerRows, err := tx.Query(markerSQL, markerArgs...) + if err != nil { + return err + } + resetTasks := map[string]bool{} + for markerRows.Next() { + var id string + if err := markerRows.Scan(&id); err != nil { + markerRows.Close() + return err + } + resetTasks[id] = true + } + markerErr := markerRows.Err() + markerRows.Close() + if markerErr != nil { + return markerErr + } retryStartedAt := s.time.Now().Unix() emptyJSONArray := "[]" for _, task := range tasks { + + // A committed snapshot is the idempotency marker for this retry boundary. + if resetTasks[task.UUID] { + continue + } + switch task.State { + case model.TaskStatus(apiv2.PipelineTask_SUCCEEDED), model.TaskStatus(apiv2.PipelineTask_CACHED), model.TaskStatus(apiv2.PipelineTask_SKIPPED): + continue + } + snapshot, err := retryLogSnapshot(task, retryGeneration, retryStartedAt) + if err != nil { + return err + } + encoded, err := protojson.Marshal(snapshot) + if err != nil { + return err + } + insertSQL, insertArgs, err := qb.Insert(q("task_log_history")).Columns(q("TaskID"), q("BeforeRetryGeneration"), q("RunUUID"), q("Snapshot")).Values(task.UUID, retryGeneration, runUUID, string(encoded)).ToSql() + if err != nil { + return err + } + if _, err := tx.Exec(insertSQL, insertArgs...); err != nil { + return err + } + deleteSQL, deleteArgs, err := qb.Delete(q("artifact_tasks")).Where(sq.Eq{q("TaskID"): task.UUID, q("RunUUID"): runUUID}).ToSql() + if err != nil { + return err + } + if _, err := tx.Exec(deleteSQL, deleteArgs...); err != nil { + return err + } mergedHistory := task.StateHistory if len(mergedHistory) == 0 || getLastTaskState(mergedHistory) != model.TaskStatus(apiv2.PipelineTask_RUNNING) { taskStatus := &apiv2.PipelineTask_TaskStatus{ @@ -1670,19 +1758,34 @@ func getLastTaskState(history model.JSONSlice) model.TaskStatus { // DeleteTasksForRun deletes all tasks associated with a specific run. // This should be called before deleting a run to avoid foreign key constraint violations. func (s *TaskStore) DeleteTasksForRun(tx *sql.Tx, runUUID string) error { + if tx == nil { + ownTx, err := s.db.Begin() + if err != nil { + return err + } + defer ownTx.Rollback() + if err := s.DeleteTasksForRun(ownTx, runUUID); err != nil { + return err + } + return ownTx.Commit() + } + q := s.dbDialect.QuoteIdentifier qb := s.dbDialect.QueryBuilder() + historySQL, historyArgs, err := qb.Delete(q("task_log_history")).Where(sq.Eq{q("RunUUID"): runUUID}).ToSql() + if err != nil { + return err + } + if _, err := tx.Exec(historySQL, historyArgs...); err != nil { + return err + } + deleteSQL, deleteArgs, err := qb.Delete(q(tableName)).Where(sq.Eq{q("RunUUID"): runUUID}).ToSql() if err != nil { return util.NewInternalServerError(err, "Failed to create query to delete tasks for run: %s", runUUID) } - var result sql.Result - if tx != nil { - result, err = tx.Exec(deleteSQL, deleteArgs...) - } else { - result, err = s.db.Exec(deleteSQL, deleteArgs...) - } + result, err := tx.Exec(deleteSQL, deleteArgs...) if err != nil { return util.NewInternalServerError(err, "Failed to delete tasks for run %s from table", runUUID) @@ -1697,3 +1800,93 @@ func (s *TaskStore) DeleteTasksForRun(tx *sql.Tx, runUUID string) error { return nil } + +// hydrateTaskLogHistory is shared by task and run reads, avoiding one history +// query per task. Snapshot paths are immutable references, never live outputs. +func hydrateTaskLogHistory(db taskQueryer, tasks []*model.Task, d dialect.DBDialect) error { + byID := map[string]*model.Task{} + ids := []string{} + for _, task := range tasks { + if task != nil && task.UUID != "" { + task.LogHistory = nil + if _, ok := byID[task.UUID]; !ok { + ids = append(ids, task.UUID) + } + byID[task.UUID] = task + } + } + if len(ids) == 0 { + return nil + } + q := d.QuoteIdentifier + query, args, err := d.QueryBuilder().Select(q("TaskID"), q("Snapshot")).From(q("task_log_history")).Where(sq.Eq{q("TaskID"): ids}).OrderBy(q("BeforeRetryGeneration") + " ASC").ToSql() + if err != nil { + return err + } + rows, err := db.Query(query, args...) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var id, data string + if err := rows.Scan(&id, &data); err != nil { + return err + } + history := &apiv2.PipelineTask_LogHistory{} + if err := protojson.Unmarshal([]byte(data), history); err != nil { + return fmt.Errorf("decode log history for task %s: %w", id, err) + } + // A reset followed by a failed resume can leave an empty boundary. + // Keep its database marker, but expose only actual troubleshooting references. + if len(history.Pods) > 0 || len(history.LogArtifacts) > 0 || history.DriverLogsUri != "" { + byID[id].LogHistory = append(byID[id].LogHistory, history) + } + } + return rows.Err() +} + +func retryLogSnapshot(task *model.Task, generation, archivedAt int64) (*apiv2.PipelineTask_LogHistory, error) { + snapshot := &apiv2.PipelineTask_LogHistory{BeforeRetryGeneration: generation, ArchivedAt: ×tamppb.Timestamp{Seconds: archivedAt}, State: apiv2.PipelineTask_TaskState(task.State)} + if task.StartedInSec != 0 { + snapshot.StartTime = ×tamppb.Timestamp{Seconds: task.StartedInSec} + } + if task.FinishedInSec != 0 { + snapshot.EndTime = ×tamppb.Timestamp{Seconds: task.FinishedInSec} + } + pods, err := model.JSONSliceToProtoSlice(task.Pods, func() *apiv2.PipelineTask_TaskPod { return &apiv2.PipelineTask_TaskPod{} }) + if err != nil { + return nil, err + } + snapshot.Pods = pods + metadata, err := model.JSONDataToProtoMessage(task.StatusMetadata, func() *apiv2.PipelineTask_StatusMetadata { return &apiv2.PipelineTask_StatusMetadata{} }) + if err != nil { + return nil, err + } + if metadata != nil { + snapshot.DriverLogsUri = metadata.GetCustomProperties()["driver_logs_uri"].GetStringValue() + } + for _, h := range task.OutputArtifactsHydrated { + if h.Key != "executor-logs" && h.Key != "driver-logs" { + continue + } + if h.Value == nil { + continue + } + a := h.Value + artifact := &apiv2.Artifact{ArtifactId: a.UUID, Namespace: a.Namespace, Type: apiv2.Artifact_ArtifactType(a.Type), Uri: a.URI, Name: a.Name, Description: a.Description, NumberValue: a.NumberValue, CreatedAt: ×tamppb.Timestamp{Seconds: a.CreatedAtInSec}} + if a.Metadata != nil { + metadata, err := structpb.NewStruct(a.Metadata) + if err != nil { + return nil, err + } + artifact.Metadata = metadata.Fields + } + link := &apiv2.PipelineTask_InputOutputs_IOArtifact{Artifacts: []*apiv2.Artifact{artifact}, Type: h.Type, ArtifactKey: h.Key} + if h.Producer != nil { + link.Producer = &apiv2.IOProducer{TaskName: h.Producer.TaskName, Iteration: h.Producer.Iteration} + } + snapshot.LogArtifacts = append(snapshot.LogArtifacts, link) + } + return snapshot, nil +} diff --git a/backend/src/apiserver/storage/task_store_test.go b/backend/src/apiserver/storage/task_store_test.go index a778de63991..0b226e8a190 100644 --- a/backend/src/apiserver/storage/task_store_test.go +++ b/backend/src/apiserver/storage/task_store_test.go @@ -20,6 +20,7 @@ import ( "sync" "testing" + sq "github.com/Masterminds/squirrel" apiv2 "github.com/kubeflow/pipelines/backend/api/v2/go_client" "github.com/kubeflow/pipelines/backend/src/apiserver/common/sql/dialect" "github.com/kubeflow/pipelines/backend/src/apiserver/filter" @@ -723,7 +724,7 @@ func TestResetTasksForRetry_ClearsAttemptLocalStateAndPreservesHistory(t *testin }) require.NoError(t, err) - err = taskStore.ResetTasksForRetry([]string{created.UUID}) + err = taskStore.ResetTasksForRetry("run-1", 0, []string{created.UUID}) require.NoError(t, err) retried, err := taskStore.GetTask(created.UUID) @@ -1910,3 +1911,179 @@ func TestListTasks_SortByStartTimeAndRunIdPaginates(t *testing.T) { }) } } + +func TestRetryLogHistoryLifecycle(t *testing.T) { + db, store, _ := initializeTaskStore() + defer db.Close() + task, err := store.CreateTask(&model.Task{Namespace: "ns1", RunUUID: "run-1", Name: "retry", ScopePath: "root.retry", State: model.TaskStatus(apiv2.PipelineTask_FAILED), StatusMetadata: model.JSONData{"customProperties": map[string]interface{}{"driver_logs_uri": "s3://logs/driver"}}, Pods: createTaskPodsAsJSONSlice(createTaskPod("pod-1", "uid-1", apiv2.PipelineTask_EXECUTOR))}) + require.NoError(t, err) + // Include both log artifacts and ordinary output artifacts: only logs survive + // in history, and neither remains a current-attempt output. + artifacts := NewArtifactStore(db, util.NewFakeTimeForEpoch(), util.NewFakeUUIDGeneratorOrFatal(testUUID2, nil), store.dbDialect) + uri := "s3://logs/attempt-1/executor-logs-0" + artifact, err := artifacts.CreateArtifact(&model.Artifact{Namespace: "ns1", Name: "logs", URI: &uri}) + require.NoError(t, err) + _, err = db.Exec("INSERT INTO artifact_tasks (UUID, ArtifactID, TaskID, Type, RunUUID, ArtifactKey) VALUES (?, ?, ?, ?, ?, ?)", "log-link", artifact.UUID, task.UUID, int32(apiv2.IOType_OUTPUT), "run-1", "executor-logs") + require.NoError(t, err) + _, err = db.Exec("INSERT INTO artifact_tasks (UUID, ArtifactID, TaskID, Type, RunUUID, ArtifactKey) VALUES (?, ?, ?, ?, ?, ?)", "data-link", artifact.UUID, task.UUID, int32(apiv2.IOType_OUTPUT), "run-1", "model") + require.NoError(t, err) + require.NoError(t, store.ResetTasksForRetry("run-1", 0, []string{task.UUID})) + got, err := store.GetTask(task.UUID) + require.NoError(t, err) + require.Len(t, got.LogHistory, 1) + history := got.LogHistory[0] + require.Equal(t, apiv2.PipelineTask_FAILED, history.State) + require.Equal(t, "uid-1", history.Pods[0].Uid) + require.Equal(t, "s3://logs/driver", history.DriverLogsUri) + require.Len(t, history.LogArtifacts, 1) + require.Equal(t, uri, history.LogArtifacts[0].Artifacts[0].GetUri()) + require.Empty(t, got.OutputArtifactsHydrated) + // Repeated delivery cannot snapshot or erase data already written by the new attempt. + _, err = db.Exec("UPDATE tasks SET pods = ? WHERE UUID = ?", `[{"name":"pod-2","uid":"uid-2"}]`, task.UUID) + require.NoError(t, err) + require.NoError(t, store.ResetTasksForRetry("run-1", 0, []string{task.UUID})) + got, err = store.GetTask(task.UUID) + require.NoError(t, err) + require.Len(t, got.LogHistory, 1) + require.NotEmpty(t, got.Pods) + // A later generation appends history. An older claimant may not reset it. + _, err = db.Exec("UPDATE run_details SET RetryGeneration = 1 WHERE UUID = ?", "run-1") + require.NoError(t, err) + require.Error(t, store.ResetTasksForRetry("run-1", 0, []string{task.UUID})) + require.NoError(t, store.ResetTasksForRetry("run-1", 1, []string{task.UUID})) + got, err = store.GetTask(task.UUID) + require.NoError(t, err) + require.Len(t, got.LogHistory, 2) + require.Equal(t, "uid-2", got.LogHistory[1].Pods[0].Uid) + // A new claim after reset but before workflow resume has no new references. + _, err = db.Exec("UPDATE run_details SET RetryGeneration = 2 WHERE UUID = ?", "run-1") + require.NoError(t, err) + require.NoError(t, store.ResetTasksForRetry("run-1", 2, []string{task.UUID})) + got, err = store.GetTask(task.UUID) + require.NoError(t, err) + require.Len(t, got.LogHistory, 2) + require.NoError(t, store.ResetTasksForRetry("run-1", 2, []string{task.UUID})) + // Completing the retried task retains old logs separately from fresh output. + output, err := model.ProtoSliceToJSONSlice([]*apiv2.PipelineTask_InputOutputs_IOParameter{{ParameterKey: "result", Type: apiv2.IOType_OUTPUT, Value: structpb.NewStringValue("fresh")}}) + require.NoError(t, err) + _, err = store.UpdateTask(&model.Task{UUID: task.UUID, State: model.TaskStatus(apiv2.PipelineTask_SUCCEEDED), OutputParameters: output}) + require.NoError(t, err) + _, err = db.Exec("INSERT INTO artifact_tasks (UUID, ArtifactID, TaskID, Type, RunUUID, ArtifactKey) VALUES (?, ?, ?, ?, ?, ?)", "new-output-link", artifact.UUID, task.UUID, int32(apiv2.IOType_OUTPUT), "run-1", "new-model") + require.NoError(t, err) + got, err = store.GetTask(task.UUID) + require.NoError(t, err) + require.Equal(t, model.TaskStatus(apiv2.PipelineTask_SUCCEEDED), got.State) + require.Len(t, got.LogHistory, 2) + require.Len(t, got.OutputArtifactsHydrated, 1) + require.Equal(t, "new-model", got.OutputArtifactsHydrated[0].Key) + require.NotEmpty(t, got.OutputParameters) + // Cross-run resets are refused without exposing or mutating another run. + require.Error(t, store.ResetTasksForRetry("run-2", 0, []string{task.UUID})) + require.NoError(t, store.DeleteTasksForRun(nil, "run-1")) + var count int + require.NoError(t, db.QueryRow("SELECT COUNT(*) FROM task_log_history").Scan(&count)) + require.Zero(t, count) + // Retention records are references; deleting them never deletes object-store data. + _, err = artifacts.GetArtifact(artifact.UUID) + require.NoError(t, err) +} + +func TestRetryLogHistoryResetRollsBackAtomically(t *testing.T) { + db, store, _ := initializeTaskStore() + defer db.Close() + task, err := store.CreateTask(&model.Task{Namespace: "ns1", RunUUID: "run-1", Name: "retry", ScopePath: "root.retry", State: model.TaskStatus(apiv2.PipelineTask_FAILED), Pods: createTaskPodsAsJSONSlice(createTaskPod("old-pod", "old-uid", apiv2.PipelineTask_EXECUTOR))}) + require.NoError(t, err) + // Fail after snapshot insertion and artifact unlinking, at the state reset. + _, err = db.Exec("CREATE TRIGGER fail_reset BEFORE UPDATE ON tasks BEGIN SELECT RAISE(ABORT, 'injected reset failure'); END") + require.NoError(t, err) + require.Error(t, store.ResetTasksForRetry("run-1", 0, []string{task.UUID})) + got, err := store.GetTask(task.UUID) + require.NoError(t, err) + require.Equal(t, model.TaskStatus(apiv2.PipelineTask_FAILED), got.State) + require.Empty(t, got.LogHistory) + _, err = db.Exec("DROP TRIGGER fail_reset") + require.NoError(t, err) + require.NoError(t, store.ResetTasksForRetry("run-1", 0, []string{task.UUID})) + got, err = store.GetTask(task.UUID) + require.NoError(t, err) + require.Len(t, got.LogHistory, 1) +} + +// Uses the same disposable database configuration as the recurring-run tests so +// retry history exercises real MySQL and PostgreSQL transactions in backend CI. +func TestRetryLogHistoryProductionDatabases(t *testing.T) { + for _, driver := range []string{"mysql", "pgx"} { + t.Run(driver, func(t *testing.T) { + dbs, d := recurringIntegrationDatabases(t, driver) + db := dbs[0] + q := d.QuoteIdentifier + qb := d.QueryBuilder() + clock := util.NewFakeTimeForEpoch() + runs := NewRunStore(db, clock, d) + store := NewTaskStore(db, clock, util.NewFakeUUIDGeneratorOrFatal(testUUID1, nil), d) + _, err := runs.CreateRun(&model.Run{UUID: "retry-run", Namespace: "test", DisplayName: "retry", RunDetails: model.RunDetails{State: model.RuntimeStateFailed}}) + require.NoError(t, err) + task, err := store.CreateTask(&model.Task{Namespace: "test", RunUUID: "retry-run", Name: "retry", ScopePath: "root.retry", State: model.TaskStatus(apiv2.PipelineTask_FAILED), Pods: createTaskPodsAsJSONSlice(createTaskPod("old", "uid-old", apiv2.PipelineTask_EXECUTOR))}) + require.NoError(t, err) + artifacts := NewArtifactStore(db, clock, util.NewFakeUUIDGeneratorOrFatal(testUUID2, nil), d) + uri := "s3://test/immutable/executor-logs-0" + artifact, err := artifacts.CreateArtifact(&model.Artifact{Namespace: "test", Name: "logs", URI: &uri}) + require.NoError(t, err) + links := NewArtifactTaskStore(db, util.NewFakeUUIDGeneratorOrFatal(testUUID3, nil), d) + _, err = links.CreateArtifactTask(&model.ArtifactTask{ArtifactID: artifact.UUID, TaskID: task.UUID, RunUUID: "retry-run", Type: model.IOType(apiv2.IOType_OUTPUT), ArtifactKey: "executor-logs"}) + require.NoError(t, err) + // Inject a failure after snapshot/link changes to prove the entire boundary rolls back. + if driver == "mysql" { + _, err = db.Exec("CREATE TRIGGER reject_retry BEFORE UPDATE ON tasks FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'injected retry failure'") + } else { + _, err = db.Exec("CREATE FUNCTION reject_retry() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected retry failure'; END $$") + require.NoError(t, err) + _, err = db.Exec("CREATE TRIGGER reject_retry BEFORE UPDATE ON tasks FOR EACH ROW EXECUTE FUNCTION reject_retry()") + } + require.NoError(t, err) + require.Error(t, store.ResetTasksForRetry("retry-run", 0, []string{task.UUID})) + got, err := store.GetTask(task.UUID) + require.NoError(t, err) + require.Empty(t, got.LogHistory) + require.Len(t, got.OutputArtifactsHydrated, 1) + require.Equal(t, model.TaskStatus(apiv2.PipelineTask_FAILED), got.State) + if driver == "mysql" { + _, err = db.Exec("DROP TRIGGER reject_retry") + } else { + _, err = db.Exec("DROP TRIGGER reject_retry ON tasks") + } + require.NoError(t, err) + require.NoError(t, store.ResetTasksForRetry("retry-run", 0, []string{task.UUID})) + require.NoError(t, store.ResetTasksForRetry("retry-run", 0, []string{task.UUID})) + got, err = store.GetTask(task.UUID) + require.NoError(t, err) + require.Len(t, got.LogHistory, 1) + require.Len(t, got.LogHistory[0].LogArtifacts, 1) + require.Empty(t, got.OutputArtifactsHydrated) + // Two concurrently delivered resets commit exactly one history row. + statement, args, err := qb.Update(q("run_details")).Set(q("RetryGeneration"), 1).Where(sq.Eq{q("UUID"): "retry-run"}).ToSql() + require.NoError(t, err) + _, err = db.Exec(statement, args...) + require.NoError(t, err) + statement, args, err = qb.Update(q("tasks")).Set(q("pods"), `[{"name":"new","uid":"uid-new"}]`).Where(sq.Eq{q("UUID"): task.UUID}).ToSql() + require.NoError(t, err) + _, err = db.Exec(statement, args...) + require.NoError(t, err) + errs := recurringConcurrent(2, func(i int) error { + other := NewTaskStore(dbs[i], clock, nil, d) + return other.ResetTasksForRetry("retry-run", 1, []string{task.UUID}) + }) + for _, err := range errs { + require.NoError(t, err) + } + require.Error(t, store.ResetTasksForRetry("retry-run", 0, []string{task.UUID})) + got, err = store.GetTask(task.UUID) + require.NoError(t, err) + require.Len(t, got.LogHistory, 2) + require.NoError(t, store.DeleteTasksForRun(nil, "retry-run")) + var count int + require.NoError(t, db.QueryRow("SELECT COUNT(*) FROM task_log_history").Scan(&count)) + require.Zero(t, count) + }) + } +} diff --git a/backend/src/common/client/api_server/v2/task_history_serialization_test.go b/backend/src/common/client/api_server/v2/task_history_serialization_test.go new file mode 100644 index 00000000000..faad40bdbf3 --- /dev/null +++ b/backend/src/common/client/api_server/v2/task_history_serialization_test.go @@ -0,0 +1,28 @@ +// Copyright 2026 The Kubeflow Authors +// Licensed under the Apache License, Version 2.0. + +package api_server_v2 //nolint:staticcheck // Match the existing package name. + +import ( + "encoding/json" + "testing" + + "github.com/kubeflow/pipelines/backend/api/v2/go_http_client/run_model" + "github.com/stretchr/testify/require" +) + +func TestTaskHistorySerializationCompatibility(t *testing.T) { + for _, history := range [][]*run_model.PipelineTaskLogHistory{nil, {}} { + body, err := json.Marshal(&run_model.V2PipelineTask{LogHistory: history}) + require.NoError(t, err) + var fields map[string]json.RawMessage + require.NoError(t, json.Unmarshal(body, &fields)) + require.NotContains(t, fields, "log_history", "older servers reject this unknown field even when null") + } + body, err := json.Marshal(&run_model.V2PipelineTask{LogHistory: []*run_model.PipelineTaskLogHistory{{DriverLogsURI: "s3://logs/previous"}}}) + require.NoError(t, err) + var decoded run_model.V2PipelineTask + require.NoError(t, json.Unmarshal(body, &decoded)) + require.Len(t, decoded.LogHistory, 1) + require.Equal(t, "s3://logs/previous", decoded.LogHistory[0].DriverLogsURI) +} diff --git a/backend/src/v2/component/launcher_v2.go b/backend/src/v2/component/launcher_v2.go index 5be7cc37534..b76628ad494 100644 --- a/backend/src/v2/component/launcher_v2.go +++ b/backend/src/v2/component/launcher_v2.go @@ -404,7 +404,7 @@ func (o *LauncherV2Options) validate() error { // artifacts to object storage, updates the task's recorded outputs, flushes the // queued API mutations, and finally propagates the resulting outputs up the DAG. func (l *LauncherV2) executeV2(ctx context.Context) (*pipelinespec.ExecutorOutput, error) { - qualifyExecutorLogsForRetry(ctx, l.executorInput, l.options.PublishLogs, l.options.Namespace, l.clientManager.K8sClient(), l.options.PodName) + qualifyExecutorLogsForRetry(ctx, l.executorInput, l.options.PublishLogs, l.options.Namespace, l.clientManager.K8sClient(), l.options.PodName, l.options.PodUID) // Populate component defaults for placeholders without mutating the original // executor input used by the rest of the launcher flow. @@ -543,15 +543,12 @@ const OutputMetadataFilepath = "/tmp/kfp_outputs/output_metadata.json" // We overwrite this as a DI mechanism for testing getLogWriter. var osCreateFunc = os.Create -// qualifyExecutorLogsURI appends the retry index to the executor-logs artifact -// URI so that each Argo retry attempt writes to and registers a distinct, -// human-readable path in the object store and MLMD (e.g. executor-logs-0, -// executor-logs-1, …). Without this, all retry pods share the single URI that -// the driver provisioned before any attempt ran, causing every retry to -// overwrite the same S3 key and register duplicate MLMD artifacts pointing at -// stale data. -func qualifyExecutorLogsURI(artifacts map[string]*pipelinespec.ArtifactList, retryIndex string) { - if retryIndex == "" { +// qualifyExecutorLogsURI gives every executor pod an immutable log location. +// Manual retry can retain the successful driver and reuse its output prefix and +// automatic retry index, but Kubernetes assigns the new executor a fresh UID. +// Keep the executor-logs-N basename for clients that identify log artifacts by it. +func qualifyExecutorLogsURI(artifacts map[string]*pipelinespec.ArtifactList, retryIndex, podUID string) { + if retryIndex == "" || podUID == "" { return } logsArtifactList, ok := artifacts["executor-logs"] @@ -562,23 +559,28 @@ func qualifyExecutorLogsURI(artifacts map[string]*pipelinespec.ArtifactList, ret if art == nil { return } - art.Uri = appendRetryIndexSuffix(art.Uri, retryIndex) + art.Uri = executorLogAttemptPath(art.Uri, retryIndex, podUID) if art.CustomPath != nil && *art.CustomPath != "" { - *art.CustomPath = appendRetryIndexSuffix(*art.CustomPath, retryIndex) + *art.CustomPath = executorLogAttemptPath(*art.CustomPath, retryIndex, podUID) } } -// appendRetryIndexSuffix appends "-" to a log path or URI once, -// preserving already-qualified values. -func appendRetryIndexSuffix(pathOrURI, retryIndex string) string { - if pathOrURI == "" || retryIndex == "" { +// executorLogAttemptPath qualifies a driver-provisioned URI or local path once. +func executorLogAttemptPath(pathOrURI, retryIndex, podUID string) string { + if pathOrURI == "" { return pathOrURI } + separator := strings.LastIndex(pathOrURI, "/") + parent, name := pathOrURI[:separator+1], pathOrURI[separator+1:] + attemptDirectory := "attempts/" + podUID + "/" + if !strings.HasSuffix(parent, attemptDirectory) { + parent += attemptDirectory + } suffix := "-" + retryIndex - if strings.HasSuffix(pathOrURI, suffix) { - return pathOrURI + if !strings.HasSuffix(name, suffix) { + name += suffix } - return pathOrURI + suffix + return parent + name } // qualifyExecutorLogsForRetry resolves the current retry attempt and updates the @@ -590,6 +592,7 @@ func qualifyExecutorLogsForRetry( namespace string, k8sClient kubernetes.Interface, podName string, + podUID string, ) { if publishLogs != "true" { return @@ -606,7 +609,7 @@ func qualifyExecutorLogsForRetry( if retryIndex == "" { retryIndex = "0" } - qualifyExecutorLogsURI(executorInput.GetOutputs().GetArtifacts(), retryIndex) + qualifyExecutorLogsURI(executorInput.GetOutputs().GetArtifacts(), retryIndex, podUID) } // retryIndexFromPodAnnotation reads the Argo node-name annotation on the current diff --git a/backend/src/v2/component/launcher_v2_test.go b/backend/src/v2/component/launcher_v2_test.go index 9a743fa5971..e1115daa0ea 100644 --- a/backend/src/v2/component/launcher_v2_test.go +++ b/backend/src/v2/component/launcher_v2_test.go @@ -29,6 +29,7 @@ import ( "github.com/kubeflow/pipelines/backend/src/v2/common/plugins" "github.com/stretchr/testify/require" "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" apiv2 "github.com/kubeflow/pipelines/backend/api/v2/go_client" @@ -1832,8 +1833,70 @@ func TestQualifyExecutorLogsForRetry_UsesRetryEnv(t *testing.T) { }, } - qualifyExecutorLogsForRetry(context.Background(), executorInput, "true", "", nil, "") - require.Equal(t, "minio://bucket/logs/executor-logs-2", executorInput.GetOutputs().GetArtifacts()["executor-logs"].Artifacts[0].GetUri()) + qualifyExecutorLogsForRetry(context.Background(), executorInput, "true", "", nil, "", "pod-uid") + require.Equal(t, "minio://bucket/logs/attempts/pod-uid/executor-logs-2", executorInput.GetOutputs().GetArtifacts()["executor-logs"].Artifacts[0].GetUri()) +} + +// A successful driver survives manual retry, so every executor receives the +// same original log URI even when its automatic retry index restarts at zero. +func TestQualifyExecutorLogsURI_PreservesManualRetryHistory(t *testing.T) { + base := &pipelinespec.ExecutorInput{ + Outputs: &pipelinespec.ExecutorInput_Outputs{ + Artifacts: map[string]*pipelinespec.ArtifactList{ + "executor-logs": {Artifacts: []*pipelinespec.RuntimeArtifact{{ + Uri: "minio://bucket/task/driver-prefix/executor-logs", + CustomPath: proto.String("/logs/executor-logs"), + }}}, + "model": {Artifacts: []*pipelinespec.RuntimeArtifact{{Uri: "minio://bucket/task/model"}}}, + }, + }, + } + storedLogs := make(map[string]string) + for _, attempt := range []struct{ podUID, index, output string }{ + {"initial-pod", "0", "initial failure"}, + {"automatic-pod", "1", "automatic retry failure"}, + {"manual-pod", "0", "manual retry failure"}, + {"second-manual-pod", "0", "successful retry"}, + } { + input := proto.Clone(base).(*pipelinespec.ExecutorInput) + artifacts := input.GetOutputs().GetArtifacts() + qualifyExecutorLogsURI(artifacts, attempt.index, attempt.podUID) + log := artifacts["executor-logs"].Artifacts[0] + require.Equal(t, "minio://bucket/task/driver-prefix/attempts/"+attempt.podUID+"/executor-logs-"+attempt.index, log.Uri) + require.Equal(t, "/logs/attempts/"+attempt.podUID+"/executor-logs-"+attempt.index, log.GetCustomPath()) + require.Equal(t, "minio://bucket/task/model", artifacts["model"].Artifacts[0].Uri) + require.NotContains(t, storedLogs, log.Uri, "new execution must not overwrite earlier output") + storedLogs[log.Uri] = attempt.output + qualified := proto.Clone(input) + qualifyExecutorLogsURI(artifacts, attempt.index, attempt.podUID) + require.True(t, proto.Equal(qualified, input), "qualification must be idempotent") + } + require.Len(t, storedLogs, 4) + require.Contains(t, storedLogs, "minio://bucket/task/driver-prefix/attempts/initial-pod/executor-logs-0") +} + +func TestQualifyExecutorLogsForRetry_PublicationDisabled(t *testing.T) { + input := &pipelinespec.ExecutorInput{Outputs: &pipelinespec.ExecutorInput_Outputs{ + Artifacts: map[string]*pipelinespec.ArtifactList{ + "executor-logs": {Artifacts: []*pipelinespec.RuntimeArtifact{{Uri: "minio://bucket/executor-logs"}}}, + }, + }} + original := proto.Clone(input) + qualifyExecutorLogsForRetry(context.Background(), input, "false", "", nil, "", "pod-uid") + require.True(t, proto.Equal(original, input)) +} + +func TestExecutorLogAttemptPath(t *testing.T) { + for _, tc := range []struct{ name, input, expected string }{ + {"empty", "", ""}, + {"relative", "executor-logs", "attempts/pod-uid/executor-logs-0"}, + {"already retry qualified", "s3://bucket/logs/executor-logs-0", "s3://bucket/logs/attempts/pod-uid/executor-logs-0"}, + {"already invocation qualified", "s3://bucket/logs/attempts/pod-uid/executor-logs-0", "s3://bucket/logs/attempts/pod-uid/executor-logs-0"}, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.expected, executorLogAttemptPath(tc.input, "0", "pod-uid")) + }) + } } // Tests happy and unhappy paths for constructing a new LauncherV2 diff --git a/backend/test/proto_tests/testdata/generated-1791485/pipeline_task.json b/backend/test/proto_tests/testdata/generated-1791485/pipeline_task.json index 5bbcd055df2..c549848f38d 100644 --- a/backend/test/proto_tests/testdata/generated-1791485/pipeline_task.json +++ b/backend/test/proto_tests/testdata/generated-1791485/pipeline_task.json @@ -71,5 +71,6 @@ ], "artifacts": [] }, - "scope_path": "root.trainer-task" + "scope_path": "root.trainer-task", + "log_history": [] } diff --git a/docs/_static/kfp_api_single_file.swagger.json b/docs/_static/kfp_api_single_file.swagger.json index cda52188343..c9209ca5d13 100644 --- a/docs/_static/kfp_api_single_file.swagger.json +++ b/docs/_static/kfp_api_single_file.swagger.json @@ -3347,6 +3347,51 @@ } } }, + "PipelineTaskLogHistory": { + "type": "object", + "properties": { + "before_retry_generation": { + "type": "string", + "format": "int64", + "description": "Together with task_id, identifies this snapshot. This is the manual retry\ngeneration that archived it, not an automatic pod retry attempt number." + }, + "archived_at": { + "type": "string", + "format": "date-time" + }, + "start_time": { + "type": "string", + "format": "date-time" + }, + "end_time": { + "type": "string", + "format": "date-time" + }, + "state": { + "$ref": "#/definitions/PipelineTaskTaskState" + }, + "pods": { + "type": "array", + "items": { + "type": "object", + "$ref": "#/definitions/PipelineTaskTaskPod" + } + }, + "log_artifacts": { + "type": "array", + "items": { + "type": "object", + "$ref": "#/definitions/InputOutputsIOArtifact" + }, + "description": "Log artifacts only; never includes task outputs used by downstream tasks." + }, + "driver_logs_uri": { + "type": "string", + "description": "Legacy driver log reference, when not represented by a log artifact." + } + }, + "description": "Log references from an execution archived before a manual run retry.\nReferences can outlive pods or archived objects and do not guarantee availability." + }, "PipelineTaskStatusMetadata": { "type": "object", "properties": { @@ -3636,6 +3681,15 @@ "lifecycle_category": { "type": "string", "description": "Classified category of lifecycle_message.\nOne of image-pull, scheduling, runtime, admission, or unknown.\nEmpty when lifecycle_message is empty." + }, + "log_history": { + "type": "array", + "items": { + "type": "object", + "$ref": "#/definitions/PipelineTaskLogHistory" + }, + "description": "Output only. Prior executions archived by manual retries, oldest first.\nHistory begins when preservation is enabled; older retries are not backfilled.", + "readOnly": true } }, "description": "Runtime information of a task execution used by task APIs and Run.tasks." diff --git a/docs/operator-guides/index.md b/docs/operator-guides/index.md index 959c0c56653..01cee23ea8c 100644 --- a/docs/operator-guides/index.md +++ b/docs/operator-guides/index.md @@ -14,5 +14,6 @@ multi-user high-availability rbac-migration-2.18 configure-object-store +retry-log-history mlflow-plugin ``` diff --git a/docs/operator-guides/retry-log-history.md b/docs/operator-guides/retry-log-history.md new file mode 100644 index 00000000000..3a4519cda5e --- /dev/null +++ b/docs/operator-guides/retry-log-history.md @@ -0,0 +1,88 @@ +# Retry log history + +Manual **Retry run** keeps the run and logical task identities. Before resetting +an unsuccessful task, the API server saves its troubleshooting metadata and log +artifact references in `task_log_history`. Successful, cached, and skipped tasks +are retained without a reset. The current task continues to describe only its +current execution; historical output is available through its log history. + +Each record identifies the manual retry generation that caused the snapshot. +Automatic retry attempts remain distinguishable by their executor pods and log +artifacts within that generation. A generation is an identity, not a guarantee +that the workflow operation completed: a retry claim may fail after the snapshot. +Repeated delivery must preserve the original snapshot rather than replace it +with a partially reset task. + +## Upgrade and rollback + +The schema addition is a standalone table with a composite task/generation key +and run lookup index. It has no foreign keys and does not alter existing tables +or backfill historical runs. Startup creates it under database advisory ownership +on one physical connection. Ownership acquisition fails immediately when another +replica holds it; metadata lock waits are limited to one second and the migration +has a ten-second context deadline. Failure aborts startup with an actionable error; +resolve contention and retry startup. Session settings and locks are restored, +and connections whose cleanup fails are discarded. + +PostgreSQL creates the table and index in a transaction. MySQL creates the table +with its index in the same DDL statement. Existing history tables are not +reconciled by broad `AutoMigrate`. These bounds apply to the new history-table +operation, not to unrelated schema reconciliation in the release. Database +operations still acquire brief catalog/metadata locks; this is not a promise of +zero locking or zero downtime. + +Roll out the API server and new launcher/runtime together. New launchers write +executor logs to `.../attempts//executor-logs-`. +This prevents a manual retry that reuses a successful driver's output prefix +from overwriting an earlier executor's object. Existing archived URIs remain +valid. Older launchers can still overwrite their reused paths, so snapshot +preservation alone cannot guarantee immutable log contents during a mixed-runtime +rollout. + +Rolling back the API server leaves the additive table intact but stops recording +and exposing new history. Old-server run deletion does not clean this new table; +operators who delete runs while rolled back should account for orphan history +rows before eventual data-retention cleanup. After all old servers are retired, +identify rows whose `RunUUID` has no matching `run_details.UUID` and remove them +in bounded batches under the deployment's normal database maintenance process. +Review counts first; this metadata cleanup must not delete object-store blobs. Do not drop the table as part of +routine rollback. Forward rollout can resume recording without reconstructing +already-cleared references. + +## Retention and access + +History preserves references, not a second copy of log bytes. Kubernetes pod +collection, object-store lifecycle policies, missing objects, and disabled log +publication still affect availability. Run deletion on the new server removes +its history in the same database transaction as task deletion. Artifact/object +retention is otherwise unchanged. History is exposed through the existing +run/task authorization boundary; it does not create a public log endpoint. + +Existing references or blobs already removed or overwritten before this upgrade +cannot be recovered by this migration. The guarantee applies going forward once +the updated server and runtime are active. + +Namespace export/import does not currently include the separate retry history +table. Moving a run through that mechanism preserves its supported current-task +metadata, but does not transfer historical retry snapshots. Keep the source +installation and its archive retention available when that history is needed; +a future archive-format change is separate from this migration. + +## Verification + +The migration regression test uses `KFP_RECURRING_MYSQL_TEST_DSN` and +`KFP_RECURRING_POSTGRES_TEST_DSN`, creates isolated databases/schemas, and exercises +`autoMigrate` against a populated immediately preceding schema. It checks live +read/write completion during startup, repeated startup, retained values, +competing migration ownership, cancellation, and recovery. Run it with: + +```sh +go test ./backend/src/apiserver/client_manager -run TestRetryLogHistoryMigrationProductionDatabases -count=1 -v +``` + +The output reports completed concurrent request pairs and maximum observed +latency. Storage lifecycle tests cover snapshot/reset atomicity, repeated manual +retries, successful siblings, artifact references, and deletion. Launcher tests +cover immutable paths across automatic and manual retries without changing +non-log outputs. These tests do not establish availability for every historical +schema variant or unrelated release migration. diff --git a/frontend/src/apisv2/run/apis/RunServiceApi.ts b/frontend/src/apisv2/run/apis/RunServiceApi.ts index 26fce7dc413..edbb43bc970 100644 --- a/frontend/src/apisv2/run/apis/RunServiceApi.ts +++ b/frontend/src/apisv2/run/apis/RunServiceApi.ts @@ -89,7 +89,7 @@ export interface RetryRunRequest { export interface TaskRequest { run_id: string; - task: V2PipelineTask; + task: Omit; } export interface Task0Request { @@ -100,7 +100,7 @@ export interface Task0Request { export interface Task1Request { run_id: string; task_id: string; - task: V2PipelineTask; + task: Omit; } export interface TasksRequest { diff --git a/frontend/src/apisv2/run/models/PipelineTaskLogHistory.ts b/frontend/src/apisv2/run/models/PipelineTaskLogHistory.ts new file mode 100644 index 00000000000..0bcf4c979b5 --- /dev/null +++ b/frontend/src/apisv2/run/models/PipelineTaskLogHistory.ts @@ -0,0 +1,163 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * backend/api/v2/run.proto + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: version not set + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { mapValues } from '../runtime'; +import type { InputOutputsIOArtifact } from './InputOutputsIOArtifact'; +import { + InputOutputsIOArtifactFromJSON, + InputOutputsIOArtifactFromJSONTyped, + InputOutputsIOArtifactToJSON, + InputOutputsIOArtifactToJSONTyped, +} from './InputOutputsIOArtifact'; +import type { PipelineTaskTaskPod } from './PipelineTaskTaskPod'; +import { + PipelineTaskTaskPodFromJSON, + PipelineTaskTaskPodFromJSONTyped, + PipelineTaskTaskPodToJSON, + PipelineTaskTaskPodToJSONTyped, +} from './PipelineTaskTaskPod'; +import type { PipelineTaskTaskState } from './PipelineTaskTaskState'; +import { + PipelineTaskTaskStateFromJSON, + PipelineTaskTaskStateFromJSONTyped, + PipelineTaskTaskStateToJSON, + PipelineTaskTaskStateToJSONTyped, +} from './PipelineTaskTaskState'; + +/** + * Log references from an execution archived before a manual run retry. + * References can outlive pods or archived objects and do not guarantee availability. + * @export + * @interface PipelineTaskLogHistory + */ +export interface PipelineTaskLogHistory { + /** + * Together with task_id, identifies this snapshot. This is the manual retry + * generation that archived it, not an automatic pod retry attempt number. + * @type {string} + * @memberof PipelineTaskLogHistory + */ + before_retry_generation?: string; + /** + * + * @type {Date} + * @memberof PipelineTaskLogHistory + */ + archived_at?: Date; + /** + * + * @type {Date} + * @memberof PipelineTaskLogHistory + */ + start_time?: Date; + /** + * + * @type {Date} + * @memberof PipelineTaskLogHistory + */ + end_time?: Date; + /** + * + * @type {PipelineTaskTaskState} + * @memberof PipelineTaskLogHistory + */ + state?: PipelineTaskTaskState; + /** + * + * @type {Array} + * @memberof PipelineTaskLogHistory + */ + pods?: Array; + /** + * Log artifacts only; never includes task outputs used by downstream tasks. + * @type {Array} + * @memberof PipelineTaskLogHistory + */ + log_artifacts?: Array; + /** + * Legacy driver log reference, when not represented by a log artifact. + * @type {string} + * @memberof PipelineTaskLogHistory + */ + driver_logs_uri?: string; +} + +/** + * Check if a given object implements the PipelineTaskLogHistory interface. + */ +export function instanceOfPipelineTaskLogHistory(value: object): value is PipelineTaskLogHistory { + return true; +} + +export function PipelineTaskLogHistoryFromJSON(json: any): PipelineTaskLogHistory { + return PipelineTaskLogHistoryFromJSONTyped(json, false); +} + +export function PipelineTaskLogHistoryFromJSONTyped( + json: any, + ignoreDiscriminator: boolean, +): PipelineTaskLogHistory { + if (json == null) { + return json; + } + return { + before_retry_generation: + json['before_retry_generation'] == null ? undefined : json['before_retry_generation'], + archived_at: json['archived_at'] == null ? undefined : new Date(json['archived_at']), + start_time: json['start_time'] == null ? undefined : new Date(json['start_time']), + end_time: json['end_time'] == null ? undefined : new Date(json['end_time']), + state: json['state'] == null ? undefined : PipelineTaskTaskStateFromJSON(json['state']), + pods: + json['pods'] == null + ? undefined + : (json['pods'] as Array).map(PipelineTaskTaskPodFromJSON), + log_artifacts: + json['log_artifacts'] == null + ? undefined + : (json['log_artifacts'] as Array).map(InputOutputsIOArtifactFromJSON), + driver_logs_uri: json['driver_logs_uri'] == null ? undefined : json['driver_logs_uri'], + }; +} + +export function PipelineTaskLogHistoryToJSON(json: any): PipelineTaskLogHistory { + return PipelineTaskLogHistoryToJSONTyped(json, false); +} + +export function PipelineTaskLogHistoryToJSONTyped( + value?: PipelineTaskLogHistory | null, + ignoreDiscriminator: boolean = false, +): any { + if (value == null) { + return value; + } + + return { + before_retry_generation: value['before_retry_generation'], + archived_at: + value['archived_at'] == null ? value['archived_at'] : value['archived_at'].toISOString(), + start_time: + value['start_time'] == null ? value['start_time'] : value['start_time'].toISOString(), + end_time: value['end_time'] == null ? value['end_time'] : value['end_time'].toISOString(), + state: PipelineTaskTaskStateToJSON(value['state']), + pods: + value['pods'] == null + ? undefined + : (value['pods'] as Array).map(PipelineTaskTaskPodToJSON), + log_artifacts: + value['log_artifacts'] == null + ? undefined + : (value['log_artifacts'] as Array).map(InputOutputsIOArtifactToJSON), + driver_logs_uri: value['driver_logs_uri'], + }; +} diff --git a/frontend/src/apisv2/run/models/V2PipelineTask.ts b/frontend/src/apisv2/run/models/V2PipelineTask.ts index c4559f13a0f..50e5336722d 100644 --- a/frontend/src/apisv2/run/models/V2PipelineTask.ts +++ b/frontend/src/apisv2/run/models/V2PipelineTask.ts @@ -62,6 +62,13 @@ import { PipelineTaskTypeAttributesToJSON, PipelineTaskTypeAttributesToJSONTyped, } from './PipelineTaskTypeAttributes'; +import type { PipelineTaskLogHistory } from './PipelineTaskLogHistory'; +import { + PipelineTaskLogHistoryFromJSON, + PipelineTaskLogHistoryFromJSONTyped, + PipelineTaskLogHistoryToJSON, + PipelineTaskLogHistoryToJSONTyped, +} from './PipelineTaskLogHistory'; import type { PipelineTaskTaskState } from './PipelineTaskTaskState'; import { PipelineTaskTaskStateFromJSON, @@ -202,6 +209,13 @@ export interface V2PipelineTask { * @memberof V2PipelineTask */ lifecycle_category?: string; + /** + * Output only. Prior executions archived by manual retries, oldest first. + * History begins when preservation is enabled; older retries are not backfilled. + * @type {Array} + * @memberof V2PipelineTask + */ + readonly log_history?: Array; } /** @@ -259,6 +273,10 @@ export function V2PipelineTaskFromJSONTyped( scope_path: json['scope_path'] == null ? undefined : json['scope_path'], lifecycle_message: json['lifecycle_message'] == null ? undefined : json['lifecycle_message'], lifecycle_category: json['lifecycle_category'] == null ? undefined : json['lifecycle_category'], + log_history: + json['log_history'] == null + ? undefined + : (json['log_history'] as Array).map(PipelineTaskLogHistoryFromJSON), }; } @@ -267,7 +285,7 @@ export function V2PipelineTaskToJSON(json: any): V2PipelineTask { } export function V2PipelineTaskToJSONTyped( - value?: V2PipelineTask | null, + value?: Omit | null, ignoreDiscriminator: boolean = false, ): any { if (value == null) { diff --git a/frontend/src/apisv2/run/models/index.ts b/frontend/src/apisv2/run/models/index.ts index a850fc36f9d..8b1be8bb3ee 100644 --- a/frontend/src/apisv2/run/models/index.ts +++ b/frontend/src/apisv2/run/models/index.ts @@ -7,6 +7,7 @@ export * from './InputOutputsIOParameter'; export * from './MetadataValueRenderType'; export * from './PipelineTaskChildTask'; export * from './PipelineTaskInputOutputs'; +export * from './PipelineTaskLogHistory'; export * from './PipelineTaskStatusMetadata'; export * from './PipelineTaskTaskPod'; export * from './PipelineTaskTaskPodType'; diff --git a/sdk/python/kfp/server_api/__init__.py b/sdk/python/kfp/server_api/__init__.py index 3821f27799d..ba49345d4a4 100644 --- a/sdk/python/kfp/server_api/__init__.py +++ b/sdk/python/kfp/server_api/__init__.py @@ -48,6 +48,7 @@ from kfp.server_api.models.metadata_value_render_type import MetadataValueRenderType from kfp.server_api.models.pipeline_task_child_task import PipelineTaskChildTask from kfp.server_api.models.pipeline_task_input_outputs import PipelineTaskInputOutputs +from kfp.server_api.models.pipeline_task_log_history import PipelineTaskLogHistory from kfp.server_api.models.pipeline_task_status_metadata import PipelineTaskStatusMetadata from kfp.server_api.models.pipeline_task_task_pod import PipelineTaskTaskPod from kfp.server_api.models.pipeline_task_task_pod_type import PipelineTaskTaskPodType diff --git a/sdk/python/kfp/server_api/models/__init__.py b/sdk/python/kfp/server_api/models/__init__.py index 0d412164adf..77dbae5df3f 100644 --- a/sdk/python/kfp/server_api/models/__init__.py +++ b/sdk/python/kfp/server_api/models/__init__.py @@ -25,6 +25,7 @@ from kfp.server_api.models.metadata_value_render_type import MetadataValueRenderType from kfp.server_api.models.pipeline_task_child_task import PipelineTaskChildTask from kfp.server_api.models.pipeline_task_input_outputs import PipelineTaskInputOutputs +from kfp.server_api.models.pipeline_task_log_history import PipelineTaskLogHistory from kfp.server_api.models.pipeline_task_status_metadata import PipelineTaskStatusMetadata from kfp.server_api.models.pipeline_task_task_pod import PipelineTaskTaskPod from kfp.server_api.models.pipeline_task_task_pod_type import PipelineTaskTaskPodType diff --git a/sdk/python/kfp/server_api/models/pipeline_task_log_history.py b/sdk/python/kfp/server_api/models/pipeline_task_log_history.py new file mode 100644 index 00000000000..1bf3465a2bf --- /dev/null +++ b/sdk/python/kfp/server_api/models/pipeline_task_log_history.py @@ -0,0 +1,308 @@ +# coding: utf-8 + +""" + Kubeflow Pipelines API + + This file contains REST API specification for Kubeflow Pipelines. The file is autogenerated from the swagger definition. + + Contact: kubeflow-pipelines@google.com + Generated by: https://openapi-generator.tech +""" + + +import pprint +import re # noqa: F401 + +import six + +from kfp.server_api.configuration import Configuration + + +class PipelineTaskLogHistory(object): + """NOTE: This class is auto generated by OpenAPI Generator. + Ref: https://openapi-generator.tech + + Do not edit the class manually. + """ + + """ + Attributes: + openapi_types (dict): The key is attribute name + and the value is attribute type. + attribute_map (dict): The key is attribute name + and the value is json key in definition. + """ + openapi_types = { + 'before_retry_generation': 'str', + 'archived_at': 'datetime', + 'start_time': 'datetime', + 'end_time': 'datetime', + 'state': 'PipelineTaskTaskState', + 'pods': 'list[PipelineTaskTaskPod]', + 'log_artifacts': 'list[InputOutputsIOArtifact]', + 'driver_logs_uri': 'str' + } + + attribute_map = { + 'before_retry_generation': 'before_retry_generation', + 'archived_at': 'archived_at', + 'start_time': 'start_time', + 'end_time': 'end_time', + 'state': 'state', + 'pods': 'pods', + 'log_artifacts': 'log_artifacts', + 'driver_logs_uri': 'driver_logs_uri' + } + + def __init__(self, before_retry_generation=None, archived_at=None, start_time=None, end_time=None, state=None, pods=None, log_artifacts=None, driver_logs_uri=None, local_vars_configuration=None): # noqa: E501 + """PipelineTaskLogHistory - a model defined in OpenAPI""" # noqa: E501 + if local_vars_configuration is None: + local_vars_configuration = Configuration() + self.local_vars_configuration = local_vars_configuration + + self._before_retry_generation = None + self._archived_at = None + self._start_time = None + self._end_time = None + self._state = None + self._pods = None + self._log_artifacts = None + self._driver_logs_uri = None + self.discriminator = None + + if before_retry_generation is not None: + self.before_retry_generation = before_retry_generation + if archived_at is not None: + self.archived_at = archived_at + if start_time is not None: + self.start_time = start_time + if end_time is not None: + self.end_time = end_time + if state is not None: + self.state = state + if pods is not None: + self.pods = pods + if log_artifacts is not None: + self.log_artifacts = log_artifacts + if driver_logs_uri is not None: + self.driver_logs_uri = driver_logs_uri + + @property + def before_retry_generation(self): + """Gets the before_retry_generation of this PipelineTaskLogHistory. # noqa: E501 + + Together with task_id, identifies this snapshot. This is the manual retry generation that archived it, not an automatic pod retry attempt number. # noqa: E501 + + :return: The before_retry_generation of this PipelineTaskLogHistory. # noqa: E501 + :rtype: str + """ + return self._before_retry_generation + + @before_retry_generation.setter + def before_retry_generation(self, before_retry_generation): + """Sets the before_retry_generation of this PipelineTaskLogHistory. + + Together with task_id, identifies this snapshot. This is the manual retry generation that archived it, not an automatic pod retry attempt number. # noqa: E501 + + :param before_retry_generation: The before_retry_generation of this PipelineTaskLogHistory. # noqa: E501 + :type before_retry_generation: str + """ + + self._before_retry_generation = before_retry_generation + + @property + def archived_at(self): + """Gets the archived_at of this PipelineTaskLogHistory. # noqa: E501 + + + :return: The archived_at of this PipelineTaskLogHistory. # noqa: E501 + :rtype: datetime + """ + return self._archived_at + + @archived_at.setter + def archived_at(self, archived_at): + """Sets the archived_at of this PipelineTaskLogHistory. + + + :param archived_at: The archived_at of this PipelineTaskLogHistory. # noqa: E501 + :type archived_at: datetime + """ + + self._archived_at = archived_at + + @property + def start_time(self): + """Gets the start_time of this PipelineTaskLogHistory. # noqa: E501 + + + :return: The start_time of this PipelineTaskLogHistory. # noqa: E501 + :rtype: datetime + """ + return self._start_time + + @start_time.setter + def start_time(self, start_time): + """Sets the start_time of this PipelineTaskLogHistory. + + + :param start_time: The start_time of this PipelineTaskLogHistory. # noqa: E501 + :type start_time: datetime + """ + + self._start_time = start_time + + @property + def end_time(self): + """Gets the end_time of this PipelineTaskLogHistory. # noqa: E501 + + + :return: The end_time of this PipelineTaskLogHistory. # noqa: E501 + :rtype: datetime + """ + return self._end_time + + @end_time.setter + def end_time(self, end_time): + """Sets the end_time of this PipelineTaskLogHistory. + + + :param end_time: The end_time of this PipelineTaskLogHistory. # noqa: E501 + :type end_time: datetime + """ + + self._end_time = end_time + + @property + def state(self): + """Gets the state of this PipelineTaskLogHistory. # noqa: E501 + + + :return: The state of this PipelineTaskLogHistory. # noqa: E501 + :rtype: PipelineTaskTaskState + """ + return self._state + + @state.setter + def state(self, state): + """Sets the state of this PipelineTaskLogHistory. + + + :param state: The state of this PipelineTaskLogHistory. # noqa: E501 + :type state: PipelineTaskTaskState + """ + + self._state = state + + @property + def pods(self): + """Gets the pods of this PipelineTaskLogHistory. # noqa: E501 + + + :return: The pods of this PipelineTaskLogHistory. # noqa: E501 + :rtype: list[PipelineTaskTaskPod] + """ + return self._pods + + @pods.setter + def pods(self, pods): + """Sets the pods of this PipelineTaskLogHistory. + + + :param pods: The pods of this PipelineTaskLogHistory. # noqa: E501 + :type pods: list[PipelineTaskTaskPod] + """ + + self._pods = pods + + @property + def log_artifacts(self): + """Gets the log_artifacts of this PipelineTaskLogHistory. # noqa: E501 + + Log artifacts only; never includes task outputs used by downstream tasks. # noqa: E501 + + :return: The log_artifacts of this PipelineTaskLogHistory. # noqa: E501 + :rtype: list[InputOutputsIOArtifact] + """ + return self._log_artifacts + + @log_artifacts.setter + def log_artifacts(self, log_artifacts): + """Sets the log_artifacts of this PipelineTaskLogHistory. + + Log artifacts only; never includes task outputs used by downstream tasks. # noqa: E501 + + :param log_artifacts: The log_artifacts of this PipelineTaskLogHistory. # noqa: E501 + :type log_artifacts: list[InputOutputsIOArtifact] + """ + + self._log_artifacts = log_artifacts + + @property + def driver_logs_uri(self): + """Gets the driver_logs_uri of this PipelineTaskLogHistory. # noqa: E501 + + Legacy driver log reference, when not represented by a log artifact. # noqa: E501 + + :return: The driver_logs_uri of this PipelineTaskLogHistory. # noqa: E501 + :rtype: str + """ + return self._driver_logs_uri + + @driver_logs_uri.setter + def driver_logs_uri(self, driver_logs_uri): + """Sets the driver_logs_uri of this PipelineTaskLogHistory. + + Legacy driver log reference, when not represented by a log artifact. # noqa: E501 + + :param driver_logs_uri: The driver_logs_uri of this PipelineTaskLogHistory. # noqa: E501 + :type driver_logs_uri: str + """ + + self._driver_logs_uri = driver_logs_uri + + def to_dict(self): + """Returns the model properties as a dict""" + result = {} + + for attr, _ in six.iteritems(self.openapi_types): + value = getattr(self, attr) + if isinstance(value, list): + result[attr] = list(map( + lambda x: x.to_dict() if hasattr(x, "to_dict") else x, + value + )) + elif hasattr(value, "to_dict"): + result[attr] = value.to_dict() + elif isinstance(value, dict): + result[attr] = dict(map( + lambda item: (item[0], item[1].to_dict()) + if hasattr(item[1], "to_dict") else item, + value.items() + )) + else: + result[attr] = value + + return result + + def to_str(self): + """Returns the string representation of the model""" + return pprint.pformat(self.to_dict()) + + def __repr__(self): + """For `print` and `pprint`""" + return self.to_str() + + def __eq__(self, other): + """Returns true if both objects are equal""" + if not isinstance(other, PipelineTaskLogHistory): + return False + + return self.to_dict() == other.to_dict() + + def __ne__(self, other): + """Returns true if both objects are not equal""" + if not isinstance(other, PipelineTaskLogHistory): + return True + + return self.to_dict() != other.to_dict() diff --git a/sdk/python/kfp/server_api/models/v2_pipeline_task.py b/sdk/python/kfp/server_api/models/v2_pipeline_task.py index e77b8b065df..b1796e3ae9e 100644 --- a/sdk/python/kfp/server_api/models/v2_pipeline_task.py +++ b/sdk/python/kfp/server_api/models/v2_pipeline_task.py @@ -52,7 +52,8 @@ class V2PipelineTask(object): 'outputs': 'PipelineTaskInputOutputs', 'scope_path': 'str', 'lifecycle_message': 'str', - 'lifecycle_category': 'str' + 'lifecycle_category': 'str', + 'log_history': 'list[PipelineTaskLogHistory]' } attribute_map = { @@ -75,10 +76,11 @@ class V2PipelineTask(object): 'outputs': 'outputs', 'scope_path': 'scope_path', 'lifecycle_message': 'lifecycle_message', - 'lifecycle_category': 'lifecycle_category' + 'lifecycle_category': 'lifecycle_category', + 'log_history': 'log_history' } - def __init__(self, name=None, display_name=None, task_id=None, run_id=None, pods=None, cache_fingerprint=None, create_time=None, end_time=None, state=None, status_metadata=None, state_history=None, type=None, type_attributes=None, parent_task_id=None, child_tasks=None, inputs=None, outputs=None, scope_path=None, lifecycle_message=None, lifecycle_category=None, local_vars_configuration=None): # noqa: E501 + def __init__(self, name=None, display_name=None, task_id=None, run_id=None, pods=None, cache_fingerprint=None, create_time=None, end_time=None, state=None, status_metadata=None, state_history=None, type=None, type_attributes=None, parent_task_id=None, child_tasks=None, inputs=None, outputs=None, scope_path=None, lifecycle_message=None, lifecycle_category=None, log_history=None, local_vars_configuration=None): # noqa: E501 """V2PipelineTask - a model defined in OpenAPI""" # noqa: E501 if local_vars_configuration is None: local_vars_configuration = Configuration() @@ -104,6 +106,7 @@ def __init__(self, name=None, display_name=None, task_id=None, run_id=None, pods self._scope_path = None self._lifecycle_message = None self._lifecycle_category = None + self._log_history = None self.discriminator = None if name is not None: @@ -146,6 +149,8 @@ def __init__(self, name=None, display_name=None, task_id=None, run_id=None, pods self.lifecycle_message = lifecycle_message if lifecycle_category is not None: self.lifecycle_category = lifecycle_category + if log_history is not None: + self.log_history = log_history @property def name(self): @@ -587,6 +592,29 @@ def lifecycle_category(self, lifecycle_category): self._lifecycle_category = lifecycle_category + @property + def log_history(self): + """Gets the log_history of this V2PipelineTask. # noqa: E501 + + Output only. Prior executions archived by manual retries, oldest first. History begins when preservation is enabled; older retries are not backfilled. # noqa: E501 + + :return: The log_history of this V2PipelineTask. # noqa: E501 + :rtype: list[PipelineTaskLogHistory] + """ + return self._log_history + + @log_history.setter + def log_history(self, log_history): + """Sets the log_history of this V2PipelineTask. + + Output only. Prior executions archived by manual retries, oldest first. History begins when preservation is enabled; older retries are not backfilled. # noqa: E501 + + :param log_history: The log_history of this V2PipelineTask. # noqa: E501 + :type log_history: list[PipelineTaskLogHistory] + """ + + self._log_history = log_history + def to_dict(self): """Returns the model properties as a dict""" result = {}