diff --git a/client/llb/marshal.go b/client/llb/marshal.go index d447031108a4..58fff4e364a7 100644 --- a/client/llb/marshal.go +++ b/client/llb/marshal.go @@ -7,6 +7,7 @@ import ( cerrdefs "github.com/containerd/errdefs" "github.com/containerd/platforms" + "github.com/moby/buildkit/identity" "github.com/moby/buildkit/solver/pb" digest "github.com/opencontainers/go-digest" "google.golang.org/protobuf/proto" @@ -109,12 +110,16 @@ func MarshalConstraints(base, override *Constraints) (*pb.Op, *pb.OpMetadata) { opPlatform.OSFeatures = slices.Clone(c.Platform.OSFeatures) } - return &pb.Op{ + op := &pb.Op{ Platform: &opPlatform, Constraints: &pb.WorkerConstraints{ Filter: c.WorkerConstraints, }, - }, c.Metadata.ToPB() + } + if c.GenerateIdentities { + op.Identity = identity.NewID() + } + return op, c.Metadata.ToPB() } type MarshalCache struct { diff --git a/client/llb/state.go b/client/llb/state.go index 7e5022c00a1c..9a02750c128d 100644 --- a/client/llb/state.go +++ b/client/llb/state.go @@ -644,6 +644,14 @@ func WithCaps(caps apicaps.CapSet) ConstraintsOpt { }) } +// WithIdentities will add an identity to this operation that +// will force it to be unique. +func WithIdentities() ConstraintsOpt { + return constraintsOptFunc(func(c *Constraints) { + c.GenerateIdentities = true + }) +} + type constraintsWrapper struct { Constraints } @@ -653,12 +661,13 @@ func (cw *constraintsWrapper) applyConstraints(f func(c *Constraints)) { } type Constraints struct { - Platform *ocispecs.Platform - WorkerConstraints []string - Metadata OpMetadata - LocalUniqueID string - Caps *apicaps.CapSet - SourceLocations []*SourceLocation + Platform *ocispecs.Platform + WorkerConstraints []string + Metadata OpMetadata + LocalUniqueID string + Caps *apicaps.CapSet + SourceLocations []*SourceLocation + GenerateIdentities bool } // OpMetadata has a more friendly interface for pb.OpMetadata. diff --git a/frontend/dockerfile/builder/build.go b/frontend/dockerfile/builder/build.go index 26b019df42d1..e2f7d1c99679 100644 --- a/frontend/dockerfile/builder/build.go +++ b/frontend/dockerfile/builder/build.go @@ -2,6 +2,7 @@ package builder import ( "context" + "strconv" "strings" "sync" @@ -30,7 +31,8 @@ import ( const ( // Don't forget to update frontend documentation if you add // a new build-arg: frontend/dockerfile/docs/reference.md - keySyntaxArg = "build-arg:BUILDKIT_SYNTAX" + keySyntaxArg = "build-arg:BUILDKIT_SYNTAX" + keyWithIdentities = "build-arg:BUILDKIT_WITH_IDENTITIES" ) func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) { @@ -67,6 +69,15 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) { } } + var withIdentities bool + if v, ok := opts[keyWithIdentities]; ok { + b, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Errorf("invalid boolean value for with identities: %s", v) + } + withIdentities = b + } + if capsError != nil { return nil, capsError } @@ -138,7 +149,12 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) { return nil, nil, nil, err } - def, err := st.Marshal(ctx) + var co []llb.ConstraintsOpt + if withIdentities { + co = append(co, llb.WithIdentities()) + } + + def, err := st.Marshal(ctx, co...) if err != nil { return nil, nil, nil, errors.Wrapf(err, "failed to marshal LLB definition") } diff --git a/frontend/dockerfile/docs/reference.md b/frontend/dockerfile/docs/reference.md index 5b32b7f01e13..3521aaa9ca3e 100644 --- a/frontend/dockerfile/docs/reference.md +++ b/frontend/dockerfile/docs/reference.md @@ -2617,6 +2617,7 @@ RUN echo "I'm building for $TARGETPLATFORM" | `BUILDKIT_MULTI_PLATFORM` | Bool | Opt into deterministic output regardless of multi-platform output or not. | | `BUILDKIT_SANDBOX_HOSTNAME` | String | Set the hostname (default `buildkitsandbox`) | | `BUILDKIT_SYNTAX` | String | Set frontend image | +| `BUILDKIT_WITH_IDENTITIES` | Bool | Add unique ids for each LLB operation to prevent collisions. | | `SOURCE_DATE_EPOCH` | Int | Set the Unix timestamp for created image and layers. More info from [reproducible builds](https://reproducible-builds.org/docs/source-date-epoch/). Supported since Dockerfile 1.5, BuildKit 0.11 | #### Example: keep `.git` dir diff --git a/solver/pb/json.go b/solver/pb/json.go index 63886c0c8511..b5c5b034857e 100644 --- a/solver/pb/json.go +++ b/solver/pb/json.go @@ -14,6 +14,7 @@ type jsonOp struct { } Platform *Platform `json:"platform,omitempty"` Constraints *WorkerConstraints `json:"constraints,omitempty"` + Identity string `json:"identity,omitempty"` } func (m *Op) MarshalJSON() ([]byte, error) { @@ -35,6 +36,7 @@ func (m *Op) MarshalJSON() ([]byte, error) { } v.Platform = m.Platform v.Constraints = m.Constraints + v.Identity = m.Identity return json.Marshal(v) } @@ -61,6 +63,7 @@ func (m *Op) UnmarshalJSON(data []byte) error { } m.Platform = v.Platform m.Constraints = v.Constraints + m.Identity = v.Identity return nil } diff --git a/solver/pb/ops.pb.go b/solver/pb/ops.pb.go index db85d91c2418..7d3644105755 100644 --- a/solver/pb/ops.pb.go +++ b/solver/pb/ops.pb.go @@ -295,6 +295,7 @@ type Op struct { Op isOp_Op `protobuf_oneof:"op"` Platform *Platform `protobuf:"bytes,10,opt,name=platform,proto3" json:"platform,omitempty"` Constraints *WorkerConstraints `protobuf:"bytes,11,opt,name=constraints,proto3" json:"constraints,omitempty"` + Identity string `protobuf:"bytes,12,opt,name=identity,proto3" json:"identity,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -411,6 +412,13 @@ func (x *Op) GetConstraints() *WorkerConstraints { return nil } +func (x *Op) GetIdentity() string { + if x != nil { + return x.Identity + } + return "" +} + type isOp_Op interface { isOp_Op() } @@ -3384,7 +3392,7 @@ var File_github_com_moby_buildkit_solver_pb_ops_proto protoreflect.FileDescripto const file_github_com_moby_buildkit_solver_pb_ops_proto_rawDesc = "" + "\n" + - ",github.com/moby/buildkit/solver/pb/ops.proto\x12\x02pb\"\xe8\x02\n" + + ",github.com/moby/buildkit/solver/pb/ops.proto\x12\x02pb\"\x84\x03\n" + "\x02Op\x12!\n" + "\x06inputs\x18\x01 \x03(\v2\t.pb.InputR\x06inputs\x12 \n" + "\x04exec\x18\x02 \x01(\v2\n" + @@ -3398,7 +3406,8 @@ const file_github_com_moby_buildkit_solver_pb_ops_proto_rawDesc = "" + ".pb.DiffOpH\x00R\x04diff\x12(\n" + "\bplatform\x18\n" + " \x01(\v2\f.pb.PlatformR\bplatform\x127\n" + - "\vconstraints\x18\v \x01(\v2\x15.pb.WorkerConstraintsR\vconstraintsB\x04\n" + + "\vconstraints\x18\v \x01(\v2\x15.pb.WorkerConstraintsR\vconstraints\x12\x1a\n" + + "\bidentity\x18\f \x01(\tR\bidentityB\x04\n" + "\x02op\"\x96\x01\n" + "\bPlatform\x12\"\n" + "\fArchitecture\x18\x01 \x01(\tR\fArchitecture\x12\x0e\n" + diff --git a/solver/pb/ops.proto b/solver/pb/ops.proto index 90b69560e005..423ad36f9f28 100644 --- a/solver/pb/ops.proto +++ b/solver/pb/ops.proto @@ -21,6 +21,7 @@ message Op { } Platform platform = 10; WorkerConstraints constraints = 11; + string identity = 12; } // Platform is github.com/opencontainers/image-spec/specs-go/v1.Platform diff --git a/solver/pb/ops_vtproto.pb.go b/solver/pb/ops_vtproto.pb.go index 1a6c89c3690f..f73c480aa70d 100644 --- a/solver/pb/ops_vtproto.pb.go +++ b/solver/pb/ops_vtproto.pb.go @@ -26,6 +26,7 @@ func (m *Op) CloneVT() *Op { r := new(Op) r.Platform = m.Platform.CloneVT() r.Constraints = m.Constraints.CloneVT() + r.Identity = m.Identity if rhs := m.Inputs; rhs != nil { tmpContainer := make([]*Input, len(rhs)) for k, v := range rhs { @@ -1186,6 +1187,9 @@ func (this *Op) EqualVT(that *Op) bool { if !this.Constraints.EqualVT(that.Constraints) { return false } + if this.Identity != that.Identity { + return false + } return string(this.unknownFields) == string(that.unknownFields) } @@ -2966,6 +2970,13 @@ func (m *Op) MarshalToSizedBufferVT(dAtA []byte) (int, error) { } i -= size } + if len(m.Identity) > 0 { + i -= len(m.Identity) + copy(dAtA[i:], m.Identity) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Identity))) + i-- + dAtA[i] = 0x62 + } if m.Constraints != nil { size, err := m.Constraints.MarshalToSizedBufferVT(dAtA[:i]) if err != nil { @@ -5984,6 +5995,10 @@ func (m *Op) SizeVT() (n int) { l = m.Constraints.SizeVT() n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + l = len(m.Identity) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } n += len(m.unknownFields) return n } @@ -7579,6 +7594,38 @@ func (m *Op) UnmarshalVT(dAtA []byte) error { return err } iNdEx = postIndex + case 12: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Identity", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Identity = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:])