diff --git a/cmd/buildah/commit.go b/cmd/buildah/commit.go index e32cc979765..f92563a03c7 100644 --- a/cmd/buildah/commit.go +++ b/cmd/buildah/commit.go @@ -19,7 +19,9 @@ import ( "go.podman.io/buildah/util" "go.podman.io/common/pkg/auth" "go.podman.io/common/pkg/completion" + "go.podman.io/common/pkg/config" "go.podman.io/image/v5/manifest" + "go.podman.io/image/v5/pkg/compression" "go.podman.io/image/v5/pkg/shortnames" storageTransport "go.podman.io/image/v5/storage" "go.podman.io/image/v5/transports/alltransports" @@ -27,49 +29,52 @@ import ( ) type commitInputOptions struct { - authfile string - omitHistory bool - blobCache string - certDir string - changes []string - configFile string - creds string - cwOptions string - disableCompression bool - format string - iidfile string - manifest string - omitTimestamp bool - timestamp int64 - sourceDateEpoch string - rewriteTimestamp bool - quiet bool - referenceTime string - rm bool - pull string - pullAlways bool - pullNever bool - sbomImgOutput string - sbomImgPurlOutput string - sbomMergeStrategy string - sbomOutput string - sbomPreset string - sbomPurlOutput string - sbomScannerCommand []string - sbomScannerImage string - signaturePolicy string - signBy string - squash bool - tlsVerify bool - identityLabel bool - encryptionKeys []string - encryptLayers []int - unsetenvs []string - addFile []string - unsetAnnotation []string - annotation []string - createdAnnotation bool - metadataFile string + authfile string + omitHistory bool + blobCache string + certDir string + changes []string + compressionFormat string + compressionLevel int + configFile string + creds string + cwOptions string + disableCompression bool + forceCompressionFormat bool + format string + iidfile string + manifest string + omitTimestamp bool + timestamp int64 + sourceDateEpoch string + rewriteTimestamp bool + quiet bool + referenceTime string + rm bool + pull string + pullAlways bool + pullNever bool + sbomImgOutput string + sbomImgPurlOutput string + sbomMergeStrategy string + sbomOutput string + sbomPreset string + sbomPurlOutput string + sbomScannerCommand []string + sbomScannerImage string + signaturePolicy string + signBy string + squash bool + tlsVerify bool + identityLabel bool + encryptionKeys []string + encryptLayers []int + unsetenvs []string + addFile []string + unsetAnnotation []string + annotation []string + createdAnnotation bool + metadataFile string } func commitInit() { @@ -117,8 +122,13 @@ func commitListFlagSet(cmd *cobra.Command, opts *commitInputOptions) { _ = cmd.RegisterFlagCompletionFunc("config", completion.AutocompleteDefault) flags.StringVar(&opts.creds, "creds", "", "use `[username[:password]]` for accessing the registry") _ = cmd.RegisterFlagCompletionFunc("creds", completion.AutocompleteNone) + flags.StringVar(&opts.compressionFormat, "compression-format", "", "compression format to use") + _ = cmd.RegisterFlagCompletionFunc("compression-format", completion.AutocompleteNone) + flags.IntVar(&opts.compressionLevel, "compression-level", 0, "compression level to use") + _ = cmd.RegisterFlagCompletionFunc("compression-level", completion.AutocompleteNone) flags.StringVar(&opts.cwOptions, "cw", "", "confidential workload `options`") flags.BoolVarP(&opts.disableCompression, "disable-compression", "D", true, "don't compress layers") + flags.BoolVar(&opts.forceCompressionFormat, "force-compression", false, "use the specified compression algorithm if the destination contains a differently-compressed variant already") flags.StringVarP(&opts.format, "format", "f", defaultFormat(), "`format` of the image manifest and metadata") _ = cmd.RegisterFlagCompletionFunc("format", completion.AutocompleteNone) flags.StringVar(&opts.manifest, "manifest", "", "adds created image to the specified manifest list. Creates manifest list if it does not exist") @@ -228,6 +238,15 @@ func commitCmd(c *cobra.Command, args []string, iopts commitInputOptions) error compress = define.Uncompressed } + if c.Flag("disable-compression").Changed && iopts.disableCompression { + if c.Flag("compression-format").Changed { + return errors.New("--disable-compression and --compression-format cannot be used together") + } + if c.Flag("force-compression").Changed { + return errors.New("--disable-compression and --force-compression cannot be used together") + } + } + format, err := cli.GetFormat(iopts.format) if err != nil { return err @@ -308,25 +327,54 @@ func commitCmd(c *cobra.Command, args []string, iopts commitInputOptions) error } options := buildah.CommitOptions{ - PreferredManifestType: format, - Manifest: iopts.manifest, - Compression: compress, - SignaturePolicyPath: iopts.signaturePolicy, - SystemContext: systemContext, - IIDFile: iopts.iidfile, - Squash: iopts.squash, - BlobDirectory: iopts.blobCache, - OmitHistory: iopts.omitHistory, - SignBy: iopts.signBy, - OciEncryptConfig: encConfig, - OciEncryptLayers: encLayers, - UnsetEnvs: iopts.unsetenvs, - OverrideChanges: iopts.changes, - OverrideConfig: overrideConfig, - ExtraImageContent: addFiles, - UnsetAnnotations: iopts.unsetAnnotation, - Annotations: iopts.annotation, - CreatedAnnotation: types.NewOptionalBool(iopts.createdAnnotation), + PreferredManifestType: format, + Manifest: iopts.manifest, + Compression: compress, + ForceCompressionFormat: iopts.forceCompressionFormat, + SignaturePolicyPath: iopts.signaturePolicy, + SystemContext: systemContext, + IIDFile: iopts.iidfile, + Squash: iopts.squash, + BlobDirectory: iopts.blobCache, + OmitHistory: iopts.omitHistory, + SignBy: iopts.signBy, + OciEncryptConfig: encConfig, + OciEncryptLayers: encLayers, + UnsetEnvs: iopts.unsetenvs, + OverrideChanges: iopts.changes, + OverrideConfig: overrideConfig, + ExtraImageContent: addFiles, + UnsetAnnotations: iopts.unsetAnnotation, + Annotations: iopts.annotation, + CreatedAnnotation: types.NewOptionalBool(iopts.createdAnnotation), + } + defaultContainerConfig, err := config.Default() + if err != nil { + return fmt.Errorf("failed to get container config: %w", err) + } + if iopts.compressionFormat != "" { + algo, err := compression.AlgorithmByName(iopts.compressionFormat) + if err != nil { + return err + } + options.CompressionFormat = &algo + if !c.Flag("force-compression").Changed { + options.ForceCompressionFormat = true + } + } else if defaultContainerConfig.Engine.CompressionFormat != "" && defaultContainerConfig.Engine.CompressionFormat != "gzip" { + algo, err := compression.AlgorithmByName(defaultContainerConfig.Engine.CompressionFormat) + if err != nil { + return fmt.Errorf("parsing compression_format from containers.conf: %w", err) + } + options.CompressionFormat = &algo + if !c.Flag("force-compression").Changed { + options.ForceCompressionFormat = true + } + } + if c.Flag("compression-level").Changed { + options.CompressionLevel = &iopts.compressionLevel + } else { + options.CompressionLevel = defaultContainerConfig.Engine.CompressionLevel } exclusiveFlags := 0 if c.Flag("reference-time").Changed { diff --git a/cmd/buildah/push.go b/cmd/buildah/push.go index 51b84cf3b9e..8c38e09f099 100644 --- a/cmd/buildah/push.go +++ b/cmd/buildah/push.go @@ -16,6 +16,7 @@ import ( "go.podman.io/buildah/pkg/parse" util "go.podman.io/buildah/util" "go.podman.io/common/pkg/auth" + "go.podman.io/common/pkg/config" "go.podman.io/image/v5/manifest" "go.podman.io/image/v5/pkg/compression" "go.podman.io/image/v5/transports" @@ -194,14 +195,6 @@ func pushCmd(c *cobra.Command, args []string, iopts pushOptions) error { return fmt.Errorf("unable to obtain encryption config: %w", err) } - if c.Flag("compression-format").Changed { - if !c.Flag("force-compression").Changed { - // If `compression-format` is set and no value for `--force-compression` - // is selected then defaults to `true`. - iopts.forceCompressionFormat = true - } - } - options := buildah.PushOptions{ Compression: compress, ManifestType: manifestType, @@ -225,15 +218,33 @@ func pushCmd(c *cobra.Command, args []string, iopts pushOptions) error { if !iopts.quiet { options.ReportWriter = os.Stderr } + defaultContainerConfig, err := config.Default() + if err != nil { + return fmt.Errorf("failed to get container config: %w", err) + } if iopts.compressionFormat != "" { algo, err := compression.AlgorithmByName(iopts.compressionFormat) if err != nil { return err } options.CompressionFormat = &algo + if !c.Flag("force-compression").Changed { + options.ForceCompressionFormat = true + } + } else if defaultContainerConfig.Engine.CompressionFormat != "" && defaultContainerConfig.Engine.CompressionFormat != "gzip" { + algo, err := compression.AlgorithmByName(defaultContainerConfig.Engine.CompressionFormat) + if err != nil { + return fmt.Errorf("parsing compression_format from containers.conf: %w", err) + } + options.CompressionFormat = &algo + if !c.Flag("force-compression").Changed { + options.ForceCompressionFormat = true + } } if c.Flag("compression-level").Changed { options.CompressionLevel = &iopts.compressionLevel + } else { + options.CompressionLevel = defaultContainerConfig.Engine.CompressionLevel } ref, digest, err := buildah.Push(getContext(), src, dest, options) diff --git a/commit.go b/commit.go index 4e7b82f09e7..9a616cae0b9 100644 --- a/commit.go +++ b/commit.go @@ -22,6 +22,7 @@ import ( "go.podman.io/image/v5/docker" "go.podman.io/image/v5/docker/reference" "go.podman.io/image/v5/manifest" + "go.podman.io/image/v5/pkg/compression" "go.podman.io/image/v5/signature" is "go.podman.io/image/v5/storage" "go.podman.io/image/v5/transports" @@ -47,6 +48,14 @@ type CommitOptions struct { // layer blobs. The default is to not use compression, but // archive.Gzip is recommended. Compression archive.Compression + // CompressionFormat is the format to use for the compression of the blobs. + CompressionFormat *compression.Algorithm + // CompressionLevel specifies what compression level is used. + CompressionLevel *int + // ForceCompressionFormat ensures that the compression algorithm set in + // CompressionFormat is used exclusively, and blobs of other compression + // algorithms are not reused. + ForceCompressionFormat bool // SignaturePolicyPath specifies an override location for the signature // policy which should be used for verifying the new image as it is // being written. Except in specific circumstances, no value should be @@ -456,12 +465,16 @@ func (b *Builder) CommitResults(ctx context.Context, dest types.ImageReference, if options.Compression != archive.Uncompressed { compress = types.Compress } - cache, err := blobcache.NewBlobCache(src, options.BlobDirectory, compress) + var cacheOpts []blobcache.Option + if options.CompressionFormat != nil { + cacheOpts = append(cacheOpts, blobcache.WithCompressAlgorithm(options.CompressionFormat)) + } + cache, err := blobcache.NewBlobCache(src, options.BlobDirectory, compress, cacheOpts...) if err != nil { return nil, fmt.Errorf("wrapping image reference %q in blob cache at %q: %w", transports.ImageName(src), options.BlobDirectory, err) } maybeCachedSrc = cache - cache, err = blobcache.NewBlobCache(dest, options.BlobDirectory, compress) + cache, err = blobcache.NewBlobCache(dest, options.BlobDirectory, compress, cacheOpts...) if err != nil { return nil, fmt.Errorf("wrapping image reference %q in blob cache at %q: %w", transports.ImageName(dest), options.BlobDirectory, err) } @@ -471,8 +484,9 @@ func (b *Builder) CommitResults(ctx context.Context, dest types.ImageReference, switch options.Compression { case archive.Uncompressed: systemContext.OCIAcceptUncompressedLayers = true - case archive.Gzip: + default: systemContext.DirForceCompress = true + systemContext.OCIAcceptUncompressedLayers = false } if systemContext.ArchitectureChoice != b.Architecture() { @@ -482,8 +496,16 @@ func (b *Builder) CommitResults(ctx context.Context, dest types.ImageReference, systemContext.OSChoice = b.OS() } + copyOptions := getCopyOptions(b.store, options.ReportWriter, nil, systemContext, "", false, options.SignBy, options.OciEncryptLayers, options.OciEncryptConfig, nil, destinationTimestamp) + if options.CompressionFormat != nil { + copyOptions.DestinationCtx.CompressionFormat = options.CompressionFormat + copyOptions.DestinationCtx.CompressionLevel = options.CompressionLevel + } + if options.ForceCompressionFormat { + copyOptions.ForceCompressionFormat = true + } var manifestBytes []byte - if manifestBytes, err = retryCopyImage(ctx, policyContext, maybeCachedDest, maybeCachedSrc, dest, getCopyOptions(b.store, options.ReportWriter, nil, systemContext, "", false, options.SignBy, options.OciEncryptLayers, options.OciEncryptConfig, nil, destinationTimestamp), options.MaxRetries, options.RetryDelay); err != nil { + if manifestBytes, err = retryCopyImage(ctx, policyContext, maybeCachedDest, maybeCachedSrc, dest, copyOptions, options.MaxRetries, options.RetryDelay); err != nil { return nil, fmt.Errorf("copying layers and metadata for container %q: %w", b.ContainerID, err) } // If we've got more names to attach, and we know how to do that for diff --git a/define/build.go b/define/build.go index bd44383a8cf..1a452067780 100644 --- a/define/build.go +++ b/define/build.go @@ -8,6 +8,7 @@ import ( "go.podman.io/common/libimage/manifests" nettypes "go.podman.io/common/libnetwork/types" "go.podman.io/image/v5/docker/reference" + "go.podman.io/image/v5/pkg/compression" "go.podman.io/image/v5/types" "go.podman.io/storage/pkg/archive" "golang.org/x/sync/semaphore" @@ -165,6 +166,14 @@ type BuildOptions struct { // layer blobs. The default is to not use compression, but // archive.Gzip is recommended. Compression archive.Compression + // CompressionFormat is the format to use for the compression of the blobs. + CompressionFormat *compression.Algorithm + // CompressionLevel specifies what compression level is used. + CompressionLevel *int + // ForceCompressionFormat ensures that the compression algorithm set in + // CompressionFormat is used exclusively, and blobs of other compression + // algorithms are not reused. + ForceCompressionFormat bool // Arguments which can be interpolated into Dockerfiles Args map[string]string // Map of external additional build contexts diff --git a/docs/buildah-build.1.md b/docs/buildah-build.1.md index 6667a0ef6ac..91892d72bd4 100644 --- a/docs/buildah-build.1.md +++ b/docs/buildah-build.1.md @@ -218,6 +218,20 @@ This option is added to be aligned with other containers CLIs. Buildah doesn't send a copy of the context directory to a daemon or a remote server. Thus, compressing the data before sending it is irrelevant to Buildah. +**--compression-format** *format* + +Specifies the compression format to use. Supported values are: `gzip`, `zstd` and `zstd:chunked`. +`zstd:chunked` is incompatible with encrypting images, and will be treated as `zstd` with a warning in that case. +If not specified, the format is read from the `compression_format` setting in containers.conf. + +This option affects cache pushes with `--cache-to` and the final image when it is written to a non-local destination (e.g., `dir:`, `oci:`, `oci-archive:`, or a registry). +When the output is local container storage (the default), layers are always decompressed on ingest, so compression is applied at `buildah push` time instead. + +**--compression-level** *level* + +Specifies the compression level to use. The value is specific to the compression algorithm used, e.g. for zstd the accepted values are in the range 1-20 (inclusive), while for gzip it is 1-9 (inclusive). +If not specified, the level is read from the `compression_level` setting in containers.conf. + **--cpp-flag**="" Set additional flags to pass to the C Preprocessor cpp(1). @@ -444,6 +458,11 @@ context directory will be prepended to the local file value. If you specify `-f -`, the Containerfile contents will be read from stdin. +**--force-compression** + +If set, uses the specified compression algorithm even if the destination contains a differently-compressed variant already. +Defaults to `true` if `--compression-format` is explicitly specified on the command-line or `compression_format` is set in containers.conf, `false` otherwise. + **--force-rm** *bool-value* Always remove intermediate containers after a build, even if the build fails (default false). diff --git a/docs/buildah-commit.1.md b/docs/buildah-commit.1.md index 532916f87df..2e6ba6a74f6 100644 --- a/docs/buildah-commit.1.md +++ b/docs/buildah-commit.1.md @@ -56,6 +56,17 @@ Apply the change to the committed image that would have been made if it had been built using a Containerfile which included the specified instruction. This option can be specified multiple times. +**--compression-format** *format* + +Specifies the compression format to use. Supported values are: `gzip`, `zstd` and `zstd:chunked`. +If not specified, the format is read from the `compression_format` setting in containers.conf. +Cannot be used together with **--disable-compression**. + +**--compression-level** *level* + +Specifies the compression level to use. The value is specific to the compression algorithm used, e.g. for zstd the accepted values are in the range 1-20 (inclusive), while for gzip it is 1-9 (inclusive). +If not specified, the level is read from the `compression_level` setting in containers.conf. + **--config** *filename* Read a JSON-encoded version of an image configuration object from the specified @@ -143,6 +154,7 @@ because image layers are compressed automatically when they are pushed to registries, and images being written to local storage would only need to be decompressed again to be stored. Compression can be forced in all cases by specifying **--disable-compression=false**. +Cannot be used together with **--compression-format** or **--force-compression**. **--encrypt-layer** *layer(s)* @@ -152,6 +164,11 @@ Layer(s) to encrypt: 0-indexed layer indices with support for negative indexing The [protocol:keyfile] specifies the encryption protocol, which can be JWE (RFC7516), PGP (RFC4880), and PKCS7 (RFC2315) and the key material required for image encryption. For instance, jwe:/path/to/key.pem or pgp:admin@example.com or pkcs7:/path/to/x509-file. +**--force-compression** + +If set, commit uses the specified compression algorithm even if the destination contains a differently-compressed variant already. +Defaults to `true` if **--compression-format** is explicitly specified on the command-line or `compression_format` is set in containers.conf, `false` otherwise. + **--format**, **-f** *[oci | docker]* Control the format for the image manifest and configuration data. Recognized diff --git a/docs/buildah-push.1.md b/docs/buildah-push.1.md index 5d000693ada..7dffd9aaf9d 100644 --- a/docs/buildah-push.1.md +++ b/docs/buildah-push.1.md @@ -42,12 +42,12 @@ The default certificates directory is _/etc/containers/certs.d_. Specifies the compression format to use. Supported values are: `gzip`, `zstd` and `zstd:chunked`. `zstd:chunked` is incompatible with encrypting images, and will be treated as `zstd` with a warning in that case. +If not specified, the format is read from the `compression_format` setting in containers.conf. **--compression-level** *level* -Specify the compression level used with the compression. - Specifies the compression level to use. The value is specific to the compression algorithm used, e.g. for zstd the accepted values are in the range 1-20 (inclusive), while for gzip it is 1-9 (inclusive). +If not specified, the level is read from the `compression_level` setting in containers.conf. **--creds** *creds* @@ -74,7 +74,7 @@ The [protocol:keyfile] specifies the encryption protocol, which can be JWE (RFC7 **--force-compression** If set, push uses the specified compression algorithm even if the destination contains a differently-compressed variant already. -Defaults to `true` if `--compression-format` is explicitly specified on the command-line, `false` otherwise. +Defaults to `true` if `--compression-format` is explicitly specified on the command-line or `compression_format` is set in containers.conf, `false` otherwise. **--format**, **-f** diff --git a/imagebuildah/executor.go b/imagebuildah/executor.go index e3d49f034e0..737a6335ead 100644 --- a/imagebuildah/executor.go +++ b/imagebuildah/executor.go @@ -33,6 +33,7 @@ import ( "go.podman.io/common/pkg/config" "go.podman.io/image/v5/docker/reference" "go.podman.io/image/v5/manifest" + "go.podman.io/image/v5/pkg/compression" storageTransport "go.podman.io/image/v5/storage" "go.podman.io/image/v5/transports" "go.podman.io/image/v5/transports/alltransports" @@ -84,6 +85,9 @@ type executor struct { transientMounts []Mount transientRunMounts []string compression archive.Compression + compressionFormat *compression.Algorithm + compressionLevel *int + forceCompressionFormat bool output string outputFormat string additionalTags []string @@ -286,6 +290,9 @@ func newExecutor(logger *logrus.Logger, logPrefix string, store storage.Store, o transientMounts: transientMounts, transientRunMounts: options.TransientRunMounts, compression: options.Compression, + compressionFormat: options.CompressionFormat, + compressionLevel: options.CompressionLevel, + forceCompressionFormat: options.ForceCompressionFormat, output: options.Output, outputFormat: options.OutputFormat, additionalTags: options.AdditionalTags, diff --git a/imagebuildah/stage_executor.go b/imagebuildah/stage_executor.go index 67c728b621d..58df665fb37 100644 --- a/imagebuildah/stage_executor.go +++ b/imagebuildah/stage_executor.go @@ -2390,14 +2390,17 @@ func (s *stageExecutor) pushCache(ctx context.Context, src, cacheKey string) err for _, dest := range destList { logrus.Debugf("trying to push cache to dest: %+v from src:%+v", dest, src) options := buildah.PushOptions{ - Compression: s.executor.compression, - SignaturePolicyPath: s.executor.signaturePolicyPath, - Store: s.executor.store, - SystemContext: s.systemContext, - BlobDirectory: s.executor.blobDirectory, - SignBy: s.executor.signBy, - MaxRetries: s.executor.maxPullPushRetries, - RetryDelay: s.executor.retryPullPushDelay, + Compression: s.executor.compression, + CompressionFormat: s.executor.compressionFormat, + CompressionLevel: s.executor.compressionLevel, + ForceCompressionFormat: s.executor.forceCompressionFormat, + SignaturePolicyPath: s.executor.signaturePolicyPath, + Store: s.executor.store, + SystemContext: s.systemContext, + BlobDirectory: s.executor.blobDirectory, + SignBy: s.executor.signBy, + MaxRetries: s.executor.maxPullPushRetries, + RetryDelay: s.executor.retryPullPushDelay, } if s.executor.cachePushSourceLookupReferenceFunc != nil { options.SourceLookupReferenceFunc = s.executor.cachePushSourceLookupReferenceFunc(dest) @@ -2729,6 +2732,15 @@ func (s *stageExecutor) commit(ctx context.Context, createdBy string, emptyLayer if finalInstruction { options.ConfidentialWorkloadOptions = s.executor.confidentialWorkload options.SBOMScanOptions = s.executor.sbomScanOptions + // Apply compression settings only when committing to a non-local + // transport (registry, dir:, oci-archive:, etc.). Local storage + // decompresses layers on write, so specifying compression there + // would have no effect. + if imageRef != nil && imageRef.Transport().Name() != is.Transport.Name() { + options.CompressionFormat = s.executor.compressionFormat + options.CompressionLevel = s.executor.compressionLevel + options.ForceCompressionFormat = s.executor.forceCompressionFormat + } } results, err := s.builder.CommitResults(ctx, imageRef, options) if err != nil { diff --git a/pkg/blobcache/blobcache.go b/pkg/blobcache/blobcache.go index b2723c1b3e6..ad8bc7e87a9 100644 --- a/pkg/blobcache/blobcache.go +++ b/pkg/blobcache/blobcache.go @@ -5,6 +5,13 @@ import ( "go.podman.io/image/v5/types" ) +// Option configures optional BlobCache behavior. +type Option = imageBlobCache.BlobCacheOption + +// WithCompressAlgorithm sets compression algorithm for compressed blobs +// when compress is set to Compress. +var WithCompressAlgorithm = imageBlobCache.WithCompressAlgorithm + // BlobCache is an object which saves copies of blobs that are written to it while passing them // through to some real destination, and which can be queried directly in order to read them // back. @@ -26,6 +33,7 @@ type BlobCache interface { // as-is to the specified directory or a temporary directory. // The compress argument controls whether or not the cache will try to substitute a compressed // or different version of a blob when preparing the list of layers when reading an image. -func NewBlobCache(ref types.ImageReference, directory string, compress types.LayerCompression) (BlobCache, error) { - return imageBlobCache.NewBlobCache(ref, directory, compress) +// The optional Option values can further refine behavior. +func NewBlobCache(ref types.ImageReference, directory string, compress types.LayerCompression, opts ...Option) (BlobCache, error) { + return imageBlobCache.NewBlobCache(ref, directory, compress, opts...) } diff --git a/pkg/cli/build.go b/pkg/cli/build.go index aeb77f9d598..886812f16eb 100644 --- a/pkg/cli/build.go +++ b/pkg/cli/build.go @@ -25,7 +25,9 @@ import ( "go.podman.io/buildah/pkg/parse" "go.podman.io/buildah/pkg/util" "go.podman.io/common/pkg/auth" + "go.podman.io/common/pkg/config" "go.podman.io/image/v5/docker/reference" + imgCompression "go.podman.io/image/v5/pkg/compression" "go.podman.io/image/v5/types" ) @@ -236,6 +238,40 @@ func GenBuildOptions(c *cobra.Command, inputArgs []string, iopts BuildOptions) ( compression = define.Uncompressed } + var compressionFormat *imgCompression.Algorithm + forceCompressionFormat := iopts.ForceCompressionFormat + defaultContainerConfig, err := config.Default() + if err != nil { + return options, nil, nil, fmt.Errorf("failed to get container config: %w", err) + } + if iopts.CompressionFormat != "" { + algo, err := imgCompression.AlgorithmByName(iopts.CompressionFormat) + if err != nil { + return options, nil, nil, err + } + compressionFormat = &algo + compression = define.Gzip + if !c.Flag("force-compression").Changed { + forceCompressionFormat = true + } + } else if defaultContainerConfig.Engine.CompressionFormat != "" && defaultContainerConfig.Engine.CompressionFormat != "gzip" { + algo, err := imgCompression.AlgorithmByName(defaultContainerConfig.Engine.CompressionFormat) + if err != nil { + return options, nil, nil, fmt.Errorf("parsing compression_format from containers.conf: %w", err) + } + compressionFormat = &algo + compression = define.Gzip + if !c.Flag("force-compression").Changed { + forceCompressionFormat = true + } + } + var compressionLevel *int + if c.Flag("compression-level").Changed { + compressionLevel = &iopts.CompressionLevel + } else { + compressionLevel = defaultContainerConfig.Engine.CompressionLevel + } + if c.Flag("disable-content-trust").Changed { logrus.Debugf("--disable-content-trust option specified but is ignored") } @@ -399,6 +435,9 @@ func GenBuildOptions(c *cobra.Command, inputArgs []string, iopts BuildOptions) ( CPPFlags: iopts.CPPFlags, CommonBuildOpts: commonOpts, Compression: compression, + CompressionFormat: compressionFormat, + CompressionLevel: compressionLevel, + ForceCompressionFormat: forceCompressionFormat, ConfigureNetwork: networkPolicy, ContextDirectory: contextDir, CreatedAnnotation: createdAnnotation, diff --git a/pkg/cli/common.go b/pkg/cli/common.go index db5f1711b0f..7d283c57cdc 100644 --- a/pkg/cli/common.go +++ b/pkg/cli/common.go @@ -57,85 +57,88 @@ type NameSpaceResults struct { // BudResults represents the results for Build flags type BudResults struct { - AllPlatforms bool - Annotation []string - Authfile string - BuildArg []string - BuildArgFile []string - BuildContext []string - CacheFrom []string - CacheTo []string - CacheTTL string - CertDir string - Compress bool - Creds string - CPPFlags []string - DisableCompression bool - DisableContentTrust bool - IgnoreFile string - File []string - Format string - From string - Iidfile string - IidfileRaw string - InheritLabels bool - InheritAnnotations bool - Label []string - LayerLabel []string - Logfile string - LogSplitByPlatform bool - Manifest string - MetadataFile string - NoHostname bool - NoHosts bool - NoCache bool - Timestamp int64 - OmitHistory bool - OCIHooksDir []string - Pull string - PullAlways bool - PullNever bool - Quiet bool - IdentityLabel bool - Rm bool - Runtime string - RuntimeFlags []string - SbomPreset string - SbomScannerImage string - SbomScannerCommand []string - SbomMergeStrategy string - SbomOutput string - SbomImgOutput string - SbomPurlOutput string - SbomImgPurlOutput string - Secrets []string - SSH []string - SignaturePolicy string - SignBy string - Squash bool - SkipUnusedStages bool - Stdin bool - Tag []string - BuildOutputs []string - Target string - TLSVerify bool - Jobs int - LogRusage bool - RusageLogFile string - UnsetEnvs []string - UnsetLabels []string - UnsetAnnotations []string - Envs []string - OSFeatures []string - OSVersion string - CWOptions string - SBOMOptions []string - CompatVolumes bool - SourceDateEpoch string - RewriteTimestamp bool - CreatedAnnotation bool - SourcePolicyFile string - TransientRunMounts []string + AllPlatforms bool + Annotation []string + Authfile string + BuildArg []string + BuildArgFile []string + BuildContext []string + CacheFrom []string + CacheTo []string + CacheTTL string + CertDir string + Compress bool + Creds string + CPPFlags []string + CompressionFormat string + CompressionLevel int + ForceCompressionFormat bool + DisableCompression bool + DisableContentTrust bool + IgnoreFile string + File []string + Format string + From string + Iidfile string + IidfileRaw string + InheritLabels bool + InheritAnnotations bool + Label []string + LayerLabel []string + Logfile string + LogSplitByPlatform bool + Manifest string + MetadataFile string + NoHostname bool + NoHosts bool + NoCache bool + Timestamp int64 + OmitHistory bool + OCIHooksDir []string + Pull string + PullAlways bool + PullNever bool + Quiet bool + IdentityLabel bool + Rm bool + Runtime string + RuntimeFlags []string + SbomPreset string + SbomScannerImage string + SbomScannerCommand []string + SbomMergeStrategy string + SbomOutput string + SbomImgOutput string + SbomPurlOutput string + SbomImgPurlOutput string + Secrets []string + SSH []string + SignaturePolicy string + SignBy string + Squash bool + SkipUnusedStages bool + Stdin bool + Tag []string + BuildOutputs []string + Target string + TLSVerify bool + Jobs int + LogRusage bool + RusageLogFile string + UnsetEnvs []string + UnsetLabels []string + UnsetAnnotations []string + Envs []string + OSFeatures []string + OSVersion string + CWOptions string + SBOMOptions []string + CompatVolumes bool + SourceDateEpoch string + RewriteTimestamp bool + CreatedAnnotation bool + SourcePolicyFile string + TransientRunMounts []string } // FromAndBugResults represents the results for common flags @@ -250,6 +253,9 @@ func GetBudFlags(flags *BudResults) pflag.FlagSet { fs.BoolVar(&flags.CreatedAnnotation, "created-annotation", true, `set an "org.opencontainers.image.created" annotation in the image`) fs.StringVar(&flags.Creds, "creds", "", "use `[username[:password]]` for accessing the registry") fs.StringVarP(&flags.CWOptions, "cw", "", "", "confidential workload `options`") + fs.StringVar(&flags.CompressionFormat, "compression-format", "", "compression format to use for layers and cache") + fs.IntVar(&flags.CompressionLevel, "compression-level", 0, "compression level to use for layers and cache") + fs.BoolVar(&flags.ForceCompressionFormat, "force-compression", false, "use the specified compression algorithm even if the destination contains a differently-compressed variant already") fs.BoolVarP(&flags.DisableCompression, "disable-compression", "D", true, "don't compress layers by default") fs.BoolVar(&flags.DisableContentTrust, "disable-content-trust", false, "this is a Docker specific option and is a NOOP") fs.StringArrayVar(&flags.Envs, "env", []string{}, "set environment variable for the image") @@ -355,6 +361,8 @@ func GetBudFlagsCompletions() commonComp.FlagCompletions { flagCompletion["cache-to"] = commonComp.AutocompleteNone flagCompletion["cache-ttl"] = commonComp.AutocompleteNone flagCompletion["cert-dir"] = commonComp.AutocompleteDefault + flagCompletion["compression-format"] = commonComp.AutocompleteNone + flagCompletion["compression-level"] = commonComp.AutocompleteNone flagCompletion["cpp-flag"] = commonComp.AutocompleteNone flagCompletion["creds"] = commonComp.AutocompleteNone flagCompletion["cw"] = commonComp.AutocompleteNone diff --git a/push.go b/push.go index c316d4de1de..7177417f574 100644 --- a/push.go +++ b/push.go @@ -23,14 +23,14 @@ import ( // cacheLookupReferenceFunc wraps a BlobCache into a // libimage.LookupReferenceFunc to allow for using a BlobCache during // image-copy operations. -func cacheLookupReferenceFunc(directory string, compress types.LayerCompression) libimage.LookupReferenceFunc { +func cacheLookupReferenceFunc(directory string, compress types.LayerCompression, opts ...blobcache.Option) libimage.LookupReferenceFunc { // Using a closure here allows us to reference a BlobCache without // having to explicitly maintain it in the libimage API. return func(ref types.ImageReference) (types.ImageReference, error) { if directory == "" { return ref, nil } - ref, err := blobcache.NewBlobCache(ref, directory, compress) + ref, err := blobcache.NewBlobCache(ref, directory, compress, opts...) if err != nil { return nil, fmt.Errorf("using blobcache %q: %w", directory, err) } @@ -130,13 +130,17 @@ func Push(ctx context.Context, image string, dest types.ImageReference, options } compress := types.PreserveOriginal - if options.Compression == archive.Gzip { + if options.Compression != archive.Uncompressed { compress = types.Compress } if options.SourceLookupReferenceFunc != nil { libimageOptions.SourceLookupReferenceFunc = options.SourceLookupReferenceFunc } else { - libimageOptions.SourceLookupReferenceFunc = cacheLookupReferenceFunc(options.BlobDirectory, compress) + var cacheOpts []blobcache.Option + if options.CompressionFormat != nil { + cacheOpts = append(cacheOpts, blobcache.WithCompressAlgorithm(options.CompressionFormat)) + } + libimageOptions.SourceLookupReferenceFunc = cacheLookupReferenceFunc(options.BlobDirectory, compress, cacheOpts...) } libimageOptions.DestinationLookupReferenceFunc = options.DestinationLookupReferenceFunc diff --git a/tests/bud.bats b/tests/bud.bats index 0b2c3f3ad14..ef9e9fb29f7 100644 --- a/tests/bud.bats +++ b/tests/bud.bats @@ -6474,6 +6474,131 @@ _EOF } +# Sets shell variable: contextdir +# Creates a Containerfile with alpine base and a unique layer. +function _setup_compression_context() { + _prefetch alpine + contextdir=${TEST_SCRATCH_DIR}/bud/compression-test + mkdir -p $contextdir + echo "unique-content-$(random_string)-$(date +%s)" > $contextdir/testfile + cat > $contextdir/Containerfile << _EOF +FROM alpine +RUN echo "layer1-$(random_string)" +COPY testfile /testfile +_EOF +} + +# Sets shell variables: contextdir, authfile +# Creates a Containerfile and starts a registry for push/cache tests. +function _setup_compression_test() { + which skopeo || skip "skopeo is not installed" + _setup_compression_context + start_registry + authfile=${TEST_SCRATCH_DIR}/test.auth + run_buildah login --tls-verify=false --authfile $authfile --username testuser --password testpassword localhost:${REGISTRY_PORT} +} + +function _build_with_cache() { + local imgname=$1; shift + local cacherepo=$1; shift + run_buildah build \ + $WITH_POLICY_JSON \ + --tls-verify=false \ + --authfile $authfile \ + --layers \ + --no-cache \ + --cache-to ${cacherepo} \ + "$@" \ + -t ${imgname} \ + -f $contextdir/Containerfile \ + $contextdir + expect_output --substring "Pushing cache" +} + +# Inspect a cache entry from a --cache-to registry repo. +# Uses skopeo list-tags to find a cache tag, then inspects its raw manifest. +# Sets $output to the raw manifest JSON for subsequent assertions. +# Usage: _inspect_cache_entry cacherepo +function _inspect_cache_entry() { + local cacherepo=$1 + + run skopeo list-tags \ + --authfile $authfile \ + --tls-verify=false \ + docker://${cacherepo} + assert $status -eq 0 "listing cache tags" + local tag + tag=$(jq -r '.Tags[0]' <<< "$output") + assert "$tag" != "null" "cache repo should have at least one tag" + + run skopeo inspect \ + --authfile $authfile \ + --tls-verify=false \ + --raw \ + docker://${cacherepo}:${tag} + assert $status -eq 0 "skopeo inspect of cache entry should succeed" +} + +@test "build --cache-to should respect compression_format from containers.conf" { + _setup_compression_test + cat > $contextdir/containers.conf << _EOF +[engine] +compression_format="zstd" +_EOF + local imgname="img-cache-conf-$(safename)" + local cacherepo="localhost:${REGISTRY_PORT}/cache-conf-test" + + CONTAINERS_CONF=$contextdir/containers.conf \ + _build_with_cache ${imgname} ${cacherepo} + + _inspect_cache_entry ${cacherepo} + expect_output --substring "zstd" \ + "cache layers should use zstd compression per containers.conf" + assert "$output" !~ "tar+gzip" \ + "cache layers should NOT use gzip when containers.conf specifies zstd" +} + +@test "build --cache-to with --compression-format and --force-compression" { + _setup_compression_test + local imgname="img-cache-cflag-$(safename)" + + for flags in "--compression-format zstd" "--compression-format zstd --force-compression"; do + local cacherepo="localhost:${REGISTRY_PORT}/cache-test-${flags// /-}" + + _build_with_cache ${imgname} ${cacherepo} $flags + + _inspect_cache_entry ${cacherepo} + expect_output --substring "zstd" \ + "cache layers should use zstd with: $flags" + assert "$output" !~ "tar+gzip" \ + "cache layers should NOT use gzip with: $flags" + done +} + +@test "build --compression-format zstd to dir and oci-archive" { + which skopeo || skip "skopeo is not installed" + _setup_compression_context + + for transport in dir oci-archive; do + local dest=${TEST_SCRATCH_DIR}/${transport}-out + + run_buildah build \ + $WITH_POLICY_JSON \ + --no-cache \ + --compression-format zstd \ + -t ${transport}:${dest} \ + -f $contextdir/Containerfile \ + $contextdir + + run skopeo inspect --raw ${transport}:${dest} + assert $status -eq 0 "$transport: skopeo inspect should succeed" + assert "$output" =~ "zstd" \ + "$transport: layers should use zstd mediaType" + assert "$output" !~ "gzip" \ + "$transport: layers should NOT use gzip" + done +} + @test "bud with undefined build arg directory" { _prefetch alpine mytmpdir=${TEST_SCRATCH_DIR}/my-dir1 @@ -10176,7 +10301,7 @@ _EOF @test "[test caching] Build 1: --layers | Build 2: --save-stages --stage-labels --layers | Cache: No cache reuse - added labels are causing cache miss for each stage" { _prefetch alpine target="cache-no-reuse-$(safename)" - + # First build run_buildah build $WITH_POLICY_JSON --layers -t ${target}-first -f $BUDFILES/save-stages/Dockerfile.single-build-stage $BUDFILES/save-stages @@ -10227,7 +10352,7 @@ _EOF # Verify cache was used 3 times cache_count=$(echo "$output" | grep -c "Using cache") - assert "$cache_count" -eq 3 "should use cache 3 times (LABEL final, COPY final, RUN final)" + assert "$cache_count" -eq 3 "should use cache 3 times (LABEL final, COPY final, RUN final)" run_buildah images -a intermediate_images_second=$(echo "$output" | awk '$1 == "" && $2 == "" {print $3}') diff --git a/tests/commit.bats b/tests/commit.bats index 5751f52809e..9c80a3fdc8a 100644 --- a/tests/commit.bats +++ b/tests/commit.bats @@ -642,3 +642,82 @@ load helpers fi done } + +@test "commit --disable-compression conflicts with compression flags" { + _prefetch alpine + run_buildah from $WITH_POLICY_JSON alpine + cid=$output + + run_buildah 125 commit \ + $WITH_POLICY_JSON \ + --disable-compression \ + --compression-format zstd \ + $cid \ + dir:${TEST_SCRATCH_DIR}/should-fail + expect_output --substring "cannot be used together" + + run_buildah 125 commit \ + $WITH_POLICY_JSON \ + --disable-compression \ + --force-compression \ + $cid \ + dir:${TEST_SCRATCH_DIR}/should-fail + expect_output --substring "cannot be used together" +} + +@test "commit --compression-format zstd to dir, oci, and oci-archive" { + which skopeo || skip "skopeo is not installed" + _prefetch alpine + run_buildah from $WITH_POLICY_JSON alpine + cid=$output + run_buildah run $cid touch /testfile + + for transport in dir oci oci-archive; do + local dest=${TEST_SCRATCH_DIR}/commit-zstd-${transport} + + run_buildah commit \ + $WITH_POLICY_JSON \ + --compression-format zstd \ + --disable-compression=false \ + $cid \ + ${transport}:$dest + + run skopeo inspect --raw ${transport}:${dest} + assert $status -eq 0 "$transport: skopeo inspect should succeed" + expect_output --substring "zstd" \ + "$transport: manifest should reference zstd-compressed layers" + assert "$output" !~ "gzip" \ + "$transport: manifest should NOT reference gzip layers" + done +} + +@test "commit should respect compression_format from containers.conf" { + which skopeo || skip "skopeo is not installed" + _prefetch alpine + run_buildah from $WITH_POLICY_JSON alpine + cid=$output + run_buildah run $cid touch /testfile + + local confdir=${TEST_SCRATCH_DIR}/commit-conf-zstd + mkdir -p $confdir + cat > $confdir/containers.conf << _EOF +[engine] +compression_format="zstd" +_EOF + + local dest=${TEST_SCRATCH_DIR}/commit-conf-zstd-output + + CONTAINERS_CONF=$confdir/containers.conf \ + run_buildah commit \ + $WITH_POLICY_JSON \ + --disable-compression=false \ + $cid \ + dir:$dest + + run skopeo inspect --raw dir:$dest + assert $status -eq 0 "skopeo inspect should succeed" + expect_output --substring "zstd" \ + "manifest should reference zstd-compressed layers per containers.conf" + assert "$output" !~ "gzip" \ + "manifest should NOT reference gzip layers when containers.conf specifies zstd" +} diff --git a/tests/push.bats b/tests/push.bats index 69ec345fe59..d7c53d9370f 100644 --- a/tests/push.bats +++ b/tests/push.bats @@ -224,3 +224,94 @@ load helpers # Verify there is some zstd compressed layer. grep application/vnd.oci.image.layer.v1.tar+zstd ${TEST_SCRATCH_DIR}/zstd/manifest.json } + +@test "push should respect compression_format from containers.conf" { + which skopeo || skip "skopeo is not installed" + _prefetch alpine + + local contextdir=${TEST_SCRATCH_DIR}/push-compression-conf + mkdir -p $contextdir + cat > $contextdir/Containerfile << _EOF +FROM alpine +RUN echo "layer1-$(random_string)" +_EOF + cat > $contextdir/containers.conf << _EOF +[engine] +compression_format="zstd" +_EOF + start_registry + run_buildah login --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth --username testuser --password testpassword localhost:${REGISTRY_PORT} + local imgname="img-push-compress-$(safename)" + local finalrepo="localhost:${REGISTRY_PORT}/${imgname}" + + CONTAINERS_CONF=$contextdir/containers.conf \ + run_buildah build \ + $WITH_POLICY_JSON \ + -t ${imgname} \ + --no-cache \ + -f $contextdir/Containerfile \ + $contextdir + + CONTAINERS_CONF=$contextdir/containers.conf \ + run_buildah push \ + $WITH_POLICY_JSON \ + --tls-verify=false \ + --authfile ${TEST_SCRATCH_DIR}/test.auth \ + ${imgname} \ + docker://${finalrepo} + + run skopeo inspect \ + --authfile ${TEST_SCRATCH_DIR}/test.auth \ + --tls-verify=false \ + --raw \ + docker://${finalrepo} + assert $status -eq 0 "skopeo inspect should succeed" + + expect_output --substring "zstd" \ + "layers should use zstd compression per containers.conf" + assert "$output" !~ "tar+gzip" \ + "layers should NOT use gzip when containers.conf specifies zstd" +} + +@test "push should respect --compression-format flag with registry" { + which skopeo || skip "skopeo is not installed" + _prefetch alpine + + local contextdir=${TEST_SCRATCH_DIR}/push-compression-flag + mkdir -p $contextdir + cat > $contextdir/Containerfile << _EOF +FROM alpine +RUN echo "layer1-$(random_string)" +_EOF + start_registry + run_buildah login --tls-verify=false --authfile ${TEST_SCRATCH_DIR}/test.auth --username testuser --password testpassword localhost:${REGISTRY_PORT} + local imgname="img-push-compress-flag-$(safename)" + local finalrepo="localhost:${REGISTRY_PORT}/${imgname}" + + run_buildah build \ + $WITH_POLICY_JSON \ + -t ${imgname} \ + --no-cache \ + -f $contextdir/Containerfile \ + $contextdir + + run_buildah push \ + $WITH_POLICY_JSON \ + --tls-verify=false \ + --authfile ${TEST_SCRATCH_DIR}/test.auth \ + --compression-format zstd \ + ${imgname} \ + docker://${finalrepo} + + run skopeo inspect \ + --authfile ${TEST_SCRATCH_DIR}/test.auth \ + --tls-verify=false \ + --raw \ + docker://${finalrepo} + assert $status -eq 0 "skopeo inspect should succeed" + + expect_output --substring "zstd" \ + "layers should use zstd compression per --compression-format flag" + assert "$output" !~ "tar+gzip" \ + "layers should NOT use gzip when --compression-format zstd is specified" +}