Respect compression_format from containers.conf in push, build --cache-to, and commit

cmd/buildah/commit.go

@@ -19,57 +19,62 @@ import (
 	"go.podman.io/buildah/util"
 	"go.podman.io/common/pkg/auth"
 	"go.podman.io/common/pkg/completion"
+	"go.podman.io/common/pkg/config"
 	"go.podman.io/image/v5/manifest"
+	"go.podman.io/image/v5/pkg/compression"
 	"go.podman.io/image/v5/pkg/shortnames"
 	storageTransport "go.podman.io/image/v5/storage"
 	"go.podman.io/image/v5/transports/alltransports"
 	"go.podman.io/image/v5/types"
 )
 
 type commitInputOptions struct {
-	authfile           string
-	omitHistory        bool
-	blobCache          string
-	certDir            string
-	changes            []string
-	configFile         string
-	creds              string
-	cwOptions          string
-	disableCompression bool
-	format             string
-	iidfile            string
-	manifest           string
-	omitTimestamp      bool
-	timestamp          int64
-	sourceDateEpoch    string
-	rewriteTimestamp   bool
-	quiet              bool
-	referenceTime      string
-	rm                 bool
-	pull               string
-	pullAlways         bool
-	pullNever          bool
-	sbomImgOutput      string
-	sbomImgPurlOutput  string
-	sbomMergeStrategy  string
-	sbomOutput         string
-	sbomPreset         string
-	sbomPurlOutput     string
-	sbomScannerCommand []string
-	sbomScannerImage   string
-	signaturePolicy    string
-	signBy             string
-	squash             bool
-	tlsVerify          bool
-	identityLabel      bool
-	encryptionKeys     []string
-	encryptLayers      []int
-	unsetenvs          []string
-	addFile            []string
-	unsetAnnotation    []string
-	annotation         []string
-	createdAnnotation  bool
-	metadataFile       string
+	authfile               string
+	omitHistory            bool
+	blobCache              string
+	certDir                string
+	changes                []string
+	compressionFormat      string
+	compressionLevel       int
+	configFile             string
+	creds                  string
+	cwOptions              string
+	disableCompression     bool
+	forceCompressionFormat bool
+	format                 string
+	iidfile                string
+	manifest               string
+	omitTimestamp          bool
+	timestamp              int64
+	sourceDateEpoch        string
+	rewriteTimestamp       bool
+	quiet                  bool
+	referenceTime          string
+	rm                     bool
+	pull                   string
+	pullAlways             bool
+	pullNever              bool
+	sbomImgOutput          string
+	sbomImgPurlOutput      string
+	sbomMergeStrategy      string
+	sbomOutput             string
+	sbomPreset             string
+	sbomPurlOutput         string
+	sbomScannerCommand     []string
+	sbomScannerImage       string
+	signaturePolicy        string
+	signBy                 string
+	squash                 bool
+	tlsVerify              bool
+	identityLabel          bool
+	encryptionKeys         []string
+	encryptLayers          []int
+	unsetenvs              []string
+	addFile                []string
+	unsetAnnotation        []string
+	annotation             []string
+	createdAnnotation      bool
+	metadataFile           string
 }
 
 func commitInit() {
@@ -117,8 +122,13 @@ func commitListFlagSet(cmd *cobra.Command, opts *commitInputOptions) {
 	_ = cmd.RegisterFlagCompletionFunc("config", completion.AutocompleteDefault)
 	flags.StringVar(&opts.creds, "creds", "", "use `[username[:password]]` for accessing the registry")
 	_ = cmd.RegisterFlagCompletionFunc("creds", completion.AutocompleteNone)
+	flags.StringVar(&opts.compressionFormat, "compression-format", "", "compression format to use")
+	_ = cmd.RegisterFlagCompletionFunc("compression-format", completion.AutocompleteNone)
+	flags.IntVar(&opts.compressionLevel, "compression-level", 0, "compression level to use")
+	_ = cmd.RegisterFlagCompletionFunc("compression-level", completion.AutocompleteNone)
 	flags.StringVar(&opts.cwOptions, "cw", "", "confidential workload `options`")
 	flags.BoolVarP(&opts.disableCompression, "disable-compression", "D", true, "don't compress layers")
+	flags.BoolVar(&opts.forceCompressionFormat, "force-compression", false, "use the specified compression algorithm if the destination contains a differently-compressed variant already")
 	flags.StringVarP(&opts.format, "format", "f", defaultFormat(), "`format` of the image manifest and metadata")
 	_ = cmd.RegisterFlagCompletionFunc("format", completion.AutocompleteNone)
 	flags.StringVar(&opts.manifest, "manifest", "", "adds created image to the specified manifest list. Creates manifest list if it does not exist")
@@ -228,6 +238,15 @@ func commitCmd(c *cobra.Command, args []string, iopts commitInputOptions) error
 		compress = define.Uncompressed
 	}
 
+	if c.Flag("disable-compression").Changed && iopts.disableCompression {
+		if c.Flag("compression-format").Changed {
+			return errors.New("--disable-compression and --compression-format cannot be used together")
+		}
+		if c.Flag("force-compression").Changed {
+			return errors.New("--disable-compression and --force-compression cannot be used together")
+		}
+	}
+
 	format, err := cli.GetFormat(iopts.format)
 	if err != nil {
 		return err
@@ -308,25 +327,54 @@ func commitCmd(c *cobra.Command, args []string, iopts commitInputOptions) error
 	}
 
 	options := buildah.CommitOptions{
-		PreferredManifestType: format,
-		Manifest:              iopts.manifest,
-		Compression:           compress,
-		SignaturePolicyPath:   iopts.signaturePolicy,
-		SystemContext:         systemContext,
-		IIDFile:               iopts.iidfile,
-		Squash:                iopts.squash,
-		BlobDirectory:         iopts.blobCache,
-		OmitHistory:           iopts.omitHistory,
-		SignBy:                iopts.signBy,
-		OciEncryptConfig:      encConfig,
-		OciEncryptLayers:      encLayers,
-		UnsetEnvs:             iopts.unsetenvs,
-		OverrideChanges:       iopts.changes,
-		OverrideConfig:        overrideConfig,
-		ExtraImageContent:     addFiles,
-		UnsetAnnotations:      iopts.unsetAnnotation,
-		Annotations:           iopts.annotation,
-		CreatedAnnotation:     types.NewOptionalBool(iopts.createdAnnotation),
+		PreferredManifestType:  format,
+		Manifest:               iopts.manifest,
+		Compression:            compress,
+		ForceCompressionFormat: iopts.forceCompressionFormat,
+		SignaturePolicyPath:    iopts.signaturePolicy,
+		SystemContext:          systemContext,
+		IIDFile:                iopts.iidfile,
+		Squash:                 iopts.squash,
+		BlobDirectory:          iopts.blobCache,
+		OmitHistory:            iopts.omitHistory,
+		SignBy:                 iopts.signBy,
+		OciEncryptConfig:       encConfig,
+		OciEncryptLayers:       encLayers,
+		UnsetEnvs:              iopts.unsetenvs,
+		OverrideChanges:        iopts.changes,
+		OverrideConfig:         overrideConfig,
+		ExtraImageContent:      addFiles,
+		UnsetAnnotations:       iopts.unsetAnnotation,
+		Annotations:            iopts.annotation,
+		CreatedAnnotation:      types.NewOptionalBool(iopts.createdAnnotation),
+	}
+	defaultContainerConfig, err := config.Default()
+	if err != nil {
+		return fmt.Errorf("failed to get container config: %w", err)
+	}
+	if iopts.compressionFormat != "" {
+		algo, err := compression.AlgorithmByName(iopts.compressionFormat)
+		if err != nil {
+			return err
+		}
+		options.CompressionFormat = &algo
+		if !c.Flag("force-compression").Changed {
+			options.ForceCompressionFormat = true
+		}
+	} else if defaultContainerConfig.Engine.CompressionFormat != "" && defaultContainerConfig.Engine.CompressionFormat != "gzip" {
+		algo, err := compression.AlgorithmByName(defaultContainerConfig.Engine.CompressionFormat)
+		if err != nil {
+			return fmt.Errorf("parsing compression_format from containers.conf: %w", err)
+		}
+		options.CompressionFormat = &algo
+		if !c.Flag("force-compression").Changed {
+			options.ForceCompressionFormat = true
+		}
+	}
+	if c.Flag("compression-level").Changed {
+		options.CompressionLevel = &iopts.compressionLevel
+	} else {
+		options.CompressionLevel = defaultContainerConfig.Engine.CompressionLevel
 	}
 	exclusiveFlags := 0
 	if c.Flag("reference-time").Changed {

cmd/buildah/push.go

@@ -16,6 +16,7 @@ import (
 	"go.podman.io/buildah/pkg/parse"
 	util "go.podman.io/buildah/util"
 	"go.podman.io/common/pkg/auth"
+	"go.podman.io/common/pkg/config"
 	"go.podman.io/image/v5/manifest"
 	"go.podman.io/image/v5/pkg/compression"
 	"go.podman.io/image/v5/transports"
@@ -194,14 +195,6 @@ func pushCmd(c *cobra.Command, args []string, iopts pushOptions) error {
 		return fmt.Errorf("unable to obtain encryption config: %w", err)
 	}
 
-	if c.Flag("compression-format").Changed {
-		if !c.Flag("force-compression").Changed {
-			// If `compression-format` is set and no value for `--force-compression`
-			// is selected then defaults to `true`.
-			iopts.forceCompressionFormat = true
-		}
-	}
-
 	options := buildah.PushOptions{
 		Compression:            compress,
 		ManifestType:           manifestType,
@@ -225,15 +218,33 @@ func pushCmd(c *cobra.Command, args []string, iopts pushOptions) error {
 	if !iopts.quiet {
 		options.ReportWriter = os.Stderr
 	}
+	defaultContainerConfig, err := config.Default()
+	if err != nil {
+		return fmt.Errorf("failed to get container config: %w", err)
+	}
 	if iopts.compressionFormat != "" {
 		algo, err := compression.AlgorithmByName(iopts.compressionFormat)
 		if err != nil {
 			return err
 		}
 		options.CompressionFormat = &algo
+		if !c.Flag("force-compression").Changed {
+			options.ForceCompressionFormat = true
+		}
+	} else if defaultContainerConfig.Engine.CompressionFormat != "" && defaultContainerConfig.Engine.CompressionFormat != "gzip" {
+		algo, err := compression.AlgorithmByName(defaultContainerConfig.Engine.CompressionFormat)
+		if err != nil {
+			return fmt.Errorf("parsing compression_format from containers.conf: %w", err)
+		}
+		options.CompressionFormat = &algo
+		if !c.Flag("force-compression").Changed {
+			options.ForceCompressionFormat = true
+		}
 	}
 	if c.Flag("compression-level").Changed {
 		options.CompressionLevel = &iopts.compressionLevel
+	} else {
+		options.CompressionLevel = defaultContainerConfig.Engine.CompressionLevel
 	}
 
 	ref, digest, err := buildah.Push(getContext(), src, dest, options)

commit.go

@@ -22,6 +22,7 @@ import (
 	"go.podman.io/image/v5/docker"
 	"go.podman.io/image/v5/docker/reference"
 	"go.podman.io/image/v5/manifest"
+	"go.podman.io/image/v5/pkg/compression"
 	"go.podman.io/image/v5/signature"
 	is "go.podman.io/image/v5/storage"
 	"go.podman.io/image/v5/transports"
@@ -47,6 +48,14 @@ type CommitOptions struct {
 	// layer blobs.  The default is to not use compression, but
 	// archive.Gzip is recommended.
 	Compression archive.Compression
+	// CompressionFormat is the format to use for the compression of the blobs.
+	CompressionFormat *compression.Algorithm
+	// CompressionLevel specifies what compression level is used.
+	CompressionLevel *int
+	// ForceCompressionFormat ensures that the compression algorithm set in
+	// CompressionFormat is used exclusively, and blobs of other compression
+	// algorithms are not reused.
+	ForceCompressionFormat bool
 	// SignaturePolicyPath specifies an override location for the signature
 	// policy which should be used for verifying the new image as it is
 	// being written.  Except in specific circumstances, no value should be
@@ -456,12 +465,16 @@ func (b *Builder) CommitResults(ctx context.Context, dest types.ImageReference,
 		if options.Compression != archive.Uncompressed {
 			compress = types.Compress
 		}
-		cache, err := blobcache.NewBlobCache(src, options.BlobDirectory, compress)
+		var cacheOpts []blobcache.Option
+		if options.CompressionFormat != nil {
+			cacheOpts = append(cacheOpts, blobcache.WithCompressAlgorithm(options.CompressionFormat))
+		}
+		cache, err := blobcache.NewBlobCache(src, options.BlobDirectory, compress, cacheOpts...)
 		if err != nil {
 			return nil, fmt.Errorf("wrapping image reference %q in blob cache at %q: %w", transports.ImageName(src), options.BlobDirectory, err)
 		}
 		maybeCachedSrc = cache
-		cache, err = blobcache.NewBlobCache(dest, options.BlobDirectory, compress)
+		cache, err = blobcache.NewBlobCache(dest, options.BlobDirectory, compress, cacheOpts...)
 		if err != nil {
 			return nil, fmt.Errorf("wrapping image reference %q in blob cache at %q: %w", transports.ImageName(dest), options.BlobDirectory, err)
 		}
@@ -471,8 +484,9 @@ func (b *Builder) CommitResults(ctx context.Context, dest types.ImageReference,
 	switch options.Compression {
 	case archive.Uncompressed:
 		systemContext.OCIAcceptUncompressedLayers = true
-	case archive.Gzip:
+	default:
 		systemContext.DirForceCompress = true
+		systemContext.OCIAcceptUncompressedLayers = false
 	}
 
 	if systemContext.ArchitectureChoice != b.Architecture() {
@@ -482,8 +496,16 @@ func (b *Builder) CommitResults(ctx context.Context, dest types.ImageReference,
 		systemContext.OSChoice = b.OS()
 	}
 
+	copyOptions := getCopyOptions(b.store, options.ReportWriter, nil, systemContext, "", false, options.SignBy, options.OciEncryptLayers, options.OciEncryptConfig, nil, destinationTimestamp)
+	if options.CompressionFormat != nil {
+		copyOptions.DestinationCtx.CompressionFormat = options.CompressionFormat
+		copyOptions.DestinationCtx.CompressionLevel = options.CompressionLevel
+	}
+	if options.ForceCompressionFormat {
+		copyOptions.ForceCompressionFormat = true
+	}
 	var manifestBytes []byte
-	if manifestBytes, err = retryCopyImage(ctx, policyContext, maybeCachedDest, maybeCachedSrc, dest, getCopyOptions(b.store, options.ReportWriter, nil, systemContext, "", false, options.SignBy, options.OciEncryptLayers, options.OciEncryptConfig, nil, destinationTimestamp), options.MaxRetries, options.RetryDelay); err != nil {
+	if manifestBytes, err = retryCopyImage(ctx, policyContext, maybeCachedDest, maybeCachedSrc, dest, copyOptions, options.MaxRetries, options.RetryDelay); err != nil {
 		return nil, fmt.Errorf("copying layers and metadata for container %q: %w", b.ContainerID, err)
 	}
 	// If we've got more names to attach, and we know how to do that for

define/build.go

@@ -8,6 +8,7 @@ import (
 	"go.podman.io/common/libimage/manifests"
 	nettypes "go.podman.io/common/libnetwork/types"
 	"go.podman.io/image/v5/docker/reference"
+	"go.podman.io/image/v5/pkg/compression"
 	"go.podman.io/image/v5/types"
 	"go.podman.io/storage/pkg/archive"
 	"golang.org/x/sync/semaphore"
@@ -165,6 +166,14 @@ type BuildOptions struct {
 	// layer blobs.  The default is to not use compression, but
 	// archive.Gzip is recommended.
 	Compression archive.Compression
+	// CompressionFormat is the format to use for the compression of the blobs.
+	CompressionFormat *compression.Algorithm
+	// CompressionLevel specifies what compression level is used.
+	CompressionLevel *int
+	// ForceCompressionFormat ensures that the compression algorithm set in
+	// CompressionFormat is used exclusively, and blobs of other compression
+	// algorithms are not reused.
+	ForceCompressionFormat bool
 	// Arguments which can be interpolated into Dockerfiles
 	Args map[string]string
 	// Map of external additional build contexts

docs/buildah-build.1.md

@@ -218,6 +218,20 @@ This option is added to be aligned with other containers CLIs.
 Buildah doesn't send a copy of the context directory to a daemon or a remote server.
 Thus, compressing the data before sending it is irrelevant to Buildah.
 
+**--compression-format** *format*
+
+Specifies the compression format to use.  Supported values are: `gzip`, `zstd` and `zstd:chunked`.
+`zstd:chunked` is incompatible with encrypting images, and will be treated as `zstd` with a warning in that case.
+If not specified, the format is read from the `compression_format` setting in containers.conf.
+
+This option affects cache pushes with `--cache-to` and the final image when it is written to a non-local destination (e.g., `dir:`, `oci:`, `oci-archive:`, or a registry).
+When the output is local container storage (the default), layers are always decompressed on ingest, so compression is applied at `buildah push` time instead.
+
+**--compression-level** *level*
+
+Specifies the compression level to use.  The value is specific to the compression algorithm used, e.g. for zstd the accepted values are in the range 1-20 (inclusive), while for gzip it is 1-9 (inclusive).
+If not specified, the level is read from the `compression_level` setting in containers.conf.
+
 **--cpp-flag**=""
 
 Set additional flags to pass to the C Preprocessor cpp(1).
@@ -444,6 +458,11 @@ context directory will be prepended to the local file value.
 
 If you specify `-f -`, the Containerfile contents will be read from stdin.
 
+**--force-compression**
+
+If set, uses the specified compression algorithm even if the destination contains a differently-compressed variant already.
+Defaults to `true` if `--compression-format` is explicitly specified on the command-line or `compression_format` is set in containers.conf, `false` otherwise.
+
 **--force-rm** *bool-value*
 
 Always remove intermediate containers after a build, even if the build fails (default false).