Ignore .containerignore for git repositories in ADD

add.go

@@ -29,13 +29,13 @@ import (
 	"go.podman.io/buildah/copier"
 	"go.podman.io/buildah/define"
 	"go.podman.io/buildah/internal/tmpdir"
+	"go.podman.io/buildah/internal/urlsource"
 	"go.podman.io/buildah/pkg/chrootuser"
 	"go.podman.io/common/pkg/retry"
 	"go.podman.io/image/v5/pkg/tlsclientconfig"
 	"go.podman.io/image/v5/types"
 	"go.podman.io/storage/pkg/fileutils"
 	"go.podman.io/storage/pkg/idtools"
-	"go.podman.io/storage/pkg/regexp"
 )
 
 // AddAndCopyOptions holds options for add and copy commands.
@@ -60,7 +60,8 @@ type AddAndCopyOptions struct {
 	// If the sources include directory trees, Hasher will be passed
 	// tar-format archives of the directory trees.
 	Hasher io.Writer
-	// Excludes is the contents of the .containerignore file.
+	// Excludes is a list of patterns to be excluded, which has the
+	// format of lines of .containerignore file.
 	Excludes []string
 	// IgnoreFile is the path to the .containerignore file.
 	IgnoreFile string
@@ -114,29 +115,6 @@ type AddAndCopyOptions struct {
 	BuildMetadata string
 }
 
-// gitURLFragmentSuffix matches fragments to use as Git reference and build
-// context from the Git repository e.g.
-//
-//	github.com/containers/buildah.git
-//	github.com/containers/buildah.git#main
-//	github.com/containers/buildah.git#v1.35.0
-var gitURLFragmentSuffix = regexp.Delayed(`\.git(?:#.+)?$`)
-
-// sourceIsGit returns true if "source" is a git location.
-func sourceIsGit(source string) bool {
-	return isURL(source) && gitURLFragmentSuffix.MatchString(source)
-}
-
-func isURL(url string) bool {
-	return strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://")
-}
-
-// sourceIsRemote returns true if "source" is a remote location
-// and *not* a git repo. Certain github urls such as raw.github.* are allowed.
-func sourceIsRemote(source string) bool {
-	return isURL(source) && !gitURLFragmentSuffix.MatchString(source)
-}
-
 // getURL writes a tar archive containing the named content
 func getURL(src string, chown *idtools.IDPair, mountpoint, renameTarget string, writer io.Writer, chmod *os.FileMode, srcDigest digest.Digest, certPath string, insecureSkipTLSVerify types.OptionalBool, timestamp *time.Time) error {
 	url, err := url.Parse(src)
@@ -349,11 +327,11 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 		if src == "" {
 			return errors.New("empty source location")
 		}
-		if sourceIsRemote(src) {
+		if urlsource.IsRemote(src) {
 			remoteSources = append(remoteSources, src)
 			continue
 		}
-		if sourceIsGit(src) {
+		if urlsource.IsGit(src) {
 			gitSources = append(gitSources, src)
 			continue
 		}
@@ -448,7 +426,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 	destCanBeFile := false
 	if len(sources) == 1 {
 		if len(remoteSources) == 1 {
-			destCanBeFile = sourceIsRemote(sources[0])
+			destCanBeFile = urlsource.IsRemote(sources[0])
 		}
 		if len(localSources) == 1 {
 			item := localSourceStats[0].Results[localSourceStats[0].Globbed[0]]
@@ -589,7 +567,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 		var multiErr *multierror.Error
 		var getErr, closeErr, renameErr, putErr error
 		var wg sync.WaitGroup
-		if sourceIsRemote(src) || sourceIsGit(src) {
+		if urlsource.IsRemote(src) || urlsource.IsGit(src) {
 			pipeReader, pipeWriter := io.Pipe()
 			var srcDigest digest.Digest
 			if options.Checksum != "" {
@@ -600,7 +578,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 			}
 
 			wg.Add(1)
-			if sourceIsGit(src) {
+			if urlsource.IsGit(src) {
 				go func() {
 					defer wg.Done()
 					defer pipeWriter.Close()

define/types.go

@@ -17,6 +17,7 @@ import (
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
+	"go.podman.io/buildah/internal/urlsource"
 	"go.podman.io/image/v5/manifest"
 	"go.podman.io/storage/pkg/archive"
 	"go.podman.io/storage/pkg/chrootarchive"
@@ -193,8 +194,7 @@ type SBOMScanOptions struct {
 // directory is the responsibility of the caller.  If the string doesn't look
 // like a URL or "-", TempDirForURL returns empty strings and a nil error code.
 func TempDirForURL(dir, prefix, url string) (name string, subdir string, err error) {
-	if !strings.HasPrefix(url, "http://") &&
-		!strings.HasPrefix(url, "https://") &&
+	if !urlsource.IsHTTPOrHTTPS(url) &&
 		!strings.HasPrefix(url, "git://") &&
 		!strings.HasPrefix(url, "github.com/") &&
 		url != "-" {
@@ -229,7 +229,7 @@ func TempDirForURL(dir, prefix, url string) (name string, subdir string, err err
 		logrus.Debugf("resolving url %q to %q", ghurl, url)
 		subdir = path.Base(ghurl) + "-master"
 	}
-	if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
+	if urlsource.IsHTTPOrHTTPS(url) {
 		err = downloadToDirectory(url, downloadDir)
 		if err != nil {
 			if err2 := os.RemoveAll(name); err2 != nil {

docs/buildah-build.1.md

@@ -1684,6 +1684,8 @@ This functionality is compatible with the handling of .containerignore files des
 
 https://github.com/containers/common/blob/main/docs/containerignore.5.md
 
+Note: when the argument to ADD is a git repository, the local .containerignore is not applied.
+
 **registries.conf** (`/etc/containers/registries.conf`)
 
 registries.conf is the configuration file which specifies which container registries should be consulted when completing image names which do not include a registry or domain portion.

imagebuildah/stage_executor.go

@@ -32,6 +32,7 @@ import (
 	"go.podman.io/buildah/internal/output"
 	"go.podman.io/buildah/internal/sanitize"
 	"go.podman.io/buildah/internal/tmpdir"
+	"go.podman.io/buildah/internal/urlsource"
 	internalUtil "go.podman.io/buildah/internal/util"
 	"go.podman.io/buildah/pkg/parse"
 	"go.podman.io/buildah/pkg/rusage"
@@ -406,12 +407,12 @@ func (s *stageExecutor) performCopy(excludes []string, copies ...imagebuilder.Co
 		if err := s.volumeCacheInvalidate(copy.Dest); err != nil {
 			return err
 		}
-		var sources []string
 		// The From field says to read the content from another
 		// container.  Update the ID mappings and
 		// all-content-comes-from-below-this-directory value.
 		var idMappingOptions *define.IDMappingOptions
 		var copyExcludes []string
+		var copyExcludesWithoutContainerIgnore []string
 		stripSetuid := false
 		stripSetgid := false
 		preserveOwnership := false
@@ -567,16 +568,24 @@ func (s *stageExecutor) performCopy(excludes []string, copies ...imagebuilder.Co
 			stripSetuid = true // did this change between 18.06 and 19.03?
 			stripSetgid = true // did this change between 18.06 and 19.03?
 		}
+		copyExcludesWithoutContainerIgnore = excludes
 		if copy.Download {
 			logrus.Debugf("ADD %#v, %#v", excludes, copy)
 		} else {
 			logrus.Debugf("COPY %#v, %#v", excludes, copy)
 		}
+
+		var gitSources []string
+		var nonGitSources []string
 		for _, src := range copy.Src {
-			if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
+			if urlsource.IsHTTPOrHTTPS(src) {
 				// Source is a URL, allowed for ADD but not COPY.
 				if copy.Download {
-					sources = append(sources, src)
+					if urlsource.IsGit(src) {
+						gitSources = append(gitSources, src)
+					} else {
+						nonGitSources = append(nonGitSources, src)
+					}
 				} else {
 					// returns an error to be compatible with docker
 					return fmt.Errorf("source can't be a URL for COPY")
@@ -592,7 +601,7 @@ func (s *stageExecutor) performCopy(excludes []string, copies ...imagebuilder.Co
 				} else {
 					src = filepath.Join(contextDir, src)
 				}
-				sources = append(sources, src)
+				nonGitSources = append(nonGitSources, src)
 			}
 		}
 		labelsAndAnnotations := s.buildMetadata(s.isLastStep, true)
@@ -627,8 +636,21 @@ func (s *stageExecutor) performCopy(excludes []string, copies ...imagebuilder.Co
 			options.Excludes = nil
 			options.IgnoreFile = ""
 		}
-		if err := s.builder.Add(copy.Dest, copy.Download, options, sources...); err != nil {
-			return err
+
+		if len(nonGitSources) > 0 {
+			if err := s.builder.Add(copy.Dest, copy.Download, options, nonGitSources...); err != nil {
+				return err
+			}
+		}
+
+		// Special handling for Git sources, local .containerignore is not applied.
+		if len(gitSources) > 0 {
+			gitOptions := options
+			gitOptions.Excludes = copyExcludesWithoutContainerIgnore
+			gitOptions.IgnoreFile = ""
+			if err := s.builder.Add(copy.Dest, copy.Download, gitOptions, gitSources...); err != nil {
+				return err
+			}
 		}
 	}
 	if len(copiesExtend) > 0 {

internal/urlsource/urlsource.go

@@ -0,0 +1,31 @@
+package urlsource
+
+import (
+	"strings"
+
+	"go.podman.io/storage/pkg/regexp"
+)
+
+// gitURLFragmentSuffix matches fragments to use as Git reference and build
+// context from the Git repository e.g.
+//
+//	github.com/containers/buildah.git
+//	github.com/containers/buildah.git#main
+//	github.com/containers/buildah.git#v1.35.0
+var gitURLFragmentSuffix = regexp.Delayed(`\.git(?:#.+)?$`)
+
+// IsHTTPOrHTTPS reports whether the source is an HTTP(S) URL.
+func IsHTTPOrHTTPS(source string) bool {
+	return strings.HasPrefix(source, "http://") || strings.HasPrefix(source, "https://")
+}
+
+// IsGit reports whether the source is an HTTP(S) Git URL.
+func IsGit(source string) bool {
+	return IsHTTPOrHTTPS(source) && gitURLFragmentSuffix.MatchString(source)
+}
+
+// IsRemote reports whether the source is a remote HTTP(S) URL
+// and *not* a Git repository. Certain GitHub URLs such as raw.github.* are allowed.
+func IsRemote(source string) bool {
+	return IsHTTPOrHTTPS(source) && !gitURLFragmentSuffix.MatchString(source)
+}

tests/bud.bats

@@ -8010,6 +8010,29 @@ _EOF
   expect_output --substring "couldn't find remote ref nosuchbranch"
 }
 
+@test "bud with ADD with git repository source - .containerignore not applied" {
+  _prefetch busybox
+
+  local repodir=${TEST_SCRATCH_DIR}/repository-add-git-ignoring-files
+  mkdir -p ${repodir}/podman.git
+  tar -C ${repodir}/podman.git -xz < ${TEST_SOURCES}/git-daemon/bare-podman-repo.tar.gz
+  starthttpd /git/=${repodir}:"git http-backend":GIT_HTTP_EXPORT_ALL=1:GIT_PROJECT_ROOT=${repodir} ${repodir}
+
+  local contextdir=${TEST_SCRATCH_DIR}/add-git-ignoring-files
+  mkdir -p $contextdir
+  cat > $contextdir/Dockerfile << _EOF
+FROM busybox
+ADD http://0.0.0.0:${HTTP_SERVER_PORT}/git/podman.git#v5.0.0 /podman-tag
+RUN test -f /podman-tag/file.txt
+_EOF
+
+  cat > $contextdir/.containerignore << _EOF
+file.txt
+_EOF
+
+  run_buildah build -f $contextdir/Dockerfile -t add-git-ignoring-files $contextdir
+}
+
 @test "build-validates-bind-bind-propagation" {
   _prefetch alpine