diff --git a/add.go b/add.go index 3e7a8f3937b..fd5812c9437 100644 --- a/add.go +++ b/add.go @@ -29,13 +29,13 @@ import ( "go.podman.io/buildah/copier" "go.podman.io/buildah/define" "go.podman.io/buildah/internal/tmpdir" + "go.podman.io/buildah/internal/urlsource" "go.podman.io/buildah/pkg/chrootuser" "go.podman.io/common/pkg/retry" "go.podman.io/image/v5/pkg/tlsclientconfig" "go.podman.io/image/v5/types" "go.podman.io/storage/pkg/fileutils" "go.podman.io/storage/pkg/idtools" - "go.podman.io/storage/pkg/regexp" ) // AddAndCopyOptions holds options for add and copy commands. @@ -60,7 +60,8 @@ type AddAndCopyOptions struct { // If the sources include directory trees, Hasher will be passed // tar-format archives of the directory trees. Hasher io.Writer - // Excludes is the contents of the .containerignore file. + // Excludes is a list of patterns to be excluded, which has the + // format of lines of .containerignore file. Excludes []string // IgnoreFile is the path to the .containerignore file. IgnoreFile string @@ -114,29 +115,6 @@ type AddAndCopyOptions struct { BuildMetadata string } -// gitURLFragmentSuffix matches fragments to use as Git reference and build -// context from the Git repository e.g. -// -// github.com/containers/buildah.git -// github.com/containers/buildah.git#main -// github.com/containers/buildah.git#v1.35.0 -var gitURLFragmentSuffix = regexp.Delayed(`\.git(?:#.+)?$`) - -// sourceIsGit returns true if "source" is a git location. -func sourceIsGit(source string) bool { - return isURL(source) && gitURLFragmentSuffix.MatchString(source) -} - -func isURL(url string) bool { - return strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") -} - -// sourceIsRemote returns true if "source" is a remote location -// and *not* a git repo. Certain github urls such as raw.github.* are allowed. -func sourceIsRemote(source string) bool { - return isURL(source) && !gitURLFragmentSuffix.MatchString(source) -} - // getURL writes a tar archive containing the named content func getURL(src string, chown *idtools.IDPair, mountpoint, renameTarget string, writer io.Writer, chmod *os.FileMode, srcDigest digest.Digest, certPath string, insecureSkipTLSVerify types.OptionalBool, timestamp *time.Time) error { url, err := url.Parse(src) @@ -349,11 +327,11 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption if src == "" { return errors.New("empty source location") } - if sourceIsRemote(src) { + if urlsource.IsRemote(src) { remoteSources = append(remoteSources, src) continue } - if sourceIsGit(src) { + if urlsource.IsGit(src) { gitSources = append(gitSources, src) continue } @@ -448,7 +426,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption destCanBeFile := false if len(sources) == 1 { if len(remoteSources) == 1 { - destCanBeFile = sourceIsRemote(sources[0]) + destCanBeFile = urlsource.IsRemote(sources[0]) } if len(localSources) == 1 { item := localSourceStats[0].Results[localSourceStats[0].Globbed[0]] @@ -589,7 +567,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption var multiErr *multierror.Error var getErr, closeErr, renameErr, putErr error var wg sync.WaitGroup - if sourceIsRemote(src) || sourceIsGit(src) { + if urlsource.IsRemote(src) || urlsource.IsGit(src) { pipeReader, pipeWriter := io.Pipe() var srcDigest digest.Digest if options.Checksum != "" { @@ -600,7 +578,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption } wg.Add(1) - if sourceIsGit(src) { + if urlsource.IsGit(src) { go func() { defer wg.Done() defer pipeWriter.Close() diff --git a/define/types.go b/define/types.go index 6a3057a9a3f..de30e0ba99b 100644 --- a/define/types.go +++ b/define/types.go @@ -17,6 +17,7 @@ import ( v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" + "go.podman.io/buildah/internal/urlsource" "go.podman.io/image/v5/manifest" "go.podman.io/storage/pkg/archive" "go.podman.io/storage/pkg/chrootarchive" @@ -193,8 +194,7 @@ type SBOMScanOptions struct { // directory is the responsibility of the caller. If the string doesn't look // like a URL or "-", TempDirForURL returns empty strings and a nil error code. func TempDirForURL(dir, prefix, url string) (name string, subdir string, err error) { - if !strings.HasPrefix(url, "http://") && - !strings.HasPrefix(url, "https://") && + if !urlsource.IsHTTPOrHTTPS(url) && !strings.HasPrefix(url, "git://") && !strings.HasPrefix(url, "github.com/") && url != "-" { @@ -229,7 +229,7 @@ func TempDirForURL(dir, prefix, url string) (name string, subdir string, err err logrus.Debugf("resolving url %q to %q", ghurl, url) subdir = path.Base(ghurl) + "-master" } - if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") { + if urlsource.IsHTTPOrHTTPS(url) { err = downloadToDirectory(url, downloadDir) if err != nil { if err2 := os.RemoveAll(name); err2 != nil { diff --git a/docs/buildah-build.1.md b/docs/buildah-build.1.md index 8ca32812894..6667a0ef6ac 100644 --- a/docs/buildah-build.1.md +++ b/docs/buildah-build.1.md @@ -1684,6 +1684,8 @@ This functionality is compatible with the handling of .containerignore files des https://github.com/containers/common/blob/main/docs/containerignore.5.md +Note: when the argument to ADD is a git repository, the local .containerignore is not applied. + **registries.conf** (`/etc/containers/registries.conf`) registries.conf is the configuration file which specifies which container registries should be consulted when completing image names which do not include a registry or domain portion. diff --git a/imagebuildah/stage_executor.go b/imagebuildah/stage_executor.go index 8cf55bbbd84..67c728b621d 100644 --- a/imagebuildah/stage_executor.go +++ b/imagebuildah/stage_executor.go @@ -32,6 +32,7 @@ import ( "go.podman.io/buildah/internal/output" "go.podman.io/buildah/internal/sanitize" "go.podman.io/buildah/internal/tmpdir" + "go.podman.io/buildah/internal/urlsource" internalUtil "go.podman.io/buildah/internal/util" "go.podman.io/buildah/pkg/parse" "go.podman.io/buildah/pkg/rusage" @@ -406,12 +407,12 @@ func (s *stageExecutor) performCopy(excludes []string, copies ...imagebuilder.Co if err := s.volumeCacheInvalidate(copy.Dest); err != nil { return err } - var sources []string // The From field says to read the content from another // container. Update the ID mappings and // all-content-comes-from-below-this-directory value. var idMappingOptions *define.IDMappingOptions var copyExcludes []string + var copyExcludesWithoutContainerIgnore []string stripSetuid := false stripSetgid := false preserveOwnership := false @@ -567,16 +568,24 @@ func (s *stageExecutor) performCopy(excludes []string, copies ...imagebuilder.Co stripSetuid = true // did this change between 18.06 and 19.03? stripSetgid = true // did this change between 18.06 and 19.03? } + copyExcludesWithoutContainerIgnore = excludes if copy.Download { logrus.Debugf("ADD %#v, %#v", excludes, copy) } else { logrus.Debugf("COPY %#v, %#v", excludes, copy) } + + var gitSources []string + var nonGitSources []string for _, src := range copy.Src { - if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") { + if urlsource.IsHTTPOrHTTPS(src) { // Source is a URL, allowed for ADD but not COPY. if copy.Download { - sources = append(sources, src) + if urlsource.IsGit(src) { + gitSources = append(gitSources, src) + } else { + nonGitSources = append(nonGitSources, src) + } } else { // returns an error to be compatible with docker return fmt.Errorf("source can't be a URL for COPY") @@ -592,7 +601,7 @@ func (s *stageExecutor) performCopy(excludes []string, copies ...imagebuilder.Co } else { src = filepath.Join(contextDir, src) } - sources = append(sources, src) + nonGitSources = append(nonGitSources, src) } } labelsAndAnnotations := s.buildMetadata(s.isLastStep, true) @@ -627,8 +636,21 @@ func (s *stageExecutor) performCopy(excludes []string, copies ...imagebuilder.Co options.Excludes = nil options.IgnoreFile = "" } - if err := s.builder.Add(copy.Dest, copy.Download, options, sources...); err != nil { - return err + + if len(nonGitSources) > 0 { + if err := s.builder.Add(copy.Dest, copy.Download, options, nonGitSources...); err != nil { + return err + } + } + + // Special handling for Git sources, local .containerignore is not applied. + if len(gitSources) > 0 { + gitOptions := options + gitOptions.Excludes = copyExcludesWithoutContainerIgnore + gitOptions.IgnoreFile = "" + if err := s.builder.Add(copy.Dest, copy.Download, gitOptions, gitSources...); err != nil { + return err + } } } if len(copiesExtend) > 0 { diff --git a/internal/urlsource/urlsource.go b/internal/urlsource/urlsource.go new file mode 100644 index 00000000000..b217d1aa9de --- /dev/null +++ b/internal/urlsource/urlsource.go @@ -0,0 +1,31 @@ +package urlsource + +import ( + "strings" + + "go.podman.io/storage/pkg/regexp" +) + +// gitURLFragmentSuffix matches fragments to use as Git reference and build +// context from the Git repository e.g. +// +// github.com/containers/buildah.git +// github.com/containers/buildah.git#main +// github.com/containers/buildah.git#v1.35.0 +var gitURLFragmentSuffix = regexp.Delayed(`\.git(?:#.+)?$`) + +// IsHTTPOrHTTPS reports whether the source is an HTTP(S) URL. +func IsHTTPOrHTTPS(source string) bool { + return strings.HasPrefix(source, "http://") || strings.HasPrefix(source, "https://") +} + +// IsGit reports whether the source is an HTTP(S) Git URL. +func IsGit(source string) bool { + return IsHTTPOrHTTPS(source) && gitURLFragmentSuffix.MatchString(source) +} + +// IsRemote reports whether the source is a remote HTTP(S) URL +// and *not* a Git repository. Certain GitHub URLs such as raw.github.* are allowed. +func IsRemote(source string) bool { + return IsHTTPOrHTTPS(source) && !gitURLFragmentSuffix.MatchString(source) +} diff --git a/tests/bud.bats b/tests/bud.bats index d7b9f71af84..0b2c3f3ad14 100644 --- a/tests/bud.bats +++ b/tests/bud.bats @@ -8010,6 +8010,29 @@ _EOF expect_output --substring "couldn't find remote ref nosuchbranch" } +@test "bud with ADD with git repository source - .containerignore not applied" { + _prefetch busybox + + local repodir=${TEST_SCRATCH_DIR}/repository-add-git-ignoring-files + mkdir -p ${repodir}/podman.git + tar -C ${repodir}/podman.git -xz < ${TEST_SOURCES}/git-daemon/bare-podman-repo.tar.gz + starthttpd /git/=${repodir}:"git http-backend":GIT_HTTP_EXPORT_ALL=1:GIT_PROJECT_ROOT=${repodir} ${repodir} + + local contextdir=${TEST_SCRATCH_DIR}/add-git-ignoring-files + mkdir -p $contextdir + cat > $contextdir/Dockerfile << _EOF +FROM busybox +ADD http://0.0.0.0:${HTTP_SERVER_PORT}/git/podman.git#v5.0.0 /podman-tag +RUN test -f /podman-tag/file.txt +_EOF + + cat > $contextdir/.containerignore << _EOF +file.txt +_EOF + + run_buildah build -f $contextdir/Dockerfile -t add-git-ignoring-files $contextdir +} + @test "build-validates-bind-bind-propagation" { _prefetch alpine