Feature: support to replace a package path using a local repo

conf/config.go

@@ -1,17 +1,18 @@
 package conf
 
 import (
-	"github.com/genshen/pkg"
-	"gopkg.in/yaml.v3"
 	"os"
 	"path/filepath"
+
+	"github.com/genshen/pkg"
+	"gopkg.in/yaml.v3"
 )
 
 const ConfigFileName = "pkg.config.yaml"
 
 type PkgConfig struct {
-	Auth       map[string]Auth   `yaml:"auth"`
-	GitReplace map[string]string `yaml:"git-replace"`
+	Auth       map[string]Auth                   `yaml:"auth"`
+	GitReplace map[string]pkg.GitReplaceMetadata `yaml:"git-replace"`
 }
 
 func ParseConfig(projectHome string) (*PkgConfig, error) {

dep_tree.go

@@ -6,11 +6,12 @@ package pkg
 
 import (
 	"fmt"
-	"gopkg.in/yaml.v3"
 	"io"
 	"log"
 	"os"
 	"strings"
+
+	"gopkg.in/yaml.v3"
 )
 
 const (
@@ -19,6 +20,11 @@ const (
 	DlStatusOk
 )
 
+const (
+	SrcLocationVendor = 0
+	SrcLocationLocal  = 1
+)
+
 type DependencyTree struct {
 	Dependencies []*DependencyTree
 	Context      PackageMeta
@@ -28,16 +34,17 @@ type DependencyTree struct {
 
 // package metadata used in sum file.
 type PackageMeta struct {
-	PackageName string   `yaml:"pkg"`    // package name (usually it is a path)
-	TargetName  string   `yaml:"target"` // cmake package name
-	Features    []string `yaml:"features"`
-	Optional    bool     `yaml:"optional"` // if true: this package is optional
-	//	SrcPath      string   `yaml:"-"`
-	Version      string   `yaml:"version"`
-	Builder      []string `yaml:"builder"`        // outer builder (lib used by others, specified by others pkg)
-	SelfBuild    []string `yaml:"self_build"`     // inner builder (shows how this package is built, specified in package's pkg.yaml file)
-	CMakeLib     string   `yaml:"cmake_lib"`      // outer cmake script to add this lib.
-	SelfCMakeLib string   `yaml:"self_cmake_lib"` // inner cmake script to add this lib.
+	PackageName     string   `yaml:"pkg"`    // package name (usually it is a path)
+	TargetName      string   `yaml:"target"` // cmake package name
+	Features        []string `yaml:"features"`
+	Optional        bool     `yaml:"optional"`                    // if true: this package is optional
+	SrcPath         string   `yaml:"src_path,omitempty"`          // not in yaml: the absolute path to place the source code (default: vendor/src/{pkg_name})
+	SrcLocationType int      `yaml:"src_location_type,omitempty"` // the location of src code.
+	Version         string   `yaml:"version"`
+	Builder         []string `yaml:"builder"`        // outer builder (lib used by others, specified by others pkg)
+	SelfBuild       []string `yaml:"self_build"`     // inner builder (shows how this package is built, specified in package's pkg.yaml file)
+	CMakeLib        string   `yaml:"cmake_lib"`      // outer cmake script to add this lib.
+	SelfCMakeLib    string   `yaml:"self_cmake_lib"` // inner cmake script to add this lib.
 }
 
 func (ctx *PackageMeta) SetPackageName(key string) error {
@@ -80,8 +87,17 @@ func (ctx *PackageMeta) HomeCacheSrcPath() string {
 	}
 }
 
-// return directory path of source in vendor
-func (ctx *PackageMeta) VendorSrcPath(base string) string {
+func (ctx *PackageMeta) PackageSrcPath(base string) string {
+	if ctx.SrcLocationType == SrcLocationLocal {
+		return ctx.SrcPath // return full path
+	} else {
+		// use vendor path.
+		return getPackageVendorSrcPath(base, ctx.PackageName, ctx.Version)
+	}
+}
+
+// DefaultPackageSrcPathVendor returns the default directory path of source in vendor
+func (ctx *PackageMeta) DefaultPackageSrcPathVendor(base string) string {
 	return getPackageVendorSrcPath(base, ctx.PackageName, ctx.Version)
 }
 

example/pkg.config.yaml

@@ -5,3 +5,4 @@ auth:
 
 git-replace:
   github.com/google/googletest: gitee.com/mirrors/googletest
+  github.com/fmtlib/fmt: local://{{.PKG_ROOT}}/../fmt-main/

pkg/fetch/cache_strategy.go

@@ -1,7 +1,9 @@
 package fetch
 
 import (
+	"errors"
 	"os"
+	"strings"
 
 	"github.com/genshen/pkg"
 )
@@ -11,16 +13,48 @@ const (
 	CacheStrategyCopyFromGlobalCache CacheStrategy = iota                            // package exits at user home's global cache directory.
 	CacheStrategyUserLocalVendor     CacheStrategy = iota                            // package exist in project's vendor/src directory.
 	CacheStrategySkip                CacheStrategy = iota                            // skip package downloading or local copying
+	CacheStrategyUseLocalReplace     CacheStrategy = iota                            // use local repo
 	CacheStrategyDefault                           = CacheStrategyDownloadFromRemote // package exist in project's vendor/src directory.
 )
 
 type CacheStrategy int
 
 // determinePackageCacheStrategy determines package cache strategy via package filesystem path.
 // noCache: dont use the global cache
-func determinePackageCacheStrategy(packageMeta pkg.PackageMeta, projectRoot string, noCache bool) (error, CacheStrategy) {
+func determinePackageCacheStrategy(packageMeta *pkg.PackageMeta, projectRoot string, noCache bool, localReplace, globalReplace map[string]pkg.GitReplaceMetadata) (error, CacheStrategy) {
+	// check local replace:
+	var localRepoPath string = ""
+	if replaceAddr, ok := localReplace[packageMeta.PackageName]; ok && replaceAddr.TargetType == pkg.GitReplaceTypeLocalGit {
+		localRepoPath = replaceAddr.Target
+	}
+	if replaceAddr, ok := globalReplace[packageMeta.PackageName]; ok && replaceAddr.TargetType == pkg.GitReplaceTypeLocalGit {
+		localRepoPath = replaceAddr.Target
+	}
+
+	if localRepoPath != "" {
+		parts := strings.SplitN(localRepoPath, "://", 2)
+		if len(parts) == 2 {
+			// replace the env in the path string.
+			packageEnv := pkg.NewPackageEnvs(projectRoot, packageMeta.PackageName, packageMeta.DefaultPackageSrcPathVendor(projectRoot))
+			if realReplacePath, err := pkg.ExpandEnv(parts[1], packageEnv); err != nil {
+				return err, CacheStrategySkip
+			} else { // get the real path
+				if _, err := os.Stat(realReplacePath); err != nil {
+					return err, CacheStrategySkip
+				}
+				packageMeta.SrcPath = realReplacePath
+				packageMeta.SrcLocationType = pkg.SrcLocationLocal
+				return nil, CacheStrategyUseLocalReplace
+			}
+		}
+		return errors.New("invalid path format: " + localRepoPath), CacheStrategySkip
+	}
+
 	pkgCachePath := packageMeta.HomeCacheSrcPath() // package global cache dir
-	pkgVendorPath := packageMeta.VendorSrcPath(projectRoot)
+	pkgVendorPath := packageMeta.PackageSrcPath(projectRoot)
+	packageMeta.SrcLocationType = pkg.SrcLocationVendor
+
+	packageMeta.SrcPath = pkgVendorPath // the src path should be the vendor src path
 
 	// copy only when global cache exists
 	if _, err := os.Stat(pkgVendorPath); os.IsNotExist(err) { // vendor src does not exist.

pkg/fetch/cmake_lib.go

@@ -95,7 +95,7 @@ func createPkgDepCmake(pkgHome string, rootDep *pkg.DependencyTree, findPackageO
 	}
 
 	for _, depTree := range depsList {
-		var packageSrcPath = depTree.Context.VendorSrcPath(pkgHome)
+		var packageSrcPath = depTree.Context.PackageSrcPath(pkgHome)
 		if depTree.Context.PackageName == pkg.RootPKG {
 			packageSrcPath = pkgHome
 		}
@@ -155,18 +155,20 @@ func cmakeLib(depTree *pkg.DependencyTree, pkgHome string, findPackageOptions st
 
 	basePath := "${PROJECT_HOME_PATH}"
 	for _, dep := range depsList {
-		src := dep.Context.VendorSrcPath(basePath) // vendor/src/@pkg@version,using relative path.
+		src := dep.Context.PackageSrcPath(basePath) // usually vendor/src/@pkg@version, or other path.
 		// add env variables for this package, using relative path.
 		packageEnv := pkg.NewPackageEnvs(basePath, dep.Context.PackageName, src)
 		// generating cmake script.
 		toFile := cmakeDepData{
 			PackageMeta: pkg.PackageMeta{
-				PackageName:  dep.Context.PackageName,
-				Version:      dep.Context.Version,
-				TargetName:   dep.Context.TargetName,
-				SelfCMakeLib: dep.Context.SelfCMakeLib,
-				CMakeLib:     dep.Context.CMakeLib,
-				Features:     dep.Context.Features,
+				PackageName:     dep.Context.PackageName,
+				SrcPath:         dep.Context.SrcPath,
+				SrcLocationType: dep.Context.SrcLocationType,
+				Version:         dep.Context.Version,
+				TargetName:      dep.Context.TargetName,
+				SelfCMakeLib:    dep.Context.SelfCMakeLib,
+				CMakeLib:        dep.Context.CMakeLib,
+				Features:        dep.Context.Features,
 			},
 			SrcDir:             src,
 			DepsDir:            pkg.GetPackageDepsPath(basePath, dep.Context.PackageName),

pkg/fetch/fetch.go

@@ -60,7 +60,7 @@ type fetch struct {
 	NoCache                bool     // download package without using global cache
 	DepTree                pkg.DependencyTree
 	Auth                   map[string]conf.Auth
-	GlobalReplace          map[string]string
+	GlobalReplace          map[string]pkg.GitReplaceMetadata
 }
 
 func (f *fetch) PreRun() error {
@@ -249,7 +249,7 @@ func (f *fetch) fetchSubDependency(pkgPath string, pkgVendorSrcPath string, acti
 
 			// initial empty local replace list
 			if pkgYaml.GitReplace == nil {
-				pkgYaml.GitReplace = make(map[string]string)
+				pkgYaml.GitReplace = make(map[string]pkg.GitReplaceMetadata)
 			}
 
 			// compare min pkg version in yaml file
@@ -279,7 +279,7 @@ func (f *fetch) fetchSubDependency(pkgPath string, pkgVendorSrcPath string, acti
 				depTree.Dependencies = append(depTree.Dependencies, deps...)
 				for _, dep := range deps {
 					// todo: currently, we disable features for sub dependencies.
-					if err := f.fetchSubDependency(dep.Context.PackageName, dep.Context.VendorSrcPath(f.PkgHome), nil, pkgLock, dep); err != nil {
+					if err := f.fetchSubDependency(dep.Context.PackageName, dep.Context.PackageSrcPath(f.PkgHome), nil, pkgLock, dep); err != nil {
 						return err
 					}
 				}
@@ -304,7 +304,7 @@ func (f *fetch) fetchSubDependency(pkgPath string, pkgVendorSrcPath string, acti
 // download a package source to destination refer to installPath, including source code and installed files.
 // usually src files are located at 'vendor/src/PackageName/', installed files are located at 'vendor/pkg/PackageName/'.
 // pkgHome: project root direction.
-func (f *fetch) dlPackagesDepSrc(pkgLock *map[string]string, featPkgList []string, localReplace, globalReplace map[string]string,
+func (f *fetch) dlPackagesDepSrc(pkgLock *map[string]string, featPkgList []string, localReplace, globalReplace map[string]pkg.GitReplaceMetadata,
 	packages map[string]PackageFetcher) ([]*pkg.DependencyTree, error) {
 	var deps []*pkg.DependencyTree
 	// todo check install.
@@ -330,36 +330,40 @@ func (f *fetch) dlPackagesDepSrc(pkgLock *map[string]string, featPkgList []strin
 
 		// src path in (global) user home
 		srcDes := context.HomeCacheSrcPath()
-		vendorSrcDes := context.VendorSrcPath(f.PkgHome)
 
-		err, strategy := determinePackageCacheStrategy(context, f.PkgHome, f.NoCache)
+		err, strategy := determinePackageCacheStrategy(&context, f.PkgHome, f.NoCache, localReplace, globalReplace)
 		if err != nil {
 			return nil, err
 		}
 
+		pkgSrcPath := context.PackageSrcPath(f.PkgHome)
 		switch strategy {
 		case CacheStrategyDownloadFromRemote:
 			log.WithFields(log.Fields{"pkg": context.PackageName, "storage": srcDes}).Info("downloading dependencies.")
 			if err := p.fetch(f.Auth, localReplace, globalReplace, srcDes, context); err != nil {
 				return nil, err
 			}
 			// copy package from system global cache to project's vendor/src
-			if err := copy.Copy(srcDes, vendorSrcDes); err != nil {
+			if err := copy.Copy(srcDes, pkgSrcPath); err != nil {
 				return nil, err
 			}
 			status = pkg.DlStatusOk
 		case CacheStrategyCopyFromGlobalCache:
 			log.WithFields(log.Fields{"pkg": key, "src_path": srcDes}).Info("skipped fetching package, because it already exists.")
 			// copy downloaded packages from global cache to vendor directory
-			if err := copy.Copy(srcDes, vendorSrcDes); err != nil {
+			if err := copy.Copy(srcDes, pkgSrcPath); err != nil {
+				println("could not copy package", err.Error(), "src#", srcDes, "dest#", pkgSrcPath)
 				return nil, err
 			}
 			status = pkg.DlStatusSkip
 		case CacheStrategyUserLocalVendor:
-			log.WithFields(log.Fields{"pkg": key, "src_path": vendorSrcDes}).Info("skipped fetching package, because it already exists.")
+			log.WithFields(log.Fields{"pkg": key, "src_path": pkgSrcPath}).Info("skipped fetching package, because it already exists.")
 			status = pkg.DlStatusSkip
 		case CacheStrategySkip:
 			// not handled: skip with error.
+		case CacheStrategyUseLocalReplace:
+			log.WithFields(log.Fields{"pkg": key, "src_path": pkgSrcPath}).Info("skipped fetching package, because it already exists.")
+			status = pkg.DlStatusSkip
 		default:
 			return nil, fmt.Errorf("unknown package cache strategy: %d", strategy)
 		}

pkg/fetch/fetcher.go

@@ -11,7 +11,7 @@ import (
 
 type PackageFetcher interface {
 	setPackageMeta(pkgPath string, meta *pkg.PackageMeta) error
-	fetch(auth map[string]conf.Auth, localReplace, globalReplace map[string]string, srcDes string, meta pkg.PackageMeta) error
+	fetch(auth map[string]conf.Auth, localReplace, globalReplace map[string]pkg.GitReplaceMetadata, srcDes string, meta pkg.PackageMeta) error
 }
 
 type YamlGitPkgFetcher pkg.YamlGitPackage
@@ -44,17 +44,17 @@ func (git *YamlGitPkgFetcher) setPackageMeta(pkgPath string, meta *pkg.PackageMe
 	return nil
 }
 
-func (git *YamlGitPkgFetcher) fetch(auth map[string]conf.Auth, localReplace, globalReplace map[string]string, srcDes string, meta pkg.PackageMeta) error {
+func (git *YamlGitPkgFetcher) fetch(auth map[string]conf.Auth, localReplace, globalReplace map[string]pkg.GitReplaceMetadata, srcDes string, meta pkg.PackageMeta) error {
 	// replace priority: package.path in package's pkg.yaml < local replace in pkg.yaml
 	// < local replace in `pkg.config.yaml` < replace in global config
 	if git.Path == "" {
 		git.Path = fmt.Sprintf("https://%s.git", meta.PackageName)
 	}
-	if replaceAddr, ok := localReplace[meta.PackageName]; ok {
-		git.Path = fmt.Sprintf("https://%s.git", replaceAddr)
+	if replaceAddr, ok := localReplace[meta.PackageName]; ok && replaceAddr.TargetType == pkg.GitReplaceTypeRemoteGit {
+		git.Path = fmt.Sprintf("https://%s.git", replaceAddr.Target)
 	}
-	if replaceAddr, ok := globalReplace[meta.PackageName]; ok {
-		git.Path = fmt.Sprintf("https://%s.git", replaceAddr)
+	if replaceAddr, ok := globalReplace[meta.PackageName]; ok && replaceAddr.TargetType == pkg.GitReplaceTypeRemoteGit {
+		git.Path = fmt.Sprintf("https://%s.git", replaceAddr.Target)
 	}
 
 	log.WithFields(log.Fields{
@@ -80,7 +80,7 @@ func (files *YamlFilesPkgFetcher) setPackageMeta(pkgPath string, meta *pkg.Packa
 	return nil
 }
 
-func (files *YamlFilesPkgFetcher) fetch(auth map[string]conf.Auth, localReplace, globalReplace map[string]string, srcDes string, meta pkg.PackageMeta) error {
+func (files *YamlFilesPkgFetcher) fetch(auth map[string]conf.Auth, localReplace, globalReplace map[string]pkg.GitReplaceMetadata, srcDes string, meta pkg.PackageMeta) error {
 	if err := filesSrc(srcDes, meta.PackageName, files.Path, files.Files); err != nil {
 		_ = os.RemoveAll(srcDes)
 		return err
@@ -99,7 +99,7 @@ func (archive *YamlArchivePkgFetcher) setPackageMeta(pkgPath string, meta *pkg.P
 	return nil
 }
 
-func (archive *YamlArchivePkgFetcher) fetch(auth map[string]conf.Auth, localReplace, globalReplace map[string]string, srcDes string, meta pkg.PackageMeta) error {
+func (archive *YamlArchivePkgFetcher) fetch(auth map[string]conf.Auth, localReplace, globalReplace map[string]pkg.GitReplaceMetadata, srcDes string, meta pkg.PackageMeta) error {
 	if err := archiveSrc(archive.Type, srcDes, meta.PackageName, archive.Path); err != nil {
 		_ = os.RemoveAll(srcDes)
 		return err

pkg/import/import.go

@@ -138,7 +138,7 @@ func (i *_import) Run() error {
 		}
 		// todo use pkg api to get from path
 		packagePathFrom := filepath.Join(importCache, pkg.VendorSrc, meta.PackageName+"@"+meta.Version)
-		targetSrcPath := meta.VendorSrcPath(i.home)
+		targetSrcPath := meta.PackageSrcPath(i.home)
 		// check source directory
 		if _, err := os.Stat(packagePathFrom); err != nil {
 			return err

pkg/install/command.go

@@ -41,7 +41,7 @@ func (in *InsExecutor) PkgPreInstall(meta *pkg.PackageMeta) (*pkg.PackageEnvs, e
 		"pkg": meta.PackageName,
 	}).Info("installing package.")
 	// package env
-	packageEnv := pkg.NewPackageEnvs(in.pkgHome, meta.PackageName, meta.VendorSrcPath(in.pkgHome))
+	packageEnv := pkg.NewPackageEnvs(in.pkgHome, meta.PackageName, meta.PackageSrcPath(in.pkgHome))
 	return packageEnv, nil
 }
 
@@ -57,7 +57,7 @@ func (in *InsExecutor) InsCp(triple pkg.InsTriple, meta *pkg.PackageMeta) error
 		return errors.New("CP instruction must have src and des")
 	}
 	// run copy.
-	srcPath := meta.VendorSrcPath(in.pkgHome)
+	srcPath := meta.PackageSrcPath(in.pkgHome)
 	if err := runInsCopy(filepath.Join(srcPath, triple.Second), triple.Third); err != nil {
 		return err
 	}
@@ -86,7 +86,7 @@ func (in *InsExecutor) InsRun(triple pkg.InsTriple, meta *pkg.PackageMeta) error
 // Triple Third: cmake build arguments
 func (in *InsExecutor) InsCMake(triple pkg.InsTriple, meta *pkg.PackageMeta) error {
 	packageCacheDir := pkg.GetCachePath(in.pkgHome, meta.PackageName)
-	srcPath := meta.VendorSrcPath(in.pkgHome)
+	srcPath := meta.PackageSrcPath(in.pkgHome)
 
 	// make sure the dirs exist.
 	if _, err := os.Stat(packageCacheDir); err != nil {

pkg/install/install_shell.go

@@ -49,7 +49,7 @@ PKG_SRC_PATH=%s
 
 func (sh *InsShellWriter) PkgPreInstall(meta *pkg.PackageMeta) (*pkg.PackageEnvs, error) {
 	// using short path with env '$PKG_SRC_PATH'.
-	packageSrcPath := strings.Replace(meta.VendorSrcPath(sh.pkgHome), pkg.GetPkgSrcPath(sh.pkgHome), "$PKG_SRC_PATH", 1)
+	packageSrcPath := strings.Replace(meta.PackageSrcPath(sh.pkgHome), pkg.GetPkgSrcPath(sh.pkgHome), "$PKG_SRC_PATH", 1)
 	// package env
 	packageEnv := pkg.NewPackageEnvs("$PROJECT_HOME", meta.PackageName, packageSrcPath)
 	if _, err := sh.writer.WriteString(fmt.Sprintf("\n## pacakge %s\n", meta.PackageName)); err != nil {
@@ -68,7 +68,7 @@ func (sh *InsShellWriter) InsCp(triple pkg.InsTriple, meta *pkg.PackageMeta) err
 	}
 
 	pathBase := "${PROJECT_HOME}"
-	srcPath := meta.VendorSrcPath(pathBase)
+	srcPath := meta.DefaultPackageSrcPathVendor(pathBase)
 	if _, err := sh.writer.WriteString(fmt.Sprintf("mkdir -p \"%s\"\n", pkg.GetIncludePath(pathBase))); err != nil {
 		return err
 	}
@@ -93,7 +93,7 @@ func (sh *InsShellWriter) InsRun(triple pkg.InsTriple, meta *pkg.PackageMeta) er
 
 func (sh *InsShellWriter) InsCMake(triple pkg.InsTriple, meta *pkg.PackageMeta) error {
 	pathBase := "${PROJECT_HOME}"
-	srcPath := meta.VendorSrcPath(pathBase)
+	srcPath := meta.PackageSrcPath(pathBase)
 
 	// prepare cmake config from "features" in pkg.yaml
 	if meta.Features != nil && len(meta.Features) != 0 {