From d8cd308ce1e60a5e8caaf1a24c2ca40bdffa2782 Mon Sep 17 00:00:00 2001 From: tongjingran Date: Mon, 14 Dec 2020 10:05:59 +0800 Subject: [PATCH] optmise the copy stratgy --- pkg/build/gomodules.go | 18 ---------- pkg/build/gomodules_test.go | 3 +- pkg/build/legacy.go | 65 +++++++++++++++++++++++++++++-------- pkg/build/legacy_test.go | 21 +++++++++--- pkg/build/tmpfolder.go | 4 +-- pkg/cover/cover.go | 15 +++++++-- 6 files changed, 85 insertions(+), 41 deletions(-) diff --git a/pkg/build/gomodules.go b/pkg/build/gomodules.go index 32c9b79..ac9c07c 100644 --- a/pkg/build/gomodules.go +++ b/pkg/build/gomodules.go @@ -20,27 +20,9 @@ import ( "io/ioutil" "path/filepath" - "github.com/otiai10/copy" - log "github.com/sirupsen/logrus" "golang.org/x/mod/modfile" ) -func (b *Build) cpGoModulesProject() { - for _, v := range b.Pkgs { - if v.Name == "main" { - dst := b.TmpDir - src := v.Module.Dir - - if err := copy.Copy(src, dst); err != nil { - log.Errorf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err) - } - break - } else { - continue - } - } -} - // updateGoModFile rewrites the go.mod file in the temporary directory, // if it has a 'replace' directive, and the directive has a relative local path // it will be rewritten with a absolute path. diff --git a/pkg/build/gomodules_test.go b/pkg/build/gomodules_test.go index e0e6a6d..c629ec3 100644 --- a/pkg/build/gomodules_test.go +++ b/pkg/build/gomodules_test.go @@ -45,6 +45,7 @@ func TestModProjectCopyWithUnexistedDir(t *testing.T) { Module: &cover.ModulePublic{ Dir: "not exied, ia mas duser", // not real one, should fail copy }, + GoFiles: []string{"empty.go"}, } pkgs["another"] = &cover.Package{} b := &Build{ @@ -52,7 +53,7 @@ func TestModProjectCopyWithUnexistedDir(t *testing.T) { Pkgs: pkgs, } - output := captureOutput(b.cpGoModulesProject) + output := captureOutput(b.cpLegacyProject) assert.Equal(t, strings.Contains(output, "Failed to Copy"), true) } diff --git a/pkg/build/legacy.go b/pkg/build/legacy.go index 72f5c55..8583895 100644 --- a/pkg/build/legacy.go +++ b/pkg/build/legacy.go @@ -24,6 +24,7 @@ import ( "github.com/otiai10/copy" "github.com/qiniu/goc/pkg/cover" + "strings" ) func (b *Build) cpLegacyProject() { @@ -37,18 +38,39 @@ func (b *Build) cpLegacyProject() { continue } - if err := copy.Copy(src, dst); err != nil { + if err := b.copyDir(v, b.TmpDir); err != nil { log.Errorf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err) } visited[src] = true - b.cpDepPackages(v, visited) + //b.cpDepPackages(v, visited) + } + if b.IsMod { + for _, v := range b.Pkgs { + if v.Name == "main" { + dst := filepath.Join(b.TmpDir, "go.mod") + src := filepath.Join(v.Module.Dir, "go.mod") + if err := copy.Copy(src, dst); err != nil { + log.Errorf("Failed to Copy the go mod file from %v to %v, the error is: %v ", src, dst, err) + } + + dst = filepath.Join(b.TmpDir, "go.sum") + src = filepath.Join(v.Module.Dir, "go.sum") + if err := copy.Copy(src, dst); err != nil && !strings.Contains(err.Error(), "no such file or directory") { + log.Errorf("Failed to Copy the go mod file from %v to %v, the error is: %v ", src, dst, err) + } + break + } else { + continue + } + } } } // only cp dependency in root(current gopath), // skip deps in other GOPATHs +// only used for version before go 1.11.4 func (b *Build) cpDepPackages(pkg *cover.Package, visited map[string]bool) { gopath := pkg.Root for _, dep := range pkg.Deps { @@ -74,18 +96,35 @@ func (b *Build) cpDepPackages(pkg *cover.Package, visited map[string]bool) { } } -func (b *Build) cpNonStandardLegacy() { - for _, v := range b.Pkgs { - if v.Name == "main" { - dst := b.TmpDir - src := v.Dir - - if err := copy.Copy(src, dst); err != nil { - log.Printf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err) - } - break +func (b *Build) copyDir(pkg *cover.Package, tmpDir string) error { + fileList := []string{} + dir := pkg.Dir + fileList = append(fileList, pkg.GoFiles...) + fileList = append(fileList, pkg.CompiledGoFiles...) + fileList = append(fileList, pkg.IgnoredGoFiles...) + fileList = append(fileList, pkg.CFiles...) + fileList = append(fileList, pkg.CXXFiles...) + fileList = append(fileList, pkg.MFiles...) + fileList = append(fileList, pkg.HFiles...) + fileList = append(fileList, pkg.FFiles...) + fileList = append(fileList, pkg.SFiles...) + fileList = append(fileList, pkg.SwigCXXFiles...) + fileList = append(fileList, pkg.SwigFiles...) + fileList = append(fileList, pkg.SysoFiles...) + for _, file := range fileList { + p := filepath.Join(dir, file) + var src, root string + if pkg.Root == "" { + root = b.WorkingDir // use workingDir instead when the root is empty. } else { - continue + root = pkg.Root + } + src = strings.TrimPrefix(pkg.Dir, root) // get the relative path of the files + dst := filepath.Join(tmpDir, src, file) // it will adapt the case where src is "" + if err := copy.Copy(p, dst); err != nil { + log.Errorf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err) + return err } } + return nil } diff --git a/pkg/build/legacy_test.go b/pkg/build/legacy_test.go index 53f9cc1..1c5f6c6 100644 --- a/pkg/build/legacy_test.go +++ b/pkg/build/legacy_test.go @@ -23,6 +23,7 @@ import ( "github.com/qiniu/goc/pkg/cover" "github.com/stretchr/testify/assert" + "os" ) // copy in cpLegacyProject/cpNonStandardLegacy of invalid src, dst name @@ -32,8 +33,9 @@ func TestLegacyProjectCopyWithUnexistedDir(t *testing.T) { Module: &cover.ModulePublic{ Dir: "not exied, ia mas duser", // not real one, should fail copy }, - Dir: "not exit, iasdfs", - Name: "main", + Dir: "not exit, iasdfs", + Name: "main", + GoFiles: []string{"not_exist.go"}, } pkgs["another"] = &cover.Package{} b := &Build{ @@ -43,9 +45,6 @@ func TestLegacyProjectCopyWithUnexistedDir(t *testing.T) { output := captureOutput(b.cpLegacyProject) assert.Equal(t, strings.Contains(output, "Failed to Copy"), true) - - output = captureOutput(b.cpNonStandardLegacy) - assert.Equal(t, strings.Contains(output, "Failed to Copy"), true) } // copy in cpDepPackages of invalid dst name @@ -69,3 +68,15 @@ func TestDepPackagesCopyWithInvalidDir(t *testing.T) { }) assert.Equal(t, strings.Contains(output, "Failed to Copy"), true) } + +func TestCopyDir(t *testing.T) { + wd, _ := os.Getwd() + pkg := &cover.Package{Dir: wd, Root: wd, GoFiles: []string{"build.go", "legacy.go"}, CgoFiles: []string{"run.go"}} + tmpDir := "/tmp/test/" + b := &Build{ + WorkingDir: "empty", + } + assert.NoError(t, os.MkdirAll(tmpDir, os.ModePerm)) + defer os.RemoveAll(tmpDir) + assert.NoError(t, b.copyDir(pkg, tmpDir)) +} diff --git a/pkg/build/tmpfolder.go b/pkg/build/tmpfolder.go index 9cc8fbe..e4687ef 100644 --- a/pkg/build/tmpfolder.go +++ b/pkg/build/tmpfolder.go @@ -99,7 +99,7 @@ func (b *Build) mvProjectsToTmp() error { if b.IsMod == false && b.Root != "" { b.cpLegacyProject() } else if b.IsMod == true { // go 1.11, 1.12 has no Build.Root - b.cpGoModulesProject() + b.cpLegacyProject() updated, newGoModContent, err := b.updateGoModFile() if err != nil { return fmt.Errorf("fail to generate new go.mod: %v", err) @@ -114,7 +114,7 @@ func (b *Build) mvProjectsToTmp() error { } } else if b.IsMod == false && b.Root == "" { b.TmpWorkingDir = b.TmpDir - b.cpNonStandardLegacy() + b.cpLegacyProject() } else { return fmt.Errorf("unknown project type: %w", ErrShouldNotReached) } diff --git a/pkg/cover/cover.go b/pkg/cover/cover.go index 742e352..f9fa12d 100644 --- a/pkg/cover/cover.go +++ b/pkg/cover/cover.go @@ -85,8 +85,19 @@ type Package struct { DepOnly bool `json:"DepOnly,omitempty"` // package is only a dependency, not explicitly listed // Source files - GoFiles []string `json:"GoFiles,omitempty"` // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) - CgoFiles []string `json:"CgoFiles,omitempty"` // .go source files that import "C" + GoFiles []string `json:",omitempty"` // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) + CgoFiles []string `json:",omitempty"` // .go source files that import "C" + CompiledGoFiles []string `json:",omitempty"` // .go output from running cgo on CgoFiles + IgnoredGoFiles []string `json:",omitempty"` // .go source files ignored due to build constraints + CFiles []string `json:",omitempty"` // .c source files + CXXFiles []string `json:",omitempty"` // .cc, .cpp and .cxx source files + MFiles []string `json:",omitempty"` // .m source files + HFiles []string `json:",omitempty"` // .h, .hh, .hpp and .hxx source files + FFiles []string `json:",omitempty"` // .f, .F, .for and .f90 Fortran source files + SFiles []string `json:",omitempty"` // .s source files + SwigFiles []string `json:",omitempty"` // .swig files + SwigCXXFiles []string `json:",omitempty"` // .swigcxx files + SysoFiles []string `json:",omitempty"` // .syso system object files added to package // Dependency information Deps []string `json:"Deps,omitempty"` // all (recursively) imported dependencies