ar0c/goc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge pull request #150 from tongjingran/buildCopy2

Browse Source 
optimize the file copy strategy of `goc build` and fix bugs
This commit is contained in:
qiniu-bot 2020-12-31 16:54:19 +08:00 committed by GitHub
commit bc7cae11cb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 128 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -9,13 +9,13 @@ require (
github.com/julienschmidt/httprouter v1.2.0
github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/olekukonko/tablewriter v0.0.4
github.com/otiai10/copy v1.0.2
github.com/qiniu/api.v7/v7 v7.5.0
github.com/sirupsen/logrus v1.6.0
github.com/spf13/cobra v1.0.0
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.6.2
github.com/stretchr/testify v1.5.1
github.com/tongjingran/copy v1.3.3
golang.org/x/mod v0.3.0
golang.org/x/net v0.0.0-20200625001655-4c5254603344
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d

10
go.sum
View File

@ -325,6 +325,7 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@ -578,8 +579,12 @@ github.com/otiai10/copy v1.0.2 h1:DDNipYy6RkIkjMwy+AWzgKiNTyj2RUI9yEMeETEpVyc=
github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY=
github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95 h1:+OLn68pqasWca0z5ryit9KGfp3sUsW4Lqg32iRMJyzs=
github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
github.com/otiai10/curr v1.0.0 h1:TJIWdbX0B+kpNagQrjgq8bCMrbhiuX73M2XwgtDMoOI=
github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
github.com/otiai10/mint v1.3.0 h1:Ady6MKVezQwHBkGzLFbrsywyp09Ah7rkmfjV3Bcr5uc=
github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
github.com/otiai10/mint v1.3.2 h1:VYWnrP5fXmz1MXvjuUvcBrXSjGE6xjON+axB/UrpO3E=
github.com/otiai10/mint v1.3.2/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@ -714,6 +719,8 @@ github.com/tektoncd/plumbing/pipelinerun-logs v0.0.0-20191206114338-712d544c2c21
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tongjingran/copy v1.3.3 h1:Oha32tKfDrLTSXGaxW+jaTSmGc+hiHFRf/STZ4x9b4g=
github.com/tongjingran/copy v1.3.3/go.mod h1:Njma1OR5OuzB8pLAmQSzonHXzba+DDiPVmMSonpSpy4=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
@ -739,7 +746,6 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMx
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
@ -962,8 +968,6 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapK
golang.org/x/tools v0.0.0-20200214144324-88be01311a71/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200303214625-2b0b585e22fe h1:Kh3iY7o/2bMfQXZdwLdL9jDMU1k9HoVn0P1mGCfoFLc=
golang.org/x/tools v0.0.0-20200303214625-2b0b585e22fe/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65 h1:1KSbntBked74wYsKq0jzXYy7ZwcjAUtrl7EmPE97Iiw=
golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
golang.org/x/tools v0.0.0-20200730221956-1ac65761fe2c h1:c5JjBOQWM2pRemVbog00sS+oAdi8tTR+NNRFDwUOrTQ=
golang.org/x/tools v0.0.0-20200730221956-1ac65761fe2c/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

7
pkg/build/build.go
View File

@ -133,8 +133,11 @@ func (b *Build) determineOutputDir(outputDir string) (string, error) {
targetName := ""
for _, pkg := range b.Pkgs {
if pkg.Name == "main" {
_, file := filepath.Split(pkg.Target)
targetName = file
if pkg.Target != "" {
targetName = filepath.Base(pkg.Target)
} else {
targetName = filepath.Base(pkg.Dir)
}
break
}
}

4
pkg/build/gomodules.go
View File

@ -20,8 +20,8 @@ import (
"io/ioutil"
"path/filepath"
"github.com/otiai10/copy"
log "github.com/sirupsen/logrus"
"github.com/tongjingran/copy"
"golang.org/x/mod/modfile"
)
@ -31,7 +31,7 @@ func (b *Build) cpGoModulesProject() {
dst := b.TmpDir
src := v.Module.Dir
if err := copy.Copy(src, dst); err != nil {
if err := copy.Copy(src, dst, copy.Options{Skip: skipCopy}); err != nil {
log.Errorf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err)
}
break

23
pkg/build/legacy.go
View File

@ -19,11 +19,12 @@ package build
import (
"os"
"path/filepath"
"strings"
log "github.com/sirupsen/logrus"
"github.com/otiai10/copy"
"github.com/qiniu/goc/pkg/cover"
"github.com/tongjingran/copy"
)
func (b *Build) cpLegacyProject() {
@ -37,7 +38,7 @@ func (b *Build) cpLegacyProject() {
continue
}
if err := copy.Copy(src, dst); err != nil {
if err := copy.Copy(src, dst, copy.Options{Skip: skipCopy}); err != nil {
log.Errorf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err)
}
@ -66,7 +67,7 @@ func (b *Build) cpDepPackages(pkg *cover.Package, visited map[string]bool) {
dst := filepath.Join(b.TmpDir, "src", dep)
if err := copy.Copy(src, dst); err != nil {
if err := copy.Copy(src, dst, copy.Options{Skip: skipCopy}); err != nil {
log.Errorf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err)
}
@ -80,7 +81,7 @@ func (b *Build) cpNonStandardLegacy() {
dst := b.TmpDir
src := v.Dir
if err := copy.Copy(src, dst); err != nil {
if err := copy.Copy(src, dst, copy.Options{Skip: skipCopy}); err != nil {
log.Printf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err)
}
break
@ -89,3 +90,17 @@ func (b *Build) cpNonStandardLegacy() {
}
}
}
// skipCopy skip copy .git dir and irregular files
func skipCopy(src string, info os.FileInfo) (bool, error) {
irregularModeType := os.ModeNamedPipe | os.ModeSocket | os.ModeDevice | os.ModeCharDevice | os.ModeIrregular
if strings.HasSuffix(src, "/.git") {
log.Infof("Skip .git dir [%s]", src)
return true, nil
}
if info.Mode()&irregularModeType != 0 {
log.Warnf("Skip file [%s], the file mode is [%s]", src, info.Mode().String())
return true, nil
}
return false, nil
}

62
pkg/build/legacy_test.go
View File

@ -17,9 +17,11 @@
package build
import (
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/qiniu/goc/pkg/cover"
"github.com/stretchr/testify/assert"
@ -69,3 +71,63 @@ func TestDepPackagesCopyWithInvalidDir(t *testing.T) {
})
assert.Equal(t, strings.Contains(output, "Failed to Copy"), true)
}
type MockFile struct {
name string
size int64
mode os.FileMode
modTime time.Time
isDir bool
}
func (m MockFile) Name() string {
return m.name
}
func (m MockFile) Size() int64 {
return m.size
}
func (m MockFile) Mode() os.FileMode {
return m.mode
}
func (m MockFile) ModTime() time.Time {
return m.modTime
}
func (m MockFile) IsDir() bool {
return m.isDir
}
func (m MockFile) Sys() interface{} {
return nil
}
// skipCopy verify
func TestSkipCopy(t *testing.T) {
testCases := map[string]struct {
inputSrc string
inputInfo MockFile
expected bool
}{
"src with /.git suffix": {inputSrc: "/test/.git", inputInfo: MockFile{mode: 0}, expected: true},
"src with ./git suffix": {inputSrc: "/test.git", inputInfo: MockFile{mode: 0}, expected: false},
"src with /.gita suffix": {inputSrc: "/test/.gita", inputInfo: MockFile{mode: 0}, expected: false},
"src with /.git in middle": {inputSrc: "/test/.git/test", inputInfo: MockFile{mode: 0}, expected: false},
"irregular file": {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeIrregular}, expected: true},
"dir file": {inputSrc: "/test", inputInfo: MockFile{isDir: true, mode: os.ModeDir}, expected: false},
"temporary file": {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeTemporary}, expected: false},
"symlink file": {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeSymlink}, expected: false},
"device file": {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeDevice}, expected: true},
"named pipe file": {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeNamedPipe}, expected: true},
"socket file": {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeSocket}, expected: true},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
output, err := skipCopy(tc.inputSrc, tc.inputInfo)
assert.NoError(t, err)
assert.Equal(t, output, tc.expected)
})
}
}

2
pkg/build/tmpfolder.go
View File

@ -115,8 +115,6 @@ func (b *Build) mvProjectsToTmp() error {
} else if b.IsMod == false && b.Root == "" {
b.TmpWorkingDir = b.TmpDir
b.cpNonStandardLegacy()
} else {
return fmt.Errorf("unknown project type: %w", ErrShouldNotReached)
}
log.Infof("New workingdir in tmp directory in: %v", b.TmpWorkingDir)

2
pkg/cover/cover_test.go
View File

@ -27,8 +27,8 @@ import (
log "github.com/sirupsen/logrus"
"github.com/otiai10/copy"
"github.com/stretchr/testify/assert"
"github.com/tongjingran/copy"
)
func testCoverage() (c *Coverage) {

13
tests/build.bats
View File

@ -98,5 +98,18 @@ setup() {
info build5 output: $output
[ "$status" -eq 0 ]
wait $profile_pid
}
@test "test goc build on reference other package project" {
cd samples/reference_other_package_project/app
wait_profile_backend "build6" &
profile_pid=$!
run gocc build --debug --debugcisyncfile ci-sync.bak;
info build5 output: $output
[ "$status" -eq 0 ]
wait $profile_pid
}

8
tests/samples/reference_other_package_project/app/main.go Normal file
View File

@ -0,0 +1,8 @@
package main
import "example.com/reference_other_package_project/foo"
func main() {
foo.PrintFoo()
return
}

7
tests/samples/reference_other_package_project/foo/foo.go Normal file
View File

@ -0,0 +1,7 @@
package foo
import "fmt"
func PrintFoo() {
fmt.Println("hello, world.")
}

3
tests/samples/reference_other_package_project/go.mod Normal file
View File

@ -0,0 +1,3 @@
module example.com/reference_other_package_project
go 1.11