ar0c/goc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge pull request #146 from lyyyuna/perf

Browse Source 
Improve performance of getting filelist to copy
This commit is contained in:
qiniu-bot 2020-12-21 20:39:25 +08:00 committed by GitHub
commit 692c3c806e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 97 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
pkg/build/legacy.go
View File

@ -20,10 +20,9 @@ import (
"path/filepath"
"strings"
log "github.com/sirupsen/logrus"
"github.com/otiai10/copy"
cc "github.com/otiai10/copy" // conflit with builtin copy
"github.com/qiniu/goc/pkg/cover"
log "github.com/sirupsen/logrus"
)
func (b *Build) cpProject() {
@ -48,13 +47,13 @@ func (b *Build) cpProject() {
if v.Name == "main" {
dst := filepath.Join(b.TmpDir, "go.mod")
src := filepath.Join(v.Module.Dir, "go.mod")
if err := copy.Copy(src, dst); err != nil {
if err := cc.Copy(src, dst); err != nil {
log.Errorf("Failed to Copy the go mod file from %v to %v, the error is: %v ", src, dst, err)
}
dst = filepath.Join(b.TmpDir, "go.sum")
src = filepath.Join(v.Module.Dir, "go.sum")
if err := copy.Copy(src, dst); err != nil && !strings.Contains(err.Error(), "no such file or directory") {
if err := cc.Copy(src, dst); err != nil && !strings.Contains(err.Error(), "no such file or directory") {
log.Errorf("Failed to Copy the go mod file from %v to %v, the error is: %v ", src, dst, err)
}
break
@ -68,18 +67,7 @@ func (b *Build) cpProject() {
func (b *Build) copyDir(pkg *cover.Package) error {
fileList := []string{}
dir := pkg.Dir
fileList = append(fileList, pkg.GoFiles...)
fileList = append(fileList, pkg.CompiledGoFiles...)
fileList = append(fileList, pkg.IgnoredGoFiles...)
fileList = append(fileList, pkg.CFiles...)
fileList = append(fileList, pkg.CXXFiles...)
fileList = append(fileList, pkg.MFiles...)
fileList = append(fileList, pkg.HFiles...)
fileList = append(fileList, pkg.FFiles...)
fileList = append(fileList, pkg.SFiles...)
fileList = append(fileList, pkg.SwigCXXFiles...)
fileList = append(fileList, pkg.SwigFiles...)
fileList = append(fileList, pkg.SysoFiles...)
fileList = getFileListNeedsCopy(pkg)
for _, file := range fileList {
p := filepath.Join(dir, file)
var src, root string
@ -90,10 +78,69 @@ func (b *Build) copyDir(pkg *cover.Package) error {
}
src = strings.TrimPrefix(pkg.Dir, root) // get the relative path of the files
dst := filepath.Join(b.TmpDir, src, file) // it will adapt the case where src is ""
if err := copy.Copy(p, dst); err != nil {
if err := cc.Copy(p, dst); err != nil {
log.Errorf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err)
return err
}
}
return nil
}
// original implementation
// goos: darwin
// goarch: amd64
// pkg: github.com/qiniu/goc/pkg/build
// BenchmarkGetFileList-4 1535029 778 ns/op 1488 B/op 5 allocs/op
// func getFileListNeedsCopy(pkg *cover.Package) []string {
// fileList := []string{}
// fileList = append(fileList, pkg.GoFiles...)
// fileList = append(fileList, pkg.CompiledGoFiles...)
// fileList = append(fileList, pkg.IgnoredGoFiles...)
// fileList = append(fileList, pkg.CFiles...)
// fileList = append(fileList, pkg.CXXFiles...)
// fileList = append(fileList, pkg.MFiles...)
// fileList = append(fileList, pkg.HFiles...)
// fileList = append(fileList, pkg.FFiles...)
// fileList = append(fileList, pkg.SFiles...)
// fileList = append(fileList, pkg.SwigCXXFiles...)
// fileList = append(fileList, pkg.SwigFiles...)
// fileList = append(fileList, pkg.SysoFiles...)
// return fileList
// }
// new implementation
// goos: darwin
// goarch: amd64
// pkg: github.com/qiniu/goc/pkg/build
// BenchmarkGetFileList-4 3884557 298 ns/op 576 B/op 1 allocs/op
func getFileListNeedsCopy(pkg *cover.Package) []string {
fileSlices := [][]string{
pkg.GoFiles,
pkg.CompiledGoFiles,
pkg.IgnoredGoFiles,
pkg.CFiles,
pkg.CXXFiles,
pkg.MFiles,
pkg.HFiles,
pkg.FFiles,
pkg.SFiles,
pkg.SwigCXXFiles,
pkg.SwigFiles,
pkg.SysoFiles,
}
var totalLen int
for _, s := range fileSlices {
totalLen += len(s)
}
tmp := make([]string, totalLen)
var i int
for _, s := range fileSlices {
i += copy(tmp[i:], s)
}
return tmp
}

33
pkg/build/legacy_test.go
View File

@ -17,12 +17,12 @@
package build
import (
"os"
"strings"
"testing"
"github.com/qiniu/goc/pkg/cover"
"github.com/stretchr/testify/assert"
"os"
)
// copy in cpProject of invalid src, dst name
@ -80,3 +80,34 @@ func TestCopyDir(t *testing.T) {
defer os.RemoveAll(tmpDir)
assert.NoError(t, b.copyDir(pkg))
}
func initSlicesForTest() *cover.Package {
var pkg cover.Package
pkg.GoFiles = []string{"a1.go", "b1.go", "c1.go"}
pkg.CompiledGoFiles = []string{"a2.go", "b2.go", "c2.go"}
pkg.IgnoredGoFiles = []string{"a3.go", "b3.go", "c3.go"}
pkg.CFiles = []string{"a4.go", "b4.go", "c4.go"}
pkg.CXXFiles = []string{"a5.go", "b5.go", "c5.go"}
pkg.MFiles = []string{"a6.go", "b6.go", "c6.go"}
pkg.HFiles = []string{"a7.go", "b7.go", "c7.go"}
pkg.FFiles = []string{"a8.go", "b8.go", "c8.go"}
pkg.SFiles = []string{"a9.go", "b9.go", "c9.go"}
pkg.SwigCXXFiles = []string{"a10.go", "b10.go", "c10.go"}
pkg.SwigFiles = []string{"a11.go", "b11.go", "c11.go"}
pkg.SysoFiles = []string{"a12.go", "b12.go", "c12.go"}
return &pkg
}
// benchmark getFileListNeedsCopy
// goos: darwin
// goarch: amd64
// pkg: github.com/qiniu/goc/pkg/build
// BenchmarkGetFileList-4 3884557 298 ns/op 576 B/op 1 allocs/op
func BenchmarkGetFileList(b *testing.B) {
//var files []string
pkg := initSlicesForTest()
for n := 0; n < b.N; n++ {
getFileListNeedsCopy(pkg)
}
}