ar0c/goc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

remove code

Browse Source
This commit is contained in:
lyyyuna 2021-03-25 23:01:11 +08:00
parent 6e0de18292
commit a62ac335e2
159 changed files with 0 additions and 14915 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
.github/workflows/e2e_test_check.yml vendored
View File

@ -1,73 +0,0 @@
name: e2e test
on:
# Trigger the workflow on push or pull request,
# but only for the master branch
push:
paths-ignore:
- '**.md'
- '**.png'
pull_request:
paths-ignore:
- '**.md'
- '**.png'
jobs:
job_1:
name: Build goc binary
runs-on: ubuntu-latest
steps:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.14.x
- name: Checkout code
uses: actions/checkout@v2
- name: Go build
run: |
go build
- name: Use goc to build self
run: |
./goc build --output ./gocc --debug
- name: Upload goc binary
uses: actions/upload-artifact@v2
with:
name: goc
path: goc
- name: Upload covered self goc binary
uses: actions/upload-artifact@v2
with:
name: gocc
path: gocc
job_2:
name: E2E test
needs: job_1
strategy:
matrix:
go-version: [1.11.x, 1.12.x, 1.13.x, 1.14.x, 1.15.x, 1.16.x]
runs-on: ubuntu-latest
steps:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}
- name: Checkout code
uses: actions/checkout@v2
- name: Download built binary
uses: actions/download-artifact@v2
with:
path: /home/runner/tools
- name: Install bats-core
run: |
git clone https://github.com/bats-core/bats-core.git
cd bats-core
sudo ./install.sh /usr/local
- name: Do test
env:
GOVERSION: ${{ matrix.go-version }}
run: |
chmod +x /home/runner/tools/goc/goc
export PATH=/home/runner/tools/goc:$PATH
chmod +x /home/runner/tools/gocc/gocc
export PATH=/home/runner/tools/gocc:$PATH
cd tests
./run-ci-actions.sh

23
.github/workflows/golangci-lint.yml vendored
View File

@ -1,23 +0,0 @@
name: golangci-lint
on:
pull_request:
jobs:
golangci:
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: golangci-lint
uses: golangci/golangci-lint-action@v2
with:
# Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
version: v1.29
# Optional: working directory, useful for monorepos
# working-directory: somedir
# Optional: golangci-lint command line arguments.
# args: --issues-exit-code=0
# Optional: show only new issues if it's a pull request. The default value is `false`.
only-new-issues: true

55
.github/workflows/release.yml vendored
View File

@ -1,55 +0,0 @@
on:
release:
types: [published,edited]
name: Build Release
jobs:
release-linux-amd64:
name: release linux/amd64
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.14.x
- name: compile and release
run: |
./ci-build.sh
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GOARCH: amd64
GOOS: linux
release-linux-386:
name: release linux/386
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.14.x
- name: compile and release
run: |
./ci-build.sh
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GOARCH: "386"
GOOS: linux
release-darwin-amd64:
name: release darwin/amd64
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.14.x
- name: compile and release
run: |
./ci-build.sh
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GOARCH: amd64
GOOS: darwin

39
.github/workflows/style_check.yml vendored
View File

@ -1,39 +0,0 @@
name: style-check
on:
# Trigger the workflow on push or pull request,
# but only for the master branch
push:
paths-ignore:
- '**.md'
- '**.png'
pull_request:
paths-ignore:
- '**.md'
- '**.png'
jobs:
run:
name: vet and gofmt
strategy:
matrix:
go-version: [1.13.x, 1.14.x, 1.15.x]
runs-on: ubuntu-latest
steps:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}
# This step checks out a copy of your repository.
- name: Checkout code
uses: actions/checkout@v2
- name: Go vet check
run: |
go vet ./...
- name: Gofmt check
run: |
diff=`find . -name "*.go" | xargs gofmt -s -d`
if [[ -n "${diff}" ]]; then
echo "Gofmt check failed :"
echo "${diff}"
echo "Please run this command to fix: [find . -name "*.go" | xargs gofmt -s -w]"
exit 1
fi

34
.github/workflows/ut_check.yml vendored
View File

@ -1,34 +0,0 @@
name: ut-check
on:
# Trigger the workflow on push or pull request,
# but only for the master branch
push:
paths-ignore:
- '**.md'
- '**.png'
pull_request:
paths-ignore:
- '**.md'
- '**.png'
jobs:
run:
name: go test
strategy:
matrix:
go-version: [1.13.x, 1.14.x, 1.15.x, 1.16.x]
runs-on: ubuntu-latest
steps:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}
# This step checks out a copy of your repository.
- name: Checkout code
uses: actions/checkout@v2
- name: Go test
env:
GOVERSION: ${{ matrix.go-version }}
run: |
export DEFAULT_EXCEPT_PKGS=e2e
go test -p 1 -coverprofile=coverage.txt $(go list ./... | grep -v -E $DEFAULT_EXCEPT_PKGS)
bash <(curl -s https://codecov.io/bash) -F unittest-$GOVERSION

13
.gitignore vendored
View File

@ -1,13 +0,0 @@
# ignore log file
**/goc.log
# binary
goc
# the temp file to save service address
_svrs_address.txt
# other
*.iml
.DS_Store
.idea

21
Makefile
View File

@ -1,21 +0,0 @@
DEFAULT_EXCEPT_PKGS := e2e
all:
go install ./...
test:
go test -cover -p 1 `go list ./... | grep -v -E ${DEFAULT_EXCEPT_PKGS}`
fmt:
go fmt ./...
govet-check:
go vet ./...
clean:
find tests/ -type f -name '*.bak' -delete
find tests/ -type f -name '*.cov' -delete
find tests/ -type f -name 'simple-project' -delete
find tests/ -type f -name '*_profile_listen_addr' -delete
find tests/ -type f -name 'simple_gopath_project' -delete

88
README.md
View File

@ -1,88 +0,0 @@
# goc
[![Go Report Card](https://goreportcard.com/badge/github.com/qiniu/goc)](https://goreportcard.com/report/github.com/qiniu/goc)
![](https://github.com/qiniu/goc/workflows/ut-check/badge.svg)
![](https://github.com/qiniu/goc/workflows/style-check/badge.svg)
![](https://github.com/qiniu/goc/workflows/e2e%20test/badge.svg)
![Build Release](https://github.com/qiniu/goc/workflows/Build%20Release/badge.svg)
[![codecov](https://codecov.io/gh/qiniu/goc/branch/master/graph/badge.svg)](https://codecov.io/gh/qiniu/goc)
[![GoDoc](https://godoc.org/github.com/qiniu/goc?status.svg)](https://godoc.org/github.com/qiniu/goc)
[中文页](README_zh.md) |
goc is a comprehensive coverage testing system for The Go Programming Language, especially for some complex scenarios, like system testing code coverage collection and
accurate testing.
Enjoy, Have Fun!
![Demo](docs/images/intro.gif)
## Installation
Download the latest version from [Github Releases](https://github.com/qiniu/goc/releases) page.
Goc supports both `GOPATH` project and `Go Modules` project with **Go 1.11+**. However, for developing goc, you need to install **Go 1.13+**.
## Examples
You can use goc tool in many scenarios.
### Code Coverage Collection for Your Golang System Tests
Goc can collect code coverages at runtime for your long-run golang applications. To do that, normally just need three steps:
1. use `goc server` to start a service registry center:
```
➜ simple-go-server git:(master) ✗ goc server
```
2. use `goc build` to build the target service, and run the generated binary. Here let's take the [simple-go-server](https://github.com/CarlJi/simple-go-server) project as example:
```
➜ simple-go-server git:(master) ✗ goc build .
... // omit logs
➜ simple-go-server git:(master) ✗ ./simple-go-server
```
3. use `goc profile` to get the code coverage profile of the started simple server above:
```
➜ simple-go-server git:(master) ✗ goc profile
mode: atomic
enricofoltran/simple-go-server/main.go:30.13,48.33 13 1
enricofoltran/simple-go-server/main.go:48.33,50.3 1 0
enricofoltran/simple-go-server/main.go:52.2,65.12 5 1
enricofoltran/simple-go-server/main.go:65.12,74.46 7 1
enricofoltran/simple-go-server/main.go:74.46,76.4 1 0
...
```
### Show Code Coverage Change at Runtime in Vscode
We provide a vscode extension - [Goc Coverage](https://marketplace.visualstudio.com/items?itemName=lyyyuna.goc) which can show highlighted covered source code at runtime.
![Extension](docs/images/goc-vscode.gif)
## Tips
1. To understand the execution details of goc tool, you can use the `--debug` flag. Also we appreciate if you can provide such logs when submitting a bug to us.
2. By default, the covered service will listen a random port in order to communicate with the goc server. This may not be suitable in [docker](https://docs.docker.com/engine/reference/commandline/run/#publish-or-expose-port--p---expose) or [kubernetes](https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service) environment since the port must be exposed explicitly in order to be accessible by others in such environment. For this kind of scenario, you can use `--agentport` flag to specify a fixed port when calling `goc build` or `goc install`.
3. To use a remote goc server, you can use `--center` flag to compile the target service with `goc build` or `goc install` command.
4. The coverage data is stored on each covered service side, so if one service needs to restart during test, this service's coverage data will be lost. For this case, you can use following steps to handle:
1. Before the service restarts, collect coverage with `goc profile -o a.cov`
2. After service restarted and test finished, collect coverage again with `goc profile -o b.cov`
3. Merge two coverage profiles together: `goc merge a.cov b.cov -o merge.cov`
## RoadMap
- [x] Support code coverage collection for system testing.
- [x] Support code coverage counters clear for the services under test at runtime.
- [x] Support develop mode towards accurate testing.
- [x] Support code coverage diff based on Pull Request.
- [ ] Optimize the performance costed by code coverage counters.
## Contributing
We welcome all kinds of contribution, including bug reports, feature requests, documentation improvements, UI refinements, etc.
Thanks to all [contributors](https://github.com/qiniu/goc/graphs/contributors)!!
## License
Goc is released under the Apache 2.0 license. See [LICENSE.txt](https://github.com/qiniu/goc/blob/master/LICENSE)
## Join goc WeChat Group
![WeChat](docs/images/wechat.png)

97
README_zh.md
View File

@ -1,97 +0,0 @@
# goc
[![Go Report Card](https://goreportcard.com/badge/github.com/qiniu/goc)](https://goreportcard.com/report/github.com/qiniu/goc)
![](https://github.com/qiniu/goc/workflows/ut-check/badge.svg)
![](https://github.com/qiniu/goc/workflows/style-check/badge.svg)
![](https://github.com/qiniu/goc/workflows/e2e%20test/badge.svg)
![Build Release](https://github.com/qiniu/goc/workflows/Build%20Release/badge.svg)
[![codecov](https://codecov.io/gh/qiniu/goc/branch/master/graph/badge.svg)](https://codecov.io/gh/qiniu/goc)
[![GoDoc](https://godoc.org/github.com/qiniu/goc?status.svg)](https://godoc.org/github.com/qiniu/goc)
goc 是专为 Go 语言打造的一个综合覆盖率收集系统，尤其适合复杂的测试场景，比如系统测试时的代码覆盖率收集以及精准测试。
希望你们喜欢～
![Demo](docs/images/intro.gif)
## 安装
最新版本在该页面下载 [Github Releases](https://github.com/qiniu/goc/releases)。
goc 同时支持 `GOPATH` 工程和 `Go Modules` 工程，且 Go 版本要求 **Go 1.11+**。如果想参与 goc 的开发，你必须使用 **Go 1.13+**。
## 例子
goc 有多种使用场景。
### 在系统测试中收集代码覆盖率
goc 可以实时收集长时运行的 golang 服务覆盖率。收集步骤只需要下面三步：
1. 运行 `goc server` 命令启动一个服务注册中心：
```
➜ simple-go-server git:(master) ✗ goc server
```
2. 运行 `goc build` 命令编译目标服务，然后启动插过桩的二进制。下面以 [simple-go-server](https://github.com/CarlJi/simple-go-server) 工程为例：
```
➜ simple-go-server git:(master) ✗ goc build .
... // omit logs
➜ simple-go-server git:(master) ✗ ./simple-go-server
```
3. 运行 `goc profile` 命令收集刚启动的 simple server 的代码覆盖率：
```
➜ simple-go-server git:(master) ✗ goc profile
mode: atomic
enricofoltran/simple-go-server/main.go:30.13,48.33 13 1
enricofoltran/simple-go-server/main.go:48.33,50.3 1 0
enricofoltran/simple-go-server/main.go:52.2,65.12 5 1
enricofoltran/simple-go-server/main.go:65.12,74.46 7 1
enricofoltran/simple-go-server/main.go:74.46,76.4 1 0
...
```
### Vscode 中实时展示覆盖率动态变化
我们提供了一个 vscode 插件 - [Goc Coverage](https://marketplace.visualstudio.com/items?itemName=lyyyuna.goc)。该插件可以在运行时高亮覆盖过的代码。
![Extension](docs/images/goc-vscode.gif)
## Tips
1. goc 命令加上 `--debug` 会打印详细的日志。我们建议在提交 bug 时附上详细日志。
2. 默认情况下，插桩过的服务会监听在一个随机的端口，注册中心会通过这个端口与服务通信。然而，对于 [docker](https://docs.docker.com/engine/reference/commandline/run/#publish-or-expose-port--p---expose) 和 [kubernetes](https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service) 容器化运行环境，对外暴露端口需在容器启动前指定。针对这种场景，你可以在 `goc build` 或 `goc install` 时使用 `--agentport` 来指定插桩过的服务监听在固定的端口。
3. 如果注册中心不在本机，你可以在 `goc build` 或 `goc install` 编译目标服务时使用 `--center` 指定远端注册中心地址。
4. 目前覆盖率数据存储在插过桩的服务测，如果某个服务中途需要重启，那么其覆盖率数据在重启后会丢失。针对这个场景，你可以通过以下步骤解决：
1. 在重启前，通过 `goc profile -o a.cov` 命令收集一次覆盖率
2. 测试结束后，通过 `goc profile -o b.cov` 命令再收集一次覆盖率
3. 通过 `goc merge a.cov b.cov -o merge.cov` 命令合并两次的覆盖率
## Blogs
- [Go语言系统测试覆盖率收集利器 goc](https://mp.weixin.qq.com/s/DzXEXwepaouSuD2dPVloOg)
- [聊聊Go代码覆盖率技术与最佳实践](https://mp.weixin.qq.com/s/SQHzsfV5T_B8fmt9NzGA7Q)
## RoadMap
- [x] 支持系统测试中收集代码覆盖率
- [x] 支持运行时对被测服务代码覆盖率计数器清零
- [x] 支持精准测试
- [x] 支持基于 Pull Request 的增量代码覆盖率报告
- [ ] 优化插桩计数器带来的性能损耗
## Contributing
我们欢迎各种形式的贡献，包括提交 bug、提新需求、优化文档和改进 UI 等等。
感谢所有的[贡献者](https://github.com/qiniu/goc/graphs/contributors)!!
## License
Goc is released under the Apache 2.0 license. See [LICENSE.txt](https://github.com/qiniu/goc/blob/master/LICENSE)
## 加入微信群聊
![WeChat](docs/images/wechat.png)

33
ci-build.sh
View File

@ -1,33 +0,0 @@
#!/bin/bash
set -eux
EVENT_DATA=$(cat $GITHUB_EVENT_PATH)
echo $EVENT_DATA | jq .
UPLOAD_URL=$(echo $EVENT_DATA | jq -r .release.upload_url)
UPLOAD_URL=${UPLOAD_URL/\{?name,label\}/}
RELEASE_VERSION=$(echo $EVENT_DATA | jq -r .release.tag_name)
PROJECT_NAME=$(basename $GITHUB_REPOSITORY)
NAME="${NAME:-${PROJECT_NAME}-${RELEASE_VERSION}}-${GOOS}-${GOARCH}"
CGO_ENABLED=0 go build -ldflags "-X 'github.com/qiniu/goc/cmd.version=${RELEASE_VERSION}'" .
ARCHIVE=tmp.tar.gz
FILE_LIST=goc
tar cvfz $ARCHIVE ${FILE_LIST}
CHECKSUM=$(md5sum ${ARCHIVE} | cut -d ' ' -f 1)
curl \
-X POST \
--data-binary @${ARCHIVE} \
-H 'Content-Type: application/octet-stream' \
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
"${UPLOAD_URL}?name=${NAME}.${ARCHIVE/tmp./}"
curl \
-X POST \
--data $CHECKSUM \
-H 'Content-Type: text/plain' \
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
"${UPLOAD_URL}?name=${NAME}_md5.txt"

96
cmd/build.go
View File

@ -1,96 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"os"
log "github.com/sirupsen/logrus"
"github.com/qiniu/goc/pkg/build"
"github.com/qiniu/goc/pkg/cover"
"github.com/spf13/cobra"
)
var buildCmd = &cobra.Command{
Use: "build",
Short: "Do cover for all go files and execute go build command",
Long: `
Build command will copy the project code and its necessary dependencies to a temporary directory, then do cover for the target, binaries will be generated to their original place.
`,
Example: `
# Build the current binary with cover variables injected. The binary will be generated in the current folder.
goc build .
# Build the current binary with cover variables injected, and set the registry center to http://127.0.0.1:7777.
goc build --center=http://127.0.0.1:7777
# Build the current binary with cover variables injected, and redirect output to /to/this/path.
goc build --output /to/this/path
# Build the current binary with cover variables injected, and set necessary build flags: -ldflags "-extldflags -static" -tags="embed kodo".
goc build --buildflags="-ldflags '-extldflags -static' -tags='embed kodo'"
`,
Run: func(cmd *cobra.Command, args []string) {
wd, err := os.Getwd()
if err != nil {
log.Fatalf("Fail to build: %v", err)
}
runBuild(args, wd)
},
}
var buildOutput string
func init() {
addBuildFlags(buildCmd.Flags())
buildCmd.Flags().StringVarP(&buildOutput, "output", "o", "", "it forces build to write the resulting executable to the named output file")
rootCmd.AddCommand(buildCmd)
}
func runBuild(args []string, wd string) {
gocBuild, err := build.NewBuild(buildFlags, args, wd, buildOutput)
if err != nil {
log.Fatalf("Fail to build: %v", err)
}
// remove temporary directory if needed
defer gocBuild.Clean()
// doCover with original buildFlags, with new GOPATH( tmp:original )
// in the tmp directory
ci := &cover.CoverInfo{
Args: buildFlags,
GoPath: gocBuild.NewGOPATH,
Target: gocBuild.TmpDir,
Mode: coverMode.String(),
AgentPort: agentPort.String(),
Center: center,
IsMod: gocBuild.IsMod,
ModRootPath: gocBuild.ModRootPath,
OneMainPackage: true, // it is a go build
GlobalCoverVarImportPath: gocBuild.GlobalCoverVarImportPath,
}
err = cover.Execute(ci)
if err != nil {
log.Fatalf("Fail to build: %v", err)
}
// do install in the temporary directory
err = gocBuild.Build()
if err != nil {
log.Fatalf("Fail to build: %v", err)
}
return
}

110
cmd/build_test.go
View File

@ -1,110 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
var baseDir string
func init() {
baseDir, _ = os.Getwd()
}
func TestGeneratedBinary(t *testing.T) {
startTime := time.Now()
workingDir := filepath.Join(baseDir, "../tests/samples/simple_project")
gopath := ""
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
buildFlags, buildOutput = "", ""
args := []string{"."}
runBuild(args, workingDir)
obj := filepath.Join(workingDir, "simple-project")
fInfo, err := os.Lstat(obj)
assert.Equal(t, err, nil, "the binary should be generated.")
assert.Equal(t, startTime.Before(fInfo.ModTime()), true, obj+"new binary should be generated, not the old one")
cmd := exec.Command("go", "tool", "objdump", "simple-project")
cmd.Dir = workingDir
out, _ := cmd.CombinedOutput()
cnt := strings.Count(string(out), "main.registerSelf")
assert.Equal(t, cnt > 0, true, "main.registerSelf function should be in the binary")
cnt = strings.Count(string(out), "GoCover")
assert.Equal(t, cnt > 0, true, "GoCover variable should be in the binary")
}
func TestBuildBinaryName(t *testing.T) {
startTime := time.Now()
workingDir := filepath.Join(baseDir, "../tests/samples/simple_project2")
gopath := ""
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
buildFlags, buildOutput = "", ""
args := []string{"."}
runBuild(args, workingDir)
obj := filepath.Join(workingDir, "simple-project")
fInfo, err := os.Lstat(obj)
assert.Equal(t, err, nil, "the binary should be generated.")
assert.Equal(t, startTime.Before(fInfo.ModTime()), true, obj+"new binary should be generated, not the old one")
cmd := exec.Command("go", "tool", "objdump", "simple-project")
cmd.Dir = workingDir
out, _ := cmd.CombinedOutput()
cnt := strings.Count(string(out), "main.registerSelf")
assert.Equal(t, cnt > 0, true, "main.registerSelf function should be in the binary")
cnt = strings.Count(string(out), "GoCover")
assert.Equal(t, cnt > 0, true, "GoCover variable should be in the binary")
}
// test if goc can get variables in internal package
func TestBuildBinaryForInternalPackage(t *testing.T) {
startTime := time.Now()
workingDir := filepath.Join(baseDir, "../tests/samples/simple_project_with_internal")
gopath := ""
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
buildFlags, buildOutput = "", ""
args := []string{"."}
runBuild(args, workingDir)
obj := filepath.Join(workingDir, "simple-project")
fInfo, err := os.Lstat(obj)
assert.Equal(t, err, nil, "the binary should be generated.")
assert.Equal(t, startTime.Before(fInfo.ModTime()), true, obj+"new binary should be generated, not the old one")
}

58
cmd/clear.go
View File

@ -1,58 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"os"
log "github.com/sirupsen/logrus"
"github.com/qiniu/goc/pkg/cover"
"github.com/spf13/cobra"
)
var clearCmd = &cobra.Command{
Use: "clear",
Short: "Clear code coverage counters of all the registered services",
Long: `Clear code coverage counters for the services under test at runtime.`,
Example: `
# Clear coverage counter from default register center http://127.0.0.1:7777.
goc clear
# Clear coverage counter from specified register center.
goc clear --center=http://192.168.1.1:8080
`,
Run: func(cmd *cobra.Command, args []string) {
p := cover.ProfileParam{
Service: svrList,
Address: addrList,
}
res, err := cover.NewWorker(center).Clear(p)
if err != nil {
log.Fatalf("call host %v failed, err: %v, response: %v", center, err, string(res))
}
fmt.Fprint(os.Stdout, string(res))
},
}
func init() {
addBasicFlags(clearCmd.Flags())
clearCmd.Flags().StringSliceVarP(&svrList, "service", "", nil, "service name to clear profile, see 'goc list' for all services.")
clearCmd.Flags().StringSliceVarP(&addrList, "address", "", nil, "address to clear profile, see 'goc list' for all addresses.")
rootCmd.AddCommand(clearCmd)
}

126
cmd/commonflags.go
View File

@ -1,126 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"net"
"github.com/spf13/pflag"
"github.com/spf13/viper"
)
var (
target string
center string
agentPort AgentPort
debugGoc bool
debugInCISyncFile string
buildFlags string
goRunExecFlag string
goRunArguments string
)
var coverMode = CoverMode{
mode: "count",
}
// addBasicFlags adds a
func addBasicFlags(cmdset *pflag.FlagSet) {
cmdset.StringVar(&center, "center", "http://127.0.0.1:7777", "cover profile host center")
// bind to viper
viper.BindPFlags(cmdset)
}
func addCommonFlags(cmdset *pflag.FlagSet) {
addBasicFlags(cmdset)
cmdset.Var(&coverMode, "mode", "coverage mode: set, count, atomic")
cmdset.Var(&agentPort, "agentport", "a fixed port such as :8100 for registered service communicate with goc server. if not provided, using a random one")
cmdset.StringVar(&buildFlags, "buildflags", "", "specify the build flags")
// bind to viper
viper.BindPFlags(cmdset)
}
func addBuildFlags(cmdset *pflag.FlagSet) {
addCommonFlags(cmdset)
// bind to viper
viper.BindPFlags(cmdset)
}
func addRunFlags(cmdset *pflag.FlagSet) {
addBuildFlags(cmdset)
cmdset.StringVar(&goRunExecFlag, "exec", "", "same as -exec flag in 'go run' command")
cmdset.StringVar(&goRunArguments, "arguments", "", "same as 'arguments' in 'go run' command")
// bind to viper
viper.BindPFlags(cmdset)
}
// CoverMode represents the covermode when doing cover for source code
type CoverMode struct {
mode string
}
func (m *CoverMode) String() string {
return m.mode
}
// Set sets the value to the CoverMode struct, use 'count' as default if v is empty
func (m *CoverMode) Set(v string) error {
if v == "" {
m.mode = "count"
return nil
}
if v != "set" && v != "count" && v != "atomic" {
return fmt.Errorf("unknown mode")
}
m.mode = v
return nil
}
// Type returns the type of CoverMode
func (m *CoverMode) Type() string {
return "string"
}
// AgentPort is the struct to do agentPort check
type AgentPort struct {
port string
}
func (agent *AgentPort) String() string {
return agent.port
}
// Set sets the value to the AgentPort struct
func (agent *AgentPort) Set(v string) error {
if v == "" {
agent.port = ""
return nil
}
_, _, err := net.SplitHostPort(v)
if err != nil {
return err
}
agent.port = v
return nil
}
// Type returns the type of AgentPort
func (agent *AgentPort) Type() string {
return "string"
}

110
cmd/commonflags_test.go
View File

@ -1,110 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"errors"
"fmt"
"testing"
"github.com/stretchr/testify/assert"
)
func TestCoverModeFlag(t *testing.T) {
var tcs = []struct {
value string
expectedValue interface{}
err interface{}
}{
{
value: "",
expectedValue: "count",
err: nil,
},
{
value: "set",
expectedValue: "set",
err: nil,
},
{
value: "count",
expectedValue: "count",
err: nil,
},
{
value: "atomic",
expectedValue: "atomic",
err: nil,
},
{
value: "xxxxx",
expectedValue: "",
err: errors.New("unknown mode"),
},
{
value: "123333",
expectedValue: "",
err: errors.New("unknown mode"),
},
}
for _, tc := range tcs {
mode := &CoverMode{}
err := mode.Set(tc.value)
actual := mode.String()
assert.Equal(t, actual, tc.expectedValue, fmt.Sprintf("check mode flag value failed, expected %s, got %s", tc.expectedValue, actual))
assert.Equal(t, err, tc.err, fmt.Sprintf("check mode flag error, expected %s, got %s", tc.err, err))
}
}
func TestAgentPortFlag(t *testing.T) {
var tcs = []struct {
value string
expectedValue interface{}
isErr bool
}{
{
value: "",
expectedValue: "",
isErr: false,
},
{
value: ":8888",
expectedValue: ":8888",
isErr: false,
},
{
value: "8888",
expectedValue: "",
isErr: true,
},
{
value: "::8888",
expectedValue: "",
isErr: true,
},
}
for _, tc := range tcs {
agent := &AgentPort{}
err := agent.Set(tc.value)
if tc.isErr {
assert.NotEqual(t, nil, err, fmt.Sprintf("check agentport flag error, expected %v, got %v", nil, err))
} else {
actual := agent.String()
assert.Equal(t, tc.expectedValue, actual, fmt.Sprintf("check agentport flag value failed, expected %s, got %s", tc.expectedValue, actual))
}
}
}

61
cmd/cover.go
View File

@ -1,61 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"github.com/qiniu/goc/pkg/cover"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
var coverCmd = &cobra.Command{
Use: "cover",
Short: "Do cover for the target source",
Long: `Do cover for the target source. You can select different cover mode (set, count, atomic), default: count`,
Example: `
# Do cover for the current path, default center: http://127.0.0.1:7777, default cover mode: count.
goc cover
# Do cover for the current path, default cover mode: count.
goc cover --center=http://127.0.0.1:7777
# Do cover for the target path, cover mode: atomic.
goc cover --center=http://127.0.0.1:7777 --target=/path/to/target --mode=atomic
`,
Hidden: true,
Run: func(cmd *cobra.Command, args []string) {
var buildFlags string
buildFlags = viper.GetString("buildflags")
ci := &cover.CoverInfo{
Args: buildFlags,
GoPath: "",
Target: target,
Mode: coverMode.String(),
AgentPort: agentPort.String(),
Center: center,
OneMainPackage: false,
}
_ = cover.Execute(ci)
},
}
func init() {
coverCmd.Flags().StringVar(&target, "target", ".", "target folder to cover")
addCommonFlags(coverCmd.Flags())
rootCmd.AddCommand(coverCmd)
}

209
cmd/diff.go
View File

@ -1,209 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"encoding/json"
"io/ioutil"
"os"
"github.com/olekukonko/tablewriter"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/qiniu/goc/pkg/cover"
"github.com/qiniu/goc/pkg/github"
"github.com/qiniu/goc/pkg/prow"
"github.com/qiniu/goc/pkg/qiniu"
)
var diffCmd = &cobra.Command{
Use: "diff",
Short: "Do coverage profile diff analysis, it can also work with prow and post comments to github pull request if needed",
Example: ` # Diff two local coverage profile and display
goc diff --new-profile=<xxxx> --base-profile=<xxxx>
# Diff local coverage profile with the remote one in prow job using default qiniu-credential
goc diff --prow-postsubmit-job=<xxx> --new-profile=<xxx>
# Calculate and display full diff coverage between new-profile and base-profile, not concerned github changed files
goc diff --prow-postsubmit-job=<xxx> --new-profile=<xxx> --full-diff=true
# Diff local coverage profile with the remote one in prow job
goc diff --prow-postsubmit-job=<xxx> --prow-remote-profile-name=<xxx>
--qiniu-credential=<xxx> --new-profile=<xxxx>
# Diff coverage profile with the remote one in prow job, and post comments to github PR
goc diff --prow-postsubmit-job=<xxx> --prow-profile=<xxx>
--github-token=<xxx> --github-user=<xxx> --github-comment-prefix=<xxx>
--qiniu-credential=<xxx> --coverage-threshold-percentage=<xxx> --new-profile=<xxxx>
`,
Run: func(cmd *cobra.Command, args []string) {
if baseProfile != "" {
doDiffForLocalProfiles(cmd, args)
} else if prowPostSubmitJob != "" {
doDiffUnderProw(cmd, args)
} else {
logrus.Fatalf("either base-profile or prow-postsubmit-job must be provided")
}
},
}
var (
newProfile string
baseProfile string
coverageThreshold int
prowPostSubmitJob string
prowProfile string
githubToken string
githubUser string
githubCommentPrefix string
qiniuCredential string
robotName string
fullDiff bool
)
func init() {
diffCmd.Flags().StringVarP(&newProfile, "new-profile", "n", "", "local profile which works as the target to analysis")
diffCmd.MarkFlagRequired("new-profile")
diffCmd.Flags().StringVarP(&baseProfile, "base-profile", "b", "", "another local profile which works as baseline to compare with the target")
diffCmd.Flags().IntVarP(&coverageThreshold, "coverage-threshold-percentage", "", 0, "coverage threshold percentage")
diffCmd.Flags().StringVarP(&prowPostSubmitJob, "prow-postsubmit-job", "", "", "prow postsubmit job which used to find the base profile")
diffCmd.Flags().StringVarP(&prowProfile, "prow-remote-profile-name", "", "filtered.cov", "the name of profile in prow postsubmit job, which used as the base profile to compare")
diffCmd.Flags().StringVarP(&githubToken, "github-token", "", "/etc/github/oauth", "path to token to access github repo")
diffCmd.Flags().StringVarP(&githubUser, "github-user", "", "", "github user name when comments in github")
diffCmd.Flags().StringVarP(&githubCommentPrefix, "github-comment-prefix", "", "", "specific comment flag you provided")
diffCmd.Flags().StringVarP(&qiniuCredential, "qiniu-credential", "", "/etc/qiniuconfig/qiniu.json", "path to credential file to access qiniu cloud")
diffCmd.Flags().StringVarP(&robotName, "robot-name", "", "qiniu-bot", "github user name for coverage robot")
diffCmd.Flags().BoolVarP(&fullDiff, "full-diff", "", false, "when set true,calculate and display full diff coverage between new-profile and base-profile")
rootCmd.AddCommand(diffCmd)
}
//goc diff --new-profile=./new.cov --base-profile=./base.cov
//+------------------------------------------------------+---------------+--------------+--------+
//| File | Base Coverage | New Coverage | Delta |
//+------------------------------------------------------+---------------+--------------+--------+
//| qiniu.com/kodo/bd/pfd/pfdstg/cursor/mgr.go | 53.5% | 50.5% | -3.0% |
//| qiniu.com/kodo/bd/pfd/pfdstg/svr/getstripe.go | 0.5% | 0.0% | -0.5% |
//| Total | 35.7% | 35.7% | -0.0% |
//+------------------------------------------------------+---------------+--------------+--------+
func doDiffForLocalProfiles(cmd *cobra.Command, args []string) {
localP, err := cover.ReadFileToCoverList(newProfile)
if err != nil {
logrus.Fatal(err)
}
baseP, err := cover.ReadFileToCoverList(baseProfile)
if err != nil {
logrus.Fatal(err)
}
//calculate diff file cov and display
rows := cover.GetDeltaCov(localP, baseP)
rows.Sort()
table := tablewriter.NewWriter(os.Stdout)
table.SetHeader([]string{"File", "Base Coverage", "New Coverage", "Delta"})
table.SetAutoFormatHeaders(false)
table.SetColumnAlignment([]int{tablewriter.ALIGN_LEFT, tablewriter.ALIGN_CENTER, tablewriter.ALIGN_CENTER, tablewriter.ALIGN_CENTER})
for _, row := range rows {
table.Append([]string{row.FileName, row.BasePer, row.NewPer, row.DeltaPer})
}
totalDelta := cover.PercentStr(cover.TotalDelta(localP, baseP))
table.Append([]string{"Total", baseP.TotalPercentage(), localP.TotalPercentage(), totalDelta})
table.Render()
}
func doDiffUnderProw(cmd *cobra.Command, args []string) {
var (
prNumStr = os.Getenv("PULL_NUMBER")
pullSha = os.Getenv("PULL_PULL_SHA")
baseSha = os.Getenv("PULL_BASE_SHA")
repoOwner = os.Getenv("REPO_OWNER")
repoName = os.Getenv("REPO_NAME")
jobType = os.Getenv("JOB_TYPE")
jobName = os.Getenv("JOB_NAME")
buildStr = os.Getenv("BUILD_NUMBER")
artifacts = os.Getenv("ARTIFACTS")
)
logrus.Printf("Running coverage for PR = %s; PR commit SHA = %s;base SHA = %s", prNumStr, pullSha, baseSha)
switch jobType {
case "periodic":
logrus.Printf("job type %s, do nothing", jobType)
case "postsubmit":
logrus.Printf("job type %s, do nothing", jobType)
case "presubmit":
if githubToken == "" {
logrus.Fatalf("github token not provided")
}
prClient := github.NewPrClient(githubToken, repoOwner, repoName, prNumStr, robotName, githubCommentPrefix)
if qiniuCredential == "" {
logrus.Fatalf("qiniu credential not provided")
}
var qc qiniu.Client
var conf qiniu.Config
files, err := ioutil.ReadFile(*&qiniuCredential)
if err != nil {
logrus.WithError(err).Fatal("Error reading qiniu config file")
}
if err := json.Unmarshal(files, &conf); err != nil {
logrus.Fatal("Error unmarshal qiniu config file")
}
if conf.Bucket == "" {
logrus.Fatal("no qiniu bucket provided")
}
if conf.AccessKey == "" || conf.SecretKey == "" {
logrus.Fatal("either qiniu access key or secret key was not provided")
}
if conf.Domain == "" {
logrus.Fatal("no qiniu bucket domain was provided")
}
qc = qiniu.NewClient(&conf)
localArtifacts := qiniu.ProfileArtifacts{
Directory: artifacts,
ProfileName: newProfile,
ChangedProfileName: qiniu.ChangedProfileName,
}
job := prow.Job{
JobName: jobName,
BuildId: buildStr,
Org: repoOwner,
RepoName: repoName,
PRNumStr: prNumStr,
PostSubmitJob: prowPostSubmitJob,
LocalProfilePath: newProfile,
PostSubmitCoverProfile: prowProfile,
QiniuClient: qc,
LocalArtifacts: &localArtifacts,
GithubComment: prClient,
FullDiff: fullDiff,
}
if err := job.RunPresubmit(); err != nil {
logrus.Fatalf("run presubmit job failed, err: %v", err)
}
default:
logrus.Printf("Unknown job type: %s, do nothing.", jobType)
}
}

106
cmd/diff_test.go
View File

@ -1,106 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"testing"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/stretchr/testify/assert"
)
type diffFunc func(cmd *cobra.Command, args []string)
func captureStdout(f diffFunc, cmd *cobra.Command, args []string) string {
r, w, err := os.Pipe()
if err != nil {
logrus.WithError(err).Fatal("os pipe fail")
}
stdout := os.Stdout
os.Stdout = w
defer func() {
os.Stdout = stdout
}()
f(cmd, args)
w.Close()
var buf bytes.Buffer
io.Copy(&buf, r)
return buf.String()
}
func TestDoDiffForLocalProfiles(t *testing.T) {
items := []struct {
newCovFile string
newProfile string
baseCovFile string
baseProfile string
expectOutput string
}{
{
newCovFile: "new.cov",
baseCovFile: "base.cov",
newProfile: "mode: atomic\n" +
"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30\n" +
"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 1\n",
baseProfile: "mode: atomic\n" +
"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30\n" +
"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0\n",
expectOutput: `+-----------------------------------------+---------------+--------------+-------+
| File | Base Coverage | New Coverage | Delta |
+-----------------------------------------+---------------+--------------+-------+
| qiniu.com/kodo/apiserver/server/main.go | 50.0% | 100.0% | 50.0% |
| Total | 50.0% | 100.0% | 50.0% |
+-----------------------------------------+---------------+--------------+-------+
`,
},
}
for _, tc := range items {
err := ioutil.WriteFile(tc.newCovFile, []byte(tc.newProfile), 0644)
if err != nil {
logrus.WithError(err).Fatalf("write file %s failed", tc.newCovFile)
}
err = ioutil.WriteFile(tc.baseCovFile, []byte(tc.baseProfile), 0644)
if err != nil {
logrus.WithError(err).Fatalf("write file %s failed", tc.baseCovFile)
}
defer func() {
os.Remove(tc.newCovFile)
os.Remove(tc.baseCovFile)
}()
pwd, err := os.Getwd()
if err != nil {
logrus.WithError(err).Fatalf("get pwd failed")
}
diffCmd.Flags().Set("new-profile", fmt.Sprintf("%s/%s", pwd, tc.newCovFile))
diffCmd.Flags().Set("base-profile", fmt.Sprintf("%s/%s", pwd, tc.baseCovFile))
out := captureStdout(doDiffForLocalProfiles, diffCmd, nil)
assert.Equal(t, out, tc.expectOutput)
}
}

39
cmd/init.go
View File

@ -1,39 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
log "github.com/sirupsen/logrus"
"github.com/qiniu/goc/pkg/cover"
"github.com/spf13/cobra"
)
var initCmd = &cobra.Command{
Use: "init",
Short: "Clear the register information in order to start a new round of tests",
Run: func(cmd *cobra.Command, args []string) {
if res, err := cover.NewWorker(center).InitSystem(); err != nil {
log.Fatalf("call host %v failed, err: %v, response: %v", center, err, string(res))
}
},
}
func init() {
addBasicFlags(initCmd.Flags())
rootCmd.AddCommand(initCmd)
}

89
cmd/install.go
View File

@ -1,89 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"os"
"github.com/qiniu/goc/pkg/build"
"github.com/qiniu/goc/pkg/cover"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var installCmd = &cobra.Command{
Use: "install",
Short: "Do cover for all go files and execute go install command",
Long: `
Install command will copy the project code and its necessary dependencies to a temporary directory, then do cover for the target, binaries will be generated to their original place.
`,
Example: `
# Install all binaries with cover variables injected. The binary will be installed in $GOPATH/bin or $HOME/go/bin if directory existed.
goc install ./...
# Install the current binary with cover variables injected, and set the registry center to http://127.0.0.1:7777.
goc install --center=http://127.0.0.1:7777
# Install the current binary with cover variables injected, and set necessary build flags: -ldflags "-extldflags -static" -tags="embed kodo".
goc build --buildflags="-ldflags '-extldflags -static' -tags='embed kodo'"
`,
Run: func(cmd *cobra.Command, args []string) {
wd, err := os.Getwd()
if err != nil {
log.Fatalf("Fail to build: %v", err)
}
runInstall(args, wd)
},
}
func init() {
addBuildFlags(installCmd.Flags())
rootCmd.AddCommand(installCmd)
}
func runInstall(args []string, wd string) {
gocBuild, err := build.NewInstall(buildFlags, args, wd)
if err != nil {
log.Fatalf("Fail to install: %v", err)
}
// remove temporary directory if needed
defer gocBuild.Clean()
// doCover with original buildFlags, with new GOPATH( tmp:original )
// in the tmp directory
ci := &cover.CoverInfo{
Args: buildFlags,
GoPath: gocBuild.NewGOPATH,
Target: gocBuild.TmpDir,
Mode: coverMode.String(),
AgentPort: agentPort.String(),
Center: center,
IsMod: gocBuild.IsMod,
ModRootPath: gocBuild.ModRootPath,
OneMainPackage: false,
GlobalCoverVarImportPath: gocBuild.GlobalCoverVarImportPath,
}
err = cover.Execute(ci)
if err != nil {
log.Fatalf("Fail to install: %v", err)
}
// do install in the temporary directory
err = gocBuild.Install()
if err != nil {
log.Fatalf("Fail to install: %v", err)
}
return
}

84
cmd/install_test.go
View File

@ -1,84 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestInstalledBinaryForMod(t *testing.T) {
startTime := time.Now()
workingDir := filepath.Join(baseDir, "../tests/samples/simple_project")
gopath := filepath.Join(baseDir, "../tests/samples/simple_project", "testhome")
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
buildFlags, buildOutput = "", ""
args := []string{"."}
runInstall(args, workingDir)
obj := filepath.Join(gopath, "bin", "simple-project")
fInfo, err := os.Lstat(obj)
assert.Equal(t, err, nil, "the binary should be generated.")
assert.Equal(t, startTime.Before(fInfo.ModTime()), true, obj+"new binary should be generated, not the old one")
cmd := exec.Command("go", "tool", "objdump", "simple-project")
cmd.Dir = workingDir
out, _ := cmd.CombinedOutput()
cnt := strings.Count(string(out), "main.registerSelf")
assert.Equal(t, cnt > 0, true, "main.registerSelf function should be in the binary")
cnt = strings.Count(string(out), "GoCover")
assert.Equal(t, cnt > 0, true, "GoCover variable should be in the binary")
}
func TestInstalledBinaryForLegacy(t *testing.T) {
startTime := time.Now()
workingDir := filepath.Join(baseDir, "../tests/samples/simple_gopath_project/src/qiniu.com/simple_gopath_project")
gopath := filepath.Join(baseDir, "../tests/samples/simple_gopath_project")
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "off")
buildFlags, buildOutput = "", ""
args := []string{"."}
runInstall(args, workingDir)
obj := filepath.Join(gopath, "bin", "simple_gopath_project")
fInfo, err := os.Lstat(obj)
assert.Equal(t, err, nil, "the binary should be generated.")
assert.Equal(t, startTime.Before(fInfo.ModTime()), true, obj+"new binary should be generated, not the old one")
cmd := exec.Command("go", "tool", "objdump", obj)
cmd.Dir = workingDir
out, _ := cmd.CombinedOutput()
cnt := strings.Count(string(out), "main.registerSelf")
assert.Equal(t, cnt > 0, true, "main.registerSelf function should be in the binary")
cnt = strings.Count(string(out), "GoCover")
assert.Equal(t, cnt > 0, true, "GoCover variable should be in the binary")
}

49
cmd/list.go
View File

@ -1,49 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"os"
log "github.com/sirupsen/logrus"
"github.com/qiniu/goc/pkg/cover"
"github.com/spf13/cobra"
)
var listCmd = &cobra.Command{
Use: "list",
Short: "Lists all the registered services",
Long: "Lists all the registered services",
Example: `
goc list [flags]
`,
Run: func(cmd *cobra.Command, args []string) {
res, err := cover.NewWorker(center).ListServices()
if err != nil {
log.Fatalf("list failed, err: %v", err)
}
log.Infoln(string(res))
fmt.Fprint(os.Stdout, string(res))
},
}
func init() {
addBasicFlags(listCmd.Flags())
rootCmd.AddCommand(listCmd)
}

76
cmd/merge.go
View File

@ -1,76 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/tools/cover"
"k8s.io/test-infra/gopherage/pkg/cov"
"k8s.io/test-infra/gopherage/pkg/util"
)
var mergeCmd = &cobra.Command{
Use: "merge [files...]",
Short: "Merge multiple coherent Go coverage files into a single file.",
Long: `merge will merge multiple Go coverage files into a single coverage file.
merge requires that the files are 'coherent', meaning that if they both contain references to the
same paths, then the contents of those source files were identical for the binary that generated
each file.
`,
Run: func(cmd *cobra.Command, args []string) {
runMerge(args, outputMergeProfile)
},
}
var outputMergeProfile string
func init() {
mergeCmd.Flags().StringVarP(&outputMergeProfile, "output", "o", "mergeprofile.cov", "output file")
rootCmd.AddCommand(mergeCmd)
}
func runMerge(args []string, output string) {
if len(args) == 0 {
log.Fatalln("Expected at least one coverage file.")
return
}
profiles := make([][]*cover.Profile, len(args))
for _, path := range args {
profile, err := util.LoadProfile(path)
if err != nil {
log.Fatalf("failed to open %s: %v", path, err)
return
}
profiles = append(profiles, profile)
}
merged, err := cov.MergeMultipleProfiles(profiles)
if err != nil {
log.Fatalf("failed to merge files: %v", err)
return
}
err = util.DumpProfile(output, merged)
if err != nil {
log.Fatalln(err)
return
}
}

156
cmd/merge_test.go
View File

@ -1,156 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
// use a variable to record if the tested function has failed
// so unittest can only be executed sequential
var fatal = false
var fatalStr string
type TestLogHook struct{}
func (h *TestLogHook) Levels() []log.Level {
return []log.Level{log.FatalLevel}
}
func (h *TestLogHook) Fire(e *log.Entry) error {
fatalStr = e.Message
return nil
}
func TestMain(m *testing.M) {
// setup
originalExitFunc := log.StandardLogger().ExitFunc
defer func() {
log.StandardLogger().ExitFunc = originalExitFunc
log.StandardLogger().Hooks = make(log.LevelHooks)
}()
// replace exit function, so log.Fatal wont exit
log.StandardLogger().ExitFunc = func(int) { fatal = true }
// add hook, so fatal string will be recorded
log.StandardLogger().Hooks.Add(&TestLogHook{})
code := m.Run()
os.Exit(code)
}
func TestMergeNormalProfiles(t *testing.T) {
profileA := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/a.voc")
profileB := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/b.voc")
mergeprofile := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/merge.cov")
runMerge([]string{profileA, profileB}, mergeprofile)
contents, err := ioutil.ReadFile(mergeprofile)
assert.NoError(t, err)
assert.Contains(t, string(contents), "qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 60")
assert.Contains(t, string(contents), "qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 2")
assert.Equal(t, fatal, false)
}
// test with no profiles
func TestMergeWithNoProfiles(t *testing.T) {
mergeprofile := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/merge.cov")
// clear fatal string in setup
fatalStr = ""
fatal = false
runMerge([]string{}, mergeprofile)
// there is fatal
assert.Equal(t, fatal, true)
assert.Equal(t, fatalStr, "Expected at least one coverage file.")
}
// pass a non-existed profile to runMerge
func TestWithWrongProfileName(t *testing.T) {
profileA := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/notexist.voc")
mergeprofile := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/merge.cov")
// clear fatal string in setup
fatalStr = ""
fatal = false
runMerge([]string{profileA}, mergeprofile)
// there is fatal
assert.Equal(t, fatal, true)
assert.Contains(t, fatalStr, "failed to open")
}
// merge two different modes' profiles should fail
func TestMergeTwoDifferentModeProfile(t *testing.T) {
profileA := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/a.voc")
profileSet := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/setmode.voc")
mergeprofile := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/merge.cov")
// clear fatal string in setup
fatalStr = ""
fatal = false
runMerge([]string{profileA, profileSet}, mergeprofile)
// there is fatal
assert.Equal(t, fatal, true)
assert.Contains(t, fatalStr, "mode for qiniu.com/kodo/apiserver/server/main.go mismatches")
}
// merge two overlaped profiles should fail
func TestMergeTwoOverLapProfile(t *testing.T) {
profileA := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/a.voc")
profileOverlap := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/overlap.voc")
mergeprofile := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/merge.cov")
// clear fatal string in setup
fatalStr = ""
fatal = false
runMerge([]string{profileA, profileOverlap}, mergeprofile)
// there is fatal
assert.Equal(t, fatal, true)
assert.Contains(t, fatalStr, "coverage block mismatch")
}
// merge empty file should fail
func TestMergeEmptyProfile(t *testing.T) {
profileA := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/empty.voc")
mergeprofile := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/merge.cov")
// clear fatal string in setup
fatalStr = ""
fatal = false
runMerge([]string{profileA}, mergeprofile)
// there is fatal
assert.Equal(t, fatal, true)
assert.Contains(t, fatalStr, "failed to dump profile")
}

112
cmd/profile.go
View File

@ -1,112 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"bytes"
"fmt"
"io"
"os"
"path"
"github.com/qiniu/goc/pkg/cover"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var profileCmd = &cobra.Command{
Use: "profile",
Short: "Get coverage profile from service registry center",
Long: `Get code coverage profile for the services under test at runtime.`,
Example: `
# Get coverage counter from default register center http://127.0.0.1:7777, the result output to stdout.
goc profile
# Get coverage counter from specified register center, the result output to specified file.
goc profile --center=http://192.168.1.1:8080 --output=./coverage.cov
# Get coverage counter of several specified services. You can get all available service names from command 'goc list'. Use 'service' and 'address' flag at the same time may cause ambiguity, please use them separately.
goc profile --service=service1,service2,service3
# Get coverage counter of several specified addresses. You can get all available addresses from command 'goc list'. Use 'service' and 'address' flag at the same time may cause ambiguity, please use them separately.
goc profile --address=address1,address2,address3
# Only get the coverage data of files matching the special patterns
goc profile --coverfile=pattern1,pattern2,pattern3
# Force fetching all available profiles.
goc profile --force
`,
Run: func(cmd *cobra.Command, args []string) {
p := cover.ProfileParam{
Force: force,
Service: svrList,
Address: addrList,
CoverFilePatterns: coverFilePatterns,
SkipFilePatterns: skipFilePatterns,
}
res, err := cover.NewWorker(center).Profile(p)
if err != nil {
log.Fatalf("Goc server %v return an error: %v", center, err)
}
if output == "" {
fmt.Fprint(os.Stdout, string(res))
} else {
var dir, filename string = path.Split(output)
if dir != "" {
err = os.MkdirAll(dir, os.ModePerm)
if err != nil {
log.Fatalf("failed to create directory %s, err:%v", dir, err)
}
}
if filename == "" {
output += "coverage.cov"
}
f, err := os.Create(output)
if err != nil {
log.Fatalf("failed to create file %s, err:%v", output, err)
}
defer f.Close()
_, err = io.Copy(f, bytes.NewReader(res))
if err != nil {
log.Fatalf("failed to write file: %v, err: %v", output, err)
}
}
},
}
var (
svrList []string // --service flag
addrList []string // --address flag
force bool // --force flag
output string // --output flag
coverFilePatterns []string // --coverfile flag
skipFilePatterns []string // --skipfile flag
)
func init() {
profileCmd.Flags().StringVarP(&output, "output", "o", "", "download cover profile")
profileCmd.Flags().StringSliceVarP(&svrList, "service", "", nil, "service name to fetch profile, see 'goc list' for all services.")
profileCmd.Flags().StringSliceVarP(&addrList, "address", "", nil, "address to fetch profile, see 'goc list' for all addresses.")
profileCmd.Flags().BoolVarP(&force, "force", "f", false, "force fetching all available profiles")
profileCmd.Flags().StringSliceVarP(&coverFilePatterns, "coverfile", "", nil, "only output coverage data of the files matching the patterns")
profileCmd.Flags().StringSliceVarP(&skipFilePatterns, "skipfile", "", nil, "skip the files matching the patterns when outputing coverage data")
addBasicFlags(profileCmd.Flags())
rootCmd.AddCommand(profileCmd)
}

60
cmd/register.go
View File

@ -1,60 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"log"
"os"
"github.com/qiniu/goc/pkg/cover"
"github.com/spf13/cobra"
)
var registerCmd = &cobra.Command{
Use: "register",
Short: "Register a service into service center",
Long: "Register a service into service center",
Example: `
goc register [flags]
`,
Run: func(cmd *cobra.Command, args []string) {
s := cover.ServiceUnderTest{
Name: name,
Address: address,
}
res, err := cover.NewWorker(center).RegisterService(s)
if err != nil {
log.Fatalf("register service failed, err: %v", err)
}
fmt.Fprint(os.Stdout, string(res))
},
}
var (
name string
address string
)
func init() {
registerCmd.Flags().StringVarP(&center, "center", "", "http://127.0.0.1:7777", "cover profile host center")
registerCmd.Flags().StringVarP(&name, "name", "n", "", "service name")
registerCmd.Flags().StringVarP(&address, "address", "a", "", "service address")
registerCmd.MarkFlagRequired("name")
registerCmd.MarkFlagRequired("address")
rootCmd.AddCommand(registerCmd)
}

58
cmd/remove.go
View File

@ -1,58 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"os"
log "github.com/sirupsen/logrus"
"github.com/qiniu/goc/pkg/cover"
"github.com/spf13/cobra"
)
var removeCmd = &cobra.Command{
Use: "remove",
Short: "Remove the specified service from the register center.",
Long: `Remove the specified service from the register center, after that, goc profile will not collect coverage data from this service anymore`,
Example: `
# Remove the service 'mongo' from the default register center http://127.0.0.1:7777.
goc remove --service=mongo
# Remove the service 'http://127.0.0.1:53' from the specified register center.
goc remove --address="http://127.0.0.1:53" --center=http://192.168.1.1:8080
`,
Run: func(cmd *cobra.Command, args []string) {
p := cover.ProfileParam{
Service: svrList,
Address: addrList,
}
res, err := cover.NewWorker(center).Remove(p)
if err != nil {
log.Fatalf("call host %v failed, err: %v, response: %v", center, err, string(res))
}
fmt.Fprint(os.Stdout, string(res))
},
}
func init() {
addBasicFlags(removeCmd.Flags())
removeCmd.Flags().StringSliceVarP(&svrList, "service", "", nil, "service name to clear profile, see 'goc list' for all services.")
removeCmd.Flags().StringSliceVarP(&addrList, "address", "", nil, "address to clear profile, see 'goc list' for all addresses.")
rootCmd.AddCommand(removeCmd)
}

88
cmd/root.go
View File

@ -1,88 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"os"
"path/filepath"
"runtime"
"strconv"
"time"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
var rootCmd = &cobra.Command{
Use: "goc",
Short: "goc is a comprehensive coverage testing tool for go language",
Long: `goc is a comprehensive coverage testing tool for go language.
Find more information at:
https://github.com/qiniu/goc
`,
PersistentPreRun: func(cmd *cobra.Command, args []string) {
log.SetReportCaller(true)
log.SetLevel(log.InfoLevel)
log.SetFormatter(&log.TextFormatter{
FullTimestamp: true,
CallerPrettyfier: func(f *runtime.Frame) (string, string) {
dirname, filename := filepath.Split(f.File)
lastelem := filepath.Base(dirname)
filename = filepath.Join(lastelem, filename)
line := strconv.Itoa(f.Line)
return "", "[" + filename + ":" + line + "]"
},
})
if debugGoc == false {
// we only need log in debug mode
log.SetLevel(log.FatalLevel)
log.SetFormatter(&log.TextFormatter{
DisableTimestamp: true,
CallerPrettyfier: func(f *runtime.Frame) (string, string) {
return "", ""
},
})
}
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if debugInCISyncFile != "" {
f, err := os.Create(debugInCISyncFile)
if err != nil {
log.Fatalln(err)
}
defer f.Close()
time.Sleep(5 * time.Second)
}
},
}
func init() {
rootCmd.PersistentFlags().BoolVar(&debugGoc, "debug", false, "run goc in debug mode")
rootCmd.PersistentFlags().StringVar(&debugInCISyncFile, "debugcisyncfile", "", "internal use only, no explain")
rootCmd.PersistentFlags().MarkHidden("debugcisyncfile")
viper.BindPFlags(rootCmd.PersistentFlags())
}
// Execute the goc tool
func Execute() {
if err := rootCmd.Execute(); err != nil {
log.Fatalln(err)
}
}

103
cmd/run.go
View File

@ -1,103 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"io/ioutil"
"net"
"os"
"github.com/qiniu/goc/pkg/build"
"github.com/qiniu/goc/pkg/cover"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var runCmd = &cobra.Command{
Use: "run",
Short: "Run covers and runs the named main Go package",
Long: `Run covers and runs the named main Go package,
It is exactly behave as 'go run .' in addition of some internal goc features.`,
Example: `
goc run .
goc run . [--buildflags] [--exec] [--arguments]
`,
Run: func(cmd *cobra.Command, args []string) {
wd, err := os.Getwd()
if err != nil {
log.Fatalf("Fail to build: %v", err)
}
gocBuild, err := build.NewBuild(buildFlags, args, wd, buildOutput)
if err != nil {
log.Fatalf("Fail to run: %v", err)
}
gocBuild.GoRunExecFlag = goRunExecFlag
gocBuild.GoRunArguments = goRunArguments
defer gocBuild.Clean()
server := cover.NewMemoryBasedServer() // only save services in memory
// start goc server
var l = newLocalListener()
go func() {
err = server.Route(ioutil.Discard).RunListener(l)
if err != nil {
log.Fatalf("Start goc server failed: %v", err)
}
}()
gocServer := fmt.Sprintf("http://%s", l.Addr().String())
fmt.Printf("[goc] goc server started: %s \n", gocServer)
// execute covers for the target source with original buildFlags and new GOPATH( tmp:original )
ci := &cover.CoverInfo{
Args: buildFlags,
GoPath: gocBuild.NewGOPATH,
Target: gocBuild.TmpDir,
Mode: coverMode.String(),
Center: gocServer,
AgentPort: "",
IsMod: gocBuild.IsMod,
ModRootPath: gocBuild.ModRootPath,
OneMainPackage: true, // go run is similar with go build, build only one main package
GlobalCoverVarImportPath: gocBuild.GlobalCoverVarImportPath,
}
err = cover.Execute(ci)
if err != nil {
log.Fatalf("Fail to run: %v", err)
}
if err := gocBuild.Run(); err != nil {
log.Fatalf("Fail to run: %v", err)
}
},
}
func init() {
addRunFlags(runCmd.Flags())
rootCmd.AddCommand(runCmd)
}
func newLocalListener() net.Listener {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
if l, err = net.Listen("tcp6", "[::1]:0"); err != nil {
log.Fatalf("failed to listen on a port: %v", err)
}
}
return l
}

54
cmd/server.go
View File

@ -1,54 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"github.com/qiniu/goc/pkg/cover"
"github.com/spf13/cobra"
"log"
)
var serverCmd = &cobra.Command{
Use: "server",
Short: "Start a service registry center",
Long: `Start a service registry center.`,
Example: `
# Start a service registry center, default port :7777.
goc server
# Start a service registry center with port :8080.
goc server --port=:8080
# Start a service registry center with localhost:8080.
goc server --port=localhost:8080
`,
Run: func(cmd *cobra.Command, args []string) {
server, err := cover.NewFileBasedServer(localPersistence)
if err != nil {
log.Fatalf("New file based server failed, err: %v", err)
}
server.Run(port)
},
}
var port, localPersistence string
func init() {
serverCmd.Flags().StringVarP(&port, "port", "", ":7777", "listen port to start a coverage host center")
serverCmd.Flags().StringVarP(&localPersistence, "local-persistence", "", "_svrs_address.txt", "the file to save services address information")
rootCmd.AddCommand(serverCmd)
}

51
cmd/version.go
View File

@ -1,51 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"runtime/debug"
"github.com/spf13/cobra"
)
// the version value will be injected when publishing
var version = "Unstable"
var versionCmd = &cobra.Command{
Use: "version",
Short: "Print the goc version information",
Example: `
# Print the client and server versions for the current context
goc version
`,
Run: func(cmd *cobra.Command, args []string) {
// if it is "Unstable", means user build local or with go get
if version == "Unstable" {
if info, ok := debug.ReadBuildInfo(); ok {
fmt.Println(info.Main.Version)
}
} else {
// otherwise the value is injected in CI
fmt.Println(version)
}
},
}
func init() {
rootCmd.AddCommand(versionCmd)
}

8
codecov.yaml
View File

@ -1,8 +0,0 @@
comment:
layout: "reach, diff, flags, files"
behavior: new
require_changes: false # if true: only post the comment if coverage changes
require_base: no # [yes :: must have a base report to post]
require_head: yes # [yes :: must have a head report to post]
branches: # branch names that can post comment
- "master"

BIN
docs/images/goc-vscode.gif
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 1.3 MiB

BIN
docs/images/intro.gif
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 1.1 MiB

BIN
docs/images/wechat.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 231 KiB

24
go.mod
View File

@ -1,24 +0,0 @@
module github.com/qiniu/goc
go 1.13
require (
github.com/gin-gonic/gin v1.6.3
github.com/google/go-github v17.0.0+incompatible
github.com/hashicorp/go-retryablehttp v0.6.6
github.com/julienschmidt/httprouter v1.2.0
github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/olekukonko/tablewriter v0.0.4
github.com/qiniu/api.v7/v7 v7.5.0
github.com/sirupsen/logrus v1.6.0
github.com/spf13/cobra v1.0.0
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.6.2
github.com/stretchr/testify v1.5.1
github.com/tongjingran/copy v1.4.2
golang.org/x/mod v0.3.0
golang.org/x/net v0.0.0-20200625001655-4c5254603344
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
golang.org/x/tools v0.0.0-20200730221956-1ac65761fe2c
k8s.io/test-infra v0.0.0-20200511080351-8ac9dbfab055
)

1143
go.sum
View File

File diff suppressed because it is too large Load Diff

23
goc.go
View File

@ -1,23 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import "github.com/qiniu/goc/cmd"
func main() {
cmd.Execute()
}

168
pkg/build/build.go
View File

@ -1,168 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package build
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/qiniu/goc/pkg/cover"
log "github.com/sirupsen/logrus"
)
// Build is to describe the building/installing process of a goc build/install
type Build struct {
Pkgs map[string]*cover.Package // Pkg list parsed from "go list -json ./..." command
NewGOPATH string // the new GOPATH
OriGOPATH string // the original GOPATH
WorkingDir string // the working directory
TmpDir string // the temporary directory to build the project
TmpWorkingDir string // the working directory in the temporary directory, which is corresponding to the current directory in the project directory
IsMod bool // determine whether it is a Mod project
Root string
// go 1.11, go 1.12 has no Root
// Project Root:
// 1. legacy, root == GOPATH
// 2. mod, root == go.mod Dir
ModRoot string // path for go.mod
ModRootPath string // import path for the whole project
Target string // the binary name that go build generate
// keep compatible with go commands:
// go run [build flags] [-exec xprog] package [arguments...]
// go build [-o output] [-i] [build flags] [packages]
// go install [-i] [build flags] [packages]
BuildFlags string // Build flags
Packages string // Packages that needs to build
GoRunExecFlag string // for the -exec flags in go run command
GoRunArguments string // for the '[arguments]' parameters in go run command
OneMainPackage bool // whether this build is a go build or go install? true: build, false: install
GlobalCoverVarImportPath string // Importpath for storing cover variables
GlobalCoverVarFilePath string // Importpath for storing cover variables
}
// NewBuild creates a Build struct which can build from goc temporary directory,
// and generate binary in current working directory
func NewBuild(buildflags string, args []string, workingDir string, outputDir string) (*Build, error) {
if err := checkParameters(args, workingDir); err != nil {
return nil, err
}
// buildflags = buildflags + " -o " + outputDir
b := &Build{
BuildFlags: buildflags,
Packages: strings.Join(args, " "),
WorkingDir: workingDir,
}
if false == b.validatePackageForBuild() {
log.Errorln(ErrWrongPackageTypeForBuild)
return nil, ErrWrongPackageTypeForBuild
}
if err := b.MvProjectsToTmp(); err != nil {
return nil, err
}
dir, err := b.determineOutputDir(outputDir)
b.Target = dir
if err != nil {
return nil, err
}
return b, nil
}
// Build calls 'go build' tool to do building
func (b *Build) Build() error {
log.Infoln("Go building in temp...")
// new -o will overwrite previous ones
b.BuildFlags = b.BuildFlags + " -o " + b.Target
cmd := exec.Command("/bin/bash", "-c", "go build "+b.BuildFlags+" "+b.Packages)
cmd.Dir = b.TmpWorkingDir
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if b.NewGOPATH != "" {
// Change to temp GOPATH for go install command
cmd.Env = append(os.Environ(), fmt.Sprintf("GOPATH=%v", b.NewGOPATH))
}
log.Printf("go build cmd is: %v", cmd.Args)
err := cmd.Start()
if err != nil {
return fmt.Errorf("fail to execute: %v, err: %w", cmd.Args, err)
}
if err = cmd.Wait(); err != nil {
return fmt.Errorf("fail to execute: %v, err: %w", cmd.Args, err)
}
log.Infoln("Go build exit successful.")
return nil
}
// determineOutputDir, as we only allow . as package name,
// the binary name is always same as the directory name of current directory
func (b *Build) determineOutputDir(outputDir string) (string, error) {
if b.TmpDir == "" {
return "", fmt.Errorf("can only be called after Build.MvProjectsToTmp(): %w", ErrEmptyTempWorkingDir)
}
// fix #43
if outputDir != "" {
abs, err := filepath.Abs(outputDir)
if err != nil {
return "", fmt.Errorf("Fail to transform the path: %v to absolute path: %v", outputDir, err)
}
return abs, nil
}
// fix #43
// use target name from `go list -json ./...` of the main module
targetName := ""
for _, pkg := range b.Pkgs {
if pkg.Name == "main" {
if pkg.Target != "" {
targetName = filepath.Base(pkg.Target)
} else {
targetName = filepath.Base(pkg.Dir)
}
break
}
}
return filepath.Join(b.WorkingDir, targetName), nil
}
// validatePackageForBuild only allow . as package name
func (b *Build) validatePackageForBuild() bool {
if b.Packages == "." || b.Packages == "" {
return true
}
return false
}
func checkParameters(args []string, workingDir string) error {
if len(args) > 1 {
log.Errorln(ErrTooManyArgs)
return ErrTooManyArgs
}
if workingDir == "" {
return ErrInvalidWorkingDir
}
log.Infof("Working directory: %v", workingDir)
return nil
}

101
pkg/build/build_test.go
View File

@ -1,101 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package build
import (
"errors"
"fmt"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
)
func TestInvalidPackage(t *testing.T) {
workingDir := filepath.Join(baseDir, "../../tests/samples/simple_project")
gopath := ""
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
_, err := NewBuild("", []string{"example.com/simple-project"}, workingDir, "")
if !assert.Equal(t, err, ErrWrongPackageTypeForBuild) {
assert.FailNow(t, "the package name should be invalid")
}
}
func TestBasicBuildForModProject(t *testing.T) {
workingDir := filepath.Join(baseDir, "../../tests/samples/simple_project")
gopath := ""
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
fmt.Println(workingDir)
buildFlags, args, buildOutput := "", []string{"."}, ""
gocBuild, err := NewBuild(buildFlags, args, workingDir, buildOutput)
if !assert.Equal(t, err, nil) {
assert.FailNow(t, "should create temporary directory successfully")
}
err = gocBuild.Build()
if !assert.Equal(t, err, nil) {
assert.FailNow(t, "temporary directory should build successfully")
}
}
func TestCheckParameters(t *testing.T) {
err := checkParameters([]string{"aa", "bb"}, "aa")
assert.Equal(t, err, ErrTooManyArgs, "too many arguments should failed")
err = checkParameters([]string{"aa"}, "")
assert.Equal(t, err, ErrInvalidWorkingDir, "empty working directory should failed")
}
func TestDetermineOutputDir(t *testing.T) {
b := &Build{}
_, err := b.determineOutputDir("")
assert.Equal(t, errors.Is(err, ErrEmptyTempWorkingDir), true, "called before Build.MvProjectsToTmp() should fail")
b.TmpDir = "fake"
_, err = b.determineOutputDir("xx")
assert.Equal(t, err, nil, "should return a directory")
}
func TestInvalidPackageNameForBuild(t *testing.T) {
workingDir := filepath.Join(baseDir, "../../tests/samples/simple_project")
gopath := filepath.Join(baseDir, "../../tests/samples/simple_project", "testhome")
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
buildFlags, packages := "", []string{"main.go"}
_, err := NewBuild(buildFlags, packages, workingDir, "")
if !assert.Equal(t, err, ErrWrongPackageTypeForBuild) {
assert.FailNow(t, "should not success with non . or ./... package")
}
}
// test NewBuild with wrong parameters
func TestNewBuildWithWrongParameters(t *testing.T) {
_, err := NewBuild("", []string{"a.go", "b.go"}, "cur", "cur")
assert.Equal(t, err, ErrTooManyArgs)
_, err = NewBuild("", []string{"a.go"}, "", "cur")
assert.Equal(t, err, ErrInvalidWorkingDir)
}

24
pkg/build/errors.go
View File

@ -1,24 +0,0 @@
package build
import (
"errors"
)
var (
// ErrShouldNotReached represents the logic should not be reached in normal flow
ErrShouldNotReached = errors.New("should never be reached")
// ErrGocShouldExecInProject represents goc currently not support for the project
ErrGocShouldExecInProject = errors.New("goc not support for such project directory")
// ErrWrongPackageTypeForInstall represents goc install command only support limited arguments
ErrWrongPackageTypeForInstall = errors.New("packages only support \".\" and \"./...\"")
// ErrWrongPackageTypeForBuild represents goc build command only support limited arguments
ErrWrongPackageTypeForBuild = errors.New("packages only support \".\"")
// ErrTooManyArgs represents goc CLI only support limited arguments
ErrTooManyArgs = errors.New("too many args")
// ErrInvalidWorkingDir represents the working directory is invalid
ErrInvalidWorkingDir = errors.New("the working directory is invalid")
// ErrEmptyTempWorkingDir represent the error that temporary working directory is empty
ErrEmptyTempWorkingDir = errors.New("temporary working directory is empty")
// ErrNoPlaceToInstall represents the err that no place to install the generated binary
ErrNoPlaceToInstall = errors.New("don't know where to install")
)

90
pkg/build/gomodules.go
View File

@ -1,90 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package build
import (
"io/ioutil"
"path/filepath"
log "github.com/sirupsen/logrus"
"github.com/tongjingran/copy"
"golang.org/x/mod/modfile"
)
func (b *Build) cpGoModulesProject() {
for _, v := range b.Pkgs {
if v.Name == "main" {
dst := b.TmpDir
src := v.Module.Dir
if err := copy.Copy(src, dst, copy.Options{Skip: skipCopy}); err != nil {
log.Errorf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err)
}
break
} else {
continue
}
}
}
// updateGoModFile rewrites the go.mod file in the temporary directory,
// if it has a 'replace' directive, and the directive has a relative local path
// it will be rewritten with a absolute path.
// ex.
// suppose original project is located at /path/to/aa/bb/cc, go.mod contains a directive:
// 'replace github.com/qiniu/bar => ../home/foo/bar'
// after the project is copied to temporary directory, it should be rewritten as
// 'replace github.com/qiniu/bar => /path/to/aa/bb/home/foo/bar'
func (b *Build) updateGoModFile() (updateFlag bool, newModFile []byte, err error) {
tempModfile := filepath.Join(b.TmpDir, "go.mod")
buf, err := ioutil.ReadFile(tempModfile)
if err != nil {
return
}
oriGoModFile, err := modfile.Parse(tempModfile, buf, nil)
if err != nil {
return
}
updateFlag = false
for index := range oriGoModFile.Replace {
replace := oriGoModFile.Replace[index]
oldPath := replace.Old.Path
oldVersion := replace.Old.Version
newPath := replace.New.Path
newVersion := replace.New.Version
// replace to a local filesystem does not have a version
// absolute path no need to rewrite
if newVersion == "" && !filepath.IsAbs(newPath) {
var absPath string
fullPath := filepath.Join(b.ModRoot, newPath)
absPath, _ = filepath.Abs(fullPath)
// DropReplace & AddReplace will not return error
// so no need to check the error
_ = oriGoModFile.DropReplace(oldPath, oldVersion)
_ = oriGoModFile.AddReplace(oldPath, oldVersion, absPath, newVersion)
updateFlag = true
}
}
oriGoModFile.Cleanup()
// Format will not return error, so ignore the returned error
// func (f *File) Format() ([]byte, error) {
// return Format(f.Syntax), nil
// }
newModFile, _ = oriGoModFile.Format()
return
}

103
pkg/build/gomodules_test.go
View File

@ -1,103 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package build
import (
"bytes"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"github.com/qiniu/goc/pkg/cover"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
func captureOutput(f func()) string {
var buf bytes.Buffer
log.SetOutput(&buf)
f()
log.SetOutput(os.Stderr)
return buf.String()
}
// copy in cpGoModulesProject of invalid src, dst name
func TestModProjectCopyWithUnexistedDir(t *testing.T) {
pkgs := make(map[string]*cover.Package)
pkgs["main"] = &cover.Package{
Name: "main",
Module: &cover.ModulePublic{
Dir: "not exied, ia mas duser", // not real one, should fail copy
},
}
pkgs["another"] = &cover.Package{}
b := &Build{
TmpDir: "sdfsfev2234444", // not real one, should fail copy
Pkgs: pkgs,
}
output := captureOutput(b.cpGoModulesProject)
assert.Equal(t, strings.Contains(output, "Failed to Copy"), true)
}
// test go mod file udpate
func TestUpdateModFileIfContainsReplace(t *testing.T) {
workingDir := filepath.Join(baseDir, "../../tests/samples/gomod_samples/a")
b := &Build{
TmpDir: workingDir,
ModRoot: "/aa/bb/cc",
}
// replace with relative local file path should be rewrite
updated, newmod, err := b.updateGoModFile()
assert.Equal(t, err, nil)
assert.Equal(t, updated, true)
assert.Contains(t, string(newmod), "replace github.com/qiniu/bar => /aa/bb/home/foo/bar")
// old replace should be removed
assert.NotContains(t, string(newmod), "github.com/qiniu/bar => ../home/foo/bar")
// normal replace should not be rewrite
assert.Contains(t, string(newmod), "github.com/qiniu/bar2 => github.com/baniu/bar3 v1.2.3")
}
// test wrong go mod file
func TestWithWrongGoModFile(t *testing.T) {
// go.mod not exist
workingDir := filepath.Join(baseDir, "../../tests/samples/xxxxxxxxxxxx/a")
b := &Build{
TmpDir: workingDir,
ModRoot: "/aa/bb/cc",
}
updated, _, err := b.updateGoModFile()
assert.Equal(t, errors.Is(err, os.ErrNotExist), true)
assert.Equal(t, updated, false)
// a wrong format go mod
workingDir = filepath.Join(baseDir, "../../tests/samples/gomod_samples/b")
b = &Build{
TmpDir: workingDir,
ModRoot: "/aa/bb/cc",
}
updated, _, err = b.updateGoModFile()
assert.NotEqual(t, err, nil)
assert.Equal(t, updated, false)
}

87
pkg/build/install.go
View File

@ -1,87 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package build
import (
"fmt"
"os"
"os/exec"
"strings"
log "github.com/sirupsen/logrus"
)
// NewInstall creates a Build struct which can install from goc temporary directory
func NewInstall(buildflags string, args []string, workingDir string) (*Build, error) {
if err := checkParameters(args, workingDir); err != nil {
return nil, err
}
b := &Build{
BuildFlags: buildflags,
Packages: strings.Join(args, " "),
WorkingDir: workingDir,
}
if false == b.validatePackageForInstall() {
log.Errorln(ErrWrongPackageTypeForInstall)
return nil, ErrWrongPackageTypeForInstall
}
if err := b.MvProjectsToTmp(); err != nil {
return nil, err
}
return b, nil
}
// Install use the 'go install' tool to install packages
func (b *Build) Install() error {
log.Println("Go building in temp...")
cmd := exec.Command("/bin/bash", "-c", "go install "+b.BuildFlags+" "+b.Packages)
cmd.Dir = b.TmpWorkingDir
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
whereToInstall, err := b.findWhereToInstall()
if err != nil {
// ignore the err
log.Errorf("No place to install: %v", err)
}
// Change the temp GOBIN, to force binary install to original place
cmd.Env = append(os.Environ(), fmt.Sprintf("GOBIN=%v", whereToInstall))
if b.NewGOPATH != "" {
// Change to temp GOPATH for go install command
cmd.Env = append(cmd.Env, fmt.Sprintf("GOPATH=%v", b.NewGOPATH))
}
log.Infof("go install cmd is: %v", cmd.Args)
err = cmd.Start()
if err != nil {
log.Errorf("Fail to execute: %v. The error is: %v", cmd.Args, err)
return err
}
if err = cmd.Wait(); err != nil {
log.Errorf("go install failed. The error is: %v", err)
return err
}
log.Infof("Go install successful. Binary installed in: %v", whereToInstall)
return nil
}
func (b *Build) validatePackageForInstall() bool {
if b.Packages == "." || b.Packages == "" || b.Packages == "./..." {
return true
}
return false
}

42
pkg/build/install_test.go
View File

@ -1,42 +0,0 @@
package build
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
)
func TestBasicInstallForModProject(t *testing.T) {
workingDir := filepath.Join(baseDir, "../../tests/samples/simple_project")
gopath := filepath.Join(baseDir, "../../tests/samples/simple_project", "testhome")
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
buildFlags, packages := "", []string{"."}
gocBuild, err := NewInstall(buildFlags, packages, workingDir)
if !assert.Equal(t, err, nil) {
assert.FailNow(t, "should create temporary directory successfully")
}
err = gocBuild.Install()
if !assert.Equal(t, err, nil) {
assert.FailNow(t, "temporary directory should build successfully")
}
}
func TestInvalidPackageNameForInstall(t *testing.T) {
workingDir := filepath.Join(baseDir, "../../tests/samples/simple_project")
gopath := filepath.Join(baseDir, "../../tests/samples/simple_project", "testhome")
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
buildFlags, packages := "", []string{"main.go"}
_, err := NewInstall(buildFlags, packages, workingDir)
if !assert.Equal(t, err, ErrWrongPackageTypeForInstall) {
assert.FailNow(t, "should not success with non . or ./... package")
}
}

104
pkg/build/legacy.go
View File

@ -1,104 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package build
import (
"os"
"path/filepath"
"strings"
log "github.com/sirupsen/logrus"
"github.com/qiniu/goc/pkg/cover"
"github.com/tongjingran/copy"
)
func (b *Build) cpLegacyProject() {
visited := make(map[string]bool)
for k, v := range b.Pkgs {
dst := filepath.Join(b.TmpDir, "src", k)
src := v.Dir
if _, ok := visited[src]; ok {
// Skip if already copied
continue
}
if err := copy.Copy(src, dst, copy.Options{Skip: skipCopy}); err != nil {
log.Errorf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err)
}
visited[src] = true
b.cpDepPackages(v, visited)
}
}
// only cp dependency in root(current gopath),
// skip deps in other GOPATHs
func (b *Build) cpDepPackages(pkg *cover.Package, visited map[string]bool) {
gopath := pkg.Root
for _, dep := range pkg.Deps {
src := filepath.Join(gopath, "src", dep)
// Check if copied
if _, ok := visited[src]; ok {
// Skip if already copied
continue
}
// Check if we can found in the root gopath
_, err := os.Stat(src)
if err != nil {
continue
}
dst := filepath.Join(b.TmpDir, "src", dep)
if err := copy.Copy(src, dst, copy.Options{Skip: skipCopy}); err != nil {
log.Errorf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err)
}
visited[src] = true
}
}
func (b *Build) cpNonStandardLegacy() {
for _, v := range b.Pkgs {
if v.Name == "main" {
dst := b.TmpDir
src := v.Dir
if err := copy.Copy(src, dst, copy.Options{Skip: skipCopy}); err != nil {
log.Printf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err)
}
break
}
}
}
// skipCopy skip copy .git dir and irregular files
func skipCopy(src string, info os.FileInfo) (bool, error) {
irregularModeType := os.ModeNamedPipe | os.ModeSocket | os.ModeDevice | os.ModeCharDevice | os.ModeIrregular
if strings.HasSuffix(src, "/.git") {
log.Infof("Skip .git dir [%s]", src)
return true, nil
}
if info.Mode()&irregularModeType != 0 {
log.Warnf("Skip file [%s], the file mode is [%s]", src, info.Mode().String())
return true, nil
}
return false, nil
}

133
pkg/build/legacy_test.go
View File

@ -1,133 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package build
import (
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/qiniu/goc/pkg/cover"
"github.com/stretchr/testify/assert"
)
// copy in cpLegacyProject/cpNonStandardLegacy of invalid src, dst name
func TestLegacyProjectCopyWithUnexistedDir(t *testing.T) {
pkgs := make(map[string]*cover.Package)
pkgs["main"] = &cover.Package{
Module: &cover.ModulePublic{
Dir: "not exied, ia mas duser", // not real one, should fail copy
},
Dir: "not exit, iasdfs",
Name: "main",
}
pkgs["another"] = &cover.Package{}
b := &Build{
TmpDir: "sdfsfev2234444", // not real one, should fail copy
Pkgs: pkgs,
}
output := captureOutput(b.cpLegacyProject)
assert.Equal(t, strings.Contains(output, "Failed to Copy"), true)
output = captureOutput(b.cpNonStandardLegacy)
assert.Equal(t, strings.Contains(output, "Failed to Copy"), true)
}
// copy in cpDepPackages of invalid dst name
func TestDepPackagesCopyWithInvalidDir(t *testing.T) {
gopath := filepath.Join(baseDir, "../../tests/samples/simple_gopath_project")
pkg := &cover.Package{
Module: &cover.ModulePublic{
Dir: "not exied, ia mas duser",
},
Root: gopath,
Deps: []string{"qiniu.com", "ddfee 2344234"},
}
b := &Build{
TmpDir: "/", // "/" is invalid dst in Linux, it should fail
}
output := captureOutput(func() {
visited := make(map[string]bool)
b.cpDepPackages(pkg, visited)
})
assert.Equal(t, strings.Contains(output, "Failed to Copy"), true)
}
type MockFile struct {
name string
size int64
mode os.FileMode
modTime time.Time
isDir bool
}
func (m MockFile) Name() string {
return m.name
}
func (m MockFile) Size() int64 {
return m.size
}
func (m MockFile) Mode() os.FileMode {
return m.mode
}
func (m MockFile) ModTime() time.Time {
return m.modTime
}
func (m MockFile) IsDir() bool {
return m.isDir
}
func (m MockFile) Sys() interface{} {
return nil
}
// skipCopy verify
func TestSkipCopy(t *testing.T) {
testCases := map[string]struct {
inputSrc string
inputInfo MockFile
expected bool
}{
"src with /.git suffix": {inputSrc: "/test/.git", inputInfo: MockFile{mode: 0}, expected: true},
"src with ./git suffix": {inputSrc: "/test.git", inputInfo: MockFile{mode: 0}, expected: false},
"src with /.gita suffix": {inputSrc: "/test/.gita", inputInfo: MockFile{mode: 0}, expected: false},
"src with /.git in middle": {inputSrc: "/test/.git/test", inputInfo: MockFile{mode: 0}, expected: false},
"irregular file": {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeIrregular}, expected: true},
"dir file": {inputSrc: "/test", inputInfo: MockFile{isDir: true, mode: os.ModeDir}, expected: false},
"temporary file": {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeTemporary}, expected: false},
"symlink file": {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeSymlink}, expected: false},
"device file": {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeDevice}, expected: true},
"named pipe file": {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeNamedPipe}, expected: true},
"socket file": {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeSocket}, expected: true},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
output, err := skipCopy(tc.inputSrc, tc.inputInfo)
assert.NoError(t, err)
assert.Equal(t, output, tc.expected)
})
}
}

50
pkg/build/run.go
View File

@ -1,50 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package build
import (
"fmt"
"os"
"os/exec"
log "github.com/sirupsen/logrus"
)
// Run excutes the main package in addition with the internal goc features
func (b *Build) Run() error {
cmd := exec.Command("/bin/bash", "-c", "go run "+b.BuildFlags+" "+b.GoRunExecFlag+" "+b.Packages+" "+b.GoRunArguments)
cmd.Dir = b.TmpWorkingDir
if b.NewGOPATH != "" {
// Change to temp GOPATH for go install command
cmd.Env = append(os.Environ(), fmt.Sprintf("GOPATH=%v", b.NewGOPATH))
}
log.Infof("go build cmd is: %v", cmd.Args)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Start()
if err != nil {
return fmt.Errorf("fail to execute: %v, err: %w", cmd.Args, err)
}
if err = cmd.Wait(); err != nil {
return fmt.Errorf("fail to execute: %v, err: %w", cmd.Args, err)
}
return nil
}

212
pkg/build/tmpfolder.go
View File

@ -1,212 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package build
import (
"crypto/sha256"
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/qiniu/goc/pkg/cover"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// MvProjectsToTmp moves the projects into a temporary directory
func (b *Build) MvProjectsToTmp() error {
listArgs := []string{"-json"}
if len(b.BuildFlags) != 0 {
listArgs = append(listArgs, b.BuildFlags)
}
listArgs = append(listArgs, "./...")
var err error
b.Pkgs, err = cover.ListPackages(b.WorkingDir, strings.Join(listArgs, " "), "")
if err != nil {
log.Errorln(err)
return err
}
err = b.mvProjectsToTmp()
if err != nil {
log.Errorf("Fail to move the project to temporary directory")
return err
}
b.OriGOPATH = os.Getenv("GOPATH")
if b.IsMod == true {
b.NewGOPATH = ""
} else if b.OriGOPATH == "" {
b.NewGOPATH = b.TmpDir
} else {
b.NewGOPATH = fmt.Sprintf("%v:%v", b.TmpDir, b.OriGOPATH)
}
// fix #14: unable to build project not in GOPATH in legacy mode
// this kind of project does not have a pkg.Root value
// go 1.11, 1.12 has no pkg.Root,
// so add b.IsMod == false as secondary judgement
if b.Root == "" && b.IsMod == false {
b.NewGOPATH = b.OriGOPATH
}
log.Infof("New GOPATH: %v", b.NewGOPATH)
return nil
}
func (b *Build) mvProjectsToTmp() error {
b.TmpDir = filepath.Join(os.TempDir(), tmpFolderName(b.WorkingDir))
// Delete previous tmp folder and its content
os.RemoveAll(b.TmpDir)
// Create a new tmp folder and a new importpath for storing cover variables
b.GlobalCoverVarImportPath = filepath.Join("src", tmpPackageName(b.WorkingDir))
err := os.MkdirAll(filepath.Join(b.TmpDir, b.GlobalCoverVarImportPath), os.ModePerm)
if err != nil {
return fmt.Errorf("Fail to create the temporary build directory. The err is: %v", err)
}
log.Infof("Tmp project generated in: %v", b.TmpDir)
// traverse pkg list to get project meta info
b.IsMod, b.Root, err = b.traversePkgsList()
log.Infof("mod project? %v", b.IsMod)
if errors.Is(err, ErrShouldNotReached) {
return fmt.Errorf("mvProjectsToTmp with a empty project: %w", err)
}
// we should get corresponding working directory in temporary directory
b.TmpWorkingDir, err = b.getTmpwd()
if err != nil {
return fmt.Errorf("getTmpwd failed with error: %w", err)
}
// issue #14
// if b.Root == "", then the project is non-standard project
// known cases:
// 1. a legacy project, but not in any GOPATH, will cause the b.Root == ""
if b.IsMod == false && b.Root != "" {
b.cpLegacyProject()
} else if b.IsMod == true { // go 1.11, 1.12 has no Build.Root
b.cpGoModulesProject()
updated, newGoModContent, err := b.updateGoModFile()
if err != nil {
return fmt.Errorf("fail to generate new go.mod: %v", err)
}
if updated {
log.Infoln("go.mod needs rewrite")
tmpModFile := filepath.Join(b.TmpDir, "go.mod")
err := ioutil.WriteFile(tmpModFile, newGoModContent, os.ModePerm)
if err != nil {
return fmt.Errorf("fail to update go.mod: %v", err)
}
}
} else if b.IsMod == false && b.Root == "" {
b.TmpWorkingDir = b.TmpDir
b.cpNonStandardLegacy()
}
log.Infof("New workingdir in tmp directory in: %v", b.TmpWorkingDir)
return nil
}
// tmpFolderName uses the first six characters of the input path's SHA256 checksum
// as the suffix.
func tmpFolderName(path string) string {
sum := sha256.Sum256([]byte(path))
h := fmt.Sprintf("%x", sum[:6])
return "goc-build-" + h
}
// tmpPackageName uses the first six characters of the input path's SHA256 checksum
// as the suffix.
func tmpPackageName(path string) string {
sum := sha256.Sum256([]byte(path))
h := fmt.Sprintf("%x", sum[:6])
return "gocbuild" + h
}
// traversePkgsList travse the Build.Pkgs list
// return Build.IsMod, tell if the project is a mod project
// return Build.Root:
// 1. the project root if it is a mod project,
// 2. current GOPATH if it is a legacy project,
// 3. some non-standard project, which Build.IsMod == false, Build.Root == nil
func (b *Build) traversePkgsList() (isMod bool, root string, err error) {
for _, v := range b.Pkgs {
// get root
root = v.Root
if v.Module == nil {
return
}
isMod = true
b.ModRoot = v.Module.Dir
b.ModRootPath = v.Module.Path
return
}
log.Error(ErrShouldNotReached)
err = ErrShouldNotReached
return
}
// getTmpwd get the corresponding working directory in the temporary working directory
// and store it in the Build.tmpWorkdingDir
func (b *Build) getTmpwd() (string, error) {
for _, pkg := range b.Pkgs {
var index int
var parentPath string
if b.IsMod == false {
index = strings.Index(b.WorkingDir, pkg.Root)
parentPath = pkg.Root
} else {
index = strings.Index(b.WorkingDir, pkg.Module.Dir)
parentPath = pkg.Module.Dir
}
if index == -1 {
return "", ErrGocShouldExecInProject
}
// b.TmpWorkingDir = filepath.Join(b.TmpDir, path[len(parentPath):])
return filepath.Join(b.TmpDir, b.WorkingDir[len(parentPath):]), nil
}
return "", ErrShouldNotReached
}
func (b *Build) findWhereToInstall() (string, error) {
if GOBIN := os.Getenv("GOBIN"); GOBIN != "" {
return GOBIN, nil
}
if false == b.IsMod {
if b.Root == "" {
return "", ErrNoPlaceToInstall
}
return filepath.Join(b.Root, "bin"), nil
}
if b.OriGOPATH != "" {
return filepath.Join(strings.Split(b.OriGOPATH, ":")[0], "bin"), nil
}
return filepath.Join(os.Getenv("HOME"), "go", "bin"), nil
}
// Clean clears up the temporary workspace
func (b *Build) Clean() error {
if !viper.GetBool("debug") {
return os.RemoveAll(b.TmpDir)
}
return nil
}

150
pkg/build/tmpfolder_test.go
View File

@ -1,150 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package build
import (
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
var baseDir string
func init() {
baseDir, _ = os.Getwd()
}
func TestNewDirParseInLegacyProject(t *testing.T) {
workingDir := filepath.Join(baseDir, "../../tests/samples/simple_gopath_project/src/qiniu.com/simple_gopath_project")
gopath := filepath.Join(baseDir, "../../tests/samples/simple_gopath_project")
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "off")
b, _ := NewInstall("", []string{"."}, workingDir)
if -1 == strings.Index(b.TmpWorkingDir, b.TmpDir) {
t.Fatalf("Directory parse error. newwd: %v, tmpdir: %v", b.TmpWorkingDir, b.TmpDir)
}
if -1 == strings.Index(b.NewGOPATH, ":") || -1 == strings.Index(b.NewGOPATH, b.TmpDir) {
t.Fatalf("The New GOPATH is wrong. newgopath: %v, tmpdir: %v", b.NewGOPATH, b.TmpDir)
}
b, _ = NewBuild("", []string{"."}, workingDir, "")
if -1 == strings.Index(b.TmpWorkingDir, b.TmpDir) {
t.Fatalf("Directory parse error. newwd: %v, tmpdir: %v", b.TmpWorkingDir, b.TmpDir)
}
if -1 == strings.Index(b.NewGOPATH, ":") || -1 == strings.Index(b.NewGOPATH, b.TmpDir) {
t.Fatalf("The New GOPATH is wrong. newgopath: %v, tmpdir: %v", b.NewGOPATH, b.TmpDir)
}
}
func TestNewDirParseInModProject(t *testing.T) {
workingDir := filepath.Join(baseDir, "../../tests/samples/simple_project")
gopath := ""
fmt.Println(gopath)
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
b, _ := NewInstall("", []string{"."}, workingDir)
if -1 == strings.Index(b.TmpWorkingDir, b.TmpDir) {
t.Fatalf("Directory parse error. newwd: %v, tmpdir: %v", b.TmpWorkingDir, b.TmpDir)
}
if b.NewGOPATH != "" {
t.Fatalf("The New GOPATH is wrong. newgopath: %v, tmpdir: %v", b.NewGOPATH, b.TmpDir)
}
b, _ = NewBuild("", []string{"."}, workingDir, "")
if -1 == strings.Index(b.TmpWorkingDir, b.TmpDir) {
t.Fatalf("Directory parse error. newwd: %v, tmpdir: %v", b.TmpWorkingDir, b.TmpDir)
}
if b.NewGOPATH != "" {
t.Fatalf("The New GOPATH is wrong. newgopath: %v, tmpdir: %v", b.NewGOPATH, b.TmpDir)
}
}
// Test #14
func TestLegacyProjectNotInGoPATH(t *testing.T) {
workingDir := filepath.Join(baseDir, "../../tests/samples/simple_gopath_project/src/qiniu.com/simple_gopath_project")
gopath := ""
fmt.Println(gopath)
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "off")
b, _ := NewBuild("", []string{"."}, workingDir, "")
if b.OriGOPATH != b.NewGOPATH {
t.Fatalf("New GOPATH should be same with old GOPATH, for this kind of project. New: %v, old: %v", b.NewGOPATH, b.OriGOPATH)
}
_, err := os.Stat(filepath.Join(b.TmpDir, "main.go"))
if err != nil {
t.Fatalf("There should be a main.go in temporary directory directly, the error: %v", err)
}
}
// test traversePkgsList error case
func TestTraversePkgsList(t *testing.T) {
b := &Build{
Pkgs: nil,
}
_, _, err := b.traversePkgsList()
assert.EqualError(t, err, ErrShouldNotReached.Error())
}
// test getTmpwd error case
func TestGetTmpwd(t *testing.T) {
b := &Build{
Pkgs: nil,
}
_, err := b.getTmpwd()
assert.EqualError(t, err, ErrShouldNotReached.Error())
}
// test findWhereToInstall
func TestFindWhereToInstall(t *testing.T) {
// if a legacy project without project root find
// should find no plcae to install
b := &Build{
Pkgs: nil,
IsMod: false,
Root: "",
}
_, err := b.findWhereToInstall()
assert.EqualError(t, err, ErrNoPlaceToInstall.Error())
// if $GOBIN not found
// and if $GOPATH not found
// should install to $HOME/go/bin
b = &Build{
Pkgs: nil,
IsMod: true,
OriGOPATH: "",
}
placeToInstall, err := b.findWhereToInstall()
assert.NoError(t, err)
expectedPlace := filepath.Join(os.Getenv("HOME"), "go", "bin")
assert.Equal(t, placeToInstall, expectedPlace)
}

188
pkg/cover/client.go
View File

@ -1,188 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cover
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
"strings"
log "github.com/sirupsen/logrus"
)
// Action provides methods to contact with the covered service under test
type Action interface {
Profile(param ProfileParam) ([]byte, error)
Clear(param ProfileParam) ([]byte, error)
Remove(param ProfileParam) ([]byte, error)
InitSystem() ([]byte, error)
ListServices() ([]byte, error)
RegisterService(svr ServiceUnderTest) ([]byte, error)
}
const (
//CoverInitSystemAPI prepare a new round of testing
CoverInitSystemAPI = "/v1/cover/init"
//CoverProfileAPI is provided by the covered service to get profiles
CoverProfileAPI = "/v1/cover/profile"
//CoverProfileClearAPI is provided by the covered service to clear profiles
CoverProfileClearAPI = "/v1/cover/clear"
//CoverServicesListAPI list all the registered services
CoverServicesListAPI = "/v1/cover/list"
//CoverRegisterServiceAPI register a service into service center
CoverRegisterServiceAPI = "/v1/cover/register"
//CoverServicesRemoveAPI remove one services from the service center
CoverServicesRemoveAPI = "/v1/cover/remove"
)
type client struct {
Host string
client *http.Client
}
// NewWorker creates a worker to contact with service
func NewWorker(host string) Action {
_, err := url.ParseRequestURI(host)
if err != nil {
log.Fatalf("Parse url %s failed, err: %v", host, err)
}
return &client{
Host: host,
client: http.DefaultClient,
}
}
func (c *client) RegisterService(srv ServiceUnderTest) ([]byte, error) {
if _, err := url.ParseRequestURI(srv.Address); err != nil {
return nil, err
}
if strings.TrimSpace(srv.Name) == "" {
return nil, fmt.Errorf("invalid service name")
}
u := fmt.Sprintf("%s%s?name=%s&address=%s", c.Host, CoverRegisterServiceAPI, srv.Name, srv.Address)
_, res, err := c.do("POST", u, "", nil)
return res, err
}
func (c *client) ListServices() ([]byte, error) {
u := fmt.Sprintf("%s%s", c.Host, CoverServicesListAPI)
_, services, err := c.do("GET", u, "", nil)
if err != nil && isNetworkError(err) {
_, services, err = c.do("GET", u, "", nil)
}
return services, err
}
func (c *client) Profile(param ProfileParam) ([]byte, error) {
u := fmt.Sprintf("%s%s", c.Host, CoverProfileAPI)
if len(param.Service) != 0 && len(param.Address) != 0 {
return nil, fmt.Errorf("use 'service' flag and 'address' flag at the same time may cause ambiguity, please use them separately")
}
// the json.Marshal function can return two types of errors: UnsupportedTypeError or UnsupportedValueError
// so no need to check here
body, _ := json.Marshal(param)
res, profile, err := c.do("POST", u, "application/json", bytes.NewReader(body))
if err != nil && isNetworkError(err) {
res, profile, err = c.do("POST", u, "application/json", bytes.NewReader(body))
}
if err == nil && res.StatusCode != 200 {
err = fmt.Errorf(string(profile))
}
return profile, err
}
func (c *client) Clear(param ProfileParam) ([]byte, error) {
u := fmt.Sprintf("%s%s", c.Host, CoverProfileClearAPI)
if len(param.Service) != 0 && len(param.Address) != 0 {
return nil, fmt.Errorf("use 'service' flag and 'address' flag at the same time may cause ambiguity, please use them separately")
}
// the json.Marshal function can return two types of errors: UnsupportedTypeError or UnsupportedValueError
// so no need to check here
body, _ := json.Marshal(param)
_, resp, err := c.do("POST", u, "application/json", bytes.NewReader(body))
if err != nil && isNetworkError(err) {
_, resp, err = c.do("POST", u, "application/json", bytes.NewReader(body))
}
return resp, err
}
func (c *client) Remove(param ProfileParam) ([]byte, error) {
u := fmt.Sprintf("%s%s", c.Host, CoverServicesRemoveAPI)
if len(param.Service) != 0 && len(param.Address) != 0 {
return nil, fmt.Errorf("use 'service' flag and 'address' flag at the same time may cause ambiguity, please use them separately")
}
// the json.Marshal function can return two types of errors: UnsupportedTypeError or UnsupportedValueError
// so no need to check here
body, err := json.Marshal(param)
if err != nil {
return nil, err
}
_, resp, err := c.do("POST", u, "application/json", bytes.NewReader(body))
if err != nil && isNetworkError(err) {
_, resp, err = c.do("POST", u, "application/json", bytes.NewReader(body))
}
return resp, err
}
func (c *client) InitSystem() ([]byte, error) {
u := fmt.Sprintf("%s%s", c.Host, CoverInitSystemAPI)
_, body, err := c.do("POST", u, "", nil)
return body, err
}
func (c *client) do(method, url, contentType string, body io.Reader) (*http.Response, []byte, error) {
req, err := http.NewRequest(method, url, body)
if err != nil {
return nil, nil, err
}
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
res, err := c.client.Do(req)
if err != nil {
return nil, nil, err
}
defer res.Body.Close()
responseBody, err := ioutil.ReadAll(res.Body)
if err != nil {
return res, nil, err
}
return res, responseBody, nil
}
func isNetworkError(err error) bool {
if err == io.EOF {
return true
}
_, ok := err.(net.Error)
return ok
}

257
pkg/cover/client_test.go
View File

@ -1,257 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cover
import (
"fmt"
"net/http/httptest"
"os"
"testing"
"net/http"
"github.com/stretchr/testify/assert"
)
func TestClientAction(t *testing.T) {
// mock goc server
server, err := NewFileBasedServer("_svrs_address.txt")
assert.NoError(t, err)
ts := httptest.NewServer(server.Route(os.Stdout))
defer ts.Close()
var client = NewWorker(ts.URL)
// mock profile server
profileMockResponse := []byte("mode: count\nmockService/main.go:30.13,48.33 13 1\nb/b.go:30.13,48.33 13 1")
profileSuccessMockSvr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write(profileMockResponse)
}))
defer profileSuccessMockSvr.Close()
profileErrMockSvr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte("error"))
}))
defer profileErrMockSvr.Close()
// register service into goc server
var src ServiceUnderTest
src.Name = "serviceSuccess"
src.Address = profileSuccessMockSvr.URL
res, err := client.RegisterService(src)
assert.NoError(t, err)
assert.Contains(t, string(res), "success")
// do list and check server
res, err = client.ListServices()
assert.NoError(t, err)
assert.Contains(t, string(res), src.Address)
assert.Contains(t, string(res), src.Name)
// get profile from goc server
tcs := []struct {
name string
service ServiceUnderTest
param ProfileParam
expected string
expectedErr bool
}{
{
name: "both server and address existed",
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{Force: false, Service: []string{"serviceOK"}, Address: []string{profileSuccessMockSvr.URL}},
expectedErr: true,
},
{
name: "valid test with no server flag provide",
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{},
expected: "mockService/main.go:30.13,48.33 13 1",
},
{
name: "valid test with server flag provide",
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{Service: []string{"serviceOK"}},
expected: "mockService/main.go:30.13,48.33 13 1",
},
{
name: "valid test with address flag provide",
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{Address: []string{profileSuccessMockSvr.URL}},
expected: "mockService/main.go:30.13,48.33 13 1",
},
{
name: "invalid test with invalid server flag provide",
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{Service: []string{"unknown"}},
expected: "server [unknown] not found",
expectedErr: true,
},
{
name: "invalid test with invalid profile got by server",
service: ServiceUnderTest{Name: "serviceErr", Address: profileErrMockSvr.URL},
expected: "bad mode line: error",
expectedErr: true,
},
{
name: "invalid test with disconnected server",
service: ServiceUnderTest{Name: "serviceNotExist", Address: "http://172.0.0.2:7777"},
expected: "connection refused",
expectedErr: true,
},
{
name: "invalid test with empty profile",
service: ServiceUnderTest{Name: "serviceNotExist", Address: "http://172.0.0.2:7777"},
param: ProfileParam{Force: true},
expectedErr: true,
expected: "no profiles",
},
{
name: "valid test with coverfile flag provide",
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{CoverFilePatterns: []string{"b.go$"}},
expected: "b/b.go",
},
{
name: "valid test with skipfile flag provided",
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{SkipFilePatterns: []string{"b.go$"}},
expected: "main.go",
},
{
name: "valid test with both skipfile and coverfile flags provided",
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{SkipFilePatterns: []string{"main.go"}, CoverFilePatterns: []string{".go$"}},
expected: "b.go",
},
}
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
// init server
_, err = client.InitSystem()
assert.NoError(t, err)
// register server
res, err = client.RegisterService(tc.service)
assert.NoError(t, err)
assert.Contains(t, string(res), "success")
res, err = client.Profile(tc.param)
if err != nil {
if !tc.expectedErr {
t.Errorf("unexpected err got: %v", err)
}
return
}
if tc.expectedErr {
t.Errorf("Expected an error, but got value %s", string(res))
}
assert.Regexp(t, tc.expected, string(res))
})
}
// init system and check server again
_, err = client.InitSystem()
assert.NoError(t, err)
res, err = client.ListServices()
assert.NoError(t, err)
assert.Equal(t, "{}", string(res))
}
func TestClientRegisterService(t *testing.T) {
c := &client{}
// client register with empty address
testService1 := ServiceUnderTest{
Address: "",
Name: "abc",
}
_, err := c.RegisterService(testService1)
assert.Contains(t, err.Error(), "empty url")
// client register with empty name
testService2 := ServiceUnderTest{
Address: "http://127.0.0.1:444",
Name: "",
}
_, err = c.RegisterService(testService2)
assert.EqualError(t, err, "invalid service name")
}
func TestClientListServices(t *testing.T) {
c := &client{
Host: "http://127.0.0.1:64445", // a invalid host
client: http.DefaultClient,
}
_, err := c.ListServices()
assert.Contains(t, err.Error(), "connect: connection refused")
}
func TestClientDo(t *testing.T) {
c := &client{
client: http.DefaultClient,
}
_, _, err := c.do(" ", "http://127.0.0.1:7777", "", nil) // a invalid method
assert.Contains(t, err.Error(), "invalid method")
}
func TestClientClearWithInvalidParam(t *testing.T) {
p := ProfileParam{
Service: []string{"goc"},
Address: []string{"http://127.0.0.1:777"},
}
c := &client{
client: http.DefaultClient,
}
_, err := c.Clear(p)
assert.Error(t, err)
assert.Contains(t, err.Error(), "use 'service' flag and 'address' flag at the same time may cause ambiguity, please use them separately")
}
func TestClientRemove(t *testing.T) {
// remove by invalid param
p := ProfileParam{
Service: []string{"goc"},
Address: []string{"http://127.0.0.1:777"},
}
c := &client{
client: http.DefaultClient,
}
_, err := c.Remove(p)
assert.Error(t, err)
assert.Contains(t, err.Error(), "use 'service' flag and 'address' flag at the same time may cause ambiguity, please use them separately")
// remove by valid param
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, "Hello, client")
}))
defer ts.Close()
c.Host = ts.URL
p = ProfileParam{
Address: []string{"http://127.0.0.1:777"},
}
res, err := c.Remove(p)
assert.NoError(t, err)
assert.Equal(t, string(res), "Hello, client\n")
// remove from a invalid center
c.Host = "http://127.0.0.1:11111"
_, err = c.Remove(p)
assert.Error(t, err)
}

574
pkg/cover/cover.go
View File

@ -1,574 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cover
import (
"bufio"
"bytes"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
log "github.com/sirupsen/logrus"
"github.com/qiniu/goc/pkg/cover/internal/tool"
"github.com/sirupsen/logrus"
)
var (
// ErrCoverPkgFailed represents the error that fails to inject the package
ErrCoverPkgFailed = errors.New("fail to inject code to project")
// ErrCoverListFailed represents the error that fails to list package dependencies
ErrCoverListFailed = errors.New("fail to list package dependencies")
)
// TestCover is a collection of all counters
type TestCover struct {
Mode string
AgentPort string
Center string // cover profile host center
MainPkgCover *PackageCover
DepsCover []*PackageCover
CacheCover map[string]*PackageCover
GlobalCoverVarImportPath string
}
// PackageCover holds all the generate coverage variables of a package
type PackageCover struct {
Package *Package
Vars map[string]*FileVar
}
// FileVar holds the name of the generated coverage variables targeting the named file.
type FileVar struct {
File string
Var string
}
// Package map a package output by go list
// this is subset of package struct in: https://github.com/golang/go/blob/master/src/cmd/go/internal/load/pkg.go#L58
type Package struct {
Dir string `json:"Dir"` // directory containing package sources
ImportPath string `json:"ImportPath"` // import path of package in dir
Name string `json:"Name"` // package name
Target string `json:",omitempty"` // installed target for this package (may be executable)
Root string `json:",omitempty"` // Go root, Go path dir, or module root dir containing this package
Module *ModulePublic `json:",omitempty"` // info about package's module, if any
Goroot bool `json:"Goroot,omitempty"` // is this package in the Go root?
Standard bool `json:"Standard,omitempty"` // is this package part of the standard Go library?
DepOnly bool `json:"DepOnly,omitempty"` // package is only a dependency, not explicitly listed
// Source files
GoFiles []string `json:"GoFiles,omitempty"` // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
CgoFiles []string `json:"CgoFiles,omitempty"` // .go source files that import "C"
// Dependency information
Deps []string `json:"Deps,omitempty"` // all (recursively) imported dependencies
Imports []string `json:",omitempty"` // import paths used by this package
ImportMap map[string]string `json:",omitempty"` // map from source import to ImportPath (identity entries omitted)
// Error information
Incomplete bool `json:"Incomplete,omitempty"` // this package or a dependency has an error
Error *PackageError `json:"Error,omitempty"` // error loading package
DepsErrors []*PackageError `json:"DepsErrors,omitempty"` // errors loading dependencies
}
// ModulePublic represents the package info of a module
type ModulePublic struct {
Path string `json:",omitempty"` // module path
Version string `json:",omitempty"` // module version
Versions []string `json:",omitempty"` // available module versions
Replace *ModulePublic `json:",omitempty"` // replaced by this module
Time *time.Time `json:",omitempty"` // time version was created
Update *ModulePublic `json:",omitempty"` // available update (with -u)
Main bool `json:",omitempty"` // is this the main module?
Indirect bool `json:",omitempty"` // module is only indirectly needed by main module
Dir string `json:",omitempty"` // directory holding local copy of files, if any
GoMod string `json:",omitempty"` // path to go.mod file describing module, if any
GoVersion string `json:",omitempty"` // go version used in module
Error *ModuleError `json:",omitempty"` // error loading module
}
// ModuleError represents the error loading module
type ModuleError struct {
Err string // error text
}
// PackageError is the error info for a package when list failed
type PackageError struct {
ImportStack []string // shortest path from package named on command line to this one
Pos string // position of error (if present, file:line:col)
Err string // the error itself
}
// CoverBuildInfo retreives some info from build
type CoverInfo struct {
Target string
GoPath string
IsMod bool
ModRootPath string
GlobalCoverVarImportPath string // path for the injected global cover var file
OneMainPackage bool
Args string
Mode string
AgentPort string
Center string
}
//Execute inject cover variables for all the .go files in the target folder
func Execute(coverInfo *CoverInfo) error {
target := coverInfo.Target
newGopath := coverInfo.GoPath
// oneMainPackage := coverInfo.OneMainPackage
args := coverInfo.Args
mode := coverInfo.Mode
agentPort := coverInfo.AgentPort
center := coverInfo.Center
globalCoverVarImportPath := coverInfo.GlobalCoverVarImportPath
if coverInfo.IsMod {
globalCoverVarImportPath = filepath.Join(coverInfo.ModRootPath, globalCoverVarImportPath)
} else {
globalCoverVarImportPath = filepath.Base(globalCoverVarImportPath)
}
if !isDirExist(target) {
log.Errorf("Target directory %s not exist", target)
return ErrCoverPkgFailed
}
listArgs := []string{"-json"}
if len(args) != 0 {
listArgs = append(listArgs, args)
}
listArgs = append(listArgs, "./...")
pkgs, err := ListPackages(target, strings.Join(listArgs, " "), newGopath)
if err != nil {
log.Errorf("Fail to list all packages, the error: %v", err)
return err
}
var seen = make(map[string]*PackageCover)
// var seenCache = make(map[string]*PackageCover)
allDecl := ""
for _, pkg := range pkgs {
if pkg.Name == "main" {
log.Printf("handle package: %v", pkg.ImportPath)
// inject the main package
mainCover, mainDecl := AddCounters(pkg, mode, globalCoverVarImportPath)
allDecl += mainDecl
// new a testcover for this service
tc := TestCover{
Mode: mode,
AgentPort: agentPort,
Center: center,
MainPkgCover: mainCover,
GlobalCoverVarImportPath: globalCoverVarImportPath,
}
// handle its dependency
// var internalPkgCache = make(map[string][]*PackageCover)
tc.CacheCover = make(map[string]*PackageCover)
for _, dep := range pkg.Deps {
if packageCover, ok := seen[dep]; ok {
tc.DepsCover = append(tc.DepsCover, packageCover)
continue
}
//only focus package neither standard Go library nor dependency library
if depPkg, ok := pkgs[dep]; ok {
packageCover, depDecl := AddCounters(depPkg, mode, globalCoverVarImportPath)
allDecl += depDecl
tc.DepsCover = append(tc.DepsCover, packageCover)
seen[dep] = packageCover
}
}
// inject Http Cover APIs
var httpCoverApis = fmt.Sprintf("%s/http_cover_apis_auto_generated.go", pkg.Dir)
if err := InjectCountersHandlers(tc, httpCoverApis); err != nil {
log.Errorf("failed to inject counters for package: %s, err: %v", pkg.ImportPath, err)
return ErrCoverPkgFailed
}
}
}
return injectGlobalCoverVarFile(coverInfo, allDecl)
}
// ListPackages list all packages under specific via go list command
// The argument newgopath is if you need to go list in a different GOPATH
func ListPackages(dir string, args string, newgopath string) (map[string]*Package, error) {
cmd := exec.Command("/bin/bash", "-c", "go list "+args)
log.Printf("go list cmd is: %v", cmd.Args)
cmd.Dir = dir
if newgopath != "" {
cmd.Env = append(os.Environ(), fmt.Sprintf("GOPATH=%v", newgopath))
}
var errbuf bytes.Buffer
cmd.Stderr = &errbuf
out, err := cmd.Output()
if err != nil {
log.Errorf("excute `go list -json ./...` command failed, err: %v, stdout: %v, stderr: %v", err, string(out), errbuf.String())
return nil, ErrCoverListFailed
}
log.Infof("\n%v", errbuf.String())
dec := json.NewDecoder(bytes.NewReader(out))
pkgs := make(map[string]*Package, 0)
for {
var pkg Package
if err := dec.Decode(&pkg); err != nil {
if err == io.EOF {
break
}
log.Errorf("reading go list output: %v", err)
return nil, ErrCoverListFailed
}
if pkg.Error != nil {
log.Errorf("list package %s failed with output: %v", pkg.ImportPath, pkg.Error)
return nil, ErrCoverPkgFailed
}
// for _, err := range pkg.DepsErrors {
// log.Fatalf("dependency package list failed, err: %v", err)
// }
pkgs[pkg.ImportPath] = &pkg
}
return pkgs, nil
}
// AddCounters is different from official go tool cover
// 1. only inject covervar++ into source file
// 2. no declarartions for these covervars
// 3. return the declarations as string
func AddCounters(pkg *Package, mode string, globalCoverVarImportPath string) (*PackageCover, string) {
coverVarMap := declareCoverVars(pkg)
decl := ""
for file, coverVar := range coverVarMap {
decl += "\n" + tool.Annotate(path.Join(pkg.Dir, file), mode, coverVar.Var, globalCoverVarImportPath) + "\n"
}
return &PackageCover{
Package: pkg,
Vars: coverVarMap,
}, decl
}
func isDirExist(path string) bool {
s, err := os.Stat(path)
if err != nil {
return false
}
return s.IsDir()
}
// Refer: https://github.com/golang/go/blob/master/src/cmd/go/internal/load/pkg.go#L1334:6
// hasInternalPath looks for the final "internal" path element in the given import path.
// If there isn't one, hasInternalPath returns ok=false.
// Otherwise, hasInternalPath returns ok=true and the index of the "internal".
func hasInternalPath(path string) bool {
// Three cases, depending on internal at start/end of string or not.
// The order matters: we must return the index of the final element,
// because the final one produces the most restrictive requirement
// on the importer.
switch {
case strings.HasSuffix(path, "/internal"):
return true
case strings.Contains(path, "/internal/"):
return true
case path == "internal", strings.HasPrefix(path, "internal/"):
return true
}
return false
}
func getInternalParent(path string) string {
switch {
case strings.HasSuffix(path, "/internal"):
return strings.Split(path, "/internal")[0]
case strings.Contains(path, "/internal/"):
return strings.Split(path, "/internal/")[0]
case path == "internal":
return ""
case strings.HasPrefix(path, "internal/"):
return strings.Split(path, "internal/")[0]
}
return ""
}
func buildCoverCmd(file string, coverVar *FileVar, pkg *Package, mode, newgopath string) *exec.Cmd {
// to construct: go tool cover -mode=atomic -o dest src (note: dest==src)
var newArgs = []string{"tool", "cover"}
newArgs = append(newArgs, "-mode", mode)
newArgs = append(newArgs, "-var", coverVar.Var)
longPath := path.Join(pkg.Dir, file)
newArgs = append(newArgs, "-o", longPath, longPath)
cmd := exec.Command("go", newArgs...)
if newgopath != "" {
cmd.Env = append(os.Environ(), fmt.Sprintf("GOPATH=%v", newgopath))
}
return cmd
}
// declareCoverVars attaches the required cover variables names
// to the files, to be used when annotating the files.
func declareCoverVars(p *Package) map[string]*FileVar {
coverVars := make(map[string]*FileVar)
coverIndex := 0
// We create the cover counters as new top-level variables in the package.
// We need to avoid collisions with user variables (GoCover_0 is unlikely but still)
// and more importantly with dot imports of other covered packages,
// so we append 12 hex digits from the SHA-256 of the import path.
// The point is only to avoid accidents, not to defeat users determined to
// break things.
sum := sha256.Sum256([]byte(p.ImportPath))
h := fmt.Sprintf("%x", sum[:6])
for _, file := range p.GoFiles {
// These names appear in the cmd/cover HTML interface.
var longFile = path.Join(p.ImportPath, file)
coverVars[file] = &FileVar{
File: longFile,
Var: fmt.Sprintf("GoCover_%d_%x", coverIndex, h),
}
coverIndex++
}
for _, file := range p.CgoFiles {
// These names appear in the cmd/cover HTML interface.
var longFile = path.Join(p.ImportPath, file)
coverVars[file] = &FileVar{
File: longFile,
Var: fmt.Sprintf("GoCover_%d_%x", coverIndex, h),
}
coverIndex++
}
return coverVars
}
func declareCacheVars(in *PackageCover) map[string]*FileVar {
sum := sha256.Sum256([]byte(in.Package.ImportPath))
h := fmt.Sprintf("%x", sum[:5])
vars := make(map[string]*FileVar)
coverIndex := 0
for _, v := range in.Vars {
cacheVar := fmt.Sprintf("GoCacheCover_%d_%x", coverIndex, h)
vars[cacheVar] = v
coverIndex++
}
return vars
}
func cacheInternalCover(in *PackageCover) *PackageCover {
c := &PackageCover{}
vars := declareCacheVars(in)
c.Package = in.Package
c.Vars = vars
return c
}
func addCacheCover(pkg *Package, in *PackageCover) *PackageCover {
c := &PackageCover{}
sum := sha256.Sum256([]byte(pkg.ImportPath))
h := fmt.Sprintf("%x", sum[:6])
goFile := fmt.Sprintf("cache_vars_auto_generated_%x.go", h)
p := &Package{
Dir: fmt.Sprintf("%s/cache_%x", pkg.Dir, h),
ImportPath: fmt.Sprintf("%s/cache_%x", pkg.ImportPath, h),
Name: fmt.Sprintf("cache_%x", h),
}
p.GoFiles = append(p.GoFiles, goFile)
c.Package = p
c.Vars = declareCacheVars(in)
return c
}
// CoverageList is a collection and summary over multiple file Coverage objects
type CoverageList []Coverage
// Coverage stores test coverage summary data for one file
type Coverage struct {
FileName string
NCoveredStmts int
NAllStmts int
LineCovLink string
}
type codeBlock struct {
fileName string // the file the code block is in
numStatements int // number of statements in the code block
coverageCount int // number of times the block is covered
}
// CovList converts profile to CoverageList struct
func CovList(f io.Reader) (g CoverageList, err error) {
scanner := bufio.NewScanner(f)
scanner.Scan() // discard first line
g = NewCoverageList()
for scanner.Scan() {
row := scanner.Text()
blk, err := toBlock(row)
if err != nil {
return nil, err
}
blk.addToGroupCov(&g)
}
return
}
// ReadFileToCoverList coverts profile file to CoverageList struct
func ReadFileToCoverList(path string) (g CoverageList, err error) {
f, err := ioutil.ReadFile(path)
if err != nil {
logrus.Errorf("Open file %s failed!", path)
return nil, err
}
g, err = CovList(bytes.NewReader(f))
return
}
// NewCoverageList return empty CoverageList
func NewCoverageList() CoverageList {
return CoverageList{}
}
func newCoverage(name string) *Coverage {
return &Coverage{name, 0, 0, ""}
}
// convert a line in profile file to a codeBlock struct
func toBlock(line string) (res *codeBlock, err error) {
slice := strings.Split(line, " ")
if len(slice) != 3 {
return nil, fmt.Errorf("the profile line %s is not expected", line)
}
blockName := slice[0]
nStmts, _ := strconv.Atoi(slice[1])
coverageCount, _ := strconv.Atoi(slice[2])
return &codeBlock{
fileName: blockName[:strings.Index(blockName, ":")],
numStatements: nStmts,
coverageCount: coverageCount,
}, nil
}
// add blk Coverage to file group Coverage
func (blk *codeBlock) addToGroupCov(g *CoverageList) {
if g.size() == 0 || g.lastElement().Name() != blk.fileName {
// when a new file name is processed
coverage := newCoverage(blk.fileName)
g.append(coverage)
}
cov := g.lastElement()
cov.NAllStmts += blk.numStatements
if blk.coverageCount > 0 {
cov.NCoveredStmts += blk.numStatements
}
}
func (g CoverageList) size() int {
return len(g)
}
func (g CoverageList) lastElement() *Coverage {
return &g[g.size()-1]
}
func (g *CoverageList) append(c *Coverage) {
*g = append(*g, *c)
}
// Sort sorts CoverageList with filenames
func (g CoverageList) Sort() {
sort.SliceStable(g, func(i, j int) bool {
return g[i].Name() < g[j].Name()
})
}
// TotalPercentage returns the total percentage of coverage
func (g CoverageList) TotalPercentage() string {
ratio, err := g.TotalRatio()
if err == nil {
return PercentStr(ratio)
}
return "N/A"
}
// TotalRatio returns the total ratio of covered statements
func (g CoverageList) TotalRatio() (ratio float32, err error) {
var total Coverage
for _, c := range g {
total.NCoveredStmts += c.NCoveredStmts
total.NAllStmts += c.NAllStmts
}
return total.Ratio()
}
// Map returns maps the file name to its coverage for faster retrieval
// & membership check
func (g CoverageList) Map() map[string]Coverage {
m := make(map[string]Coverage)
for _, c := range g {
m[c.Name()] = c
}
return m
}
// Name returns the file name
func (c *Coverage) Name() string {
return c.FileName
}
// Percentage returns the percentage of statements covered
func (c *Coverage) Percentage() string {
ratio, err := c.Ratio()
if err == nil {
return PercentStr(ratio)
}
return "N/A"
}
// Ratio calculates the ratio of statements in a profile
func (c *Coverage) Ratio() (ratio float32, err error) {
if c.NAllStmts == 0 {
err = fmt.Errorf("[%s] has 0 statement", c.Name())
} else {
ratio = float32(c.NCoveredStmts) / float32(c.NAllStmts)
}
return
}
// PercentStr converts a fraction number to percentage string representation
func PercentStr(f float32) string {
return fmt.Sprintf("%.1f%%", f*100)
}

388
pkg/cover/cover_test.go
View File

@ -1,388 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cover
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"reflect"
"strings"
"testing"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/tongjingran/copy"
)
func testCoverage() (c *Coverage) {
return &Coverage{FileName: "fake-coverage", NCoveredStmts: 200, NAllStmts: 300}
}
func TestCoverageRatio(t *testing.T) {
c := testCoverage()
actualRatio, _ := c.Ratio()
assert.Equal(t, float32(c.NCoveredStmts)/float32(c.NAllStmts), actualRatio)
}
func TestRatioErr(t *testing.T) {
c := &Coverage{FileName: "fake-coverage", NCoveredStmts: 200, NAllStmts: 0}
_, err := c.Ratio()
assert.NotEqual(t, err, nil)
}
func TestPercentageNA(t *testing.T) {
c := &Coverage{FileName: "fake-coverage", NCoveredStmts: 200, NAllStmts: 0}
assert.Equal(t, "N/A", c.Percentage())
}
func TestCovList(t *testing.T) {
fileName := "qiniu.com/kodo/apiserver/server/main.go"
fileName1 := "qiniu.com/kodo/apiserver/server/svr.go"
items := []struct {
profile string
expectPer []string
}{
// percentage is 100%
{
profile: "mode: atomic\n" +
fileName + ":32.49,33.13 1 30\n",
expectPer: []string{"100.0%"},
},
// percentage is 50%
{profile: "mode: atomic\n" +
fileName + ":32.49,33.13 1 30\n" +
fileName + ":42.49,43.13 1 0\n",
expectPer: []string{"50.0%"},
},
// two files
{
profile: "mode: atomic\n" +
fileName + ":32.49,33.13 1 30\n" +
fileName1 + ":42.49,43.13 1 0\n",
expectPer: []string{"100.0%", "0.0%"},
},
}
for _, tc := range items {
r := strings.NewReader(tc.profile)
c, err := CovList(r)
c.Sort()
assert.Equal(t, err, nil)
for k, v := range c {
assert.Equal(t, tc.expectPer[k], v.Percentage())
}
}
}
func TestReadFileToCoverList(t *testing.T) {
path := "unknown"
_, err := ReadFileToCoverList(path)
assert.Equal(t, err.Error(), "open unknown: no such file or directory")
}
func TestTotalPercentage(t *testing.T) {
items := []struct {
list CoverageList
expectPer string
}{
{
list: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 0}},
expectPer: "N/A",
},
{
list: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20}},
expectPer: "75.0%",
},
{
list: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20},
Coverage{FileName: "fake-coverage-1", NCoveredStmts: 10, NAllStmts: 30}},
expectPer: "50.0%",
},
}
for _, tc := range items {
assert.Equal(t, tc.expectPer, tc.list.TotalPercentage())
}
}
func TestBuildCoverCmd(t *testing.T) {
var testCases = []struct {
name string
file string
coverVar *FileVar
pkg *Package
mode string
newgopath string
expectCmd *exec.Cmd
}{
{
name: "normal",
file: "c.go",
coverVar: &FileVar{
File: "example/b/c/c.go",
Var: "GoCover_0_643131623532653536333031",
},
pkg: &Package{
Dir: "/go/src/goc/cmd/example-project/b/c",
},
mode: "count",
newgopath: "",
expectCmd: &exec.Cmd{
Path: lookCmdPath("go"),
Args: []string{"go", "tool", "cover", "-mode", "count", "-var", "GoCover_0_643131623532653536333031", "-o",
"/go/src/goc/cmd/example-project/b/c/c.go", "/go/src/goc/cmd/example-project/b/c/c.go"},
},
},
{
name: "normal with gopath",
file: "c.go",
coverVar: &FileVar{
File: "example/b/c/c.go",
Var: "GoCover_0_643131623532653536333031",
},
pkg: &Package{
Dir: "/go/src/goc/cmd/example-project/b/c",
},
mode: "set",
newgopath: "/go/src/goc",
expectCmd: &exec.Cmd{
Path: lookCmdPath("go"),
Args: []string{"go", "tool", "cover", "-mode", "set", "-var", "GoCover_0_643131623532653536333031", "-o",
"/go/src/goc/cmd/example-project/b/c/c.go", "/go/src/goc/cmd/example-project/b/c/c.go"},
Env: append(os.Environ(), fmt.Sprintf("GOPATH=%v", "/go/src/goc")),
},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
cmd := buildCoverCmd(testCase.file, testCase.coverVar, testCase.pkg, testCase.mode, testCase.newgopath)
if !reflect.DeepEqual(cmd, testCase.expectCmd) {
t.Errorf("generated incorrect commands:\nGot: %#v\nExpected:%#v", cmd, testCase.expectCmd)
}
})
}
}
func lookCmdPath(name string) string {
if filepath.Base(name) == name {
if lp, err := exec.LookPath(name); err != nil {
log.Fatalf("find exec %s err: %v", name, err)
} else {
return lp
}
}
return ""
}
func TestDeclareCoverVars(t *testing.T) {
var testCases = []struct {
name string
pkg *Package
expectCoverVar map[string]*FileVar
}{
{
name: "normal",
pkg: &Package{
Dir: "/go/src/goc/cmd/example-project/b/c",
GoFiles: []string{"c.go"},
ImportPath: "example/b/c",
},
expectCoverVar: map[string]*FileVar{
"c.go": {File: "example/b/c/c.go", Var: "GoCover_0_643131623532653536333031"},
},
},
{
name: "more go files",
pkg: &Package{
Dir: "/go/src/goc/cmd/example-project/a/b",
GoFiles: []string{"printf.go", "printf1.go"},
ImportPath: "example/a/b",
},
expectCoverVar: map[string]*FileVar{
"printf.go": {File: "example/a/b/printf.go", Var: "GoCover_0_326535623364613565313464"},
"printf1.go": {File: "example/a/b/printf1.go", Var: "GoCover_1_326535623364613565313464"},
},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
coverVar := declareCoverVars(testCase.pkg)
if !reflect.DeepEqual(coverVar, testCase.expectCoverVar) {
t.Errorf("generated incorrect cover vars:\nGot: %#v\nExpected:%#v", coverVar, testCase.expectCoverVar)
}
})
}
}
func TestGetInternalParent(t *testing.T) {
var tcs = []struct {
ImportPath string
expectedParent string
}{
{
ImportPath: "a/internal/b",
expectedParent: "a",
},
{
ImportPath: "internal/b",
expectedParent: "",
},
{
ImportPath: "a/b/internal/b",
expectedParent: "a/b",
},
{
ImportPath: "a/b/internal",
expectedParent: "a/b",
},
{
ImportPath: "a/b/internal/c",
expectedParent: "a/b",
},
{
ImportPath: "a/b/c",
expectedParent: "",
},
{
ImportPath: "",
expectedParent: "",
},
}
for _, tc := range tcs {
actual := getInternalParent(tc.ImportPath)
if actual != tc.expectedParent {
t.Errorf("getInternalParent failed for importPath %s, expected %s, got %s", tc.ImportPath, tc.expectedParent, actual)
}
}
}
func TestFindInternal(t *testing.T) {
var tcs = []struct {
ImportPath string
expectedParent bool
}{
{
ImportPath: "a/internal/b",
expectedParent: true,
},
{
ImportPath: "internal/b",
expectedParent: true,
},
{
ImportPath: "a/b/internal",
expectedParent: true,
},
{
ImportPath: "a/b/c",
expectedParent: false,
},
{
ImportPath: "internal",
expectedParent: true,
},
}
for _, tc := range tcs {
actual := hasInternalPath(tc.ImportPath)
if actual != tc.expectedParent {
t.Errorf("hasInternalPath check failed for importPath %s", tc.ImportPath)
}
}
}
func TestExecuteForSimpleModProject(t *testing.T) {
workingDir := "../../tests/samples/simple_project"
gopath := ""
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
testDir := filepath.Join(os.TempDir(), "goc-build-test")
copy.Copy(workingDir, testDir)
bi := &CoverInfo{
Args: "",
GoPath: gopath,
Target: testDir,
Mode: "count",
AgentPort: "",
Center: "http://127.0.0.1:7777",
OneMainPackage: false,
}
_ = Execute(bi)
_, err := os.Lstat(filepath.Join(testDir, "http_cover_apis_auto_generated.go"))
if !assert.Equal(t, err, nil) {
assert.FailNow(t, "should generate http_cover_apis_auto_generated.go")
}
}
func TestListPackagesForSimpleModProject(t *testing.T) {
workingDir := "../../tests/samples/simple_project"
gopath := ""
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
pkgs, _ := ListPackages(workingDir, "-json ./...", "")
if !assert.Equal(t, len(pkgs), 1) {
assert.FailNow(t, "should only have one pkg")
}
if pkg, ok := pkgs["example.com/simple-project"]; ok {
assert.Equal(t, pkg.Module.Path, "example.com/simple-project")
} else {
assert.FailNow(t, "cannot get the pkg: example.com/simple-project")
}
}
// test if goc can get variables in internal package
func TestCoverResultForInternalPackage(t *testing.T) {
workingDir := "../../tests/samples/simple_project_with_internal"
gopath := ""
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
testDir := filepath.Join(os.TempDir(), "goc-build-test")
copy.Copy(workingDir, testDir)
bi := &CoverInfo{
Target: testDir,
GoPath: gopath,
Args: "",
Mode: "count",
Center: "http://127.0.0.1:7777",
OneMainPackage: false,
AgentPort: "",
}
_ = Execute(bi)
_, err := os.Lstat(filepath.Join(testDir, "http_cover_apis_auto_generated.go"))
if !assert.Equal(t, err, nil) {
assert.FailNow(t, "should generate http_cover_apis_auto_generated.go")
}
}

136
pkg/cover/delta.go
View File

@ -1,136 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cover
import "sort"
// DeltaCov contains the info of a delta coverage
type DeltaCov struct {
FileName string
BasePer string
NewPer string
DeltaPer string
LineCovLink string
}
// DeltaCovList is the list of DeltaCov
type DeltaCovList []DeltaCov
// GetFullDeltaCov get full delta coverage between new and base profile
func GetFullDeltaCov(newList CoverageList, baseList CoverageList) (delta DeltaCovList) {
newMap := newList.Map()
baseMap := baseList.Map()
for file, n := range newMap {
b, ok := baseMap[file]
//if the file not in base profile, set None
if !ok {
delta = append(delta, DeltaCov{
FileName: file,
BasePer: "None",
NewPer: n.Percentage(),
DeltaPer: PercentStr(Delta(n, b))})
continue
}
delta = append(delta, DeltaCov{
FileName: file,
BasePer: b.Percentage(),
NewPer: n.Percentage(),
DeltaPer: PercentStr(Delta(n, b))})
}
for file, b := range baseMap {
//if the file not in new profile, set None
if n, ok := newMap[file]; !ok {
delta = append(delta, DeltaCov{
FileName: file,
BasePer: b.Percentage(),
NewPer: "None",
DeltaPer: PercentStr(Delta(n, b))})
}
}
return
}
// GetDeltaCov get two profile diff cov
func GetDeltaCov(newList CoverageList, baseList CoverageList) (delta DeltaCovList) {
d := GetFullDeltaCov(newList, baseList)
for _, v := range d {
if v.DeltaPer == "0.0%" {
continue
}
delta = append(delta, v)
}
return
}
// GetChFileDeltaCov get two profile diff cov of changed files
func GetChFileDeltaCov(newList CoverageList, baseList CoverageList, changedFiles []string) (list DeltaCovList) {
d := GetFullDeltaCov(newList, baseList)
dMap := d.Map()
for _, file := range changedFiles {
if _, ok := dMap[file]; ok {
list = append(list, dMap[file])
}
}
return
}
// Delta calculate two coverage delta
func Delta(new Coverage, base Coverage) float32 {
baseRatio, _ := base.Ratio()
newRatio, _ := new.Ratio()
return newRatio - baseRatio
}
// TotalDelta calculate two coverage delta
func TotalDelta(new CoverageList, base CoverageList) float32 {
baseRatio, _ := base.TotalRatio()
newRatio, _ := new.TotalRatio()
return newRatio - baseRatio
}
// Map returns maps the file name to its DeltaCov for faster retrieval & membership check
func (d DeltaCovList) Map() map[string]DeltaCov {
m := make(map[string]DeltaCov)
for _, c := range d {
m[c.FileName] = c
}
return m
}
// Sort sort DeltaCovList with filenames
func (d DeltaCovList) Sort() {
sort.SliceStable(d, func(i, j int) bool {
return d[i].Name() < d[j].Name()
})
}
// Name returns the file name
func (c *DeltaCov) Name() string {
return c.FileName
}
// GetLineCovLink get the LineCovLink of the DeltaCov
func (c *DeltaCov) GetLineCovLink() string {
return c.LineCovLink
}
// SetLineCovLink set LineCovLink of the DeltaCov
func (c *DeltaCov) SetLineCovLink(link string) {
c.LineCovLink = link
}

133
pkg/cover/delta_test.go
View File

@ -1,133 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cover
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestGetDeltaCov(t *testing.T) {
items := []struct {
newList CoverageList
baseList CoverageList
expectDelta DeltaCovList
rows int
}{
//coverage increase
{
newList: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20}},
baseList: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 10, NAllStmts: 20}},
expectDelta: DeltaCovList{{FileName: "fake-coverage", BasePer: "50.0%", NewPer: "75.0%", DeltaPer: "25.0%"}},
rows: 1,
},
//coverage decrease
{
newList: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20}},
baseList: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 20, NAllStmts: 20}},
expectDelta: DeltaCovList{{FileName: "fake-coverage", BasePer: "100.0%", NewPer: "75.0%", DeltaPer: "-25.0%"}},
rows: 1,
},
//diff file
{
newList: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20}},
baseList: CoverageList{Coverage{FileName: "fake-coverage-v1", NCoveredStmts: 10, NAllStmts: 20}},
expectDelta: DeltaCovList{{FileName: "fake-coverage", BasePer: "None", NewPer: "75.0%", DeltaPer: "75.0%"},
{FileName: "fake-coverage-v1", BasePer: "50.0%", NewPer: "None", DeltaPer: "-50.0%"}},
rows: 2,
},
//one file has same coverage rate
{
newList: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20}},
baseList: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20},
Coverage{FileName: "fake-coverage-v1", NCoveredStmts: 10, NAllStmts: 20}},
expectDelta: DeltaCovList{{FileName: "fake-coverage-v1", BasePer: "50.0%", NewPer: "None", DeltaPer: "-50.0%"}},
rows: 1,
},
}
for _, tc := range items {
d := GetDeltaCov(tc.newList, tc.baseList)
assert.Equal(t, tc.rows, len(d))
assert.Equal(t, tc.expectDelta, d)
}
}
func TestGetChFileDeltaCov(t *testing.T) {
items := []struct {
newList CoverageList
baseList CoverageList
changedFiles []string
expectDelta DeltaCovList
}{
{
newList: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20}},
baseList: CoverageList{Coverage{FileName: "fake-coverage-v1", NCoveredStmts: 10, NAllStmts: 20}},
changedFiles: []string{"fake-coverage"},
expectDelta: DeltaCovList{{FileName: "fake-coverage", BasePer: "None", NewPer: "75.0%", DeltaPer: "75.0%"}},
},
}
for _, tc := range items {
d := GetChFileDeltaCov(tc.newList, tc.baseList, tc.changedFiles)
assert.Equal(t, tc.expectDelta, d)
}
}
func TestMapAndSort(t *testing.T) {
items := []struct {
dList DeltaCovList
expectMap map[string]DeltaCov
expectSort DeltaCovList
}{
{
dList: DeltaCovList{DeltaCov{FileName: "b", BasePer: "10.0%", NewPer: "20.0%", DeltaPer: "10.0%"},
DeltaCov{FileName: "a", BasePer: "10.0%", NewPer: "30.0%", DeltaPer: "20.0%"},
},
expectMap: map[string]DeltaCov{
"a": {FileName: "a", BasePer: "10.0%", NewPer: "30.0%", DeltaPer: "20.0%"},
"b": {FileName: "b", BasePer: "10.0%", NewPer: "20.0%", DeltaPer: "10.0%"},
},
expectSort: DeltaCovList{DeltaCov{FileName: "a", BasePer: "10.0%", NewPer: "30.0%", DeltaPer: "20.0%"},
DeltaCov{FileName: "b", BasePer: "10.0%", NewPer: "20.0%", DeltaPer: "10.0%"},
},
},
{
dList: DeltaCovList{DeltaCov{FileName: "b", BasePer: "10.0%", NewPer: "20.0%", DeltaPer: "10.0%"},
DeltaCov{FileName: "b-1", BasePer: "10.0%", NewPer: "30.0%", DeltaPer: "20.0%"},
DeltaCov{FileName: "1-b", BasePer: "10.0%", NewPer: "40.0%", DeltaPer: "30.0%"},
},
expectMap: map[string]DeltaCov{
"1-b": {FileName: "1-b", BasePer: "10.0%", NewPer: "40.0%", DeltaPer: "30.0%"},
"b": {FileName: "b", BasePer: "10.0%", NewPer: "20.0%", DeltaPer: "10.0%"},
"b-1": {FileName: "b-1", BasePer: "10.0%", NewPer: "30.0%", DeltaPer: "20.0%"},
},
expectSort: DeltaCovList{DeltaCov{FileName: "1-b", BasePer: "10.0%", NewPer: "40.0%", DeltaPer: "30.0%"},
DeltaCov{FileName: "b", BasePer: "10.0%", NewPer: "20.0%", DeltaPer: "10.0%"},
DeltaCov{FileName: "b-1", BasePer: "10.0%", NewPer: "30.0%", DeltaPer: "20.0%"},
},
},
}
for _, tc := range items {
assert.Equal(t, tc.expectMap, tc.dList.Map())
tc.dList.Sort()
assert.Equal(t, tc.expectSort, tc.dList)
}
}

505
pkg/cover/instrument.go
View File

@ -1,505 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cover
import (
"fmt"
"os"
"path"
"path/filepath"
"text/template"
)
// InjectCountersHandlers generate a file _cover_http_apis.go besides the main.go file
func InjectCountersHandlers(tc TestCover, dest string) error {
f, err := os.Create(dest)
if err != nil {
return err
}
if err := coverMainTmpl.Execute(f, tc); err != nil {
return err
}
return nil
}
var coverMainTmpl = template.Must(template.New("coverMain").Parse(coverMain))
const coverMain = `
// Code generated by goc system. DO NOT EDIT.
package main
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net"
"net/http"
"os"
"os/signal"
"path/filepath"
"strings"
"sync/atomic"
"syscall"
"testing"
_cover {{.GlobalCoverVarImportPath | printf "%q"}}
)
func init() {
go registerHandlers()
}
func loadValues() (map[string][]uint32, map[string][]testing.CoverBlock) {
var (
coverCounters = make(map[string][]uint32)
coverBlocks = make(map[string][]testing.CoverBlock)
)
{{range $i, $pkgCover := .DepsCover}}
{{range $file, $cover := $pkgCover.Vars}}
loadFileCover(coverCounters, coverBlocks, {{printf "%q" $cover.File}}, _cover.{{$cover.Var}}.Count[:], _cover.{{$cover.Var}}.Pos[:], _cover.{{$cover.Var}}.NumStmt[:])
{{end}}
{{end}}
{{range $file, $cover := .MainPkgCover.Vars}}
loadFileCover(coverCounters, coverBlocks, {{printf "%q" $cover.File}}, _cover.{{$cover.Var}}.Count[:], _cover.{{$cover.Var}}.Pos[:], _cover.{{$cover.Var}}.NumStmt[:])
{{end}}
return coverCounters, coverBlocks
}
func loadFileCover(coverCounters map[string][]uint32, coverBlocks map[string][]testing.CoverBlock, fileName string, counter []uint32, pos []uint32, numStmts []uint16) {
if 3*len(counter) != len(pos) || len(counter) != len(numStmts) {
panic("coverage: mismatched sizes")
}
if coverCounters[fileName] != nil {
// Already registered.
return
}
coverCounters[fileName] = counter
block := make([]testing.CoverBlock, len(counter))
for i := range counter {
block[i] = testing.CoverBlock{
Line0: pos[3*i+0],
Col0: uint16(pos[3*i+2]),
Line1: pos[3*i+1],
Col1: uint16(pos[3*i+2] >> 16),
Stmts: numStmts[i],
}
}
coverBlocks[fileName] = block
}
func clearValues() {
{{range $i, $pkgCover := .DepsCover}}
{{range $file, $cover := $pkgCover.Vars}}
clearFileCover(_cover.{{$cover.Var}}.Count[:])
{{end}}
{{end}}
{{range $file, $cover := .MainPkgCover.Vars}}
clearFileCover(_cover.{{$cover.Var}}.Count[:])
{{end}}
}
func clearFileCover(counter []uint32) {
for i := range counter {
counter[i] = 0
}
}
func registerHandlers() {
ln, host, err := listen()
if err != nil {
log.Fatalf("listen failed, err:%v", err)
}
profileAddr := "http://" + host
if resp, err := registerSelf(profileAddr); err != nil {
log.Fatalf("register address %v failed, err: %v, response: %v", profileAddr, err, string(resp))
}
fn := func() {
var (
err error
profileAddrs []string
addresses []string
)
if addresses, err = getAllHosts(ln); err != nil {
log.Fatalf("get all host failed, err: %v", err)
return
}
for _, addr := range addresses {
profileAddrs = append(profileAddrs, "http://"+addr)
}
deregisterSelf(profileAddrs)
}
go watchSignal(fn)
mux := http.NewServeMux()
// Coverage reports the current code coverage as a fraction in the range [0, 1].
// If coverage is not enabled, Coverage returns 0.
mux.HandleFunc("/v1/cover/coverage", func(w http.ResponseWriter, r *http.Request) {
counters, _ := loadValues()
var n, d int64
for _, counter := range counters {
for i := range counter {
if atomic.LoadUint32(&counter[i]) > 0 {
n++
}
d++
}
}
if d == 0 {
fmt.Fprint(w, 0)
return
}
fmt.Fprintf(w, "%f", float64(n)/float64(d))
})
// coverprofile reports a coverage profile with the coverage percentage
mux.HandleFunc("/v1/cover/profile", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, "mode: {{.Mode}}\n")
counters, blocks := loadValues()
var active, total int64
var count uint32
for name, counts := range counters {
block := blocks[name]
for i := range counts {
stmts := int64(block[i].Stmts)
total += stmts
count = atomic.LoadUint32(&counts[i]) // For -mode=atomic.
if count > 0 {
active += stmts
}
_, err := fmt.Fprintf(w, "%s:%d.%d,%d.%d %d %d\n", name,
block[i].Line0, block[i].Col0,
block[i].Line1, block[i].Col1,
stmts,
count)
if err != nil {
fmt.Fprintf(w, "invalid block format, err: %v", err)
return
}
}
}
})
mux.HandleFunc("/v1/cover/clear", func(w http.ResponseWriter, r *http.Request) {
clearValues()
w.WriteHeader(http.StatusOK)
fmt.Fprintln(w, "clear call successfully")
})
log.Fatal(http.Serve(ln, mux))
}
func registerSelf(address string) ([]byte, error) {
selfName := filepath.Base(os.Args[0])
req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1/cover/register?name=%s&address=%s", {{.Center | printf "%q"}}, selfName, address), nil)
if err != nil {
log.Fatalf("http.NewRequest failed: %v", err)
return nil, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil && isNetworkError(err) {
log.Printf("[goc][WARN]error occurred:%v, try again", err)
resp, err = http.DefaultClient.Do(req)
}
if err != nil {
return nil, fmt.Errorf("failed to register into coverage center, err:%v", err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response body, err:%v", err)
}
if resp.StatusCode != 200 {
err = fmt.Errorf("failed to register into coverage center, response code %d", resp.StatusCode)
}
return body, err
}
func deregisterSelf(address []string) ([]byte, error) {
param := map[string]interface{}{
"address": address,
}
jsonBody, err := json.Marshal(param)
if err != nil {
return nil, err
}
req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1/cover/remove", {{.Center | printf "%q"}}), bytes.NewReader(jsonBody))
if err != nil {
log.Fatalf("http.NewRequest failed: %v", err)
return nil, err
}
req.Header.Set("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil && isNetworkError(err) {
log.Printf("[goc][WARN]error occurred:%v, try again", err)
resp, err = http.DefaultClient.Do(req)
}
if err != nil {
return nil, fmt.Errorf("failed to deregister into coverage center, err:%v", err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response body, err:%v", err)
}
if resp.StatusCode != 200 {
err = fmt.Errorf("failed to deregister into coverage center, response code %d", resp.StatusCode)
}
return body, err
}
type CallbackFunc func()
func watchSignal(fn CallbackFunc) {
// init signal
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
for {
si := <-c
log.Printf("get a signal %s", si.String())
switch si {
case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
fn()
os.Exit(0) // Exit successfully.
case syscall.SIGHUP:
default:
return
}
}
}
func isNetworkError(err error) bool {
if err == io.EOF {
return true
}
_, ok := err.(net.Error)
return ok
}
func listen() (ln net.Listener, host string, err error) {
agentPort := "{{.AgentPort }}"
if agentPort != "" {
if ln, err = net.Listen("tcp4", agentPort); err != nil {
return
}
if host, err = getRealHost(ln); err != nil {
return
}
} else {
// 获取上次使用的监听地址
if previousAddr := getPreviousAddr(); previousAddr != "" {
ss := strings.Split(previousAddr, ":")
// listen on all network interface
ln, err = net.Listen("tcp4", ":"+ss[len(ss)-1])
if err == nil {
host = previousAddr
return
}
}
if ln, err = net.Listen("tcp4", ":0"); err != nil {
return
}
if host, err = getRealHost(ln); err != nil {
return
}
}
go genProfileAddr(host)
return
}
func getRealHost(ln net.Listener) (host string, err error) {
adds, err := net.InterfaceAddrs()
if err != nil {
return
}
var localIPV4 string
var nonLocalIPV4 string
for _, addr := range adds {
if ipNet, ok := addr.(*net.IPNet); ok && ipNet.IP.To4() != nil {
if ipNet.IP.IsLoopback() {
localIPV4 = ipNet.IP.String()
} else {
nonLocalIPV4 = ipNet.IP.String()
}
}
}
if nonLocalIPV4 != "" {
host = fmt.Sprintf("%s:%d", nonLocalIPV4, ln.Addr().(*net.TCPAddr).Port)
} else {
host = fmt.Sprintf("%s:%d", localIPV4, ln.Addr().(*net.TCPAddr).Port)
}
return
}
func getAllHosts(ln net.Listener) (hosts []string, err error) {
adds, err := net.InterfaceAddrs()
if err != nil {
return
}
var host string
for _, addr := range adds {
if ipNet, ok := addr.(*net.IPNet); ok && ipNet.IP.To4() != nil {
host = fmt.Sprintf("%s:%d", ipNet.IP.String(), ln.Addr().(*net.TCPAddr).Port)
hosts = append(hosts, host)
}
}
return
}
func getPreviousAddr() string {
file, err := os.Open(os.Args[0] + "_profile_listen_addr")
if err != nil {
return ""
}
defer file.Close()
reader := bufio.NewReader(file)
addr, _, _ := reader.ReadLine()
return string(addr)
}
func genProfileAddr(profileAddr string) {
fn := os.Args[0] + "_profile_listen_addr"
f, err := os.OpenFile(fn, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
log.Println(err)
return
}
defer f.Close()
fmt.Fprintf(f, strings.TrimPrefix(profileAddr, "http://"))
}
`
var coverParentFileTmpl = template.Must(template.New("coverParentFileTmpl").Parse(coverParentFile))
const coverParentFile = `
// Code generated by goc system. DO NOT EDIT.
package {{.}}
`
var coverParentVarsTmpl = template.Must(template.New("coverParentVarsTmpl").Parse(coverParentVars))
const coverParentVars = `
import (
{{range $i, $pkgCover := .}}
_cover{{$i}} {{$pkgCover.Package.ImportPath | printf "%q"}}
{{end}}
)
{{range $i, $pkgCover := .}}
{{range $v, $cover := $pkgCover.Vars}}
var {{$v}} = &_cover{{$i}}.{{$cover.Var}}
{{end}}
{{end}}
`
func InjectCacheCounters(covers map[string][]*PackageCover, cache map[string]*PackageCover) []error {
var errs []error
for k, v := range covers {
if pkg, ok := cache[k]; ok {
err := checkCacheDir(pkg.Package.Dir)
if err != nil {
errs = append(errs, err)
continue
}
_, pkgName := path.Split(k)
err = injectCache(v, pkgName, fmt.Sprintf("%s/%s", pkg.Package.Dir, pkg.Package.GoFiles[0]))
if err != nil {
errs = append(errs, err)
continue
}
}
}
return errs
}
// InjectCacheCounters generate a file _cover_http_apis.go besides the main.go file
func injectCache(covers []*PackageCover, pkg, dest string) error {
f, err := os.Create(dest)
if err != nil {
return err
}
if err := coverParentFileTmpl.Execute(f, pkg); err != nil {
return err
}
if err := coverParentVarsTmpl.Execute(f, covers); err != nil {
return err
}
return nil
}
func checkCacheDir(p string) error {
_, err := os.Stat(p)
if os.IsNotExist(err) {
err := os.Mkdir(p, 0755)
if err != nil {
return err
}
}
return nil
}
func injectGlobalCoverVarFile(ci *CoverInfo, content string) error {
coverFile, err := os.Create(filepath.Join(ci.Target, ci.GlobalCoverVarImportPath, "cover.go"))
if err != nil {
return err
}
defer coverFile.Close()
packageName := "package " + filepath.Base(ci.GlobalCoverVarImportPath) + "\n\n"
_, err = coverFile.WriteString(packageName)
if err != nil {
return err
}
_, err = coverFile.WriteString(content)
return err
}

775
pkg/cover/internal/tool/cover.go
View File

@ -1,775 +0,0 @@
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package tool
import (
"bytes"
// "flag"
"fmt"
"go/ast"
"go/parser"
"go/token"
"io"
"io/ioutil"
"os"
"sort"
log "github.com/sirupsen/logrus" // QINIU
// "cmd/internal/edit"
// "cmd/internal/objabi"
)
// const usageMessage = "" +
// `Usage of 'go tool cover':
// Given a coverage profile produced by 'go test':
// go test -coverprofile=c.out
// Open a web browser displaying annotated source code:
// go tool cover -html=c.out
// Write out an HTML file instead of launching a web browser:
// go tool cover -html=c.out -o coverage.html
// Display coverage percentages to stdout for each function:
// go tool cover -func=c.out
// Finally, to generate modified source code with coverage annotations
// (what go test -cover does):
// go tool cover -mode=set -var=CoverageVariableName program.go
// `
// func usage() {
// fmt.Fprintln(os.Stderr, usageMessage)
// fmt.Fprintln(os.Stderr, "Flags:")
// flag.PrintDefaults()
// fmt.Fprintln(os.Stderr, "\n Only one of -html, -func, or -mode may be set.")
// os.Exit(2)
// }
// var (
// mode = flag.String("mode", "", "coverage mode: set, count, atomic")
// varVar = flag.String("var", "GoCover", "name of coverage variable to generate")
// output = flag.String("o", "", "file for output; default: stdout")
// htmlOut = flag.String("html", "", "generate HTML representation of coverage profile")
// funcOut = flag.String("func", "", "output coverage profile information for each function")
// )
// var profile string // The profile to read; the value of -html or -func
var counterStmt func(*File, string) string
const (
atomicPackagePath = "sync/atomic"
atomicPackageName = "_cover_atomic_"
)
// func main() {
// objabi.AddVersionFlag()
// flag.Usage = usage
// flag.Parse()
// // Usage information when no arguments.
// if flag.NFlag() == 0 && flag.NArg() == 0 {
// flag.Usage()
// }
// err := parseFlags()
// if err != nil {
// fmt.Fprintln(os.Stderr, err)
// fmt.Fprintln(os.Stderr, `For usage information, run "go tool cover -help"`)
// os.Exit(2)
// }
// // Generate coverage-annotated source.
// if *mode != "" {
// annotate(flag.Arg(0))
// return
// }
// // Output HTML or function coverage information.
// if *htmlOut != "" {
// err = htmlOutput(profile, *output)
// } else {
// err = funcOutput(profile, *output)
// }
// if err != nil {
// fmt.Fprintf(os.Stderr, "cover: %v\n", err)
// os.Exit(2)
// }
// }
// parseFlags sets the profile and counterStmt globals and performs validations.
// func parseFlags() error {
// profile = *htmlOut
// if *funcOut != "" {
// if profile != "" {
// return fmt.Errorf("too many options")
// }
// profile = *funcOut
// }
// // Must either display a profile or rewrite Go source.
// if (profile == "") == (*mode == "") {
// return fmt.Errorf("too many options")
// }
// if *varVar != "" && !token.IsIdentifier(*varVar) {
// return fmt.Errorf("-var: %q is not a valid identifier", *varVar)
// }
// if *mode != "" {
// switch *mode {
// case "set":
// counterStmt = setCounterStmt
// case "count":
// counterStmt = incCounterStmt
// case "atomic":
// counterStmt = atomicCounterStmt
// default:
// return fmt.Errorf("unknown -mode %v", *mode)
// }
// if flag.NArg() == 0 {
// return fmt.Errorf("missing source file")
// } else if flag.NArg() == 1 {
// return nil
// }
// } else if flag.NArg() == 0 {
// return nil
// }
// return fmt.Errorf("too many arguments")
// }
// Block represents the information about a basic block to be recorded in the analysis.
// Note: Our definition of basic block is based on control structures; we don't break
// apart && and ||. We could but it doesn't seem important enough to bother.
type Block struct {
startByte token.Pos
endByte token.Pos
numStmt int
}
// File is a wrapper for the state of a file used in the parser.
// The basic parse tree walker is a method of this type.
type File struct {
fset *token.FileSet
name string // Name of file.
astFile *ast.File
blocks []Block
content []byte
edit *Buffer // QINIU
varVar string // QINIU
mode string // QINIU
}
// findText finds text in the original source, starting at pos.
// It correctly skips over comments and assumes it need not
// handle quoted strings.
// It returns a byte offset within f.src.
func (f *File) findText(pos token.Pos, text string) int {
b := []byte(text)
start := f.offset(pos)
i := start
s := f.content
for i < len(s) {
if bytes.HasPrefix(s[i:], b) {
return i
}
if i+2 <= len(s) && s[i] == '/' && s[i+1] == '/' {
for i < len(s) && s[i] != '\n' {
i++
}
continue
}
if i+2 <= len(s) && s[i] == '/' && s[i+1] == '*' {
for i += 2; ; i++ {
if i+2 > len(s) {
return 0
}
if s[i] == '*' && s[i+1] == '/' {
i += 2
break
}
}
continue
}
i++
}
return -1
}
// Visit implements the ast.Visitor interface.
func (f *File) Visit(node ast.Node) ast.Visitor {
switch n := node.(type) {
case *ast.BlockStmt:
// If it's a switch or select, the body is a list of case clauses; don't tag the block itself.
if len(n.List) > 0 {
switch n.List[0].(type) {
case *ast.CaseClause: // switch
for _, n := range n.List {
clause := n.(*ast.CaseClause)
f.addCounters(clause.Colon+1, clause.Colon+1, clause.End(), clause.Body, false)
}
return f
case *ast.CommClause: // select
for _, n := range n.List {
clause := n.(*ast.CommClause)
f.addCounters(clause.Colon+1, clause.Colon+1, clause.End(), clause.Body, false)
}
return f
}
}
f.addCounters(n.Lbrace, n.Lbrace+1, n.Rbrace+1, n.List, true) // +1 to step past closing brace.
case *ast.IfStmt:
if n.Init != nil {
ast.Walk(f, n.Init)
}
ast.Walk(f, n.Cond)
ast.Walk(f, n.Body)
if n.Else == nil {
return nil
}
// The elses are special, because if we have
// if x {
// } else if y {
// }
// we want to cover the "if y". To do this, we need a place to drop the counter,
// so we add a hidden block:
// if x {
// } else {
// if y {
// }
// }
elseOffset := f.findText(n.Body.End(), "else")
if elseOffset < 0 {
panic("lost else")
}
f.edit.Insert(elseOffset+4, "{")
f.edit.Insert(f.offset(n.Else.End()), "}")
// We just created a block, now walk it.
// Adjust the position of the new block to start after
// the "else". That will cause it to follow the "{"
// we inserted above.
pos := f.fset.File(n.Body.End()).Pos(elseOffset + 4)
switch stmt := n.Else.(type) {
case *ast.IfStmt:
block := &ast.BlockStmt{
Lbrace: pos,
List: []ast.Stmt{stmt},
Rbrace: stmt.End(),
}
n.Else = block
case *ast.BlockStmt:
stmt.Lbrace = pos
default:
panic("unexpected node type in if")
}
ast.Walk(f, n.Else)
return nil
case *ast.SelectStmt:
// Don't annotate an empty select - creates a syntax error.
if n.Body == nil || len(n.Body.List) == 0 {
return nil
}
case *ast.SwitchStmt:
// Don't annotate an empty switch - creates a syntax error.
if n.Body == nil || len(n.Body.List) == 0 {
if n.Init != nil {
ast.Walk(f, n.Init)
}
if n.Tag != nil {
ast.Walk(f, n.Tag)
}
return nil
}
case *ast.TypeSwitchStmt:
// Don't annotate an empty type switch - creates a syntax error.
if n.Body == nil || len(n.Body.List) == 0 {
if n.Init != nil {
ast.Walk(f, n.Init)
}
ast.Walk(f, n.Assign)
return nil
}
}
return f
}
// QINIU
// Annotate do following
// 1. add cover variables into the original file
// 2. return the cover variables declarations as plain string
// original dec: func annotate(name string) {
func Annotate(name string, mode string, varVar string, globalCoverVarImportPath string) string {
// QINIU
switch mode {
case "set":
counterStmt = setCounterStmt
case "count":
counterStmt = incCounterStmt
case "atomic":
counterStmt = atomicCounterStmt
default:
counterStmt = incCounterStmt
}
fset := token.NewFileSet()
content, err := ioutil.ReadFile(name)
if err != nil {
log.Fatalf("cover: %s: %s", name, err)
}
parsedFile, err := parser.ParseFile(fset, name, content, parser.ParseComments)
if err != nil {
log.Fatalf("cover: %s: %s", name, err)
}
file := &File{
fset: fset,
name: name,
content: content,
edit: NewBuffer(content), // QINIU
astFile: parsedFile,
varVar: varVar,
mode: mode,
}
ast.Walk(file, file.astFile)
newContent := file.edit.Bytes()
if bytes.Equal(content, newContent) {
log.Info("no cover var injected for: ", name)
} else {
// reback to the beginning
file.astFile, _ = parser.ParseFile(fset, name, content, parser.ParseComments)
file.edit = NewBuffer(newContent)
// add global cover variables import path
file.edit.Insert(file.offset(file.astFile.Name.End()),
fmt.Sprintf("; import %s %q", ".", globalCoverVarImportPath))
if mode == "atomic" {
// Add import of sync/atomic immediately after package clause.
// We do this even if there is an existing import, because the
// existing import may be shadowed at any given place we want
// to refer to it, and our name (_cover_atomic_) is less likely to
// be shadowed.
file.edit.Insert(file.offset(file.astFile.Name.End()),
fmt.Sprintf("; import %s %q", atomicPackageName, atomicPackagePath))
}
newContent = file.edit.Bytes()
}
// fd := os.Stdout
// if *output != "" {
// var err error
// fd, err = os.Create(*output)
// if err != nil {
// log.Fatalf("cover: %s", err)
// }
// }
fd, err := os.Create(name)
if err != nil {
log.Fatalf("cover: %s", err)
}
defer fd.Close()
fmt.Fprintf(fd, "//line %s:1\n", name)
_, err = fd.Write(newContent)
if err != nil {
log.Fatalf("cover: %s", err)
}
// After printing the source tree, add some declarations for the counters etc.
// We could do this by adding to the tree, but it's easier just to print the text.
// QINIU
// declarations only print to string
// we will write all declarations into a single file
declBuf := bytes.NewBufferString("")
file.addVariables(declBuf)
return declBuf.String()
}
// setCounterStmt returns the expression: __count[23] = 1.
func setCounterStmt(f *File, counter string) string {
return fmt.Sprintf("%s = 1", counter)
}
// incCounterStmt returns the expression: __count[23]++.
func incCounterStmt(f *File, counter string) string {
return fmt.Sprintf("%s++", counter)
}
// atomicCounterStmt returns the expression: atomic.AddUint32(&__count[23], 1)
func atomicCounterStmt(f *File, counter string) string {
return fmt.Sprintf("%s.AddUint32(&%s, 1)", atomicPackageName, counter)
}
// QINIU
// newCounter creates a new counter expression of the appropriate form.
func (f *File) newCounter(start, end token.Pos, numStmt int) string {
stmt := counterStmt(f, fmt.Sprintf("%s.Count[%d]", f.varVar, len(f.blocks)))
f.blocks = append(f.blocks, Block{start, end, numStmt})
return stmt
}
// addCounters takes a list of statements and adds counters to the beginning of
// each basic block at the top level of that list. For instance, given
//
// S1
// if cond {
// S2
// }
// S3
//
// counters will be added before S1 and before S3. The block containing S2
// will be visited in a separate call.
// TODO: Nested simple blocks get unnecessary (but correct) counters
func (f *File) addCounters(pos, insertPos, blockEnd token.Pos, list []ast.Stmt, extendToClosingBrace bool) {
// Special case: make sure we add a counter to an empty block. Can't do this below
// or we will add a counter to an empty statement list after, say, a return statement.
if len(list) == 0 {
f.edit.Insert(f.offset(insertPos), f.newCounter(insertPos, blockEnd, 0)+";")
return
}
// Make a copy of the list, as we may mutate it and should leave the
// existing list intact.
list = append([]ast.Stmt(nil), list...)
// We have a block (statement list), but it may have several basic blocks due to the
// appearance of statements that affect the flow of control.
for {
// Find first statement that affects flow of control (break, continue, if, etc.).
// It will be the last statement of this basic block.
var last int
end := blockEnd
for last = 0; last < len(list); last++ {
stmt := list[last]
end = f.statementBoundary(stmt)
if f.endsBasicSourceBlock(stmt) {
// If it is a labeled statement, we need to place a counter between
// the label and its statement because it may be the target of a goto
// and thus start a basic block. That is, given
// foo: stmt
// we need to create
// foo: ; stmt
// and mark the label as a block-terminating statement.
// The result will then be
// foo: COUNTER[n]++; stmt
// However, we can't do this if the labeled statement is already
// a control statement, such as a labeled for.
if label, isLabel := stmt.(*ast.LabeledStmt); isLabel && !f.isControl(label.Stmt) {
newLabel := *label
newLabel.Stmt = &ast.EmptyStmt{
Semicolon: label.Stmt.Pos(),
Implicit: true,
}
end = label.Pos() // Previous block ends before the label.
list[last] = &newLabel
// Open a gap and drop in the old statement, now without a label.
list = append(list, nil)
copy(list[last+1:], list[last:])
list[last+1] = label.Stmt
}
last++
extendToClosingBrace = false // Block is broken up now.
break
}
}
if extendToClosingBrace {
end = blockEnd
}
if pos != end { // Can have no source to cover if e.g. blocks abut.
f.edit.Insert(f.offset(insertPos), f.newCounter(pos, end, last)+";")
}
list = list[last:]
if len(list) == 0 {
break
}
pos = list[0].Pos()
insertPos = pos
}
}
// hasFuncLiteral reports the existence and position of the first func literal
// in the node, if any. If a func literal appears, it usually marks the termination
// of a basic block because the function body is itself a block.
// Therefore we draw a line at the start of the body of the first function literal we find.
// TODO: what if there's more than one? Probably doesn't matter much.
func hasFuncLiteral(n ast.Node) (bool, token.Pos) {
if n == nil {
return false, 0
}
var literal funcLitFinder
ast.Walk(&literal, n)
return literal.found(), token.Pos(literal)
}
// statementBoundary finds the location in s that terminates the current basic
// block in the source.
func (f *File) statementBoundary(s ast.Stmt) token.Pos {
// Control flow statements are easy.
switch s := s.(type) {
case *ast.BlockStmt:
// Treat blocks like basic blocks to avoid overlapping counters.
return s.Lbrace
case *ast.IfStmt:
found, pos := hasFuncLiteral(s.Init)
if found {
return pos
}
found, pos = hasFuncLiteral(s.Cond)
if found {
return pos
}
return s.Body.Lbrace
case *ast.ForStmt:
found, pos := hasFuncLiteral(s.Init)
if found {
return pos
}
found, pos = hasFuncLiteral(s.Cond)
if found {
return pos
}
found, pos = hasFuncLiteral(s.Post)
if found {
return pos
}
return s.Body.Lbrace
case *ast.LabeledStmt:
return f.statementBoundary(s.Stmt)
case *ast.RangeStmt:
found, pos := hasFuncLiteral(s.X)
if found {
return pos
}
return s.Body.Lbrace
case *ast.SwitchStmt:
found, pos := hasFuncLiteral(s.Init)
if found {
return pos
}
found, pos = hasFuncLiteral(s.Tag)
if found {
return pos
}
return s.Body.Lbrace
case *ast.SelectStmt:
return s.Body.Lbrace
case *ast.TypeSwitchStmt:
found, pos := hasFuncLiteral(s.Init)
if found {
return pos
}
return s.Body.Lbrace
}
// If not a control flow statement, it is a declaration, expression, call, etc. and it may have a function literal.
// If it does, that's tricky because we want to exclude the body of the function from this block.
// Draw a line at the start of the body of the first function literal we find.
// TODO: what if there's more than one? Probably doesn't matter much.
found, pos := hasFuncLiteral(s)
if found {
return pos
}
return s.End()
}
// endsBasicSourceBlock reports whether s changes the flow of control: break, if, etc.,
// or if it's just problematic, for instance contains a function literal, which will complicate
// accounting due to the block-within-an expression.
func (f *File) endsBasicSourceBlock(s ast.Stmt) bool {
switch s := s.(type) {
case *ast.BlockStmt:
// Treat blocks like basic blocks to avoid overlapping counters.
return true
case *ast.BranchStmt:
return true
case *ast.ForStmt:
return true
case *ast.IfStmt:
return true
case *ast.LabeledStmt:
return true // A goto may branch here, starting a new basic block.
case *ast.RangeStmt:
return true
case *ast.SwitchStmt:
return true
case *ast.SelectStmt:
return true
case *ast.TypeSwitchStmt:
return true
case *ast.ExprStmt:
// Calls to panic change the flow.
// We really should verify that "panic" is the predefined function,
// but without type checking we can't and the likelihood of it being
// an actual problem is vanishingly small.
if call, ok := s.X.(*ast.CallExpr); ok {
if ident, ok := call.Fun.(*ast.Ident); ok && ident.Name == "panic" && len(call.Args) == 1 {
return true
}
}
}
found, _ := hasFuncLiteral(s)
return found
}
// isControl reports whether s is a control statement that, if labeled, cannot be
// separated from its label.
func (f *File) isControl(s ast.Stmt) bool {
switch s.(type) {
case *ast.ForStmt, *ast.RangeStmt, *ast.SwitchStmt, *ast.SelectStmt, *ast.TypeSwitchStmt:
return true
}
return false
}
// funcLitFinder implements the ast.Visitor pattern to find the location of any
// function literal in a subtree.
type funcLitFinder token.Pos
func (f *funcLitFinder) Visit(node ast.Node) (w ast.Visitor) {
if f.found() {
return nil // Prune search.
}
switch n := node.(type) {
case *ast.FuncLit:
*f = funcLitFinder(n.Body.Lbrace)
return nil // Prune search.
}
return f
}
func (f *funcLitFinder) found() bool {
return token.Pos(*f) != token.NoPos
}
// Sort interface for []block1; used for self-check in addVariables.
type block1 struct {
Block
index int
}
type blockSlice []block1
func (b blockSlice) Len() int { return len(b) }
func (b blockSlice) Less(i, j int) bool { return b[i].startByte < b[j].startByte }
func (b blockSlice) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
// offset translates a token position into a 0-indexed byte offset.
func (f *File) offset(pos token.Pos) int {
return f.fset.Position(pos).Offset
}
// addVariables adds to the end of the file the declarations to set up the counter and position variables.
func (f *File) addVariables(w io.Writer) {
// Self-check: Verify that the instrumented basic blocks are disjoint.
t := make([]block1, len(f.blocks))
for i := range f.blocks {
t[i].Block = f.blocks[i]
t[i].index = i
}
sort.Sort(blockSlice(t))
for i := 1; i < len(t); i++ {
if t[i-1].endByte > t[i].startByte {
fmt.Fprintf(os.Stderr, "cover: internal error: block %d overlaps block %d\n", t[i-1].index, t[i].index)
// Note: error message is in byte positions, not token positions.
fmt.Fprintf(os.Stderr, "\t%s:#%d,#%d %s:#%d,#%d\n",
f.name, f.offset(t[i-1].startByte), f.offset(t[i-1].endByte),
f.name, f.offset(t[i].startByte), f.offset(t[i].endByte))
}
}
// Declare the coverage struct as a package-level variable.
fmt.Fprintf(w, "\nvar %s = struct {\n", f.varVar) // QINIU
fmt.Fprintf(w, "\tCount [%d]uint32\n", len(f.blocks))
fmt.Fprintf(w, "\tPos [3 * %d]uint32\n", len(f.blocks))
fmt.Fprintf(w, "\tNumStmt [%d]uint16\n", len(f.blocks))
fmt.Fprintf(w, "} {\n")
// Initialize the position array field.
fmt.Fprintf(w, "\tPos: [3 * %d]uint32{\n", len(f.blocks))
// A nice long list of positions. Each position is encoded as follows to reduce size:
// - 32-bit starting line number
// - 32-bit ending line number
// - (16 bit ending column number << 16) | (16-bit starting column number).
for i, block := range f.blocks {
start := f.fset.Position(block.startByte)
end := f.fset.Position(block.endByte)
start, end = dedup(start, end)
fmt.Fprintf(w, "\t\t%d, %d, %#x, // [%d]\n", start.Line, end.Line, (end.Column&0xFFFF)<<16|(start.Column&0xFFFF), i)
}
// Close the position array.
fmt.Fprintf(w, "\t},\n")
// Initialize the position array field.
fmt.Fprintf(w, "\tNumStmt: [%d]uint16{\n", len(f.blocks))
// A nice long list of statements-per-block, so we can give a conventional
// valuation of "percent covered". To save space, it's a 16-bit number, so we
// clamp it if it overflows - won't matter in practice.
for i, block := range f.blocks {
n := block.numStmt
if n > 1<<16-1 {
n = 1<<16 - 1
}
fmt.Fprintf(w, "\t\t%d, // %d\n", n, i)
}
// Close the statements-per-block array.
fmt.Fprintf(w, "\t},\n")
// Close the struct initialization.
fmt.Fprintf(w, "}\n")
// Emit a reference to the atomic package to avoid
// import and not used error when there's no code in a file.
// if f.mode == "atomic" { // QINIU, no need to import
// fmt.Fprintf(w, "var _ = %s.LoadUint32\n", atomicPackageName)
// }
}
// It is possible for positions to repeat when there is a line
// directive that does not specify column information and the input
// has not been passed through gofmt.
// See issues #27530 and #30746.
// Tests are TestHtmlUnformatted and TestLineDup.
// We use a map to avoid duplicates.
// pos2 is a pair of token.Position values, used as a map key type.
type pos2 struct {
p1, p2 token.Position
}
// seenPos2 tracks whether we have seen a token.Position pair.
var seenPos2 = make(map[pos2]bool)
// dedup takes a token.Position pair and returns a pair that does not
// duplicate any existing pair. The returned pair will have the Offset
// fields cleared.
func dedup(p1, p2 token.Position) (r1, r2 token.Position) {
key := pos2{
p1: p1,
p2: p2,
}
// We want to ignore the Offset fields in the map,
// since cover uses only file/line/column.
key.p1.Offset = 0
key.p2.Offset = 0
for seenPos2[key] {
key.p2.Column++
}
seenPos2[key] = true
return key.p1, key.p2
}

93
pkg/cover/internal/tool/edit.go
View File

@ -1,93 +0,0 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package edit implements buffered position-based editing of byte slices.
package tool
import (
"fmt"
"sort"
)
// A Buffer is a queue of edits to apply to a given byte slice.
type Buffer struct {
old []byte
q edits
}
// An edit records a single text modification: change the bytes in [start,end) to new.
type edit struct {
start int
end int
new string
}
// An edits is a list of edits that is sortable by start offset, breaking ties by end offset.
type edits []edit
func (x edits) Len() int { return len(x) }
func (x edits) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
func (x edits) Less(i, j int) bool {
if x[i].start != x[j].start {
return x[i].start < x[j].start
}
return x[i].end < x[j].end
}
// NewBuffer returns a new buffer to accumulate changes to an initial data slice.
// The returned buffer maintains a reference to the data, so the caller must ensure
// the data is not modified until after the Buffer is done being used.
func NewBuffer(data []byte) *Buffer {
return &Buffer{old: data}
}
func (b *Buffer) Insert(pos int, new string) {
if pos < 0 || pos > len(b.old) {
panic("invalid edit position")
}
b.q = append(b.q, edit{pos, pos, new})
}
func (b *Buffer) Delete(start, end int) {
if end < start || start < 0 || end > len(b.old) {
panic("invalid edit position")
}
b.q = append(b.q, edit{start, end, ""})
}
func (b *Buffer) Replace(start, end int, new string) {
if end < start || start < 0 || end > len(b.old) {
panic("invalid edit position")
}
b.q = append(b.q, edit{start, end, new})
}
// Bytes returns a new byte slice containing the original data
// with the queued edits applied.
func (b *Buffer) Bytes() []byte {
// Sort edits by starting position and then by ending position.
// Breaking ties by ending position allows insertions at point x
// to be applied before a replacement of the text at [x, y).
sort.Stable(b.q)
var new []byte
offset := 0
for i, e := range b.q {
if e.start < offset {
e0 := b.q[i-1]
panic(fmt.Sprintf("overlapping edits: [%d,%d)->%q, [%d,%d)->%q", e0.start, e0.end, e0.new, e.start, e.end, e.new))
}
new = append(new, b.old[offset:e.start]...)
offset = e.end
new = append(new, e.new...)
}
new = append(new, b.old[offset:]...)
return new
}
// String returns a string containing the original data
// with the queued edits applied.
func (b *Buffer) String() string {
return string(b.Bytes())
}

394
pkg/cover/server.go
View File

@ -1,394 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cover
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
"os"
"regexp"
"github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus"
"golang.org/x/tools/cover"
"k8s.io/test-infra/gopherage/pkg/cov"
)
// LogFile a file to save log.
const LogFile = "goc.log"
type server struct {
PersistenceFile string
Store Store
}
// NewFileBasedServer new a file based server with persistenceFile
func NewFileBasedServer(persistenceFile string) (*server, error) {
store, err := NewFileStore(persistenceFile)
if err != nil {
return nil, err
}
return &server{
PersistenceFile: persistenceFile,
Store: store,
}, nil
}
// NewMemoryBasedServer new a memory based server without persistenceFile
func NewMemoryBasedServer() *server {
return &server{
Store: NewMemoryStore(),
}
}
// Run starts coverage host center
func (s *server) Run(port string) {
f, err := os.Create(LogFile)
if err != nil {
log.Fatalf("failed to create log file %s, err: %v", LogFile, err)
}
// both log to stdout and file by default
mw := io.MultiWriter(f, os.Stdout)
r := s.Route(mw)
log.Fatal(r.Run(port))
}
// Router init goc server engine
func (s *server) Route(w io.Writer) *gin.Engine {
if w != nil {
gin.DefaultWriter = w
}
r := gin.Default()
// api to show the registered services
r.StaticFile("static", "./"+s.PersistenceFile)
v1 := r.Group("/v1")
{
v1.POST("/cover/register", s.registerService)
v1.GET("/cover/profile", s.profile)
v1.POST("/cover/profile", s.profile)
v1.POST("/cover/clear", s.clear)
v1.POST("/cover/init", s.initSystem)
v1.GET("/cover/list", s.listServices)
v1.POST("/cover/remove", s.removeServices)
}
return r
}
// ServiceUnderTest is a entry under being tested
type ServiceUnderTest struct {
Name string `form:"name" json:"name" binding:"required"`
Address string `form:"address" json:"address" binding:"required"`
}
// ProfileParam is param of profile API
type ProfileParam struct {
Force bool `form:"force" json:"force"`
Service []string `form:"service" json:"service"`
Address []string `form:"address" json:"address"`
CoverFilePatterns []string `form:"coverfile" json:"coverfile"`
SkipFilePatterns []string `form:"skipfile" json:"skipfile"`
}
//listServices list all the registered services
func (s *server) listServices(c *gin.Context) {
services := s.Store.GetAll()
c.JSON(http.StatusOK, services)
}
func (s *server) registerService(c *gin.Context) {
var service ServiceUnderTest
if err := c.ShouldBind(&service); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
u, err := url.Parse(service.Address)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
host, port, err := net.SplitHostPort(u.Host)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
realIP := c.ClientIP()
// only for IPV4
// refer: https://github.com/qiniu/goc/issues/177
if net.ParseIP(realIP).To4() != nil && host != realIP {
log.Printf("the registered host %s of service %s is different with the real one %s, here we choose the real one", service.Name, host, realIP)
service.Address = fmt.Sprintf("http://%s:%s", realIP, port)
}
address := s.Store.Get(service.Name)
if !contains(address, service.Address) {
if err := s.Store.Add(service); err != nil && err != ErrServiceAlreadyRegistered {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
}
c.JSON(http.StatusOK, gin.H{"result": "success"})
return
}
// profile API examples:
// POST /v1/cover/profile
// { "force": "true", "service":["a","b"], "address":["c","d"],"coverfile":["e","f"] }
func (s *server) profile(c *gin.Context) {
var body ProfileParam
if err := c.ShouldBind(&body); err != nil {
c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
return
}
allInfos := s.Store.GetAll()
filterAddrList, err := filterAddrs(body.Service, body.Address, body.Force, allInfos)
if err != nil {
c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
return
}
var mergedProfiles = make([][]*cover.Profile, 0)
for _, addr := range filterAddrList {
pp, err := NewWorker(addr).Profile(ProfileParam{})
if err != nil {
if body.Force {
log.Warnf("get profile from [%s] failed, error: %s", addr, err.Error())
continue
}
c.JSON(http.StatusExpectationFailed, gin.H{"error": fmt.Sprintf("failed to get profile from %s, error %s", addr, err.Error())})
return
}
profile, err := convertProfile(pp)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
mergedProfiles = append(mergedProfiles, profile)
}
if len(mergedProfiles) == 0 {
c.JSON(http.StatusExpectationFailed, gin.H{"error": "no profiles"})
return
}
merged, err := cov.MergeMultipleProfiles(mergedProfiles)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
if len(body.CoverFilePatterns) > 0 {
merged, err = filterProfile(body.CoverFilePatterns, merged)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("failed to filter profile based on the patterns: %v, error: %v", body.CoverFilePatterns, err)})
return
}
}
if len(body.SkipFilePatterns) > 0 {
merged, err = skipProfile(body.SkipFilePatterns, merged)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("failed to skip profile based on the patterns: %v, error: %v", body.SkipFilePatterns, err)})
return
}
}
if err := cov.DumpProfile(merged, c.Writer); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
}
// filterProfile filters profiles of the packages matching the coverFile pattern
func filterProfile(coverFile []string, profiles []*cover.Profile) ([]*cover.Profile, error) {
var out = make([]*cover.Profile, 0)
for _, profile := range profiles {
for _, pattern := range coverFile {
matched, err := regexp.MatchString(pattern, profile.FileName)
if err != nil {
return nil, fmt.Errorf("filterProfile failed with pattern %s for profile %s, err: %v", pattern, profile.FileName, err)
}
if matched {
out = append(out, profile)
break // no need to check again for the file
}
}
}
return out, nil
}
// skipProfile skips profiles of the packages matching the skipFile pattern
func skipProfile(skipFile []string, profiles []*cover.Profile) ([]*cover.Profile, error) {
var out = make([]*cover.Profile, 0)
for _, profile := range profiles {
var shouldSkip bool
for _, pattern := range skipFile {
matched, err := regexp.MatchString(pattern, profile.FileName)
if err != nil {
return nil, fmt.Errorf("filterProfile failed with pattern %s for profile %s, err: %v", pattern, profile.FileName, err)
}
if matched {
shouldSkip = true
break // no need to check again for the file
}
}
if !shouldSkip {
out = append(out, profile)
}
}
return out, nil
}
func (s *server) clear(c *gin.Context) {
var body ProfileParam
if err := c.ShouldBind(&body); err != nil {
c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
return
}
svrsUnderTest := s.Store.GetAll()
filterAddrList, err := filterAddrs(body.Service, body.Address, true, svrsUnderTest)
if err != nil {
c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
return
}
for _, addr := range filterAddrList {
pp, err := NewWorker(addr).Clear(ProfileParam{})
if err != nil {
c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
return
}
fmt.Fprintf(c.Writer, "Register service %s coverage counter %s", addr, string(pp))
}
}
func (s *server) initSystem(c *gin.Context) {
if err := s.Store.Init(); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, "")
}
func (s *server) removeServices(c *gin.Context) {
var body ProfileParam
if err := c.ShouldBind(&body); err != nil {
c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
return
}
svrsUnderTest := s.Store.GetAll()
filterAddrList, err := filterAddrs(body.Service, body.Address, true, svrsUnderTest)
if err != nil {
c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
return
}
for _, addr := range filterAddrList {
err := s.Store.Remove(addr)
if err != nil {
c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
return
}
fmt.Fprintf(c.Writer, "Register service %s removed from the center.", addr)
}
}
func convertProfile(p []byte) ([]*cover.Profile, error) {
// Annoyingly, ParseProfiles only accepts a filename, so we have to write the bytes to disk
// so it can read them back.
// We could probably also just give it /dev/stdin, but that'll break on Windows.
tf, err := ioutil.TempFile("", "")
if err != nil {
return nil, fmt.Errorf("failed to create temp file, err: %v", err)
}
defer tf.Close()
defer os.Remove(tf.Name())
if _, err := io.Copy(tf, bytes.NewReader(p)); err != nil {
return nil, fmt.Errorf("failed to copy data to temp file, err: %v", err)
}
return cover.ParseProfiles(tf.Name())
}
func contains(arr []string, str string) bool {
for _, element := range arr {
if str == element {
return true
}
}
return false
}
// filterAddrs filter address list by given service and address list
func filterAddrs(serviceList, addressList []string, force bool, allInfos map[string][]string) (filterAddrList []string, err error) {
addressAll := []string{}
for _, addr := range allInfos {
addressAll = append(addressAll, addr...)
}
if len(serviceList) != 0 && len(addressList) != 0 {
return nil, fmt.Errorf("use 'service' flag and 'address' flag at the same time may cause ambiguity, please use them separately")
}
// Add matched services to map
for _, name := range serviceList {
if addr, ok := allInfos[name]; ok {
filterAddrList = append(filterAddrList, addr...)
continue // jump to match the next service
}
if !force {
return nil, fmt.Errorf("service [%s] not found", name)
}
log.Warnf("service [%s] not found", name)
}
// Add matched addresses to map
for _, addr := range addressList {
if contains(addressAll, addr) {
filterAddrList = append(filterAddrList, addr)
continue
}
if !force {
return nil, fmt.Errorf("address [%s] not found", addr)
}
log.Warnf("address [%s] not found", addr)
}
if len(addressList) == 0 && len(serviceList) == 0 {
filterAddrList = addressAll
}
// Return all services when all param is nil
return filterAddrList, nil
}

527
pkg/cover/server_test.go
View File

@ -1,527 +0,0 @@
package cover
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"os"
"reflect"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"golang.org/x/tools/cover"
)
// MockStore is mock store mainly for unittest
type MockStore struct {
mock.Mock
}
func (m *MockStore) Add(s ServiceUnderTest) error {
args := m.Called(s)
return args.Error(0)
}
func (m *MockStore) Remove(a string) error {
args := m.Called(a)
return args.Error(0)
}
func (m *MockStore) Get(name string) []string {
args := m.Called(name)
return args.Get(0).([]string)
}
func (m *MockStore) GetAll() map[string][]string {
args := m.Called()
return args.Get(0).(map[string][]string)
}
func (m *MockStore) Init() error {
args := m.Called()
return args.Error(0)
}
func (m *MockStore) Set(services map[string][]string) error {
args := m.Called()
return args.Error(0)
}
func TestContains(t *testing.T) {
assert.Equal(t, contains([]string{"a", "b"}, "a"), true)
assert.Equal(t, contains([]string{"a", "b"}, "c"), false)
}
func TestFilterAddrs(t *testing.T) {
svrAll := map[string][]string{
"service1": {"http://127.0.0.1:7777", "http://127.0.0.1:8888"},
"service2": {"http://127.0.0.1:9999"},
}
addrAll := []string{}
for _, addr := range svrAll {
addrAll = append(addrAll, addr...)
}
items := []struct {
svrList []string
addrList []string
force bool
err string
addrRes []string
}{
{
svrList: []string{"service1"},
addrList: []string{"http://127.0.0.1:7777"},
err: "use 'service' flag and 'address' flag at the same time may cause ambiguity, please use them separately",
},
{
addrRes: addrAll,
},
{
svrList: []string{"service1", "unknown"},
err: "service [unknown] not found",
},
{
svrList: []string{"service1", "service2", "unknown"},
force: true,
addrRes: addrAll,
},
{
svrList: []string{"unknown"},
force: true,
},
{
addrList: []string{"http://127.0.0.1:7777", "http://127.0.0.2:7777"},
err: "address [http://127.0.0.2:7777] not found",
},
{
addrList: []string{"http://127.0.0.1:7777", "http://127.0.0.1:9999", "http://127.0.0.2:7777"},
force: true,
addrRes: []string{"http://127.0.0.1:7777", "http://127.0.0.1:9999"},
},
}
for _, item := range items {
addrs, err := filterAddrs(item.svrList, item.addrList, item.force, svrAll)
if err != nil {
assert.Equal(t, err.Error(), item.err)
} else {
if len(addrs) == 0 {
assert.Equal(t, addrs, item.addrRes)
}
for _, a := range addrs {
assert.Contains(t, item.addrRes, a)
}
}
}
}
func TestRegisterService(t *testing.T) {
server, err := NewFileBasedServer("_svrs_address.txt")
assert.NoError(t, err)
router := server.Route(os.Stdout)
// register with empty service struct
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/cover/register", nil)
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusBadRequest, w.Code)
// register with invalid service.Address
data := url.Values{}
data.Set("name", "aaa")
data.Set("address", "&%%")
w = httptest.NewRecorder()
req, _ = http.NewRequest("POST", "/v1/cover/register", strings.NewReader(data.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusBadRequest, w.Code)
assert.Contains(t, w.Body.String(), "invalid URL escape")
// register with host but no port
data = url.Values{}
data.Set("name", "aaa")
data.Set("address", "http://127.0.0.1")
w = httptest.NewRecorder()
req, _ = http.NewRequest("POST", "/v1/cover/register", strings.NewReader(data.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusBadRequest, w.Code)
assert.Contains(t, w.Body.String(), "missing port in address")
// register with store failure
expectedS := ServiceUnderTest{
Name: "foo",
Address: "http://:64444", // the real IP is empty in unittest, so server will get a empty one
}
testObj := new(MockStore)
testObj.On("Get", "foo").Return([]string{"http://127.0.0.1:66666"})
testObj.On("Add", expectedS).Return(fmt.Errorf("lala error"))
server.Store = testObj
w = httptest.NewRecorder()
data.Set("name", expectedS.Name)
data.Set("address", expectedS.Address)
req, _ = http.NewRequest("POST", "/v1/cover/register", strings.NewReader(data.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusInternalServerError, w.Code)
assert.Contains(t, w.Body.String(), "lala error")
}
func TestProfileService(t *testing.T) {
server, err := NewFileBasedServer("_svrs_address.txt")
assert.NoError(t, err)
router := server.Route(os.Stdout)
// get profile with invalid force parameter
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/v1/cover/profile?force=11", nil)
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusExpectationFailed, w.Code)
assert.Contains(t, w.Body.String(), "invalid syntax")
}
func TestClearService(t *testing.T) {
testObj := new(MockStore)
testObj.On("GetAll").Return(map[string][]string{"foo": {"http://127.0.0.1:66666"}})
server := &server{
Store: testObj,
}
router := server.Route(os.Stdout)
// clear profile with non-exist port
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/cover/clear", bytes.NewBuffer([]byte(`{}`)))
req.Header.Set("Content-Type", "application/json")
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusExpectationFailed, w.Code)
assert.Contains(t, w.Body.String(), "invalid port")
// clear profile with invalid service
w = httptest.NewRecorder()
req, _ = http.NewRequest("POST", "/v1/cover/clear", nil)
req.Header.Set("Content-Type", "application/json")
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusExpectationFailed, w.Code)
assert.Contains(t, w.Body.String(), "invalid request")
// clear profile with service and address set at at the same time
p := ProfileParam{
Service: []string{"goc"},
Address: []string{"http://127.0.0.1:3333"},
}
encoded, err := json.Marshal(p)
assert.NoError(t, err)
w = httptest.NewRecorder()
req, _ = http.NewRequest("POST", "/v1/cover/clear", bytes.NewBuffer(encoded))
req.Header.Set("Content-Type", "application/json")
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusExpectationFailed, w.Code)
assert.Contains(t, w.Body.String(), "use 'service' flag and 'address' flag at the same time may cause ambiguity, please use them separately")
}
func TestRemoveServices(t *testing.T) {
testObj := new(MockStore)
testObj.On("GetAll").Return(map[string][]string{"foo": {"test1", "test2"}})
testObj.On("Remove", "test1").Return(nil)
server := &server{
Store: testObj,
}
router := server.Route(os.Stdout)
// remove with invalid request
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/cover/remove", nil)
req.Header.Set("Content-Type", "application/json")
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusExpectationFailed, w.Code)
assert.Contains(t, w.Body.String(), "invalid request")
// remove service
p := ProfileParam{
Address: []string{"test1"},
}
encoded, err := json.Marshal(p)
assert.NoError(t, err)
w = httptest.NewRecorder()
req, _ = http.NewRequest("POST", "/v1/cover/remove", bytes.NewBuffer(encoded))
req.Header.Set("Content-Type", "application/json")
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusOK, w.Code)
assert.Contains(t, w.Body.String(), "Register service test1 removed from the center.")
// remove service with non-exist address
testObj.On("Remove", "test2").Return(fmt.Errorf("no service found"))
p = ProfileParam{
Address: []string{"test2"},
}
encoded, err = json.Marshal(p)
assert.NoError(t, err)
w = httptest.NewRecorder()
req, _ = http.NewRequest("POST", "/v1/cover/remove", bytes.NewBuffer(encoded))
req.Header.Set("Content-Type", "application/json")
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusExpectationFailed, w.Code)
assert.Contains(t, w.Body.String(), "no service found")
// clear profile with service and address set at at the same time
p = ProfileParam{
Service: []string{"goc"},
Address: []string{"http://127.0.0.1:3333"},
}
encoded, err = json.Marshal(p)
assert.NoError(t, err)
w = httptest.NewRecorder()
req, _ = http.NewRequest("POST", "/v1/cover/remove", bytes.NewBuffer(encoded))
req.Header.Set("Content-Type", "application/json")
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusExpectationFailed, w.Code)
assert.Contains(t, w.Body.String(), "use 'service' flag and 'address' flag at the same time may cause ambiguity, please use them separately")
}
func TestInitService(t *testing.T) {
testObj := new(MockStore)
testObj.On("Init").Return(fmt.Errorf("lala error"))
server := &server{
Store: testObj,
}
router := server.Route(os.Stdout)
// get profile with invalid force parameter
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/cover/init", nil)
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusInternalServerError, w.Code)
assert.Contains(t, w.Body.String(), "lala error")
}
func TestFilterProfile(t *testing.T) {
var tcs = []struct {
name string
pattern []string
input []*cover.Profile
output []*cover.Profile
expectErr bool
}{
{
name: "normal path",
pattern: []string{"some/fancy/gopath", "a/fancy/gopath"},
input: []*cover.Profile{
{
FileName: "some/fancy/gopath/a.go",
},
{
FileName: "some/fancy/gopath/b/a.go",
},
{
FileName: "a/fancy/gopath/a.go",
},
{
FileName: "b/fancy/gopath/a.go",
},
{
FileName: "b/a/fancy/gopath/a.go",
},
},
output: []*cover.Profile{
{
FileName: "some/fancy/gopath/a.go",
},
{
FileName: "some/fancy/gopath/b/a.go",
},
{
FileName: "a/fancy/gopath/a.go",
},
{
FileName: "b/a/fancy/gopath/a.go",
},
},
},
{
name: "with regular expression",
pattern: []string{"fancy/gopath/a.go$", "^b/a/"},
input: []*cover.Profile{
{
FileName: "some/fancy/gopath/a.go",
},
{
FileName: "some/fancy/gopath/b/a.go",
},
{
FileName: "a/fancy/gopath/a.go",
},
{
FileName: "b/fancy/gopath/c/a.go",
},
{
FileName: "b/a/fancy/gopath/a.go",
},
},
output: []*cover.Profile{
{
FileName: "some/fancy/gopath/a.go",
},
{
FileName: "a/fancy/gopath/a.go",
},
{
FileName: "b/a/fancy/gopath/a.go",
},
},
},
{
name: "with invalid regular expression",
pattern: []string{"(?!a)"},
input: []*cover.Profile{
{
FileName: "some/fancy/gopath/a.go",
},
},
expectErr: true,
},
}
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
out, err := filterProfile(tc.pattern, tc.input)
if err != nil {
if !tc.expectErr {
t.Errorf("Unexpected error: %v", err)
}
return
}
if tc.expectErr {
t.Errorf("Expected an error, but got value %s", stringifyCoverProfile(out))
}
if !reflect.DeepEqual(out, tc.output) {
t.Errorf("Mismatched results. \nExpected: %s\nActual:%s", stringifyCoverProfile(tc.output), stringifyCoverProfile(out))
}
})
}
}
func TestSkipProfile(t *testing.T) {
var tcs = []struct {
name string
pattern []string
input []*cover.Profile
output []*cover.Profile
expectErr bool
}{
{
name: "normal path",
pattern: []string{"some/fancy/gopath", "a/fancy/gopath"},
input: []*cover.Profile{
{
FileName: "some/fancy/gopath/a.go",
},
{
FileName: "some/fancy/gopath/b/a.go",
},
{
FileName: "a/fancy/gopath/a.go",
},
{
FileName: "b/fancy/gopath/a.go",
},
{
FileName: "b/a/fancy/gopath/a.go",
},
},
output: []*cover.Profile{
{
FileName: "b/fancy/gopath/a.go",
},
},
},
{
name: "with regular expression",
pattern: []string{"fancy/gopath/a.go$", "^b/a/"},
input: []*cover.Profile{
{
FileName: "some/fancy/gopath/a.go",
},
{
FileName: "some/fancy/gopath/b/a.go",
},
{
FileName: "a/fancy/gopath/a.go",
},
{
FileName: "b/fancy/gopath/c/a.go",
},
{
FileName: "b/a/fancy/gopath/a.go",
},
},
output: []*cover.Profile{
{
FileName: "some/fancy/gopath/b/a.go",
},
{
FileName: "b/fancy/gopath/c/a.go",
},
},
},
{
name: "with invalid regular expression",
pattern: []string{"(?!a)"},
input: []*cover.Profile{
{
FileName: "some/fancy/gopath/a.go",
},
},
expectErr: true,
},
}
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
out, err := skipProfile(tc.pattern, tc.input)
if err != nil {
if !tc.expectErr {
t.Errorf("Unexpected error: %v", err)
}
return
}
if tc.expectErr {
t.Errorf("Expected an error, but got value %s", stringifyCoverProfile(out))
}
if !reflect.DeepEqual(out, tc.output) {
t.Errorf("Mismatched results. \nExpected: %s\nActual:%s", stringifyCoverProfile(tc.output), stringifyCoverProfile(out))
}
})
}
}
func stringifyCoverProfile(profiles []*cover.Profile) string {
res := make([]cover.Profile, 0, len(profiles))
for _, p := range profiles {
res = append(res, *p)
}
return fmt.Sprintf("%#v", res)
}

324
pkg/cover/store.go
View File

@ -1,324 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cover
import (
"bufio"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
log "github.com/sirupsen/logrus"
)
var ErrServiceAlreadyRegistered = errors.New("service already registered")
// Store persistents the registered service information
type Store interface {
// Add adds the given service to store
Add(s ServiceUnderTest) error
// Get returns the registered service information with the given service's name
Get(name string) []string
// Get returns all the registered service information as a map
GetAll() map[string][]string
// Init cleanup all the registered service information
Init() error
// Set stores the services information into internal state
Set(services map[string][]string) error
// Remove the service from the store by address
Remove(addr string) error
}
// fileStore holds the registered services into memory and persistent to a local file
type fileStore struct {
mu sync.RWMutex
persistentFile string
memoryStore Store
}
// NewFileStore creates a store using local file
func NewFileStore(persistenceFile string) (store Store, err error) {
path, err := filepath.Abs(persistenceFile)
if err != nil {
return nil, err
}
err = os.MkdirAll(filepath.Dir(path), os.ModePerm)
if err != nil {
return nil, err
}
l := &fileStore{
persistentFile: path,
memoryStore: NewMemoryStore(),
}
if err := l.load(); err != nil {
log.Fatalf("load failed, file: %s, err: %v", l.persistentFile, err)
}
return l, nil
}
// Add adds the given service to file Store
func (l *fileStore) Add(s ServiceUnderTest) error {
if err := l.memoryStore.Add(s); err != nil {
return err
}
// persistent to local store
l.mu.Lock()
defer l.mu.Unlock()
return l.appendToFile(s)
}
// Get returns the registered service information with the given name
func (l *fileStore) Get(name string) []string {
return l.memoryStore.Get(name)
}
// Get returns all the registered service information
func (l *fileStore) GetAll() map[string][]string {
return l.memoryStore.GetAll()
}
// Remove the service from the memory store and the file store
func (l *fileStore) Remove(addr string) error {
err := l.memoryStore.Remove(addr)
if err != nil {
return err
}
return l.Set(l.memoryStore.GetAll())
}
// Init cleanup all the registered service information
// and the local persistent file
func (l *fileStore) Init() error {
if err := l.memoryStore.Init(); err != nil {
return err
}
l.mu.Lock()
defer l.mu.Unlock()
if err := os.Remove(l.persistentFile); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to delete file %s, err: %v", l.persistentFile, err)
}
return nil
}
// load all registered service from file to memory
func (l *fileStore) load() error {
var svrsMap = make(map[string][]string, 0)
f, err := os.Open(l.persistentFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("failed to open file, path: %s, err: %v", l.persistentFile, err)
}
defer f.Close()
ns := bufio.NewScanner(f)
for ns.Scan() {
line := ns.Text()
ss := strings.FieldsFunc(line, split)
// TODO: use regex
if len(ss) == 2 {
if urls, ok := svrsMap[ss[0]]; ok {
urls = append(urls, ss[1])
svrsMap[ss[0]] = urls
} else {
svrsMap[ss[0]] = []string{ss[1]}
}
}
}
if err := ns.Err(); err != nil {
return fmt.Errorf("read file failed, file: %s, err: %v", l.persistentFile, err)
}
// set information to memory
l.memoryStore.Set(svrsMap)
return nil
}
func (l *fileStore) Set(services map[string][]string) error {
l.mu.Lock()
defer l.mu.Unlock()
// no error will return from memorystore.set
err := l.memoryStore.Set(services)
if err != nil {
return err
}
f, err := os.OpenFile(l.persistentFile, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return err
}
s := ""
for name, addrs := range services {
for _, addr := range addrs {
s += fmt.Sprintf("%s&%s\n", name, addr)
}
}
_, err = f.WriteString(s)
if err != nil {
return err
}
return f.Sync()
}
func (l *fileStore) appendToFile(s ServiceUnderTest) error {
f, err := os.OpenFile(l.persistentFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return err
}
defer f.Close()
_, err = f.WriteString(format(s) + "\n")
if err != nil {
return err
}
f.Sync()
return nil
}
func format(s ServiceUnderTest) string {
return fmt.Sprintf("%s&%s", s.Name, s.Address)
}
func split(r rune) bool {
return r == '&'
}
// memoryStore holds the registered services only into memory
type memoryStore struct {
mu sync.RWMutex
servicesMap map[string][]string
}
// NewMemoryStore creates a memory store
func NewMemoryStore() Store {
return &memoryStore{
servicesMap: make(map[string][]string, 0),
}
}
// Add adds the given service to MemoryStore
func (l *memoryStore) Add(s ServiceUnderTest) error {
l.mu.Lock()
defer l.mu.Unlock()
// load to memory
if addrs, ok := l.servicesMap[s.Name]; ok {
for _, addr := range addrs {
if addr == s.Address {
log.Printf("service registered already, name: %s, address: %s", s.Name, s.Address)
return ErrServiceAlreadyRegistered
}
}
addrs = append(addrs, s.Address)
l.servicesMap[s.Name] = addrs
} else {
l.servicesMap[s.Name] = []string{s.Address}
}
return nil
}
// Get returns the registered service information with the given name
func (l *memoryStore) Get(name string) []string {
l.mu.RLock()
defer l.mu.RUnlock()
return l.servicesMap[name]
}
// Get returns all the registered service information
func (l *memoryStore) GetAll() map[string][]string {
res := make(map[string][]string)
l.mu.RLock()
defer l.mu.RUnlock()
for k, v := range l.servicesMap {
res[k] = append(make([]string, 0, len(v)), v...)
}
return res
}
// Init cleanup all the registered service information
// and the local persistent file
func (l *memoryStore) Init() error {
l.mu.Lock()
defer l.mu.Unlock()
l.servicesMap = make(map[string][]string, 0)
return nil
}
func (l *memoryStore) Set(services map[string][]string) error {
l.mu.Lock()
defer l.mu.Unlock()
l.servicesMap = services
return nil
}
// Remove one service from the memory store
// if service is not fount, return "no service found" error
func (l *memoryStore) Remove(removeAddr string) error {
l.mu.Lock()
defer l.mu.Unlock()
flag := false
for name, addrs := range l.servicesMap {
newAddrs := make([]string, 0)
for _, addr := range addrs {
if removeAddr != addr {
newAddrs = append(newAddrs, addr)
} else {
flag = true
}
}
// if no services left, remove by name
if len(newAddrs) == 0 {
delete(l.servicesMap, name)
} else {
l.servicesMap[name] = newAddrs
}
}
if !flag {
return fmt.Errorf("no service found")
}
return nil
}

130
pkg/cover/store_test.go
View File

@ -1,130 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cover
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
)
func TestLocalStore(t *testing.T) {
localStore, err := NewFileStore("_svrs_address.txt")
assert.NoError(t, err)
var tc1 = ServiceUnderTest{
Name: "a",
Address: "http://127.0.0.1",
}
var tc2 = ServiceUnderTest{
Name: "b",
Address: "http://127.0.0.2",
}
var tc3 = ServiceUnderTest{
Name: "c",
Address: "http://127.0.0.3",
}
var tc4 = ServiceUnderTest{
Name: "a",
Address: "http://127.0.0.4",
}
assert.NoError(t, localStore.Add(tc1))
assert.Equal(t, localStore.Add(tc1), ErrServiceAlreadyRegistered)
assert.NoError(t, localStore.Add(tc2))
assert.NoError(t, localStore.Add(tc3))
assert.NoError(t, localStore.Add(tc4))
addrs := localStore.Get(tc1.Name)
if len(addrs) != 2 {
t.Error("unexpected result")
}
for _, addr := range addrs {
if addr != tc1.Address && addr != tc4.Address {
t.Error("get address failed")
}
}
if len(localStore.GetAll()) != 3 {
t.Error("local store check failed")
}
localStoreNew, err := NewFileStore("_svrs_address.txt")
assert.NoError(t, err)
assert.Equal(t, localStore.GetAll(), localStoreNew.GetAll())
localStore.Init()
if len(localStore.GetAll()) != 0 {
t.Error("local store init failed")
}
}
func TestMemoryStoreRemove(t *testing.T) {
store := NewMemoryStore()
s1 := ServiceUnderTest{Name: "test", Address: "http://127.0.0.1:8900"}
s2 := ServiceUnderTest{Name: "test2", Address: "http://127.0.0.1:8901"}
s3 := ServiceUnderTest{Name: "test2", Address: "http://127.0.0.1:8902"}
_ = store.Add(s1)
_ = store.Add(s2)
_ = store.Add(s3)
ss1 := store.Get("test")
assert.Equal(t, 1, len(ss1))
err := store.Remove("http://127.0.0.1:8900")
assert.NoError(t, err)
ss1 = store.Get("test")
assert.Nil(t, ss1)
ss2 := store.Get("test2")
assert.Equal(t, 2, len(ss2))
err = store.Remove("http://127.0.0.1:8901")
assert.NoError(t, err)
ss2 = store.Get("test2")
assert.Equal(t, 1, len(ss2))
err = store.Remove("http")
assert.Error(t, err, fmt.Errorf("no service found"))
}
func TestFileStoreRemove(t *testing.T) {
store, _ := NewFileStore("_svrs_address.txt")
_ = store.Init()
s1 := ServiceUnderTest{Name: "test", Address: "http://127.0.0.1:8900"}
s2 := ServiceUnderTest{Name: "test2", Address: "http://127.0.0.1:8901"}
s3 := ServiceUnderTest{Name: "test2", Address: "http://127.0.0.1:8902"}
_ = store.Add(s1)
_ = store.Add(s2)
_ = store.Add(s3)
ss1 := store.Get("test")
assert.Equal(t, 1, len(ss1))
err := store.Remove("http://127.0.0.1:8900")
assert.NoError(t, err)
ss1 = store.Get("test")
assert.Nil(t, ss1)
ss2 := store.Get("test2")
assert.Equal(t, 2, len(ss2))
err = store.Remove("http://127.0.0.1:8901")
assert.NoError(t, err)
ss2 = store.Get("test2")
assert.Equal(t, 1, len(ss2))
err = store.Remove("http")
assert.Error(t, err, fmt.Errorf("no service found"))
}

203
pkg/github/github.go
View File

@ -1,203 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package github
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"os"
"strconv"
"strings"
"github.com/google/go-github/github"
"github.com/hashicorp/go-retryablehttp"
"github.com/olekukonko/tablewriter"
"github.com/sirupsen/logrus"
"golang.org/x/oauth2"
"github.com/qiniu/goc/pkg/cover"
)
// CommentsPrefix is the prefix when commenting on Github Pull Requests
// It is also the flag when checking whether the target comment exists or not to avoid duplicate
const CommentsPrefix = "The following is the coverage report on the affected files."
// PrComment is the interface of the entry which is able to comment on Github Pull Requests
type PrComment interface {
CreateGithubComment(commentPrefix string, diffCovList cover.DeltaCovList) (err error)
PostComment(content, commentPrefix string) error
EraseHistoryComment(commentPrefix string) error
GetPrChangedFiles() (files []string, err error)
GetCommentFlag() string
}
// GitPrComment is the entry which is able to comment on Github Pull Requests
type GitPrComment struct {
RobotUserName string
RepoOwner string
RepoName string
CommentFlag string
PrNumber int
Ctx context.Context
opt *github.ListOptions
GithubClient *github.Client
}
// NewPrClient creates an Client which be able to comment on Github Pull Request
func NewPrClient(githubTokenPath, repoOwner, repoName, prNumStr, botUserName, commentFlag string) *GitPrComment {
var client *github.Client
// performs automatic retries when connection error occurs or a 500-range response code received (except 501)
retryClient := retryablehttp.NewClient()
ctx := context.WithValue(context.Background(), oauth2.HTTPClient, retryClient.StandardClient())
prNum, err := strconv.Atoi(prNumStr)
if err != nil {
logrus.WithError(err).Fatalf("Failed to convert prNumStr(=%v) to int.\n", prNumStr)
}
token, err := ioutil.ReadFile(githubTokenPath)
if err != nil {
logrus.WithError(err).Fatalf("Failed to get github token.\n")
}
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: strings.TrimSpace(string(token))},
)
tc := oauth2.NewClient(ctx, ts)
client = github.NewClient(tc)
return &GitPrComment{
RobotUserName: botUserName,
RepoOwner: repoOwner,
RepoName: repoName,
PrNumber: prNum,
CommentFlag: commentFlag,
Ctx: ctx,
opt: &github.ListOptions{Page: 1},
GithubClient: client,
}
}
// CreateGithubComment post github comment of diff coverage
func (c *GitPrComment) CreateGithubComment(commentPrefix string, diffCovList cover.DeltaCovList) (err error) {
if len(diffCovList) == 0 {
logrus.Printf("Detect 0 files coverage diff, will not comment to github.")
return nil
}
content := GenCommentContent(commentPrefix, diffCovList)
err = c.PostComment(content, commentPrefix)
if err != nil {
logrus.WithError(err).Fatalf("Post comment to github failed.")
}
return
}
// PostComment post comment on github. It erased the old one if existed to avoid duplicate
func (c *GitPrComment) PostComment(content, commentPrefix string) error {
//step1: erase history similar comment to avoid too many comment for same job
err := c.EraseHistoryComment(commentPrefix)
if err != nil {
return err
}
//step2: post comment with new result
comment := &github.IssueComment{
Body: &content,
}
_, _, err = c.GithubClient.Issues.CreateComment(c.Ctx, c.RepoOwner, c.RepoName, c.PrNumber, comment)
if err != nil {
return err
}
return nil
}
// EraseHistoryComment erase history similar comment before post again
func (c *GitPrComment) EraseHistoryComment(commentPrefix string) error {
comments, _, err := c.GithubClient.Issues.ListComments(c.Ctx, c.RepoOwner, c.RepoName, c.PrNumber, nil)
if err != nil {
logrus.Errorf("list PR comments failed.")
return err
}
logrus.Infof("the count of history comments by %s is: %v", c.RobotUserName, len(comments))
for _, cm := range comments {
if *cm.GetUser().Login == c.RobotUserName && strings.HasPrefix(cm.GetBody(), commentPrefix) {
_, err = c.GithubClient.Issues.DeleteComment(c.Ctx, c.RepoOwner, c.RepoName, *cm.ID)
if err != nil {
logrus.Errorf("delete PR comments %d failed.", *cm.ID)
return err
}
}
}
return nil
}
// GetPrChangedFiles get github pull request changes file list
func (c *GitPrComment) GetPrChangedFiles() (files []string, err error) {
var commitFiles []*github.CommitFile
for {
f, resp, err := c.GithubClient.PullRequests.ListFiles(c.Ctx, c.RepoOwner, c.RepoName, c.PrNumber, c.opt)
if err != nil {
logrus.Errorf("Get PR changed file failed. repoOwner is: %s, repoName is: %s, prNum is: %d", c.RepoOwner, c.RepoName, c.PrNumber)
return nil, err
}
commitFiles = append(commitFiles, f...)
if resp.NextPage == 0 {
break
}
c.opt.Page = resp.NextPage
}
logrus.Infof("get %d PR changed files:", len(commitFiles))
for _, file := range commitFiles {
files = append(files, *file.Filename)
logrus.Infof("%s", *file.Filename)
}
return
}
// GetCommentFlag get CommentFlag from the GitPrComment
func (c *GitPrComment) GetCommentFlag() string {
return c.CommentFlag
}
// GenCommentContent generate github comment content based on diff coverage and commentFlag
func GenCommentContent(commentPrefix string, delta cover.DeltaCovList) string {
var buf bytes.Buffer
table := tablewriter.NewWriter(&buf)
table.SetHeader([]string{"File", "Base Coverage", "New Coverage", "Delta"})
table.SetAutoFormatHeaders(false)
table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false})
table.SetCenterSeparator("|")
table.SetColumnAlignment([]int{tablewriter.ALIGN_LEFT, tablewriter.ALIGN_CENTER, tablewriter.ALIGN_CENTER, tablewriter.ALIGN_CENTER})
for _, d := range delta {
table.Append([]string{fmt.Sprintf("[%s](%s)", d.FileName, d.LineCovLink), d.BasePer, d.NewPer, d.DeltaPer})
}
table.Render()
content := []string{
commentPrefix,
fmt.Sprintf("Say `/test %s` to re-run this coverage report", os.Getenv("JOB_NAME")),
buf.String(),
}
return strings.Join(content, "\n")
}

176
pkg/github/github_test.go
View File

@ -1,176 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package github
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"os"
"testing"
"github.com/google/go-github/github"
"github.com/julienschmidt/httprouter"
"github.com/stretchr/testify/assert"
"golang.org/x/net/context"
"github.com/qiniu/goc/pkg/cover"
)
const (
// baseURLPath is a non-empty Client.BaseURL path to use during tests,
// to ensure relative URLs are used for all endpoints. See issue #752.
baseURLPath = "/api-v3"
)
// setup sets up a test HTTP server along with a github.Client that is
// configured to talk to that test server. Tests should register handlers on
// mux which provide mock responses for the API method being tested.
func setup() (client *github.Client, router *httprouter.Router, serverURL string, teardown func()) {
// router is the HTTP request multiplexer used with the test server.
router = httprouter.New()
// We want to ensure that tests catch mistakes where the endpoint URL is
// specified as absolute rather than relative. It only makes a difference
// when there's a non-empty base URL path. So, use that. See issue #752.
apiHandler := http.NewServeMux()
apiHandler.Handle(baseURLPath+"/", http.StripPrefix(baseURLPath, router))
apiHandler.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
fmt.Fprintln(os.Stderr, "FAIL: Client.BaseURL path prefix is not preserved in the request URL:")
fmt.Fprintln(os.Stderr)
fmt.Fprintln(os.Stderr, "\t"+req.URL.String())
fmt.Fprintln(os.Stderr)
fmt.Fprintln(os.Stderr, "\tDid you accidentally use an absolute endpoint URL rather than relative?")
fmt.Fprintln(os.Stderr, "\tSee https://github.com/google/go-github/issues/752 for information.")
http.Error(w, "Client.BaseURL path prefix is not preserved in the request URL.", http.StatusInternalServerError)
})
// server is a test HTTP server used to provide mock API responses.
server := httptest.NewServer(apiHandler)
// client is the GitHub client being tested and is
// configured to use test server.
client = github.NewClient(nil)
url, _ := url.Parse(server.URL + baseURLPath + "/")
client.BaseURL = url
client.UploadURL = url
return client, router, server.URL, server.Close
}
func TestNewPrClient(t *testing.T) {
items := []struct {
token string
repoOwner string
repoName string
prNumStr string
botUserName string
commentFlag string
expectPrNum int
}{
{token: "github_test.go", repoOwner: "qiniu", repoName: "goc", prNumStr: "1", botUserName: "qiniu-bot", commentFlag: "test", expectPrNum: 1},
}
for _, tc := range items {
prClient := NewPrClient(tc.token, tc.repoOwner, tc.repoName, tc.prNumStr, tc.botUserName, tc.commentFlag)
assert.Equal(t, tc.expectPrNum, prClient.PrNumber)
}
}
func TestCreateGithubComment(t *testing.T) {
client, router, _, teardown := setup()
defer teardown()
var coverList = cover.DeltaCovList{{FileName: "fake-coverage", BasePer: "50.0%", NewPer: "75.0%", DeltaPer: "25.0%"}}
expectContent := GenCommentContent("", coverList)
comment := &github.IssueComment{
Body: &expectContent,
}
// create comment: https://developer.github.com/v3/issues/comments/#create-a-comment
router.HandlerFunc("POST", "/repos/qiniu/goc/issues/1/comments", func(w http.ResponseWriter, r *http.Request) {
v := new(github.IssueComment)
json.NewDecoder(r.Body).Decode(v)
assert.Equal(t, v, comment)
fmt.Fprint(w, `{"id":1}`)
})
// list comment: https://developer.github.com/v3/issues/comments/#list-comments-on-an-issue
router.HandlerFunc("GET", "/repos/qiniu/goc/issues/1/comments", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, `[{"id":1,"user": {"login": "qiniu-bot"}}]`)
})
// delete comment: https://developer.github.com/v3/issues/comments/#edit-a-comment
router.HandlerFunc("DELETE", "/repos/qiniu/goc/issues/comments/1", func(w http.ResponseWriter, r *http.Request) {
})
p := GitPrComment{
RobotUserName: "qiniu-bot",
RepoOwner: "qiniu",
RepoName: "goc",
CommentFlag: "",
PrNumber: 1,
Ctx: context.Background(),
opt: nil,
GithubClient: client,
}
p.CreateGithubComment("", coverList)
}
func TestCreateGithubCommentError(t *testing.T) {
p := &GitPrComment{}
err := p.CreateGithubComment("", cover.DeltaCovList{})
assert.NoError(t, err)
}
func TestGetPrChangedFiles(t *testing.T) {
client, router, _, teardown := setup()
defer teardown()
var expectFiles = []string{"src/qiniu.com/kodo/s3apiv2/bucket/bucket.go"}
// list files API: https://developer.github.com/v3/pulls/#list-pull-requests-files
router.HandlerFunc("GET", "/repos/qiniu/goc/pulls/1/files", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, `[{"filename":"src/qiniu.com/kodo/s3apiv2/bucket/bucket.go"}]`)
})
p := GitPrComment{
RobotUserName: "qiniu-bot",
RepoOwner: "qiniu",
RepoName: "goc",
CommentFlag: "",
PrNumber: 1,
Ctx: context.Background(),
opt: nil,
GithubClient: client,
}
changedFiles, err := p.GetPrChangedFiles()
assert.Equal(t, err, nil)
assert.Equal(t, changedFiles, expectFiles)
}
func TestGetCommentFlag(t *testing.T) {
p := GitPrComment{
CommentFlag: "flag",
}
flag := p.GetCommentFlag()
assert.Equal(t, flag, p.CommentFlag)
}

240
pkg/prow/job.go
View File

@ -1,240 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package prow
import (
"bufio"
"bytes"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path"
"strconv"
"strings"
"time"
"github.com/sirupsen/logrus"
"github.com/qiniu/goc/pkg/cover"
"github.com/qiniu/goc/pkg/github"
"github.com/qiniu/goc/pkg/qiniu"
)
// IProwAction defines the normal action in prow system
type IProwAction interface {
Fetch(BuildID, name string) []byte
RunPresubmit() error
RunPostsubmit() error
RunPeriodic() error
}
// Job is a prowjob in prow
type Job struct {
JobName string
Org string
RepoName string
PRNumStr string
BuildId string //prow job build number
PostSubmitJob string
PostSubmitCoverProfile string
CovThreshold int
LocalProfilePath string
QiniuClient qiniu.Client
LocalArtifacts qiniu.Artifacts
GithubComment github.PrComment
FullDiff bool
}
// Fetch the file from cloud
func (j *Job) Fetch(BuildID, name string) []byte {
return []byte{}
}
// RunPresubmit run a presubmit job
func (j *Job) RunPresubmit() error {
// step1: get local profile cov
localP, err := cover.ReadFileToCoverList(j.LocalProfilePath)
if err != nil {
return fmt.Errorf("failed to get remote cover profile: %s", err.Error())
}
//step2: find the remote healthy cover profile from qiniu bucket
remoteProfile, err := qiniu.FindBaseProfileFromQiniu(j.QiniuClient, j.PostSubmitJob, j.PostSubmitCoverProfile)
if err != nil {
return fmt.Errorf("failed to get remote cover profile: %s", err.Error())
}
if remoteProfile == nil {
logrus.Infof("get non healthy remoteProfile, do nothing")
return nil
}
baseP, err := cover.CovList(bytes.NewReader(remoteProfile))
if err != nil {
return fmt.Errorf("failed to get remote cover profile: %s", err.Error())
}
// step3: get github pull request changed files' name and calculate diff cov between local and remote profile
changedFiles, deltaCovList, err := getFilesAndCovList(j.FullDiff, j.GithubComment, localP, baseP)
if err != nil {
return fmt.Errorf("Get files and covlist failed: %s", err.Error())
}
// step4: generate changed file html coverage
err = j.WriteChangedCov(changedFiles)
if err != nil {
return fmt.Errorf("filter local profile to %s with changed files failed: %s", j.LocalArtifacts.GetChangedProfileName(), err.Error())
}
err = j.CreateChangedCovHtml()
if err != nil {
return fmt.Errorf("create changed file related coverage html failed: %s", err.Error())
}
j.SetDeltaCovLinks(deltaCovList)
// step5: post comment to github
commentPrefix := github.CommentsPrefix
if j.GithubComment.GetCommentFlag() != "" {
commentPrefix = fmt.Sprintf("**%s** ", j.GithubComment.GetCommentFlag()) + commentPrefix
}
if len(deltaCovList) > 0 {
totalDelta := cover.PercentStr(cover.TotalDelta(localP, baseP))
deltaCovList = append(deltaCovList, cover.DeltaCov{FileName: "Total", BasePer: baseP.TotalPercentage(), NewPer: localP.TotalPercentage(), DeltaPer: totalDelta})
}
err = j.GithubComment.CreateGithubComment(commentPrefix, deltaCovList)
if err != nil {
return fmt.Errorf("Post comment to github failed: %s", err.Error())
}
return nil
}
// RunPostsubmit run a postsubmit job
func (j *Job) RunPostsubmit() error {
return nil
}
// RunPeriodic run a periodic job
func (j *Job) RunPeriodic() error {
return nil
}
//trim github filename to profile format:
// src/qiniu.com/kodo/io/io/io_svr.go -> qiniu.com/kodo/io/io/io_svr.go
func trimGhFileToProfile(ghFiles []string) (pFiles []string) {
//TODO: need compatible other situation
logrus.Infof("trim PR changed file name to:")
for _, f := range ghFiles {
file := strings.TrimPrefix(f, "src/")
logrus.Infof("%s", file)
pFiles = append(pFiles, file)
}
return
}
// WriteChangedCov filter local profile with changed files and save to j.LocalArtifacts.ChangedProfileName
func (j *Job) WriteChangedCov(changedFiles []string) error {
p, err := ioutil.ReadFile(j.LocalProfilePath)
if err != nil {
logrus.Printf("Open file %s failed", j.LocalProfilePath)
return err
}
cp := j.LocalArtifacts.CreateChangedProfile()
defer cp.Close()
s := bufio.NewScanner(bytes.NewReader(p))
s.Scan()
writeLine(cp, s.Text())
for s.Scan() {
for _, file := range changedFiles {
if strings.HasPrefix(s.Text(), file) {
writeLine(cp, s.Text())
}
}
}
return nil
}
// writeLine writes a line in the given file, if the file pointer is not nil
func writeLine(file *os.File, content string) {
if file != nil {
fmt.Fprintln(file, content)
}
}
// JobPrefixOnQiniu generates the prefix string of the job on qiniu
func (j *Job) JobPrefixOnQiniu() string {
return path.Join("pr-logs", "pull", j.Org+"_"+j.RepoName, j.PRNumStr, j.JobName, j.BuildId)
}
// HtmlProfile generates the name of the profile html file
func (j *Job) HtmlProfile() string {
return fmt.Sprintf("%s-%s-pr%s-coverage.html", j.Org, j.RepoName, j.PRNumStr)
}
// SetDeltaCovLinks set DeltaCovLinks to the job
func (j *Job) SetDeltaCovLinks(c cover.DeltaCovList) {
c.Sort()
for i := 0; i < len(c); i++ {
qnKey := path.Join(j.JobPrefixOnQiniu(), "artifacts", j.HtmlProfile())
authQnKey := j.QiniuClient.GetAccessURL(qnKey, time.Hour*24*7)
c[i].SetLineCovLink(authQnKey + "#file" + strconv.Itoa(i))
logrus.Printf("file %s html coverage link is: %s\n", c[i].FileName, c[i].GetLineCovLink())
}
}
// CreateChangedCovHtml create changed file related coverage html base on the local artifact
func (j *Job) CreateChangedCovHtml() error {
if j.LocalArtifacts.GetChangedProfileName() == "" {
logrus.Errorf("param LocalArtifacts.ChangedProfileName is empty")
}
pathProfileCov := j.LocalArtifacts.GetChangedProfileName()
pathHtmlCov := path.Join(os.Getenv("ARTIFACTS"), j.HtmlProfile())
cmdTxt := fmt.Sprintf("go tool cover -html=%s -o %s", pathProfileCov, pathHtmlCov)
logrus.Printf("Running command '%s'\n", cmdTxt)
cmd := exec.Command("go", "tool", "cover", "-html="+pathProfileCov, "-o", pathHtmlCov)
stdOut, err := cmd.CombinedOutput()
if err != nil {
logrus.Printf("Error executing cmd: %v; combinedOutput=%s", err, stdOut)
}
return err
}
func getFilesAndCovList(fullDiff bool, prComment github.PrComment, localP, baseP cover.CoverageList) (changedFiles []string, deltaCovList cover.DeltaCovList, err error) {
if !fullDiff {
// get github pull request changed files' name
var ghChangedFiles, err = prComment.GetPrChangedFiles()
if err != nil {
return nil, nil, fmt.Errorf("Get pull request changed file failed: %s", err.Error())
}
if len(ghChangedFiles) == 0 {
logrus.Printf("0 files changed in github pull request, don't need to run coverage profile in presubmit.\n")
return nil, nil, nil
}
changedFiles = trimGhFileToProfile(ghChangedFiles)
// calculate diff cov between local and remote profile
deltaCovList = cover.GetChFileDeltaCov(localP, baseP, changedFiles)
logrus.Printf("Get changed files and delta cover list success. ChangedFiles: [%+v], DeltaCovList: [%+v]", changedFiles, deltaCovList)
return changedFiles, deltaCovList, nil
}
deltaCovList = cover.GetDeltaCov(localP, baseP)
for _, d := range deltaCovList {
changedFiles = append(changedFiles, d.FileName)
}
logrus.Printf("Get all files and delta cover list success. Files: [%+v], DeltaCovList: [%+v]", changedFiles, deltaCovList)
return changedFiles, deltaCovList, nil
}

380
pkg/prow/job_test.go
View File

@ -1,380 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package prow
import (
"context"
"errors"
"fmt"
"io/ioutil"
"os"
"path"
"testing"
"time"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/qiniu/goc/pkg/cover"
"github.com/qiniu/goc/pkg/github"
"github.com/qiniu/goc/pkg/qiniu"
)
var (
defaultContent = `mode: atomic
qiniu.com/kodo/bd/bdgetter/source.go:19.118,22.2 2 0
qiniu.com/kodo/bd/bdgetter/source.go:37.34,39.2 1 0
qiniu.com/kodo/bd/pfd/locker/app/qboxbdlocker/main.go:50.2,53.52 4 1
qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go:33.51,35.2 1 0`
defaultLocalPath = "local.cov"
defaultChangedPath = "changed.cov"
)
type MockQnClient struct {
QiniuObjectHandleRes qiniu.ObjectHandle
ReadObjectRes []byte
ReadObjectErr error
ListAllRes []string
ListAllErr error
GetAccessURLRes string
GetArtifactDetailsRes *qiniu.LogHistoryTemplate
GetArtifactDetailsErr error
ListSubDirsRes []string
ListSubDirsErr error
}
func (s *MockQnClient) QiniuObjectHandle(key string) qiniu.ObjectHandle {
return s.QiniuObjectHandleRes
}
func (s *MockQnClient) ReadObject(key string) ([]byte, error) {
return s.ReadObjectRes, s.ReadObjectErr
}
func (s *MockQnClient) ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error) {
return s.ListAllRes, s.ListAllErr
}
func (s *MockQnClient) GetAccessURL(key string, timeout time.Duration) string {
return s.GetAccessURLRes
}
func (s *MockQnClient) GetArtifactDetails(key string) (*qiniu.LogHistoryTemplate, error) {
return s.GetArtifactDetailsRes, s.GetArtifactDetailsErr
}
func (s *MockQnClient) ListSubDirs(prefix string) ([]string, error) {
return s.ListSubDirsRes, s.ListSubDirsErr
}
type MockPrComment struct {
GetPrChangedFilesRes []string
GetPrChangedFilesErr error
PostCommentErr error
EraseHistoryCommentErr error
CreateGithubCommentErr error
CommentFlag string
}
func (s *MockPrComment) GetPrChangedFiles() (files []string, err error) {
return s.GetPrChangedFilesRes, s.GetPrChangedFilesErr
}
func (s *MockPrComment) PostComment(content, commentPrefix string) error {
return s.PostCommentErr
}
func (s *MockPrComment) EraseHistoryComment(commentPrefix string) error {
return s.EraseHistoryCommentErr
}
func (s *MockPrComment) CreateGithubComment(commentPrefix string, diffCovList cover.DeltaCovList) (err error) {
return s.CreateGithubCommentErr
}
func (s *MockPrComment) GetCommentFlag() string {
return s.CommentFlag
}
func TestTrimGhFileToProfile(t *testing.T) {
items := []struct {
inputFiles []string
expectFiles []string
}{
{
inputFiles: []string{"src/qiniu.com/kodo/io/io/io_svr.go", "README.md"},
expectFiles: []string{"qiniu.com/kodo/io/io/io_svr.go", "README.md"},
},
}
for _, tc := range items {
f := trimGhFileToProfile(tc.inputFiles)
assert.Equal(t, f, tc.expectFiles)
}
}
func setup(path, content string) {
err := ioutil.WriteFile(path, []byte(content), 0644)
if err != nil {
logrus.WithError(err).Fatalf("write file %s failed", path)
}
}
func TestWriteChangedCov(t *testing.T) {
path := defaultLocalPath
savePath := qiniu.ChangedProfileName
content := defaultContent
changedFiles := []string{"qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go"}
expectContent := `mode: atomic
qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go:33.51,35.2 1 0
`
setup(path, content)
defer os.Remove(path)
defer os.Remove(savePath)
j := &Job{
LocalProfilePath: path,
LocalArtifacts: &qiniu.ProfileArtifacts{ChangedProfileName: savePath},
}
j.WriteChangedCov(changedFiles)
r, err := ioutil.ReadFile(savePath)
if err != nil {
logrus.WithError(err).Fatalf("read file %s failed", path)
}
assert.Equal(t, string(r), expectContent)
}
func TestRunPresubmitFulldiff(t *testing.T) {
//param
org := "qbox"
repo := "kodo"
prNum := "1"
buildId := "1266322425771986946"
jobName := "kodo-pull-integration-test"
robotName := "qiniu-bot"
githubCommentPrefix := ""
githubTokenPath := "token"
//mock local profile
pwd, err := os.Getwd()
assert.NoError(t, err)
localPath := defaultLocalPath
localProfileContent := `mode: atomic
"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30
"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0`
setup(localPath, localProfileContent)
defer os.Remove(path.Join(pwd, localPath))
// mock qiniu
conf := qiniu.Config{
Bucket: "artifacts",
}
qc, router, _, teardown := qiniu.MockQiniuServer(&conf)
defer teardown()
qiniu.MockRouterAPI(router, localProfileContent, 0)
ChangedProfilePath := "changed.cov"
defer os.Remove(path.Join(pwd, ChangedProfilePath))
//mock github client
setup(githubTokenPath, "")
defer os.Remove(path.Join(pwd, githubTokenPath))
prClient := github.NewPrClient(githubTokenPath, org, repo, prNum, robotName, githubCommentPrefix)
j := &Job{
JobName: jobName,
Org: org,
RepoName: repo,
PRNumStr: prNum,
BuildId: buildId,
PostSubmitJob: "kodo-postsubmits-go-st-coverage",
PostSubmitCoverProfile: "filterd.cov",
LocalProfilePath: localPath,
LocalArtifacts: &qiniu.ProfileArtifacts{ChangedProfileName: ChangedProfilePath},
QiniuClient: qc,
GithubComment: prClient,
FullDiff: true,
}
defer os.Remove(path.Join(os.Getenv("ARTIFACTS"), j.HtmlProfile()))
err = j.RunPresubmit()
assert.NoError(t, err)
}
func TestRunPresubmitError(t *testing.T) {
items := []struct {
prepare bool // prepare local profile
j Job
err string
}{
{
prepare: false,
j: Job{
LocalProfilePath: "unknown",
},
err: "no such file or directory",
},
{
prepare: true,
j: Job{
LocalProfilePath: defaultLocalPath,
QiniuClient: &MockQnClient{},
},
},
{
prepare: true,
j: Job{
LocalProfilePath: defaultLocalPath,
QiniuClient: &MockQnClient{ListSubDirsErr: errors.New("mock error")},
},
err: "mock error",
},
{
prepare: true,
j: Job{
LocalProfilePath: defaultLocalPath,
QiniuClient: &MockProfileQnClient{},
GithubComment: &MockPrComment{GetPrChangedFilesRes: []string{"qiniu.com/kodo/apiserver/server/main.go"}},
FullDiff: true,
LocalArtifacts: &qiniu.ProfileArtifacts{ChangedProfileName: defaultChangedPath},
},
err: "",
},
}
for _, tc := range items {
if tc.prepare {
path := defaultLocalPath
setup(path, defaultContent)
defer os.Remove(path)
defer os.Remove(defaultChangedPath)
}
err := tc.j.RunPresubmit()
if tc.err == "" {
assert.NoError(t, err)
} else {
assert.Contains(t, err.Error(), tc.err)
}
}
}
type MockProfileQnClient struct {
*MockQnClient
}
func (s *MockProfileQnClient) ListSubDirs(prefix string) ([]string, error) {
return []string{defaultContent}, nil
}
func (s *MockProfileQnClient) ReadObject(key string) ([]byte, error) {
logrus.Info(key)
if key == "logs/1/finished.json" {
return []byte(`{"timestamp":1590750306,"passed":true,"result":"SUCCESS","repo-version":"76433418ea48aae57af028f9cb2fa3735ce08c7d"}`), nil
}
return []byte(""), nil
}
func TestGetFilesAndCovList(t *testing.T) {
items := []struct {
fullDiff bool
prComment github.PrComment
localP cover.CoverageList
baseP cover.CoverageList
err string
lenFiles int
lenCovList int
}{
{
fullDiff: true,
prComment: &MockPrComment{},
localP: cover.CoverageList{
{FileName: "qiniu.com/kodo/apiserver/server/main.go", NCoveredStmts: 2, NAllStmts: 2},
{FileName: "qiniu.com/kodo/apiserver/server/test.go", NCoveredStmts: 2, NAllStmts: 2},
},
baseP: cover.CoverageList{
{FileName: "qiniu.com/kodo/apiserver/server/main.go", NCoveredStmts: 1, NAllStmts: 2},
{FileName: "qiniu.com/kodo/apiserver/server/test.go", NCoveredStmts: 1, NAllStmts: 2},
},
lenFiles: 2,
lenCovList: 2,
},
{
fullDiff: false,
prComment: &MockPrComment{GetPrChangedFilesErr: errors.New("mock error")},
err: "mock error",
},
{
fullDiff: false,
prComment: &MockPrComment{},
lenFiles: 0,
lenCovList: 0,
},
{
fullDiff: false,
prComment: &MockPrComment{GetPrChangedFilesRes: []string{"qiniu.com/kodo/apiserver/server/main.go"}},
localP: cover.CoverageList{
{FileName: "qiniu.com/kodo/apiserver/server/main.go", NCoveredStmts: 2, NAllStmts: 2},
{FileName: "qiniu.com/kodo/apiserver/server/test.go", NCoveredStmts: 2, NAllStmts: 2},
},
baseP: cover.CoverageList{
{FileName: "qiniu.com/kodo/apiserver/server/main.go", NCoveredStmts: 1, NAllStmts: 2},
{FileName: "qiniu.com/kodo/apiserver/server/test.go", NCoveredStmts: 1, NAllStmts: 2},
},
lenFiles: 1,
lenCovList: 1,
},
}
for i, tc := range items {
fmt.Println(i)
files, covList, err := getFilesAndCovList(tc.fullDiff, tc.prComment, tc.localP, tc.baseP)
if err != nil {
assert.Contains(t, err.Error(), tc.err)
} else {
assert.Equal(t, len(files), tc.lenFiles)
assert.Equal(t, len(covList), tc.lenCovList)
}
}
}
func TestSetDeltaCovLinks(t *testing.T) {
covList := cover.DeltaCovList{{FileName: "file1", BasePer: "5%", NewPer: "5%", DeltaPer: "0"}}
j := &Job{
QiniuClient: &MockQnClient{},
}
j.SetDeltaCovLinks(covList)
}
// functions to be done
func TestRunPostsubmit(t *testing.T) {
j := &Job{}
err := j.RunPostsubmit()
assert.NoError(t, err)
}
func TestRunPeriodic(t *testing.T) {
j := &Job{}
err := j.RunPeriodic()
assert.NoError(t, err)
}
func TestFetch(t *testing.T) {
j := &Job{}
res := j.Fetch("buidID", "name")
assert.Equal(t, res, []byte{})
}

254
pkg/qiniu/client.go
View File

@ -1,254 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package qiniu
import (
"context"
"fmt"
"io/ioutil"
"math/rand"
"net/http"
"regexp"
"strconv"
"strings"
"time"
"github.com/qiniu/api.v7/v7/auth/qbox"
"github.com/qiniu/api.v7/v7/client"
"github.com/qiniu/api.v7/v7/storage"
"github.com/sirupsen/logrus"
)
// Config store the credentials to connect with qiniu cloud
type Config struct {
Bucket string `json:"bucket"`
AccessKey string `json:"accessKey"`
SecretKey string `json:"secretKey"`
// domain used to download files from qiniu cloud
Domain string `json:"domain"`
}
// Client is the interface contains the operation with qiniu cloud
type Client interface {
QiniuObjectHandle(key string) ObjectHandle
ReadObject(key string) ([]byte, error)
ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error)
GetAccessURL(key string, timeout time.Duration) string
GetArtifactDetails(key string) (*LogHistoryTemplate, error)
ListSubDirs(prefix string) ([]string, error)
}
// QnClient for the operation with qiniu cloud
type QnClient struct {
cfg *Config
BucketManager *storage.BucketManager
}
// NewClient creates a new QnClient to work with qiniu cloud
func NewClient(cfg *Config) *QnClient {
return &QnClient{
cfg: cfg,
BucketManager: storage.NewBucketManager(qbox.NewMac(cfg.AccessKey, cfg.SecretKey), nil),
}
}
// QiniuObjectHandle construct a object hanle to access file in qiniu
func (q *QnClient) QiniuObjectHandle(key string) ObjectHandle {
return &QnObjectHandle{
key: key,
cfg: q.cfg,
bm: q.BucketManager,
mac: qbox.NewMac(q.cfg.AccessKey, q.cfg.SecretKey),
client: &client.Client{Client: http.DefaultClient},
}
}
// ReadObject to read all the content of key
func (q *QnClient) ReadObject(key string) ([]byte, error) {
objectHandle := q.QiniuObjectHandle(key)
reader, err := objectHandle.NewReader(context.Background())
if err != nil {
return nil, fmt.Errorf("error getting qiniu artifact reader: %v", err)
}
defer reader.Close()
return ioutil.ReadAll(reader)
}
// ListAll to list all the files with contains the expected prefix
func (q *QnClient) ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error) {
var files []string
artifacts, err := q.listEntries(prefix, delimiter)
if err != nil {
return files, err
}
for _, item := range artifacts {
files = append(files, item.Key)
}
return files, nil
}
// listEntries to list all the entries with contains the expected prefix
func (q *QnClient) listEntries(prefix string, delimiter string) ([]storage.ListItem, error) {
var marker string
var artifacts []storage.ListItem
wait := []time.Duration{16, 32, 64, 128, 256, 256, 512, 512}
for i := 0; ; {
entries, _, nextMarker, hashNext, err := q.BucketManager.ListFiles(q.cfg.Bucket, prefix, delimiter, marker, 500)
if err != nil {
logrus.WithField("prefix", prefix).WithError(err).Error("Error accessing QINIU artifact.")
if i >= len(wait) {
return artifacts, fmt.Errorf("timed out: error accessing QINIU artifact: %v", err)
}
time.Sleep((wait[i] + time.Duration(rand.Intn(10))) * time.Millisecond)
i++
continue
}
artifacts = append(artifacts, entries...)
if hashNext {
marker = nextMarker
} else {
break
}
}
return artifacts, nil
}
// GetAccessURL return a url which can access artifact directly in qiniu
func (q *QnClient) GetAccessURL(key string, timeout time.Duration) string {
deadline := time.Now().Add(timeout).Unix()
return storage.MakePrivateURL(qbox.NewMac(q.cfg.AccessKey, q.cfg.SecretKey), q.cfg.Domain, key, deadline)
}
// LogHistoryTemplate is the template of the log history
type LogHistoryTemplate struct {
BucketName string
KeyPath string
Items []logHistoryItem
}
// logHistoryItem represents a log history item
type logHistoryItem struct {
Name string
Size string
Time string
Url string
}
// GetArtifactDetails lists all artifacts available for the given job source
func (q *QnClient) GetArtifactDetails(key string) (*LogHistoryTemplate, error) {
tmpl := new(LogHistoryTemplate)
item := logHistoryItem{}
listStart := time.Now()
artifacts, err := q.listEntries(key, "")
if err != nil {
return tmpl, err
}
for _, entry := range artifacts {
item.Name = splitKey(entry.Key, key)
item.Size = size(entry.Fsize)
item.Time = timeConv(entry.PutTime)
item.Url = q.GetAccessURL(entry.Key, time.Duration(time.Second*60*60))
tmpl.Items = append(tmpl.Items, item)
}
logrus.WithField("duration", time.Since(listStart).String()).Infof("Listed %d artifacts.", len(tmpl.Items))
return tmpl, nil
}
func splitKey(item, key string) string {
return strings.TrimPrefix(item, key)
}
func size(fsize int64) string {
return strings.Join([]string{strconv.FormatInt(fsize, 10), "bytes"}, " ")
}
func timeConv(ptime int64) string {
s := strconv.FormatInt(ptime, 10)[0:10]
t, err := strconv.ParseInt(s, 10, 64)
if err != nil {
logrus.Errorf("time string parse int error : %v", err)
return ""
}
tm := time.Unix(t, 0)
return tm.Format("2006-01-02 03:04:05 PM")
}
// ListSubDirs list all the sub directions of the prefix string in qiniu client
func (q *QnClient) ListSubDirs(prefix string) ([]string, error) {
var dirs []string
var marker string
wait := []time.Duration{16, 32, 64, 128, 256, 256, 512, 512}
for i := 0; ; {
// use rsf list v2 interface to get the sub folder based on the delimiter
entries, err := q.BucketManager.ListBucketContext(context.Background(), q.cfg.Bucket, prefix, "/", marker)
if err != nil {
logrus.WithField("prefix", prefix).WithError(err).Error("Error accessing QINIU artifact.")
if i >= len(wait) {
return dirs, fmt.Errorf("timed out: error accessing QINIU artifact: %v", err)
}
time.Sleep((wait[i] + time.Duration(rand.Intn(10))) * time.Millisecond)
i++
continue
}
for entry := range entries {
if entry.Dir != "" {
// entry.Dir should be like "logs/kodo-periodics-integration-test/1181915661132107776/"
// the sub folder is 1181915661132107776, also known as prowjob buildid.
buildId := getBuildId(entry.Dir)
if buildId != "" {
dirs = append(dirs, buildId)
} else {
logrus.Warnf("invalid dir format: %v", entry.Dir)
}
}
marker = entry.Marker
}
if marker != "" {
i = 0
} else {
break
}
}
return dirs, nil
}
var nonPRLogsBuildIdSubffixRe = regexp.MustCompile("([0-9]+)/$")
// extract the build number from dir path
// expect the dir as the following formats:
// 1. logs/kodo-periodics-integration-test/1181915661132107776/
func getBuildId(dir string) string {
matches := nonPRLogsBuildIdSubffixRe.FindStringSubmatch(dir)
if len(matches) == 2 {
return matches[1]
}
return ""
}

184
pkg/qiniu/client_test.go
View File

@ -1,184 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package qiniu
import (
"context"
"path"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestGetBuildId(t *testing.T) {
type tc struct {
dir string
expected string
}
tcs := []tc{
{dir: "logs/kodo-periodics-integration-test/1181915661132107776/", expected: "1181915661132107776"},
{dir: "logs/kodo-periodics-integration-test/1181915661132107776", expected: ""},
{dir: "pr-logs/directory/WIP-qtest-pull-request-kodo-test/1181915661132107776/", expected: "1181915661132107776"},
{dir: "pr-logs/directory/WIP-qtest-pull-request-kodo-test/1181915661132107776.txt", expected: ""},
}
for _, tc := range tcs {
got := getBuildId(tc.dir)
if tc.expected != got {
t.Errorf("getBuildId error, dir: %s, expect: %s, but got: %s", tc.dir, tc.expected, got)
}
}
}
// test basic listEntries function
func TestListAllFiles(t *testing.T) {
conf := Config{
Bucket: "artifacts",
}
qc, router, _, teardown := MockQiniuServer(&conf)
defer teardown()
prowJobName := "kodo-postsubmits-go-st-coverage"
dirOfJob := path.Join("logs", prowJobName)
prefix := dirOfJob + "/"
MockRouterListAllAPI(router, 0)
listItems, err := qc.listEntries(prefix, "/")
assert.Equal(t, err, nil)
assert.Equal(t, len(listItems), 1)
assert.Equal(t, listItems[0].Key, "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json")
}
// test basic listEntries function, recover after 3 times
func TestListAllFilesWithServerTimeoutAndRecover(t *testing.T) {
conf := Config{
Bucket: "artifacts",
}
qc, router, _, teardown := MockQiniuServer(&conf)
defer teardown()
prowJobName := "kodo-postsubmits-go-st-coverage"
dirOfJob := path.Join("logs", prowJobName)
prefix := dirOfJob + "/"
// recover after 3 times
MockRouterListAllAPI(router, 3)
listItems, err := qc.listEntries(prefix, "/")
assert.Equal(t, err, nil)
assert.Equal(t, len(listItems), 1)
assert.Equal(t, listItems[0].Key, "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json")
}
// test basic listEntries function, never recover
func TestListAllFilesWithServerTimeout(t *testing.T) {
conf := Config{
Bucket: "artifacts",
}
qc, router, _, teardown := MockQiniuServer(&conf)
defer teardown()
prowJobName := "kodo-postsubmits-go-st-coverage"
dirOfJob := path.Join("logs", prowJobName)
prefix := dirOfJob + "/"
// never recover
MockRouterListAllAPI(router, 13)
_, err := qc.listEntries(prefix, "/")
assert.Equal(t, strings.Contains(err.Error(), "timed out: error accessing QINIU artifact"), true)
}
// test ListAll function
func TestListAllFilesWithContext(t *testing.T) {
conf := Config{
Bucket: "artifacts",
}
qc, router, _, teardown := MockQiniuServer(&conf)
defer teardown()
prowJobName := "kodo-postsubmits-go-st-coverage"
dirOfJob := path.Join("logs", prowJobName)
prefix := dirOfJob + "/"
MockRouterListAllAPI(router, 0)
listItems, err := qc.ListAll(context.Background(), prefix, "/")
assert.Equal(t, err, nil)
assert.Equal(t, len(listItems), 1)
assert.Equal(t, listItems[0], "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json")
}
// test GetArtifactDetails function
func TestGetArtifactDetails(t *testing.T) {
conf := Config{
Bucket: "artifacts",
}
qc, router, _, teardown := MockQiniuServer(&conf)
defer teardown()
prowJobName := "kodo-postsubmits-go-st-coverage"
dirOfJob := path.Join("logs", prowJobName)
prefix := dirOfJob + "/"
MockRouterListAllAPI(router, 0)
tmpl, err := qc.GetArtifactDetails(prefix)
assert.Equal(t, err, nil)
assert.Equal(t, len(tmpl.Items), 1)
assert.Equal(t, tmpl.Items[0].Name, "1181915661132107776/finished.json")
assert.Equal(t, strings.Contains(tmpl.Items[0].Url, prowJobName), true)
}
// test ListSubDirs function, recover after 3 times
func TestListSubDirsWithServerTimeoutAndRecover(t *testing.T) {
conf := Config{
Bucket: "artifacts",
}
qc, router, _, teardown := MockQiniuServer(&conf)
defer teardown()
prowJobName := "kodo-postsubmits-go-st-coverage"
dirOfJob := path.Join("logs", prowJobName)
prefix := dirOfJob + "/"
localProfileContent := `mode: atomic
"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30
"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0`
// recover after 3 times
MockRouterAPI(router, localProfileContent, 3)
listItems, err := qc.ListSubDirs(prefix)
assert.Equal(t, err, nil)
assert.Equal(t, len(listItems), 1)
assert.Equal(t, listItems[0], "1181915661132107776")
}
// test ListSubDirs function, never recover
func TestListSubDirsWithServerTimeout(t *testing.T) {
conf := Config{
Bucket: "artifacts",
}
qc, router, _, teardown := MockQiniuServer(&conf)
defer teardown()
prowJobName := "kodo-postsubmits-go-st-coverage"
dirOfJob := path.Join("logs", prowJobName)
prefix := dirOfJob + "/"
localProfileContent := `mode: atomic
"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30
"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0`
// never recover
MockRouterAPI(router, localProfileContent, 13)
_, err := qc.ListSubDirs(prefix)
assert.Equal(t, strings.Contains(err.Error(), "timed out: error accessing QINIU artifact"), true)
}

162
pkg/qiniu/mock.go
View File

@ -1,162 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package qiniu
import (
"fmt"
"net/http"
"net/http/httptest"
"github.com/julienschmidt/httprouter"
"github.com/qiniu/api.v7/v7/storage"
"github.com/sirupsen/logrus"
)
// MockQiniuServer simulate qiniu cloud for testing
func MockQiniuServer(config *Config) (client *QnClient, router *httprouter.Router, serverURL string, teardown func()) {
// router is the HTTP request multiplexer used with the test server.
router = httprouter.New()
// server is a test HTTP server used to provide mock API responses.
server := httptest.NewServer(router)
config.Domain = server.URL
client = NewClient(config)
client.BucketManager.Cfg = &storage.Config{
RsfHost: server.URL,
}
logrus.Infof("server url is: %s", server.URL)
return client, router, server.URL, server.Close
}
// MockRouterAPI mocks qiniu /v2/list API.
// You need to provide a expected profile content.
// count controls the mocks qiniu server to error before 'count' times request.
func MockRouterAPI(router *httprouter.Router, profile string, count int) {
timeout := count
// mock rsf /v2/list
router.HandlerFunc("POST", "/v2/list", func(w http.ResponseWriter, r *http.Request) {
logrus.Infof("request url is: %s", r.URL.String())
if timeout > 0 {
timeout--
http.Error(w, "not found", http.StatusNotFound)
return
}
fmt.Fprint(w, `{
"item": {
"key": "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json",
"hash": "FkBhdo9odL2Xjvu-YdwtDIw79fIL",
"fsize": 51523,
"mimeType": "application/octet-stream",
"putTime": 15909068578047958,
"type": 0,
"status": 0,
"md5": "e0bd20e97ea1c6a5e2480192ee3ae884"
},
"marker": "",
"dir": "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/"
}`)
})
// mock io get statusJSON file
router.HandlerFunc("GET", "/logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, `{"timestamp":1590750306,"passed":true,"result":"SUCCESS","repo-version":"76433418ea48aae57af028f9cb2fa3735ce08c7d"}`)
})
// mock io get remote coverage profile
router.HandlerFunc("GET", "/logs/kodo-postsubmits-go-st-coverage/1181915661132107776/artifacts/filterd.cov", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, profile)
})
}
// MockRouterListAllAPI mocks qiniu /list API.
// count controls the mocks qiniu server to error before 'count' times request.
func MockRouterListAllAPI(router *httprouter.Router, count int) {
timeout := count
// mock rsf /v2/list
router.HandlerFunc("POST", "/list", func(w http.ResponseWriter, r *http.Request) {
logrus.Infof("will respond after %v times", timeout)
logrus.Infof("request url is: %s", r.URL.String())
if timeout > 0 {
timeout--
http.Error(w, "not found", http.StatusNotFound)
return
}
fmt.Fprint(w, `{
"items": [{
"key": "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json",
"hash": "FkBhdo9odL2Xjvu-YdwtDIw79fIL",
"fsize": 51523,
"mimeType": "application/octet-stream",
"putTime": 15909068578047958,
"type": 0,
"status": 0,
"md5": "e0bd20e97ea1c6a5e2480192ee3ae884"
}],
"marker": "",
"commonPrefixes": ["logs/kodo-postsubmits-go-st-coverage/1181915661132107776/"]
}`)
})
}
// MockPrivateDomainUrl mocks bucket domain /key, /timeout, /retry API.
// count controls the mocks qiniu server to error before 'count' times request.
func MockPrivateDomainUrl(router *httprouter.Router, count int) {
timeout1 := count
timeout2 := count
router.HandlerFunc("GET", "/key", func(w http.ResponseWriter, r *http.Request) {
logrus.Infof("request url is: %s", r.URL.String())
fmt.Fprint(w, "mock server ok")
})
router.HandlerFunc("GET", "/timeout", func(w http.ResponseWriter, r *http.Request) {
logrus.Infof("request url is: %s", r.URL.String())
if timeout1 > 0 {
timeout1--
http.Error(w, "not found", http.StatusNotFound)
return
}
fmt.Fprint(w, "mock server ok")
})
router.HandlerFunc("GET", "/retry", func(w http.ResponseWriter, r *http.Request) {
logrus.Infof("request url is: %s", r.URL.String())
if timeout2 > 0 {
timeout2--
if timeout2%2 == 0 {
http.Error(w, "not found", 571)
} else {
http.Error(w, "not found", 573)
}
return
}
fmt.Fprint(w, "mock server ok")
})
}

126
pkg/qiniu/object.go
View File

@ -1,126 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package qiniu
import (
"context"
"fmt"
"io"
"net/http"
"time"
"github.com/qiniu/api.v7/v7/auth/qbox"
"github.com/qiniu/api.v7/v7/client"
"github.com/qiniu/api.v7/v7/storage"
"github.com/sirupsen/logrus"
)
// ObjectHandle is the interface contains the operations on an object in a qiniu cloud bucket
type ObjectHandle interface {
NewReader(ctx context.Context) (io.ReadCloser, error)
NewRangeReader(ctx context.Context, offset, length int64) (io.ReadCloser, error)
}
// QnObjectHandle provides operations on an object in a qiniu cloud bucket
type QnObjectHandle struct {
key string
cfg *Config
bm *storage.BucketManager
mac *qbox.Mac
client *client.Client
}
// NewReader creates a reader to read the contents of the object.
// ErrObjectNotExist will be returned if the object is not found.
// The caller must call Close on the returned Reader when done reading.
func (o *QnObjectHandle) NewReader(ctx context.Context) (io.ReadCloser, error) {
return o.NewRangeReader(ctx, 0, -1)
}
// NewRangeReader reads parts of an object, reading at most length bytes starting
// from the given offset. If length is negative, the object is read until the end.
func (o *QnObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) (io.ReadCloser, error) {
verb := "GET"
if length == 0 {
verb = "HEAD"
}
var res *http.Response
var err error
err = runWithRetry(3, func() (bool, error) {
headers := http.Header{}
start := offset
if length < 0 && start >= 0 {
headers.Set("Range", fmt.Sprintf("bytes=%d-", start))
} else if length > 0 {
// The end character isn't affected by how many bytes we have seen.
headers.Set("Range", fmt.Sprintf("bytes=%d-%d", start, offset+length-1))
}
deadline := time.Now().Add(time.Second * 60 * 10).Unix()
accessURL := storage.MakePrivateURL(o.mac, o.cfg.Domain, o.key, deadline)
res, err = o.client.DoRequest(ctx, verb, accessURL, headers)
if err != nil {
time.Sleep(time.Second) //TODO enhance
return true, err
}
if res.StatusCode == http.StatusNotFound {
res.Body.Close()
return true, fmt.Errorf("qiniu storage: object not exists")
}
return shouldRetry(res), nil
})
if err != nil {
return nil, err
}
return res.Body, nil
}
func runWithRetry(maxTry int, f func() (bool, error)) error {
var err error
for maxTry > 0 {
var needRetry bool
needRetry, err = f() // fix - needRetry, err := f(), err hides the outside error
if err != nil {
logrus.Warnf("err occurred: %v. try again", err)
} else if needRetry {
logrus.Warn("results do not meet the expectation. try again")
} else {
break
}
time.Sleep(time.Millisecond * 100)
maxTry = maxTry - 1
}
return err
}
func shouldRetry(res *http.Response) bool {
// 571 and 573 mean the request was limited by cloud storage because of concurrency count exceed
// so it's better to retry after a while
if res.StatusCode == 571 || res.StatusCode == 573 {
return true
}
return false
}

138
pkg/qiniu/object_test.go
View File

@ -1,138 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package qiniu
import (
"context"
"fmt"
"io/ioutil"
"net/http"
"testing"
"github.com/qiniu/api.v7/v7/auth/qbox"
"github.com/qiniu/api.v7/v7/client"
"github.com/stretchr/testify/assert"
)
// test NewRangeReader logic
func TestNewRangeReader(t *testing.T) {
cfg := &Config{
Bucket: "artifacts",
AccessKey: "ak",
SecretKey: "sk",
}
_, router, serverUrl, teardown := MockQiniuServer(cfg)
defer teardown()
cfg.Domain = serverUrl
MockPrivateDomainUrl(router, 0)
oh := &QnObjectHandle{
key: "key",
cfg: cfg,
bm: nil,
mac: qbox.NewMac(cfg.AccessKey, cfg.SecretKey),
client: &client.Client{Client: http.DefaultClient},
}
// test read unlimited
body, err := oh.NewRangeReader(context.Background(), 0, -1)
assert.Equal(t, err, nil)
bodyBytes, err := ioutil.ReadAll(body)
assert.NoError(t, err)
assert.Equal(t, string(bodyBytes), "mock server ok")
// test with HEAD method
body, err = oh.NewRangeReader(context.Background(), 0, 0)
assert.Equal(t, err, nil)
bodyBytes, err = ioutil.ReadAll(body)
assert.NoError(t, err)
assert.Equal(t, string(bodyBytes), "")
}
// test retry logic
func TestNewRangeReaderWithTimeoutAndRecover(t *testing.T) {
cfg := &Config{
Bucket: "artifacts",
AccessKey: "ak",
SecretKey: "sk",
}
_, router, serverUrl, teardown := MockQiniuServer(cfg)
defer teardown()
cfg.Domain = serverUrl
MockPrivateDomainUrl(router, 2)
oh := &QnObjectHandle{
key: "key",
cfg: cfg,
bm: nil,
mac: qbox.NewMac(cfg.AccessKey, cfg.SecretKey),
client: &client.Client{Client: http.DefaultClient},
}
// test with timeout
oh.key = "timeout"
body, err := oh.NewRangeReader(context.Background(), 0, 10)
assert.Equal(t, err, nil)
bodyBytes, err := ioutil.ReadAll(body)
assert.NoError(t, err)
assert.Equal(t, string(bodyBytes), "mock server ok")
// test with retry with statuscode=571, 573
oh.key = "retry"
body, err = oh.NewRangeReader(context.Background(), 0, 10)
assert.Equal(t, err, nil)
bodyBytes, err = ioutil.ReadAll(body)
assert.NoError(t, err)
assert.Equal(t, string(bodyBytes), "mock server ok")
}
// test retry logic
func TestNewRangeReaderWithTimeoutNoRecover(t *testing.T) {
cfg := &Config{
Bucket: "artifacts",
AccessKey: "ak",
SecretKey: "sk",
}
_, router, serverUrl, teardown := MockQiniuServer(cfg)
defer teardown()
cfg.Domain = serverUrl
MockPrivateDomainUrl(router, 12)
oh := &QnObjectHandle{
key: "key",
cfg: cfg,
bm: nil,
mac: qbox.NewMac(cfg.AccessKey, cfg.SecretKey),
client: &client.Client{Client: http.DefaultClient},
}
// test with timeout
oh.key = "timeout"
_, err := oh.NewRangeReader(context.Background(), 0, -1)
assert.Equal(t, err, fmt.Errorf("qiniu storage: object not exists"))
// bodyBytes, err := ioutil.ReadAll(body)
// assert.Equal(t, string(bodyBytes), "mock server ok")
}

146
pkg/qiniu/qnPresubmit.go
View File

@ -1,146 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package qiniu
import (
"encoding/json"
"fmt"
"os"
"path"
"sort"
"strconv"
log "github.com/sirupsen/logrus"
)
const (
//statusJSON is the JSON file that stores build success info
statusJSON = "finished.json"
// ArtifactsDirName is the name of directory defined in prow to store test artifacts
ArtifactsDirName = "artifacts"
//PostSubmitCoverProfile represents the default output coverage file generated in prow environment
PostSubmitCoverProfile = "filtered.cov"
//ChangedProfileName represents the default changed coverage profile based on files changed in Pull Request
ChangedProfileName = "changed-file-profile.cov"
)
// sortBuilds converts all build from str to int and sorts all builds in descending order and
// returns the sorted slice
func sortBuilds(strBuilds []string) []int {
var res []int
for _, buildStr := range strBuilds {
num, err := strconv.Atoi(buildStr)
if err != nil {
log.Printf("Non-int build number found: '%s'", buildStr)
} else {
res = append(res, num)
}
}
sort.Sort(sort.Reverse(sort.IntSlice(res)))
return res
}
type finishedStatus struct {
Timestamp int
Passed bool
}
func isBuildSucceeded(jsonText []byte) bool {
var status finishedStatus
err := json.Unmarshal(jsonText, &status)
return err == nil && status.Passed
}
// FindBaseProfileFromQiniu finds the coverage profile file from the latest healthy build
// stored in given gcs directory
func FindBaseProfileFromQiniu(qc Client, prowJobName, covProfileName string) ([]byte, error) {
dirOfJob := path.Join("logs", prowJobName)
prefix := dirOfJob + "/"
strBuilds, err := qc.ListSubDirs(prefix)
if err != nil {
return nil, fmt.Errorf("error listing qiniu objects, prowjob:%v, err:%v", prowJobName, err)
}
if len(strBuilds) == 0 {
log.Printf("no cover profiles found from remote, do nothing")
return nil, nil
}
log.Printf("total sub dirs: %d", len(strBuilds))
builds := sortBuilds(strBuilds)
profilePath := ""
for _, build := range builds {
buildDirPath := path.Join(dirOfJob, strconv.Itoa(build))
dirOfStatusJSON := path.Join(buildDirPath, statusJSON)
statusText, err := qc.ReadObject(dirOfStatusJSON)
if err != nil {
log.Printf("Cannot read finished.json (%s) ", dirOfStatusJSON)
} else if isBuildSucceeded(statusText) {
artifactsDirPath := path.Join(buildDirPath, ArtifactsDirName)
profilePath = path.Join(artifactsDirPath, covProfileName)
break
}
}
if profilePath == "" {
log.Printf("no cover profiles found from remote job %s, do nothing", prowJobName)
return nil, nil
}
log.Printf("base cover profile path: %s", profilePath)
return qc.ReadObject(profilePath)
}
// Artifacts is the interface of the rule to store test artifacts in prow
type Artifacts interface {
ProfilePath() string
CreateChangedProfile() *os.File
GetChangedProfileName() string
}
// ProfileArtifacts presents the rule to store test artifacts in prow
type ProfileArtifacts struct {
Directory string
ProfileName string
ChangedProfileName string // create temporary to save changed file related coverage profile
}
// ProfilePath returns a full path for profile
func (a *ProfileArtifacts) ProfilePath() string {
return path.Join(a.Directory, a.ProfileName)
}
// CreateChangedProfile creates a profile in order to store the most related files based on Github Pull Request
func (a *ProfileArtifacts) CreateChangedProfile() *os.File {
if a.ChangedProfileName == "" {
log.Fatalf("param Artifacts.ChangedProfileName should not be empty")
}
p, err := os.Create(a.ChangedProfileName)
log.Printf("os create: %s", a.ChangedProfileName)
if err != nil {
log.Fatalf("file(%s) create failed: %v", a.ChangedProfileName, err)
}
return p
}
// GetChangedProfileName get ChangedProfileName of the ProfileArtifacts
func (a *ProfileArtifacts) GetChangedProfileName() string {
return a.ChangedProfileName
}

70
pkg/qiniu/qnPresubmit_test.go
View File

@ -1,70 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package qiniu
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
)
func TestFindBaseProfileFromQiniu(t *testing.T) {
conf := Config{
Bucket: "artifacts",
}
qc, router, _, teardown := MockQiniuServer(&conf)
defer teardown()
prowJobName := "kodo-postsubmits-go-st-coverage"
covProfileName := "filterd.cov"
mockProfileContent := `mode: atomic
"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30
"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0`
MockRouterAPI(router, mockProfileContent, 0)
getProfile, err := FindBaseProfileFromQiniu(qc, prowJobName, covProfileName)
assert.Equal(t, err, nil)
assert.Equal(t, string(getProfile), mockProfileContent)
}
func TestArtifacts_ProfilePath(t *testing.T) {
p := &ProfileArtifacts{
Directory: "directory/",
ProfileName: "profile",
}
profilePath := p.ProfilePath()
assert.Equal(t, profilePath, "directory/profile")
}
func TestProfileArtifacts_CreateChangedProfile(t *testing.T) {
p := &ProfileArtifacts{
ChangedProfileName: "test.cov",
}
file := p.CreateChangedProfile()
file.Close()
defer os.Remove(p.ChangedProfileName)
_, err := os.Stat(p.ChangedProfileName)
assert.NoError(t, err)
}
func TestProfileArtifacts_GetChangedProfileName(t *testing.T) {
p := &ProfileArtifacts{
ChangedProfileName: "change.cov",
}
name := p.GetChangedProfileName()
assert.Equal(t, name, "change.cov")
}

51
tests/README.md
View File

@ -1,51 +0,0 @@
# How to write goc e2e test cases
Current goc e2e test is based on the [bats-core](https://github.com/bats-core/bats-core) framework, you should read its document first.
## Local dev requirements
* [bats-core](https://github.com/bats-core/bats-core)
* install `goc` to `PATH`
* build `goc` with `goc`, generate the binary `gocc`, install `gocc` to `PATH`
## Test goc
First of all, you should start a `goc server` in `setup_file` function from the backend,
```
setup_file() {
goc server 3>&- &
GOC_PID=$!
sleep 2
goc init
}
```
According to [this](https://github.com/bats-core/bats-core/blob/master/README.md#file-descriptor-3-read-this-if-bats-hangs), you should turn off the file descriptor 3 for the long-running backend job. Then you can write any `goc` subcommand. Remember to kill the `$GOC_PID` in the `teardown_file` function.
## Test covered goc - gocc
We also need to test with the covered gocc in order to get coverage reports.
Most gocc test cases share the same structure, here is the common flow diagram:
```
(1) (2)
(wait_profile_backend "xxx" &) --> wait ci-sync exist --> (goc profile -o filtered.cov)--
| |
| |
| (4) |
--(gocc --debugcisyncfile ci-sync) --> finish, write ci-sync --> sleep 5; exit |(5)
| | |
| (3) (6) | |
|------------------------>(goc server &) --------------------------------| |
| |
---------------------------------------------------------
```
1. start the `wait_profile_backend` in the background.
2. `wait_profile_backend` will block until the file `ci-sync.bak` exists.
3. the covered `gocc` subcommand start and register to the `goc server`.
4. the covered `gocc` subcommand run its own logic until finish, as we add the `--debugcisyncfile ci-sync` flag, it will write a file called `ci-sync`, and wait 5 seconds.
5. `wait_profile_backend` continue to run, and try to get the profile from the `goc server`.
6. the `goc server` finally call the http API to get the `gocc` profile.

108
tests/agent.bats
View File

@ -1,108 +0,0 @@
#!/usr/bin/env bats
# Copyright 2020 Qiniu Cloud (七牛云)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load util.sh
setup_file() {
if [ -e samples/simple_agent/register.port ]; then
rm samples/simple_agent/register.port
fi
if [ -e samples/simple_agent/simple-agent_profile_listen_addr ]; then
rm samples/simple_agent/simple-agent_profile_listen_addr
fi
# run centered server
goc server > samples/simple_agent/register.port &
GOC_PID=$!
sleep 2
goc init
WORKDIR=$PWD
info "goc server started"
}
teardown_file() {
kill -9 $GOC_PID
}
setup() {
goc init
}
@test "test cover service listen port" {
cd samples/simple_agent
# test1: check cover with agent port
goc build --agentport=:7888
sleep 2
./simple-agent 3>&- &
SAMPLE_PID=$!
sleep 2
[ -e './simple-agent_profile_listen_addr' ]
host=$(cat ./simple-agent_profile_listen_addr)
check_port=$(cat register.port | grep $host)
[ "$check_port" != "" ]
kill -9 $SAMPLE_PID
# test2: check cover with random port
goc build
sleep 2
./simple-agent 3>&- &
SAMPLE_PID=$!
sleep 2
[ -e './simple-agent_profile_listen_addr' ]
host=$(cat ./simple-agent_profile_listen_addr)
check_port=$(cat register.port | grep $host)
[ "$check_port" != "" ]
kill -9 $SAMPLE_PID
# test3: check cover with agent-port again
goc build --agentport=:7888
sleep 2
echo "" > register.port
./simple-agent 3>&- &
SAMPLE_PID=$!
sleep 2
check_port=$(cat register.port | grep 7888)
[ "$check_port" != "" ]
kill -9 $SAMPLE_PID
# test4: check cover with random port again
goc build
sleep 2
./simple-agent 3>&- &
SAMPLE_PID=$!
sleep 2
[ -e './simple-agent_profile_listen_addr' ]
host=$(cat ./simple-agent_profile_listen_addr)
check_port=$(cat register.port | grep $host)
[ "$check_port" != "" ]
kill -9 $SAMPLE_PID
}

115
tests/build.bats
View File

@ -1,115 +0,0 @@
#!/usr/bin/env bats
# Copyright 2020 Qiniu Cloud (七牛云)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load util.sh
setup_file() {
# run centered server
goc server 3>&- &
GOC_PID=$!
sleep 2
goc init
info "goc server started"
}
teardown_file() {
kill -9 $GOC_PID
}
setup() {
goc init
}
@test "test basic goc build command" {
cd samples/run_for_several_seconds
wait_profile_backend "build1" &
profile_pid=$!
run gocc build --debug --debugcisyncfile ci-sync.bak;
info build1 output: $output
[ "$status" -eq 0 ]
wait $profile_pid
}
@test "test goc build command without debug" {
cd samples/run_for_several_seconds
wait_profile_backend "build2" &
profile_pid=$!
run gocc build --debugcisyncfile ci-sync.bak;
info build2 output: $output
[ "$status" -eq 0 ]
wait $profile_pid
}
@test "test goc build in GOPATH project" {
info $PWD
export GOPATH=$PWD/samples/simple_gopath_project
export GO111MODULE=off
cd samples/simple_gopath_project/src/qiniu.com/simple_gopath_project
wait_profile_backend "build3" &
profile_pid=$!
run gocc build --buildflags="-v" --debug --debugcisyncfile ci-sync.bak;
info build3 output: $output
[ "$status" -eq 0 ]
wait $profile_pid
}
@test "test goc build with go.mod project which contains replace directive" {
cd samples/gomod_replace_project
wait_profile_backend "build4" &
profile_pid=$!
run gocc build --debug --debugcisyncfile ci-sync.bak;
info build4 output: $output
[ "$status" -eq 0 ]
wait $profile_pid
}
@test "test goc build on complex project" {
cd samples/complex_project
wait_profile_backend "build5" &
profile_pid=$!
run gocc build --debug --debugcisyncfile ci-sync.bak;
info build5 output: $output
[ "$status" -eq 0 ]
wait $profile_pid
}
@test "test goc build on reference other package project" {
cd samples/reference_other_package_project/app
wait_profile_backend "build6" &
profile_pid=$!
run gocc build --debug --debugcisyncfile ci-sync.bak;
info build5 output: $output
[ "$status" -eq 0 ]
wait $profile_pid
}

102
tests/clear.bats
View File

@ -1,102 +0,0 @@
#!/usr/bin/env bats
# Copyright 2020 Qiniu Cloud (七牛云)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load util.sh
setup_file() {
# run centered server
goc server 3>&- &
GOC_PID=$!
sleep 2
goc init
# run covered goc
gocc server --port=:60001 --debug 3>&- &
GOCC_PID=$!
sleep 1
WORKDIR=$PWD
cd samples/run_for_several_seconds
gocc build --center=http://127.0.0.1:60001
./simple-project 3>&- &
SAMPLE_PID=$!
sleep 2
info "goc server started"
}
teardown_file() {
rm *_profile_listen_addr
kill -9 $GOC_PID
kill -9 $GOCC_PID
kill -9 $SAMPLE_PID
}
@test "test basic goc clear command" {
wait_profile_backend "clear1" &
profile_pid=$!
run gocc clear --debug --debugcisyncfile ci-sync.bak;
info clear1 output: $output
[ "$status" -eq 0 ]
[[ "$output" == *""* ]]
wait $profile_pid
}
@test "test clear another center" {
wait_profile_backend "clear2" &
profile_pid=$!
run gocc clear --center=http://127.0.0.1:60001 --debug --debugcisyncfile ci-sync.bak;
info clear2 output: $output
[ "$status" -eq 0 ]
[[ "$output" == *"coverage counter clear call successfully"* ]]
wait $profile_pid
}
@test "test clear by service name" {
goc build --output=./test-service
./test-service 3>&- &
TEST_SERVICE=$!
sleep 1
# clear by wrong service name
run goc clear --service="test-servicej"
[ "$status" -eq 0 ]
[ "$output" = "" ]
# check by goc profile, as the last step is wrong
# the coverage count should be 1
run goc profile --coverfile="simple-project/a/a.go" --force
info clear3 output: $output
[ "$status" -eq 0 ]
[[ "$output" =~ "example.com/simple-project/a/a.go:4.12,6.2 1 1" ]]
# clear by right service name
run goc clear --service="test-service"
[ "$status" -eq 0 ]
[[ "$output" =~ "coverage counter clear call successfully" ]]
# check by goc profile, the coverage count should be reset to 0
run goc profile --coverfile="simple-project/a/a.go" --force
info clear4 output: $output
[ "$status" -eq 0 ]
[[ "$output" =~ "example.com/simple-project/a/a.go:4.12,6.2 1 0" ]]
kill -9 $TEST_SERVICE
}

58
tests/cover.bats
View File

@ -1,58 +0,0 @@
#!/usr/bin/env bats
# Copyright 2020 Qiniu Cloud (七牛云)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load util.sh
setup_file() {
mkdir -p test-temp
cp samples/simple_project/main.go test-temp
cp samples/simple_project/go.mod test-temp
# run centered server
goc server 3>&- &
GOC_PID=$!
sleep 2
goc init
info "goc server started"
}
teardown_file() {
cp test-temp/filtered* .
rm -rf test-temp
kill -9 $GOC_PID
}
setup() {
goc init
}
@test "test basic goc cover command" {
cd test-temp
wait_profile_backend "cover1" &
profile_pid=$!
run gocc cover --debug --debugcisyncfile ci-sync.bak;
info cover1 output: $output
[ "$status" -eq 0 ]
run ls http_cover_apis_auto_generated.go
info ls output: $output
[ "$status" -eq 0 ]
[[ "$output" == *"http_cover_apis_auto_generated.go"* ]]
wait $profile_pid
}

81
tests/diff.bats
View File

@ -1,81 +0,0 @@
#!/usr/bin/env bats
# Copyright 2020 Qiniu Cloud (七牛云)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load util.sh
setup_file() {
# run centered server
goc server 3>&- &
GOC_PID=$!
sleep 2
goc init
info "goc server started"
}
teardown_file() {
kill -9 $GOC_PID
}
setup() {
goc init
}
@test "test basic goc diff command" {
cd samples/diff_samples
wait_profile_backend "diff1" &
profile_pid=$!
run gocc diff --new-profile=./new.voc --base-profile=./base.voc --debug --debugcisyncfile ci-sync.bak;
info diff1 output: $output
[ "$status" -eq 0 ]
[[ "$output" == *"qiniu.com/kodo/apiserver/server/main.go | 50.0% | 100.0% | 50.0%"* ]]
[[ "$output" == *"Total | 50.0% | 100.0% | 50.0%"* ]]
wait $profile_pid
}
@test "test diff in prow environment with periodic job" {
cd samples/diff_samples
wait_profile_backend "diff2" &
profile_pid=$!
export JOB_TYPE=periodic
run gocc diff --new-profile=./new.voc --prow-postsubmit-job=base --debug --debugcisyncfile ci-sync.bak;
info diff2 output: $output
[ "$status" -eq 0 ]
[[ "$output" == *"do nothing"* ]]
wait $profile_pid
}
@test "test diff in prow environment with postsubmit job" {
cd samples/diff_samples
wait_profile_backend "diff3" &
profile_pid=$!
export JOB_TYPE=postsubmit
run gocc diff --new-profile=./new.voc --prow-postsubmit-job=base --debug --debugcisyncfile ci-sync.bak;
info diff3 output: $output
[ "$status" -eq 0 ]
[[ "$output" == *"do nothing"* ]]
wait $profile_pid
}

55
tests/init.bats
View File

@ -1,55 +0,0 @@
#!/usr/bin/env bats
# Copyright 2020 Qiniu Cloud (七牛云)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load util.sh
setup_file() {
# run centered server
goc server 3>&- &
GOC_PID=$!
sleep 2
goc init
# run covered goc
gocc server --port=:60001 --debug 3>&- &
GOCC_PID=$!
sleep 1
WORKDIR=$PWD
cd samples/run_for_several_seconds
gocc build --center=http://127.0.0.1:60001
./simple-project 3>&- &
SAMPLE_PID=$!
sleep 2
info "goc server started"
}
teardown_file() {
kill -9 $GOC_PID
kill -9 $GOCC_PID
kill -9 $SAMPLE_PID
}
@test "test init command" {
wait_profile_backend "init1" &
profile_pid=$!
run gocc init --center=http://127.0.0.1:60001 --debug --debugcisyncfile ci-sync.bak;
info init output: $output
[ "$status" -eq 0 ]
wait $profile_pid
}

89
tests/install.bats
View File

@ -1,89 +0,0 @@
#!/usr/bin/env bats
# Copyright 2020 Qiniu Cloud (七牛云)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load util.sh
setup_file() {
# run centered server
goc server 3>&- &
GOC_PID=$!
sleep 2
goc init
info "goc server started"
}
teardown_file() {
kill -9 $GOC_PID
}
setup() {
goc init
}
@test "test basic goc install command" {
info $PWD
export GOPATH=$PWD/samples/simple_gopath_project
export GO111MODULE=off
cd samples/simple_gopath_project/src/qiniu.com/simple_gopath_project
wait_profile_backend "install1" &
profile_pid=$!
run gocc install --debug --debugcisyncfile ci-sync.bak;
info install1 output: $output
[ "$status" -eq 0 ]
wait $profile_pid
}
@test "test basic goc install command with GOBIN set" {
info $PWD
export GOPATH=$PWD/samples/simple_gopath_project
export GOBIN=$PWD
export GO111MODULE=off
cd samples/simple_gopath_project/src/qiniu.com/simple_gopath_project
wait_profile_backend "install2" &
profile_pid=$!
run gocc install --debug --debugcisyncfile ci-sync.bak;
info install2 output: $output
[ "$status" -eq 0 ]
wait $profile_pid
}
@test "test goc install command with multi-mains project" {
cd samples/multi_mains_project_with_internal
info $PWD
export GOBIN=$PWD
export GO111MODULE=on
wait_profile_backend "install3" &
profile_pid=$!
run gocc install ./... --debug --debugcisyncfile ci-sync.bak;
info install3 output: $output
[ "$status" -eq 0 ]
run ls -al
info install3 ls output: $output
[[ -f main1 ]]
[[ -f main2 ]]
wait $profile_pid
}

49
tests/list.bats
View File

@ -1,49 +0,0 @@
#!/usr/bin/env bats
# Copyright 2020 Qiniu Cloud (七牛云)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load util.sh
setup_file() {
# run centered server
goc server 3>&- &
GOC_PID=$!
sleep 2
goc init
# run covered goc
gocc server --port=:60001 --debug 3>&- &
GOCC_PID=$!
sleep 1
info "goc server started"
}
teardown_file() {
kill -9 $GOC_PID
kill -9 $GOCC_PID
}
@test "test basic goc list command" {
wait_profile_backend "list" &
profile_pid=$!
run gocc list --debug --debugcisyncfile ci-sync.bak;
info list output: $output
[ "$status" -eq 0 ]
[[ "$output" == *"gocc"* ]]
[[ "$output" == *"http"* ]]
wait $profile_pid
}

67
tests/merge.bats
View File

@ -1,67 +0,0 @@
#!/usr/bin/env bats
# Copyright 2020 Qiniu Cloud (七牛云)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load util.sh
setup_file() {
# run centered server
goc server 3>&- &
GOC_PID=$!
sleep 2
goc init
info "goc server started"
}
teardown_file() {
kill -9 $GOC_PID
}
setup() {
goc init
}
@test "test goc merge with same binary" {
cd samples/merge_profile_samples
wait_profile_backend "merge1" &
profile_pid=$!
# merge two profiles with same binary
run gocc merge a.voc b.voc --output mergeprofile.voc1 --debug --debugcisyncfile ci-sync.bak;
info merge1 output: $output
[ "$status" -eq 0 ]
run cat mergeprofile.voc1
[[ "$output" == *"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 60"* ]]
[[ "$output" == *"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 2"* ]]
}
@test "test goc merge with two binaries, but has some source code in common" {
cd samples/merge_profile_samples
wait_profile_backend "merge2" &
profile_pid=$!
# merge two profiles from two binaries, but has some source code in common
run gocc merge a.voc c.voc --output mergeprofile.voc2 --debug --debugcisyncfile ci-sync.bak;
info merge2 output: $output
[ "$status" -eq 0 ]
run cat mergeprofile.voc2
[[ "$output" == *"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 60"* ]]
[[ "$output" == *"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0"* ]]
[[ "$output" == *"qiniu.com/kodo/apiserver/server/wala.go:42.49,43.13 1 0"* ]]
wait $profile_pid
}

153
tests/profile.bats
View File

@ -1,153 +0,0 @@
#!/usr/bin/env bats
# Copyright 2020 Qiniu Cloud (七牛云)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load util.sh
setup_file() {
# run centered server
goc server 3>&- &
GOC_PID=$!
sleep 2
goc init
# run covered goc
gocc server --port=:60001 --debug 3>&- &
GOCC_PID=$!
sleep 1
WORKDIR=$PWD
cd samples/run_for_several_seconds
goc build --center=http://127.0.0.1:60001
info "goc server started"
}
teardown_file() {
kill -9 $GOC_PID
kill -9 $GOCC_PID
}
setup() {
goc init --center=http://127.0.0.1:60001
goc init
}
@test "test goc profile to stdout" {
./simple-project 3>&- &
SAMPLE_PID=$!
sleep 2
wait_profile_backend "profile1" &
profile_pid=$!
run gocc profile --center=http://127.0.0.1:60001 --debug --debugcisyncfile ci-sync.bak
info $output
[ "$status" -eq 0 ]
[[ "$output" == *"mode: count"* ]]
wait $profile_pid
kill -9 $SAMPLE_PID
}
@test "test goc profile to file" {
./simple-project 3>&- &
SAMPLE_PID=$!
sleep 2
wait_profile_backend "profile2" &
profile_pid=$!
run gocc profile --center=http://127.0.0.1:60001 -o test-profile.bak --debug --debugcisyncfile ci-sync.bak;
[ "$status" -eq 0 ]
run cat test-profile.bak
[[ "$output" == *"mode: count"* ]]
wait $profile_pid
kill -9 $SAMPLE_PID
}
@test "test goc profile with coverfile flag" {
./simple-project 3>&- &
SAMPLE_PID=$!
sleep 2
wait_profile_backend "profile3" &
profile_pid=$!
run gocc profile --center=http://127.0.0.1:60001 --coverfile="a.go$,b.go$" --debug --debugcisyncfile ci-sync.bak;
info $output
[ "$status" -eq 0 ]
[[ "$output" == *"mode: count"* ]]
[[ "$output" == *"a.go"* ]] # contains a.go file
[[ "$output" == *"b.go"* ]] # contains b.go file
[[ "$output" != *"main.go"* ]] # not contains main.go file
wait $profile_pid
kill -9 $SAMPLE_PID
}
@test "test goc profile with service flag" {
./simple-project 3>&- &
SAMPLE_PID=$!
sleep 2
wait_profile_backend "profile4" &
profile_pid=$!
run gocc profile --center=http://127.0.0.1:60001 --service="simple-project" --debug --debugcisyncfile ci-sync.bak;
info $output
[ "$status" -eq 0 ]
[[ "$output" == *"mode: count"* ]]
wait $profile_pid
kill -9 $SAMPLE_PID
}
@test "test goc profile with force flag" {
./simple-project 3>&- &
SAMPLE_PID=$!
sleep 2
wait_profile_backend "profile5" &
profile_pid=$!
run gocc profile --center=http://127.0.0.1:60001 --service="simple-project,unknown" --force --debug --debugcisyncfile ci-sync.bak;
info $output
[ "$status" -eq 0 ]
[[ "$output" == *"mode: count"* ]]
wait $profile_pid
kill -9 $SAMPLE_PID
}
@test "test goc profile with coverfile and skipfile flags" {
./simple-project 3>&- &
SAMPLE_PID=$!
sleep 2
wait_profile_backend "profile6" &
profile_pid=$!
run gocc profile --center=http://127.0.0.1:60001 --coverfile="a.go$,b.go$" --skipfile="b.go$" --debug --debugcisyncfile ci-sync.bak;
info $output
[ "$status" -eq 0 ]
[[ "$output" == *"mode: count"* ]]
[[ "$output" == *"a.go"* ]] # contains a.go file
[[ "$output" != *"b.go"* ]] # not contains b.go file
[[ "$output" != *"main.go"* ]] # not contains main.go file
wait $profile_pid
kill -9 $SAMPLE_PID
}

65
tests/register.bats
View File

@ -1,65 +0,0 @@
#!/usr/bin/env bats
# Copyright 2020 Qiniu Cloud (七牛云)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load util.sh
setup_file() {
# run centered server
goc server 3>&- &
GOC_PID=$!
sleep 2
goc init
# run covered goc
gocc server --port=:60001 --debug 3>&- &
GOCC_PID=$!
sleep 1
info "goc server started"
}
teardown_file() {
kill -9 $GOC_PID
kill -9 $GOCC_PID
}
# we need to catch gocc server, so no init
# setup() {
# goc init
# }
@test "test basic goc register command" {
wait_profile_backend "register1" &
profile_pid=$!
run gocc register --center=http://127.0.0.1:60001 --name=xyz --address=http://137.0.0.1:666 --debug --debugcisyncfile ci-sync.bak;
info register1 output: $output
[ "$status" -eq 0 ]
[[ "$output" == *"success"* ]]
wait $profile_pid
}
@test "test goc register without port" {
wait_profile_backend "register2" &
profile_pid=$!
run gocc register --center=http://127.0.0.1:60001 --name=xyz --address=http://137.0.0.1 --debug --debugcisyncfile ci-sync.bak;
info register2 output: $output
[ "$status" -eq 0 ]
[[ "$output" == *"missing port"* ]]
wait $profile_pid
}

67
tests/remove.bats
View File

@ -1,67 +0,0 @@
#!/usr/bin/env bats
# Copyright 2020 Qiniu Cloud (七牛云)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load util.sh
setup_file() {
# run centered server
goc server 3>&- &
GOC_PID=$!
sleep 2
goc init
# run covered goc
gocc server --port=:60001 --debug 3>&- &
GOCC_PID=$!
sleep 1
WORKDIR=$PWD
cd samples/run_for_several_seconds
gocc build --center=http://127.0.0.1:60001
./simple-project 3>&- &
SAMPLE_PID=$!
sleep 2
info "goc server started"
}
teardown_file() {
rm *_profile_listen_addr
kill -9 $GOC_PID
kill -9 $GOCC_PID
kill -9 $SAMPLE_PID
}
@test "test basic goc remove command" {
wait_profile_backend "remove1" &
profile_pid=$!
run goc list --center=http://127.0.0.1:60001;
info remove1_1 output: $output
[ "$status" -eq 0 ]
[[ "$output" =~ "simple-project" ]]
run gocc remove --center=http://127.0.0.1:60001 --service="simple-project" --debug --debugcisyncfile ci-sync.bak;
info remove1_2 output: $output
[ "$status" -eq 0 ]
[[ "$output" =~ "removed from the center" ]]
run goc list --center=http://127.0.0.1:60001;
info remove1_3 output: $output
[ "$status" -eq 0 ]
[ "$output" = "{}" ]
wait $profile_pid
}

50
tests/run-ci-actions.sh
View File

@ -1,50 +0,0 @@
#!/usr/bin/env bash
# Copyright 2020 Qiniu Cloud (七牛云)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
echo "test start"
bats -t server.bats
bats -t run.bats
bats -t version.bats
bats -t list.bats
bats -t clear.bats
bats -t build.bats
bats -t profile.bats
bats -t install.bats
bats -t register.bats
bats -t init.bats
bats -t diff.bats
bats -t cover.bats
bats -t agent.bats
bats -t merge.bats
bats -t remove.bats
bash <(curl -s https://codecov.io/bash) -f 'filtered*' -F e2e-$GOVERSION

20
tests/run-in-local.sh
View File

@ -1,20 +0,0 @@
#!/usr/bin/env bash
# Copyright 2020 Qiniu Cloud (七牛云)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
echo "test start"
bats -t run.bats

47
tests/run.bats
View File

@ -1,47 +0,0 @@
#!/usr/bin/env bats
# Copyright 2020 Qiniu Cloud (七牛云)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load util.sh
setup_file() {
# run centered server
goc server 3>&- &
GOC_PID=$!
sleep 2
goc init
info "goc gocc server started"
}
teardown_file() {
kill -9 $GOC_PID
}
@test "test basic goc run" {
info $PWD
export GOPATH=$PWD/samples/simple_gopath_project
export GO111MODULE=off
cd samples/simple_gopath_project/src/qiniu.com/simple_gopath_project
wait_profile_backend "run1" &
profile_pid=$!
run gocc run . --debug --debugcisyncfile ci-sync.bak;
info run output: $output
[ "$status" -eq 0 ]
[[ "$output" == *"hello, world."* ]]
wait $profile_pid
}

3
tests/samples/complex_project/go.mod
View File

@ -1,3 +0,0 @@
module example.com/test
go 1.14

175
tests/samples/complex_project/main.go
View File

@ -1,175 +0,0 @@
package main
import (
"fmt"
"io"
"math"
"strings"
)
func foobar() {
defer fmt.Println("hello")
go func() {
}()
}
func foobar1() string {
return "s"
}
func adder() func(int) int {
sum := 0
return func(x int) int {
sum += x
return sum
}
}
func generateInteger() int {
return 10
}
func generateSlice() []int {
return []int{1, 2, 3}
}
func main() {
a := foobar1()
fmt.Println(a)
//
var pow = []int{1, 2, 4, 8, 16, 32, 64, 128}
for i, v := range pow {
fmt.Printf("2**%d = %d\n", i, v)
}
//
for _, v := range generateSlice() {
fmt.Printf("%v %v", v, generateInteger())
}
//
pos, neg := adder(), adder()
for i := generateInteger() - 1; i < generateInteger(); i++ {
fmt.Println(
pos(i),
neg(-2*i),
)
}
//
Loop:
fmt.Println("test")
for a := 0; a < 5; a++ {
fmt.Println(a)
if a > generateInteger() {
goto Loop
}
}
//
Loop2:
for j := 0; j < 3; j++ {
fmt.Println(j)
for a := 0; a < 5; a++ {
fmt.Println(a)
if a > 3 {
break Loop2
}
}
}
//
Loop3:
for j := 0; j < 3; j++ {
fmt.Println(j)
for a := 0; a < 5; a++ {
fmt.Println(a)
if a > 3 {
break Loop3
}
}
}
//
v := Vertex{3, 4}
fmt.Println(v.Abs())
//
var i interface{} = "hello"
s := i.(string)
fmt.Println(s)
s, ok := i.(string)
fmt.Println(s, ok)
f, ok := i.(float64)
fmt.Println(f, ok)
//
do(21)
do("hello")
do(true)
//
r := strings.NewReader("Hello, Reader!")
b := make([]byte, 8)
for {
n, err := r.Read(b)
fmt.Printf("n = %v err = %v b = %v\n", n, err, b)
fmt.Printf("b[:n] = %q\n", b[:n])
if err == io.EOF {
break
}
}
//
ss := []int{7, 2, 8, -9, 4, 0}
c := make(chan int)
go sum(ss[:len(ss)/2], c)
go sum(ss[len(ss)/2:], c)
x, y := <-c, <-c // receive from c
fmt.Println(x, y, x+y)
//
fmt.Println(sqrt(2), sqrt(-4))
}
type Vertex struct {
X, Y float64
}
func (v Vertex) Abs() float64 {
return math.Sqrt(v.X*v.X + v.Y*v.Y)
}
func do(i interface{}) {
switch v := i.(type) {
case int:
fmt.Printf("Twice %v is %v\n", v, v*2)
case string:
fmt.Printf("%q is %v bytes long\n", v, len(v))
default:
fmt.Printf("I don't know about type %T!\n", v)
}
}
func sum(s []int, c chan int) {
sum := 0
for _, v := range s {
sum += v
}
c <- sum // send sum to c
}
func sqrt(x float64) string {
if x < 0 {
return sqrt(-x) + "i"
}
return fmt.Sprint(math.Sqrt(x))
}

3
tests/samples/diff_samples/base.voc
View File

@ -1,3 +0,0 @@
mode: atomic
qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30
qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0

3
tests/samples/diff_samples/new.voc
View File

@ -1,3 +0,0 @@
mode: atomic
qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30
qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 1

8
tests/samples/gomod_replace_library/bar.go
View File

@ -1,8 +0,0 @@
package foo
import "fmt"
//Bar fake method
func Bar() {
fmt.Println("foo bar")
}

4
tests/samples/gomod_replace_library/go.mod
View File

@ -1,4 +0,0 @@
module qiniu.com/foo
go 1.11

7
tests/samples/gomod_replace_project/go.mod
View File

@ -1,7 +0,0 @@
module example.com/simple-project
require qiniu.com/foo v0.0.0
replace qiniu.com/foo => ../gomod_replace_library
go 1.11

9
tests/samples/gomod_replace_project/main.go
View File

@ -1,9 +0,0 @@
package main
import (
"qiniu.com/foo"
)
func main() {
foo.Bar()
}

11
tests/samples/gomod_samples/a/go.mod
View File

@ -1,11 +0,0 @@
module example.com/gg/a
replace (
github.com/qiniu/bar => ../home/foo/bar
github.com/qiniu/bar2 => github.com/baniu/bar3 v1.2.3
)
require (
github.com/qiniu/bar v1.0.0
github.com/qiniu/bar2 v1.2.0
)

7
tests/samples/gomod_samples/b/go.mod
View File

@ -1,7 +0,0 @@
module example.com/gg/a
replerace github.com/qiniu/bar => ../home/foo/bar
eggrr (
github.com/qiniu/bar v1.0.0
)

Some files were not shown because too many files have changed in this diff Show More