ar0c/goc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge pull request #196 from lyyyuna/v2-init

Browse Source 
goc v2 init
This commit is contained in:
Li Yiyang 2021-06-28 10:28:00 +08:00 committed by GitHub
commit 2459e2b8bb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
192 changed files with 4071 additions and 12696 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
.github/workflows/e2e-linux.yml vendored Normal file
View File

@ -0,0 +1,32 @@
name: linux/macos e2e test
on:
# Trigger the workflow on push or pull request,
# but only for the master branch
push:
paths-ignore:
- '**.md'
- '**.png'
pull_request:
paths-ignore:
- '**.md'
- '**.png'
jobs:
job_1:
name: Build goc binary
strategy:
matrix:
os: [ubuntu-latest, macos-latest]
runs-on: ${{ matrix.os }}
steps:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.16.x
- name: Checkout code
uses: actions/checkout@v2
- name: Go build
run: |
go build .
- name: Use goc to build self
run: |
./goc build -o ./gocc .

32
.github/workflows/e2e-wins.yml vendored Normal file
View File

@ -0,0 +1,32 @@
name: windows e2e test
on:
# Trigger the workflow on push or pull request,
# but only for the master branch
push:
paths-ignore:
- '**.md'
- '**.png'
pull_request:
paths-ignore:
- '**.md'
- '**.png'
jobs:
job_1:
name: Build goc binary
strategy:
matrix:
os: [windows-latest]
runs-on: ${{ matrix.os }}
steps:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.16.x
- name: Checkout code
uses: actions/checkout@v2
- name: Go build
run: |
go build .
- name: Use goc to build self
run: |
.\goc.exe build -o gocc .

73
.github/workflows/e2e_test_check.yml vendored
View File

@ -1,73 +0,0 @@
name: e2e test
on:
# Trigger the workflow on push or pull request,
# but only for the master branch
push:
paths-ignore:
- '**.md'
- '**.png'
pull_request:
paths-ignore:
- '**.md'
- '**.png'
jobs:
job_1:
name: Build goc binary
runs-on: ubuntu-latest
steps:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.14.x
- name: Checkout code
uses: actions/checkout@v2
- name: Go build
run: |
go build
- name: Use goc to build self
run: |
./goc build --output ./gocc --debug
- name: Upload goc binary
uses: actions/upload-artifact@v2
with:
name: goc
path: goc
- name: Upload covered self goc binary
uses: actions/upload-artifact@v2
with:
name: gocc
path: gocc
job_2:
name: E2E test
needs: job_1
strategy:
matrix:
go-version: [1.11.x, 1.12.x, 1.13.x, 1.14.x, 1.15.x, 1.16.x]
runs-on: ubuntu-latest
steps:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}
- name: Checkout code
uses: actions/checkout@v2
- name: Download built binary
uses: actions/download-artifact@v2
with:
path: /home/runner/tools
- name: Install bats-core
run: |
git clone https://github.com/bats-core/bats-core.git
cd bats-core
sudo ./install.sh /usr/local
- name: Do test
env:
GOVERSION: ${{ matrix.go-version }}
run: |
chmod +x /home/runner/tools/goc/goc
export PATH=/home/runner/tools/goc:$PATH
chmod +x /home/runner/tools/gocc/gocc
export PATH=/home/runner/tools/gocc:$PATH
cd tests
./run-ci-actions.sh

23
.github/workflows/golangci-lint.yml vendored
View File

@ -1,23 +0,0 @@
name: golangci-lint
on:
pull_request:
jobs:
golangci:
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: golangci-lint
uses: golangci/golangci-lint-action@v2
with:
# Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
version: v1.29
# Optional: working directory, useful for monorepos
# working-directory: somedir
# Optional: golangci-lint command line arguments.
# args: --issues-exit-code=0
# Optional: show only new issues if it's a pull request. The default value is `false`.
only-new-issues: true

55
.github/workflows/release.yml vendored
View File

@ -1,55 +0,0 @@
on:
release:
types: [published,edited]
name: Build Release
jobs:
release-linux-amd64:
name: release linux/amd64
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.14.x
- name: compile and release
run: |
./ci-build.sh
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GOARCH: amd64
GOOS: linux
release-linux-386:
name: release linux/386
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.14.x
- name: compile and release
run: |
./ci-build.sh
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GOARCH: "386"
GOOS: linux
release-darwin-amd64:
name: release darwin/amd64
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.14.x
- name: compile and release
run: |
./ci-build.sh
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GOARCH: amd64
GOOS: darwin

4
.github/workflows/style_check.yml vendored
View File

@ -15,7 +15,7 @@ jobs:
name: vet and gofmt
strategy:
matrix:
go-version: [1.13.x, 1.14.x, 1.15.x]
go-version: [1.16.x]
runs-on: ubuntu-latest
steps:
- name: Install Go
@ -36,4 +36,4 @@ jobs:
echo "${diff}"
echo "Please run this command to fix: [find . -name "*.go" | xargs gofmt -s -w]"
exit 1
fi
fi

4
.github/workflows/ut_check.yml → .github/workflows/ut-check.yml vendored
View File

@ -15,7 +15,7 @@ jobs:
name: go test
strategy:
matrix:
go-version: [1.13.x, 1.14.x, 1.15.x, 1.16.x]
go-version: [1.16.x]
runs-on: ubuntu-latest
steps:
- name: Install Go
@ -31,4 +31,4 @@ jobs:
run: |
export DEFAULT_EXCEPT_PKGS=e2e
go test -p 1 -coverprofile=coverage.txt $(go list ./... | grep -v -E $DEFAULT_EXCEPT_PKGS)
bash <(curl -s https://codecov.io/bash) -F unittest-$GOVERSION
bash <(curl -s https://codecov.io/bash) -F unittest-$GOVERSION

13
.gitignore vendored
View File

@ -1,13 +0,0 @@
# ignore log file
**/goc.log
# binary
goc
# the temp file to save service address
_svrs_address.txt
# other
*.iml
.DS_Store
.idea

21
Makefile
View File

@ -1,21 +0,0 @@
DEFAULT_EXCEPT_PKGS := e2e
all:
go install ./...
test:
go test -cover -p 1 `go list ./... | grep -v -E ${DEFAULT_EXCEPT_PKGS}`
fmt:
go fmt ./...
govet-check:
go vet ./...
clean:
find tests/ -type f -name '*.bak' -delete
find tests/ -type f -name '*.cov' -delete
find tests/ -type f -name 'simple-project' -delete
find tests/ -type f -name '*_profile_listen_addr' -delete
find tests/ -type f -name 'simple_gopath_project' -delete

113
README.md
View File

@ -1,88 +1,73 @@
# goc
[![Go Report Card](https://goreportcard.com/badge/github.com/qiniu/goc)](https://goreportcard.com/report/github.com/qiniu/goc)
![](https://github.com/qiniu/goc/workflows/ut-check/badge.svg)
![](https://github.com/qiniu/goc/workflows/style-check/badge.svg)
![](https://github.com/qiniu/goc/workflows/e2e%20test/badge.svg)
![Build Release](https://github.com/qiniu/goc/workflows/Build%20Release/badge.svg)
[![codecov](https://codecov.io/gh/qiniu/goc/branch/master/graph/badge.svg)](https://codecov.io/gh/qiniu/goc)
[![GoDoc](https://godoc.org/github.com/qiniu/goc?status.svg)](https://godoc.org/github.com/qiniu/goc)
[中文页](README_zh.md) |
goc v2 版本开发中
goc is a comprehensive coverage testing system for The Go Programming Language, especially for some complex scenarios, like system testing code coverage collection and
accurate testing.
## Quick Start
Enjoy, Have Fun!
![Demo](docs/images/intro.gif)
### 编译要求
## Installation
`Go 1.16+`
Download the latest version from [Github Releases](https://github.com/qiniu/goc/releases) page.
### 新特性一览
Goc supports both `GOPATH` project and `Go Modules` project with **Go 1.11+**. However, for developing goc, you need to install **Go 1.13+**.
#### 1. 只支持 go module 工程
## Examples
You can use goc tool in many scenarios.
考虑到 GOPATH 已被官方明确将淘汰，以及支持 GOPATH 工程带来的巨大工作量，v2 不再支持 GOPATH 工程。
### Code Coverage Collection for Your Golang System Tests
Goc can collect code coverages at runtime for your long-run golang applications. To do that, normally just need three steps:
#### 2. 命令行 flag 解析优化
1. use `goc server` to start a service registry center:
```
➜ simple-go-server git:(master) ✗ goc server
```
2. use `goc build` to build the target service, and run the generated binary. Here let's take the [simple-go-server](https://github.com/CarlJi/simple-go-server) project as example:
```
➜ simple-go-server git:(master) ✗ goc build .
... // omit logs
➜ simple-go-server git:(master) ✗ ./simple-go-server
```
3. use `goc profile` to get the code coverage profile of the started simple server above:
```
➜ simple-go-server git:(master) ✗ goc profile
mode: atomic
enricofoltran/simple-go-server/main.go:30.13,48.33 13 1
enricofoltran/simple-go-server/main.go:48.33,50.3 1 0
enricofoltran/simple-go-server/main.go:52.2,65.12 5 1
enricofoltran/simple-go-server/main.go:65.12,74.46 7 1
enricofoltran/simple-go-server/main.go:74.46,76.4 1 0
...
```
在 v1 版本中，`go build -o -ldflags 'foo=bar' ./app/main.go` 与 goc 命令并不等价，首先你需要切换到 `.app/` 目录中，然后执行 `goc build --buildflags="-o -ldflags 'foo=bar' ."`。这个转换给使用者带来不小的负担，特别是 `'"` 混杂在一起时，感觉会更难受。
### Show Code Coverage Change at Runtime in Vscode
在 v2 版本中，goc 编译命令和 go 编译命令已经极为相似，例如
We provide a vscode extension - [Goc Coverage](https://marketplace.visualstudio.com/items?itemName=lyyyuna.goc) which can show highlighted covered source code at runtime.
```bash
go build -o -ldflags 'foo=bar' ./app/main.go
# 等价于
goc build -o -ldflags 'foo=bar' ./app/main.go
#
go build -o -ldflags 'foo=bar' ./app
# 等价于
goc build -o -ldflags 'foo=bar' ./app
#
go install ./app/...
# 等价于
goc install ./app/...
#
```
![Extension](docs/images/goc-vscode.gif)
由于 go 命令对 flags 和 args 的相对位置有着严格要求：`go build [-o output] [build flags] [packages]`，所以在指定 goc 自己的 flags （所有 goc flags 都是 `--` 开头）必须和 `build flags` 位置保持相同，即：
## Tips
```bash
goc build --debug -o /home/app . # 合法
1. To understand the execution details of goc tool, you can use the `--debug` flag. Also we appreciate if you can provide such logs when submitting a bug to us.
goc build -o /home/app . --debug # 非法
```
2. By default, the covered service will listen a random port in order to communicate with the goc server. This may not be suitable in [docker](https://docs.docker.com/engine/reference/commandline/run/#publish-or-expose-port--p---expose) or [kubernetes](https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service) environment since the port must be exposed explicitly in order to be accessible by others in such environment. For this kind of scenario, you can use `--agentport` flag to specify a fixed port when calling `goc build` or `goc install`.
#### 3. 日志优化
3. To use a remote goc server, you can use `--center` flag to compile the target service with `goc build` or `goc install` command.
带颜色日志，以及长时间操作时（例如 build, copy）会有转圈动画。
4. The coverage data is stored on each covered service side, so if one service needs to restart during test, this service's coverage data will be lost. For this case, you can use following steps to handle:
#### 4. 被测服务部署优化
1. Before the service restarts, collect coverage with `goc profile -o a.cov`
2. After service restarted and test finished, collect coverage again with `goc profile -o b.cov`
3. Merge two coverage profiles together: `goc merge a.cov b.cov -o merge.cov`
在 v1 版本中，当被测服务在 docker 中，goc server 在外部时，会要求在容器启动时额外开启端口转发，并且编译还需带额外参数。这给部署带来不便。
## RoadMap
- [x] Support code coverage collection for system testing.
- [x] Support code coverage counters clear for the services under test at runtime.
- [x] Support develop mode towards accurate testing.
- [x] Support code coverage diff based on Pull Request.
- [ ] Optimize the performance costed by code coverage counters.
在 v2 版本中，不再有这一限制，只需要 goc server 能够被被测服务访问即可。
## Contributing
We welcome all kinds of contribution, including bug reports, feature requests, documentation improvements, UI refinements, etc.
#### 5. watch 模式
Thanks to all [contributors](https://github.com/qiniu/goc/graphs/contributors)!!
当使用 `goc build --mode watch .` 编译后，被测服务任何覆盖率变化都将实时推送到 goc server。
## License
Goc is released under the Apache 2.0 license. See [LICENSE.txt](https://github.com/qiniu/goc/blob/master/LICENSE)
用户可以使用该 websocket 连接 `ws://[goc_server_host]/cover/ws/watch` 观察到被测服务的新触发代码块，推送信息格式如下：
## Join goc WeChat Group
![WeChat](docs/images/wechat.png)
```bash
qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0
#
# importpath/filename.go:Line0.Col1,Line1,Col1 1 0
```
除此之外，原来的全局整体覆盖率可正常获取，不受影响。
#### 6. 跨平台支持
1. 支持 `Linux/Macos/Windows`
2. 支持 go 的交叉编译

97
README_zh.md
View File

@ -1,97 +0,0 @@
# goc
[![Go Report Card](https://goreportcard.com/badge/github.com/qiniu/goc)](https://goreportcard.com/report/github.com/qiniu/goc)
![](https://github.com/qiniu/goc/workflows/ut-check/badge.svg)
![](https://github.com/qiniu/goc/workflows/style-check/badge.svg)
![](https://github.com/qiniu/goc/workflows/e2e%20test/badge.svg)
![Build Release](https://github.com/qiniu/goc/workflows/Build%20Release/badge.svg)
[![codecov](https://codecov.io/gh/qiniu/goc/branch/master/graph/badge.svg)](https://codecov.io/gh/qiniu/goc)
[![GoDoc](https://godoc.org/github.com/qiniu/goc?status.svg)](https://godoc.org/github.com/qiniu/goc)
goc 是专为 Go 语言打造的一个综合覆盖率收集系统，尤其适合复杂的测试场景，比如系统测试时的代码覆盖率收集以及精准测试。
希望你们喜欢～
![Demo](docs/images/intro.gif)
## 安装
最新版本在该页面下载 [Github Releases](https://github.com/qiniu/goc/releases)。
goc 同时支持 `GOPATH` 工程和 `Go Modules` 工程，且 Go 版本要求 **Go 1.11+**。如果想参与 goc 的开发，你必须使用 **Go 1.13+**。
## 例子
goc 有多种使用场景。
### 在系统测试中收集代码覆盖率
goc 可以实时收集长时运行的 golang 服务覆盖率。收集步骤只需要下面三步：
1. 运行 `goc server` 命令启动一个服务注册中心：
```
➜ simple-go-server git:(master) ✗ goc server
```
2. 运行 `goc build` 命令编译目标服务，然后启动插过桩的二进制。下面以 [simple-go-server](https://github.com/CarlJi/simple-go-server) 工程为例：
```
➜ simple-go-server git:(master) ✗ goc build .
... // omit logs
➜ simple-go-server git:(master) ✗ ./simple-go-server
```
3. 运行 `goc profile` 命令收集刚启动的 simple server 的代码覆盖率：
```
➜ simple-go-server git:(master) ✗ goc profile
mode: atomic
enricofoltran/simple-go-server/main.go:30.13,48.33 13 1
enricofoltran/simple-go-server/main.go:48.33,50.3 1 0
enricofoltran/simple-go-server/main.go:52.2,65.12 5 1
enricofoltran/simple-go-server/main.go:65.12,74.46 7 1
enricofoltran/simple-go-server/main.go:74.46,76.4 1 0
...
```
### Vscode 中实时展示覆盖率动态变化
我们提供了一个 vscode 插件 - [Goc Coverage](https://marketplace.visualstudio.com/items?itemName=lyyyuna.goc)。该插件可以在运行时高亮覆盖过的代码。
![Extension](docs/images/goc-vscode.gif)
## Tips
1. goc 命令加上 `--debug` 会打印详细的日志。我们建议在提交 bug 时附上详细日志。
2. 默认情况下，插桩过的服务会监听在一个随机的端口，注册中心会通过这个端口与服务通信。然而，对于 [docker](https://docs.docker.com/engine/reference/commandline/run/#publish-or-expose-port--p---expose) 和 [kubernetes](https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service) 容器化运行环境，对外暴露端口需在容器启动前指定。针对这种场景，你可以在 `goc build` 或 `goc install` 时使用 `--agentport` 来指定插桩过的服务监听在固定的端口。
3. 如果注册中心不在本机，你可以在 `goc build` 或 `goc install` 编译目标服务时使用 `--center` 指定远端注册中心地址。
4. 目前覆盖率数据存储在插过桩的服务测，如果某个服务中途需要重启，那么其覆盖率数据在重启后会丢失。针对这个场景，你可以通过以下步骤解决：
1. 在重启前，通过 `goc profile -o a.cov` 命令收集一次覆盖率
2. 测试结束后，通过 `goc profile -o b.cov` 命令再收集一次覆盖率
3. 通过 `goc merge a.cov b.cov -o merge.cov` 命令合并两次的覆盖率
## Blogs
- [Go语言系统测试覆盖率收集利器 goc](https://mp.weixin.qq.com/s/DzXEXwepaouSuD2dPVloOg)
- [聊聊Go代码覆盖率技术与最佳实践](https://mp.weixin.qq.com/s/SQHzsfV5T_B8fmt9NzGA7Q)
## RoadMap
- [x] 支持系统测试中收集代码覆盖率
- [x] 支持运行时对被测服务代码覆盖率计数器清零
- [x] 支持精准测试
- [x] 支持基于 Pull Request 的增量代码覆盖率报告
- [ ] 优化插桩计数器带来的性能损耗
## Contributing
我们欢迎各种形式的贡献，包括提交 bug、提新需求、优化文档和改进 UI 等等。
感谢所有的[贡献者](https://github.com/qiniu/goc/graphs/contributors)!!
## License
Goc is released under the Apache 2.0 license. See [LICENSE.txt](https://github.com/qiniu/goc/blob/master/LICENSE)
## 加入微信群聊
![WeChat](docs/images/wechat.png)

33
ci-build.sh
View File

@ -1,33 +0,0 @@
#!/bin/bash
set -eux
EVENT_DATA=$(cat $GITHUB_EVENT_PATH)
echo $EVENT_DATA | jq .
UPLOAD_URL=$(echo $EVENT_DATA | jq -r .release.upload_url)
UPLOAD_URL=${UPLOAD_URL/\{?name,label\}/}
RELEASE_VERSION=$(echo $EVENT_DATA | jq -r .release.tag_name)
PROJECT_NAME=$(basename $GITHUB_REPOSITORY)
NAME="${NAME:-${PROJECT_NAME}-${RELEASE_VERSION}}-${GOOS}-${GOARCH}"
CGO_ENABLED=0 go build -ldflags "-X 'github.com/qiniu/goc/cmd.version=${RELEASE_VERSION}'" .
ARCHIVE=tmp.tar.gz
FILE_LIST=goc
tar cvfz $ARCHIVE ${FILE_LIST}
CHECKSUM=$(md5sum ${ARCHIVE} | cut -d ' ' -f 1)
curl \
-X POST \
--data-binary @${ARCHIVE} \
-H 'Content-Type: application/octet-stream' \
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
"${UPLOAD_URL}?name=${NAME}.${ARCHIVE/tmp./}"
curl \
-X POST \
--data $CHECKSUM \
-H 'Content-Type: text/plain' \
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
"${UPLOAD_URL}?name=${NAME}_md5.txt"

91
cmd/build.go
View File

@ -1,96 +1,25 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"os"
log "github.com/sirupsen/logrus"
"github.com/qiniu/goc/pkg/build"
"github.com/qiniu/goc/pkg/cover"
"github.com/qiniu/goc/v2/pkg/build"
"github.com/qiniu/goc/v2/pkg/config"
"github.com/spf13/cobra"
)
var buildCmd = &cobra.Command{
Use: "build",
Short: "Do cover for all go files and execute go build command",
Long: `
Build command will copy the project code and its necessary dependencies to a temporary directory, then do cover for the target, binaries will be generated to their original place.
`,
Example: `
# Build the current binary with cover variables injected. The binary will be generated in the current folder.
goc build .
Use: "build",
Run: buildAction,
# Build the current binary with cover variables injected, and set the registry center to http://127.0.0.1:7777.
goc build --center=http://127.0.0.1:7777
# Build the current binary with cover variables injected, and redirect output to /to/this/path.
goc build --output /to/this/path
# Build the current binary with cover variables injected, and set necessary build flags: -ldflags "-extldflags -static" -tags="embed kodo".
goc build --buildflags="-ldflags '-extldflags -static' -tags='embed kodo'"
`,
Run: func(cmd *cobra.Command, args []string) {
wd, err := os.Getwd()
if err != nil {
log.Fatalf("Fail to build: %v", err)
}
runBuild(args, wd)
},
DisableFlagParsing: true, // build 命令需要用原生 go 的方式处理 flags
}
var buildOutput string
func init() {
addBuildFlags(buildCmd.Flags())
buildCmd.Flags().StringVarP(&buildOutput, "output", "o", "", "it forces build to write the resulting executable to the named output file")
buildCmd.Flags().StringVarP(&config.GocConfig.Mode, "mode", "", "count", "coverage mode: set, count, atomic, watch")
buildCmd.Flags().StringVarP(&config.GocConfig.Host, "host", "", "127.0.0.1:7777", "specify the host of the goc sever")
rootCmd.AddCommand(buildCmd)
}
func runBuild(args []string, wd string) {
gocBuild, err := build.NewBuild(buildFlags, args, wd, buildOutput)
if err != nil {
log.Fatalf("Fail to build: %v", err)
}
// remove temporary directory if needed
defer gocBuild.Clean()
// doCover with original buildFlags, with new GOPATH( tmp:original )
// in the tmp directory
ci := &cover.CoverInfo{
Args: buildFlags,
GoPath: gocBuild.NewGOPATH,
Target: gocBuild.TmpDir,
Mode: coverMode.String(),
AgentPort: agentPort.String(),
Center: center,
IsMod: gocBuild.IsMod,
ModRootPath: gocBuild.ModRootPath,
OneMainPackage: true, // it is a go build
GlobalCoverVarImportPath: gocBuild.GlobalCoverVarImportPath,
}
err = cover.Execute(ci)
if err != nil {
log.Fatalf("Fail to build: %v", err)
}
// do install in the temporary directory
err = gocBuild.Build()
if err != nil {
log.Fatalf("Fail to build: %v", err)
}
return
func buildAction(cmd *cobra.Command, args []string) {
b := build.NewBuild(cmd, args)
b.Build()
}

110
cmd/build_test.go
View File

@ -1,110 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
var baseDir string
func init() {
baseDir, _ = os.Getwd()
}
func TestGeneratedBinary(t *testing.T) {
startTime := time.Now()
workingDir := filepath.Join(baseDir, "../tests/samples/simple_project")
gopath := ""
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
buildFlags, buildOutput = "", ""
args := []string{"."}
runBuild(args, workingDir)
obj := filepath.Join(workingDir, "simple-project")
fInfo, err := os.Lstat(obj)
assert.Equal(t, err, nil, "the binary should be generated.")
assert.Equal(t, startTime.Before(fInfo.ModTime()), true, obj+"new binary should be generated, not the old one")
cmd := exec.Command("go", "tool", "objdump", "simple-project")
cmd.Dir = workingDir
out, _ := cmd.CombinedOutput()
cnt := strings.Count(string(out), "main.registerSelf")
assert.Equal(t, cnt > 0, true, "main.registerSelf function should be in the binary")
cnt = strings.Count(string(out), "GoCover")
assert.Equal(t, cnt > 0, true, "GoCover variable should be in the binary")
}
func TestBuildBinaryName(t *testing.T) {
startTime := time.Now()
workingDir := filepath.Join(baseDir, "../tests/samples/simple_project2")
gopath := ""
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
buildFlags, buildOutput = "", ""
args := []string{"."}
runBuild(args, workingDir)
obj := filepath.Join(workingDir, "simple-project")
fInfo, err := os.Lstat(obj)
assert.Equal(t, err, nil, "the binary should be generated.")
assert.Equal(t, startTime.Before(fInfo.ModTime()), true, obj+"new binary should be generated, not the old one")
cmd := exec.Command("go", "tool", "objdump", "simple-project")
cmd.Dir = workingDir
out, _ := cmd.CombinedOutput()
cnt := strings.Count(string(out), "main.registerSelf")
assert.Equal(t, cnt > 0, true, "main.registerSelf function should be in the binary")
cnt = strings.Count(string(out), "GoCover")
assert.Equal(t, cnt > 0, true, "GoCover variable should be in the binary")
}
// test if goc can get variables in internal package
func TestBuildBinaryForInternalPackage(t *testing.T) {
startTime := time.Now()
workingDir := filepath.Join(baseDir, "../tests/samples/simple_project_with_internal")
gopath := ""
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
buildFlags, buildOutput = "", ""
args := []string{"."}
runBuild(args, workingDir)
obj := filepath.Join(workingDir, "simple-project")
fInfo, err := os.Lstat(obj)
assert.Equal(t, err, nil, "the binary should be generated.")
assert.Equal(t, startTime.Before(fInfo.ModTime()), true, obj+"new binary should be generated, not the old one")
}

58
cmd/clear.go
View File

@ -1,58 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"os"
log "github.com/sirupsen/logrus"
"github.com/qiniu/goc/pkg/cover"
"github.com/spf13/cobra"
)
var clearCmd = &cobra.Command{
Use: "clear",
Short: "Clear code coverage counters of all the registered services",
Long: `Clear code coverage counters for the services under test at runtime.`,
Example: `
# Clear coverage counter from default register center http://127.0.0.1:7777.
goc clear
# Clear coverage counter from specified register center.
goc clear --center=http://192.168.1.1:8080
`,
Run: func(cmd *cobra.Command, args []string) {
p := cover.ProfileParam{
Service: svrList,
Address: addrList,
}
res, err := cover.NewWorker(center).Clear(p)
if err != nil {
log.Fatalf("call host %v failed, err: %v, response: %v", center, err, string(res))
}
fmt.Fprint(os.Stdout, string(res))
},
}
func init() {
addBasicFlags(clearCmd.Flags())
clearCmd.Flags().StringSliceVarP(&svrList, "service", "", nil, "service name to clear profile, see 'goc list' for all services.")
clearCmd.Flags().StringSliceVarP(&addrList, "address", "", nil, "address to clear profile, see 'goc list' for all addresses.")
rootCmd.AddCommand(clearCmd)
}

126
cmd/commonflags.go
View File

@ -1,126 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"net"
"github.com/spf13/pflag"
"github.com/spf13/viper"
)
var (
target string
center string
agentPort AgentPort
debugGoc bool
debugInCISyncFile string
buildFlags string
goRunExecFlag string
goRunArguments string
)
var coverMode = CoverMode{
mode: "count",
}
// addBasicFlags adds a
func addBasicFlags(cmdset *pflag.FlagSet) {
cmdset.StringVar(&center, "center", "http://127.0.0.1:7777", "cover profile host center")
// bind to viper
viper.BindPFlags(cmdset)
}
func addCommonFlags(cmdset *pflag.FlagSet) {
addBasicFlags(cmdset)
cmdset.Var(&coverMode, "mode", "coverage mode: set, count, atomic")
cmdset.Var(&agentPort, "agentport", "a fixed port such as :8100 for registered service communicate with goc server. if not provided, using a random one")
cmdset.StringVar(&buildFlags, "buildflags", "", "specify the build flags")
// bind to viper
viper.BindPFlags(cmdset)
}
func addBuildFlags(cmdset *pflag.FlagSet) {
addCommonFlags(cmdset)
// bind to viper
viper.BindPFlags(cmdset)
}
func addRunFlags(cmdset *pflag.FlagSet) {
addBuildFlags(cmdset)
cmdset.StringVar(&goRunExecFlag, "exec", "", "same as -exec flag in 'go run' command")
cmdset.StringVar(&goRunArguments, "arguments", "", "same as 'arguments' in 'go run' command")
// bind to viper
viper.BindPFlags(cmdset)
}
// CoverMode represents the covermode when doing cover for source code
type CoverMode struct {
mode string
}
func (m *CoverMode) String() string {
return m.mode
}
// Set sets the value to the CoverMode struct, use 'count' as default if v is empty
func (m *CoverMode) Set(v string) error {
if v == "" {
m.mode = "count"
return nil
}
if v != "set" && v != "count" && v != "atomic" {
return fmt.Errorf("unknown mode")
}
m.mode = v
return nil
}
// Type returns the type of CoverMode
func (m *CoverMode) Type() string {
return "string"
}
// AgentPort is the struct to do agentPort check
type AgentPort struct {
port string
}
func (agent *AgentPort) String() string {
return agent.port
}
// Set sets the value to the AgentPort struct
func (agent *AgentPort) Set(v string) error {
if v == "" {
agent.port = ""
return nil
}
_, _, err := net.SplitHostPort(v)
if err != nil {
return err
}
agent.port = v
return nil
}
// Type returns the type of AgentPort
func (agent *AgentPort) Type() string {
return "string"
}

110
cmd/commonflags_test.go
View File

@ -1,110 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"errors"
"fmt"
"testing"
"github.com/stretchr/testify/assert"
)
func TestCoverModeFlag(t *testing.T) {
var tcs = []struct {
value string
expectedValue interface{}
err interface{}
}{
{
value: "",
expectedValue: "count",
err: nil,
},
{
value: "set",
expectedValue: "set",
err: nil,
},
{
value: "count",
expectedValue: "count",
err: nil,
},
{
value: "atomic",
expectedValue: "atomic",
err: nil,
},
{
value: "xxxxx",
expectedValue: "",
err: errors.New("unknown mode"),
},
{
value: "123333",
expectedValue: "",
err: errors.New("unknown mode"),
},
}
for _, tc := range tcs {
mode := &CoverMode{}
err := mode.Set(tc.value)
actual := mode.String()
assert.Equal(t, actual, tc.expectedValue, fmt.Sprintf("check mode flag value failed, expected %s, got %s", tc.expectedValue, actual))
assert.Equal(t, err, tc.err, fmt.Sprintf("check mode flag error, expected %s, got %s", tc.err, err))
}
}
func TestAgentPortFlag(t *testing.T) {
var tcs = []struct {
value string
expectedValue interface{}
isErr bool
}{
{
value: "",
expectedValue: "",
isErr: false,
},
{
value: ":8888",
expectedValue: ":8888",
isErr: false,
},
{
value: "8888",
expectedValue: "",
isErr: true,
},
{
value: "::8888",
expectedValue: "",
isErr: true,
},
}
for _, tc := range tcs {
agent := &AgentPort{}
err := agent.Set(tc.value)
if tc.isErr {
assert.NotEqual(t, nil, err, fmt.Sprintf("check agentport flag error, expected %v, got %v", nil, err))
} else {
actual := agent.String()
assert.Equal(t, tc.expectedValue, actual, fmt.Sprintf("check agentport flag value failed, expected %s, got %s", tc.expectedValue, actual))
}
}
}

61
cmd/cover.go
View File

@ -1,61 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"github.com/qiniu/goc/pkg/cover"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
var coverCmd = &cobra.Command{
Use: "cover",
Short: "Do cover for the target source",
Long: `Do cover for the target source. You can select different cover mode (set, count, atomic), default: count`,
Example: `
# Do cover for the current path, default center: http://127.0.0.1:7777, default cover mode: count.
goc cover
# Do cover for the current path, default cover mode: count.
goc cover --center=http://127.0.0.1:7777
# Do cover for the target path, cover mode: atomic.
goc cover --center=http://127.0.0.1:7777 --target=/path/to/target --mode=atomic
`,
Hidden: true,
Run: func(cmd *cobra.Command, args []string) {
var buildFlags string
buildFlags = viper.GetString("buildflags")
ci := &cover.CoverInfo{
Args: buildFlags,
GoPath: "",
Target: target,
Mode: coverMode.String(),
AgentPort: agentPort.String(),
Center: center,
OneMainPackage: false,
}
_ = cover.Execute(ci)
},
}
func init() {
coverCmd.Flags().StringVar(&target, "target", ".", "target folder to cover")
addCommonFlags(coverCmd.Flags())
rootCmd.AddCommand(coverCmd)
}

209
cmd/diff.go
View File

@ -1,209 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"encoding/json"
"io/ioutil"
"os"
"github.com/olekukonko/tablewriter"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/qiniu/goc/pkg/cover"
"github.com/qiniu/goc/pkg/github"
"github.com/qiniu/goc/pkg/prow"
"github.com/qiniu/goc/pkg/qiniu"
)
var diffCmd = &cobra.Command{
Use: "diff",
Short: "Do coverage profile diff analysis, it can also work with prow and post comments to github pull request if needed",
Example: ` # Diff two local coverage profile and display
goc diff --new-profile=<xxxx> --base-profile=<xxxx>
# Diff local coverage profile with the remote one in prow job using default qiniu-credential
goc diff --prow-postsubmit-job=<xxx> --new-profile=<xxx>
# Calculate and display full diff coverage between new-profile and base-profile, not concerned github changed files
goc diff --prow-postsubmit-job=<xxx> --new-profile=<xxx> --full-diff=true
# Diff local coverage profile with the remote one in prow job
goc diff --prow-postsubmit-job=<xxx> --prow-remote-profile-name=<xxx>
--qiniu-credential=<xxx> --new-profile=<xxxx>
# Diff coverage profile with the remote one in prow job, and post comments to github PR
goc diff --prow-postsubmit-job=<xxx> --prow-profile=<xxx>
--github-token=<xxx> --github-user=<xxx> --github-comment-prefix=<xxx>
--qiniu-credential=<xxx> --coverage-threshold-percentage=<xxx> --new-profile=<xxxx>
`,
Run: func(cmd *cobra.Command, args []string) {
if baseProfile != "" {
doDiffForLocalProfiles(cmd, args)
} else if prowPostSubmitJob != "" {
doDiffUnderProw(cmd, args)
} else {
logrus.Fatalf("either base-profile or prow-postsubmit-job must be provided")
}
},
}
var (
newProfile string
baseProfile string
coverageThreshold int
prowPostSubmitJob string
prowProfile string
githubToken string
githubUser string
githubCommentPrefix string
qiniuCredential string
robotName string
fullDiff bool
)
func init() {
diffCmd.Flags().StringVarP(&newProfile, "new-profile", "n", "", "local profile which works as the target to analysis")
diffCmd.MarkFlagRequired("new-profile")
diffCmd.Flags().StringVarP(&baseProfile, "base-profile", "b", "", "another local profile which works as baseline to compare with the target")
diffCmd.Flags().IntVarP(&coverageThreshold, "coverage-threshold-percentage", "", 0, "coverage threshold percentage")
diffCmd.Flags().StringVarP(&prowPostSubmitJob, "prow-postsubmit-job", "", "", "prow postsubmit job which used to find the base profile")
diffCmd.Flags().StringVarP(&prowProfile, "prow-remote-profile-name", "", "filtered.cov", "the name of profile in prow postsubmit job, which used as the base profile to compare")
diffCmd.Flags().StringVarP(&githubToken, "github-token", "", "/etc/github/oauth", "path to token to access github repo")
diffCmd.Flags().StringVarP(&githubUser, "github-user", "", "", "github user name when comments in github")
diffCmd.Flags().StringVarP(&githubCommentPrefix, "github-comment-prefix", "", "", "specific comment flag you provided")
diffCmd.Flags().StringVarP(&qiniuCredential, "qiniu-credential", "", "/etc/qiniuconfig/qiniu.json", "path to credential file to access qiniu cloud")
diffCmd.Flags().StringVarP(&robotName, "robot-name", "", "qiniu-bot", "github user name for coverage robot")
diffCmd.Flags().BoolVarP(&fullDiff, "full-diff", "", false, "when set true,calculate and display full diff coverage between new-profile and base-profile")
rootCmd.AddCommand(diffCmd)
}
//goc diff --new-profile=./new.cov --base-profile=./base.cov
//+------------------------------------------------------+---------------+--------------+--------+
//| File | Base Coverage | New Coverage | Delta |
//+------------------------------------------------------+---------------+--------------+--------+
//| qiniu.com/kodo/bd/pfd/pfdstg/cursor/mgr.go | 53.5% | 50.5% | -3.0% |
//| qiniu.com/kodo/bd/pfd/pfdstg/svr/getstripe.go | 0.5% | 0.0% | -0.5% |
//| Total | 35.7% | 35.7% | -0.0% |
//+------------------------------------------------------+---------------+--------------+--------+
func doDiffForLocalProfiles(cmd *cobra.Command, args []string) {
localP, err := cover.ReadFileToCoverList(newProfile)
if err != nil {
logrus.Fatal(err)
}
baseP, err := cover.ReadFileToCoverList(baseProfile)
if err != nil {
logrus.Fatal(err)
}
//calculate diff file cov and display
rows := cover.GetDeltaCov(localP, baseP)
rows.Sort()
table := tablewriter.NewWriter(os.Stdout)
table.SetHeader([]string{"File", "Base Coverage", "New Coverage", "Delta"})
table.SetAutoFormatHeaders(false)
table.SetColumnAlignment([]int{tablewriter.ALIGN_LEFT, tablewriter.ALIGN_CENTER, tablewriter.ALIGN_CENTER, tablewriter.ALIGN_CENTER})
for _, row := range rows {
table.Append([]string{row.FileName, row.BasePer, row.NewPer, row.DeltaPer})
}
totalDelta := cover.PercentStr(cover.TotalDelta(localP, baseP))
table.Append([]string{"Total", baseP.TotalPercentage(), localP.TotalPercentage(), totalDelta})
table.Render()
}
func doDiffUnderProw(cmd *cobra.Command, args []string) {
var (
prNumStr = os.Getenv("PULL_NUMBER")
pullSha = os.Getenv("PULL_PULL_SHA")
baseSha = os.Getenv("PULL_BASE_SHA")
repoOwner = os.Getenv("REPO_OWNER")
repoName = os.Getenv("REPO_NAME")
jobType = os.Getenv("JOB_TYPE")
jobName = os.Getenv("JOB_NAME")
buildStr = os.Getenv("BUILD_NUMBER")
artifacts = os.Getenv("ARTIFACTS")
)
logrus.Printf("Running coverage for PR = %s; PR commit SHA = %s;base SHA = %s", prNumStr, pullSha, baseSha)
switch jobType {
case "periodic":
logrus.Printf("job type %s, do nothing", jobType)
case "postsubmit":
logrus.Printf("job type %s, do nothing", jobType)
case "presubmit":
if githubToken == "" {
logrus.Fatalf("github token not provided")
}
prClient := github.NewPrClient(githubToken, repoOwner, repoName, prNumStr, robotName, githubCommentPrefix)
if qiniuCredential == "" {
logrus.Fatalf("qiniu credential not provided")
}
var qc qiniu.Client
var conf qiniu.Config
files, err := ioutil.ReadFile(*&qiniuCredential)
if err != nil {
logrus.WithError(err).Fatal("Error reading qiniu config file")
}
if err := json.Unmarshal(files, &conf); err != nil {
logrus.Fatal("Error unmarshal qiniu config file")
}
if conf.Bucket == "" {
logrus.Fatal("no qiniu bucket provided")
}
if conf.AccessKey == "" || conf.SecretKey == "" {
logrus.Fatal("either qiniu access key or secret key was not provided")
}
if conf.Domain == "" {
logrus.Fatal("no qiniu bucket domain was provided")
}
qc = qiniu.NewClient(&conf)
localArtifacts := qiniu.ProfileArtifacts{
Directory: artifacts,
ProfileName: newProfile,
ChangedProfileName: qiniu.ChangedProfileName,
}
job := prow.Job{
JobName: jobName,
BuildId: buildStr,
Org: repoOwner,
RepoName: repoName,
PRNumStr: prNumStr,
PostSubmitJob: prowPostSubmitJob,
LocalProfilePath: newProfile,
PostSubmitCoverProfile: prowProfile,
QiniuClient: qc,
LocalArtifacts: &localArtifacts,
GithubComment: prClient,
FullDiff: fullDiff,
}
if err := job.RunPresubmit(); err != nil {
logrus.Fatalf("run presubmit job failed, err: %v", err)
}
default:
logrus.Printf("Unknown job type: %s, do nothing.", jobType)
}
}

106
cmd/diff_test.go
View File

@ -1,106 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"testing"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/stretchr/testify/assert"
)
type diffFunc func(cmd *cobra.Command, args []string)
func captureStdout(f diffFunc, cmd *cobra.Command, args []string) string {
r, w, err := os.Pipe()
if err != nil {
logrus.WithError(err).Fatal("os pipe fail")
}
stdout := os.Stdout
os.Stdout = w
defer func() {
os.Stdout = stdout
}()
f(cmd, args)
w.Close()
var buf bytes.Buffer
io.Copy(&buf, r)
return buf.String()
}
func TestDoDiffForLocalProfiles(t *testing.T) {
items := []struct {
newCovFile string
newProfile string
baseCovFile string
baseProfile string
expectOutput string
}{
{
newCovFile: "new.cov",
baseCovFile: "base.cov",
newProfile: "mode: atomic\n" +
"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30\n" +
"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 1\n",
baseProfile: "mode: atomic\n" +
"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30\n" +
"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0\n",
expectOutput: `+-----------------------------------------+---------------+--------------+-------+
| File | Base Coverage | New Coverage | Delta |
+-----------------------------------------+---------------+--------------+-------+
| qiniu.com/kodo/apiserver/server/main.go | 50.0% | 100.0% | 50.0% |
| Total | 50.0% | 100.0% | 50.0% |
+-----------------------------------------+---------------+--------------+-------+
`,
},
}
for _, tc := range items {
err := ioutil.WriteFile(tc.newCovFile, []byte(tc.newProfile), 0644)
if err != nil {
logrus.WithError(err).Fatalf("write file %s failed", tc.newCovFile)
}
err = ioutil.WriteFile(tc.baseCovFile, []byte(tc.baseProfile), 0644)
if err != nil {
logrus.WithError(err).Fatalf("write file %s failed", tc.baseCovFile)
}
defer func() {
os.Remove(tc.newCovFile)
os.Remove(tc.baseCovFile)
}()
pwd, err := os.Getwd()
if err != nil {
logrus.WithError(err).Fatalf("get pwd failed")
}
diffCmd.Flags().Set("new-profile", fmt.Sprintf("%s/%s", pwd, tc.newCovFile))
diffCmd.Flags().Set("base-profile", fmt.Sprintf("%s/%s", pwd, tc.baseCovFile))
out := captureStdout(doDiffForLocalProfiles, diffCmd, nil)
assert.Equal(t, out, tc.expectOutput)
}
}

39
cmd/init.go
View File

@ -1,39 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
log "github.com/sirupsen/logrus"
"github.com/qiniu/goc/pkg/cover"
"github.com/spf13/cobra"
)
var initCmd = &cobra.Command{
Use: "init",
Short: "Clear the register information in order to start a new round of tests",
Run: func(cmd *cobra.Command, args []string) {
if res, err := cover.NewWorker(center).InitSystem(); err != nil {
log.Fatalf("call host %v failed, err: %v, response: %v", center, err, string(res))
}
},
}
func init() {
addBasicFlags(initCmd.Flags())
rootCmd.AddCommand(initCmd)
}

84
cmd/install.go
View File

@ -1,89 +1,25 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"os"
"github.com/qiniu/goc/pkg/build"
"github.com/qiniu/goc/pkg/cover"
log "github.com/sirupsen/logrus"
"github.com/qiniu/goc/v2/pkg/build"
"github.com/qiniu/goc/v2/pkg/config"
"github.com/spf13/cobra"
)
var installCmd = &cobra.Command{
Use: "install",
Short: "Do cover for all go files and execute go install command",
Long: `
Install command will copy the project code and its necessary dependencies to a temporary directory, then do cover for the target, binaries will be generated to their original place.
`,
Example: `
# Install all binaries with cover variables injected. The binary will be installed in $GOPATH/bin or $HOME/go/bin if directory existed.
goc install ./...
Use: "install",
Run: installAction,
# Install the current binary with cover variables injected, and set the registry center to http://127.0.0.1:7777.
goc install --center=http://127.0.0.1:7777
# Install the current binary with cover variables injected, and set necessary build flags: -ldflags "-extldflags -static" -tags="embed kodo".
goc build --buildflags="-ldflags '-extldflags -static' -tags='embed kodo'"
`,
Run: func(cmd *cobra.Command, args []string) {
wd, err := os.Getwd()
if err != nil {
log.Fatalf("Fail to build: %v", err)
}
runInstall(args, wd)
},
DisableFlagParsing: true, // install 命令需要用原生 go 的方式处理 flags
}
func init() {
addBuildFlags(installCmd.Flags())
installCmd.Flags().StringVarP(&config.GocConfig.Mode, "mode", "", "count", "coverage mode: set, count, atomic, watch")
installCmd.Flags().StringVarP(&config.GocConfig.Host, "host", "", "127.0.0.1:7777", "specify the host of the goc sever")
rootCmd.AddCommand(installCmd)
}
func runInstall(args []string, wd string) {
gocBuild, err := build.NewInstall(buildFlags, args, wd)
if err != nil {
log.Fatalf("Fail to install: %v", err)
}
// remove temporary directory if needed
defer gocBuild.Clean()
// doCover with original buildFlags, with new GOPATH( tmp:original )
// in the tmp directory
ci := &cover.CoverInfo{
Args: buildFlags,
GoPath: gocBuild.NewGOPATH,
Target: gocBuild.TmpDir,
Mode: coverMode.String(),
AgentPort: agentPort.String(),
Center: center,
IsMod: gocBuild.IsMod,
ModRootPath: gocBuild.ModRootPath,
OneMainPackage: false,
GlobalCoverVarImportPath: gocBuild.GlobalCoverVarImportPath,
}
err = cover.Execute(ci)
if err != nil {
log.Fatalf("Fail to install: %v", err)
}
// do install in the temporary directory
err = gocBuild.Install()
if err != nil {
log.Fatalf("Fail to install: %v", err)
}
return
func installAction(cmd *cobra.Command, args []string) {
b := build.NewInstall(cmd, args)
b.Install()
}

84
cmd/install_test.go
View File

@ -1,84 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestInstalledBinaryForMod(t *testing.T) {
startTime := time.Now()
workingDir := filepath.Join(baseDir, "../tests/samples/simple_project")
gopath := filepath.Join(baseDir, "../tests/samples/simple_project", "testhome")
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
buildFlags, buildOutput = "", ""
args := []string{"."}
runInstall(args, workingDir)
obj := filepath.Join(gopath, "bin", "simple-project")
fInfo, err := os.Lstat(obj)
assert.Equal(t, err, nil, "the binary should be generated.")
assert.Equal(t, startTime.Before(fInfo.ModTime()), true, obj+"new binary should be generated, not the old one")
cmd := exec.Command("go", "tool", "objdump", "simple-project")
cmd.Dir = workingDir
out, _ := cmd.CombinedOutput()
cnt := strings.Count(string(out), "main.registerSelf")
assert.Equal(t, cnt > 0, true, "main.registerSelf function should be in the binary")
cnt = strings.Count(string(out), "GoCover")
assert.Equal(t, cnt > 0, true, "GoCover variable should be in the binary")
}
func TestInstalledBinaryForLegacy(t *testing.T) {
startTime := time.Now()
workingDir := filepath.Join(baseDir, "../tests/samples/simple_gopath_project/src/qiniu.com/simple_gopath_project")
gopath := filepath.Join(baseDir, "../tests/samples/simple_gopath_project")
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "off")
buildFlags, buildOutput = "", ""
args := []string{"."}
runInstall(args, workingDir)
obj := filepath.Join(gopath, "bin", "simple_gopath_project")
fInfo, err := os.Lstat(obj)
assert.Equal(t, err, nil, "the binary should be generated.")
assert.Equal(t, startTime.Before(fInfo.ModTime()), true, obj+"new binary should be generated, not the old one")
cmd := exec.Command("go", "tool", "objdump", obj)
cmd.Dir = workingDir
out, _ := cmd.CombinedOutput()
cnt := strings.Count(string(out), "main.registerSelf")
assert.Equal(t, cnt > 0, true, "main.registerSelf function should be in the binary")
cnt = strings.Count(string(out), "GoCover")
assert.Equal(t, cnt > 0, true, "GoCover variable should be in the binary")
}

49
cmd/list.go
View File

@ -1,49 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"os"
log "github.com/sirupsen/logrus"
"github.com/qiniu/goc/pkg/cover"
"github.com/spf13/cobra"
)
var listCmd = &cobra.Command{
Use: "list",
Short: "Lists all the registered services",
Long: "Lists all the registered services",
Example: `
goc list [flags]
`,
Run: func(cmd *cobra.Command, args []string) {
res, err := cover.NewWorker(center).ListServices()
if err != nil {
log.Fatalf("list failed, err: %v", err)
}
log.Infoln(string(res))
fmt.Fprint(os.Stdout, string(res))
},
}
func init() {
addBasicFlags(listCmd.Flags())
rootCmd.AddCommand(listCmd)
}

76
cmd/merge.go
View File

@ -1,76 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/tools/cover"
"k8s.io/test-infra/gopherage/pkg/cov"
"k8s.io/test-infra/gopherage/pkg/util"
)
var mergeCmd = &cobra.Command{
Use: "merge [files...]",
Short: "Merge multiple coherent Go coverage files into a single file.",
Long: `merge will merge multiple Go coverage files into a single coverage file.
merge requires that the files are 'coherent', meaning that if they both contain references to the
same paths, then the contents of those source files were identical for the binary that generated
each file.
`,
Run: func(cmd *cobra.Command, args []string) {
runMerge(args, outputMergeProfile)
},
}
var outputMergeProfile string
func init() {
mergeCmd.Flags().StringVarP(&outputMergeProfile, "output", "o", "mergeprofile.cov", "output file")
rootCmd.AddCommand(mergeCmd)
}
func runMerge(args []string, output string) {
if len(args) == 0 {
log.Fatalln("Expected at least one coverage file.")
return
}
profiles := make([][]*cover.Profile, len(args))
for _, path := range args {
profile, err := util.LoadProfile(path)
if err != nil {
log.Fatalf("failed to open %s: %v", path, err)
return
}
profiles = append(profiles, profile)
}
merged, err := cov.MergeMultipleProfiles(profiles)
if err != nil {
log.Fatalf("failed to merge files: %v", err)
return
}
err = util.DumpProfile(output, merged)
if err != nil {
log.Fatalln(err)
return
}
}

156
cmd/merge_test.go
View File

@ -1,156 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
// use a variable to record if the tested function has failed
// so unittest can only be executed sequential
var fatal = false
var fatalStr string
type TestLogHook struct{}
func (h *TestLogHook) Levels() []log.Level {
return []log.Level{log.FatalLevel}
}
func (h *TestLogHook) Fire(e *log.Entry) error {
fatalStr = e.Message
return nil
}
func TestMain(m *testing.M) {
// setup
originalExitFunc := log.StandardLogger().ExitFunc
defer func() {
log.StandardLogger().ExitFunc = originalExitFunc
log.StandardLogger().Hooks = make(log.LevelHooks)
}()
// replace exit function, so log.Fatal wont exit
log.StandardLogger().ExitFunc = func(int) { fatal = true }
// add hook, so fatal string will be recorded
log.StandardLogger().Hooks.Add(&TestLogHook{})
code := m.Run()
os.Exit(code)
}
func TestMergeNormalProfiles(t *testing.T) {
profileA := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/a.voc")
profileB := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/b.voc")
mergeprofile := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/merge.cov")
runMerge([]string{profileA, profileB}, mergeprofile)
contents, err := ioutil.ReadFile(mergeprofile)
assert.NoError(t, err)
assert.Contains(t, string(contents), "qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 60")
assert.Contains(t, string(contents), "qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 2")
assert.Equal(t, fatal, false)
}
// test with no profiles
func TestMergeWithNoProfiles(t *testing.T) {
mergeprofile := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/merge.cov")
// clear fatal string in setup
fatalStr = ""
fatal = false
runMerge([]string{}, mergeprofile)
// there is fatal
assert.Equal(t, fatal, true)
assert.Equal(t, fatalStr, "Expected at least one coverage file.")
}
// pass a non-existed profile to runMerge
func TestWithWrongProfileName(t *testing.T) {
profileA := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/notexist.voc")
mergeprofile := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/merge.cov")
// clear fatal string in setup
fatalStr = ""
fatal = false
runMerge([]string{profileA}, mergeprofile)
// there is fatal
assert.Equal(t, fatal, true)
assert.Contains(t, fatalStr, "failed to open")
}
// merge two different modes' profiles should fail
func TestMergeTwoDifferentModeProfile(t *testing.T) {
profileA := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/a.voc")
profileSet := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/setmode.voc")
mergeprofile := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/merge.cov")
// clear fatal string in setup
fatalStr = ""
fatal = false
runMerge([]string{profileA, profileSet}, mergeprofile)
// there is fatal
assert.Equal(t, fatal, true)
assert.Contains(t, fatalStr, "mode for qiniu.com/kodo/apiserver/server/main.go mismatches")
}
// merge two overlaped profiles should fail
func TestMergeTwoOverLapProfile(t *testing.T) {
profileA := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/a.voc")
profileOverlap := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/overlap.voc")
mergeprofile := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/merge.cov")
// clear fatal string in setup
fatalStr = ""
fatal = false
runMerge([]string{profileA, profileOverlap}, mergeprofile)
// there is fatal
assert.Equal(t, fatal, true)
assert.Contains(t, fatalStr, "coverage block mismatch")
}
// merge empty file should fail
func TestMergeEmptyProfile(t *testing.T) {
profileA := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/empty.voc")
mergeprofile := filepath.Join(baseDir, "../tests/samples/merge_profile_samples/merge.cov")
// clear fatal string in setup
fatalStr = ""
fatal = false
runMerge([]string{profileA}, mergeprofile)
// there is fatal
assert.Equal(t, fatal, true)
assert.Contains(t, fatalStr, "failed to dump profile")
}

112
cmd/profile.go
View File

@ -1,112 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"bytes"
"fmt"
"io"
"os"
"path"
"github.com/qiniu/goc/pkg/cover"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var profileCmd = &cobra.Command{
Use: "profile",
Short: "Get coverage profile from service registry center",
Long: `Get code coverage profile for the services under test at runtime.`,
Example: `
# Get coverage counter from default register center http://127.0.0.1:7777, the result output to stdout.
goc profile
# Get coverage counter from specified register center, the result output to specified file.
goc profile --center=http://192.168.1.1:8080 --output=./coverage.cov
# Get coverage counter of several specified services. You can get all available service names from command 'goc list'. Use 'service' and 'address' flag at the same time may cause ambiguity, please use them separately.
goc profile --service=service1,service2,service3
# Get coverage counter of several specified addresses. You can get all available addresses from command 'goc list'. Use 'service' and 'address' flag at the same time may cause ambiguity, please use them separately.
goc profile --address=address1,address2,address3
# Only get the coverage data of files matching the special patterns
goc profile --coverfile=pattern1,pattern2,pattern3
# Force fetching all available profiles.
goc profile --force
`,
Run: func(cmd *cobra.Command, args []string) {
p := cover.ProfileParam{
Force: force,
Service: svrList,
Address: addrList,
CoverFilePatterns: coverFilePatterns,
SkipFilePatterns: skipFilePatterns,
}
res, err := cover.NewWorker(center).Profile(p)
if err != nil {
log.Fatalf("Goc server %v return an error: %v", center, err)
}
if output == "" {
fmt.Fprint(os.Stdout, string(res))
} else {
var dir, filename string = path.Split(output)
if dir != "" {
err = os.MkdirAll(dir, os.ModePerm)
if err != nil {
log.Fatalf("failed to create directory %s, err:%v", dir, err)
}
}
if filename == "" {
output += "coverage.cov"
}
f, err := os.Create(output)
if err != nil {
log.Fatalf("failed to create file %s, err:%v", output, err)
}
defer f.Close()
_, err = io.Copy(f, bytes.NewReader(res))
if err != nil {
log.Fatalf("failed to write file: %v, err: %v", output, err)
}
}
},
}
var (
svrList []string // --service flag
addrList []string // --address flag
force bool // --force flag
output string // --output flag
coverFilePatterns []string // --coverfile flag
skipFilePatterns []string // --skipfile flag
)
func init() {
profileCmd.Flags().StringVarP(&output, "output", "o", "", "download cover profile")
profileCmd.Flags().StringSliceVarP(&svrList, "service", "", nil, "service name to fetch profile, see 'goc list' for all services.")
profileCmd.Flags().StringSliceVarP(&addrList, "address", "", nil, "address to fetch profile, see 'goc list' for all addresses.")
profileCmd.Flags().BoolVarP(&force, "force", "f", false, "force fetching all available profiles")
profileCmd.Flags().StringSliceVarP(&coverFilePatterns, "coverfile", "", nil, "only output coverage data of the files matching the patterns")
profileCmd.Flags().StringSliceVarP(&skipFilePatterns, "skipfile", "", nil, "skip the files matching the patterns when outputing coverage data")
addBasicFlags(profileCmd.Flags())
rootCmd.AddCommand(profileCmd)
}

60
cmd/register.go
View File

@ -1,60 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"log"
"os"
"github.com/qiniu/goc/pkg/cover"
"github.com/spf13/cobra"
)
var registerCmd = &cobra.Command{
Use: "register",
Short: "Register a service into service center",
Long: "Register a service into service center",
Example: `
goc register [flags]
`,
Run: func(cmd *cobra.Command, args []string) {
s := cover.ServiceUnderTest{
Name: name,
Address: address,
}
res, err := cover.NewWorker(center).RegisterService(s)
if err != nil {
log.Fatalf("register service failed, err: %v", err)
}
fmt.Fprint(os.Stdout, string(res))
},
}
var (
name string
address string
)
func init() {
registerCmd.Flags().StringVarP(&center, "center", "", "http://127.0.0.1:7777", "cover profile host center")
registerCmd.Flags().StringVarP(&name, "name", "n", "", "service name")
registerCmd.Flags().StringVarP(&address, "address", "a", "", "service address")
registerCmd.MarkFlagRequired("name")
registerCmd.MarkFlagRequired("address")
rootCmd.AddCommand(registerCmd)
}

58
cmd/remove.go
View File

@ -1,58 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"os"
log "github.com/sirupsen/logrus"
"github.com/qiniu/goc/pkg/cover"
"github.com/spf13/cobra"
)
var removeCmd = &cobra.Command{
Use: "remove",
Short: "Remove the specified service from the register center.",
Long: `Remove the specified service from the register center, after that, goc profile will not collect coverage data from this service anymore`,
Example: `
# Remove the service 'mongo' from the default register center http://127.0.0.1:7777.
goc remove --service=mongo
# Remove the service 'http://127.0.0.1:53' from the specified register center.
goc remove --address="http://127.0.0.1:53" --center=http://192.168.1.1:8080
`,
Run: func(cmd *cobra.Command, args []string) {
p := cover.ProfileParam{
Service: svrList,
Address: addrList,
}
res, err := cover.NewWorker(center).Remove(p)
if err != nil {
log.Fatalf("call host %v failed, err: %v, response: %v", center, err, string(res))
}
fmt.Fprint(os.Stdout, string(res))
},
}
func init() {
addBasicFlags(removeCmd.Flags())
removeCmd.Flags().StringSliceVarP(&svrList, "service", "", nil, "service name to clear profile, see 'goc list' for all services.")
removeCmd.Flags().StringSliceVarP(&addrList, "address", "", nil, "address to clear profile, see 'goc list' for all addresses.")
rootCmd.AddCommand(removeCmd)
}

55
cmd/root.go
View File

@ -1,12 +1,9 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -17,15 +14,9 @@
package cmd
import (
"os"
"path/filepath"
"runtime"
"strconv"
"time"
log "github.com/sirupsen/logrus"
"github.com/qiniu/goc/v2/pkg/config"
"github.com/qiniu/goc/v2/pkg/log"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
var rootCmd = &cobra.Command{
@ -37,52 +28,22 @@ Find more information at:
https://github.com/qiniu/goc
`,
PersistentPreRun: func(cmd *cobra.Command, args []string) {
log.SetReportCaller(true)
log.SetLevel(log.InfoLevel)
log.SetFormatter(&log.TextFormatter{
FullTimestamp: true,
CallerPrettyfier: func(f *runtime.Frame) (string, string) {
dirname, filename := filepath.Split(f.File)
lastelem := filepath.Base(dirname)
filename = filepath.Join(lastelem, filename)
line := strconv.Itoa(f.Line)
return "", "[" + filename + ":" + line + "]"
},
})
if debugGoc == false {
// we only need log in debug mode
log.SetLevel(log.FatalLevel)
log.SetFormatter(&log.TextFormatter{
DisableTimestamp: true,
CallerPrettyfier: func(f *runtime.Frame) (string, string) {
return "", ""
},
})
}
log.DisplayGoc()
// init logger
log.NewLogger()
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if debugInCISyncFile != "" {
f, err := os.Create(debugInCISyncFile)
if err != nil {
log.Fatalln(err)
}
defer f.Close()
time.Sleep(5 * time.Second)
}
PersistentPostRun: func(cmd *cobra.Command, args []string) {
log.Sync()
},
}
func init() {
rootCmd.PersistentFlags().BoolVar(&debugGoc, "debug", false, "run goc in debug mode")
rootCmd.PersistentFlags().StringVar(&debugInCISyncFile, "debugcisyncfile", "", "internal use only, no explain")
rootCmd.PersistentFlags().MarkHidden("debugcisyncfile")
viper.BindPFlags(rootCmd.PersistentFlags())
rootCmd.PersistentFlags().BoolVar(&config.GocConfig.Debug, "debug", false, "run goc in debug mode")
}
// Execute the goc tool
func Execute() {
if err := rootCmd.Execute(); err != nil {
log.Fatalln(err)
}
}

103
cmd/run.go
View File

@ -1,103 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"io/ioutil"
"net"
"os"
"github.com/qiniu/goc/pkg/build"
"github.com/qiniu/goc/pkg/cover"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var runCmd = &cobra.Command{
Use: "run",
Short: "Run covers and runs the named main Go package",
Long: `Run covers and runs the named main Go package,
It is exactly behave as 'go run .' in addition of some internal goc features.`,
Example: `
goc run .
goc run . [--buildflags] [--exec] [--arguments]
`,
Run: func(cmd *cobra.Command, args []string) {
wd, err := os.Getwd()
if err != nil {
log.Fatalf("Fail to build: %v", err)
}
gocBuild, err := build.NewBuild(buildFlags, args, wd, buildOutput)
if err != nil {
log.Fatalf("Fail to run: %v", err)
}
gocBuild.GoRunExecFlag = goRunExecFlag
gocBuild.GoRunArguments = goRunArguments
defer gocBuild.Clean()
server := cover.NewMemoryBasedServer() // only save services in memory
// start goc server
var l = newLocalListener()
go func() {
err = server.Route(ioutil.Discard).RunListener(l)
if err != nil {
log.Fatalf("Start goc server failed: %v", err)
}
}()
gocServer := fmt.Sprintf("http://%s", l.Addr().String())
fmt.Printf("[goc] goc server started: %s \n", gocServer)
// execute covers for the target source with original buildFlags and new GOPATH( tmp:original )
ci := &cover.CoverInfo{
Args: buildFlags,
GoPath: gocBuild.NewGOPATH,
Target: gocBuild.TmpDir,
Mode: coverMode.String(),
Center: gocServer,
AgentPort: "",
IsMod: gocBuild.IsMod,
ModRootPath: gocBuild.ModRootPath,
OneMainPackage: true, // go run is similar with go build, build only one main package
GlobalCoverVarImportPath: gocBuild.GlobalCoverVarImportPath,
}
err = cover.Execute(ci)
if err != nil {
log.Fatalf("Fail to run: %v", err)
}
if err := gocBuild.Run(); err != nil {
log.Fatalf("Fail to run: %v", err)
}
},
}
func init() {
addRunFlags(runCmd.Flags())
rootCmd.AddCommand(runCmd)
}
func newLocalListener() net.Listener {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
if l, err = net.Listen("tcp6", "[::1]:0"); err != nil {
log.Fatalf("failed to listen on a port: %v", err)
}
}
return l
}

55
cmd/server.go
View File

@ -1,54 +1,27 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"github.com/qiniu/goc/pkg/cover"
"github.com/qiniu/goc/v2/pkg/config"
"github.com/qiniu/goc/v2/pkg/server"
"github.com/spf13/cobra"
"log"
)
var serverCmd = &cobra.Command{
Use: "server",
Short: "Start a service registry center",
Long: `Start a service registry center.`,
Example: `
# Start a service registry center, default port :7777.
goc server
Use: "server",
Short: "start a service registry center",
Long: "start a service registry center",
Example: "",
# Start a service registry center with port :8080.
goc server --port=:8080
# Start a service registry center with localhost:8080.
goc server --port=localhost:8080
`,
Run: func(cmd *cobra.Command, args []string) {
server, err := cover.NewFileBasedServer(localPersistence)
if err != nil {
log.Fatalf("New file based server failed, err: %v", err)
}
server.Run(port)
},
Run: serve,
}
var port, localPersistence string
func init() {
serverCmd.Flags().StringVarP(&port, "port", "", ":7777", "listen port to start a coverage host center")
serverCmd.Flags().StringVarP(&localPersistence, "local-persistence", "", "_svrs_address.txt", "the file to save services address information")
// serverCmd.Flags().IntVarP(&config.GocConfig.Port, "port", "", 7777, "listen port to start a coverage host center")
// serverCmd.Flags().StringVarP(&config.GocConfig.StorePath, "storepath", "", "goc.store", "the file to save all goc server information")
serverCmd.Flags().StringVarP(&config.GocConfig.Host, "host", "", "127.0.0.1:7777", "specify the host of the goc server")
rootCmd.AddCommand(serverCmd)
}
func serve(cmd *cobra.Command, args []string) {
server.RunGocServerUntilExit(config.GocConfig.Host)
}

51
cmd/version.go
View File

@ -1,51 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"runtime/debug"
"github.com/spf13/cobra"
)
// the version value will be injected when publishing
var version = "Unstable"
var versionCmd = &cobra.Command{
Use: "version",
Short: "Print the goc version information",
Example: `
# Print the client and server versions for the current context
goc version
`,
Run: func(cmd *cobra.Command, args []string) {
// if it is "Unstable", means user build local or with go get
if version == "Unstable" {
if info, ok := debug.ReadBuildInfo(); ok {
fmt.Println(info.Main.Version)
}
} else {
// otherwise the value is injected in CI
fmt.Println(version)
}
},
}
func init() {
rootCmd.AddCommand(versionCmd)
}

25
cmd/watch.go Normal file
View File

@ -0,0 +1,25 @@
package cmd
import (
"github.com/qiniu/goc/v2/pkg/config"
cli "github.com/qiniu/goc/v2/pkg/watch"
"github.com/spf13/cobra"
)
var watchCmd = &cobra.Command{
Use: "watch",
Short: "watch for profile's real time update",
Long: "watch for profile's real time update",
Example: "",
Run: watch,
}
func init() {
watchCmd.Flags().StringVarP(&config.GocConfig.Host, "host", "", "127.0.0.1:7777", "specify the host of the goc server")
rootCmd.AddCommand(watchCmd)
}
func watch(cmd *cobra.Command, args []string) {
cli.Watch()
}

8
codecov.yaml
View File

@ -1,8 +0,0 @@
comment:
layout: "reach, diff, flags, files"
behavior: new
require_changes: false # if true: only post the comment if coverage changes
require_base: no # [yes :: must have a base report to post]
require_head: yes # [yes :: must have a head report to post]
branches: # branch names that can post comment
- "master"

14
doc/abstract.md Normal file
View File

@ -0,0 +1,14 @@
# 摘要 - 设计原则
goc 的定位是一个专注提升测试体验和项目质量的工具，它不用于生产环境。
当用户从 go 切换为 goc 时，**成本应越小越好**。如果是生产环境无法替代的工具，那使用部署再怎么不便，用户也会趋之若鹜。
因此 v2 版本在如下：
1. goc 命令行使用
2. goc 部署方式（即 agent <-> server 通信方式）
做了大量重构甚至重写。
得益于重写的通信方式，v2 还提供了 watch 模式，为第三方开发自己的实时代码染色系统提供了接口。

16
doc/how_to_parse_args_and_flags_in_goc.md Normal file
View File

@ -0,0 +1,16 @@
# goc 中的参数处理设计
## 背景
goc build/install/run 有不少人反馈使用起来和 go build/install/run 相比，还是有不少的差异。这种差异导致在日常开发、CI/CD 中替换不便，有些带引号的参数会被改写的面目全非。
## 原则
goc build/install/run 会尽可能的模仿 go 原生的方式去处理参数。
## 主要问题
1. goc 使用 cobra 库来组织各个子命令。cobra 对 flag 处理采用的是 posix 风格（两个短横线），和 go 的 flag 处理差异很大（一个短横线）。
2. go 命令中 args 和 flags 有着严格先后顺序。而 cobra 库对 flags 和 args 的位置没有要求。
3. 参数中 `[packages]` 有多种组合情况，会影响到插桩的起始位置。
4. goc 还有自己参数，且需要和**非** goc build/install/run 的子命令保持一致（两个短横线）。

9
doc/log.md Normal file
View File

@ -0,0 +1,9 @@
# 日志设计
## 酷炫的终端
1. 非 debug 模式模仿 devspace：
1. 支持不同级别
2. 不同颜色
3. 长时间操作有转圈动画
2. debug 模式下为常见日志库行为，这里内部选用的是`go.uber.org/zap`

88
doc/protocol.md Normal file
View File

@ -0,0 +1,88 @@
# 通信协议设计
## 背景
v1 版本中，被插桩的服务会暴露一个 HTTP 接口，由 goc server 来访问获取覆盖率。
该方式要求被插桩的服务要有一个外界可访问的 ip + port。
如果被测服务躲在 NAT 网络下，该方式就不可行了，典型的就是被测服务由 docker 拉起，而 goc server 部署在另外的网络。
## 新设计选择
新设计只需要暴露 goc server 的地址，由插桩服务发起连接，然后保持长连接，在该长连接上构建 goc 自己的业务逻辑。
### 自行设计 TCP 应用层协议
go 语言做网络编程非常适合，非阻塞地处理“粘包”也不麻烦。但设计出来不管是纯二进制的、还是类似 HTTP 的，都不会是通用协议，后续维护和扩展估计是个大坑。
### websocket + net/rpc
与 `websocket + jsonrpc2` 的区别就是没有流式调用，纯 rpc 调用。
在这种模式下，agent 和 goc server 的角色和 rpc 中的角色并不对应。agent 是 rpc server， goc server 是 rpc client。由于 websocket 是长连接，上述角色的倒换是可行的。
1. goc server 发起获取覆盖率 rpc，agent 响应 rpc，goc server 汇总覆盖率
2. watch 模式下，agent 再开一条 websocket 连接到 goc server，这条连接中角色不再颠倒，goc server 就是 rpc server
### websocket + jsonrpc2
websocket + jsonrpc2 有流式调用，消息边界。非常适合
我找到 `github.com/goriila/websocket` 和 `github.com/sourcegraph/jsonrpc2` 库，后者 import 了前者，前者没有 import 任何其它外部库，全是标准库。把两个库的代码合并一下：`github.com/lyyyuna/jsonrpc2` 就是一个无任何外部库引用的 jsonrpc 实现。
这非常适合由插桩代码来使用，因为该库没有再引用其它库，**不会污染原服务的依赖关系**。
### gRPC
老实说 gRPC 在这里更适合作为通信协议来使用，更快更通用，流式调用也有，上一小节的 `github.com/sourcegraph/jsonrpc2` 使用广度就很低。
但 gRPC 的 go 实现有一个很大的缺点，用了一些非标准库，且有版本依赖。我们不清楚原服务是不是有特定 gRPC 要求，或是 goc 插入的 gRPC 库会导致编译依赖冲突，或者是编译后运行冲突。
所以不适合。
### 结论
先使用 websocket + net/rpc 来做吧。
## 协议内容
### 注册
注册信息放入 websocket url 中，例如：
```
/v2/internal/ws/rpcstream?cmdline=.%2Fcmd&hostname=nuc&pid=1699804
```
注册信息为：
1. 完整的命令行
2. hostname
3. 进程 PID
goc server 再加上 remote ip 对四个元信息生成一个唯一 hash id，作为该 agent 的 ID。
### 获取覆盖率
```
GocAgent.GetProfile
ProfileReq: getprofile
```
### 清空覆盖率
```
GocAgent.ResetProfile
ProfileReq: resetprofile
```
### watch
### 异常处理
goc server 端遇到 err 就关闭对应 agent 的 websocket 连接。

16
doc/return_error_or_fatal.md Normal file
View File

@ -0,0 +1,16 @@
# 返回 error 还是原地 fatal
哪种好呢？在一个纯命令行应用中，及时 fatal，并能在 panic 中打印出堆栈对调试即为有益。
若是返回 error，虽然看起来：
1. 对每种异常处理都定义的很清楚（有明确的错误消息）
2. 写单测也更容易
如果函数调用嵌套很深，导致 error 被一层层包裹返回，以至于在最上层拿到 error 时你都分不清到底出错点在哪 -_-，虽然 Go 1.13 的 error wrap 一定程度消减了这个问题，但还是不如 panic 来的好调试。（如果是一个 web 应用，则做法相反，web 的查错主要靠详细的日志，这时候不应鼓励过多的 panic 或 os.Exit(1))
而且 goc 并不是一个供大家使用的库，很多 error 没必要往上抛。
目前来看 fatal 还是更适合。
那如何保障测试质量呢？我们不应被覆盖率所绑架，只要测试用例全面，一样可以保证质量。

BIN
docs/images/goc-vscode.gif
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 1.3 MiB

BIN
docs/images/intro.gif
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 1.1 MiB

BIN
docs/images/wechat.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 231 KiB

35
go.mod
View File

@ -1,24 +1,23 @@
module github.com/qiniu/goc
module github.com/qiniu/goc/v2
go 1.13
go 1.16
require (
github.com/gin-gonic/gin v1.6.3
github.com/google/go-github v17.0.0+incompatible
github.com/hashicorp/go-retryablehttp v0.6.6
github.com/julienschmidt/httprouter v1.2.0
github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/olekukonko/tablewriter v0.0.4
github.com/qiniu/api.v7/v7 v7.5.0
github.com/sirupsen/logrus v1.6.0
github.com/spf13/cobra v1.0.0
github.com/gin-gonic/gin v1.7.2
github.com/gorilla/websocket v1.4.2
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213
github.com/mattn/go-colorable v0.1.8 // indirect
github.com/mattn/go-isatty v0.0.13 // indirect
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
github.com/spf13/cobra v1.1.3
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.6.2
github.com/stretchr/testify v1.5.1
github.com/tongjingran/copy v1.4.2
golang.org/x/mod v0.3.0
golang.org/x/net v0.0.0-20200625001655-4c5254603344
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
golang.org/x/tools v0.0.0-20200730221956-1ac65761fe2c
k8s.io/test-infra v0.0.0-20200511080351-8ac9dbfab055
go.uber.org/zap v1.17.0
golang.org/x/mod v0.4.2
golang.org/x/sys v0.0.0-20210608053332-aa57babbf139 // indirect
golang.org/x/tools v0.1.3
k8s.io/kubectl v0.21.2
k8s.io/test-infra v0.0.0-20210618100605-34aa2f2aa75b
)
replace k8s.io/client-go => k8s.io/client-go v0.21.1

1070
go.sum
View File

File diff suppressed because it is too large Load Diff

7
goc.go → main.go
View File

@ -1,12 +1,9 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -16,7 +13,9 @@
package main
import "github.com/qiniu/goc/cmd"
import (
"github.com/qiniu/goc/v2/cmd"
)
func main() {
cmd.Execute()

223
pkg/build/build.go
View File

@ -1,168 +1,93 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package build
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/qiniu/goc/pkg/cover"
log "github.com/sirupsen/logrus"
"github.com/qiniu/goc/v2/pkg/config"
"github.com/qiniu/goc/v2/pkg/cover"
"github.com/qiniu/goc/v2/pkg/flag"
"github.com/qiniu/goc/v2/pkg/log"
"github.com/spf13/cobra"
)
// Build is to describe the building/installing process of a goc build/install
// Build struct a build
// most configurations are stored in global variables: config.GocConfig & config.GoConfig
type Build struct {
Pkgs map[string]*cover.Package // Pkg list parsed from "go list -json ./..." command
NewGOPATH string // the new GOPATH
OriGOPATH string // the original GOPATH
WorkingDir string // the working directory
TmpDir string // the temporary directory to build the project
TmpWorkingDir string // the working directory in the temporary directory, which is corresponding to the current directory in the project directory
IsMod bool // determine whether it is a Mod project
Root string
// go 1.11, go 1.12 has no Root
// Project Root:
// 1. legacy, root == GOPATH
// 2. mod, root == go.mod Dir
ModRoot string // path for go.mod
ModRootPath string // import path for the whole project
Target string // the binary name that go build generate
// keep compatible with go commands:
// go run [build flags] [-exec xprog] package [arguments...]
// go build [-o output] [-i] [build flags] [packages]
// go install [-i] [build flags] [packages]
BuildFlags string // Build flags
Packages string // Packages that needs to build
GoRunExecFlag string // for the -exec flags in go run command
GoRunArguments string // for the '[arguments]' parameters in go run command
OneMainPackage bool // whether this build is a go build or go install? true: build, false: install
GlobalCoverVarImportPath string // Importpath for storing cover variables
GlobalCoverVarFilePath string // Importpath for storing cover variables
}
// NewBuild creates a Build struct which can build from goc temporary directory,
// and generate binary in current working directory
func NewBuild(buildflags string, args []string, workingDir string, outputDir string) (*Build, error) {
if err := checkParameters(args, workingDir); err != nil {
return nil, err
}
// buildflags = buildflags + " -o " + outputDir
b := &Build{
BuildFlags: buildflags,
Packages: strings.Join(args, " "),
WorkingDir: workingDir,
}
if false == b.validatePackageForBuild() {
log.Errorln(ErrWrongPackageTypeForBuild)
return nil, ErrWrongPackageTypeForBuild
}
if err := b.MvProjectsToTmp(); err != nil {
return nil, err
}
dir, err := b.determineOutputDir(outputDir)
b.Target = dir
if err != nil {
return nil, err
}
return b, nil
// NewBuild creates a Build struct
//
// consumes args, get package dirs, read project meta info.
func NewBuild(cmd *cobra.Command, args []string) *Build {
b := &Build{}
// 1. 解析 goc 命令行和 go 命令行
remainedArgs := flag.BuildCmdArgsParse(cmd, args)
// 2. 解析 go 包位置
flag.GetPackagesDir(remainedArgs)
// 3. 读取工程元信息：go.mod, pkgs list ...
b.readProjectMetaInfo()
// 4. 展示元信息
b.displayProjectMetaInfo()
return b
}
// Build calls 'go build' tool to do building
func (b *Build) Build() error {
log.Infoln("Go building in temp...")
// new -o will overwrite previous ones
b.BuildFlags = b.BuildFlags + " -o " + b.Target
cmd := exec.Command("/bin/bash", "-c", "go build "+b.BuildFlags+" "+b.Packages)
cmd.Dir = b.TmpWorkingDir
// Build starts go build
//
// 1. copy project to temp,
// 2. inject cover variables and functions into the project,
// 3. build the project in temp.
func (b *Build) Build() {
// 1. 拷贝至临时目录
b.copyProjectToTmp()
defer b.clean()
log.Donef("project copied to temporary directory")
// 2. inject cover vars
cover.Inject()
// 3. build in the temp project
b.doBuildInTemp()
}
func (b *Build) doBuildInTemp() {
log.StartWait("building the injected project")
goflags := config.GocConfig.Goflags
// 检查用户是否自定义了 -o
oSet := false
for _, flag := range goflags {
if flag == "-o" {
oSet = true
}
}
// 如果没被设置就加一个至原命令执行的目录
if !oSet {
goflags = append(goflags, "-o", config.GocConfig.CurWd)
}
pacakges := config.GocConfig.Packages
goflags = append(goflags, pacakges...)
args := []string{"build"}
args = append(args, goflags...)
// go 命令行由 go build [-o output] [build flags] [packages] 组成
cmd := exec.Command("go", args...)
cmd.Dir = config.GocConfig.TmpWd
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if b.NewGOPATH != "" {
// Change to temp GOPATH for go install command
cmd.Env = append(os.Environ(), fmt.Sprintf("GOPATH=%v", b.NewGOPATH))
log.Infof("go build cmd is: %v, in path [%v]", cmd.Args, cmd.Dir)
if err := cmd.Start(); err != nil {
log.Fatalf("fail to execute go build: %v", err)
}
if err := cmd.Wait(); err != nil {
log.Fatalf("fail to execute go build: %v", err)
}
log.Printf("go build cmd is: %v", cmd.Args)
err := cmd.Start()
if err != nil {
return fmt.Errorf("fail to execute: %v, err: %w", cmd.Args, err)
}
if err = cmd.Wait(); err != nil {
return fmt.Errorf("fail to execute: %v, err: %w", cmd.Args, err)
}
log.Infoln("Go build exit successful.")
return nil
}
// determineOutputDir, as we only allow . as package name,
// the binary name is always same as the directory name of current directory
func (b *Build) determineOutputDir(outputDir string) (string, error) {
if b.TmpDir == "" {
return "", fmt.Errorf("can only be called after Build.MvProjectsToTmp(): %w", ErrEmptyTempWorkingDir)
}
// fix #43
if outputDir != "" {
abs, err := filepath.Abs(outputDir)
if err != nil {
return "", fmt.Errorf("Fail to transform the path: %v to absolute path: %v", outputDir, err)
}
return abs, nil
}
// fix #43
// use target name from `go list -json ./...` of the main module
targetName := ""
for _, pkg := range b.Pkgs {
if pkg.Name == "main" {
if pkg.Target != "" {
targetName = filepath.Base(pkg.Target)
} else {
targetName = filepath.Base(pkg.Dir)
}
break
}
}
return filepath.Join(b.WorkingDir, targetName), nil
}
// validatePackageForBuild only allow . as package name
func (b *Build) validatePackageForBuild() bool {
if b.Packages == "." || b.Packages == "" {
return true
}
return false
}
func checkParameters(args []string, workingDir string) error {
if len(args) > 1 {
log.Errorln(ErrTooManyArgs)
return ErrTooManyArgs
}
if workingDir == "" {
return ErrInvalidWorkingDir
}
log.Infof("Working directory: %v", workingDir)
return nil
// done
log.StopWait()
log.Donef("go build done")
}

101
pkg/build/build_test.go
View File

@ -1,101 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package build
import (
"errors"
"fmt"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
)
func TestInvalidPackage(t *testing.T) {
workingDir := filepath.Join(baseDir, "../../tests/samples/simple_project")
gopath := ""
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
_, err := NewBuild("", []string{"example.com/simple-project"}, workingDir, "")
if !assert.Equal(t, err, ErrWrongPackageTypeForBuild) {
assert.FailNow(t, "the package name should be invalid")
}
}
func TestBasicBuildForModProject(t *testing.T) {
workingDir := filepath.Join(baseDir, "../../tests/samples/simple_project")
gopath := ""
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
fmt.Println(workingDir)
buildFlags, args, buildOutput := "", []string{"."}, ""
gocBuild, err := NewBuild(buildFlags, args, workingDir, buildOutput)
if !assert.Equal(t, err, nil) {
assert.FailNow(t, "should create temporary directory successfully")
}
err = gocBuild.Build()
if !assert.Equal(t, err, nil) {
assert.FailNow(t, "temporary directory should build successfully")
}
}
func TestCheckParameters(t *testing.T) {
err := checkParameters([]string{"aa", "bb"}, "aa")
assert.Equal(t, err, ErrTooManyArgs, "too many arguments should failed")
err = checkParameters([]string{"aa"}, "")
assert.Equal(t, err, ErrInvalidWorkingDir, "empty working directory should failed")
}
func TestDetermineOutputDir(t *testing.T) {
b := &Build{}
_, err := b.determineOutputDir("")
assert.Equal(t, errors.Is(err, ErrEmptyTempWorkingDir), true, "called before Build.MvProjectsToTmp() should fail")
b.TmpDir = "fake"
_, err = b.determineOutputDir("xx")
assert.Equal(t, err, nil, "should return a directory")
}
func TestInvalidPackageNameForBuild(t *testing.T) {
workingDir := filepath.Join(baseDir, "../../tests/samples/simple_project")
gopath := filepath.Join(baseDir, "../../tests/samples/simple_project", "testhome")
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
buildFlags, packages := "", []string{"main.go"}
_, err := NewBuild(buildFlags, packages, workingDir, "")
if !assert.Equal(t, err, ErrWrongPackageTypeForBuild) {
assert.FailNow(t, "should not success with non . or ./... package")
}
}
// test NewBuild with wrong parameters
func TestNewBuildWithWrongParameters(t *testing.T) {
_, err := NewBuild("", []string{"a.go", "b.go"}, "cur", "cur")
assert.Equal(t, err, ErrTooManyArgs)
_, err = NewBuild("", []string{"a.go"}, "", "cur")
assert.Equal(t, err, ErrInvalidWorkingDir)
}

24
pkg/build/errors.go
View File

@ -1,24 +0,0 @@
package build
import (
"errors"
)
var (
// ErrShouldNotReached represents the logic should not be reached in normal flow
ErrShouldNotReached = errors.New("should never be reached")
// ErrGocShouldExecInProject represents goc currently not support for the project
ErrGocShouldExecInProject = errors.New("goc not support for such project directory")
// ErrWrongPackageTypeForInstall represents goc install command only support limited arguments
ErrWrongPackageTypeForInstall = errors.New("packages only support \".\" and \"./...\"")
// ErrWrongPackageTypeForBuild represents goc build command only support limited arguments
ErrWrongPackageTypeForBuild = errors.New("packages only support \".\"")
// ErrTooManyArgs represents goc CLI only support limited arguments
ErrTooManyArgs = errors.New("too many args")
// ErrInvalidWorkingDir represents the working directory is invalid
ErrInvalidWorkingDir = errors.New("the working directory is invalid")
// ErrEmptyTempWorkingDir represent the error that temporary working directory is empty
ErrEmptyTempWorkingDir = errors.New("temporary working directory is empty")
// ErrNoPlaceToInstall represents the err that no place to install the generated binary
ErrNoPlaceToInstall = errors.New("don't know where to install")
)

116
pkg/build/goenv.go Normal file
View File

@ -0,0 +1,116 @@
package build
import (
"bytes"
"encoding/json"
"io"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"github.com/qiniu/goc/v2/pkg/config"
"github.com/qiniu/goc/v2/pkg/log"
)
// readProjectMetaInfo reads all meta informations of the corresponding project
func (b *Build) readProjectMetaInfo() {
// get gopath & gobin
config.GocConfig.GOPATH = b.readGOPATH()
config.GocConfig.GOBIN = b.readGOBIN()
// 获取 [packages] 及其依赖的 package list
pkgs := b.listPackages(config.GocConfig.CurWd)
// get mod info
for _, pkg := range pkgs {
// check if go modules is enabled
if pkg.Module == nil {
log.Fatalf("Go module is not enabled, please set GO111MODULE=auto or on")
}
// 工程根目录
config.GocConfig.CurModProjectDir = pkg.Root
config.GocConfig.ImportPath = pkg.Module.Path
break
}
// 如果当前目录不是工程根目录，那再次 go list 一次，获取整个工程的包信息
if config.GocConfig.CurWd != config.GocConfig.CurModProjectDir {
config.GocConfig.Pkgs = b.listPackages(config.GocConfig.CurModProjectDir)
} else {
config.GocConfig.Pkgs = pkgs
}
// get tmp folder name
config.GocConfig.TmpModProjectDir = filepath.Join(os.TempDir(), tmpFolderName(config.GocConfig.CurModProjectDir))
// get working dir in the corresponding tmp dir
config.GocConfig.TmpWd = filepath.Join(config.GocConfig.TmpModProjectDir, config.GocConfig.CurWd[len(config.GocConfig.CurModProjectDir):])
// get GlobalCoverVarImportPath
config.GocConfig.GlobalCoverVarImportPath = path.Join(config.GocConfig.ImportPath, tmpFolderName(config.GocConfig.CurModProjectDir))
log.Donef("project meta information parsed")
}
// displayProjectMetaInfo prints basic infomation of this project to stdout
func (b *Build) displayProjectMetaInfo() {
log.Infof("GOPATH: %v", config.GocConfig.GOPATH)
log.Infof("GOBIN: %v", config.GocConfig.GOBIN)
log.Infof("Project Directory: %v", config.GocConfig.CurModProjectDir)
log.Infof("Temporary Project Directory: %v", config.GocConfig.TmpModProjectDir)
log.Infof("")
}
// readGOPATH reads GOPATH use go env GOPATH command
func (b *Build) readGOPATH() string {
out, err := exec.Command("go", "env", "GOPATH").Output()
if err != nil {
log.Fatalf("fail to read GOPATH: %v", err)
}
return strings.TrimSpace(string(out))
}
// readGOBIN reads GOBIN use go env GOBIN command
func (b *Build) readGOBIN() string {
out, err := exec.Command("go", "env", "GOBIN").Output()
if err != nil {
log.Fatalf("fail to read GOBIN: %v", err)
}
return strings.TrimSpace(string(out))
}
// listPackages list all packages under specific via go list command.
func (b *Build) listPackages(dir string) map[string]*config.Package {
cmd := exec.Command("go", "list", "-json", "./...")
cmd.Dir = dir
var errBuf bytes.Buffer
cmd.Stderr = &errBuf
out, err := cmd.Output()
if err != nil {
log.Fatalf("execute go list -json failed, err: %v, stdout: %v, stderr: %v", err, string(out), errBuf.String())
}
// 有些时候 go 命令会打印一些信息到 stderr，但其实命令整体是成功运行了
if errBuf.String() != "" {
log.Errorf("%v", errBuf.String())
}
dec := json.NewDecoder(bytes.NewBuffer(out))
pkgs := make(map[string]*config.Package, 0)
for {
var pkg config.Package
if err := dec.Decode(&pkg); err != nil {
if err == io.EOF {
break
}
log.Fatalf("reading go list output error: %v", err)
}
if pkg.Error != nil {
log.Fatalf("list package %s failed with output: %v", pkg.ImportPath, pkg.Error)
}
pkgs[pkg.ImportPath] = &pkg
}
return pkgs
}

90
pkg/build/gomodules.go
View File

@ -1,90 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package build
import (
"io/ioutil"
"path/filepath"
log "github.com/sirupsen/logrus"
"github.com/tongjingran/copy"
"golang.org/x/mod/modfile"
)
func (b *Build) cpGoModulesProject() {
for _, v := range b.Pkgs {
if v.Name == "main" {
dst := b.TmpDir
src := v.Module.Dir
if err := copy.Copy(src, dst, copy.Options{Skip: skipCopy}); err != nil {
log.Errorf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err)
}
break
} else {
continue
}
}
}
// updateGoModFile rewrites the go.mod file in the temporary directory,
// if it has a 'replace' directive, and the directive has a relative local path
// it will be rewritten with a absolute path.
// ex.
// suppose original project is located at /path/to/aa/bb/cc, go.mod contains a directive:
// 'replace github.com/qiniu/bar => ../home/foo/bar'
// after the project is copied to temporary directory, it should be rewritten as
// 'replace github.com/qiniu/bar => /path/to/aa/bb/home/foo/bar'
func (b *Build) updateGoModFile() (updateFlag bool, newModFile []byte, err error) {
tempModfile := filepath.Join(b.TmpDir, "go.mod")
buf, err := ioutil.ReadFile(tempModfile)
if err != nil {
return
}
oriGoModFile, err := modfile.Parse(tempModfile, buf, nil)
if err != nil {
return
}
updateFlag = false
for index := range oriGoModFile.Replace {
replace := oriGoModFile.Replace[index]
oldPath := replace.Old.Path
oldVersion := replace.Old.Version
newPath := replace.New.Path
newVersion := replace.New.Version
// replace to a local filesystem does not have a version
// absolute path no need to rewrite
if newVersion == "" && !filepath.IsAbs(newPath) {
var absPath string
fullPath := filepath.Join(b.ModRoot, newPath)
absPath, _ = filepath.Abs(fullPath)
// DropReplace & AddReplace will not return error
// so no need to check the error
_ = oriGoModFile.DropReplace(oldPath, oldVersion)
_ = oriGoModFile.AddReplace(oldPath, oldVersion, absPath, newVersion)
updateFlag = true
}
}
oriGoModFile.Cleanup()
// Format will not return error, so ignore the returned error
// func (f *File) Format() ([]byte, error) {
// return Format(f.Syntax), nil
// }
newModFile, _ = oriGoModFile.Format()
return
}

103
pkg/build/gomodules_test.go
View File

@ -1,103 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package build
import (
"bytes"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"github.com/qiniu/goc/pkg/cover"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
func captureOutput(f func()) string {
var buf bytes.Buffer
log.SetOutput(&buf)
f()
log.SetOutput(os.Stderr)
return buf.String()
}
// copy in cpGoModulesProject of invalid src, dst name
func TestModProjectCopyWithUnexistedDir(t *testing.T) {
pkgs := make(map[string]*cover.Package)
pkgs["main"] = &cover.Package{
Name: "main",
Module: &cover.ModulePublic{
Dir: "not exied, ia mas duser", // not real one, should fail copy
},
}
pkgs["another"] = &cover.Package{}
b := &Build{
TmpDir: "sdfsfev2234444", // not real one, should fail copy
Pkgs: pkgs,
}
output := captureOutput(b.cpGoModulesProject)
assert.Equal(t, strings.Contains(output, "Failed to Copy"), true)
}
// test go mod file udpate
func TestUpdateModFileIfContainsReplace(t *testing.T) {
workingDir := filepath.Join(baseDir, "../../tests/samples/gomod_samples/a")
b := &Build{
TmpDir: workingDir,
ModRoot: "/aa/bb/cc",
}
// replace with relative local file path should be rewrite
updated, newmod, err := b.updateGoModFile()
assert.Equal(t, err, nil)
assert.Equal(t, updated, true)
assert.Contains(t, string(newmod), "replace github.com/qiniu/bar => /aa/bb/home/foo/bar")
// old replace should be removed
assert.NotContains(t, string(newmod), "github.com/qiniu/bar => ../home/foo/bar")
// normal replace should not be rewrite
assert.Contains(t, string(newmod), "github.com/qiniu/bar2 => github.com/baniu/bar3 v1.2.3")
}
// test wrong go mod file
func TestWithWrongGoModFile(t *testing.T) {
// go.mod not exist
workingDir := filepath.Join(baseDir, "../../tests/samples/xxxxxxxxxxxx/a")
b := &Build{
TmpDir: workingDir,
ModRoot: "/aa/bb/cc",
}
updated, _, err := b.updateGoModFile()
assert.Equal(t, errors.Is(err, os.ErrNotExist), true)
assert.Equal(t, updated, false)
// a wrong format go mod
workingDir = filepath.Join(baseDir, "../../tests/samples/gomod_samples/b")
b = &Build{
TmpDir: workingDir,
ModRoot: "/aa/bb/cc",
}
updated, _, err = b.updateGoModFile()
assert.NotEqual(t, err, nil)
assert.Equal(t, updated, false)
}

115
pkg/build/install.go
View File

@ -1,87 +1,62 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package build
import (
"fmt"
"os"
"os/exec"
"strings"
log "github.com/sirupsen/logrus"
"github.com/qiniu/goc/v2/pkg/config"
"github.com/qiniu/goc/v2/pkg/cover"
"github.com/qiniu/goc/v2/pkg/log"
"github.com/spf13/cobra"
)
// NewInstall creates a Build struct which can install from goc temporary directory
func NewInstall(buildflags string, args []string, workingDir string) (*Build, error) {
if err := checkParameters(args, workingDir); err != nil {
return nil, err
}
b := &Build{
BuildFlags: buildflags,
Packages: strings.Join(args, " "),
WorkingDir: workingDir,
}
if false == b.validatePackageForInstall() {
log.Errorln(ErrWrongPackageTypeForInstall)
return nil, ErrWrongPackageTypeForInstall
}
if err := b.MvProjectsToTmp(); err != nil {
return nil, err
}
return b, nil
func NewInstall(cmd *cobra.Command, args []string) *Build {
return NewBuild(cmd, args)
}
// Install use the 'go install' tool to install packages
func (b *Build) Install() error {
log.Println("Go building in temp...")
cmd := exec.Command("/bin/bash", "-c", "go install "+b.BuildFlags+" "+b.Packages)
cmd.Dir = b.TmpWorkingDir
// Install starts go install
//
// 1. copy project to temp,
// 2. inject cover variables and functions into the project,
// 3. install the project in temp.
func (b *Build) Install() {
// 1. 拷贝至临时目录
b.copyProjectToTmp()
defer b.clean()
log.Donef("project copied to temporary directory")
// 2. inject cover vars
cover.Inject()
// 3. install in the temp project
b.doInstallInTemp()
}
func (b *Build) doInstallInTemp() {
log.StartWait("installing the injected project")
goflags := config.GocConfig.Goflags
pacakges := config.GocConfig.Packages
goflags = append(goflags, pacakges...)
args := []string{"install"}
args = append(args, goflags...)
// go 命令行由 go install [build flags] [packages] 组成
cmd := exec.Command("go", args...)
cmd.Dir = config.GocConfig.TmpWd
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
whereToInstall, err := b.findWhereToInstall()
if err != nil {
// ignore the err
log.Errorf("No place to install: %v", err)
log.Infof("go install cmd is: %v, in path [%v]", cmd.Args, cmd.Dir)
if err := cmd.Start(); err != nil {
log.Fatalf("fail to execute go install: %v", err)
}
// Change the temp GOBIN, to force binary install to original place
cmd.Env = append(os.Environ(), fmt.Sprintf("GOBIN=%v", whereToInstall))
if b.NewGOPATH != "" {
// Change to temp GOPATH for go install command
cmd.Env = append(cmd.Env, fmt.Sprintf("GOPATH=%v", b.NewGOPATH))
if err := cmd.Wait(); err != nil {
log.Fatalf("fail to execute go install: %v", err)
}
log.Infof("go install cmd is: %v", cmd.Args)
err = cmd.Start()
if err != nil {
log.Errorf("Fail to execute: %v. The error is: %v", cmd.Args, err)
return err
}
if err = cmd.Wait(); err != nil {
log.Errorf("go install failed. The error is: %v", err)
return err
}
log.Infof("Go install successful. Binary installed in: %v", whereToInstall)
return nil
}
func (b *Build) validatePackageForInstall() bool {
if b.Packages == "." || b.Packages == "" || b.Packages == "./..." {
return true
}
return false
// done
log.StopWait()
log.Donef("go install done")
}

42
pkg/build/install_test.go
View File

@ -1,42 +0,0 @@
package build
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
)
func TestBasicInstallForModProject(t *testing.T) {
workingDir := filepath.Join(baseDir, "../../tests/samples/simple_project")
gopath := filepath.Join(baseDir, "../../tests/samples/simple_project", "testhome")
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
buildFlags, packages := "", []string{"."}
gocBuild, err := NewInstall(buildFlags, packages, workingDir)
if !assert.Equal(t, err, nil) {
assert.FailNow(t, "should create temporary directory successfully")
}
err = gocBuild.Install()
if !assert.Equal(t, err, nil) {
assert.FailNow(t, "temporary directory should build successfully")
}
}
func TestInvalidPackageNameForInstall(t *testing.T) {
workingDir := filepath.Join(baseDir, "../../tests/samples/simple_project")
gopath := filepath.Join(baseDir, "../../tests/samples/simple_project", "testhome")
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
buildFlags, packages := "", []string{"main.go"}
_, err := NewInstall(buildFlags, packages, workingDir)
if !assert.Equal(t, err, ErrWrongPackageTypeForInstall) {
assert.FailNow(t, "should not success with non . or ./... package")
}
}

104
pkg/build/legacy.go
View File

@ -1,104 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package build
import (
"os"
"path/filepath"
"strings"
log "github.com/sirupsen/logrus"
"github.com/qiniu/goc/pkg/cover"
"github.com/tongjingran/copy"
)
func (b *Build) cpLegacyProject() {
visited := make(map[string]bool)
for k, v := range b.Pkgs {
dst := filepath.Join(b.TmpDir, "src", k)
src := v.Dir
if _, ok := visited[src]; ok {
// Skip if already copied
continue
}
if err := copy.Copy(src, dst, copy.Options{Skip: skipCopy}); err != nil {
log.Errorf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err)
}
visited[src] = true
b.cpDepPackages(v, visited)
}
}
// only cp dependency in root(current gopath),
// skip deps in other GOPATHs
func (b *Build) cpDepPackages(pkg *cover.Package, visited map[string]bool) {
gopath := pkg.Root
for _, dep := range pkg.Deps {
src := filepath.Join(gopath, "src", dep)
// Check if copied
if _, ok := visited[src]; ok {
// Skip if already copied
continue
}
// Check if we can found in the root gopath
_, err := os.Stat(src)
if err != nil {
continue
}
dst := filepath.Join(b.TmpDir, "src", dep)
if err := copy.Copy(src, dst, copy.Options{Skip: skipCopy}); err != nil {
log.Errorf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err)
}
visited[src] = true
}
}
func (b *Build) cpNonStandardLegacy() {
for _, v := range b.Pkgs {
if v.Name == "main" {
dst := b.TmpDir
src := v.Dir
if err := copy.Copy(src, dst, copy.Options{Skip: skipCopy}); err != nil {
log.Printf("Failed to Copy the folder from %v to %v, the error is: %v ", src, dst, err)
}
break
}
}
}
// skipCopy skip copy .git dir and irregular files
func skipCopy(src string, info os.FileInfo) (bool, error) {
irregularModeType := os.ModeNamedPipe | os.ModeSocket | os.ModeDevice | os.ModeCharDevice | os.ModeIrregular
if strings.HasSuffix(src, "/.git") {
log.Infof("Skip .git dir [%s]", src)
return true, nil
}
if info.Mode()&irregularModeType != 0 {
log.Warnf("Skip file [%s], the file mode is [%s]", src, info.Mode().String())
return true, nil
}
return false, nil
}

133
pkg/build/legacy_test.go
View File

@ -1,133 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package build
import (
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/qiniu/goc/pkg/cover"
"github.com/stretchr/testify/assert"
)
// copy in cpLegacyProject/cpNonStandardLegacy of invalid src, dst name
func TestLegacyProjectCopyWithUnexistedDir(t *testing.T) {
pkgs := make(map[string]*cover.Package)
pkgs["main"] = &cover.Package{
Module: &cover.ModulePublic{
Dir: "not exied, ia mas duser", // not real one, should fail copy
},
Dir: "not exit, iasdfs",
Name: "main",
}
pkgs["another"] = &cover.Package{}
b := &Build{
TmpDir: "sdfsfev2234444", // not real one, should fail copy
Pkgs: pkgs,
}
output := captureOutput(b.cpLegacyProject)
assert.Equal(t, strings.Contains(output, "Failed to Copy"), true)
output = captureOutput(b.cpNonStandardLegacy)
assert.Equal(t, strings.Contains(output, "Failed to Copy"), true)
}
// copy in cpDepPackages of invalid dst name
func TestDepPackagesCopyWithInvalidDir(t *testing.T) {
gopath := filepath.Join(baseDir, "../../tests/samples/simple_gopath_project")
pkg := &cover.Package{
Module: &cover.ModulePublic{
Dir: "not exied, ia mas duser",
},
Root: gopath,
Deps: []string{"qiniu.com", "ddfee 2344234"},
}
b := &Build{
TmpDir: "/", // "/" is invalid dst in Linux, it should fail
}
output := captureOutput(func() {
visited := make(map[string]bool)
b.cpDepPackages(pkg, visited)
})
assert.Equal(t, strings.Contains(output, "Failed to Copy"), true)
}
type MockFile struct {
name string
size int64
mode os.FileMode
modTime time.Time
isDir bool
}
func (m MockFile) Name() string {
return m.name
}
func (m MockFile) Size() int64 {
return m.size
}
func (m MockFile) Mode() os.FileMode {
return m.mode
}
func (m MockFile) ModTime() time.Time {
return m.modTime
}
func (m MockFile) IsDir() bool {
return m.isDir
}
func (m MockFile) Sys() interface{} {
return nil
}
// skipCopy verify
func TestSkipCopy(t *testing.T) {
testCases := map[string]struct {
inputSrc string
inputInfo MockFile
expected bool
}{
"src with /.git suffix": {inputSrc: "/test/.git", inputInfo: MockFile{mode: 0}, expected: true},
"src with ./git suffix": {inputSrc: "/test.git", inputInfo: MockFile{mode: 0}, expected: false},
"src with /.gita suffix": {inputSrc: "/test/.gita", inputInfo: MockFile{mode: 0}, expected: false},
"src with /.git in middle": {inputSrc: "/test/.git/test", inputInfo: MockFile{mode: 0}, expected: false},
"irregular file": {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeIrregular}, expected: true},
"dir file": {inputSrc: "/test", inputInfo: MockFile{isDir: true, mode: os.ModeDir}, expected: false},
"temporary file": {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeTemporary}, expected: false},
"symlink file": {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeSymlink}, expected: false},
"device file": {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeDevice}, expected: true},
"named pipe file": {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeNamedPipe}, expected: true},
"socket file": {inputSrc: "/test", inputInfo: MockFile{mode: os.ModeSocket}, expected: true},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
output, err := skipCopy(tc.inputSrc, tc.inputInfo)
assert.NoError(t, err)
assert.Equal(t, output, tc.expected)
})
}
}

50
pkg/build/run.go
View File

@ -1,50 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package build
import (
"fmt"
"os"
"os/exec"
log "github.com/sirupsen/logrus"
)
// Run excutes the main package in addition with the internal goc features
func (b *Build) Run() error {
cmd := exec.Command("/bin/bash", "-c", "go run "+b.BuildFlags+" "+b.GoRunExecFlag+" "+b.Packages+" "+b.GoRunArguments)
cmd.Dir = b.TmpWorkingDir
if b.NewGOPATH != "" {
// Change to temp GOPATH for go install command
cmd.Env = append(os.Environ(), fmt.Sprintf("GOPATH=%v", b.NewGOPATH))
}
log.Infof("go build cmd is: %v", cmd.Args)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Start()
if err != nil {
return fmt.Errorf("fail to execute: %v, err: %w", cmd.Args, err)
}
if err = cmd.Wait(); err != nil {
return fmt.Errorf("fail to execute: %v, err: %w", cmd.Args, err)
}
return nil
}

284
pkg/build/tmpfolder.go
View File

@ -1,212 +1,130 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package build
import (
"crypto/sha256"
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/qiniu/goc/pkg/cover"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
"github.com/qiniu/goc/v2/pkg/config"
"github.com/qiniu/goc/v2/pkg/log"
"github.com/tongjingran/copy"
"golang.org/x/mod/modfile"
)
// MvProjectsToTmp moves the projects into a temporary directory
func (b *Build) MvProjectsToTmp() error {
listArgs := []string{"-json"}
if len(b.BuildFlags) != 0 {
listArgs = append(listArgs, b.BuildFlags)
}
listArgs = append(listArgs, "./...")
var err error
b.Pkgs, err = cover.ListPackages(b.WorkingDir, strings.Join(listArgs, " "), "")
if err != nil {
log.Errorln(err)
return err
}
// copyProjectToTmp copies project files to the temporary directory
//
// It will ignore .git and irregular files, only copy source(text) files
func (b *Build) copyProjectToTmp() {
curProject := config.GocConfig.CurModProjectDir
tmpProject := config.GocConfig.TmpModProjectDir
err = b.mvProjectsToTmp()
if err != nil {
log.Errorf("Fail to move the project to temporary directory")
return err
}
b.OriGOPATH = os.Getenv("GOPATH")
if b.IsMod == true {
b.NewGOPATH = ""
} else if b.OriGOPATH == "" {
b.NewGOPATH = b.TmpDir
} else {
b.NewGOPATH = fmt.Sprintf("%v:%v", b.TmpDir, b.OriGOPATH)
}
// fix #14: unable to build project not in GOPATH in legacy mode
// this kind of project does not have a pkg.Root value
// go 1.11, 1.12 has no pkg.Root,
// so add b.IsMod == false as secondary judgement
if b.Root == "" && b.IsMod == false {
b.NewGOPATH = b.OriGOPATH
}
log.Infof("New GOPATH: %v", b.NewGOPATH)
return nil
}
func (b *Build) mvProjectsToTmp() error {
b.TmpDir = filepath.Join(os.TempDir(), tmpFolderName(b.WorkingDir))
// Delete previous tmp folder and its content
os.RemoveAll(b.TmpDir)
// Create a new tmp folder and a new importpath for storing cover variables
b.GlobalCoverVarImportPath = filepath.Join("src", tmpPackageName(b.WorkingDir))
err := os.MkdirAll(filepath.Join(b.TmpDir, b.GlobalCoverVarImportPath), os.ModePerm)
if err != nil {
return fmt.Errorf("Fail to create the temporary build directory. The err is: %v", err)
}
log.Infof("Tmp project generated in: %v", b.TmpDir)
// traverse pkg list to get project meta info
b.IsMod, b.Root, err = b.traversePkgsList()
log.Infof("mod project? %v", b.IsMod)
if errors.Is(err, ErrShouldNotReached) {
return fmt.Errorf("mvProjectsToTmp with a empty project: %w", err)
}
// we should get corresponding working directory in temporary directory
b.TmpWorkingDir, err = b.getTmpwd()
if err != nil {
return fmt.Errorf("getTmpwd failed with error: %w", err)
}
// issue #14
// if b.Root == "", then the project is non-standard project
// known cases:
// 1. a legacy project, but not in any GOPATH, will cause the b.Root == ""
if b.IsMod == false && b.Root != "" {
b.cpLegacyProject()
} else if b.IsMod == true { // go 1.11, 1.12 has no Build.Root
b.cpGoModulesProject()
updated, newGoModContent, err := b.updateGoModFile()
if _, err := os.Stat(tmpProject); !os.IsNotExist(err) {
log.Infof("find previous temporary directory, delete")
err := os.RemoveAll(tmpProject)
if err != nil {
return fmt.Errorf("fail to generate new go.mod: %v", err)
log.Fatalf("fail to remove preivous temporary directory: %v", err)
}
if updated {
log.Infoln("go.mod needs rewrite")
tmpModFile := filepath.Join(b.TmpDir, "go.mod")
err := ioutil.WriteFile(tmpModFile, newGoModContent, os.ModePerm)
if err != nil {
return fmt.Errorf("fail to update go.mod: %v", err)
}
}
} else if b.IsMod == false && b.Root == "" {
b.TmpWorkingDir = b.TmpDir
b.cpNonStandardLegacy()
}
log.Infof("New workingdir in tmp directory in: %v", b.TmpWorkingDir)
return nil
log.StartWait("coping project")
err := os.MkdirAll(tmpProject, os.ModePerm)
if err != nil {
log.Fatalf("fail to create temporary directory: %v", err)
}
// copy
if err := copy.Copy(curProject, tmpProject, copy.Options{Skip: skipCopy}); err != nil {
log.Fatalf("fail to copy the folder from %v to %v, the err: %v", curProject, tmpProject, err)
}
log.StopWait()
}
// tmpFolderName uses the first six characters of the input path's SHA256 checksum
// as the suffix.
// tmpFolderName generates a directory name according to the path
func tmpFolderName(path string) string {
sum := sha256.Sum256([]byte(path))
h := fmt.Sprintf("%x", sum[:6])
return "goc-build-" + h
}
// tmpPackageName uses the first six characters of the input path's SHA256 checksum
// as the suffix.
func tmpPackageName(path string) string {
sum := sha256.Sum256([]byte(path))
h := fmt.Sprintf("%x", sum[:6])
return "gocbuild" + h
}
// traversePkgsList travse the Build.Pkgs list
// return Build.IsMod, tell if the project is a mod project
// return Build.Root:
// 1. the project root if it is a mod project,
// 2. current GOPATH if it is a legacy project,
// 3. some non-standard project, which Build.IsMod == false, Build.Root == nil
func (b *Build) traversePkgsList() (isMod bool, root string, err error) {
for _, v := range b.Pkgs {
// get root
root = v.Root
if v.Module == nil {
return
}
isMod = true
b.ModRoot = v.Module.Dir
b.ModRootPath = v.Module.Path
return
// skipCopy skip copy .git dir and irregular files
func skipCopy(src string, info os.FileInfo) (bool, error) {
irregularModeType := os.ModeNamedPipe | os.ModeSocket | os.ModeDevice | os.ModeCharDevice | os.ModeIrregular
if strings.HasSuffix(src, "/.git") {
log.Debugf("skip .git dir [%s]", src)
return true, nil
}
log.Error(ErrShouldNotReached)
err = ErrShouldNotReached
if info.Mode()&irregularModeType != 0 {
log.Debugf("skip file [%s], the file mode is [%s]", src, info.Mode().String())
return true, nil
}
return false, nil
}
// clean clears the temporary project
func (b *Build) clean() {
if config.GocConfig.Debug != true {
if err := os.RemoveAll(config.GocConfig.TmpModProjectDir); err != nil {
log.Fatalf("fail to delete the temporary project: %v", err)
}
log.Donef("delete the temporary project")
} else {
log.Debugf("--debug is enabled, keep the temporary project")
}
}
// updateGoModFile rewrites the go.mod file in the temporary directory,
//
// if it has a 'replace' directive, and the directive has a relative local path,
// it will be rewritten with a absolute path.
//
// ex.
//
// suppose original project is located at /path/to/aa/bb/cc, go.mod contains a directive:
// 'replace github.com/qiniu/bar => ../home/foo/bar'
//
// after the project is copied to temporary directory, it should be rewritten as
// 'replace github.com/qiniu/bar => /path/to/aa/bb/home/foo/bar'
func (b *Build) updateGoModFile() (updateFlag bool, newModFile []byte) {
tempModfile := filepath.Join(config.GocConfig.TmpModProjectDir, "go.mod")
buf, err := ioutil.ReadFile(tempModfile)
if err != nil {
log.Fatalf("cannot find go.mod file in temporary directory: %v", err)
}
oriGoModFile, err := modfile.Parse(tempModfile, buf, nil)
if err != nil {
log.Fatalf("cannot parse go.mod: %v", err)
}
updateFlag = false
for index := range oriGoModFile.Replace {
replace := oriGoModFile.Replace[index]
oldPath := replace.Old.Path
oldVersion := replace.Old.Version
newPath := replace.New.Path
newVersion := replace.New.Version
// replace to a local filesystem does not have a version
// absolute path no need to rewrite
if newVersion == "" && !filepath.IsAbs(newPath) {
var absPath string
fullPath := filepath.Join(config.GocConfig.CurModProjectDir, newPath)
absPath, _ = filepath.Abs(fullPath)
// DropReplace & AddReplace will not return error
// so no need to check the error
_ = oriGoModFile.DropReplace(oldPath, oldVersion)
_ = oriGoModFile.AddReplace(oldPath, oldVersion, absPath, newVersion)
updateFlag = true
}
}
oriGoModFile.Cleanup()
// Format will not return error, so ignore the returned error
// func (f *File) Format() ([]byte, error) {
// return Format(f.Syntax), nil
// }
newModFile, _ = oriGoModFile.Format()
return
}
// getTmpwd get the corresponding working directory in the temporary working directory
// and store it in the Build.tmpWorkdingDir
func (b *Build) getTmpwd() (string, error) {
for _, pkg := range b.Pkgs {
var index int
var parentPath string
if b.IsMod == false {
index = strings.Index(b.WorkingDir, pkg.Root)
parentPath = pkg.Root
} else {
index = strings.Index(b.WorkingDir, pkg.Module.Dir)
parentPath = pkg.Module.Dir
}
if index == -1 {
return "", ErrGocShouldExecInProject
}
// b.TmpWorkingDir = filepath.Join(b.TmpDir, path[len(parentPath):])
return filepath.Join(b.TmpDir, b.WorkingDir[len(parentPath):]), nil
}
return "", ErrShouldNotReached
}
func (b *Build) findWhereToInstall() (string, error) {
if GOBIN := os.Getenv("GOBIN"); GOBIN != "" {
return GOBIN, nil
}
if false == b.IsMod {
if b.Root == "" {
return "", ErrNoPlaceToInstall
}
return filepath.Join(b.Root, "bin"), nil
}
if b.OriGOPATH != "" {
return filepath.Join(strings.Split(b.OriGOPATH, ":")[0], "bin"), nil
}
return filepath.Join(os.Getenv("HOME"), "go", "bin"), nil
}
// Clean clears up the temporary workspace
func (b *Build) Clean() error {
if !viper.GetBool("debug") {
return os.RemoveAll(b.TmpDir)
}
return nil
}

150
pkg/build/tmpfolder_test.go
View File

@ -1,150 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package build
import (
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
var baseDir string
func init() {
baseDir, _ = os.Getwd()
}
func TestNewDirParseInLegacyProject(t *testing.T) {
workingDir := filepath.Join(baseDir, "../../tests/samples/simple_gopath_project/src/qiniu.com/simple_gopath_project")
gopath := filepath.Join(baseDir, "../../tests/samples/simple_gopath_project")
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "off")
b, _ := NewInstall("", []string{"."}, workingDir)
if -1 == strings.Index(b.TmpWorkingDir, b.TmpDir) {
t.Fatalf("Directory parse error. newwd: %v, tmpdir: %v", b.TmpWorkingDir, b.TmpDir)
}
if -1 == strings.Index(b.NewGOPATH, ":") || -1 == strings.Index(b.NewGOPATH, b.TmpDir) {
t.Fatalf("The New GOPATH is wrong. newgopath: %v, tmpdir: %v", b.NewGOPATH, b.TmpDir)
}
b, _ = NewBuild("", []string{"."}, workingDir, "")
if -1 == strings.Index(b.TmpWorkingDir, b.TmpDir) {
t.Fatalf("Directory parse error. newwd: %v, tmpdir: %v", b.TmpWorkingDir, b.TmpDir)
}
if -1 == strings.Index(b.NewGOPATH, ":") || -1 == strings.Index(b.NewGOPATH, b.TmpDir) {
t.Fatalf("The New GOPATH is wrong. newgopath: %v, tmpdir: %v", b.NewGOPATH, b.TmpDir)
}
}
func TestNewDirParseInModProject(t *testing.T) {
workingDir := filepath.Join(baseDir, "../../tests/samples/simple_project")
gopath := ""
fmt.Println(gopath)
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
b, _ := NewInstall("", []string{"."}, workingDir)
if -1 == strings.Index(b.TmpWorkingDir, b.TmpDir) {
t.Fatalf("Directory parse error. newwd: %v, tmpdir: %v", b.TmpWorkingDir, b.TmpDir)
}
if b.NewGOPATH != "" {
t.Fatalf("The New GOPATH is wrong. newgopath: %v, tmpdir: %v", b.NewGOPATH, b.TmpDir)
}
b, _ = NewBuild("", []string{"."}, workingDir, "")
if -1 == strings.Index(b.TmpWorkingDir, b.TmpDir) {
t.Fatalf("Directory parse error. newwd: %v, tmpdir: %v", b.TmpWorkingDir, b.TmpDir)
}
if b.NewGOPATH != "" {
t.Fatalf("The New GOPATH is wrong. newgopath: %v, tmpdir: %v", b.NewGOPATH, b.TmpDir)
}
}
// Test #14
func TestLegacyProjectNotInGoPATH(t *testing.T) {
workingDir := filepath.Join(baseDir, "../../tests/samples/simple_gopath_project/src/qiniu.com/simple_gopath_project")
gopath := ""
fmt.Println(gopath)
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "off")
b, _ := NewBuild("", []string{"."}, workingDir, "")
if b.OriGOPATH != b.NewGOPATH {
t.Fatalf("New GOPATH should be same with old GOPATH, for this kind of project. New: %v, old: %v", b.NewGOPATH, b.OriGOPATH)
}
_, err := os.Stat(filepath.Join(b.TmpDir, "main.go"))
if err != nil {
t.Fatalf("There should be a main.go in temporary directory directly, the error: %v", err)
}
}
// test traversePkgsList error case
func TestTraversePkgsList(t *testing.T) {
b := &Build{
Pkgs: nil,
}
_, _, err := b.traversePkgsList()
assert.EqualError(t, err, ErrShouldNotReached.Error())
}
// test getTmpwd error case
func TestGetTmpwd(t *testing.T) {
b := &Build{
Pkgs: nil,
}
_, err := b.getTmpwd()
assert.EqualError(t, err, ErrShouldNotReached.Error())
}
// test findWhereToInstall
func TestFindWhereToInstall(t *testing.T) {
// if a legacy project without project root find
// should find no plcae to install
b := &Build{
Pkgs: nil,
IsMod: false,
Root: "",
}
_, err := b.findWhereToInstall()
assert.EqualError(t, err, ErrNoPlaceToInstall.Error())
// if $GOBIN not found
// and if $GOPATH not found
// should install to $HOME/go/bin
b = &Build{
Pkgs: nil,
IsMod: true,
OriGOPATH: "",
}
placeToInstall, err := b.findWhereToInstall()
assert.NoError(t, err)
expectedPlace := filepath.Join(os.Getenv("HOME"), "go", "bin")
assert.Equal(t, placeToInstall, expectedPlace)
}

134
pkg/config/config.go Normal file
View File

@ -0,0 +1,134 @@
package config
import "time"
type gocConfig struct {
Debug bool
Host string
Mode string // cover mode
GOPATH string
GOBIN string
CurWd string
TmpWd string
CurModProjectDir string
TmpModProjectDir string
Goflags []string // command line flags
Packages []string // command line [packages]
ImportPath string // the whole import path of the project
Pkgs map[string]*Package
GlobalCoverVarImportPath string
GlobalCoverVarImportPathDir string
}
// GocConfig 全局变量，存放 goc 的各种元属性
var GocConfig gocConfig
type goConfig struct {
BuildA bool
BuildBuildmode string // -buildmode flag
BuildMod string // -mod flag
BuildModReason string // reason -mod flag is set, if set by default
BuildI bool // -i flag
BuildLinkshared bool // -linkshared flag
BuildMSan bool // -msan flag
BuildN bool // -n flag
BuildO string // -o flag
BuildP int // -p flag
BuildPkgdir string // -pkgdir flag
BuildRace bool // -race flag
BuildToolexec string // -toolexec flag
BuildToolchainName string
BuildToolchainCompiler func() string
BuildToolchainLinker func() string
BuildTrimpath bool // -trimpath flag
BuildV bool // -v flag
BuildWork bool // -work flag
BuildX bool // -x flag
// from buildcontext
Installsuffix string // -installSuffix
BuildTags string // -tags
// from load
BuildAsmflags string
BuildCompiler string
BuildGcflags string
BuildGccgoflags string
BuildLdflags string
// mod related
ModCacheRW bool
ModFile string
}
var GoConfig goConfig
// PackageCover holds all the generate coverage variables of a package
type PackageCover struct {
Package *Package
Vars map[string]*FileVar
}
// FileVar holds the name of the generated coverage variables targeting the named file.
type FileVar struct {
File string // 这里其实不是文件名，是 importpath + filename
Var string
}
// Package map a package output by go list
// this is subset of package struct in: https://github.com/golang/go/blob/master/src/cmd/go/internal/load/pkg.go#L58
type Package struct {
Dir string `json:"Dir"` // directory containing package sources
ImportPath string `json:"ImportPath"` // import path of package in dir
Name string `json:"Name"` // package name
Target string `json:",omitempty"` // installed target for this package (may be executable)
Root string `json:",omitempty"` // Go root, Go path dir, or module root dir containing this package
Module *ModulePublic `json:",omitempty"` // info about package's module, if any
Goroot bool `json:"Goroot,omitempty"` // is this package in the Go root?
Standard bool `json:"Standard,omitempty"` // is this package part of the standard Go library?
DepOnly bool `json:"DepOnly,omitempty"` // package is only a dependency, not explicitly listed
// Source files
GoFiles []string `json:"GoFiles,omitempty"` // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
CgoFiles []string `json:"CgoFiles,omitempty"` // .go source files that import "C"
// Dependency information
Deps []string `json:"Deps,omitempty"` // all (recursively) imported dependencies
Imports []string `json:",omitempty"` // import paths used by this package
ImportMap map[string]string `json:",omitempty"` // map from source import to ImportPath (identity entries omitted)
// Error information
Incomplete bool `json:"Incomplete,omitempty"` // this package or a dependency has an error
Error *PackageError `json:"Error,omitempty"` // error loading package
DepsErrors []*PackageError `json:"DepsErrors,omitempty"` // errors loading dependencies
}
// ModulePublic represents the package info of a module
type ModulePublic struct {
Path string `json:",omitempty"` // module path
Version string `json:",omitempty"` // module version
Versions []string `json:",omitempty"` // available module versions
Replace *ModulePublic `json:",omitempty"` // replaced by this module
Time *time.Time `json:",omitempty"` // time version was created
Update *ModulePublic `json:",omitempty"` // available update (with -u)
Main bool `json:",omitempty"` // is this the main module?
Indirect bool `json:",omitempty"` // module is only indirectly needed by main module
Dir string `json:",omitempty"` // directory holding local copy of files, if any
GoMod string `json:",omitempty"` // path to go.mod file describing module, if any
GoVersion string `json:",omitempty"` // go version used in module
Error *ModuleError `json:",omitempty"` // error loading module
}
// ModuleError represents the error loading module
type ModuleError struct {
Err string // error text
}
// PackageError is the error info for a package when list failed
type PackageError struct {
ImportStack []string // shortest path from package named on command line to this one
Pos string // position of error (if present, file:line:col)
Err string // the error itself
}

276
pkg/cover/agent.tpl Normal file
View File

@ -0,0 +1,276 @@
package cover
import (
"fmt"
"io"
"log"
"net/rpc"
"net/rpc/jsonrpc"
"net/url"
"os"
"strconv"
"strings"
"sync/atomic"
"time"
"testing"
"{{.GlobalCoverVarImportPath}}/websocket"
_cover "{{.GlobalCoverVarImportPath}}"
)
var (
waitDelay time.Duration = 10 * time.Second
host string = "{{.Host}}"
)
func init() {
// init host
host_env := os.Getenv("GOC_CUSTOM_HOST")
if host_env != "" {
host = host_env
}
var dialer = websocket.DefaultDialer
go func() {
// 永不退出，出错后统一操作为：延时 + conitnue
for {
// 获取进程元信息用于注册
ps, err := getRegisterInfo()
if err != nil {
time.Sleep(waitDelay)
continue
}
// 注册，直接将元信息放在 ws 地址中
v := url.Values{}
v.Set("hostname", ps.hostname)
v.Set("pid", strconv.Itoa(ps.pid))
v.Set("cmdline", ps.cmdline)
v.Encode()
rpcstreamUrl := fmt.Sprintf("ws://%v/v2/internal/ws/rpcstream?%v", host, v.Encode())
ws, _, err := dialer.Dial(rpcstreamUrl, nil)
if err != nil {
log.Printf("[goc][Error] rpc fail to dial to goc server: %v", err)
time.Sleep(waitDelay)
continue
}
log.Printf("[goc][Info] rpc connected to goc server")
rwc := &ReadWriteCloser{ws: ws}
s := rpc.NewServer()
s.Register(&GocAgent{})
s.ServeCodec(jsonrpc.NewServerCodec(rwc))
// exit rpc server, close ws connection
ws.Close()
time.Sleep(waitDelay)
log.Printf("[goc][Error] rpc connection to goc server broken", )
}
}()
}
// rpc
type GocAgent struct {
}
type ProfileReq string
type ProfileRes string
// return a profile of now
func (ga *GocAgent) GetProfile(req *ProfileReq, res *ProfileRes) error {
if *req != "getprofile" {
*res = ""
return fmt.Errorf("wrong command")
}
w := new(strings.Builder)
fmt.Fprint(w, "mode: {{.Mode}}\n")
counters, blocks := loadValues()
var active, total int64
var count uint32
for name, counts := range counters {
block := blocks[name]
for i := range counts {
stmts := int64(block[i].Stmts)
total += stmts
count = atomic.LoadUint32(&counts[i]) // For -mode=atomic.
if count > 0 {
active += stmts
}
_, err := fmt.Fprintf(w, "%s:%d.%d,%d.%d %d %d\n", name,
block[i].Line0, block[i].Col0,
block[i].Line1, block[i].Col1,
stmts,
count)
if err != nil {
fmt.Fprintf(w, "invalid block format, err: %v", err)
return err
}
}
}
*res = ProfileRes(w.String())
return nil
}
// reset profile to 0
func (ga *GocAgent) ResetProfile(req *ProfileReq, res *ProfileRes) error {
if *req != "resetprofile" {
*res = ""
return fmt.Errorf("wrong command")
}
resetValues()
*res = `ok`
return nil
}
// get cover Values
func loadValues() (map[string][]uint32, map[string][]testing.CoverBlock) {
var (
coverCounters = make(map[string][]uint32)
coverBlocks = make(map[string][]testing.CoverBlock)
)
{{range $i, $pkgCover := .Covers}}
{{range $file, $cover := $pkgCover.Vars}}
loadFileCover(coverCounters, coverBlocks, "{{$cover.File}}", _cover.{{$cover.Var}}.Count[:], _cover.{{$cover.Var}}.Pos[:], _cover.{{$cover.Var}}.NumStmt[:])
{{end}}
{{end}}
return coverCounters, coverBlocks
}
func loadFileCover(coverCounters map[string][]uint32, coverBlocks map[string][]testing.CoverBlock, fileName string, counter []uint32, pos []uint32, numStmts []uint16) {
if 3*len(counter) != len(pos) || len(counter) != len(numStmts) {
panic("coverage: mismatched sizes")
}
if coverCounters[fileName] != nil {
// Already registered.
return
}
coverCounters[fileName] = counter
block := make([]testing.CoverBlock, len(counter))
for i := range counter {
block[i] = testing.CoverBlock{
Line0: pos[3*i+0],
Col0: uint16(pos[3*i+2]),
Line1: pos[3*i+1],
Col1: uint16(pos[3*i+2] >> 16),
Stmts: numStmts[i],
}
}
coverBlocks[fileName] = block
}
// reset counters
func resetValues() {
{{range $i, $pkgCover := .Covers}}
{{range $file, $cover := $pkgCover.Vars}}
clearFileCover(_cover.{{$cover.Var}}.Count[:])
{{end}}
{{end}}
}
func clearFileCover(counter []uint32) {
for i := range counter {
counter[i] = 0
}
}
// get process meta info for register
type processInfo struct {
hostname string
pid int
cmdline string
}
func getRegisterInfo() (*processInfo, error) {
hostname, err := os.Hostname()
if err != nil {
log.Printf("[goc][Error] fail to get hostname: %v", hostname)
return nil, err
}
pid := os.Getpid()
cmdline := os.Args[0]
return &processInfo{
hostname: hostname,
pid: pid,
cmdline: cmdline,
}, nil
}
/// websocket rpc readwriter closer
type ReadWriteCloser struct {
ws *websocket.Conn
r io.Reader
w io.WriteCloser
}
func (rwc *ReadWriteCloser) Read(p []byte) (n int, err error) {
if rwc.r == nil {
var _ int
_, rwc.r, err = rwc.ws.NextReader()
if err != nil {
return 0, err
}
}
for n = 0; n < len(p); {
var m int
m, err = rwc.r.Read(p[n:])
n += m
if err == io.EOF {
// done
rwc.r = nil
break
}
// ???
if err != nil {
break
}
}
return
}
func (rwc *ReadWriteCloser) Write(p []byte) (n int, err error) {
if rwc.w == nil {
rwc.w, err = rwc.ws.NextWriter(websocket.TextMessage)
if err != nil {
return 0, err
}
}
for n = 0; n < len(p); {
var m int
m, err = rwc.w.Write(p)
n += m
if err != nil {
break
}
}
if err != nil || n == len(p) {
err = rwc.Close()
}
return
}
func (rwc *ReadWriteCloser) Close() (err error) {
if rwc.w != nil {
err = rwc.w.Close()
rwc.w = nil
}
return err
}

151
pkg/cover/agentwatch.tpl Normal file
View File

@ -0,0 +1,151 @@
package coverdef
import (
"fmt"
"time"
"os"
"log"
"strconv"
"net/url"
"{{.GlobalCoverVarImportPath}}/websocket"
)
var (
watchChannel = make(chan *blockInfo, 1024)
watchEnabled = false
waitDelay time.Duration = 10 * time.Second
host string = "{{.Host}}"
)
func init() {
// init host
host_env := os.Getenv("GOC_CUSTOM_HOST")
if host_env != "" {
host = host_env
}
var dialer = websocket.DefaultDialer
go func() {
for {
// 获取进程元信息用于注册
ps, err := getRegisterInfo()
if err != nil {
time.Sleep(waitDelay)
continue
}
// 注册，直接将元信息放在 ws 地址中
v := url.Values{}
v.Set("hostname", ps.hostname)
v.Set("pid", strconv.Itoa(ps.pid))
v.Set("cmdline", ps.cmdline)
v.Encode()
watchstreamUrl := fmt.Sprintf("ws://%v/v2/internal/ws/watchstream?%v", host, v.Encode())
ws, _, err := dialer.Dial(watchstreamUrl, nil)
if err != nil {
log.Printf("[goc][Error] watch fail to dial to goc server: %v", err)
time.Sleep(waitDelay)
continue
}
// 连接成功
watchEnabled = true
log.Printf("[goc][Info] watch connected to goc server")
ticker := time.NewTicker(time.Second)
closeFlag := false
go func() {
for {
// 必须调用一下以触发 ping 的自动处理
_, _, err := ws.ReadMessage()
if err != nil {
break
}
}
closeFlag = true
}()
Loop:
for {
select {
case block := <-watchChannel:
i := block.i
cov := fmt.Sprintf("%s:%d.%d,%d.%d %d %d", block.name,
block.pos[3*i+0], uint16(block.pos[3*i+2]),
block.pos[3*i+1], uint16(block.pos[3*i+2] >> 16),
1,
0)
err = ws.WriteMessage(websocket.TextMessage, []byte(cov))
if err != nil {
watchEnabled = false
log.Println("[goc][Error] push coverage failed: %v", err)
time.Sleep(waitDelay)
break Loop
}
case <-ticker.C:
if closeFlag == true {
break Loop
}
}
}
}
}()
}
// get process meta info for register
type processInfo struct {
hostname string
pid int
cmdline string
}
func getRegisterInfo() (*processInfo, error) {
hostname, err := os.Hostname()
if err != nil {
log.Printf("[goc][Error] fail to get hostname: %v", hostname)
return nil, err
}
pid := os.Getpid()
cmdline := os.Args[0]
return &processInfo{
hostname: hostname,
pid: pid,
cmdline: cmdline,
}, nil
}
//
type blockInfo struct {
name string
pos []uint32
i int
}
// UploadCoverChangeEvent_{{.Random}} is non-blocking
func UploadCoverChangeEvent_{{.Random}}(name string, pos []uint32, i int) {
if watchEnabled == false {
return
}
// make sure send is non-blocking
select {
case watchChannel <- &blockInfo{
name: name,
pos: pos,
i: i,
}:
default:
}
}

188
pkg/cover/client.go
View File

@ -1,188 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cover
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
"strings"
log "github.com/sirupsen/logrus"
)
// Action provides methods to contact with the covered service under test
type Action interface {
Profile(param ProfileParam) ([]byte, error)
Clear(param ProfileParam) ([]byte, error)
Remove(param ProfileParam) ([]byte, error)
InitSystem() ([]byte, error)
ListServices() ([]byte, error)
RegisterService(svr ServiceUnderTest) ([]byte, error)
}
const (
//CoverInitSystemAPI prepare a new round of testing
CoverInitSystemAPI = "/v1/cover/init"
//CoverProfileAPI is provided by the covered service to get profiles
CoverProfileAPI = "/v1/cover/profile"
//CoverProfileClearAPI is provided by the covered service to clear profiles
CoverProfileClearAPI = "/v1/cover/clear"
//CoverServicesListAPI list all the registered services
CoverServicesListAPI = "/v1/cover/list"
//CoverRegisterServiceAPI register a service into service center
CoverRegisterServiceAPI = "/v1/cover/register"
//CoverServicesRemoveAPI remove one services from the service center
CoverServicesRemoveAPI = "/v1/cover/remove"
)
type client struct {
Host string
client *http.Client
}
// NewWorker creates a worker to contact with service
func NewWorker(host string) Action {
_, err := url.ParseRequestURI(host)
if err != nil {
log.Fatalf("Parse url %s failed, err: %v", host, err)
}
return &client{
Host: host,
client: http.DefaultClient,
}
}
func (c *client) RegisterService(srv ServiceUnderTest) ([]byte, error) {
if _, err := url.ParseRequestURI(srv.Address); err != nil {
return nil, err
}
if strings.TrimSpace(srv.Name) == "" {
return nil, fmt.Errorf("invalid service name")
}
u := fmt.Sprintf("%s%s?name=%s&address=%s", c.Host, CoverRegisterServiceAPI, srv.Name, srv.Address)
_, res, err := c.do("POST", u, "", nil)
return res, err
}
func (c *client) ListServices() ([]byte, error) {
u := fmt.Sprintf("%s%s", c.Host, CoverServicesListAPI)
_, services, err := c.do("GET", u, "", nil)
if err != nil && isNetworkError(err) {
_, services, err = c.do("GET", u, "", nil)
}
return services, err
}
func (c *client) Profile(param ProfileParam) ([]byte, error) {
u := fmt.Sprintf("%s%s", c.Host, CoverProfileAPI)
if len(param.Service) != 0 && len(param.Address) != 0 {
return nil, fmt.Errorf("use 'service' flag and 'address' flag at the same time may cause ambiguity, please use them separately")
}
// the json.Marshal function can return two types of errors: UnsupportedTypeError or UnsupportedValueError
// so no need to check here
body, _ := json.Marshal(param)
res, profile, err := c.do("POST", u, "application/json", bytes.NewReader(body))
if err != nil && isNetworkError(err) {
res, profile, err = c.do("POST", u, "application/json", bytes.NewReader(body))
}
if err == nil && res.StatusCode != 200 {
err = fmt.Errorf(string(profile))
}
return profile, err
}
func (c *client) Clear(param ProfileParam) ([]byte, error) {
u := fmt.Sprintf("%s%s", c.Host, CoverProfileClearAPI)
if len(param.Service) != 0 && len(param.Address) != 0 {
return nil, fmt.Errorf("use 'service' flag and 'address' flag at the same time may cause ambiguity, please use them separately")
}
// the json.Marshal function can return two types of errors: UnsupportedTypeError or UnsupportedValueError
// so no need to check here
body, _ := json.Marshal(param)
_, resp, err := c.do("POST", u, "application/json", bytes.NewReader(body))
if err != nil && isNetworkError(err) {
_, resp, err = c.do("POST", u, "application/json", bytes.NewReader(body))
}
return resp, err
}
func (c *client) Remove(param ProfileParam) ([]byte, error) {
u := fmt.Sprintf("%s%s", c.Host, CoverServicesRemoveAPI)
if len(param.Service) != 0 && len(param.Address) != 0 {
return nil, fmt.Errorf("use 'service' flag and 'address' flag at the same time may cause ambiguity, please use them separately")
}
// the json.Marshal function can return two types of errors: UnsupportedTypeError or UnsupportedValueError
// so no need to check here
body, err := json.Marshal(param)
if err != nil {
return nil, err
}
_, resp, err := c.do("POST", u, "application/json", bytes.NewReader(body))
if err != nil && isNetworkError(err) {
_, resp, err = c.do("POST", u, "application/json", bytes.NewReader(body))
}
return resp, err
}
func (c *client) InitSystem() ([]byte, error) {
u := fmt.Sprintf("%s%s", c.Host, CoverInitSystemAPI)
_, body, err := c.do("POST", u, "", nil)
return body, err
}
func (c *client) do(method, url, contentType string, body io.Reader) (*http.Response, []byte, error) {
req, err := http.NewRequest(method, url, body)
if err != nil {
return nil, nil, err
}
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
res, err := c.client.Do(req)
if err != nil {
return nil, nil, err
}
defer res.Body.Close()
responseBody, err := ioutil.ReadAll(res.Body)
if err != nil {
return res, nil, err
}
return res, responseBody, nil
}
func isNetworkError(err error) bool {
if err == io.EOF {
return true
}
_, ok := err.(net.Error)
return ok
}

257
pkg/cover/client_test.go
View File

@ -1,257 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cover
import (
"fmt"
"net/http/httptest"
"os"
"testing"
"net/http"
"github.com/stretchr/testify/assert"
)
func TestClientAction(t *testing.T) {
// mock goc server
server, err := NewFileBasedServer("_svrs_address.txt")
assert.NoError(t, err)
ts := httptest.NewServer(server.Route(os.Stdout))
defer ts.Close()
var client = NewWorker(ts.URL)
// mock profile server
profileMockResponse := []byte("mode: count\nmockService/main.go:30.13,48.33 13 1\nb/b.go:30.13,48.33 13 1")
profileSuccessMockSvr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write(profileMockResponse)
}))
defer profileSuccessMockSvr.Close()
profileErrMockSvr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte("error"))
}))
defer profileErrMockSvr.Close()
// register service into goc server
var src ServiceUnderTest
src.Name = "serviceSuccess"
src.Address = profileSuccessMockSvr.URL
res, err := client.RegisterService(src)
assert.NoError(t, err)
assert.Contains(t, string(res), "success")
// do list and check server
res, err = client.ListServices()
assert.NoError(t, err)
assert.Contains(t, string(res), src.Address)
assert.Contains(t, string(res), src.Name)
// get profile from goc server
tcs := []struct {
name string
service ServiceUnderTest
param ProfileParam
expected string
expectedErr bool
}{
{
name: "both server and address existed",
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{Force: false, Service: []string{"serviceOK"}, Address: []string{profileSuccessMockSvr.URL}},
expectedErr: true,
},
{
name: "valid test with no server flag provide",
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{},
expected: "mockService/main.go:30.13,48.33 13 1",
},
{
name: "valid test with server flag provide",
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{Service: []string{"serviceOK"}},
expected: "mockService/main.go:30.13,48.33 13 1",
},
{
name: "valid test with address flag provide",
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{Address: []string{profileSuccessMockSvr.URL}},
expected: "mockService/main.go:30.13,48.33 13 1",
},
{
name: "invalid test with invalid server flag provide",
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{Service: []string{"unknown"}},
expected: "server [unknown] not found",
expectedErr: true,
},
{
name: "invalid test with invalid profile got by server",
service: ServiceUnderTest{Name: "serviceErr", Address: profileErrMockSvr.URL},
expected: "bad mode line: error",
expectedErr: true,
},
{
name: "invalid test with disconnected server",
service: ServiceUnderTest{Name: "serviceNotExist", Address: "http://172.0.0.2:7777"},
expected: "connection refused",
expectedErr: true,
},
{
name: "invalid test with empty profile",
service: ServiceUnderTest{Name: "serviceNotExist", Address: "http://172.0.0.2:7777"},
param: ProfileParam{Force: true},
expectedErr: true,
expected: "no profiles",
},
{
name: "valid test with coverfile flag provide",
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{CoverFilePatterns: []string{"b.go$"}},
expected: "b/b.go",
},
{
name: "valid test with skipfile flag provided",
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{SkipFilePatterns: []string{"b.go$"}},
expected: "main.go",
},
{
name: "valid test with both skipfile and coverfile flags provided",
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{SkipFilePatterns: []string{"main.go"}, CoverFilePatterns: []string{".go$"}},
expected: "b.go",
},
}
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
// init server
_, err = client.InitSystem()
assert.NoError(t, err)
// register server
res, err = client.RegisterService(tc.service)
assert.NoError(t, err)
assert.Contains(t, string(res), "success")
res, err = client.Profile(tc.param)
if err != nil {
if !tc.expectedErr {
t.Errorf("unexpected err got: %v", err)
}
return
}
if tc.expectedErr {
t.Errorf("Expected an error, but got value %s", string(res))
}
assert.Regexp(t, tc.expected, string(res))
})
}
// init system and check server again
_, err = client.InitSystem()
assert.NoError(t, err)
res, err = client.ListServices()
assert.NoError(t, err)
assert.Equal(t, "{}", string(res))
}
func TestClientRegisterService(t *testing.T) {
c := &client{}
// client register with empty address
testService1 := ServiceUnderTest{
Address: "",
Name: "abc",
}
_, err := c.RegisterService(testService1)
assert.Contains(t, err.Error(), "empty url")
// client register with empty name
testService2 := ServiceUnderTest{
Address: "http://127.0.0.1:444",
Name: "",
}
_, err = c.RegisterService(testService2)
assert.EqualError(t, err, "invalid service name")
}
func TestClientListServices(t *testing.T) {
c := &client{
Host: "http://127.0.0.1:64445", // a invalid host
client: http.DefaultClient,
}
_, err := c.ListServices()
assert.Contains(t, err.Error(), "connect: connection refused")
}
func TestClientDo(t *testing.T) {
c := &client{
client: http.DefaultClient,
}
_, _, err := c.do(" ", "http://127.0.0.1:7777", "", nil) // a invalid method
assert.Contains(t, err.Error(), "invalid method")
}
func TestClientClearWithInvalidParam(t *testing.T) {
p := ProfileParam{
Service: []string{"goc"},
Address: []string{"http://127.0.0.1:777"},
}
c := &client{
client: http.DefaultClient,
}
_, err := c.Clear(p)
assert.Error(t, err)
assert.Contains(t, err.Error(), "use 'service' flag and 'address' flag at the same time may cause ambiguity, please use them separately")
}
func TestClientRemove(t *testing.T) {
// remove by invalid param
p := ProfileParam{
Service: []string{"goc"},
Address: []string{"http://127.0.0.1:777"},
}
c := &client{
client: http.DefaultClient,
}
_, err := c.Remove(p)
assert.Error(t, err)
assert.Contains(t, err.Error(), "use 'service' flag and 'address' flag at the same time may cause ambiguity, please use them separately")
// remove by valid param
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, "Hello, client")
}))
defer ts.Close()
c.Host = ts.URL
p = ProfileParam{
Address: []string{"http://127.0.0.1:777"},
}
res, err := c.Remove(p)
assert.NoError(t, err)
assert.Equal(t, string(res), "Hello, client\n")
// remove from a invalid center
c.Host = "http://127.0.0.1:11111"
_, err = c.Remove(p)
assert.Error(t, err)
}

539
pkg/cover/cover.go
View File

@ -1,346 +1,17 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cover
import (
"bufio"
"bytes"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
log "github.com/sirupsen/logrus"
"github.com/qiniu/goc/pkg/cover/internal/tool"
"github.com/sirupsen/logrus"
"github.com/qiniu/goc/v2/pkg/config"
)
var (
// ErrCoverPkgFailed represents the error that fails to inject the package
ErrCoverPkgFailed = errors.New("fail to inject code to project")
// ErrCoverListFailed represents the error that fails to list package dependencies
ErrCoverListFailed = errors.New("fail to list package dependencies")
)
// TestCover is a collection of all counters
type TestCover struct {
Mode string
AgentPort string
Center string // cover profile host center
MainPkgCover *PackageCover
DepsCover []*PackageCover
CacheCover map[string]*PackageCover
GlobalCoverVarImportPath string
}
// PackageCover holds all the generate coverage variables of a package
type PackageCover struct {
Package *Package
Vars map[string]*FileVar
}
// FileVar holds the name of the generated coverage variables targeting the named file.
type FileVar struct {
File string
Var string
}
// Package map a package output by go list
// this is subset of package struct in: https://github.com/golang/go/blob/master/src/cmd/go/internal/load/pkg.go#L58
type Package struct {
Dir string `json:"Dir"` // directory containing package sources
ImportPath string `json:"ImportPath"` // import path of package in dir
Name string `json:"Name"` // package name
Target string `json:",omitempty"` // installed target for this package (may be executable)
Root string `json:",omitempty"` // Go root, Go path dir, or module root dir containing this package
Module *ModulePublic `json:",omitempty"` // info about package's module, if any
Goroot bool `json:"Goroot,omitempty"` // is this package in the Go root?
Standard bool `json:"Standard,omitempty"` // is this package part of the standard Go library?
DepOnly bool `json:"DepOnly,omitempty"` // package is only a dependency, not explicitly listed
// Source files
GoFiles []string `json:"GoFiles,omitempty"` // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
CgoFiles []string `json:"CgoFiles,omitempty"` // .go source files that import "C"
// Dependency information
Deps []string `json:"Deps,omitempty"` // all (recursively) imported dependencies
Imports []string `json:",omitempty"` // import paths used by this package
ImportMap map[string]string `json:",omitempty"` // map from source import to ImportPath (identity entries omitted)
// Error information
Incomplete bool `json:"Incomplete,omitempty"` // this package or a dependency has an error
Error *PackageError `json:"Error,omitempty"` // error loading package
DepsErrors []*PackageError `json:"DepsErrors,omitempty"` // errors loading dependencies
}
// ModulePublic represents the package info of a module
type ModulePublic struct {
Path string `json:",omitempty"` // module path
Version string `json:",omitempty"` // module version
Versions []string `json:",omitempty"` // available module versions
Replace *ModulePublic `json:",omitempty"` // replaced by this module
Time *time.Time `json:",omitempty"` // time version was created
Update *ModulePublic `json:",omitempty"` // available update (with -u)
Main bool `json:",omitempty"` // is this the main module?
Indirect bool `json:",omitempty"` // module is only indirectly needed by main module
Dir string `json:",omitempty"` // directory holding local copy of files, if any
GoMod string `json:",omitempty"` // path to go.mod file describing module, if any
GoVersion string `json:",omitempty"` // go version used in module
Error *ModuleError `json:",omitempty"` // error loading module
}
// ModuleError represents the error loading module
type ModuleError struct {
Err string // error text
}
// PackageError is the error info for a package when list failed
type PackageError struct {
ImportStack []string // shortest path from package named on command line to this one
Pos string // position of error (if present, file:line:col)
Err string // the error itself
}
// CoverBuildInfo retreives some info from build
type CoverInfo struct {
Target string
GoPath string
IsMod bool
ModRootPath string
GlobalCoverVarImportPath string // path for the injected global cover var file
OneMainPackage bool
Args string
Mode string
AgentPort string
Center string
}
//Execute inject cover variables for all the .go files in the target folder
func Execute(coverInfo *CoverInfo) error {
target := coverInfo.Target
newGopath := coverInfo.GoPath
// oneMainPackage := coverInfo.OneMainPackage
args := coverInfo.Args
mode := coverInfo.Mode
agentPort := coverInfo.AgentPort
center := coverInfo.Center
globalCoverVarImportPath := coverInfo.GlobalCoverVarImportPath
if coverInfo.IsMod {
globalCoverVarImportPath = filepath.Join(coverInfo.ModRootPath, globalCoverVarImportPath)
} else {
globalCoverVarImportPath = filepath.Base(globalCoverVarImportPath)
}
if !isDirExist(target) {
log.Errorf("Target directory %s not exist", target)
return ErrCoverPkgFailed
}
listArgs := []string{"-json"}
if len(args) != 0 {
listArgs = append(listArgs, args)
}
listArgs = append(listArgs, "./...")
pkgs, err := ListPackages(target, strings.Join(listArgs, " "), newGopath)
if err != nil {
log.Errorf("Fail to list all packages, the error: %v", err)
return err
}
var seen = make(map[string]*PackageCover)
// var seenCache = make(map[string]*PackageCover)
allDecl := ""
for _, pkg := range pkgs {
if pkg.Name == "main" {
log.Printf("handle package: %v", pkg.ImportPath)
// inject the main package
mainCover, mainDecl := AddCounters(pkg, mode, globalCoverVarImportPath)
allDecl += mainDecl
// new a testcover for this service
tc := TestCover{
Mode: mode,
AgentPort: agentPort,
Center: center,
MainPkgCover: mainCover,
GlobalCoverVarImportPath: globalCoverVarImportPath,
}
// handle its dependency
// var internalPkgCache = make(map[string][]*PackageCover)
tc.CacheCover = make(map[string]*PackageCover)
for _, dep := range pkg.Deps {
if packageCover, ok := seen[dep]; ok {
tc.DepsCover = append(tc.DepsCover, packageCover)
continue
}
//only focus package neither standard Go library nor dependency library
if depPkg, ok := pkgs[dep]; ok {
packageCover, depDecl := AddCounters(depPkg, mode, globalCoverVarImportPath)
allDecl += depDecl
tc.DepsCover = append(tc.DepsCover, packageCover)
seen[dep] = packageCover
}
}
// inject Http Cover APIs
var httpCoverApis = fmt.Sprintf("%s/http_cover_apis_auto_generated.go", pkg.Dir)
if err := InjectCountersHandlers(tc, httpCoverApis); err != nil {
log.Errorf("failed to inject counters for package: %s, err: %v", pkg.ImportPath, err)
return ErrCoverPkgFailed
}
}
}
return injectGlobalCoverVarFile(coverInfo, allDecl)
}
// ListPackages list all packages under specific via go list command
// The argument newgopath is if you need to go list in a different GOPATH
func ListPackages(dir string, args string, newgopath string) (map[string]*Package, error) {
cmd := exec.Command("/bin/bash", "-c", "go list "+args)
log.Printf("go list cmd is: %v", cmd.Args)
cmd.Dir = dir
if newgopath != "" {
cmd.Env = append(os.Environ(), fmt.Sprintf("GOPATH=%v", newgopath))
}
var errbuf bytes.Buffer
cmd.Stderr = &errbuf
out, err := cmd.Output()
if err != nil {
log.Errorf("excute `go list -json ./...` command failed, err: %v, stdout: %v, stderr: %v", err, string(out), errbuf.String())
return nil, ErrCoverListFailed
}
log.Infof("\n%v", errbuf.String())
dec := json.NewDecoder(bytes.NewReader(out))
pkgs := make(map[string]*Package, 0)
for {
var pkg Package
if err := dec.Decode(&pkg); err != nil {
if err == io.EOF {
break
}
log.Errorf("reading go list output: %v", err)
return nil, ErrCoverListFailed
}
if pkg.Error != nil {
log.Errorf("list package %s failed with output: %v", pkg.ImportPath, pkg.Error)
return nil, ErrCoverPkgFailed
}
// for _, err := range pkg.DepsErrors {
// log.Fatalf("dependency package list failed, err: %v", err)
// }
pkgs[pkg.ImportPath] = &pkg
}
return pkgs, nil
}
// AddCounters is different from official go tool cover
// 1. only inject covervar++ into source file
// 2. no declarartions for these covervars
// 3. return the declarations as string
func AddCounters(pkg *Package, mode string, globalCoverVarImportPath string) (*PackageCover, string) {
coverVarMap := declareCoverVars(pkg)
decl := ""
for file, coverVar := range coverVarMap {
decl += "\n" + tool.Annotate(path.Join(pkg.Dir, file), mode, coverVar.Var, globalCoverVarImportPath) + "\n"
}
return &PackageCover{
Package: pkg,
Vars: coverVarMap,
}, decl
}
func isDirExist(path string) bool {
s, err := os.Stat(path)
if err != nil {
return false
}
return s.IsDir()
}
// Refer: https://github.com/golang/go/blob/master/src/cmd/go/internal/load/pkg.go#L1334:6
// hasInternalPath looks for the final "internal" path element in the given import path.
// If there isn't one, hasInternalPath returns ok=false.
// Otherwise, hasInternalPath returns ok=true and the index of the "internal".
func hasInternalPath(path string) bool {
// Three cases, depending on internal at start/end of string or not.
// The order matters: we must return the index of the final element,
// because the final one produces the most restrictive requirement
// on the importer.
switch {
case strings.HasSuffix(path, "/internal"):
return true
case strings.Contains(path, "/internal/"):
return true
case path == "internal", strings.HasPrefix(path, "internal/"):
return true
}
return false
}
func getInternalParent(path string) string {
switch {
case strings.HasSuffix(path, "/internal"):
return strings.Split(path, "/internal")[0]
case strings.Contains(path, "/internal/"):
return strings.Split(path, "/internal/")[0]
case path == "internal":
return ""
case strings.HasPrefix(path, "internal/"):
return strings.Split(path, "internal/")[0]
}
return ""
}
func buildCoverCmd(file string, coverVar *FileVar, pkg *Package, mode, newgopath string) *exec.Cmd {
// to construct: go tool cover -mode=atomic -o dest src (note: dest==src)
var newArgs = []string{"tool", "cover"}
newArgs = append(newArgs, "-mode", mode)
newArgs = append(newArgs, "-var", coverVar.Var)
longPath := path.Join(pkg.Dir, file)
newArgs = append(newArgs, "-o", longPath, longPath)
cmd := exec.Command("go", newArgs...)
if newgopath != "" {
cmd.Env = append(os.Environ(), fmt.Sprintf("GOPATH=%v", newgopath))
}
return cmd
}
// declareCoverVars attaches the required cover variables names
// to the files, to be used when annotating the files.
func declareCoverVars(p *Package) map[string]*FileVar {
coverVars := make(map[string]*FileVar)
func declareCoverVars(p *config.Package) map[string]*config.FileVar {
coverVars := make(map[string]*config.FileVar)
coverIndex := 0
// We create the cover counters as new top-level variables in the package.
// We need to avoid collisions with user variables (GoCover_0 is unlikely but still)
@ -353,7 +24,7 @@ func declareCoverVars(p *Package) map[string]*FileVar {
for _, file := range p.GoFiles {
// These names appear in the cmd/cover HTML interface.
var longFile = path.Join(p.ImportPath, file)
coverVars[file] = &FileVar{
coverVars[file] = &config.FileVar{
File: longFile,
Var: fmt.Sprintf("GoCover_%d_%x", coverIndex, h),
}
@ -363,7 +34,7 @@ func declareCoverVars(p *Package) map[string]*FileVar {
for _, file := range p.CgoFiles {
// These names appear in the cmd/cover HTML interface.
var longFile = path.Join(p.ImportPath, file)
coverVars[file] = &FileVar{
coverVars[file] = &config.FileVar{
File: longFile,
Var: fmt.Sprintf("GoCover_%d_%x", coverIndex, h),
}
@ -372,203 +43,3 @@ func declareCoverVars(p *Package) map[string]*FileVar {
return coverVars
}
func declareCacheVars(in *PackageCover) map[string]*FileVar {
sum := sha256.Sum256([]byte(in.Package.ImportPath))
h := fmt.Sprintf("%x", sum[:5])
vars := make(map[string]*FileVar)
coverIndex := 0
for _, v := range in.Vars {
cacheVar := fmt.Sprintf("GoCacheCover_%d_%x", coverIndex, h)
vars[cacheVar] = v
coverIndex++
}
return vars
}
func cacheInternalCover(in *PackageCover) *PackageCover {
c := &PackageCover{}
vars := declareCacheVars(in)
c.Package = in.Package
c.Vars = vars
return c
}
func addCacheCover(pkg *Package, in *PackageCover) *PackageCover {
c := &PackageCover{}
sum := sha256.Sum256([]byte(pkg.ImportPath))
h := fmt.Sprintf("%x", sum[:6])
goFile := fmt.Sprintf("cache_vars_auto_generated_%x.go", h)
p := &Package{
Dir: fmt.Sprintf("%s/cache_%x", pkg.Dir, h),
ImportPath: fmt.Sprintf("%s/cache_%x", pkg.ImportPath, h),
Name: fmt.Sprintf("cache_%x", h),
}
p.GoFiles = append(p.GoFiles, goFile)
c.Package = p
c.Vars = declareCacheVars(in)
return c
}
// CoverageList is a collection and summary over multiple file Coverage objects
type CoverageList []Coverage
// Coverage stores test coverage summary data for one file
type Coverage struct {
FileName string
NCoveredStmts int
NAllStmts int
LineCovLink string
}
type codeBlock struct {
fileName string // the file the code block is in
numStatements int // number of statements in the code block
coverageCount int // number of times the block is covered
}
// CovList converts profile to CoverageList struct
func CovList(f io.Reader) (g CoverageList, err error) {
scanner := bufio.NewScanner(f)
scanner.Scan() // discard first line
g = NewCoverageList()
for scanner.Scan() {
row := scanner.Text()
blk, err := toBlock(row)
if err != nil {
return nil, err
}
blk.addToGroupCov(&g)
}
return
}
// ReadFileToCoverList coverts profile file to CoverageList struct
func ReadFileToCoverList(path string) (g CoverageList, err error) {
f, err := ioutil.ReadFile(path)
if err != nil {
logrus.Errorf("Open file %s failed!", path)
return nil, err
}
g, err = CovList(bytes.NewReader(f))
return
}
// NewCoverageList return empty CoverageList
func NewCoverageList() CoverageList {
return CoverageList{}
}
func newCoverage(name string) *Coverage {
return &Coverage{name, 0, 0, ""}
}
// convert a line in profile file to a codeBlock struct
func toBlock(line string) (res *codeBlock, err error) {
slice := strings.Split(line, " ")
if len(slice) != 3 {
return nil, fmt.Errorf("the profile line %s is not expected", line)
}
blockName := slice[0]
nStmts, _ := strconv.Atoi(slice[1])
coverageCount, _ := strconv.Atoi(slice[2])
return &codeBlock{
fileName: blockName[:strings.Index(blockName, ":")],
numStatements: nStmts,
coverageCount: coverageCount,
}, nil
}
// add blk Coverage to file group Coverage
func (blk *codeBlock) addToGroupCov(g *CoverageList) {
if g.size() == 0 || g.lastElement().Name() != blk.fileName {
// when a new file name is processed
coverage := newCoverage(blk.fileName)
g.append(coverage)
}
cov := g.lastElement()
cov.NAllStmts += blk.numStatements
if blk.coverageCount > 0 {
cov.NCoveredStmts += blk.numStatements
}
}
func (g CoverageList) size() int {
return len(g)
}
func (g CoverageList) lastElement() *Coverage {
return &g[g.size()-1]
}
func (g *CoverageList) append(c *Coverage) {
*g = append(*g, *c)
}
// Sort sorts CoverageList with filenames
func (g CoverageList) Sort() {
sort.SliceStable(g, func(i, j int) bool {
return g[i].Name() < g[j].Name()
})
}
// TotalPercentage returns the total percentage of coverage
func (g CoverageList) TotalPercentage() string {
ratio, err := g.TotalRatio()
if err == nil {
return PercentStr(ratio)
}
return "N/A"
}
// TotalRatio returns the total ratio of covered statements
func (g CoverageList) TotalRatio() (ratio float32, err error) {
var total Coverage
for _, c := range g {
total.NCoveredStmts += c.NCoveredStmts
total.NAllStmts += c.NAllStmts
}
return total.Ratio()
}
// Map returns maps the file name to its coverage for faster retrieval
// & membership check
func (g CoverageList) Map() map[string]Coverage {
m := make(map[string]Coverage)
for _, c := range g {
m[c.Name()] = c
}
return m
}
// Name returns the file name
func (c *Coverage) Name() string {
return c.FileName
}
// Percentage returns the percentage of statements covered
func (c *Coverage) Percentage() string {
ratio, err := c.Ratio()
if err == nil {
return PercentStr(ratio)
}
return "N/A"
}
// Ratio calculates the ratio of statements in a profile
func (c *Coverage) Ratio() (ratio float32, err error) {
if c.NAllStmts == 0 {
err = fmt.Errorf("[%s] has 0 statement", c.Name())
} else {
ratio = float32(c.NCoveredStmts) / float32(c.NAllStmts)
}
return
}
// PercentStr converts a fraction number to percentage string representation
func PercentStr(f float32) string {
return fmt.Sprintf("%.1f%%", f*100)
}

388
pkg/cover/cover_test.go
View File

@ -1,388 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cover
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"reflect"
"strings"
"testing"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/tongjingran/copy"
)
func testCoverage() (c *Coverage) {
return &Coverage{FileName: "fake-coverage", NCoveredStmts: 200, NAllStmts: 300}
}
func TestCoverageRatio(t *testing.T) {
c := testCoverage()
actualRatio, _ := c.Ratio()
assert.Equal(t, float32(c.NCoveredStmts)/float32(c.NAllStmts), actualRatio)
}
func TestRatioErr(t *testing.T) {
c := &Coverage{FileName: "fake-coverage", NCoveredStmts: 200, NAllStmts: 0}
_, err := c.Ratio()
assert.NotEqual(t, err, nil)
}
func TestPercentageNA(t *testing.T) {
c := &Coverage{FileName: "fake-coverage", NCoveredStmts: 200, NAllStmts: 0}
assert.Equal(t, "N/A", c.Percentage())
}
func TestCovList(t *testing.T) {
fileName := "qiniu.com/kodo/apiserver/server/main.go"
fileName1 := "qiniu.com/kodo/apiserver/server/svr.go"
items := []struct {
profile string
expectPer []string
}{
// percentage is 100%
{
profile: "mode: atomic\n" +
fileName + ":32.49,33.13 1 30\n",
expectPer: []string{"100.0%"},
},
// percentage is 50%
{profile: "mode: atomic\n" +
fileName + ":32.49,33.13 1 30\n" +
fileName + ":42.49,43.13 1 0\n",
expectPer: []string{"50.0%"},
},
// two files
{
profile: "mode: atomic\n" +
fileName + ":32.49,33.13 1 30\n" +
fileName1 + ":42.49,43.13 1 0\n",
expectPer: []string{"100.0%", "0.0%"},
},
}
for _, tc := range items {
r := strings.NewReader(tc.profile)
c, err := CovList(r)
c.Sort()
assert.Equal(t, err, nil)
for k, v := range c {
assert.Equal(t, tc.expectPer[k], v.Percentage())
}
}
}
func TestReadFileToCoverList(t *testing.T) {
path := "unknown"
_, err := ReadFileToCoverList(path)
assert.Equal(t, err.Error(), "open unknown: no such file or directory")
}
func TestTotalPercentage(t *testing.T) {
items := []struct {
list CoverageList
expectPer string
}{
{
list: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 0}},
expectPer: "N/A",
},
{
list: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20}},
expectPer: "75.0%",
},
{
list: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20},
Coverage{FileName: "fake-coverage-1", NCoveredStmts: 10, NAllStmts: 30}},
expectPer: "50.0%",
},
}
for _, tc := range items {
assert.Equal(t, tc.expectPer, tc.list.TotalPercentage())
}
}
func TestBuildCoverCmd(t *testing.T) {
var testCases = []struct {
name string
file string
coverVar *FileVar
pkg *Package
mode string
newgopath string
expectCmd *exec.Cmd
}{
{
name: "normal",
file: "c.go",
coverVar: &FileVar{
File: "example/b/c/c.go",
Var: "GoCover_0_643131623532653536333031",
},
pkg: &Package{
Dir: "/go/src/goc/cmd/example-project/b/c",
},
mode: "count",
newgopath: "",
expectCmd: &exec.Cmd{
Path: lookCmdPath("go"),
Args: []string{"go", "tool", "cover", "-mode", "count", "-var", "GoCover_0_643131623532653536333031", "-o",
"/go/src/goc/cmd/example-project/b/c/c.go", "/go/src/goc/cmd/example-project/b/c/c.go"},
},
},
{
name: "normal with gopath",
file: "c.go",
coverVar: &FileVar{
File: "example/b/c/c.go",
Var: "GoCover_0_643131623532653536333031",
},
pkg: &Package{
Dir: "/go/src/goc/cmd/example-project/b/c",
},
mode: "set",
newgopath: "/go/src/goc",
expectCmd: &exec.Cmd{
Path: lookCmdPath("go"),
Args: []string{"go", "tool", "cover", "-mode", "set", "-var", "GoCover_0_643131623532653536333031", "-o",
"/go/src/goc/cmd/example-project/b/c/c.go", "/go/src/goc/cmd/example-project/b/c/c.go"},
Env: append(os.Environ(), fmt.Sprintf("GOPATH=%v", "/go/src/goc")),
},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
cmd := buildCoverCmd(testCase.file, testCase.coverVar, testCase.pkg, testCase.mode, testCase.newgopath)
if !reflect.DeepEqual(cmd, testCase.expectCmd) {
t.Errorf("generated incorrect commands:\nGot: %#v\nExpected:%#v", cmd, testCase.expectCmd)
}
})
}
}
func lookCmdPath(name string) string {
if filepath.Base(name) == name {
if lp, err := exec.LookPath(name); err != nil {
log.Fatalf("find exec %s err: %v", name, err)
} else {
return lp
}
}
return ""
}
func TestDeclareCoverVars(t *testing.T) {
var testCases = []struct {
name string
pkg *Package
expectCoverVar map[string]*FileVar
}{
{
name: "normal",
pkg: &Package{
Dir: "/go/src/goc/cmd/example-project/b/c",
GoFiles: []string{"c.go"},
ImportPath: "example/b/c",
},
expectCoverVar: map[string]*FileVar{
"c.go": {File: "example/b/c/c.go", Var: "GoCover_0_643131623532653536333031"},
},
},
{
name: "more go files",
pkg: &Package{
Dir: "/go/src/goc/cmd/example-project/a/b",
GoFiles: []string{"printf.go", "printf1.go"},
ImportPath: "example/a/b",
},
expectCoverVar: map[string]*FileVar{
"printf.go": {File: "example/a/b/printf.go", Var: "GoCover_0_326535623364613565313464"},
"printf1.go": {File: "example/a/b/printf1.go", Var: "GoCover_1_326535623364613565313464"},
},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
coverVar := declareCoverVars(testCase.pkg)
if !reflect.DeepEqual(coverVar, testCase.expectCoverVar) {
t.Errorf("generated incorrect cover vars:\nGot: %#v\nExpected:%#v", coverVar, testCase.expectCoverVar)
}
})
}
}
func TestGetInternalParent(t *testing.T) {
var tcs = []struct {
ImportPath string
expectedParent string
}{
{
ImportPath: "a/internal/b",
expectedParent: "a",
},
{
ImportPath: "internal/b",
expectedParent: "",
},
{
ImportPath: "a/b/internal/b",
expectedParent: "a/b",
},
{
ImportPath: "a/b/internal",
expectedParent: "a/b",
},
{
ImportPath: "a/b/internal/c",
expectedParent: "a/b",
},
{
ImportPath: "a/b/c",
expectedParent: "",
},
{
ImportPath: "",
expectedParent: "",
},
}
for _, tc := range tcs {
actual := getInternalParent(tc.ImportPath)
if actual != tc.expectedParent {
t.Errorf("getInternalParent failed for importPath %s, expected %s, got %s", tc.ImportPath, tc.expectedParent, actual)
}
}
}
func TestFindInternal(t *testing.T) {
var tcs = []struct {
ImportPath string
expectedParent bool
}{
{
ImportPath: "a/internal/b",
expectedParent: true,
},
{
ImportPath: "internal/b",
expectedParent: true,
},
{
ImportPath: "a/b/internal",
expectedParent: true,
},
{
ImportPath: "a/b/c",
expectedParent: false,
},
{
ImportPath: "internal",
expectedParent: true,
},
}
for _, tc := range tcs {
actual := hasInternalPath(tc.ImportPath)
if actual != tc.expectedParent {
t.Errorf("hasInternalPath check failed for importPath %s", tc.ImportPath)
}
}
}
func TestExecuteForSimpleModProject(t *testing.T) {
workingDir := "../../tests/samples/simple_project"
gopath := ""
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
testDir := filepath.Join(os.TempDir(), "goc-build-test")
copy.Copy(workingDir, testDir)
bi := &CoverInfo{
Args: "",
GoPath: gopath,
Target: testDir,
Mode: "count",
AgentPort: "",
Center: "http://127.0.0.1:7777",
OneMainPackage: false,
}
_ = Execute(bi)
_, err := os.Lstat(filepath.Join(testDir, "http_cover_apis_auto_generated.go"))
if !assert.Equal(t, err, nil) {
assert.FailNow(t, "should generate http_cover_apis_auto_generated.go")
}
}
func TestListPackagesForSimpleModProject(t *testing.T) {
workingDir := "../../tests/samples/simple_project"
gopath := ""
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
pkgs, _ := ListPackages(workingDir, "-json ./...", "")
if !assert.Equal(t, len(pkgs), 1) {
assert.FailNow(t, "should only have one pkg")
}
if pkg, ok := pkgs["example.com/simple-project"]; ok {
assert.Equal(t, pkg.Module.Path, "example.com/simple-project")
} else {
assert.FailNow(t, "cannot get the pkg: example.com/simple-project")
}
}
// test if goc can get variables in internal package
func TestCoverResultForInternalPackage(t *testing.T) {
workingDir := "../../tests/samples/simple_project_with_internal"
gopath := ""
os.Setenv("GOPATH", gopath)
os.Setenv("GO111MODULE", "on")
testDir := filepath.Join(os.TempDir(), "goc-build-test")
copy.Copy(workingDir, testDir)
bi := &CoverInfo{
Target: testDir,
GoPath: gopath,
Args: "",
Mode: "count",
Center: "http://127.0.0.1:7777",
OneMainPackage: false,
AgentPort: "",
}
_ = Execute(bi)
_, err := os.Lstat(filepath.Join(testDir, "http_cover_apis_auto_generated.go"))
if !assert.Equal(t, err, nil) {
assert.FailNow(t, "should generate http_cover_apis_auto_generated.go")
}
}

136
pkg/cover/delta.go
View File

@ -1,136 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cover
import "sort"
// DeltaCov contains the info of a delta coverage
type DeltaCov struct {
FileName string
BasePer string
NewPer string
DeltaPer string
LineCovLink string
}
// DeltaCovList is the list of DeltaCov
type DeltaCovList []DeltaCov
// GetFullDeltaCov get full delta coverage between new and base profile
func GetFullDeltaCov(newList CoverageList, baseList CoverageList) (delta DeltaCovList) {
newMap := newList.Map()
baseMap := baseList.Map()
for file, n := range newMap {
b, ok := baseMap[file]
//if the file not in base profile, set None
if !ok {
delta = append(delta, DeltaCov{
FileName: file,
BasePer: "None",
NewPer: n.Percentage(),
DeltaPer: PercentStr(Delta(n, b))})
continue
}
delta = append(delta, DeltaCov{
FileName: file,
BasePer: b.Percentage(),
NewPer: n.Percentage(),
DeltaPer: PercentStr(Delta(n, b))})
}
for file, b := range baseMap {
//if the file not in new profile, set None
if n, ok := newMap[file]; !ok {
delta = append(delta, DeltaCov{
FileName: file,
BasePer: b.Percentage(),
NewPer: "None",
DeltaPer: PercentStr(Delta(n, b))})
}
}
return
}
// GetDeltaCov get two profile diff cov
func GetDeltaCov(newList CoverageList, baseList CoverageList) (delta DeltaCovList) {
d := GetFullDeltaCov(newList, baseList)
for _, v := range d {
if v.DeltaPer == "0.0%" {
continue
}
delta = append(delta, v)
}
return
}
// GetChFileDeltaCov get two profile diff cov of changed files
func GetChFileDeltaCov(newList CoverageList, baseList CoverageList, changedFiles []string) (list DeltaCovList) {
d := GetFullDeltaCov(newList, baseList)
dMap := d.Map()
for _, file := range changedFiles {
if _, ok := dMap[file]; ok {
list = append(list, dMap[file])
}
}
return
}
// Delta calculate two coverage delta
func Delta(new Coverage, base Coverage) float32 {
baseRatio, _ := base.Ratio()
newRatio, _ := new.Ratio()
return newRatio - baseRatio
}
// TotalDelta calculate two coverage delta
func TotalDelta(new CoverageList, base CoverageList) float32 {
baseRatio, _ := base.TotalRatio()
newRatio, _ := new.TotalRatio()
return newRatio - baseRatio
}
// Map returns maps the file name to its DeltaCov for faster retrieval & membership check
func (d DeltaCovList) Map() map[string]DeltaCov {
m := make(map[string]DeltaCov)
for _, c := range d {
m[c.FileName] = c
}
return m
}
// Sort sort DeltaCovList with filenames
func (d DeltaCovList) Sort() {
sort.SliceStable(d, func(i, j int) bool {
return d[i].Name() < d[j].Name()
})
}
// Name returns the file name
func (c *DeltaCov) Name() string {
return c.FileName
}
// GetLineCovLink get the LineCovLink of the DeltaCov
func (c *DeltaCov) GetLineCovLink() string {
return c.LineCovLink
}
// SetLineCovLink set LineCovLink of the DeltaCov
func (c *DeltaCov) SetLineCovLink(link string) {
c.LineCovLink = link
}

133
pkg/cover/delta_test.go
View File

@ -1,133 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cover
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestGetDeltaCov(t *testing.T) {
items := []struct {
newList CoverageList
baseList CoverageList
expectDelta DeltaCovList
rows int
}{
//coverage increase
{
newList: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20}},
baseList: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 10, NAllStmts: 20}},
expectDelta: DeltaCovList{{FileName: "fake-coverage", BasePer: "50.0%", NewPer: "75.0%", DeltaPer: "25.0%"}},
rows: 1,
},
//coverage decrease
{
newList: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20}},
baseList: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 20, NAllStmts: 20}},
expectDelta: DeltaCovList{{FileName: "fake-coverage", BasePer: "100.0%", NewPer: "75.0%", DeltaPer: "-25.0%"}},
rows: 1,
},
//diff file
{
newList: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20}},
baseList: CoverageList{Coverage{FileName: "fake-coverage-v1", NCoveredStmts: 10, NAllStmts: 20}},
expectDelta: DeltaCovList{{FileName: "fake-coverage", BasePer: "None", NewPer: "75.0%", DeltaPer: "75.0%"},
{FileName: "fake-coverage-v1", BasePer: "50.0%", NewPer: "None", DeltaPer: "-50.0%"}},
rows: 2,
},
//one file has same coverage rate
{
newList: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20}},
baseList: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20},
Coverage{FileName: "fake-coverage-v1", NCoveredStmts: 10, NAllStmts: 20}},
expectDelta: DeltaCovList{{FileName: "fake-coverage-v1", BasePer: "50.0%", NewPer: "None", DeltaPer: "-50.0%"}},
rows: 1,
},
}
for _, tc := range items {
d := GetDeltaCov(tc.newList, tc.baseList)
assert.Equal(t, tc.rows, len(d))
assert.Equal(t, tc.expectDelta, d)
}
}
func TestGetChFileDeltaCov(t *testing.T) {
items := []struct {
newList CoverageList
baseList CoverageList
changedFiles []string
expectDelta DeltaCovList
}{
{
newList: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20}},
baseList: CoverageList{Coverage{FileName: "fake-coverage-v1", NCoveredStmts: 10, NAllStmts: 20}},
changedFiles: []string{"fake-coverage"},
expectDelta: DeltaCovList{{FileName: "fake-coverage", BasePer: "None", NewPer: "75.0%", DeltaPer: "75.0%"}},
},
}
for _, tc := range items {
d := GetChFileDeltaCov(tc.newList, tc.baseList, tc.changedFiles)
assert.Equal(t, tc.expectDelta, d)
}
}
func TestMapAndSort(t *testing.T) {
items := []struct {
dList DeltaCovList
expectMap map[string]DeltaCov
expectSort DeltaCovList
}{
{
dList: DeltaCovList{DeltaCov{FileName: "b", BasePer: "10.0%", NewPer: "20.0%", DeltaPer: "10.0%"},
DeltaCov{FileName: "a", BasePer: "10.0%", NewPer: "30.0%", DeltaPer: "20.0%"},
},
expectMap: map[string]DeltaCov{
"a": {FileName: "a", BasePer: "10.0%", NewPer: "30.0%", DeltaPer: "20.0%"},
"b": {FileName: "b", BasePer: "10.0%", NewPer: "20.0%", DeltaPer: "10.0%"},
},
expectSort: DeltaCovList{DeltaCov{FileName: "a", BasePer: "10.0%", NewPer: "30.0%", DeltaPer: "20.0%"},
DeltaCov{FileName: "b", BasePer: "10.0%", NewPer: "20.0%", DeltaPer: "10.0%"},
},
},
{
dList: DeltaCovList{DeltaCov{FileName: "b", BasePer: "10.0%", NewPer: "20.0%", DeltaPer: "10.0%"},
DeltaCov{FileName: "b-1", BasePer: "10.0%", NewPer: "30.0%", DeltaPer: "20.0%"},
DeltaCov{FileName: "1-b", BasePer: "10.0%", NewPer: "40.0%", DeltaPer: "30.0%"},
},
expectMap: map[string]DeltaCov{
"1-b": {FileName: "1-b", BasePer: "10.0%", NewPer: "40.0%", DeltaPer: "30.0%"},
"b": {FileName: "b", BasePer: "10.0%", NewPer: "20.0%", DeltaPer: "10.0%"},
"b-1": {FileName: "b-1", BasePer: "10.0%", NewPer: "30.0%", DeltaPer: "20.0%"},
},
expectSort: DeltaCovList{DeltaCov{FileName: "1-b", BasePer: "10.0%", NewPer: "40.0%", DeltaPer: "30.0%"},
DeltaCov{FileName: "b", BasePer: "10.0%", NewPer: "20.0%", DeltaPer: "10.0%"},
DeltaCov{FileName: "b-1", BasePer: "10.0%", NewPer: "30.0%", DeltaPer: "20.0%"},
},
},
}
for _, tc := range items {
assert.Equal(t, tc.expectMap, tc.dList.Map())
tc.dList.Sort()
assert.Equal(t, tc.expectSort, tc.dList)
}
}

236
pkg/cover/inject.go Normal file
View File

@ -0,0 +1,236 @@
package cover
import (
"os"
"path"
"path/filepath"
"github.com/qiniu/goc/v2/pkg/config"
"github.com/qiniu/goc/v2/pkg/cover/internal/tool"
"github.com/qiniu/goc/v2/pkg/cover/internal/websocket"
"github.com/qiniu/goc/v2/pkg/log"
)
// Inject injects cover variables for all the .go files in the target directory
func Inject() {
log.StartWait("injecting cover variables")
var seen = make(map[string]*config.PackageCover)
// 所有插桩变量定义声明
allDecl := ""
pkgs := config.GocConfig.Pkgs
for _, pkg := range pkgs {
if pkg.Name == "main" {
log.Infof("handle main package: %v", pkg.ImportPath)
// 该 main 二进制所关联的所有插桩变量的元信息
// 每个 main 之间是不相关的，需要重新定义
allMainCovers := make([]*config.PackageCover, 0)
// 注入 main package
mainCover, mainDecl := addCounters(pkg)
// 收集插桩变量的定义和元信息
allDecl += mainDecl
allMainCovers = append(allMainCovers, mainCover)
// 向 main package 的依赖注入插桩变量
for _, dep := range pkg.Deps {
if _, ok := seen[dep]; ok {
continue
}
// 依赖需要忽略 Go 标准库和 go.mod 引入的第三方
if depPkg, ok := pkgs[dep]; ok {
// 注入依赖的 package
packageCover, depDecl := addCounters(depPkg)
// 收集插桩变量的定义和元信息
allDecl += depDecl
allMainCovers = append(allMainCovers, packageCover)
// 避免重复访问
seen[dep] = packageCover
}
}
// 为每个 main 包注入 websocket handler
injectGocAgent(getPkgTmpDir(pkg.Dir), allMainCovers)
}
}
// 在工程根目录注入所有插桩变量的声明+定义
injectGlobalCoverVarFile(allDecl)
// 在工程根目录注入 watch agent 的定义
if config.GocConfig.Mode == "watch" {
log.Infof("watch mode is enabled")
injectWatchAgentFile()
log.Donef("watch handler injected")
}
// 添加自定义 websocket 依赖
// 用户代码可能有 gorrila/websocket 的依赖，为避免依赖冲突，以及可能的 replace/vendor，
// 这里直接注入一份完整的 gorrila/websocket 实现
websocket.AddCustomWebsocketDep()
log.Donef("websocket library injected")
log.StopWait()
log.Donef("cover variables injected")
}
// addCounters is different from official go tool cover
//
// 1. only inject covervar++ into source file
//
// 2. no declarartions for these covervars
//
// 3. return the declarations as string
func addCounters(pkg *config.Package) (*config.PackageCover, string) {
mode := config.GocConfig.Mode
gobalCoverVarImportPath := config.GocConfig.GlobalCoverVarImportPath
coverVarMap := declareCoverVars(pkg)
decl := ""
for file, coverVar := range coverVarMap {
decl += "\n" + tool.Annotate(filepath.Join(getPkgTmpDir(pkg.Dir), file), mode, coverVar.Var, coverVar.File, gobalCoverVarImportPath) + "\n"
}
return &config.PackageCover{
Package: pkg,
Vars: coverVarMap,
}, decl
}
// getPkgTmpDir gets corresponding pkg dir in temporary project
//
// the reason is that config.GocConfig.Pkgs is get in the original project.
// we need to transfer the direcory.
//
// 在原工程目录已经做了一次 go list -json，在临时目录没有必要再做一遍，直接转换一下就能得到
// 临时目录中的 pkg.Dir。
func getPkgTmpDir(pkgDir string) string {
relDir, err := filepath.Rel(config.GocConfig.CurModProjectDir, pkgDir)
if err != nil {
log.Fatalf("go json -list meta info wrong: %v", err)
}
return filepath.Join(config.GocConfig.TmpModProjectDir, relDir)
}
// injectGocAgent inject handlers like following
//
// - xxx.go
// - yyy_package
// - main.go
// - goc-cover-agent-apis-auto-generated-11111-22222-bridge.go
// - goc-cover-agent-apis-auto-generated-11111-22222-package
// |
// -- init.go
//
// 11111_22222_bridge.go 仅仅用于引用 11111_22222_package, where package contains ws agent main logic.
// 使用 bridge.go 文件是为了避免插桩逻辑中的变量名污染 main 包
func injectGocAgent(where string, covers []*config.PackageCover) {
injectPkgName := "goc-cover-agent-apis-auto-generated-11111-22222-package"
injectBridgeName := "goc-cover-agent-apis-auto-generated-11111-22222-bridge.go"
wherePkg := filepath.Join(where, injectPkgName)
err := os.MkdirAll(wherePkg, os.ModePerm)
if err != nil {
log.Fatalf("fail to generate %v directory: %v", injectPkgName, err)
}
// create bridge file
whereBridge := filepath.Join(where, injectBridgeName)
f1, err := os.Create(whereBridge)
if err != nil {
log.Fatalf("fail to create cover bridge file in temporary project: %v", err)
}
defer f1.Close()
tmplBridgeData := struct {
CoverImportPath string
}{
// covers[0] is the main package
CoverImportPath: covers[0].Package.ImportPath + "/" + injectPkgName,
}
if err := coverBridgeTmpl.Execute(f1, tmplBridgeData); err != nil {
log.Fatalf("fail to generate cover bridge in temporary project: %v", err)
}
// create ws agent files
dest := filepath.Join(wherePkg, "init.go")
f2, err := os.Create(dest)
if err != nil {
log.Fatalf("fail to create cover agent file in temporary project: %v", err)
}
defer f2.Close()
var _coverMode string
if config.GocConfig.Mode == "watch" {
_coverMode = "cover"
} else {
_coverMode = config.GocConfig.Mode
}
tmplData := struct {
Covers []*config.PackageCover
GlobalCoverVarImportPath string
Package string
Host string
Mode string
}{
Covers: covers,
GlobalCoverVarImportPath: config.GocConfig.GlobalCoverVarImportPath,
Package: injectPkgName,
Host: config.GocConfig.Host,
Mode: _coverMode,
}
if err := coverMainTmpl.Execute(f2, tmplData); err != nil {
log.Fatalf("fail to generate cover agent handlers in temporary project: %v", err)
}
}
// injectGlobalCoverVarFile 写入所有插桩变量的全局定义至一个单独的文件
func injectGlobalCoverVarFile(decl string) {
globalCoverVarPackage := path.Base(config.GocConfig.GlobalCoverVarImportPath)
globalCoverDef := filepath.Join(config.GocConfig.TmpModProjectDir, globalCoverVarPackage)
config.GocConfig.GlobalCoverVarImportPathDir = globalCoverDef
err := os.MkdirAll(globalCoverDef, os.ModePerm)
if err != nil {
log.Fatalf("fail to create global cover definition package dir: %v", err)
}
coverFile, err := os.Create(filepath.Join(globalCoverDef, "cover.go"))
if err != nil {
log.Fatalf("fail to create global cover definition file: %v", err)
}
defer coverFile.Close()
packageName := "package coverdef\n\n"
_, err = coverFile.WriteString(packageName + decl)
if err != nil {
log.Fatalf("fail to write to global cover definition file: %v", err)
}
}
func injectWatchAgentFile() {
globalCoverVarPackage := path.Base(config.GocConfig.GlobalCoverVarImportPath)
globalCoverDef := filepath.Join(config.GocConfig.TmpModProjectDir, globalCoverVarPackage)
f, err := os.Create(filepath.Join(globalCoverDef, "watchagent.go"))
if err != nil {
log.Fatalf("fail to create watchagent file: %v", err)
}
tmplData := struct {
Random string
Host string
GlobalCoverVarImportPath string
}{
Random: filepath.Base(config.GocConfig.TmpModProjectDir),
Host: config.GocConfig.Host,
GlobalCoverVarImportPath: config.GocConfig.GlobalCoverVarImportPath,
}
if err := coverWatchTmpl.Execute(f, tmplData); err != nil {
log.Fatalf("fail to generate watchagent in temporary project: %v", err)
}
}

505
pkg/cover/instrument.go
View File

@ -1,505 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cover
import (
"fmt"
"os"
"path"
"path/filepath"
"text/template"
)
// InjectCountersHandlers generate a file _cover_http_apis.go besides the main.go file
func InjectCountersHandlers(tc TestCover, dest string) error {
f, err := os.Create(dest)
if err != nil {
return err
}
if err := coverMainTmpl.Execute(f, tc); err != nil {
return err
}
return nil
}
var coverMainTmpl = template.Must(template.New("coverMain").Parse(coverMain))
const coverMain = `
// Code generated by goc system. DO NOT EDIT.
package main
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net"
"net/http"
"os"
"os/signal"
"path/filepath"
"strings"
"sync/atomic"
"syscall"
"testing"
_cover {{.GlobalCoverVarImportPath | printf "%q"}}
)
func init() {
go registerHandlers()
}
func loadValues() (map[string][]uint32, map[string][]testing.CoverBlock) {
var (
coverCounters = make(map[string][]uint32)
coverBlocks = make(map[string][]testing.CoverBlock)
)
{{range $i, $pkgCover := .DepsCover}}
{{range $file, $cover := $pkgCover.Vars}}
loadFileCover(coverCounters, coverBlocks, {{printf "%q" $cover.File}}, _cover.{{$cover.Var}}.Count[:], _cover.{{$cover.Var}}.Pos[:], _cover.{{$cover.Var}}.NumStmt[:])
{{end}}
{{end}}
{{range $file, $cover := .MainPkgCover.Vars}}
loadFileCover(coverCounters, coverBlocks, {{printf "%q" $cover.File}}, _cover.{{$cover.Var}}.Count[:], _cover.{{$cover.Var}}.Pos[:], _cover.{{$cover.Var}}.NumStmt[:])
{{end}}
return coverCounters, coverBlocks
}
func loadFileCover(coverCounters map[string][]uint32, coverBlocks map[string][]testing.CoverBlock, fileName string, counter []uint32, pos []uint32, numStmts []uint16) {
if 3*len(counter) != len(pos) || len(counter) != len(numStmts) {
panic("coverage: mismatched sizes")
}
if coverCounters[fileName] != nil {
// Already registered.
return
}
coverCounters[fileName] = counter
block := make([]testing.CoverBlock, len(counter))
for i := range counter {
block[i] = testing.CoverBlock{
Line0: pos[3*i+0],
Col0: uint16(pos[3*i+2]),
Line1: pos[3*i+1],
Col1: uint16(pos[3*i+2] >> 16),
Stmts: numStmts[i],
}
}
coverBlocks[fileName] = block
}
func clearValues() {
{{range $i, $pkgCover := .DepsCover}}
{{range $file, $cover := $pkgCover.Vars}}
clearFileCover(_cover.{{$cover.Var}}.Count[:])
{{end}}
{{end}}
{{range $file, $cover := .MainPkgCover.Vars}}
clearFileCover(_cover.{{$cover.Var}}.Count[:])
{{end}}
}
func clearFileCover(counter []uint32) {
for i := range counter {
counter[i] = 0
}
}
func registerHandlers() {
ln, host, err := listen()
if err != nil {
log.Fatalf("listen failed, err:%v", err)
}
profileAddr := "http://" + host
if resp, err := registerSelf(profileAddr); err != nil {
log.Fatalf("register address %v failed, err: %v, response: %v", profileAddr, err, string(resp))
}
fn := func() {
var (
err error
profileAddrs []string
addresses []string
)
if addresses, err = getAllHosts(ln); err != nil {
log.Fatalf("get all host failed, err: %v", err)
return
}
for _, addr := range addresses {
profileAddrs = append(profileAddrs, "http://"+addr)
}
deregisterSelf(profileAddrs)
}
go watchSignal(fn)
mux := http.NewServeMux()
// Coverage reports the current code coverage as a fraction in the range [0, 1].
// If coverage is not enabled, Coverage returns 0.
mux.HandleFunc("/v1/cover/coverage", func(w http.ResponseWriter, r *http.Request) {
counters, _ := loadValues()
var n, d int64
for _, counter := range counters {
for i := range counter {
if atomic.LoadUint32(&counter[i]) > 0 {
n++
}
d++
}
}
if d == 0 {
fmt.Fprint(w, 0)
return
}
fmt.Fprintf(w, "%f", float64(n)/float64(d))
})
// coverprofile reports a coverage profile with the coverage percentage
mux.HandleFunc("/v1/cover/profile", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, "mode: {{.Mode}}\n")
counters, blocks := loadValues()
var active, total int64
var count uint32
for name, counts := range counters {
block := blocks[name]
for i := range counts {
stmts := int64(block[i].Stmts)
total += stmts
count = atomic.LoadUint32(&counts[i]) // For -mode=atomic.
if count > 0 {
active += stmts
}
_, err := fmt.Fprintf(w, "%s:%d.%d,%d.%d %d %d\n", name,
block[i].Line0, block[i].Col0,
block[i].Line1, block[i].Col1,
stmts,
count)
if err != nil {
fmt.Fprintf(w, "invalid block format, err: %v", err)
return
}
}
}
})
mux.HandleFunc("/v1/cover/clear", func(w http.ResponseWriter, r *http.Request) {
clearValues()
w.WriteHeader(http.StatusOK)
fmt.Fprintln(w, "clear call successfully")
})
log.Fatal(http.Serve(ln, mux))
}
func registerSelf(address string) ([]byte, error) {
selfName := filepath.Base(os.Args[0])
req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1/cover/register?name=%s&address=%s", {{.Center | printf "%q"}}, selfName, address), nil)
if err != nil {
log.Fatalf("http.NewRequest failed: %v", err)
return nil, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil && isNetworkError(err) {
log.Printf("[goc][WARN]error occurred:%v, try again", err)
resp, err = http.DefaultClient.Do(req)
}
if err != nil {
return nil, fmt.Errorf("failed to register into coverage center, err:%v", err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response body, err:%v", err)
}
if resp.StatusCode != 200 {
err = fmt.Errorf("failed to register into coverage center, response code %d", resp.StatusCode)
}
return body, err
}
func deregisterSelf(address []string) ([]byte, error) {
param := map[string]interface{}{
"address": address,
}
jsonBody, err := json.Marshal(param)
if err != nil {
return nil, err
}
req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1/cover/remove", {{.Center | printf "%q"}}), bytes.NewReader(jsonBody))
if err != nil {
log.Fatalf("http.NewRequest failed: %v", err)
return nil, err
}
req.Header.Set("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil && isNetworkError(err) {
log.Printf("[goc][WARN]error occurred:%v, try again", err)
resp, err = http.DefaultClient.Do(req)
}
if err != nil {
return nil, fmt.Errorf("failed to deregister into coverage center, err:%v", err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response body, err:%v", err)
}
if resp.StatusCode != 200 {
err = fmt.Errorf("failed to deregister into coverage center, response code %d", resp.StatusCode)
}
return body, err
}
type CallbackFunc func()
func watchSignal(fn CallbackFunc) {
// init signal
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
for {
si := <-c
log.Printf("get a signal %s", si.String())
switch si {
case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
fn()
os.Exit(0) // Exit successfully.
case syscall.SIGHUP:
default:
return
}
}
}
func isNetworkError(err error) bool {
if err == io.EOF {
return true
}
_, ok := err.(net.Error)
return ok
}
func listen() (ln net.Listener, host string, err error) {
agentPort := "{{.AgentPort }}"
if agentPort != "" {
if ln, err = net.Listen("tcp4", agentPort); err != nil {
return
}
if host, err = getRealHost(ln); err != nil {
return
}
} else {
// 获取上次使用的监听地址
if previousAddr := getPreviousAddr(); previousAddr != "" {
ss := strings.Split(previousAddr, ":")
// listen on all network interface
ln, err = net.Listen("tcp4", ":"+ss[len(ss)-1])
if err == nil {
host = previousAddr
return
}
}
if ln, err = net.Listen("tcp4", ":0"); err != nil {
return
}
if host, err = getRealHost(ln); err != nil {
return
}
}
go genProfileAddr(host)
return
}
func getRealHost(ln net.Listener) (host string, err error) {
adds, err := net.InterfaceAddrs()
if err != nil {
return
}
var localIPV4 string
var nonLocalIPV4 string
for _, addr := range adds {
if ipNet, ok := addr.(*net.IPNet); ok && ipNet.IP.To4() != nil {
if ipNet.IP.IsLoopback() {
localIPV4 = ipNet.IP.String()
} else {
nonLocalIPV4 = ipNet.IP.String()
}
}
}
if nonLocalIPV4 != "" {
host = fmt.Sprintf("%s:%d", nonLocalIPV4, ln.Addr().(*net.TCPAddr).Port)
} else {
host = fmt.Sprintf("%s:%d", localIPV4, ln.Addr().(*net.TCPAddr).Port)
}
return
}
func getAllHosts(ln net.Listener) (hosts []string, err error) {
adds, err := net.InterfaceAddrs()
if err != nil {
return
}
var host string
for _, addr := range adds {
if ipNet, ok := addr.(*net.IPNet); ok && ipNet.IP.To4() != nil {
host = fmt.Sprintf("%s:%d", ipNet.IP.String(), ln.Addr().(*net.TCPAddr).Port)
hosts = append(hosts, host)
}
}
return
}
func getPreviousAddr() string {
file, err := os.Open(os.Args[0] + "_profile_listen_addr")
if err != nil {
return ""
}
defer file.Close()
reader := bufio.NewReader(file)
addr, _, _ := reader.ReadLine()
return string(addr)
}
func genProfileAddr(profileAddr string) {
fn := os.Args[0] + "_profile_listen_addr"
f, err := os.OpenFile(fn, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
log.Println(err)
return
}
defer f.Close()
fmt.Fprintf(f, strings.TrimPrefix(profileAddr, "http://"))
}
`
var coverParentFileTmpl = template.Must(template.New("coverParentFileTmpl").Parse(coverParentFile))
const coverParentFile = `
// Code generated by goc system. DO NOT EDIT.
package {{.}}
`
var coverParentVarsTmpl = template.Must(template.New("coverParentVarsTmpl").Parse(coverParentVars))
const coverParentVars = `
import (
{{range $i, $pkgCover := .}}
_cover{{$i}} {{$pkgCover.Package.ImportPath | printf "%q"}}
{{end}}
)
{{range $i, $pkgCover := .}}
{{range $v, $cover := $pkgCover.Vars}}
var {{$v}} = &_cover{{$i}}.{{$cover.Var}}
{{end}}
{{end}}
`
func InjectCacheCounters(covers map[string][]*PackageCover, cache map[string]*PackageCover) []error {
var errs []error
for k, v := range covers {
if pkg, ok := cache[k]; ok {
err := checkCacheDir(pkg.Package.Dir)
if err != nil {
errs = append(errs, err)
continue
}
_, pkgName := path.Split(k)
err = injectCache(v, pkgName, fmt.Sprintf("%s/%s", pkg.Package.Dir, pkg.Package.GoFiles[0]))
if err != nil {
errs = append(errs, err)
continue
}
}
}
return errs
}
// InjectCacheCounters generate a file _cover_http_apis.go besides the main.go file
func injectCache(covers []*PackageCover, pkg, dest string) error {
f, err := os.Create(dest)
if err != nil {
return err
}
if err := coverParentFileTmpl.Execute(f, pkg); err != nil {
return err
}
if err := coverParentVarsTmpl.Execute(f, covers); err != nil {
return err
}
return nil
}
func checkCacheDir(p string) error {
_, err := os.Stat(p)
if os.IsNotExist(err) {
err := os.Mkdir(p, 0755)
if err != nil {
return err
}
}
return nil
}
func injectGlobalCoverVarFile(ci *CoverInfo, content string) error {
coverFile, err := os.Create(filepath.Join(ci.Target, ci.GlobalCoverVarImportPath, "cover.go"))
if err != nil {
return err
}
defer coverFile.Close()
packageName := "package " + filepath.Base(ci.GlobalCoverVarImportPath) + "\n\n"
_, err = coverFile.WriteString(packageName)
if err != nil {
return err
}
_, err = coverFile.WriteString(content)
return err
}

53
pkg/cover/internal/tool/cover.go
View File

@ -6,6 +6,8 @@ package tool
import (
"bytes"
"path"
// "flag"
"fmt"
"go/ast"
@ -16,7 +18,7 @@ import (
"os"
"sort"
log "github.com/sirupsen/logrus" // QINIU
"github.com/qiniu/goc/v2/pkg/log" // QINIU
// "cmd/internal/edit"
// "cmd/internal/objabi"
)
@ -155,14 +157,16 @@ type Block struct {
// File is a wrapper for the state of a file used in the parser.
// The basic parse tree walker is a method of this type.
type File struct {
fset *token.FileSet
name string // Name of file.
astFile *ast.File
blocks []Block
content []byte
edit *Buffer // QINIU
varVar string // QINIU
mode string // QINIU
fset *token.FileSet
name string // Name of file.
astFile *ast.File
blocks []Block
content []byte
edit *Buffer // QINIU
varVar string // QINIU
mode string // QINIU
importpathFileName string // QINIU, importpath + filename
random string // QINIU, random == tmp dir name
}
// findText finds text in the original source, starting at pos.
@ -304,7 +308,7 @@ func (f *File) Visit(node ast.Node) ast.Visitor {
// 1. add cover variables into the original file
// 2. return the cover variables declarations as plain string
// original dec: func annotate(name string) {
func Annotate(name string, mode string, varVar string, globalCoverVarImportPath string) string {
func Annotate(name string, mode string, varVar string, importpathFilename string, globalCoverVarImportPath string) string {
// QINIU
switch mode {
case "set":
@ -313,6 +317,8 @@ func Annotate(name string, mode string, varVar string, globalCoverVarImportPath
counterStmt = incCounterStmt
case "atomic":
counterStmt = atomicCounterStmt
case "watch":
counterStmt = watchCounterStmt
default:
counterStmt = incCounterStmt
}
@ -328,20 +334,22 @@ func Annotate(name string, mode string, varVar string, globalCoverVarImportPath
}
file := &File{
fset: fset,
name: name,
content: content,
edit: NewBuffer(content), // QINIU
astFile: parsedFile,
varVar: varVar,
mode: mode,
fset: fset,
name: name,
content: content,
edit: NewBuffer(content), // QINIU
astFile: parsedFile,
varVar: varVar, // QINIU
mode: mode, // QINIU
importpathFileName: importpathFilename, // QINIU
random: path.Base(globalCoverVarImportPath), // QINIU
}
ast.Walk(file, file.astFile)
newContent := file.edit.Bytes()
if bytes.Equal(content, newContent) {
log.Info("no cover var injected for: ", name)
log.Debugf("no cover var injected for: ", name)
} else {
// reback to the beginning
file.astFile, _ = parser.ParseFile(fset, name, content, parser.ParseComments)
@ -409,6 +417,11 @@ func atomicCounterStmt(f *File, counter string) string {
return fmt.Sprintf("%s.AddUint32(&%s, 1)", atomicPackageName, counter)
}
// watchCounterStmt returns the expression: __count[23]++;UploadCoverChangeEvent(blockname, pos[:], index)
func watchCounterStmt(f *File, counter string) string {
return fmt.Sprintf("%s++; UploadCoverChangeEvent_%v(%s.BlockName, %s.Pos[:], %v)", counter, f.random, f.varVar, f.varVar, len(f.blocks))
}
// QINIU
// newCounter creates a new counter expression of the appropriate form.
func (f *File) newCounter(start, end token.Pos, numStmt int) string {
@ -689,8 +702,12 @@ func (f *File) addVariables(w io.Writer) {
fmt.Fprintf(w, "\tCount [%d]uint32\n", len(f.blocks))
fmt.Fprintf(w, "\tPos [3 * %d]uint32\n", len(f.blocks))
fmt.Fprintf(w, "\tNumStmt [%d]uint16\n", len(f.blocks))
fmt.Fprintf(w, "\tBlockName string\n") // QINIU
fmt.Fprintf(w, "} {\n")
// 写入 BlockName 初始化
fmt.Fprintf(w, "\tBlockName: \"%v\",\n", f.importpathFileName)
// Initialize the position array field.
fmt.Fprintf(w, "\tPos: [3 * %d]uint32{\n", len(f.blocks))

BIN
pkg/cover/internal/websocket/websocket.tar Normal file
View File

Binary file not shown.

68
pkg/cover/internal/websocket/wsdep.go Normal file
View File

@ -0,0 +1,68 @@
package websocket
import (
"archive/tar"
"bytes"
"embed"
"io"
"os"
"path/filepath"
"github.com/qiniu/goc/v2/pkg/config"
"github.com/qiniu/goc/v2/pkg/log"
)
//go:embed websocket.tar
var depTarFile embed.FS
// AddCustomWebsocketDep injects custom gorrila/websocket library into the temporary directory
//
// 从 embed 文件系统中解压 websocket.tar 文件，并依次写入临时工程中，作为一个单独的包存在。
// gorrila/websocket 是一个无第三方依赖的库，因此其位置可以随处移动，而不影响自身的编译。
func AddCustomWebsocketDep() {
data, err := depTarFile.ReadFile("websocket.tar")
if err != nil {
log.Fatalf("cannot find the websocket.tar in the embed file: %v", err)
}
buf := bytes.NewBuffer(data)
tr := tar.NewReader(buf)
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
log.Fatalf("cannot untar the websocket.tar: %v", err)
}
customWebsocketPath := config.GocConfig.GlobalCoverVarImportPathDir
fpath := filepath.Join(customWebsocketPath, hdr.Name)
if hdr.FileInfo().IsDir() {
// 处理目录
err := os.MkdirAll(fpath, hdr.FileInfo().Mode())
if err != nil {
log.Fatalf("fail to untar the websocket.tar: %v", err)
}
} else {
// 处理文件
fdir := filepath.Dir(fpath)
err := os.MkdirAll(fdir, hdr.FileInfo().Mode())
if err != nil {
log.Fatalf("fail to untar the websocket.tar: %v", err)
}
f, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, hdr.FileInfo().Mode())
if err != nil {
log.Fatalf("fail to untar the websocket.tar: %v", err)
}
defer f.Close()
_, err = io.Copy(f, tr)
if err != nil {
log.Fatalf("fail to untar the websocket.tar: %v", err)
}
}
}
}

394
pkg/cover/server.go
View File

@ -1,394 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cover
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
"os"
"regexp"
"github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus"
"golang.org/x/tools/cover"
"k8s.io/test-infra/gopherage/pkg/cov"
)
// LogFile a file to save log.
const LogFile = "goc.log"
type server struct {
PersistenceFile string
Store Store
}
// NewFileBasedServer new a file based server with persistenceFile
func NewFileBasedServer(persistenceFile string) (*server, error) {
store, err := NewFileStore(persistenceFile)
if err != nil {
return nil, err
}
return &server{
PersistenceFile: persistenceFile,
Store: store,
}, nil
}
// NewMemoryBasedServer new a memory based server without persistenceFile
func NewMemoryBasedServer() *server {
return &server{
Store: NewMemoryStore(),
}
}
// Run starts coverage host center
func (s *server) Run(port string) {
f, err := os.Create(LogFile)
if err != nil {
log.Fatalf("failed to create log file %s, err: %v", LogFile, err)
}
// both log to stdout and file by default
mw := io.MultiWriter(f, os.Stdout)
r := s.Route(mw)
log.Fatal(r.Run(port))
}
// Router init goc server engine
func (s *server) Route(w io.Writer) *gin.Engine {
if w != nil {
gin.DefaultWriter = w
}
r := gin.Default()
// api to show the registered services
r.StaticFile("static", "./"+s.PersistenceFile)
v1 := r.Group("/v1")
{
v1.POST("/cover/register", s.registerService)
v1.GET("/cover/profile", s.profile)
v1.POST("/cover/profile", s.profile)
v1.POST("/cover/clear", s.clear)
v1.POST("/cover/init", s.initSystem)
v1.GET("/cover/list", s.listServices)
v1.POST("/cover/remove", s.removeServices)
}
return r
}
// ServiceUnderTest is a entry under being tested
type ServiceUnderTest struct {
Name string `form:"name" json:"name" binding:"required"`
Address string `form:"address" json:"address" binding:"required"`
}
// ProfileParam is param of profile API
type ProfileParam struct {
Force bool `form:"force" json:"force"`
Service []string `form:"service" json:"service"`
Address []string `form:"address" json:"address"`
CoverFilePatterns []string `form:"coverfile" json:"coverfile"`
SkipFilePatterns []string `form:"skipfile" json:"skipfile"`
}
//listServices list all the registered services
func (s *server) listServices(c *gin.Context) {
services := s.Store.GetAll()
c.JSON(http.StatusOK, services)
}
func (s *server) registerService(c *gin.Context) {
var service ServiceUnderTest
if err := c.ShouldBind(&service); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
u, err := url.Parse(service.Address)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
host, port, err := net.SplitHostPort(u.Host)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
realIP := c.ClientIP()
// only for IPV4
// refer: https://github.com/qiniu/goc/issues/177
if net.ParseIP(realIP).To4() != nil && host != realIP {
log.Printf("the registered host %s of service %s is different with the real one %s, here we choose the real one", service.Name, host, realIP)
service.Address = fmt.Sprintf("http://%s:%s", realIP, port)
}
address := s.Store.Get(service.Name)
if !contains(address, service.Address) {
if err := s.Store.Add(service); err != nil && err != ErrServiceAlreadyRegistered {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
}
c.JSON(http.StatusOK, gin.H{"result": "success"})
return
}
// profile API examples:
// POST /v1/cover/profile
// { "force": "true", "service":["a","b"], "address":["c","d"],"coverfile":["e","f"] }
func (s *server) profile(c *gin.Context) {
var body ProfileParam
if err := c.ShouldBind(&body); err != nil {
c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
return
}
allInfos := s.Store.GetAll()
filterAddrList, err := filterAddrs(body.Service, body.Address, body.Force, allInfos)
if err != nil {
c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
return
}
var mergedProfiles = make([][]*cover.Profile, 0)
for _, addr := range filterAddrList {
pp, err := NewWorker(addr).Profile(ProfileParam{})
if err != nil {
if body.Force {
log.Warnf("get profile from [%s] failed, error: %s", addr, err.Error())
continue
}
c.JSON(http.StatusExpectationFailed, gin.H{"error": fmt.Sprintf("failed to get profile from %s, error %s", addr, err.Error())})
return
}
profile, err := convertProfile(pp)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
mergedProfiles = append(mergedProfiles, profile)
}
if len(mergedProfiles) == 0 {
c.JSON(http.StatusExpectationFailed, gin.H{"error": "no profiles"})
return
}
merged, err := cov.MergeMultipleProfiles(mergedProfiles)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
if len(body.CoverFilePatterns) > 0 {
merged, err = filterProfile(body.CoverFilePatterns, merged)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("failed to filter profile based on the patterns: %v, error: %v", body.CoverFilePatterns, err)})
return
}
}
if len(body.SkipFilePatterns) > 0 {
merged, err = skipProfile(body.SkipFilePatterns, merged)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("failed to skip profile based on the patterns: %v, error: %v", body.SkipFilePatterns, err)})
return
}
}
if err := cov.DumpProfile(merged, c.Writer); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
}
// filterProfile filters profiles of the packages matching the coverFile pattern
func filterProfile(coverFile []string, profiles []*cover.Profile) ([]*cover.Profile, error) {
var out = make([]*cover.Profile, 0)
for _, profile := range profiles {
for _, pattern := range coverFile {
matched, err := regexp.MatchString(pattern, profile.FileName)
if err != nil {
return nil, fmt.Errorf("filterProfile failed with pattern %s for profile %s, err: %v", pattern, profile.FileName, err)
}
if matched {
out = append(out, profile)
break // no need to check again for the file
}
}
}
return out, nil
}
// skipProfile skips profiles of the packages matching the skipFile pattern
func skipProfile(skipFile []string, profiles []*cover.Profile) ([]*cover.Profile, error) {
var out = make([]*cover.Profile, 0)
for _, profile := range profiles {
var shouldSkip bool
for _, pattern := range skipFile {
matched, err := regexp.MatchString(pattern, profile.FileName)
if err != nil {
return nil, fmt.Errorf("filterProfile failed with pattern %s for profile %s, err: %v", pattern, profile.FileName, err)
}
if matched {
shouldSkip = true
break // no need to check again for the file
}
}
if !shouldSkip {
out = append(out, profile)
}
}
return out, nil
}
func (s *server) clear(c *gin.Context) {
var body ProfileParam
if err := c.ShouldBind(&body); err != nil {
c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
return
}
svrsUnderTest := s.Store.GetAll()
filterAddrList, err := filterAddrs(body.Service, body.Address, true, svrsUnderTest)
if err != nil {
c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
return
}
for _, addr := range filterAddrList {
pp, err := NewWorker(addr).Clear(ProfileParam{})
if err != nil {
c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
return
}
fmt.Fprintf(c.Writer, "Register service %s coverage counter %s", addr, string(pp))
}
}
func (s *server) initSystem(c *gin.Context) {
if err := s.Store.Init(); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, "")
}
func (s *server) removeServices(c *gin.Context) {
var body ProfileParam
if err := c.ShouldBind(&body); err != nil {
c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
return
}
svrsUnderTest := s.Store.GetAll()
filterAddrList, err := filterAddrs(body.Service, body.Address, true, svrsUnderTest)
if err != nil {
c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
return
}
for _, addr := range filterAddrList {
err := s.Store.Remove(addr)
if err != nil {
c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
return
}
fmt.Fprintf(c.Writer, "Register service %s removed from the center.", addr)
}
}
func convertProfile(p []byte) ([]*cover.Profile, error) {
// Annoyingly, ParseProfiles only accepts a filename, so we have to write the bytes to disk
// so it can read them back.
// We could probably also just give it /dev/stdin, but that'll break on Windows.
tf, err := ioutil.TempFile("", "")
if err != nil {
return nil, fmt.Errorf("failed to create temp file, err: %v", err)
}
defer tf.Close()
defer os.Remove(tf.Name())
if _, err := io.Copy(tf, bytes.NewReader(p)); err != nil {
return nil, fmt.Errorf("failed to copy data to temp file, err: %v", err)
}
return cover.ParseProfiles(tf.Name())
}
func contains(arr []string, str string) bool {
for _, element := range arr {
if str == element {
return true
}
}
return false
}
// filterAddrs filter address list by given service and address list
func filterAddrs(serviceList, addressList []string, force bool, allInfos map[string][]string) (filterAddrList []string, err error) {
addressAll := []string{}
for _, addr := range allInfos {
addressAll = append(addressAll, addr...)
}
if len(serviceList) != 0 && len(addressList) != 0 {
return nil, fmt.Errorf("use 'service' flag and 'address' flag at the same time may cause ambiguity, please use them separately")
}
// Add matched services to map
for _, name := range serviceList {
if addr, ok := allInfos[name]; ok {
filterAddrList = append(filterAddrList, addr...)
continue // jump to match the next service
}
if !force {
return nil, fmt.Errorf("service [%s] not found", name)
}
log.Warnf("service [%s] not found", name)
}
// Add matched addresses to map
for _, addr := range addressList {
if contains(addressAll, addr) {
filterAddrList = append(filterAddrList, addr)
continue
}
if !force {
return nil, fmt.Errorf("address [%s] not found", addr)
}
log.Warnf("address [%s] not found", addr)
}
if len(addressList) == 0 && len(serviceList) == 0 {
filterAddrList = addressAll
}
// Return all services when all param is nil
return filterAddrList, nil
}

527
pkg/cover/server_test.go
View File

@ -1,527 +0,0 @@
package cover
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"os"
"reflect"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"golang.org/x/tools/cover"
)
// MockStore is mock store mainly for unittest
type MockStore struct {
mock.Mock
}
func (m *MockStore) Add(s ServiceUnderTest) error {
args := m.Called(s)
return args.Error(0)
}
func (m *MockStore) Remove(a string) error {
args := m.Called(a)
return args.Error(0)
}
func (m *MockStore) Get(name string) []string {
args := m.Called(name)
return args.Get(0).([]string)
}
func (m *MockStore) GetAll() map[string][]string {
args := m.Called()
return args.Get(0).(map[string][]string)
}
func (m *MockStore) Init() error {
args := m.Called()
return args.Error(0)
}
func (m *MockStore) Set(services map[string][]string) error {
args := m.Called()
return args.Error(0)
}
func TestContains(t *testing.T) {
assert.Equal(t, contains([]string{"a", "b"}, "a"), true)
assert.Equal(t, contains([]string{"a", "b"}, "c"), false)
}
func TestFilterAddrs(t *testing.T) {
svrAll := map[string][]string{
"service1": {"http://127.0.0.1:7777", "http://127.0.0.1:8888"},
"service2": {"http://127.0.0.1:9999"},
}
addrAll := []string{}
for _, addr := range svrAll {
addrAll = append(addrAll, addr...)
}
items := []struct {
svrList []string
addrList []string
force bool
err string
addrRes []string
}{
{
svrList: []string{"service1"},
addrList: []string{"http://127.0.0.1:7777"},
err: "use 'service' flag and 'address' flag at the same time may cause ambiguity, please use them separately",
},
{
addrRes: addrAll,
},
{
svrList: []string{"service1", "unknown"},
err: "service [unknown] not found",
},
{
svrList: []string{"service1", "service2", "unknown"},
force: true,
addrRes: addrAll,
},
{
svrList: []string{"unknown"},
force: true,
},
{
addrList: []string{"http://127.0.0.1:7777", "http://127.0.0.2:7777"},
err: "address [http://127.0.0.2:7777] not found",
},
{
addrList: []string{"http://127.0.0.1:7777", "http://127.0.0.1:9999", "http://127.0.0.2:7777"},
force: true,
addrRes: []string{"http://127.0.0.1:7777", "http://127.0.0.1:9999"},
},
}
for _, item := range items {
addrs, err := filterAddrs(item.svrList, item.addrList, item.force, svrAll)
if err != nil {
assert.Equal(t, err.Error(), item.err)
} else {
if len(addrs) == 0 {
assert.Equal(t, addrs, item.addrRes)
}
for _, a := range addrs {
assert.Contains(t, item.addrRes, a)
}
}
}
}
func TestRegisterService(t *testing.T) {
server, err := NewFileBasedServer("_svrs_address.txt")
assert.NoError(t, err)
router := server.Route(os.Stdout)
// register with empty service struct
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/cover/register", nil)
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusBadRequest, w.Code)
// register with invalid service.Address
data := url.Values{}
data.Set("name", "aaa")
data.Set("address", "&%%")
w = httptest.NewRecorder()
req, _ = http.NewRequest("POST", "/v1/cover/register", strings.NewReader(data.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusBadRequest, w.Code)
assert.Contains(t, w.Body.String(), "invalid URL escape")
// register with host but no port
data = url.Values{}
data.Set("name", "aaa")
data.Set("address", "http://127.0.0.1")
w = httptest.NewRecorder()
req, _ = http.NewRequest("POST", "/v1/cover/register", strings.NewReader(data.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusBadRequest, w.Code)
assert.Contains(t, w.Body.String(), "missing port in address")
// register with store failure
expectedS := ServiceUnderTest{
Name: "foo",
Address: "http://:64444", // the real IP is empty in unittest, so server will get a empty one
}
testObj := new(MockStore)
testObj.On("Get", "foo").Return([]string{"http://127.0.0.1:66666"})
testObj.On("Add", expectedS).Return(fmt.Errorf("lala error"))
server.Store = testObj
w = httptest.NewRecorder()
data.Set("name", expectedS.Name)
data.Set("address", expectedS.Address)
req, _ = http.NewRequest("POST", "/v1/cover/register", strings.NewReader(data.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusInternalServerError, w.Code)
assert.Contains(t, w.Body.String(), "lala error")
}
func TestProfileService(t *testing.T) {
server, err := NewFileBasedServer("_svrs_address.txt")
assert.NoError(t, err)
router := server.Route(os.Stdout)
// get profile with invalid force parameter
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/v1/cover/profile?force=11", nil)
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusExpectationFailed, w.Code)
assert.Contains(t, w.Body.String(), "invalid syntax")
}
func TestClearService(t *testing.T) {
testObj := new(MockStore)
testObj.On("GetAll").Return(map[string][]string{"foo": {"http://127.0.0.1:66666"}})
server := &server{
Store: testObj,
}
router := server.Route(os.Stdout)
// clear profile with non-exist port
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/cover/clear", bytes.NewBuffer([]byte(`{}`)))
req.Header.Set("Content-Type", "application/json")
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusExpectationFailed, w.Code)
assert.Contains(t, w.Body.String(), "invalid port")
// clear profile with invalid service
w = httptest.NewRecorder()
req, _ = http.NewRequest("POST", "/v1/cover/clear", nil)
req.Header.Set("Content-Type", "application/json")
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusExpectationFailed, w.Code)
assert.Contains(t, w.Body.String(), "invalid request")
// clear profile with service and address set at at the same time
p := ProfileParam{
Service: []string{"goc"},
Address: []string{"http://127.0.0.1:3333"},
}
encoded, err := json.Marshal(p)
assert.NoError(t, err)
w = httptest.NewRecorder()
req, _ = http.NewRequest("POST", "/v1/cover/clear", bytes.NewBuffer(encoded))
req.Header.Set("Content-Type", "application/json")
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusExpectationFailed, w.Code)
assert.Contains(t, w.Body.String(), "use 'service' flag and 'address' flag at the same time may cause ambiguity, please use them separately")
}
func TestRemoveServices(t *testing.T) {
testObj := new(MockStore)
testObj.On("GetAll").Return(map[string][]string{"foo": {"test1", "test2"}})
testObj.On("Remove", "test1").Return(nil)
server := &server{
Store: testObj,
}
router := server.Route(os.Stdout)
// remove with invalid request
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/cover/remove", nil)
req.Header.Set("Content-Type", "application/json")
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusExpectationFailed, w.Code)
assert.Contains(t, w.Body.String(), "invalid request")
// remove service
p := ProfileParam{
Address: []string{"test1"},
}
encoded, err := json.Marshal(p)
assert.NoError(t, err)
w = httptest.NewRecorder()
req, _ = http.NewRequest("POST", "/v1/cover/remove", bytes.NewBuffer(encoded))
req.Header.Set("Content-Type", "application/json")
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusOK, w.Code)
assert.Contains(t, w.Body.String(), "Register service test1 removed from the center.")
// remove service with non-exist address
testObj.On("Remove", "test2").Return(fmt.Errorf("no service found"))
p = ProfileParam{
Address: []string{"test2"},
}
encoded, err = json.Marshal(p)
assert.NoError(t, err)
w = httptest.NewRecorder()
req, _ = http.NewRequest("POST", "/v1/cover/remove", bytes.NewBuffer(encoded))
req.Header.Set("Content-Type", "application/json")
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusExpectationFailed, w.Code)
assert.Contains(t, w.Body.String(), "no service found")
// clear profile with service and address set at at the same time
p = ProfileParam{
Service: []string{"goc"},
Address: []string{"http://127.0.0.1:3333"},
}
encoded, err = json.Marshal(p)
assert.NoError(t, err)
w = httptest.NewRecorder()
req, _ = http.NewRequest("POST", "/v1/cover/remove", bytes.NewBuffer(encoded))
req.Header.Set("Content-Type", "application/json")
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusExpectationFailed, w.Code)
assert.Contains(t, w.Body.String(), "use 'service' flag and 'address' flag at the same time may cause ambiguity, please use them separately")
}
func TestInitService(t *testing.T) {
testObj := new(MockStore)
testObj.On("Init").Return(fmt.Errorf("lala error"))
server := &server{
Store: testObj,
}
router := server.Route(os.Stdout)
// get profile with invalid force parameter
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/v1/cover/init", nil)
router.ServeHTTP(w, req)
assert.Equal(t, http.StatusInternalServerError, w.Code)
assert.Contains(t, w.Body.String(), "lala error")
}
func TestFilterProfile(t *testing.T) {
var tcs = []struct {
name string
pattern []string
input []*cover.Profile
output []*cover.Profile
expectErr bool
}{
{
name: "normal path",
pattern: []string{"some/fancy/gopath", "a/fancy/gopath"},
input: []*cover.Profile{
{
FileName: "some/fancy/gopath/a.go",
},
{
FileName: "some/fancy/gopath/b/a.go",
},
{
FileName: "a/fancy/gopath/a.go",
},
{
FileName: "b/fancy/gopath/a.go",
},
{
FileName: "b/a/fancy/gopath/a.go",
},
},
output: []*cover.Profile{
{
FileName: "some/fancy/gopath/a.go",
},
{
FileName: "some/fancy/gopath/b/a.go",
},
{
FileName: "a/fancy/gopath/a.go",
},
{
FileName: "b/a/fancy/gopath/a.go",
},
},
},
{
name: "with regular expression",
pattern: []string{"fancy/gopath/a.go$", "^b/a/"},
input: []*cover.Profile{
{
FileName: "some/fancy/gopath/a.go",
},
{
FileName: "some/fancy/gopath/b/a.go",
},
{
FileName: "a/fancy/gopath/a.go",
},
{
FileName: "b/fancy/gopath/c/a.go",
},
{
FileName: "b/a/fancy/gopath/a.go",
},
},
output: []*cover.Profile{
{
FileName: "some/fancy/gopath/a.go",
},
{
FileName: "a/fancy/gopath/a.go",
},
{
FileName: "b/a/fancy/gopath/a.go",
},
},
},
{
name: "with invalid regular expression",
pattern: []string{"(?!a)"},
input: []*cover.Profile{
{
FileName: "some/fancy/gopath/a.go",
},
},
expectErr: true,
},
}
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
out, err := filterProfile(tc.pattern, tc.input)
if err != nil {
if !tc.expectErr {
t.Errorf("Unexpected error: %v", err)
}
return
}
if tc.expectErr {
t.Errorf("Expected an error, but got value %s", stringifyCoverProfile(out))
}
if !reflect.DeepEqual(out, tc.output) {
t.Errorf("Mismatched results. \nExpected: %s\nActual:%s", stringifyCoverProfile(tc.output), stringifyCoverProfile(out))
}
})
}
}
func TestSkipProfile(t *testing.T) {
var tcs = []struct {
name string
pattern []string
input []*cover.Profile
output []*cover.Profile
expectErr bool
}{
{
name: "normal path",
pattern: []string{"some/fancy/gopath", "a/fancy/gopath"},
input: []*cover.Profile{
{
FileName: "some/fancy/gopath/a.go",
},
{
FileName: "some/fancy/gopath/b/a.go",
},
{
FileName: "a/fancy/gopath/a.go",
},
{
FileName: "b/fancy/gopath/a.go",
},
{
FileName: "b/a/fancy/gopath/a.go",
},
},
output: []*cover.Profile{
{
FileName: "b/fancy/gopath/a.go",
},
},
},
{
name: "with regular expression",
pattern: []string{"fancy/gopath/a.go$", "^b/a/"},
input: []*cover.Profile{
{
FileName: "some/fancy/gopath/a.go",
},
{
FileName: "some/fancy/gopath/b/a.go",
},
{
FileName: "a/fancy/gopath/a.go",
},
{
FileName: "b/fancy/gopath/c/a.go",
},
{
FileName: "b/a/fancy/gopath/a.go",
},
},
output: []*cover.Profile{
{
FileName: "some/fancy/gopath/b/a.go",
},
{
FileName: "b/fancy/gopath/c/a.go",
},
},
},
{
name: "with invalid regular expression",
pattern: []string{"(?!a)"},
input: []*cover.Profile{
{
FileName: "some/fancy/gopath/a.go",
},
},
expectErr: true,
},
}
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
out, err := skipProfile(tc.pattern, tc.input)
if err != nil {
if !tc.expectErr {
t.Errorf("Unexpected error: %v", err)
}
return
}
if tc.expectErr {
t.Errorf("Expected an error, but got value %s", stringifyCoverProfile(out))
}
if !reflect.DeepEqual(out, tc.output) {
t.Errorf("Mismatched results. \nExpected: %s\nActual:%s", stringifyCoverProfile(tc.output), stringifyCoverProfile(out))
}
})
}
}
func stringifyCoverProfile(profiles []*cover.Profile) string {
res := make([]cover.Profile, 0, len(profiles))
for _, p := range profiles {
res = append(res, *p)
}
return fmt.Sprintf("%#v", res)
}

324
pkg/cover/store.go
View File

@ -1,324 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cover
import (
"bufio"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
log "github.com/sirupsen/logrus"
)
var ErrServiceAlreadyRegistered = errors.New("service already registered")
// Store persistents the registered service information
type Store interface {
// Add adds the given service to store
Add(s ServiceUnderTest) error
// Get returns the registered service information with the given service's name
Get(name string) []string
// Get returns all the registered service information as a map
GetAll() map[string][]string
// Init cleanup all the registered service information
Init() error
// Set stores the services information into internal state
Set(services map[string][]string) error
// Remove the service from the store by address
Remove(addr string) error
}
// fileStore holds the registered services into memory and persistent to a local file
type fileStore struct {
mu sync.RWMutex
persistentFile string
memoryStore Store
}
// NewFileStore creates a store using local file
func NewFileStore(persistenceFile string) (store Store, err error) {
path, err := filepath.Abs(persistenceFile)
if err != nil {
return nil, err
}
err = os.MkdirAll(filepath.Dir(path), os.ModePerm)
if err != nil {
return nil, err
}
l := &fileStore{
persistentFile: path,
memoryStore: NewMemoryStore(),
}
if err := l.load(); err != nil {
log.Fatalf("load failed, file: %s, err: %v", l.persistentFile, err)
}
return l, nil
}
// Add adds the given service to file Store
func (l *fileStore) Add(s ServiceUnderTest) error {
if err := l.memoryStore.Add(s); err != nil {
return err
}
// persistent to local store
l.mu.Lock()
defer l.mu.Unlock()
return l.appendToFile(s)
}
// Get returns the registered service information with the given name
func (l *fileStore) Get(name string) []string {
return l.memoryStore.Get(name)
}
// Get returns all the registered service information
func (l *fileStore) GetAll() map[string][]string {
return l.memoryStore.GetAll()
}
// Remove the service from the memory store and the file store
func (l *fileStore) Remove(addr string) error {
err := l.memoryStore.Remove(addr)
if err != nil {
return err
}
return l.Set(l.memoryStore.GetAll())
}
// Init cleanup all the registered service information
// and the local persistent file
func (l *fileStore) Init() error {
if err := l.memoryStore.Init(); err != nil {
return err
}
l.mu.Lock()
defer l.mu.Unlock()
if err := os.Remove(l.persistentFile); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to delete file %s, err: %v", l.persistentFile, err)
}
return nil
}
// load all registered service from file to memory
func (l *fileStore) load() error {
var svrsMap = make(map[string][]string, 0)
f, err := os.Open(l.persistentFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("failed to open file, path: %s, err: %v", l.persistentFile, err)
}
defer f.Close()
ns := bufio.NewScanner(f)
for ns.Scan() {
line := ns.Text()
ss := strings.FieldsFunc(line, split)
// TODO: use regex
if len(ss) == 2 {
if urls, ok := svrsMap[ss[0]]; ok {
urls = append(urls, ss[1])
svrsMap[ss[0]] = urls
} else {
svrsMap[ss[0]] = []string{ss[1]}
}
}
}
if err := ns.Err(); err != nil {
return fmt.Errorf("read file failed, file: %s, err: %v", l.persistentFile, err)
}
// set information to memory
l.memoryStore.Set(svrsMap)
return nil
}
func (l *fileStore) Set(services map[string][]string) error {
l.mu.Lock()
defer l.mu.Unlock()
// no error will return from memorystore.set
err := l.memoryStore.Set(services)
if err != nil {
return err
}
f, err := os.OpenFile(l.persistentFile, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return err
}
s := ""
for name, addrs := range services {
for _, addr := range addrs {
s += fmt.Sprintf("%s&%s\n", name, addr)
}
}
_, err = f.WriteString(s)
if err != nil {
return err
}
return f.Sync()
}
func (l *fileStore) appendToFile(s ServiceUnderTest) error {
f, err := os.OpenFile(l.persistentFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return err
}
defer f.Close()
_, err = f.WriteString(format(s) + "\n")
if err != nil {
return err
}
f.Sync()
return nil
}
func format(s ServiceUnderTest) string {
return fmt.Sprintf("%s&%s", s.Name, s.Address)
}
func split(r rune) bool {
return r == '&'
}
// memoryStore holds the registered services only into memory
type memoryStore struct {
mu sync.RWMutex
servicesMap map[string][]string
}
// NewMemoryStore creates a memory store
func NewMemoryStore() Store {
return &memoryStore{
servicesMap: make(map[string][]string, 0),
}
}
// Add adds the given service to MemoryStore
func (l *memoryStore) Add(s ServiceUnderTest) error {
l.mu.Lock()
defer l.mu.Unlock()
// load to memory
if addrs, ok := l.servicesMap[s.Name]; ok {
for _, addr := range addrs {
if addr == s.Address {
log.Printf("service registered already, name: %s, address: %s", s.Name, s.Address)
return ErrServiceAlreadyRegistered
}
}
addrs = append(addrs, s.Address)
l.servicesMap[s.Name] = addrs
} else {
l.servicesMap[s.Name] = []string{s.Address}
}
return nil
}
// Get returns the registered service information with the given name
func (l *memoryStore) Get(name string) []string {
l.mu.RLock()
defer l.mu.RUnlock()
return l.servicesMap[name]
}
// Get returns all the registered service information
func (l *memoryStore) GetAll() map[string][]string {
res := make(map[string][]string)
l.mu.RLock()
defer l.mu.RUnlock()
for k, v := range l.servicesMap {
res[k] = append(make([]string, 0, len(v)), v...)
}
return res
}
// Init cleanup all the registered service information
// and the local persistent file
func (l *memoryStore) Init() error {
l.mu.Lock()
defer l.mu.Unlock()
l.servicesMap = make(map[string][]string, 0)
return nil
}
func (l *memoryStore) Set(services map[string][]string) error {
l.mu.Lock()
defer l.mu.Unlock()
l.servicesMap = services
return nil
}
// Remove one service from the memory store
// if service is not fount, return "no service found" error
func (l *memoryStore) Remove(removeAddr string) error {
l.mu.Lock()
defer l.mu.Unlock()
flag := false
for name, addrs := range l.servicesMap {
newAddrs := make([]string, 0)
for _, addr := range addrs {
if removeAddr != addr {
newAddrs = append(newAddrs, addr)
} else {
flag = true
}
}
// if no services left, remove by name
if len(newAddrs) == 0 {
delete(l.servicesMap, name)
} else {
l.servicesMap[name] = newAddrs
}
}
if !flag {
return fmt.Errorf("no service found")
}
return nil
}

130
pkg/cover/store_test.go
View File

@ -1,130 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cover
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
)
func TestLocalStore(t *testing.T) {
localStore, err := NewFileStore("_svrs_address.txt")
assert.NoError(t, err)
var tc1 = ServiceUnderTest{
Name: "a",
Address: "http://127.0.0.1",
}
var tc2 = ServiceUnderTest{
Name: "b",
Address: "http://127.0.0.2",
}
var tc3 = ServiceUnderTest{
Name: "c",
Address: "http://127.0.0.3",
}
var tc4 = ServiceUnderTest{
Name: "a",
Address: "http://127.0.0.4",
}
assert.NoError(t, localStore.Add(tc1))
assert.Equal(t, localStore.Add(tc1), ErrServiceAlreadyRegistered)
assert.NoError(t, localStore.Add(tc2))
assert.NoError(t, localStore.Add(tc3))
assert.NoError(t, localStore.Add(tc4))
addrs := localStore.Get(tc1.Name)
if len(addrs) != 2 {
t.Error("unexpected result")
}
for _, addr := range addrs {
if addr != tc1.Address && addr != tc4.Address {
t.Error("get address failed")
}
}
if len(localStore.GetAll()) != 3 {
t.Error("local store check failed")
}
localStoreNew, err := NewFileStore("_svrs_address.txt")
assert.NoError(t, err)
assert.Equal(t, localStore.GetAll(), localStoreNew.GetAll())
localStore.Init()
if len(localStore.GetAll()) != 0 {
t.Error("local store init failed")
}
}
func TestMemoryStoreRemove(t *testing.T) {
store := NewMemoryStore()
s1 := ServiceUnderTest{Name: "test", Address: "http://127.0.0.1:8900"}
s2 := ServiceUnderTest{Name: "test2", Address: "http://127.0.0.1:8901"}
s3 := ServiceUnderTest{Name: "test2", Address: "http://127.0.0.1:8902"}
_ = store.Add(s1)
_ = store.Add(s2)
_ = store.Add(s3)
ss1 := store.Get("test")
assert.Equal(t, 1, len(ss1))
err := store.Remove("http://127.0.0.1:8900")
assert.NoError(t, err)
ss1 = store.Get("test")
assert.Nil(t, ss1)
ss2 := store.Get("test2")
assert.Equal(t, 2, len(ss2))
err = store.Remove("http://127.0.0.1:8901")
assert.NoError(t, err)
ss2 = store.Get("test2")
assert.Equal(t, 1, len(ss2))
err = store.Remove("http")
assert.Error(t, err, fmt.Errorf("no service found"))
}
func TestFileStoreRemove(t *testing.T) {
store, _ := NewFileStore("_svrs_address.txt")
_ = store.Init()
s1 := ServiceUnderTest{Name: "test", Address: "http://127.0.0.1:8900"}
s2 := ServiceUnderTest{Name: "test2", Address: "http://127.0.0.1:8901"}
s3 := ServiceUnderTest{Name: "test2", Address: "http://127.0.0.1:8902"}
_ = store.Add(s1)
_ = store.Add(s2)
_ = store.Add(s3)
ss1 := store.Get("test")
assert.Equal(t, 1, len(ss1))
err := store.Remove("http://127.0.0.1:8900")
assert.NoError(t, err)
ss1 = store.Get("test")
assert.Nil(t, ss1)
ss2 := store.Get("test2")
assert.Equal(t, 2, len(ss2))
err = store.Remove("http://127.0.0.1:8901")
assert.NoError(t, err)
ss2 = store.Get("test2")
assert.Equal(t, 1, len(ss2))
err = store.Remove("http")
assert.Error(t, err, fmt.Errorf("no service found"))
}

26
pkg/cover/template.go Normal file
View File

@ -0,0 +1,26 @@
package cover
import (
_ "embed"
"text/template"
)
var coverBridgeTmpl = template.Must(template.New("coverBridge").Parse(coverBridge))
const coverBridge = `
// Code generated by goc system. DO NOT EDIT.
package main
import _ "{{.CoverImportPath}}"
`
var coverMainTmpl = template.Must(template.New("coverMain").Parse(coverMain))
//go:embed agent.tpl
var coverMain string
var coverWatchTmpl = template.Must(template.New("coverWatch").Parse(coverWatch))
//go:embed agentwatch.tpl
var coverWatch string

120
pkg/flag/build_flags.go Normal file
View File

@ -0,0 +1,120 @@
package flag
import (
"flag"
"os"
"path/filepath"
"github.com/qiniu/goc/v2/pkg/config"
"github.com/qiniu/goc/v2/pkg/log"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
var buildUsage string = `Usage:
goc build [-o output] [build flags] [packages] [goc flags]
The [goc flags] can be placed in anywhere in the command line.
However, other flags' order are same with the go official command.
`
// BuildCmdArgsParse parse both go flags and goc flags, it rewrite go flags if
// necessary, and returns all non-flag arguments.
//
// 吞下 [packages] 之前所有的 flags.
func BuildCmdArgsParse(cmd *cobra.Command, args []string) []string {
// 首先解析 cobra 定义的 flag
allFlagSets := cmd.Flags()
// 因为 args 里面含有 go 的 flag，所以需要忽略解析 go flag 的错误
allFlagSets.Init("GOC", pflag.ContinueOnError)
allFlagSets.Parse(args)
// 重写 help
helpFlag := allFlagSets.Lookup("help")
if helpFlag.Changed {
printHelp(buildUsage, cmd)
}
// 删除 help flag
args = findAndDelHelpFlag(args)
// 必须手动调用
// 由于关闭了 cobra 的 flag parse，root PersistentPreRun 调用时，log.NewLogger 并没有拿到 debug 值
log.NewLogger()
// 删除 cobra 定义的 flag
allFlagSets.Visit(func(f *pflag.Flag) {
args = findAndDelGocFlag(args, f.Name, f.Value.String())
})
// 然后解析 go 的 flag
goFlagSets := flag.NewFlagSet("GO", flag.ContinueOnError)
addBuildFlags(goFlagSets)
addOutputFlags(goFlagSets)
err := goFlagSets.Parse(args)
if err != nil {
log.Fatalf("%v", err)
}
// 找出设置的 go flag
curWd, err := os.Getwd()
if err != nil {
log.Fatalf("fail to get current working directory: %v", err)
}
config.GocConfig.CurWd = curWd
flags := make([]string, 0)
goFlagSets.Visit(func(f *flag.Flag) {
// 将用户指定 -o 改成绝对目录
if f.Name == "o" {
outputDir := f.Value.String()
outputDir, err := filepath.Abs(outputDir)
if err != nil {
log.Fatalf("output flag is not valid: %v", err)
}
flags = append(flags, "-o", outputDir)
} else {
flags = append(flags, "-"+f.Name, f.Value.String())
}
})
config.GocConfig.Goflags = flags
return goFlagSets.Args()
}
func findAndDelGocFlag(a []string, x string, v string) []string {
new := make([]string, 0, len(a))
x = "--" + x
x_v := x + "=" + v
for i := 0; i < len(a); i++ {
if a[i] == "--debug" {
// debug 是 bool，就一个元素
continue
} else if a[i] == x {
// 有 goc flag 长这样 --mode watch
i++
continue
} else if a[i] == x_v {
// 有 goc flag 长这样 --mode=watch
continue
} else {
// 剩下的是 go flag
new = append(new, a[i])
}
}
return new
}
func findAndDelHelpFlag(a []string) []string {
new := make([]string, 0, len(a))
for _, v := range a {
if v == "--help" || v == "-h" {
continue
} else {
new = append(new, v)
}
}
return new
}

40
pkg/flag/flags.go Normal file
View File

@ -0,0 +1,40 @@
package flag
import (
"flag"
"github.com/qiniu/goc/v2/pkg/config"
)
func addBuildFlags(cmdSet *flag.FlagSet) {
cmdSet.BoolVar(&config.GoConfig.BuildA, "a", false, "")
cmdSet.BoolVar(&config.GoConfig.BuildN, "n", false, "")
cmdSet.IntVar(&config.GoConfig.BuildP, "p", 4, "")
cmdSet.BoolVar(&config.GoConfig.BuildV, "v", false, "")
cmdSet.BoolVar(&config.GoConfig.BuildX, "x", false, "")
cmdSet.StringVar(&config.GoConfig.BuildBuildmode, "buildmode", "default", "")
cmdSet.StringVar(&config.GoConfig.BuildMod, "mod", "", "")
cmdSet.StringVar(&config.GoConfig.Installsuffix, "installsuffix", "", "")
// 类型和 go 原生的不一样，这里纯粹是为了 parse 并传递给 go
cmdSet.StringVar(&config.GoConfig.BuildAsmflags, "asmflags", "", "")
cmdSet.StringVar(&config.GoConfig.BuildCompiler, "compiler", "", "")
cmdSet.StringVar(&config.GoConfig.BuildGcflags, "gcflags", "", "")
cmdSet.StringVar(&config.GoConfig.BuildGccgoflags, "gccgoflags", "", "")
// mod related
cmdSet.BoolVar(&config.GoConfig.ModCacheRW, "modcacherw", false, "")
cmdSet.StringVar(&config.GoConfig.ModFile, "modfile", "", "")
cmdSet.StringVar(&config.GoConfig.BuildLdflags, "ldflags", "", "")
cmdSet.BoolVar(&config.GoConfig.BuildLinkshared, "linkshared", false, "")
cmdSet.StringVar(&config.GoConfig.BuildPkgdir, "pkgdir", "", "")
cmdSet.BoolVar(&config.GoConfig.BuildRace, "race", false, "")
cmdSet.BoolVar(&config.GoConfig.BuildMSan, "msan", false, "")
cmdSet.StringVar(&config.GoConfig.BuildTags, "tags", "", "")
cmdSet.StringVar(&config.GoConfig.BuildToolexec, "toolexec", "", "")
cmdSet.BoolVar(&config.GoConfig.BuildTrimpath, "trimpath", false, "")
cmdSet.BoolVar(&config.GoConfig.BuildWork, "work", false, "")
}
func addOutputFlags(cmdSet *flag.FlagSet) {
cmdSet.StringVar(&config.GoConfig.BuildO, "o", "", "")
}

20
pkg/flag/help.go Normal file
View File

@ -0,0 +1,20 @@
package flag
import (
"fmt"
"github.com/spf13/cobra"
)
func printHelp(usage string, cmd *cobra.Command) {
fmt.Println(usage)
flags := cmd.LocalFlags()
globalFlags := cmd.Parent().PersistentFlags()
fmt.Println("Flags:")
fmt.Println(flags.FlagUsages())
fmt.Println("Global Flags:")
fmt.Println(globalFlags.FlagUsages())
}

193
pkg/flag/packages.go Normal file
View File

@ -0,0 +1,193 @@
package flag
import (
"fmt"
"os"
"path/filepath"
"strings"
"github.com/qiniu/goc/v2/pkg/config"
"github.com/qiniu/goc/v2/pkg/log"
)
// GetPackagesDir parse [pacakges] part of args, it will fatal if error encountered
//
// 函数获取 1： [packages] 所在的目录位置，供后续插桩使用。
//
// 函数获取 2： 如果参数是 *.go，第一个 .go 文件的文件名。go build 中，二进制名字既可能是目录名也可能是文件名，和参数类型有关。
//
// 如果 [packages] 非法（即不符合 go 原生的定义），则返回对应错误
// 这里只考虑 go mod 的方式
func GetPackagesDir(patterns []string) {
packages := make([]string, 0)
for _, p := range patterns {
// patterns 只支持两种格式
// 1. 要么是直接指向某些 .go 文件的相对/绝对路径
if strings.HasSuffix(p, ".go") {
if fi, err := os.Stat(p); err == nil && !fi.IsDir() {
// check if valid
if err := goFilesPackage(patterns); err != nil {
log.Fatalf("%v", err)
}
// 获取相对于 current working directory 对路径
for _, p := range patterns {
if filepath.IsAbs(p) {
relPath, err := filepath.Rel(config.GocConfig.CurWd, p)
if err != nil {
log.Fatalf("fail to get [packages] relative path from current working directory: %v", err)
}
packages = append(packages, relPath)
} else {
packages = append(packages, p)
}
}
// fix: go build ./xx/main.go 需要转换为
// go build ./xx/main.go ./xx/goc-cover-agent-apis-auto-generated-11111-22222-bridge.go
dir := filepath.Dir(packages[0])
packages = append(packages, filepath.Join(dir, "goc-cover-agent-apis-auto-generated-11111-22222-bridge.go"))
config.GocConfig.Packages = packages
return
}
}
}
// 2. 要么是 import path
config.GocConfig.Packages = patterns
}
// goFilesPackage 对一组 go 文件解析，判断是否合法
// go 本身还判断语法上是否是同一个 package，goc 这里不做解析
// 1. 都是 *.go 文件？
// 2. *.go 文件都在同一个目录？
// 3. *.go 文件存在？
func goFilesPackage(gofiles []string) error {
// 1. 必须都是 *.go 结尾
for _, f := range gofiles {
if !strings.HasSuffix(f, ".go") {
return fmt.Errorf("named files must be .go files: %s", f)
}
}
var dir string
for _, file := range gofiles {
// 3. 文件都存在？
fi, err := os.Stat(file)
if err != nil {
return err
}
// 2.1 有可能以 *.go 结尾的目录
if fi.IsDir() {
return fmt.Errorf("%s is a directory, should be a Go file", file)
}
// 2.2 所有 *.go 必须在同一个目录内
dir1, _ := filepath.Split(file)
if dir1 == "" {
dir1 = "./"
}
if dir == "" {
dir = dir1
} else if dir != dir1 {
return fmt.Errorf("named files must all be in one directory: have %s and %s", dir, dir1)
}
}
return nil
}
// getDirFromImportPaths return the import path's real abs directory
//
// 该函数接收到的只有 dir 或 import path，file 在上一步已被排除
// 只考虑 go modules 的情况
func getDirFromImportPaths(patterns []string) (string, error) {
// no import path, pattern = current wd
if len(patterns) == 0 {
wd, err := os.Getwd()
if err != nil {
return "", fmt.Errorf("fail to parse import path: %w", err)
}
return wd, nil
}
// 为了简化插桩的逻辑，goc 对 import path 要求必须都在同一个目录
// 所以干脆只允许一个 pattern 得了 -_-
// 对于 goc build/run 来说本身就是只能在一个目录内
// 对于 goc install 来讲，这个行为就和 go install 不同，不过多 import path 较少见 >_<，先忽略
if len(patterns) > 1 {
return "", fmt.Errorf("goc only support one import path now")
}
pattern := patterns[0]
switch {
// case isLocalImport(pattern) || filepath.IsAbs(pattern):
// dir1, err := filepath.Abs(pattern)
// if err != nil {
// return "", fmt.Errorf("error (%w) get directory from the import path: %v", err, pattern)
// }
// if _, err := os.Stat(dir1); err != nil {
// return "", fmt.Errorf("error (%w) get directory from the import path: %v", err, pattern)
// }
// return dir1, nil
case strings.Contains(pattern, "..."):
i := strings.Index(pattern, "...")
dir, _ := filepath.Split(pattern[:i])
dir, _ = filepath.Abs(dir)
if _, err := os.Stat(dir); err != nil {
return "", fmt.Errorf("error (%w) get directory from the import path: %v", err, pattern)
}
return dir, nil
case strings.IndexByte(pattern, '@') > 0:
return "", fmt.Errorf("import path with @ version query is not supported in goc")
case isMetaPackage(pattern):
return "", fmt.Errorf("`std`, `cmd`, `all` import path is not supported by goc")
default: // 到这一步认为 pattern 是相对路径或者绝对路径
dir1, err := filepath.Abs(pattern)
if err != nil {
return "", fmt.Errorf("error (%w) get directory from the import path: %v", err, pattern)
}
if _, err := os.Stat(dir1); err != nil {
return "", fmt.Errorf("error (%w) get directory from the import path: %v", err, pattern)
}
return dir1, nil
}
}
// isLocalImport reports whether the import path is
// a local import path, like ".", "..", "./foo", or "../foo"
func isLocalImport(path string) bool {
return path == "." || path == ".." ||
strings.HasPrefix(path, "./") || strings.HasPrefix(path, "../")
}
// isMetaPackage checks if the name is a reserved package name
func isMetaPackage(name string) bool {
return name == "std" || name == "cmd" || name == "all"
}
// find direct path of current project which contains go.mod
func findModuleRoot(dir string) string {
dir = filepath.Clean(dir)
// look for enclosing go.mod
for {
if fi, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil && !fi.IsDir() {
return dir
}
d := filepath.Dir(dir)
if d == dir {
break
}
dir = d
}
return ""
}

203
pkg/github/github.go
View File

@ -1,203 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package github
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"os"
"strconv"
"strings"
"github.com/google/go-github/github"
"github.com/hashicorp/go-retryablehttp"
"github.com/olekukonko/tablewriter"
"github.com/sirupsen/logrus"
"golang.org/x/oauth2"
"github.com/qiniu/goc/pkg/cover"
)
// CommentsPrefix is the prefix when commenting on Github Pull Requests
// It is also the flag when checking whether the target comment exists or not to avoid duplicate
const CommentsPrefix = "The following is the coverage report on the affected files."
// PrComment is the interface of the entry which is able to comment on Github Pull Requests
type PrComment interface {
CreateGithubComment(commentPrefix string, diffCovList cover.DeltaCovList) (err error)
PostComment(content, commentPrefix string) error
EraseHistoryComment(commentPrefix string) error
GetPrChangedFiles() (files []string, err error)
GetCommentFlag() string
}
// GitPrComment is the entry which is able to comment on Github Pull Requests
type GitPrComment struct {
RobotUserName string
RepoOwner string
RepoName string
CommentFlag string
PrNumber int
Ctx context.Context
opt *github.ListOptions
GithubClient *github.Client
}
// NewPrClient creates an Client which be able to comment on Github Pull Request
func NewPrClient(githubTokenPath, repoOwner, repoName, prNumStr, botUserName, commentFlag string) *GitPrComment {
var client *github.Client
// performs automatic retries when connection error occurs or a 500-range response code received (except 501)
retryClient := retryablehttp.NewClient()
ctx := context.WithValue(context.Background(), oauth2.HTTPClient, retryClient.StandardClient())
prNum, err := strconv.Atoi(prNumStr)
if err != nil {
logrus.WithError(err).Fatalf("Failed to convert prNumStr(=%v) to int.\n", prNumStr)
}
token, err := ioutil.ReadFile(githubTokenPath)
if err != nil {
logrus.WithError(err).Fatalf("Failed to get github token.\n")
}
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: strings.TrimSpace(string(token))},
)
tc := oauth2.NewClient(ctx, ts)
client = github.NewClient(tc)
return &GitPrComment{
RobotUserName: botUserName,
RepoOwner: repoOwner,
RepoName: repoName,
PrNumber: prNum,
CommentFlag: commentFlag,
Ctx: ctx,
opt: &github.ListOptions{Page: 1},
GithubClient: client,
}
}
// CreateGithubComment post github comment of diff coverage
func (c *GitPrComment) CreateGithubComment(commentPrefix string, diffCovList cover.DeltaCovList) (err error) {
if len(diffCovList) == 0 {
logrus.Printf("Detect 0 files coverage diff, will not comment to github.")
return nil
}
content := GenCommentContent(commentPrefix, diffCovList)
err = c.PostComment(content, commentPrefix)
if err != nil {
logrus.WithError(err).Fatalf("Post comment to github failed.")
}
return
}
// PostComment post comment on github. It erased the old one if existed to avoid duplicate
func (c *GitPrComment) PostComment(content, commentPrefix string) error {
//step1: erase history similar comment to avoid too many comment for same job
err := c.EraseHistoryComment(commentPrefix)
if err != nil {
return err
}
//step2: post comment with new result
comment := &github.IssueComment{
Body: &content,
}
_, _, err = c.GithubClient.Issues.CreateComment(c.Ctx, c.RepoOwner, c.RepoName, c.PrNumber, comment)
if err != nil {
return err
}
return nil
}
// EraseHistoryComment erase history similar comment before post again
func (c *GitPrComment) EraseHistoryComment(commentPrefix string) error {
comments, _, err := c.GithubClient.Issues.ListComments(c.Ctx, c.RepoOwner, c.RepoName, c.PrNumber, nil)
if err != nil {
logrus.Errorf("list PR comments failed.")
return err
}
logrus.Infof("the count of history comments by %s is: %v", c.RobotUserName, len(comments))
for _, cm := range comments {
if *cm.GetUser().Login == c.RobotUserName && strings.HasPrefix(cm.GetBody(), commentPrefix) {
_, err = c.GithubClient.Issues.DeleteComment(c.Ctx, c.RepoOwner, c.RepoName, *cm.ID)
if err != nil {
logrus.Errorf("delete PR comments %d failed.", *cm.ID)
return err
}
}
}
return nil
}
// GetPrChangedFiles get github pull request changes file list
func (c *GitPrComment) GetPrChangedFiles() (files []string, err error) {
var commitFiles []*github.CommitFile
for {
f, resp, err := c.GithubClient.PullRequests.ListFiles(c.Ctx, c.RepoOwner, c.RepoName, c.PrNumber, c.opt)
if err != nil {
logrus.Errorf("Get PR changed file failed. repoOwner is: %s, repoName is: %s, prNum is: %d", c.RepoOwner, c.RepoName, c.PrNumber)
return nil, err
}
commitFiles = append(commitFiles, f...)
if resp.NextPage == 0 {
break
}
c.opt.Page = resp.NextPage
}
logrus.Infof("get %d PR changed files:", len(commitFiles))
for _, file := range commitFiles {
files = append(files, *file.Filename)
logrus.Infof("%s", *file.Filename)
}
return
}
// GetCommentFlag get CommentFlag from the GitPrComment
func (c *GitPrComment) GetCommentFlag() string {
return c.CommentFlag
}
// GenCommentContent generate github comment content based on diff coverage and commentFlag
func GenCommentContent(commentPrefix string, delta cover.DeltaCovList) string {
var buf bytes.Buffer
table := tablewriter.NewWriter(&buf)
table.SetHeader([]string{"File", "Base Coverage", "New Coverage", "Delta"})
table.SetAutoFormatHeaders(false)
table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false})
table.SetCenterSeparator("|")
table.SetColumnAlignment([]int{tablewriter.ALIGN_LEFT, tablewriter.ALIGN_CENTER, tablewriter.ALIGN_CENTER, tablewriter.ALIGN_CENTER})
for _, d := range delta {
table.Append([]string{fmt.Sprintf("[%s](%s)", d.FileName, d.LineCovLink), d.BasePer, d.NewPer, d.DeltaPer})
}
table.Render()
content := []string{
commentPrefix,
fmt.Sprintf("Say `/test %s` to re-run this coverage report", os.Getenv("JOB_NAME")),
buf.String(),
}
return strings.Join(content, "\n")
}

176
pkg/github/github_test.go
View File

@ -1,176 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package github
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"os"
"testing"
"github.com/google/go-github/github"
"github.com/julienschmidt/httprouter"
"github.com/stretchr/testify/assert"
"golang.org/x/net/context"
"github.com/qiniu/goc/pkg/cover"
)
const (
// baseURLPath is a non-empty Client.BaseURL path to use during tests,
// to ensure relative URLs are used for all endpoints. See issue #752.
baseURLPath = "/api-v3"
)
// setup sets up a test HTTP server along with a github.Client that is
// configured to talk to that test server. Tests should register handlers on
// mux which provide mock responses for the API method being tested.
func setup() (client *github.Client, router *httprouter.Router, serverURL string, teardown func()) {
// router is the HTTP request multiplexer used with the test server.
router = httprouter.New()
// We want to ensure that tests catch mistakes where the endpoint URL is
// specified as absolute rather than relative. It only makes a difference
// when there's a non-empty base URL path. So, use that. See issue #752.
apiHandler := http.NewServeMux()
apiHandler.Handle(baseURLPath+"/", http.StripPrefix(baseURLPath, router))
apiHandler.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
fmt.Fprintln(os.Stderr, "FAIL: Client.BaseURL path prefix is not preserved in the request URL:")
fmt.Fprintln(os.Stderr)
fmt.Fprintln(os.Stderr, "\t"+req.URL.String())
fmt.Fprintln(os.Stderr)
fmt.Fprintln(os.Stderr, "\tDid you accidentally use an absolute endpoint URL rather than relative?")
fmt.Fprintln(os.Stderr, "\tSee https://github.com/google/go-github/issues/752 for information.")
http.Error(w, "Client.BaseURL path prefix is not preserved in the request URL.", http.StatusInternalServerError)
})
// server is a test HTTP server used to provide mock API responses.
server := httptest.NewServer(apiHandler)
// client is the GitHub client being tested and is
// configured to use test server.
client = github.NewClient(nil)
url, _ := url.Parse(server.URL + baseURLPath + "/")
client.BaseURL = url
client.UploadURL = url
return client, router, server.URL, server.Close
}
func TestNewPrClient(t *testing.T) {
items := []struct {
token string
repoOwner string
repoName string
prNumStr string
botUserName string
commentFlag string
expectPrNum int
}{
{token: "github_test.go", repoOwner: "qiniu", repoName: "goc", prNumStr: "1", botUserName: "qiniu-bot", commentFlag: "test", expectPrNum: 1},
}
for _, tc := range items {
prClient := NewPrClient(tc.token, tc.repoOwner, tc.repoName, tc.prNumStr, tc.botUserName, tc.commentFlag)
assert.Equal(t, tc.expectPrNum, prClient.PrNumber)
}
}
func TestCreateGithubComment(t *testing.T) {
client, router, _, teardown := setup()
defer teardown()
var coverList = cover.DeltaCovList{{FileName: "fake-coverage", BasePer: "50.0%", NewPer: "75.0%", DeltaPer: "25.0%"}}
expectContent := GenCommentContent("", coverList)
comment := &github.IssueComment{
Body: &expectContent,
}
// create comment: https://developer.github.com/v3/issues/comments/#create-a-comment
router.HandlerFunc("POST", "/repos/qiniu/goc/issues/1/comments", func(w http.ResponseWriter, r *http.Request) {
v := new(github.IssueComment)
json.NewDecoder(r.Body).Decode(v)
assert.Equal(t, v, comment)
fmt.Fprint(w, `{"id":1}`)
})
// list comment: https://developer.github.com/v3/issues/comments/#list-comments-on-an-issue
router.HandlerFunc("GET", "/repos/qiniu/goc/issues/1/comments", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, `[{"id":1,"user": {"login": "qiniu-bot"}}]`)
})
// delete comment: https://developer.github.com/v3/issues/comments/#edit-a-comment
router.HandlerFunc("DELETE", "/repos/qiniu/goc/issues/comments/1", func(w http.ResponseWriter, r *http.Request) {
})
p := GitPrComment{
RobotUserName: "qiniu-bot",
RepoOwner: "qiniu",
RepoName: "goc",
CommentFlag: "",
PrNumber: 1,
Ctx: context.Background(),
opt: nil,
GithubClient: client,
}
p.CreateGithubComment("", coverList)
}
func TestCreateGithubCommentError(t *testing.T) {
p := &GitPrComment{}
err := p.CreateGithubComment("", cover.DeltaCovList{})
assert.NoError(t, err)
}
func TestGetPrChangedFiles(t *testing.T) {
client, router, _, teardown := setup()
defer teardown()
var expectFiles = []string{"src/qiniu.com/kodo/s3apiv2/bucket/bucket.go"}
// list files API: https://developer.github.com/v3/pulls/#list-pull-requests-files
router.HandlerFunc("GET", "/repos/qiniu/goc/pulls/1/files", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, `[{"filename":"src/qiniu.com/kodo/s3apiv2/bucket/bucket.go"}]`)
})
p := GitPrComment{
RobotUserName: "qiniu-bot",
RepoOwner: "qiniu",
RepoName: "goc",
CommentFlag: "",
PrNumber: 1,
Ctx: context.Background(),
opt: nil,
GithubClient: client,
}
changedFiles, err := p.GetPrChangedFiles()
assert.Equal(t, err, nil)
assert.Equal(t, changedFiles, expectFiles)
}
func TestGetCommentFlag(t *testing.T) {
p := GitPrComment{
CommentFlag: "flag",
}
flag := p.GetCommentFlag()
assert.Equal(t, flag, p.CommentFlag)
}

53
pkg/log/ci_logger.go Normal file
View File

@ -0,0 +1,53 @@
package log
import "go.uber.org/zap"
type ciLogger struct {
logger *zap.Logger
}
func newCiLogger() *ciLogger {
logger, _ := zap.NewDevelopment()
// fix: increases the number of caller from always reporting the wrapper code as caller
logger = logger.WithOptions(zap.AddCallerSkip(2))
zap.ReplaceGlobals(logger)
return &ciLogger{
logger: logger,
}
}
func (c *ciLogger) StartWait(message string) {
}
func (c *ciLogger) StopWait() {
}
func (c *ciLogger) Sync() {
c.logger.Sync()
}
func (c *ciLogger) Debugf(format string, args ...interface{}) {
zap.S().Debugf(format, args...)
}
func (c *ciLogger) Donef(format string, args ...interface{}) {
zap.S().Infof(format, args...)
}
func (c *ciLogger) Infof(format string, args ...interface{}) {
zap.S().Infof(format, args...)
}
func (c *ciLogger) Errorf(format string, args ...interface{}) {
zap.S().Errorf(format, args...)
}
func (c *ciLogger) Warnf(format string, args ...interface{}) {
zap.S().Warnf(format, args...)
}
func (c *ciLogger) Fatalf(format string, args ...interface{}) {
zap.S().Fatalf(format, args...)
}

16
pkg/log/display.go Normal file
View File

@ -0,0 +1,16 @@
package log
import "github.com/mgutz/ansi"
const banner = `
__ _ ___ ___
/ _ |/ _ \ / __|
| (_| | (_) | (__
\__, |\___/ \___|
|___/
`
func DisplayGoc() {
stdout.Write([]byte(ansi.Color(banner, "cyan+b")))
}

115
pkg/log/loading_text.go Normal file
View File

@ -0,0 +1,115 @@
package log
import (
"fmt"
"io"
"time"
"github.com/mgutz/ansi"
)
var tty = setupTTY()
type loadingText struct {
message string
stream io.Writer
stopChan chan bool
startTimestamp int64
cnt int
}
func (l *loadingText) start() {
l.startTimestamp = time.Now().UnixNano()
if l.stopChan == nil {
l.stopChan = make(chan bool)
}
go func() {
l.render()
for {
select {
case <-l.stopChan:
return
case <-time.After(time.Millisecond * 100):
l.render()
}
}
}()
}
func (l *loadingText) stop() {
l.stopChan <- true
l.stream.Write([]byte("\r"))
for i := 0; i < len(l.message)+20; i++ {
l.stream.Write([]byte(" "))
}
l.stream.Write([]byte("\r"))
}
func (l *loadingText) render() {
l.stream.Write([]byte("\r"))
messagePrefix := []byte("[wait] ")
prefixLength := len(messagePrefix)
l.stream.Write([]byte(ansi.Color(string(messagePrefix), "cyan+b")))
timeElapsed := fmt.Sprintf("%v", (time.Now().UnixNano()-l.startTimestamp)/int64(time.Second))
message := []byte(l.getLoadingChar() + " " + l.message)
messageSuffix := " (" + timeElapsed + "s) "
suffixLength := len(messageSuffix)
terminalSize := tty.GetSize()
// if the whole message is too long, then replace last words with ...
if terminalSize != nil && terminalSize.Width < uint16(prefixLength+len(message)+suffixLength) {
dots := []byte("...")
maxMessageLength := int(terminalSize.Width) - (prefixLength + suffixLength + len(dots) + 5)
if maxMessageLength > 0 {
message = append(message[:maxMessageLength], dots...)
}
}
message = append(message, messageSuffix...)
l.stream.Write(message)
}
func (l *loadingText) getLoadingChar() string {
var loadingChar string
switch l.cnt {
case 0:
loadingChar = "⠋"
case 1:
loadingChar = "⠙"
case 2:
loadingChar = "⠹"
case 3:
loadingChar = "⠸"
case 4:
loadingChar = "⠼"
case 5:
loadingChar = "⠴"
case 6:
loadingChar = "⠦"
case 7:
loadingChar = "⠧"
case 8:
loadingChar = "⠇"
case 9:
loadingChar = "⠏"
}
l.cnt += 1
if l.cnt > 9 {
l.cnt = 0
}
return loadingChar
}

54
pkg/log/log.go Normal file
View File

@ -0,0 +1,54 @@
package log
import (
"github.com/qiniu/goc/v2/pkg/config"
"go.uber.org/zap/zapcore"
)
var g Logger
func NewLogger() {
if config.GocConfig.Debug == true {
g = newCiLogger()
} else {
g = &terminalLogger{
level: zapcore.InfoLevel,
}
}
}
func Debugf(format string, args ...interface{}) {
g.Debugf(format, args...)
}
func Donef(format string, args ...interface{}) {
g.Donef(format, args...)
}
func Infof(format string, args ...interface{}) {
g.Infof(format, args...)
}
func Warnf(format string, args ...interface{}) {
g.Warnf(format, args...)
}
func Fatalf(format string, args ...interface{}) {
g.Fatalf(format, args...)
}
func Errorf(format string, args ...interface{}) {
g.Errorf(format, args...)
}
func StartWait(message string) {
g.StartWait(message)
}
func StopWait() {
g.StopWait()
}
func Sync() {
g.Sync()
}

22
pkg/log/logger.go Normal file
View File

@ -0,0 +1,22 @@
package log
// Logger defines common interface for logging
type Logger interface {
Debugf(format string, args ...interface{})
Infof(format string, args ...interface{})
Warnf(format string, args ...interface{})
Fatalf(format string, args ...interface{})
Errorf(format string, args ...interface{})
Donef(format string, args ...interface{})
StartWait(message string)
StopWait()
// Sync flushes cached log to disk, some log library needs this step
Sync()
}

22
pkg/log/terminal.go Normal file
View File

@ -0,0 +1,22 @@
package log
import (
"os"
"k8s.io/kubectl/pkg/util/term"
)
func setupTTY() term.TTY {
t := term.TTY{
In: os.Stdin,
Out: os.Stdout,
}
if !t.IsTerminalIn() {
return t
}
t.Raw = true
return t
}

176
pkg/log/terminal_logger.go Normal file
View File

@ -0,0 +1,176 @@
package log
import (
"fmt"
"io"
"os"
"sync"
goansi "github.com/k0kubun/go-ansi"
"github.com/mgutz/ansi"
"go.uber.org/zap/zapcore"
)
// goansi works nicer on Windows platform
var stdout = goansi.NewAnsiStdout()
var stderr = goansi.NewAnsiStderr()
type terminalLogger struct {
mutex sync.Mutex
level zapcore.Level
loadingText *loadingText
}
type levelFuncType int32
const (
fatalFn levelFuncType = iota
infoFn
errorFn
warnFn
debugFn
doneFn
)
type levelFuncInfo struct {
tag string
color string
level zapcore.Level
stream io.Writer
}
var levelFuncMap = map[levelFuncType]*levelFuncInfo{
doneFn: {
tag: "[done] √ ",
color: "green+b",
level: zapcore.InfoLevel,
stream: stdout,
},
debugFn: {
tag: "[debug] ",
color: "green+b",
level: zapcore.DebugLevel,
stream: stdout,
},
infoFn: {
tag: "[info] ",
color: "cyan+b",
level: zapcore.InfoLevel,
stream: stdout,
},
warnFn: {
tag: "[warn] ",
color: "magenta+b",
level: zapcore.WarnLevel,
stream: stdout,
},
errorFn: {
tag: "[error] ",
color: "yellow+b",
level: zapcore.ErrorLevel,
stream: stdout,
},
fatalFn: {
tag: "[fatal] ",
color: "red+b",
level: zapcore.FatalLevel,
stream: stdout,
},
}
func (t *terminalLogger) writeMessage(funcType levelFuncType, message string) {
funcInfo := levelFuncMap[funcType]
if t.level <= funcInfo.level {
// 如果当前有消息在加载，需先暂停
if t.loadingText != nil {
t.loadingText.stop()
}
funcInfo.stream.Write([]byte(ansi.Color(funcInfo.tag, funcInfo.color)))
funcInfo.stream.Write([]byte(message))
// 恢复加载
if t.loadingText != nil && funcType != fatalFn {
t.loadingText.start()
}
}
}
// StartWait prints a waiting message until StopWait is called
func (t *terminalLogger) StartWait(message string) {
t.mutex.Lock()
defer t.mutex.Unlock()
// 撤销之前的加载
if t.loadingText != nil {
t.loadingText.stop()
t.loadingText = nil
}
// 创建新的加载字符串
t.loadingText = &loadingText{
message: message,
stream: goansi.NewAnsiStdout(),
}
t.loadingText.start()
}
// StopWait stops waiting
func (t *terminalLogger) StopWait() {
t.mutex.Lock()
defer t.mutex.Unlock()
if t.loadingText != nil {
t.loadingText.stop()
t.loadingText = nil
}
}
func (t *terminalLogger) Sync() {
}
func (t *terminalLogger) Debugf(format string, args ...interface{}) {
t.mutex.Lock()
defer t.mutex.Unlock()
t.writeMessage(debugFn, fmt.Sprintf(format, args...)+"\n")
}
func (t *terminalLogger) Donef(format string, args ...interface{}) {
t.mutex.Lock()
defer t.mutex.Unlock()
t.writeMessage(doneFn, fmt.Sprintf(format, args...)+"\n")
}
func (t *terminalLogger) Infof(format string, args ...interface{}) {
t.mutex.Lock()
defer t.mutex.Unlock()
t.writeMessage(infoFn, fmt.Sprintf(format, args...)+"\n")
}
func (t *terminalLogger) Errorf(format string, args ...interface{}) {
t.mutex.Lock()
defer t.mutex.Unlock()
t.writeMessage(errorFn, fmt.Sprintf(format, args...)+"\n")
}
func (t *terminalLogger) Warnf(format string, args ...interface{}) {
t.mutex.Lock()
defer t.mutex.Unlock()
t.writeMessage(warnFn, fmt.Sprintf(format, args...)+"\n")
}
func (t *terminalLogger) Fatalf(format string, args ...interface{}) {
t.mutex.Lock()
defer t.mutex.Unlock()
t.writeMessage(fatalFn, fmt.Sprintf(format, args...)+"\n")
os.Exit(1)
}

240
pkg/prow/job.go
View File

@ -1,240 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package prow
import (
"bufio"
"bytes"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path"
"strconv"
"strings"
"time"
"github.com/sirupsen/logrus"
"github.com/qiniu/goc/pkg/cover"
"github.com/qiniu/goc/pkg/github"
"github.com/qiniu/goc/pkg/qiniu"
)
// IProwAction defines the normal action in prow system
type IProwAction interface {
Fetch(BuildID, name string) []byte
RunPresubmit() error
RunPostsubmit() error
RunPeriodic() error
}
// Job is a prowjob in prow
type Job struct {
JobName string
Org string
RepoName string
PRNumStr string
BuildId string //prow job build number
PostSubmitJob string
PostSubmitCoverProfile string
CovThreshold int
LocalProfilePath string
QiniuClient qiniu.Client
LocalArtifacts qiniu.Artifacts
GithubComment github.PrComment
FullDiff bool
}
// Fetch the file from cloud
func (j *Job) Fetch(BuildID, name string) []byte {
return []byte{}
}
// RunPresubmit run a presubmit job
func (j *Job) RunPresubmit() error {
// step1: get local profile cov
localP, err := cover.ReadFileToCoverList(j.LocalProfilePath)
if err != nil {
return fmt.Errorf("failed to get remote cover profile: %s", err.Error())
}
//step2: find the remote healthy cover profile from qiniu bucket
remoteProfile, err := qiniu.FindBaseProfileFromQiniu(j.QiniuClient, j.PostSubmitJob, j.PostSubmitCoverProfile)
if err != nil {
return fmt.Errorf("failed to get remote cover profile: %s", err.Error())
}
if remoteProfile == nil {
logrus.Infof("get non healthy remoteProfile, do nothing")
return nil
}
baseP, err := cover.CovList(bytes.NewReader(remoteProfile))
if err != nil {
return fmt.Errorf("failed to get remote cover profile: %s", err.Error())
}
// step3: get github pull request changed files' name and calculate diff cov between local and remote profile
changedFiles, deltaCovList, err := getFilesAndCovList(j.FullDiff, j.GithubComment, localP, baseP)
if err != nil {
return fmt.Errorf("Get files and covlist failed: %s", err.Error())
}
// step4: generate changed file html coverage
err = j.WriteChangedCov(changedFiles)
if err != nil {
return fmt.Errorf("filter local profile to %s with changed files failed: %s", j.LocalArtifacts.GetChangedProfileName(), err.Error())
}
err = j.CreateChangedCovHtml()
if err != nil {
return fmt.Errorf("create changed file related coverage html failed: %s", err.Error())
}
j.SetDeltaCovLinks(deltaCovList)
// step5: post comment to github
commentPrefix := github.CommentsPrefix
if j.GithubComment.GetCommentFlag() != "" {
commentPrefix = fmt.Sprintf("**%s** ", j.GithubComment.GetCommentFlag()) + commentPrefix
}
if len(deltaCovList) > 0 {
totalDelta := cover.PercentStr(cover.TotalDelta(localP, baseP))
deltaCovList = append(deltaCovList, cover.DeltaCov{FileName: "Total", BasePer: baseP.TotalPercentage(), NewPer: localP.TotalPercentage(), DeltaPer: totalDelta})
}
err = j.GithubComment.CreateGithubComment(commentPrefix, deltaCovList)
if err != nil {
return fmt.Errorf("Post comment to github failed: %s", err.Error())
}
return nil
}
// RunPostsubmit run a postsubmit job
func (j *Job) RunPostsubmit() error {
return nil
}
// RunPeriodic run a periodic job
func (j *Job) RunPeriodic() error {
return nil
}
//trim github filename to profile format:
// src/qiniu.com/kodo/io/io/io_svr.go -> qiniu.com/kodo/io/io/io_svr.go
func trimGhFileToProfile(ghFiles []string) (pFiles []string) {
//TODO: need compatible other situation
logrus.Infof("trim PR changed file name to:")
for _, f := range ghFiles {
file := strings.TrimPrefix(f, "src/")
logrus.Infof("%s", file)
pFiles = append(pFiles, file)
}
return
}
// WriteChangedCov filter local profile with changed files and save to j.LocalArtifacts.ChangedProfileName
func (j *Job) WriteChangedCov(changedFiles []string) error {
p, err := ioutil.ReadFile(j.LocalProfilePath)
if err != nil {
logrus.Printf("Open file %s failed", j.LocalProfilePath)
return err
}
cp := j.LocalArtifacts.CreateChangedProfile()
defer cp.Close()
s := bufio.NewScanner(bytes.NewReader(p))
s.Scan()
writeLine(cp, s.Text())
for s.Scan() {
for _, file := range changedFiles {
if strings.HasPrefix(s.Text(), file) {
writeLine(cp, s.Text())
}
}
}
return nil
}
// writeLine writes a line in the given file, if the file pointer is not nil
func writeLine(file *os.File, content string) {
if file != nil {
fmt.Fprintln(file, content)
}
}
// JobPrefixOnQiniu generates the prefix string of the job on qiniu
func (j *Job) JobPrefixOnQiniu() string {
return path.Join("pr-logs", "pull", j.Org+"_"+j.RepoName, j.PRNumStr, j.JobName, j.BuildId)
}
// HtmlProfile generates the name of the profile html file
func (j *Job) HtmlProfile() string {
return fmt.Sprintf("%s-%s-pr%s-coverage.html", j.Org, j.RepoName, j.PRNumStr)
}
// SetDeltaCovLinks set DeltaCovLinks to the job
func (j *Job) SetDeltaCovLinks(c cover.DeltaCovList) {
c.Sort()
for i := 0; i < len(c); i++ {
qnKey := path.Join(j.JobPrefixOnQiniu(), "artifacts", j.HtmlProfile())
authQnKey := j.QiniuClient.GetAccessURL(qnKey, time.Hour*24*7)
c[i].SetLineCovLink(authQnKey + "#file" + strconv.Itoa(i))
logrus.Printf("file %s html coverage link is: %s\n", c[i].FileName, c[i].GetLineCovLink())
}
}
// CreateChangedCovHtml create changed file related coverage html base on the local artifact
func (j *Job) CreateChangedCovHtml() error {
if j.LocalArtifacts.GetChangedProfileName() == "" {
logrus.Errorf("param LocalArtifacts.ChangedProfileName is empty")
}
pathProfileCov := j.LocalArtifacts.GetChangedProfileName()
pathHtmlCov := path.Join(os.Getenv("ARTIFACTS"), j.HtmlProfile())
cmdTxt := fmt.Sprintf("go tool cover -html=%s -o %s", pathProfileCov, pathHtmlCov)
logrus.Printf("Running command '%s'\n", cmdTxt)
cmd := exec.Command("go", "tool", "cover", "-html="+pathProfileCov, "-o", pathHtmlCov)
stdOut, err := cmd.CombinedOutput()
if err != nil {
logrus.Printf("Error executing cmd: %v; combinedOutput=%s", err, stdOut)
}
return err
}
func getFilesAndCovList(fullDiff bool, prComment github.PrComment, localP, baseP cover.CoverageList) (changedFiles []string, deltaCovList cover.DeltaCovList, err error) {
if !fullDiff {
// get github pull request changed files' name
var ghChangedFiles, err = prComment.GetPrChangedFiles()
if err != nil {
return nil, nil, fmt.Errorf("Get pull request changed file failed: %s", err.Error())
}
if len(ghChangedFiles) == 0 {
logrus.Printf("0 files changed in github pull request, don't need to run coverage profile in presubmit.\n")
return nil, nil, nil
}
changedFiles = trimGhFileToProfile(ghChangedFiles)
// calculate diff cov between local and remote profile
deltaCovList = cover.GetChFileDeltaCov(localP, baseP, changedFiles)
logrus.Printf("Get changed files and delta cover list success. ChangedFiles: [%+v], DeltaCovList: [%+v]", changedFiles, deltaCovList)
return changedFiles, deltaCovList, nil
}
deltaCovList = cover.GetDeltaCov(localP, baseP)
for _, d := range deltaCovList {
changedFiles = append(changedFiles, d.FileName)
}
logrus.Printf("Get all files and delta cover list success. Files: [%+v], DeltaCovList: [%+v]", changedFiles, deltaCovList)
return changedFiles, deltaCovList, nil
}

380
pkg/prow/job_test.go
View File

@ -1,380 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package prow
import (
"context"
"errors"
"fmt"
"io/ioutil"
"os"
"path"
"testing"
"time"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/qiniu/goc/pkg/cover"
"github.com/qiniu/goc/pkg/github"
"github.com/qiniu/goc/pkg/qiniu"
)
var (
defaultContent = `mode: atomic
qiniu.com/kodo/bd/bdgetter/source.go:19.118,22.2 2 0
qiniu.com/kodo/bd/bdgetter/source.go:37.34,39.2 1 0
qiniu.com/kodo/bd/pfd/locker/app/qboxbdlocker/main.go:50.2,53.52 4 1
qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go:33.51,35.2 1 0`
defaultLocalPath = "local.cov"
defaultChangedPath = "changed.cov"
)
type MockQnClient struct {
QiniuObjectHandleRes qiniu.ObjectHandle
ReadObjectRes []byte
ReadObjectErr error
ListAllRes []string
ListAllErr error
GetAccessURLRes string
GetArtifactDetailsRes *qiniu.LogHistoryTemplate
GetArtifactDetailsErr error
ListSubDirsRes []string
ListSubDirsErr error
}
func (s *MockQnClient) QiniuObjectHandle(key string) qiniu.ObjectHandle {
return s.QiniuObjectHandleRes
}
func (s *MockQnClient) ReadObject(key string) ([]byte, error) {
return s.ReadObjectRes, s.ReadObjectErr
}
func (s *MockQnClient) ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error) {
return s.ListAllRes, s.ListAllErr
}
func (s *MockQnClient) GetAccessURL(key string, timeout time.Duration) string {
return s.GetAccessURLRes
}
func (s *MockQnClient) GetArtifactDetails(key string) (*qiniu.LogHistoryTemplate, error) {
return s.GetArtifactDetailsRes, s.GetArtifactDetailsErr
}
func (s *MockQnClient) ListSubDirs(prefix string) ([]string, error) {
return s.ListSubDirsRes, s.ListSubDirsErr
}
type MockPrComment struct {
GetPrChangedFilesRes []string
GetPrChangedFilesErr error
PostCommentErr error
EraseHistoryCommentErr error
CreateGithubCommentErr error
CommentFlag string
}
func (s *MockPrComment) GetPrChangedFiles() (files []string, err error) {
return s.GetPrChangedFilesRes, s.GetPrChangedFilesErr
}
func (s *MockPrComment) PostComment(content, commentPrefix string) error {
return s.PostCommentErr
}
func (s *MockPrComment) EraseHistoryComment(commentPrefix string) error {
return s.EraseHistoryCommentErr
}
func (s *MockPrComment) CreateGithubComment(commentPrefix string, diffCovList cover.DeltaCovList) (err error) {
return s.CreateGithubCommentErr
}
func (s *MockPrComment) GetCommentFlag() string {
return s.CommentFlag
}
func TestTrimGhFileToProfile(t *testing.T) {
items := []struct {
inputFiles []string
expectFiles []string
}{
{
inputFiles: []string{"src/qiniu.com/kodo/io/io/io_svr.go", "README.md"},
expectFiles: []string{"qiniu.com/kodo/io/io/io_svr.go", "README.md"},
},
}
for _, tc := range items {
f := trimGhFileToProfile(tc.inputFiles)
assert.Equal(t, f, tc.expectFiles)
}
}
func setup(path, content string) {
err := ioutil.WriteFile(path, []byte(content), 0644)
if err != nil {
logrus.WithError(err).Fatalf("write file %s failed", path)
}
}
func TestWriteChangedCov(t *testing.T) {
path := defaultLocalPath
savePath := qiniu.ChangedProfileName
content := defaultContent
changedFiles := []string{"qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go"}
expectContent := `mode: atomic
qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go:33.51,35.2 1 0
`
setup(path, content)
defer os.Remove(path)
defer os.Remove(savePath)
j := &Job{
LocalProfilePath: path,
LocalArtifacts: &qiniu.ProfileArtifacts{ChangedProfileName: savePath},
}
j.WriteChangedCov(changedFiles)
r, err := ioutil.ReadFile(savePath)
if err != nil {
logrus.WithError(err).Fatalf("read file %s failed", path)
}
assert.Equal(t, string(r), expectContent)
}
func TestRunPresubmitFulldiff(t *testing.T) {
//param
org := "qbox"
repo := "kodo"
prNum := "1"
buildId := "1266322425771986946"
jobName := "kodo-pull-integration-test"
robotName := "qiniu-bot"
githubCommentPrefix := ""
githubTokenPath := "token"
//mock local profile
pwd, err := os.Getwd()
assert.NoError(t, err)
localPath := defaultLocalPath
localProfileContent := `mode: atomic
"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30
"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0`
setup(localPath, localProfileContent)
defer os.Remove(path.Join(pwd, localPath))
// mock qiniu
conf := qiniu.Config{
Bucket: "artifacts",
}
qc, router, _, teardown := qiniu.MockQiniuServer(&conf)
defer teardown()
qiniu.MockRouterAPI(router, localProfileContent, 0)
ChangedProfilePath := "changed.cov"
defer os.Remove(path.Join(pwd, ChangedProfilePath))
//mock github client
setup(githubTokenPath, "")
defer os.Remove(path.Join(pwd, githubTokenPath))
prClient := github.NewPrClient(githubTokenPath, org, repo, prNum, robotName, githubCommentPrefix)
j := &Job{
JobName: jobName,
Org: org,
RepoName: repo,
PRNumStr: prNum,
BuildId: buildId,
PostSubmitJob: "kodo-postsubmits-go-st-coverage",
PostSubmitCoverProfile: "filterd.cov",
LocalProfilePath: localPath,
LocalArtifacts: &qiniu.ProfileArtifacts{ChangedProfileName: ChangedProfilePath},
QiniuClient: qc,
GithubComment: prClient,
FullDiff: true,
}
defer os.Remove(path.Join(os.Getenv("ARTIFACTS"), j.HtmlProfile()))
err = j.RunPresubmit()
assert.NoError(t, err)
}
func TestRunPresubmitError(t *testing.T) {
items := []struct {
prepare bool // prepare local profile
j Job
err string
}{
{
prepare: false,
j: Job{
LocalProfilePath: "unknown",
},
err: "no such file or directory",
},
{
prepare: true,
j: Job{
LocalProfilePath: defaultLocalPath,
QiniuClient: &MockQnClient{},
},
},
{
prepare: true,
j: Job{
LocalProfilePath: defaultLocalPath,
QiniuClient: &MockQnClient{ListSubDirsErr: errors.New("mock error")},
},
err: "mock error",
},
{
prepare: true,
j: Job{
LocalProfilePath: defaultLocalPath,
QiniuClient: &MockProfileQnClient{},
GithubComment: &MockPrComment{GetPrChangedFilesRes: []string{"qiniu.com/kodo/apiserver/server/main.go"}},
FullDiff: true,
LocalArtifacts: &qiniu.ProfileArtifacts{ChangedProfileName: defaultChangedPath},
},
err: "",
},
}
for _, tc := range items {
if tc.prepare {
path := defaultLocalPath
setup(path, defaultContent)
defer os.Remove(path)
defer os.Remove(defaultChangedPath)
}
err := tc.j.RunPresubmit()
if tc.err == "" {
assert.NoError(t, err)
} else {
assert.Contains(t, err.Error(), tc.err)
}
}
}
type MockProfileQnClient struct {
*MockQnClient
}
func (s *MockProfileQnClient) ListSubDirs(prefix string) ([]string, error) {
return []string{defaultContent}, nil
}
func (s *MockProfileQnClient) ReadObject(key string) ([]byte, error) {
logrus.Info(key)
if key == "logs/1/finished.json" {
return []byte(`{"timestamp":1590750306,"passed":true,"result":"SUCCESS","repo-version":"76433418ea48aae57af028f9cb2fa3735ce08c7d"}`), nil
}
return []byte(""), nil
}
func TestGetFilesAndCovList(t *testing.T) {
items := []struct {
fullDiff bool
prComment github.PrComment
localP cover.CoverageList
baseP cover.CoverageList
err string
lenFiles int
lenCovList int
}{
{
fullDiff: true,
prComment: &MockPrComment{},
localP: cover.CoverageList{
{FileName: "qiniu.com/kodo/apiserver/server/main.go", NCoveredStmts: 2, NAllStmts: 2},
{FileName: "qiniu.com/kodo/apiserver/server/test.go", NCoveredStmts: 2, NAllStmts: 2},
},
baseP: cover.CoverageList{
{FileName: "qiniu.com/kodo/apiserver/server/main.go", NCoveredStmts: 1, NAllStmts: 2},
{FileName: "qiniu.com/kodo/apiserver/server/test.go", NCoveredStmts: 1, NAllStmts: 2},
},
lenFiles: 2,
lenCovList: 2,
},
{
fullDiff: false,
prComment: &MockPrComment{GetPrChangedFilesErr: errors.New("mock error")},
err: "mock error",
},
{
fullDiff: false,
prComment: &MockPrComment{},
lenFiles: 0,
lenCovList: 0,
},
{
fullDiff: false,
prComment: &MockPrComment{GetPrChangedFilesRes: []string{"qiniu.com/kodo/apiserver/server/main.go"}},
localP: cover.CoverageList{
{FileName: "qiniu.com/kodo/apiserver/server/main.go", NCoveredStmts: 2, NAllStmts: 2},
{FileName: "qiniu.com/kodo/apiserver/server/test.go", NCoveredStmts: 2, NAllStmts: 2},
},
baseP: cover.CoverageList{
{FileName: "qiniu.com/kodo/apiserver/server/main.go", NCoveredStmts: 1, NAllStmts: 2},
{FileName: "qiniu.com/kodo/apiserver/server/test.go", NCoveredStmts: 1, NAllStmts: 2},
},
lenFiles: 1,
lenCovList: 1,
},
}
for i, tc := range items {
fmt.Println(i)
files, covList, err := getFilesAndCovList(tc.fullDiff, tc.prComment, tc.localP, tc.baseP)
if err != nil {
assert.Contains(t, err.Error(), tc.err)
} else {
assert.Equal(t, len(files), tc.lenFiles)
assert.Equal(t, len(covList), tc.lenCovList)
}
}
}
func TestSetDeltaCovLinks(t *testing.T) {
covList := cover.DeltaCovList{{FileName: "file1", BasePer: "5%", NewPer: "5%", DeltaPer: "0"}}
j := &Job{
QiniuClient: &MockQnClient{},
}
j.SetDeltaCovLinks(covList)
}
// functions to be done
func TestRunPostsubmit(t *testing.T) {
j := &Job{}
err := j.RunPostsubmit()
assert.NoError(t, err)
}
func TestRunPeriodic(t *testing.T) {
j := &Job{}
err := j.RunPeriodic()
assert.NoError(t, err)
}
func TestFetch(t *testing.T) {
j := &Job{}
res := j.Fetch("buidID", "name")
assert.Equal(t, res, []byte{})
}

254
pkg/qiniu/client.go
View File

@ -1,254 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package qiniu
import (
"context"
"fmt"
"io/ioutil"
"math/rand"
"net/http"
"regexp"
"strconv"
"strings"
"time"
"github.com/qiniu/api.v7/v7/auth/qbox"
"github.com/qiniu/api.v7/v7/client"
"github.com/qiniu/api.v7/v7/storage"
"github.com/sirupsen/logrus"
)
// Config store the credentials to connect with qiniu cloud
type Config struct {
Bucket string `json:"bucket"`
AccessKey string `json:"accessKey"`
SecretKey string `json:"secretKey"`
// domain used to download files from qiniu cloud
Domain string `json:"domain"`
}
// Client is the interface contains the operation with qiniu cloud
type Client interface {
QiniuObjectHandle(key string) ObjectHandle
ReadObject(key string) ([]byte, error)
ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error)
GetAccessURL(key string, timeout time.Duration) string
GetArtifactDetails(key string) (*LogHistoryTemplate, error)
ListSubDirs(prefix string) ([]string, error)
}
// QnClient for the operation with qiniu cloud
type QnClient struct {
cfg *Config
BucketManager *storage.BucketManager
}
// NewClient creates a new QnClient to work with qiniu cloud
func NewClient(cfg *Config) *QnClient {
return &QnClient{
cfg: cfg,
BucketManager: storage.NewBucketManager(qbox.NewMac(cfg.AccessKey, cfg.SecretKey), nil),
}
}
// QiniuObjectHandle construct a object hanle to access file in qiniu
func (q *QnClient) QiniuObjectHandle(key string) ObjectHandle {
return &QnObjectHandle{
key: key,
cfg: q.cfg,
bm: q.BucketManager,
mac: qbox.NewMac(q.cfg.AccessKey, q.cfg.SecretKey),
client: &client.Client{Client: http.DefaultClient},
}
}
// ReadObject to read all the content of key
func (q *QnClient) ReadObject(key string) ([]byte, error) {
objectHandle := q.QiniuObjectHandle(key)
reader, err := objectHandle.NewReader(context.Background())
if err != nil {
return nil, fmt.Errorf("error getting qiniu artifact reader: %v", err)
}
defer reader.Close()
return ioutil.ReadAll(reader)
}
// ListAll to list all the files with contains the expected prefix
func (q *QnClient) ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error) {
var files []string
artifacts, err := q.listEntries(prefix, delimiter)
if err != nil {
return files, err
}
for _, item := range artifacts {
files = append(files, item.Key)
}
return files, nil
}
// listEntries to list all the entries with contains the expected prefix
func (q *QnClient) listEntries(prefix string, delimiter string) ([]storage.ListItem, error) {
var marker string
var artifacts []storage.ListItem
wait := []time.Duration{16, 32, 64, 128, 256, 256, 512, 512}
for i := 0; ; {
entries, _, nextMarker, hashNext, err := q.BucketManager.ListFiles(q.cfg.Bucket, prefix, delimiter, marker, 500)
if err != nil {
logrus.WithField("prefix", prefix).WithError(err).Error("Error accessing QINIU artifact.")
if i >= len(wait) {
return artifacts, fmt.Errorf("timed out: error accessing QINIU artifact: %v", err)
}
time.Sleep((wait[i] + time.Duration(rand.Intn(10))) * time.Millisecond)
i++
continue
}
artifacts = append(artifacts, entries...)
if hashNext {
marker = nextMarker
} else {
break
}
}
return artifacts, nil
}
// GetAccessURL return a url which can access artifact directly in qiniu
func (q *QnClient) GetAccessURL(key string, timeout time.Duration) string {
deadline := time.Now().Add(timeout).Unix()
return storage.MakePrivateURL(qbox.NewMac(q.cfg.AccessKey, q.cfg.SecretKey), q.cfg.Domain, key, deadline)
}
// LogHistoryTemplate is the template of the log history
type LogHistoryTemplate struct {
BucketName string
KeyPath string
Items []logHistoryItem
}
// logHistoryItem represents a log history item
type logHistoryItem struct {
Name string
Size string
Time string
Url string
}
// GetArtifactDetails lists all artifacts available for the given job source
func (q *QnClient) GetArtifactDetails(key string) (*LogHistoryTemplate, error) {
tmpl := new(LogHistoryTemplate)
item := logHistoryItem{}
listStart := time.Now()
artifacts, err := q.listEntries(key, "")
if err != nil {
return tmpl, err
}
for _, entry := range artifacts {
item.Name = splitKey(entry.Key, key)
item.Size = size(entry.Fsize)
item.Time = timeConv(entry.PutTime)
item.Url = q.GetAccessURL(entry.Key, time.Duration(time.Second*60*60))
tmpl.Items = append(tmpl.Items, item)
}
logrus.WithField("duration", time.Since(listStart).String()).Infof("Listed %d artifacts.", len(tmpl.Items))
return tmpl, nil
}
func splitKey(item, key string) string {
return strings.TrimPrefix(item, key)
}
func size(fsize int64) string {
return strings.Join([]string{strconv.FormatInt(fsize, 10), "bytes"}, " ")
}
func timeConv(ptime int64) string {
s := strconv.FormatInt(ptime, 10)[0:10]
t, err := strconv.ParseInt(s, 10, 64)
if err != nil {
logrus.Errorf("time string parse int error : %v", err)
return ""
}
tm := time.Unix(t, 0)
return tm.Format("2006-01-02 03:04:05 PM")
}
// ListSubDirs list all the sub directions of the prefix string in qiniu client
func (q *QnClient) ListSubDirs(prefix string) ([]string, error) {
var dirs []string
var marker string
wait := []time.Duration{16, 32, 64, 128, 256, 256, 512, 512}
for i := 0; ; {
// use rsf list v2 interface to get the sub folder based on the delimiter
entries, err := q.BucketManager.ListBucketContext(context.Background(), q.cfg.Bucket, prefix, "/", marker)
if err != nil {
logrus.WithField("prefix", prefix).WithError(err).Error("Error accessing QINIU artifact.")
if i >= len(wait) {
return dirs, fmt.Errorf("timed out: error accessing QINIU artifact: %v", err)
}
time.Sleep((wait[i] + time.Duration(rand.Intn(10))) * time.Millisecond)
i++
continue
}
for entry := range entries {
if entry.Dir != "" {
// entry.Dir should be like "logs/kodo-periodics-integration-test/1181915661132107776/"
// the sub folder is 1181915661132107776, also known as prowjob buildid.
buildId := getBuildId(entry.Dir)
if buildId != "" {
dirs = append(dirs, buildId)
} else {
logrus.Warnf("invalid dir format: %v", entry.Dir)
}
}
marker = entry.Marker
}
if marker != "" {
i = 0
} else {
break
}
}
return dirs, nil
}
var nonPRLogsBuildIdSubffixRe = regexp.MustCompile("([0-9]+)/$")
// extract the build number from dir path
// expect the dir as the following formats:
// 1. logs/kodo-periodics-integration-test/1181915661132107776/
func getBuildId(dir string) string {
matches := nonPRLogsBuildIdSubffixRe.FindStringSubmatch(dir)
if len(matches) == 2 {
return matches[1]
}
return ""
}

184
pkg/qiniu/client_test.go
View File

@ -1,184 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package qiniu
import (
"context"
"path"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestGetBuildId(t *testing.T) {
type tc struct {
dir string
expected string
}
tcs := []tc{
{dir: "logs/kodo-periodics-integration-test/1181915661132107776/", expected: "1181915661132107776"},
{dir: "logs/kodo-periodics-integration-test/1181915661132107776", expected: ""},
{dir: "pr-logs/directory/WIP-qtest-pull-request-kodo-test/1181915661132107776/", expected: "1181915661132107776"},
{dir: "pr-logs/directory/WIP-qtest-pull-request-kodo-test/1181915661132107776.txt", expected: ""},
}
for _, tc := range tcs {
got := getBuildId(tc.dir)
if tc.expected != got {
t.Errorf("getBuildId error, dir: %s, expect: %s, but got: %s", tc.dir, tc.expected, got)
}
}
}
// test basic listEntries function
func TestListAllFiles(t *testing.T) {
conf := Config{
Bucket: "artifacts",
}
qc, router, _, teardown := MockQiniuServer(&conf)
defer teardown()
prowJobName := "kodo-postsubmits-go-st-coverage"
dirOfJob := path.Join("logs", prowJobName)
prefix := dirOfJob + "/"
MockRouterListAllAPI(router, 0)
listItems, err := qc.listEntries(prefix, "/")
assert.Equal(t, err, nil)
assert.Equal(t, len(listItems), 1)
assert.Equal(t, listItems[0].Key, "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json")
}
// test basic listEntries function, recover after 3 times
func TestListAllFilesWithServerTimeoutAndRecover(t *testing.T) {
conf := Config{
Bucket: "artifacts",
}
qc, router, _, teardown := MockQiniuServer(&conf)
defer teardown()
prowJobName := "kodo-postsubmits-go-st-coverage"
dirOfJob := path.Join("logs", prowJobName)
prefix := dirOfJob + "/"
// recover after 3 times
MockRouterListAllAPI(router, 3)
listItems, err := qc.listEntries(prefix, "/")
assert.Equal(t, err, nil)
assert.Equal(t, len(listItems), 1)
assert.Equal(t, listItems[0].Key, "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json")
}
// test basic listEntries function, never recover
func TestListAllFilesWithServerTimeout(t *testing.T) {
conf := Config{
Bucket: "artifacts",
}
qc, router, _, teardown := MockQiniuServer(&conf)
defer teardown()
prowJobName := "kodo-postsubmits-go-st-coverage"
dirOfJob := path.Join("logs", prowJobName)
prefix := dirOfJob + "/"
// never recover
MockRouterListAllAPI(router, 13)
_, err := qc.listEntries(prefix, "/")
assert.Equal(t, strings.Contains(err.Error(), "timed out: error accessing QINIU artifact"), true)
}
// test ListAll function
func TestListAllFilesWithContext(t *testing.T) {
conf := Config{
Bucket: "artifacts",
}
qc, router, _, teardown := MockQiniuServer(&conf)
defer teardown()
prowJobName := "kodo-postsubmits-go-st-coverage"
dirOfJob := path.Join("logs", prowJobName)
prefix := dirOfJob + "/"
MockRouterListAllAPI(router, 0)
listItems, err := qc.ListAll(context.Background(), prefix, "/")
assert.Equal(t, err, nil)
assert.Equal(t, len(listItems), 1)
assert.Equal(t, listItems[0], "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json")
}
// test GetArtifactDetails function
func TestGetArtifactDetails(t *testing.T) {
conf := Config{
Bucket: "artifacts",
}
qc, router, _, teardown := MockQiniuServer(&conf)
defer teardown()
prowJobName := "kodo-postsubmits-go-st-coverage"
dirOfJob := path.Join("logs", prowJobName)
prefix := dirOfJob + "/"
MockRouterListAllAPI(router, 0)
tmpl, err := qc.GetArtifactDetails(prefix)
assert.Equal(t, err, nil)
assert.Equal(t, len(tmpl.Items), 1)
assert.Equal(t, tmpl.Items[0].Name, "1181915661132107776/finished.json")
assert.Equal(t, strings.Contains(tmpl.Items[0].Url, prowJobName), true)
}
// test ListSubDirs function, recover after 3 times
func TestListSubDirsWithServerTimeoutAndRecover(t *testing.T) {
conf := Config{
Bucket: "artifacts",
}
qc, router, _, teardown := MockQiniuServer(&conf)
defer teardown()
prowJobName := "kodo-postsubmits-go-st-coverage"
dirOfJob := path.Join("logs", prowJobName)
prefix := dirOfJob + "/"
localProfileContent := `mode: atomic
"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30
"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0`
// recover after 3 times
MockRouterAPI(router, localProfileContent, 3)
listItems, err := qc.ListSubDirs(prefix)
assert.Equal(t, err, nil)
assert.Equal(t, len(listItems), 1)
assert.Equal(t, listItems[0], "1181915661132107776")
}
// test ListSubDirs function, never recover
func TestListSubDirsWithServerTimeout(t *testing.T) {
conf := Config{
Bucket: "artifacts",
}
qc, router, _, teardown := MockQiniuServer(&conf)
defer teardown()
prowJobName := "kodo-postsubmits-go-st-coverage"
dirOfJob := path.Join("logs", prowJobName)
prefix := dirOfJob + "/"
localProfileContent := `mode: atomic
"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30
"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0`
// never recover
MockRouterAPI(router, localProfileContent, 13)
_, err := qc.ListSubDirs(prefix)
assert.Equal(t, strings.Contains(err.Error(), "timed out: error accessing QINIU artifact"), true)
}

162
pkg/qiniu/mock.go
View File

@ -1,162 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package qiniu
import (
"fmt"
"net/http"
"net/http/httptest"
"github.com/julienschmidt/httprouter"
"github.com/qiniu/api.v7/v7/storage"
"github.com/sirupsen/logrus"
)
// MockQiniuServer simulate qiniu cloud for testing
func MockQiniuServer(config *Config) (client *QnClient, router *httprouter.Router, serverURL string, teardown func()) {
// router is the HTTP request multiplexer used with the test server.
router = httprouter.New()
// server is a test HTTP server used to provide mock API responses.
server := httptest.NewServer(router)
config.Domain = server.URL
client = NewClient(config)
client.BucketManager.Cfg = &storage.Config{
RsfHost: server.URL,
}
logrus.Infof("server url is: %s", server.URL)
return client, router, server.URL, server.Close
}
// MockRouterAPI mocks qiniu /v2/list API.
// You need to provide a expected profile content.
// count controls the mocks qiniu server to error before 'count' times request.
func MockRouterAPI(router *httprouter.Router, profile string, count int) {
timeout := count
// mock rsf /v2/list
router.HandlerFunc("POST", "/v2/list", func(w http.ResponseWriter, r *http.Request) {
logrus.Infof("request url is: %s", r.URL.String())
if timeout > 0 {
timeout--
http.Error(w, "not found", http.StatusNotFound)
return
}
fmt.Fprint(w, `{
"item": {
"key": "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json",
"hash": "FkBhdo9odL2Xjvu-YdwtDIw79fIL",
"fsize": 51523,
"mimeType": "application/octet-stream",
"putTime": 15909068578047958,
"type": 0,
"status": 0,
"md5": "e0bd20e97ea1c6a5e2480192ee3ae884"
},
"marker": "",
"dir": "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/"
}`)
})
// mock io get statusJSON file
router.HandlerFunc("GET", "/logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, `{"timestamp":1590750306,"passed":true,"result":"SUCCESS","repo-version":"76433418ea48aae57af028f9cb2fa3735ce08c7d"}`)
})
// mock io get remote coverage profile
router.HandlerFunc("GET", "/logs/kodo-postsubmits-go-st-coverage/1181915661132107776/artifacts/filterd.cov", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, profile)
})
}
// MockRouterListAllAPI mocks qiniu /list API.
// count controls the mocks qiniu server to error before 'count' times request.
func MockRouterListAllAPI(router *httprouter.Router, count int) {
timeout := count
// mock rsf /v2/list
router.HandlerFunc("POST", "/list", func(w http.ResponseWriter, r *http.Request) {
logrus.Infof("will respond after %v times", timeout)
logrus.Infof("request url is: %s", r.URL.String())
if timeout > 0 {
timeout--
http.Error(w, "not found", http.StatusNotFound)
return
}
fmt.Fprint(w, `{
"items": [{
"key": "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json",
"hash": "FkBhdo9odL2Xjvu-YdwtDIw79fIL",
"fsize": 51523,
"mimeType": "application/octet-stream",
"putTime": 15909068578047958,
"type": 0,
"status": 0,
"md5": "e0bd20e97ea1c6a5e2480192ee3ae884"
}],
"marker": "",
"commonPrefixes": ["logs/kodo-postsubmits-go-st-coverage/1181915661132107776/"]
}`)
})
}
// MockPrivateDomainUrl mocks bucket domain /key, /timeout, /retry API.
// count controls the mocks qiniu server to error before 'count' times request.
func MockPrivateDomainUrl(router *httprouter.Router, count int) {
timeout1 := count
timeout2 := count
router.HandlerFunc("GET", "/key", func(w http.ResponseWriter, r *http.Request) {
logrus.Infof("request url is: %s", r.URL.String())
fmt.Fprint(w, "mock server ok")
})
router.HandlerFunc("GET", "/timeout", func(w http.ResponseWriter, r *http.Request) {
logrus.Infof("request url is: %s", r.URL.String())
if timeout1 > 0 {
timeout1--
http.Error(w, "not found", http.StatusNotFound)
return
}
fmt.Fprint(w, "mock server ok")
})
router.HandlerFunc("GET", "/retry", func(w http.ResponseWriter, r *http.Request) {
logrus.Infof("request url is: %s", r.URL.String())
if timeout2 > 0 {
timeout2--
if timeout2%2 == 0 {
http.Error(w, "not found", 571)
} else {
http.Error(w, "not found", 573)
}
return
}
fmt.Fprint(w, "mock server ok")
})
}

126
pkg/qiniu/object.go
View File

@ -1,126 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package qiniu
import (
"context"
"fmt"
"io"
"net/http"
"time"
"github.com/qiniu/api.v7/v7/auth/qbox"
"github.com/qiniu/api.v7/v7/client"
"github.com/qiniu/api.v7/v7/storage"
"github.com/sirupsen/logrus"
)
// ObjectHandle is the interface contains the operations on an object in a qiniu cloud bucket
type ObjectHandle interface {
NewReader(ctx context.Context) (io.ReadCloser, error)
NewRangeReader(ctx context.Context, offset, length int64) (io.ReadCloser, error)
}
// QnObjectHandle provides operations on an object in a qiniu cloud bucket
type QnObjectHandle struct {
key string
cfg *Config
bm *storage.BucketManager
mac *qbox.Mac
client *client.Client
}
// NewReader creates a reader to read the contents of the object.
// ErrObjectNotExist will be returned if the object is not found.
// The caller must call Close on the returned Reader when done reading.
func (o *QnObjectHandle) NewReader(ctx context.Context) (io.ReadCloser, error) {
return o.NewRangeReader(ctx, 0, -1)
}
// NewRangeReader reads parts of an object, reading at most length bytes starting
// from the given offset. If length is negative, the object is read until the end.
func (o *QnObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) (io.ReadCloser, error) {
verb := "GET"
if length == 0 {
verb = "HEAD"
}
var res *http.Response
var err error
err = runWithRetry(3, func() (bool, error) {
headers := http.Header{}
start := offset
if length < 0 && start >= 0 {
headers.Set("Range", fmt.Sprintf("bytes=%d-", start))
} else if length > 0 {
// The end character isn't affected by how many bytes we have seen.
headers.Set("Range", fmt.Sprintf("bytes=%d-%d", start, offset+length-1))
}
deadline := time.Now().Add(time.Second * 60 * 10).Unix()
accessURL := storage.MakePrivateURL(o.mac, o.cfg.Domain, o.key, deadline)
res, err = o.client.DoRequest(ctx, verb, accessURL, headers)
if err != nil {
time.Sleep(time.Second) //TODO enhance
return true, err
}
if res.StatusCode == http.StatusNotFound {
res.Body.Close()
return true, fmt.Errorf("qiniu storage: object not exists")
}
return shouldRetry(res), nil
})
if err != nil {
return nil, err
}
return res.Body, nil
}
func runWithRetry(maxTry int, f func() (bool, error)) error {
var err error
for maxTry > 0 {
var needRetry bool
needRetry, err = f() // fix - needRetry, err := f(), err hides the outside error
if err != nil {
logrus.Warnf("err occurred: %v. try again", err)
} else if needRetry {
logrus.Warn("results do not meet the expectation. try again")
} else {
break
}
time.Sleep(time.Millisecond * 100)
maxTry = maxTry - 1
}
return err
}
func shouldRetry(res *http.Response) bool {
// 571 and 573 mean the request was limited by cloud storage because of concurrency count exceed
// so it's better to retry after a while
if res.StatusCode == 571 || res.StatusCode == 573 {
return true
}
return false
}

138
pkg/qiniu/object_test.go
View File

@ -1,138 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package qiniu
import (
"context"
"fmt"
"io/ioutil"
"net/http"
"testing"
"github.com/qiniu/api.v7/v7/auth/qbox"
"github.com/qiniu/api.v7/v7/client"
"github.com/stretchr/testify/assert"
)
// test NewRangeReader logic
func TestNewRangeReader(t *testing.T) {
cfg := &Config{
Bucket: "artifacts",
AccessKey: "ak",
SecretKey: "sk",
}
_, router, serverUrl, teardown := MockQiniuServer(cfg)
defer teardown()
cfg.Domain = serverUrl
MockPrivateDomainUrl(router, 0)
oh := &QnObjectHandle{
key: "key",
cfg: cfg,
bm: nil,
mac: qbox.NewMac(cfg.AccessKey, cfg.SecretKey),
client: &client.Client{Client: http.DefaultClient},
}
// test read unlimited
body, err := oh.NewRangeReader(context.Background(), 0, -1)
assert.Equal(t, err, nil)
bodyBytes, err := ioutil.ReadAll(body)
assert.NoError(t, err)
assert.Equal(t, string(bodyBytes), "mock server ok")
// test with HEAD method
body, err = oh.NewRangeReader(context.Background(), 0, 0)
assert.Equal(t, err, nil)
bodyBytes, err = ioutil.ReadAll(body)
assert.NoError(t, err)
assert.Equal(t, string(bodyBytes), "")
}
// test retry logic
func TestNewRangeReaderWithTimeoutAndRecover(t *testing.T) {
cfg := &Config{
Bucket: "artifacts",
AccessKey: "ak",
SecretKey: "sk",
}
_, router, serverUrl, teardown := MockQiniuServer(cfg)
defer teardown()
cfg.Domain = serverUrl
MockPrivateDomainUrl(router, 2)
oh := &QnObjectHandle{
key: "key",
cfg: cfg,
bm: nil,
mac: qbox.NewMac(cfg.AccessKey, cfg.SecretKey),
client: &client.Client{Client: http.DefaultClient},
}
// test with timeout
oh.key = "timeout"
body, err := oh.NewRangeReader(context.Background(), 0, 10)
assert.Equal(t, err, nil)
bodyBytes, err := ioutil.ReadAll(body)
assert.NoError(t, err)
assert.Equal(t, string(bodyBytes), "mock server ok")
// test with retry with statuscode=571, 573
oh.key = "retry"
body, err = oh.NewRangeReader(context.Background(), 0, 10)
assert.Equal(t, err, nil)
bodyBytes, err = ioutil.ReadAll(body)
assert.NoError(t, err)
assert.Equal(t, string(bodyBytes), "mock server ok")
}
// test retry logic
func TestNewRangeReaderWithTimeoutNoRecover(t *testing.T) {
cfg := &Config{
Bucket: "artifacts",
AccessKey: "ak",
SecretKey: "sk",
}
_, router, serverUrl, teardown := MockQiniuServer(cfg)
defer teardown()
cfg.Domain = serverUrl
MockPrivateDomainUrl(router, 12)
oh := &QnObjectHandle{
key: "key",
cfg: cfg,
bm: nil,
mac: qbox.NewMac(cfg.AccessKey, cfg.SecretKey),
client: &client.Client{Client: http.DefaultClient},
}
// test with timeout
oh.key = "timeout"
_, err := oh.NewRangeReader(context.Background(), 0, -1)
assert.Equal(t, err, fmt.Errorf("qiniu storage: object not exists"))
// bodyBytes, err := ioutil.ReadAll(body)
// assert.Equal(t, string(bodyBytes), "mock server ok")
}

146
pkg/qiniu/qnPresubmit.go
View File

@ -1,146 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package qiniu
import (
"encoding/json"
"fmt"
"os"
"path"
"sort"
"strconv"
log "github.com/sirupsen/logrus"
)
const (
//statusJSON is the JSON file that stores build success info
statusJSON = "finished.json"
// ArtifactsDirName is the name of directory defined in prow to store test artifacts
ArtifactsDirName = "artifacts"
//PostSubmitCoverProfile represents the default output coverage file generated in prow environment
PostSubmitCoverProfile = "filtered.cov"
//ChangedProfileName represents the default changed coverage profile based on files changed in Pull Request
ChangedProfileName = "changed-file-profile.cov"
)
// sortBuilds converts all build from str to int and sorts all builds in descending order and
// returns the sorted slice
func sortBuilds(strBuilds []string) []int {
var res []int
for _, buildStr := range strBuilds {
num, err := strconv.Atoi(buildStr)
if err != nil {
log.Printf("Non-int build number found: '%s'", buildStr)
} else {
res = append(res, num)
}
}
sort.Sort(sort.Reverse(sort.IntSlice(res)))
return res
}
type finishedStatus struct {
Timestamp int
Passed bool
}
func isBuildSucceeded(jsonText []byte) bool {
var status finishedStatus
err := json.Unmarshal(jsonText, &status)
return err == nil && status.Passed
}
// FindBaseProfileFromQiniu finds the coverage profile file from the latest healthy build
// stored in given gcs directory
func FindBaseProfileFromQiniu(qc Client, prowJobName, covProfileName string) ([]byte, error) {
dirOfJob := path.Join("logs", prowJobName)
prefix := dirOfJob + "/"
strBuilds, err := qc.ListSubDirs(prefix)
if err != nil {
return nil, fmt.Errorf("error listing qiniu objects, prowjob:%v, err:%v", prowJobName, err)
}
if len(strBuilds) == 0 {
log.Printf("no cover profiles found from remote, do nothing")
return nil, nil
}
log.Printf("total sub dirs: %d", len(strBuilds))
builds := sortBuilds(strBuilds)
profilePath := ""
for _, build := range builds {
buildDirPath := path.Join(dirOfJob, strconv.Itoa(build))
dirOfStatusJSON := path.Join(buildDirPath, statusJSON)
statusText, err := qc.ReadObject(dirOfStatusJSON)
if err != nil {
log.Printf("Cannot read finished.json (%s) ", dirOfStatusJSON)
} else if isBuildSucceeded(statusText) {
artifactsDirPath := path.Join(buildDirPath, ArtifactsDirName)
profilePath = path.Join(artifactsDirPath, covProfileName)
break
}
}
if profilePath == "" {
log.Printf("no cover profiles found from remote job %s, do nothing", prowJobName)
return nil, nil
}
log.Printf("base cover profile path: %s", profilePath)
return qc.ReadObject(profilePath)
}
// Artifacts is the interface of the rule to store test artifacts in prow
type Artifacts interface {
ProfilePath() string
CreateChangedProfile() *os.File
GetChangedProfileName() string
}
// ProfileArtifacts presents the rule to store test artifacts in prow
type ProfileArtifacts struct {
Directory string
ProfileName string
ChangedProfileName string // create temporary to save changed file related coverage profile
}
// ProfilePath returns a full path for profile
func (a *ProfileArtifacts) ProfilePath() string {
return path.Join(a.Directory, a.ProfileName)
}
// CreateChangedProfile creates a profile in order to store the most related files based on Github Pull Request
func (a *ProfileArtifacts) CreateChangedProfile() *os.File {
if a.ChangedProfileName == "" {
log.Fatalf("param Artifacts.ChangedProfileName should not be empty")
}
p, err := os.Create(a.ChangedProfileName)
log.Printf("os create: %s", a.ChangedProfileName)
if err != nil {
log.Fatalf("file(%s) create failed: %v", a.ChangedProfileName, err)
}
return p
}
// GetChangedProfileName get ChangedProfileName of the ProfileArtifacts
func (a *ProfileArtifacts) GetChangedProfileName() string {
return a.ChangedProfileName
}

70
pkg/qiniu/qnPresubmit_test.go
View File

@ -1,70 +0,0 @@
/*
Copyright 2020 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package qiniu
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
)
func TestFindBaseProfileFromQiniu(t *testing.T) {
conf := Config{
Bucket: "artifacts",
}
qc, router, _, teardown := MockQiniuServer(&conf)
defer teardown()
prowJobName := "kodo-postsubmits-go-st-coverage"
covProfileName := "filterd.cov"
mockProfileContent := `mode: atomic
"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30
"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0`
MockRouterAPI(router, mockProfileContent, 0)
getProfile, err := FindBaseProfileFromQiniu(qc, prowJobName, covProfileName)
assert.Equal(t, err, nil)
assert.Equal(t, string(getProfile), mockProfileContent)
}
func TestArtifacts_ProfilePath(t *testing.T) {
p := &ProfileArtifacts{
Directory: "directory/",
ProfileName: "profile",
}
profilePath := p.ProfilePath()
assert.Equal(t, profilePath, "directory/profile")
}
func TestProfileArtifacts_CreateChangedProfile(t *testing.T) {
p := &ProfileArtifacts{
ChangedProfileName: "test.cov",
}
file := p.CreateChangedProfile()
file.Close()
defer os.Remove(p.ChangedProfileName)
_, err := os.Stat(p.ChangedProfileName)
assert.NoError(t, err)
}
func TestProfileArtifacts_GetChangedProfileName(t *testing.T) {
p := &ProfileArtifacts{
ChangedProfileName: "change.cov",
}
name := p.GetChangedProfileName()
assert.Equal(t, name, "change.cov")
}

Some files were not shown because too many files have changed in this diff Show More