goc add diff cmd
This commit is contained in:
chupei
parent
cb74426dd4
commit
50ffb7980b
200
cmd/diff.go
Normal file
200
cmd/diff.go
Normal file
@ -0,0 +1,200 @@
|
||||
/*
|
||||
Copyright 2020 Qiniu Cloud (qiniu.com)
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
|
||||
"github.com/olekukonko/tablewriter"
|
||||
"github.com/qiniu/goc/pkg/cover"
|
||||
"github.com/qiniu/goc/pkg/prow"
|
||||
"github.com/qiniu/goc/pkg/qiniu"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/qiniu/goc/pkg/github"
|
||||
)
|
||||
|
||||
var diffCmd = &cobra.Command{
|
||||
Use: "diff",
|
||||
Short: "do coverage profile diff analysis, it can also work with prow and post comments to github pull request if needed",
|
||||
Example: ` goc diff --new-profile=<xxxx> --base-profile=<xxxx> // diff two local coverage profile and display
|
||||
goc diff --prow-postsubmit-job=<xxx> --new-profile=<xxx> // diff local coverage profile with the remote one in prow job using default qiniu-credential
|
||||
goc diff --prow-postsubmit-job=<xxx> --new-profile=<xxx> --full-diff=true // calculate and display full diff coverage between new-profile and base-profile, not concerned github changed files
|
||||
goc diff --prow-postsubmit-job=<xxx> --prow-remote-profile-name=<xxx>
|
||||
--qiniu-credential=<xxx> --new-profile=<xxxx> // diff local coverage profile with the remote one in prow job
|
||||
goc diff --prow-postsubmit-job=<xxx> --prow-profile=<xxx>
|
||||
--github-token=<xxx> --github-user=<xxx> --github-comment-prefix=<xxx>
|
||||
--qiniu-credential=<xxx> --coverage-threshold-percentage=<xxx> --new-profile=<xxxx> // diff coverage profile with the remote one in prow job, and post comments to github PR
|
||||
`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
if baseProfile != "" {
|
||||
doDiffForLocalProfiles(cmd, args)
|
||||
} else if prowPostSubmitJob != "" {
|
||||
doDiffUnderProw(cmd, args)
|
||||
} else {
|
||||
logrus.Fatalf("either base-profile or prow-postsubmit-job must be provided")
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
var (
|
||||
newProfile string
|
||||
baseProfile string
|
||||
coverageThreshold int
|
||||
|
||||
prowPostSubmitJob string
|
||||
prowProfile string
|
||||
|
||||
githubToken string
|
||||
githubUser string
|
||||
githubCommentPrefix string
|
||||
|
||||
qiniuCredential string
|
||||
|
||||
robotName string
|
||||
fullDiff bool
|
||||
)
|
||||
|
||||
func init() {
|
||||
diffCmd.Flags().StringVarP(&newProfile, "new-profile", "n", "", "local profile which works as the target to analysis")
|
||||
diffCmd.MarkFlagRequired("new-profile")
|
||||
diffCmd.Flags().StringVarP(&baseProfile, "base-profile", "b", "", "another local profile which works as baseline to compare with the target")
|
||||
diffCmd.Flags().IntVarP(&coverageThreshold, "coverage-threshold-percentage", "", 0, "coverage threshold percentage")
|
||||
diffCmd.Flags().StringVarP(&prowPostSubmitJob, "prow-postsubmit-job", "", "", "prow postsubmit job which used to find the base profile")
|
||||
diffCmd.Flags().StringVarP(&prowProfile, "prow-remote-profile-name", "", "filtered.cov", "the name of profile in prow postsubmit job, which used as the base profile to compare")
|
||||
diffCmd.Flags().StringVarP(&githubToken, "github-token", "", "/etc/github/oauth", "path to token to access github repo")
|
||||
diffCmd.Flags().StringVarP(&githubUser, "github-user", "", "", "github user name when comments in github")
|
||||
diffCmd.Flags().StringVarP(&githubCommentPrefix, "github-comment-prefix", "", "", "specific comment flag you provided")
|
||||
diffCmd.Flags().StringVarP(&qiniuCredential, "qiniu-credential", "", "/etc/qiniuconfig/qiniu.json", "path to credential file to access qiniu cloud")
|
||||
diffCmd.Flags().StringVarP(&robotName, "robot-name", "", "qiniu-bot", "github user name for coverage robot")
|
||||
diffCmd.Flags().BoolVarP(&fullDiff, "full-diff", "", false, "when set true,calculate and display full diff coverage between new-profile and base-profile")
|
||||
|
||||
rootCmd.AddCommand(diffCmd)
|
||||
}
|
||||
|
||||
//goc diff --new-profile=./new.cov --base-profile=./base.cov
|
||||
//+------------------------------------------------------+---------------+--------------+--------+
|
||||
//| File | Base Coverage | New Coverage | Delta |
|
||||
//+------------------------------------------------------+---------------+--------------+--------+
|
||||
//| qiniu.com/kodo/bd/pfd/pfdstg/cursor/mgr.go | 53.5% | 50.5% | -3.0% |
|
||||
//| qiniu.com/kodo/bd/pfd/pfdstg/svr/getstripe.go | 0.5% | 0.0% | -0.5% |
|
||||
//| Total | 35.7% | 35.7% | -0.0% |
|
||||
//+------------------------------------------------------+---------------+--------------+--------+
|
||||
func doDiffForLocalProfiles(cmd *cobra.Command, args []string) {
|
||||
localP, err := cover.ReadFileToCoverList(newProfile)
|
||||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
|
||||
baseP, err := cover.ReadFileToCoverList(baseProfile)
|
||||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
|
||||
//calculate diff file cov and display
|
||||
rows := cover.GetDeltaCov(localP, baseP)
|
||||
rows.Sort()
|
||||
table := tablewriter.NewWriter(os.Stdout)
|
||||
table.SetHeader([]string{"File", "Base Coverage", "New Coverage", "Delta"})
|
||||
table.SetAutoFormatHeaders(false)
|
||||
table.SetColumnAlignment([]int{tablewriter.ALIGN_LEFT, tablewriter.ALIGN_CENTER, tablewriter.ALIGN_CENTER, tablewriter.ALIGN_CENTER})
|
||||
for _, row := range rows {
|
||||
table.Append([]string{row.FileName, row.BasePer, row.NewPer, row.DeltaPer})
|
||||
}
|
||||
totalDelta := cover.PercentStr(cover.TotalDelta(*localP, *baseP))
|
||||
table.Append([]string{"Total", baseP.TotalPercentage(), localP.TotalPercentage(), totalDelta})
|
||||
table.Render()
|
||||
}
|
||||
|
||||
func doDiffUnderProw(cmd *cobra.Command, args []string) {
|
||||
var (
|
||||
prNumStr = os.Getenv("PULL_NUMBER")
|
||||
pullSha = os.Getenv("PULL_PULL_SHA")
|
||||
baseSha = os.Getenv("PULL_BASE_SHA")
|
||||
repoOwner = os.Getenv("REPO_OWNER")
|
||||
repoName = os.Getenv("REPO_NAME")
|
||||
jobType = os.Getenv("JOB_TYPE")
|
||||
jobName = os.Getenv("JOB_NAME")
|
||||
buildStr = os.Getenv("BUILD_NUMBER")
|
||||
artifacts = os.Getenv("ARTIFACTS")
|
||||
)
|
||||
logrus.Printf("Running coverage for PR = %s; PR commit SHA = %s;base SHA = %s", prNumStr, pullSha, baseSha)
|
||||
|
||||
switch jobType {
|
||||
case "periodic":
|
||||
logrus.Printf("job type %s, do nothing", jobType)
|
||||
case "postsubmit":
|
||||
logrus.Printf("job type %s, do nothing", jobType)
|
||||
case "presubmit":
|
||||
if githubToken == "" {
|
||||
logrus.Fatalf("github token not provided")
|
||||
}
|
||||
prClient := github.NewPrClient(githubToken, repoOwner, repoName, prNumStr, robotName, githubCommentPrefix)
|
||||
|
||||
if qiniuCredential == "" {
|
||||
logrus.Fatalf("qiniu credential not provided")
|
||||
}
|
||||
var qc *qiniu.Client
|
||||
var conf qiniu.Config
|
||||
files, err := ioutil.ReadFile(*&qiniuCredential)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatal("Error reading qiniu config file")
|
||||
}
|
||||
if err := json.Unmarshal(files, &conf); err != nil {
|
||||
logrus.Fatal("Error unmarshal qiniu config file")
|
||||
}
|
||||
if conf.Bucket == "" {
|
||||
logrus.Fatal("no qiniu bucket provided")
|
||||
}
|
||||
if conf.AccessKey == "" || conf.SecretKey == "" {
|
||||
logrus.Fatal("either qiniu access key or secret key was not provided")
|
||||
}
|
||||
if conf.Domain == "" {
|
||||
logrus.Fatal("no qiniu bucket domain was provided")
|
||||
}
|
||||
qc = qiniu.NewClient(&conf)
|
||||
|
||||
localArtifacts := qiniu.Artifacts{
|
||||
Directory: artifacts,
|
||||
ProfileName: newProfile,
|
||||
ChangedProfileName: qiniu.ChangedProfileName,
|
||||
}
|
||||
|
||||
job := prow.Job{
|
||||
JobName: jobName,
|
||||
BuildId: buildStr,
|
||||
Org: repoOwner,
|
||||
RepoName: repoName,
|
||||
PRNumStr: prNumStr,
|
||||
PostSubmitJob: prowPostSubmitJob,
|
||||
LocalProfilePath: newProfile,
|
||||
PostSubmitCoverProfile: prowProfile,
|
||||
QiniuClient: qc,
|
||||
LocalArtifacts: &localArtifacts,
|
||||
GithubComment: prClient,
|
||||
FullDiff: fullDiff,
|
||||
}
|
||||
if err := job.RunPresubmit(); err != nil {
|
||||
logrus.Fatalf("run presubmit job failed, err: %v", err)
|
||||
}
|
||||
default:
|
||||
logrus.Printf("Unknown job type: %s, do nothing.", jobType)
|
||||
}
|
||||
}
|
||||
11
go.mod
11
go.mod
@ -4,11 +4,20 @@ go 1.13
|
||||
|
||||
require (
|
||||
github.com/gin-gonic/gin v1.6.3
|
||||
github.com/google/go-github v17.0.0+incompatible
|
||||
github.com/julienschmidt/httprouter v1.2.0
|
||||
github.com/mattn/go-runewidth v0.0.9 // indirect
|
||||
github.com/olekukonko/tablewriter v0.0.4
|
||||
github.com/onsi/ginkgo v1.11.0
|
||||
github.com/onsi/gomega v1.8.1
|
||||
github.com/otiai10/copy v1.0.2
|
||||
github.com/qiniu/api.v7/v7 v7.5.0
|
||||
github.com/sirupsen/logrus v1.4.2
|
||||
github.com/spf13/cobra v1.0.0
|
||||
github.com/stretchr/testify v1.5.1
|
||||
golang.org/x/tools v0.0.0-20200303214625-2b0b585e22fe
|
||||
golang.org/x/net v0.0.0-20200301022130-244492dfa37a
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
|
||||
golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65
|
||||
k8s.io/kubernetes v1.13.0
|
||||
k8s.io/test-infra v0.0.0-20200511080351-8ac9dbfab055
|
||||
)
|
||||
|
||||
29
go.sum
29
go.sum
@ -221,6 +221,7 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI
|
||||
github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
|
||||
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
|
||||
github.com/frankban/quicktest v1.8.1/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsouza/fake-gcs-server v0.0.0-20180612165233-e85be23bdaa8/go.mod h1:1/HufuJ+eaDf4KTnYdS6HJMGvMRU8d4cYTuu/1QaBbI=
|
||||
github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
|
||||
@ -290,6 +291,7 @@ github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfT
|
||||
github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
|
||||
github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
|
||||
github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
|
||||
github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
|
||||
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
|
||||
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
|
||||
@ -343,8 +345,10 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-containerregistry v0.0.0-20200115214256-379933c9c22b/go.mod h1:Wtl/v6YdQxv397EREtzwgd9+Ud7Q5D8XMbi3Zazgkrs=
|
||||
github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
|
||||
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
|
||||
github.com/google/go-licenses v0.0.0-20191112164736-212ea350c932/go.mod h1:16wa6pRqNDUIhOtwF0GcROVqMeXHZJ7H6eGDFUh5Pfk=
|
||||
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/go-replayers/grpcreplay v0.1.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE=
|
||||
github.com/google/go-replayers/httpreplay v0.1.0/go.mod h1:YKZViNhiGgqdBlUbI2MwGpq4pXxNmhJLPHQ7cv2b5no=
|
||||
@ -435,6 +439,7 @@ github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGn
|
||||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
|
||||
github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
|
||||
@ -447,13 +452,16 @@ github.com/klauspost/compress v1.10.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs
|
||||
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
|
||||
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
|
||||
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
|
||||
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
|
||||
@ -482,7 +490,11 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA
|
||||
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0=
|
||||
github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
|
||||
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/mattn/go-shellwords v1.0.9/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
|
||||
github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo=
|
||||
@ -520,7 +532,10 @@ github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uY
|
||||
github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
|
||||
github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5 h1:58+kh9C6jJVXYjt8IE48G2eWl6BjwU5Gj0gqY84fy78=
|
||||
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
|
||||
github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8=
|
||||
github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
|
||||
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
@ -547,6 +562,7 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ
|
||||
github.com/openzipkin/zipkin-go v0.2.0/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
|
||||
github.com/otiai10/copy v1.0.2 h1:DDNipYy6RkIkjMwy+AWzgKiNTyj2RUI9yEMeETEpVyc=
|
||||
github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY=
|
||||
github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95 h1:+OLn68pqasWca0z5ryit9KGfp3sUsW4Lqg32iRMJyzs=
|
||||
github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
|
||||
github.com/otiai10/mint v1.3.0 h1:Ady6MKVezQwHBkGzLFbrsywyp09Ah7rkmfjV3Bcr5uc=
|
||||
github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
|
||||
@ -603,6 +619,8 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa
|
||||
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
|
||||
github.com/prometheus/procfs v0.0.10/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
|
||||
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
||||
github.com/qiniu/api.v7/v7 v7.5.0 h1:DY6NrIp6FZ1GP4Roc9hRnO2m+OLzASYNnvz5Mbgw1rk=
|
||||
github.com/qiniu/api.v7/v7 v7.5.0/go.mod h1:VE5oC5rkE1xul0u1S2N0b2Uxq9/6hZzhyqjgK25XDcM=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20190706150252-9beb055b7962/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
|
||||
@ -622,6 +640,7 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV
|
||||
github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
|
||||
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
@ -695,6 +714,7 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMx
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
|
||||
github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
|
||||
github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
|
||||
@ -746,6 +766,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
|
||||
golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM=
|
||||
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
@ -809,6 +830,7 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@ -905,6 +927,8 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapK
|
||||
golang.org/x/tools v0.0.0-20200214144324-88be01311a71/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200303214625-2b0b585e22fe h1:Kh3iY7o/2bMfQXZdwLdL9jDMU1k9HoVn0P1mGCfoFLc=
|
||||
golang.org/x/tools v0.0.0-20200303214625-2b0b585e22fe/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
|
||||
golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65 h1:1KSbntBked74wYsKq0jzXYy7ZwcjAUtrl7EmPE97Iiw=
|
||||
golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
@ -930,6 +954,7 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
|
||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
@ -965,6 +990,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
@ -1017,6 +1043,7 @@ k8s.io/apiextensions-apiserver v0.17.2/go.mod h1:4KdMpjkEjjDI2pPfBA15OscyNldHWdB
|
||||
k8s.io/apimachinery v0.0.0-20190703205208-4cfb76a8bf76/go.mod h1:M2fZgZL9DbLfeJaPBCDqSqNsdsmLN+V29knYJnIXlMA=
|
||||
k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
|
||||
k8s.io/apimachinery v0.17.2/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
|
||||
k8s.io/apimachinery v0.17.3 h1:f+uZV6rm4/tHE7xXgLyToprg6xWairaClGVkm2t8omg=
|
||||
k8s.io/apimachinery v0.17.3/go.mod h1:gxLnyZcGNdZTCLnq3fgzyg2A5BVCHTNDFrw8AmuJ+0g=
|
||||
k8s.io/apiserver v0.17.0/go.mod h1:ABM+9x/prjINN6iiffRVNCBR2Wk7uY4z+EtEGZD48cg=
|
||||
k8s.io/apiserver v0.17.2/go.mod h1:lBmw/TtQdtxvrTk0e2cgtOxHizXI+d0mmGQURIHQZlo=
|
||||
@ -1039,10 +1066,12 @@ k8s.io/gengo v0.0.0-20191108084044-e500ee069b5c/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8
|
||||
k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
|
||||
k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
|
||||
k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
|
||||
k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
|
||||
k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
|
||||
k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
|
||||
k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
|
||||
k8s.io/kubectl v0.17.2/go.mod h1:y4rfLV0n6aPmvbRCqZQjvOp3ezxsFgpqL+zF5jH/lxk=
|
||||
k8s.io/kubernetes v1.13.0 h1:qTfB+u5M92k2fCCCVP2iuhgwwSOv1EkAkvQY1tQODD8=
|
||||
k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
|
||||
k8s.io/legacy-cloud-providers v0.17.0/go.mod h1:DdzaepJ3RtRy+e5YhNtrCYwlgyK87j/5+Yfp0L9Syp8=
|
||||
k8s.io/metrics v0.17.2/go.mod h1:3TkNHET4ROd+NfzNxkjoVfQ0Ob4iZnaHmSEA4vYpwLw=
|
||||
|
||||
@ -23,13 +23,17 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// TestCover is a collection of all counters
|
||||
@ -251,12 +255,7 @@ func AddCacheCover(pkg *Package, in *PackageCover) *PackageCover {
|
||||
}
|
||||
|
||||
// CoverageList is a collection and summary over multiple file Coverage objects
|
||||
type CoverageList struct {
|
||||
*Coverage
|
||||
Groups []Coverage
|
||||
ConcernedFiles map[string]bool
|
||||
CovThresholdInt int
|
||||
}
|
||||
type CoverageList []Coverage
|
||||
|
||||
// Coverage stores test coverage summary data for one file
|
||||
type Coverage struct {
|
||||
@ -272,10 +271,11 @@ type codeBlock struct {
|
||||
coverageCount int // number of times the block is covered
|
||||
}
|
||||
|
||||
//convert profile to CoverageList struct
|
||||
func CovList(f io.Reader) (g *CoverageList, err error) {
|
||||
scanner := bufio.NewScanner(f)
|
||||
scanner.Scan() // discard first line
|
||||
g = NewCoverageList("", map[string]bool{}, 0)
|
||||
g = NewCoverageList()
|
||||
|
||||
for scanner.Scan() {
|
||||
row := scanner.Text()
|
||||
@ -288,14 +288,21 @@ func CovList(f io.Reader) (g *CoverageList, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
// NewCoverageList constructs new (file) group Coverage
|
||||
func NewCoverageList(name string, concernedFiles map[string]bool, covThresholdInt int) *CoverageList {
|
||||
return &CoverageList{
|
||||
Coverage: newCoverage(name),
|
||||
Groups: []Coverage{},
|
||||
ConcernedFiles: concernedFiles,
|
||||
CovThresholdInt: covThresholdInt,
|
||||
// covert profile file to CoverageList struct
|
||||
func ReadFileToCoverList(path string) (g *CoverageList, err error) {
|
||||
f, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
logrus.Errorf("Open file %s failed!", path)
|
||||
return nil, err
|
||||
}
|
||||
g, err = CovList(bytes.NewReader(f))
|
||||
return
|
||||
}
|
||||
|
||||
// NewCoverageList return empty CoverageList
|
||||
func NewCoverageList() *CoverageList {
|
||||
return &CoverageList{}
|
||||
|
||||
}
|
||||
|
||||
func newCoverage(name string) *Coverage {
|
||||
@ -333,27 +340,46 @@ func (blk *codeBlock) addToGroupCov(g *CoverageList) {
|
||||
}
|
||||
|
||||
func (g *CoverageList) size() int {
|
||||
return len(g.Groups)
|
||||
return len(*g)
|
||||
}
|
||||
|
||||
func (g *CoverageList) lastElement() *Coverage {
|
||||
return &g.Groups[g.size()-1]
|
||||
return &(*g)[(*g).size()-1]
|
||||
}
|
||||
|
||||
func (g *CoverageList) append(c *Coverage) {
|
||||
g.Groups = append(g.Groups, *c)
|
||||
*g = append(*g, *c)
|
||||
}
|
||||
|
||||
// Group returns the collection of file Coverage objects
|
||||
func (g *CoverageList) Group() *[]Coverage {
|
||||
return &g.Groups
|
||||
// sort CoverageList with filenames
|
||||
func (g *CoverageList) Sort() {
|
||||
sort.SliceStable(g, func(i, j int) bool {
|
||||
return (*g)[i].Name() < (*g)[j].Name()
|
||||
})
|
||||
}
|
||||
|
||||
func (g *CoverageList) TotalPercentage() string {
|
||||
ratio, err := g.TotalRatio()
|
||||
if err == nil {
|
||||
return PercentStr(ratio)
|
||||
}
|
||||
return "N/A"
|
||||
}
|
||||
|
||||
func (g *CoverageList) TotalRatio() (ratio float32, err error) {
|
||||
var total Coverage
|
||||
for _, c := range *g {
|
||||
total.NCoveredStmts += c.NCoveredStmts
|
||||
total.NAllStmts += c.NAllStmts
|
||||
}
|
||||
return total.Ratio()
|
||||
}
|
||||
|
||||
// Map returns maps the file name to its coverage for faster retrieval
|
||||
// & membership check
|
||||
func (g *CoverageList) Map() map[string]Coverage {
|
||||
m := make(map[string]Coverage)
|
||||
for _, c := range g.Groups {
|
||||
for _, c := range *g {
|
||||
m[c.Name()] = c
|
||||
}
|
||||
return m
|
||||
@ -370,7 +396,6 @@ func (c *Coverage) Percentage() string {
|
||||
if err == nil {
|
||||
return PercentStr(ratio)
|
||||
}
|
||||
|
||||
return "N/A"
|
||||
}
|
||||
|
||||
|
||||
@ -50,56 +50,69 @@ func TestPercentageNA(t *testing.T) {
|
||||
assert.Equal(t, "N/A", c.Percentage())
|
||||
}
|
||||
|
||||
func TestGenLocalCoverDiffReport(t *testing.T) {
|
||||
//coverage increase
|
||||
newList := &CoverageList{Groups: []Coverage{{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20}}}
|
||||
baseList := &CoverageList{Groups: []Coverage{{FileName: "fake-coverage", NCoveredStmts: 10, NAllStmts: 20}}}
|
||||
rows := GenLocalCoverDiffReport(newList, baseList)
|
||||
assert.Equal(t, 1, len(rows))
|
||||
assert.Equal(t, []string{"fake-coverage", "50.0%", "75.0%", "25.0%"}, rows[0])
|
||||
|
||||
//coverage decrease
|
||||
baseList = &CoverageList{Groups: []Coverage{{FileName: "fake-coverage", NCoveredStmts: 20, NAllStmts: 20}}}
|
||||
rows = GenLocalCoverDiffReport(newList, baseList)
|
||||
assert.Equal(t, []string{"fake-coverage", "100.0%", "75.0%", "-25.0%"}, rows[0])
|
||||
|
||||
//diff file
|
||||
baseList = &CoverageList{Groups: []Coverage{{FileName: "fake-coverage-v1", NCoveredStmts: 10, NAllStmts: 20}}}
|
||||
rows = GenLocalCoverDiffReport(newList, baseList)
|
||||
assert.Equal(t, []string{"fake-coverage", "None", "75.0%", "75.0%"}, rows[0])
|
||||
}
|
||||
|
||||
func TestCovList(t *testing.T) {
|
||||
fileName := "qiniu.com/kodo/apiserver/server/main.go"
|
||||
|
||||
// percentage is 100%
|
||||
p := strings.NewReader("mode: atomic\n" +
|
||||
fileName + ":32.49,33.13 1 30\n")
|
||||
covL, err := CovList(p)
|
||||
covF := covL.Map()[fileName]
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "100.0%", covF.Percentage())
|
||||
|
||||
// percentage is 50%
|
||||
p = strings.NewReader("mode: atomic\n" +
|
||||
fileName + ":32.49,33.13 1 30\n" +
|
||||
fileName + ":42.49,43.13 1 0\n")
|
||||
covL, err = CovList(p)
|
||||
covF = covL.Map()[fileName]
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "50.0%", covF.Percentage())
|
||||
|
||||
// two files
|
||||
fileName1 := "qiniu.com/kodo/apiserver/server/svr.go"
|
||||
p = strings.NewReader("mode: atomic\n" +
|
||||
fileName + ":32.49,33.13 1 30\n" +
|
||||
fileName1 + ":42.49,43.13 1 0\n")
|
||||
covL, err = CovList(p)
|
||||
covF = covL.Map()[fileName]
|
||||
covF1 := covL.Map()[fileName1]
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "100.0%", covF.Percentage())
|
||||
assert.Equal(t, "0.0%", covF1.Percentage())
|
||||
|
||||
items := []struct {
|
||||
profile string
|
||||
expectPer []string
|
||||
}{
|
||||
// percentage is 100%
|
||||
{
|
||||
profile: "mode: atomic\n" +
|
||||
fileName + ":32.49,33.13 1 30\n",
|
||||
expectPer: []string{"100.0%"},
|
||||
},
|
||||
// percentage is 50%
|
||||
{profile: "mode: atomic\n" +
|
||||
fileName + ":32.49,33.13 1 30\n" +
|
||||
fileName + ":42.49,43.13 1 0\n",
|
||||
expectPer: []string{"50.0%"},
|
||||
},
|
||||
// two files
|
||||
{
|
||||
profile: "mode: atomic\n" +
|
||||
fileName + ":32.49,33.13 1 30\n" +
|
||||
fileName1 + ":42.49,43.13 1 0\n",
|
||||
expectPer: []string{"100.0%", "0.0%"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range items {
|
||||
r := strings.NewReader(tc.profile)
|
||||
c, err := CovList(r)
|
||||
c.Sort()
|
||||
assert.Nil(t, err)
|
||||
for k, v := range *c {
|
||||
assert.Equal(t, tc.expectPer[k], v.Percentage())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTotalPercentage(t *testing.T) {
|
||||
items := []struct {
|
||||
list CoverageList
|
||||
expectPer string
|
||||
}{
|
||||
{
|
||||
list: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 0}},
|
||||
expectPer: "N/A",
|
||||
},
|
||||
{
|
||||
list: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20}},
|
||||
expectPer: "75.0%",
|
||||
},
|
||||
{
|
||||
list: CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20},
|
||||
Coverage{FileName: "fake-coverage-1", NCoveredStmts: 10, NAllStmts: 30}},
|
||||
expectPer: "50.0%",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range items {
|
||||
assert.Equal(t, tc.expectPer, tc.list.TotalPercentage())
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildCoverCmd(t *testing.T) {
|
||||
|
||||
@ -16,41 +16,117 @@
|
||||
|
||||
package cover
|
||||
|
||||
type GroupChanges struct {
|
||||
Added []Coverage
|
||||
Deleted []Coverage
|
||||
Unchanged []Coverage
|
||||
Changed []Incremental
|
||||
BaseGroup *CoverageList
|
||||
NewGroup *CoverageList
|
||||
import "sort"
|
||||
|
||||
type DeltaCov struct {
|
||||
FileName string
|
||||
BasePer string
|
||||
NewPer string
|
||||
DeltaPer string
|
||||
LineCovLink string
|
||||
}
|
||||
|
||||
type Incremental struct {
|
||||
base Coverage
|
||||
new Coverage
|
||||
}
|
||||
type DeltaCovList []DeltaCov
|
||||
|
||||
func GenLocalCoverDiffReport(newList *CoverageList, baseList *CoverageList) [][]string {
|
||||
var rows [][]string
|
||||
basePMap := baseList.Map()
|
||||
// get full delta coverage between new and base profile
|
||||
func GetFullDeltaCov(newList *CoverageList, baseList *CoverageList) (delta DeltaCovList) {
|
||||
newMap := newList.Map()
|
||||
baseMap := baseList.Map()
|
||||
|
||||
for _, l := range newList.Groups {
|
||||
baseCov, ok := basePMap[l.Name()]
|
||||
for file, n := range newMap {
|
||||
b, ok := baseMap[file]
|
||||
//if the file not in base profile, set None
|
||||
if !ok {
|
||||
rows = append(rows, []string{l.FileName, "None", l.Percentage(), PercentStr(Delta(l, baseCov))})
|
||||
delta = append(delta, DeltaCov{
|
||||
FileName: file,
|
||||
BasePer: "None",
|
||||
NewPer: n.Percentage(),
|
||||
DeltaPer: PercentStr(Delta(n, b))})
|
||||
continue
|
||||
}
|
||||
if l.Percentage() == baseCov.Percentage() {
|
||||
continue
|
||||
}
|
||||
rows = append(rows, []string{l.FileName, baseCov.Percentage(), l.Percentage(), PercentStr(Delta(l, baseCov))})
|
||||
delta = append(delta, DeltaCov{
|
||||
FileName: file,
|
||||
BasePer: b.Percentage(),
|
||||
NewPer: n.Percentage(),
|
||||
DeltaPer: PercentStr(Delta(n, b))})
|
||||
}
|
||||
|
||||
return rows
|
||||
for file, b := range baseMap {
|
||||
//if the file not in new profile, set None
|
||||
if n, ok := newMap[file]; !ok {
|
||||
delta = append(delta, DeltaCov{
|
||||
FileName: file,
|
||||
BasePer: b.Percentage(),
|
||||
NewPer: "None",
|
||||
DeltaPer: PercentStr(Delta(n, b))})
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
//get two profile diff cov
|
||||
func GetDeltaCov(newList *CoverageList, baseList *CoverageList) (delta DeltaCovList) {
|
||||
d := GetFullDeltaCov(newList, baseList)
|
||||
for _, v := range d {
|
||||
if v.DeltaPer == "0.0%" {
|
||||
continue
|
||||
}
|
||||
delta = append(delta, v)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
//get two profile diff cov of changed files
|
||||
func GetChFileDeltaCov(newList *CoverageList, baseList *CoverageList, changedFiles []string) (list DeltaCovList) {
|
||||
d := GetFullDeltaCov(newList, baseList)
|
||||
dMap := d.Map()
|
||||
for _, file := range changedFiles {
|
||||
if _, ok := dMap[file]; ok {
|
||||
list = append(list, dMap[file])
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
//calculate two coverage delta
|
||||
func Delta(new Coverage, base Coverage) float32 {
|
||||
baseRatio, _ := base.Ratio()
|
||||
newRatio, _ := new.Ratio()
|
||||
return newRatio - baseRatio
|
||||
}
|
||||
|
||||
//calculate two coverage delta
|
||||
func TotalDelta(new CoverageList, base CoverageList) float32 {
|
||||
baseRatio, _ := base.TotalRatio()
|
||||
newRatio, _ := new.TotalRatio()
|
||||
return newRatio - baseRatio
|
||||
}
|
||||
|
||||
// Map returns maps the file name to its DeltaCov for faster retrieval & membership check
|
||||
func (d *DeltaCovList) Map() map[string]DeltaCov {
|
||||
m := make(map[string]DeltaCov)
|
||||
for _, c := range *d {
|
||||
m[c.FileName] = c
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// sort DeltaCovList with filenames
|
||||
func (d *DeltaCovList) Sort() {
|
||||
sort.SliceStable(d, func(i, j int) bool {
|
||||
return (*d)[i].Name() < (*d)[j].Name()
|
||||
})
|
||||
}
|
||||
|
||||
// Name returns the file name
|
||||
func (c *DeltaCov) Name() string {
|
||||
return c.FileName
|
||||
}
|
||||
|
||||
func (c *DeltaCov) GetLineCovLink() string {
|
||||
return c.LineCovLink
|
||||
}
|
||||
|
||||
func (c *DeltaCov) SetLineCovLink(link string) {
|
||||
c.LineCovLink = link
|
||||
}
|
||||
|
||||
133
pkg/cover/delta_test.go
Normal file
133
pkg/cover/delta_test.go
Normal file
@ -0,0 +1,133 @@
|
||||
/*
|
||||
Copyright 2020 Qiniu Cloud (qiniu.com)
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package cover
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGetDeltaCov(t *testing.T) {
|
||||
items := []struct {
|
||||
newList *CoverageList
|
||||
baseList *CoverageList
|
||||
expectDelta DeltaCovList
|
||||
rows int
|
||||
}{
|
||||
//coverage increase
|
||||
{
|
||||
newList: &CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20}},
|
||||
baseList: &CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 10, NAllStmts: 20}},
|
||||
expectDelta: DeltaCovList{{FileName: "fake-coverage", BasePer: "50.0%", NewPer: "75.0%", DeltaPer: "25.0%"}},
|
||||
rows: 1,
|
||||
},
|
||||
//coverage decrease
|
||||
{
|
||||
newList: &CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20}},
|
||||
baseList: &CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 20, NAllStmts: 20}},
|
||||
expectDelta: DeltaCovList{{FileName: "fake-coverage", BasePer: "100.0%", NewPer: "75.0%", DeltaPer: "-25.0%"}},
|
||||
rows: 1,
|
||||
},
|
||||
//diff file
|
||||
{
|
||||
newList: &CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20}},
|
||||
baseList: &CoverageList{Coverage{FileName: "fake-coverage-v1", NCoveredStmts: 10, NAllStmts: 20}},
|
||||
expectDelta: DeltaCovList{{FileName: "fake-coverage", BasePer: "None", NewPer: "75.0%", DeltaPer: "75.0%"},
|
||||
{FileName: "fake-coverage-v1", BasePer: "50.0%", NewPer: "None", DeltaPer: "-50.0%"}},
|
||||
rows: 2,
|
||||
},
|
||||
//one file has same coverage rate
|
||||
{
|
||||
newList: &CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20}},
|
||||
baseList: &CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20},
|
||||
Coverage{FileName: "fake-coverage-v1", NCoveredStmts: 10, NAllStmts: 20}},
|
||||
expectDelta: DeltaCovList{{FileName: "fake-coverage-v1", BasePer: "50.0%", NewPer: "None", DeltaPer: "-50.0%"}},
|
||||
rows: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range items {
|
||||
d := GetDeltaCov(tc.newList, tc.baseList)
|
||||
assert.Equal(t, tc.rows, len(d))
|
||||
assert.Equal(t, tc.expectDelta, d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetChFileDeltaCov(t *testing.T) {
|
||||
items := []struct {
|
||||
newList *CoverageList
|
||||
baseList *CoverageList
|
||||
changedFiles []string
|
||||
expectDelta DeltaCovList
|
||||
}{
|
||||
{
|
||||
newList: &CoverageList{Coverage{FileName: "fake-coverage", NCoveredStmts: 15, NAllStmts: 20}},
|
||||
baseList: &CoverageList{Coverage{FileName: "fake-coverage-v1", NCoveredStmts: 10, NAllStmts: 20}},
|
||||
changedFiles: []string{"fake-coverage"},
|
||||
expectDelta: DeltaCovList{{FileName: "fake-coverage", BasePer: "None", NewPer: "75.0%", DeltaPer: "75.0%"}},
|
||||
},
|
||||
}
|
||||
for _, tc := range items {
|
||||
d := GetChFileDeltaCov(tc.newList, tc.baseList, tc.changedFiles)
|
||||
assert.Equal(t, tc.expectDelta, d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapAndSort(t *testing.T) {
|
||||
items := []struct {
|
||||
dList DeltaCovList
|
||||
expectMap map[string]DeltaCov
|
||||
expectSort DeltaCovList
|
||||
}{
|
||||
{
|
||||
dList: DeltaCovList{DeltaCov{FileName: "b", BasePer: "10.0%", NewPer: "20.0%", DeltaPer: "10.0%"},
|
||||
DeltaCov{FileName: "a", BasePer: "10.0%", NewPer: "30.0%", DeltaPer: "20.0%"},
|
||||
},
|
||||
expectMap: map[string]DeltaCov{
|
||||
"a": {FileName: "a", BasePer: "10.0%", NewPer: "30.0%", DeltaPer: "20.0%"},
|
||||
"b": {FileName: "b", BasePer: "10.0%", NewPer: "20.0%", DeltaPer: "10.0%"},
|
||||
},
|
||||
expectSort: DeltaCovList{DeltaCov{FileName: "a", BasePer: "10.0%", NewPer: "30.0%", DeltaPer: "20.0%"},
|
||||
DeltaCov{FileName: "b", BasePer: "10.0%", NewPer: "20.0%", DeltaPer: "10.0%"},
|
||||
},
|
||||
},
|
||||
{
|
||||
dList: DeltaCovList{DeltaCov{FileName: "b", BasePer: "10.0%", NewPer: "20.0%", DeltaPer: "10.0%"},
|
||||
DeltaCov{FileName: "b-1", BasePer: "10.0%", NewPer: "30.0%", DeltaPer: "20.0%"},
|
||||
DeltaCov{FileName: "1-b", BasePer: "10.0%", NewPer: "40.0%", DeltaPer: "30.0%"},
|
||||
},
|
||||
expectMap: map[string]DeltaCov{
|
||||
"1-b": {FileName: "1-b", BasePer: "10.0%", NewPer: "40.0%", DeltaPer: "30.0%"},
|
||||
"b": {FileName: "b", BasePer: "10.0%", NewPer: "20.0%", DeltaPer: "10.0%"},
|
||||
"b-1": {FileName: "b-1", BasePer: "10.0%", NewPer: "30.0%", DeltaPer: "20.0%"},
|
||||
},
|
||||
expectSort: DeltaCovList{DeltaCov{FileName: "1-b", BasePer: "10.0%", NewPer: "40.0%", DeltaPer: "30.0%"},
|
||||
DeltaCov{FileName: "b", BasePer: "10.0%", NewPer: "20.0%", DeltaPer: "10.0%"},
|
||||
DeltaCov{FileName: "b-1", BasePer: "10.0%", NewPer: "30.0%", DeltaPer: "20.0%"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range items {
|
||||
assert.True(t, reflect.DeepEqual(tc.expectMap, tc.dList.Map()))
|
||||
tc.dList.Sort()
|
||||
assert.True(t, reflect.DeepEqual(tc.expectSort, tc.dList))
|
||||
}
|
||||
|
||||
}
|
||||
179
pkg/github/github.go
Normal file
179
pkg/github/github.go
Normal file
@ -0,0 +1,179 @@
|
||||
/*
|
||||
Copyright 2020 Qiniu Cloud (qiniu.com)
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package github
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/google/go-github/github"
|
||||
"github.com/olekukonko/tablewriter"
|
||||
"github.com/sirupsen/logrus"
|
||||
"golang.org/x/oauth2"
|
||||
|
||||
"github.com/qiniu/goc/pkg/cover"
|
||||
)
|
||||
|
||||
const CommentsPrefix = "The following is the coverage report on the affected files."
|
||||
|
||||
type PrComment struct {
|
||||
RobotUserName string
|
||||
RepoOwner string
|
||||
RepoName string
|
||||
CommentFlag string
|
||||
PrNumber int
|
||||
Ctx context.Context
|
||||
opt *github.ListOptions
|
||||
GithubClient *github.Client
|
||||
}
|
||||
|
||||
func NewPrClient(githubTokenPath, repoOwner, repoName, prNumStr, botUserName, commentFlag string) *PrComment {
|
||||
var client *github.Client
|
||||
var ctx = context.Background()
|
||||
|
||||
prNum, err := strconv.Atoi(prNumStr)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatalf("Failed to convert prNumStr(=%v) to int.\n", prNumStr)
|
||||
}
|
||||
token, err := ioutil.ReadFile(githubTokenPath)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatalf("Failed to get github token.\n")
|
||||
}
|
||||
ts := oauth2.StaticTokenSource(
|
||||
&oauth2.Token{AccessToken: strings.TrimSpace(string(token))},
|
||||
)
|
||||
tc := oauth2.NewClient(ctx, ts)
|
||||
client = github.NewClient(tc)
|
||||
|
||||
return &PrComment{
|
||||
RobotUserName: botUserName,
|
||||
RepoOwner: repoOwner,
|
||||
RepoName: repoName,
|
||||
PrNumber: prNum,
|
||||
CommentFlag: commentFlag,
|
||||
Ctx: ctx,
|
||||
opt: &github.ListOptions{Page: 1},
|
||||
GithubClient: client,
|
||||
}
|
||||
}
|
||||
|
||||
//post github comment of diff coverage
|
||||
func (c *PrComment) CreateGithubComment(commentPrefix string, diffCovList cover.DeltaCovList) (err error) {
|
||||
if len(diffCovList) == 0 {
|
||||
logrus.Printf("Detect 0 files coverage diff, will not comment to github.")
|
||||
return nil
|
||||
}
|
||||
content := GenCommentContent(commentPrefix, diffCovList)
|
||||
|
||||
err = c.PostComment(content, commentPrefix)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatalf("Post comment to github failed.")
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (c *PrComment) PostComment(content, commentPrefix string) error {
|
||||
//step1: erase history similar comment to avoid too many comment for same job
|
||||
err := c.EraseHistoryComment(commentPrefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
//step2: post comment with new result
|
||||
comment := &github.IssueComment{
|
||||
Body: &content,
|
||||
}
|
||||
_, _, err = c.GithubClient.Issues.CreateComment(c.Ctx, c.RepoOwner, c.RepoName, c.PrNumber, comment)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// erase history similar comment before post again
|
||||
func (c *PrComment) EraseHistoryComment(commentPrefix string) error {
|
||||
comments, _, err := c.GithubClient.Issues.ListComments(c.Ctx, c.RepoOwner, c.RepoName, c.PrNumber, nil)
|
||||
if err != nil {
|
||||
logrus.Errorf("list PR comments failed.")
|
||||
return err
|
||||
}
|
||||
logrus.Infof("the count of history comments by %s is: %v", c.RobotUserName, len(comments))
|
||||
|
||||
for _, cm := range comments {
|
||||
if *cm.GetUser().Login == c.RobotUserName && strings.HasPrefix(cm.GetBody(), commentPrefix) {
|
||||
_, err = c.GithubClient.Issues.DeleteComment(c.Ctx, c.RepoOwner, c.RepoName, *cm.ID)
|
||||
if err != nil {
|
||||
logrus.Errorf("delete PR comments %d failed.", *cm.ID)
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
//get github pull request changes file list
|
||||
func (c *PrComment) GetPrChangedFiles() (files []string, err error) {
|
||||
var commitFiles []*github.CommitFile
|
||||
for {
|
||||
f, resp, err := c.GithubClient.PullRequests.ListFiles(c.Ctx, c.RepoOwner, c.RepoName, c.PrNumber, c.opt)
|
||||
if err != nil {
|
||||
logrus.Errorf("Get PR changed file failed. repoOwner is: %s, repoName is: %s, prNum is: %d", c.RepoOwner, c.RepoName, c.PrNumber)
|
||||
return nil, err
|
||||
}
|
||||
commitFiles = append(commitFiles, f...)
|
||||
if resp.NextPage == 0 {
|
||||
break
|
||||
}
|
||||
c.opt.Page = resp.NextPage
|
||||
}
|
||||
logrus.Infof("get %d PR changed files:", len(commitFiles))
|
||||
for _, file := range commitFiles {
|
||||
files = append(files, *file.Filename)
|
||||
logrus.Infof("%s", *file.Filename)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
//generate github comment content based on diff coverage and commentFlag
|
||||
func GenCommentContent(commentPrefix string, delta cover.DeltaCovList) string {
|
||||
var buf bytes.Buffer
|
||||
table := tablewriter.NewWriter(&buf)
|
||||
table.SetHeader([]string{"File", "BASE Coverage", "New Coverage", "Delta"})
|
||||
table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false})
|
||||
table.SetCenterSeparator("|")
|
||||
table.SetColumnAlignment([]int{tablewriter.ALIGN_LEFT, tablewriter.ALIGN_CENTER, tablewriter.ALIGN_CENTER, tablewriter.ALIGN_CENTER})
|
||||
for _, d := range delta {
|
||||
table.Append([]string{fmt.Sprintf("[%s](%s)", d.FileName, d.LineCovLink), d.BasePer, d.NewPer, d.DeltaPer})
|
||||
}
|
||||
table.Render()
|
||||
|
||||
content := []string{
|
||||
commentPrefix,
|
||||
fmt.Sprintf("Say `/test %s` to re-run this coverage report", os.Getenv("JOB_NAME")),
|
||||
buf.String(),
|
||||
}
|
||||
|
||||
return strings.Join(content, "\n")
|
||||
}
|
||||
165
pkg/github/github_test.go
Normal file
165
pkg/github/github_test.go
Normal file
@ -0,0 +1,165 @@
|
||||
/*
|
||||
Copyright 2020 Qiniu Cloud (qiniu.com)
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package github
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-github/github"
|
||||
"github.com/julienschmidt/httprouter"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"golang.org/x/net/context"
|
||||
|
||||
"github.com/qiniu/goc/pkg/cover"
|
||||
)
|
||||
|
||||
const (
|
||||
// baseURLPath is a non-empty Client.BaseURL path to use during tests,
|
||||
// to ensure relative URLs are used for all endpoints. See issue #752.
|
||||
baseURLPath = "/api-v3"
|
||||
)
|
||||
|
||||
// setup sets up a test HTTP server along with a github.Client that is
|
||||
// configured to talk to that test server. Tests should register handlers on
|
||||
// mux which provide mock responses for the API method being tested.
|
||||
func setup() (client *github.Client, router *httprouter.Router, serverURL string, teardown func()) {
|
||||
// router is the HTTP request multiplexer used with the test server.
|
||||
router = httprouter.New()
|
||||
|
||||
// We want to ensure that tests catch mistakes where the endpoint URL is
|
||||
// specified as absolute rather than relative. It only makes a difference
|
||||
// when there's a non-empty base URL path. So, use that. See issue #752.
|
||||
apiHandler := http.NewServeMux()
|
||||
apiHandler.Handle(baseURLPath+"/", http.StripPrefix(baseURLPath, router))
|
||||
apiHandler.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
|
||||
fmt.Fprintln(os.Stderr, "FAIL: Client.BaseURL path prefix is not preserved in the request URL:")
|
||||
fmt.Fprintln(os.Stderr)
|
||||
fmt.Fprintln(os.Stderr, "\t"+req.URL.String())
|
||||
fmt.Fprintln(os.Stderr)
|
||||
fmt.Fprintln(os.Stderr, "\tDid you accidentally use an absolute endpoint URL rather than relative?")
|
||||
fmt.Fprintln(os.Stderr, "\tSee https://github.com/google/go-github/issues/752 for information.")
|
||||
http.Error(w, "Client.BaseURL path prefix is not preserved in the request URL.", http.StatusInternalServerError)
|
||||
})
|
||||
|
||||
// server is a test HTTP server used to provide mock API responses.
|
||||
server := httptest.NewServer(apiHandler)
|
||||
|
||||
// client is the GitHub client being tested and is
|
||||
// configured to use test server.
|
||||
client = github.NewClient(nil)
|
||||
url, _ := url.Parse(server.URL + baseURLPath + "/")
|
||||
client.BaseURL = url
|
||||
client.UploadURL = url
|
||||
|
||||
return client, router, server.URL, server.Close
|
||||
}
|
||||
|
||||
func TestNewPrClient(t *testing.T) {
|
||||
items := []struct {
|
||||
token string
|
||||
repoOwner string
|
||||
repoName string
|
||||
prNumStr string
|
||||
botUserName string
|
||||
commentFlag string
|
||||
expectPrNum int
|
||||
}{
|
||||
{token: "github_test.go", repoOwner: "qiniu", repoName: "goc", prNumStr: "1", botUserName: "qiniu-bot", commentFlag: "test", expectPrNum: 1},
|
||||
}
|
||||
|
||||
for _, tc := range items {
|
||||
prClient := NewPrClient(tc.token, tc.repoOwner, tc.repoName, tc.prNumStr, tc.botUserName, tc.commentFlag)
|
||||
assert.Equal(t, tc.expectPrNum, prClient.PrNumber)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateGithubComment(t *testing.T) {
|
||||
client, router, _, teardown := setup()
|
||||
defer teardown()
|
||||
|
||||
var coverList = cover.DeltaCovList{{FileName: "fake-coverage", BasePer: "50.0%", NewPer: "75.0%", DeltaPer: "25.0%"}}
|
||||
expectContent := GenCommentContent("", coverList)
|
||||
comment := &github.IssueComment{
|
||||
Body: &expectContent,
|
||||
}
|
||||
|
||||
// create comment: https://developer.github.com/v3/issues/comments/#create-a-comment
|
||||
router.HandlerFunc("POST", "/repos/qiniu/goc/issues/1/comments", func(w http.ResponseWriter, r *http.Request) {
|
||||
v := new(github.IssueComment)
|
||||
json.NewDecoder(r.Body).Decode(v)
|
||||
|
||||
if !reflect.DeepEqual(v, comment) {
|
||||
t.Errorf("Request body = %+v, want %+v", v, comment)
|
||||
}
|
||||
fmt.Fprint(w, `{"id":1}`)
|
||||
})
|
||||
|
||||
// list comment: https://developer.github.com/v3/issues/comments/#list-comments-on-an-issue
|
||||
router.HandlerFunc("GET", "/repos/qiniu/goc/issues/1/comments", func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprint(w, `[{"id":1,"user": {"login": "qiniu-bot"}}]`)
|
||||
})
|
||||
|
||||
// delete comment: https://developer.github.com/v3/issues/comments/#edit-a-comment
|
||||
router.HandlerFunc("DELETE", "/repos/qiniu/goc/issues/comments/1", func(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
|
||||
p := PrComment{
|
||||
RobotUserName: "qiniu-bot",
|
||||
RepoOwner: "qiniu",
|
||||
RepoName: "goc",
|
||||
CommentFlag: "",
|
||||
PrNumber: 1,
|
||||
Ctx: context.Background(),
|
||||
opt: nil,
|
||||
GithubClient: client,
|
||||
}
|
||||
|
||||
p.CreateGithubComment("", coverList)
|
||||
}
|
||||
|
||||
func TestGetPrChangedFiles(t *testing.T) {
|
||||
client, router, _, teardown := setup()
|
||||
defer teardown()
|
||||
|
||||
var expectFiles = []string{"src/qiniu.com/kodo/s3apiv2/bucket/bucket.go"}
|
||||
|
||||
// list files API: https://developer.github.com/v3/pulls/#list-pull-requests-files
|
||||
router.HandlerFunc("GET", "/repos/qiniu/goc/pulls/1/files", func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprint(w, `[{"filename":"src/qiniu.com/kodo/s3apiv2/bucket/bucket.go"}]`)
|
||||
})
|
||||
|
||||
p := PrComment{
|
||||
RobotUserName: "qiniu-bot",
|
||||
RepoOwner: "qiniu",
|
||||
RepoName: "goc",
|
||||
CommentFlag: "",
|
||||
PrNumber: 1,
|
||||
Ctx: context.Background(),
|
||||
opt: nil,
|
||||
GithubClient: client,
|
||||
}
|
||||
changedFiles, err := p.GetPrChangedFiles()
|
||||
assert.Nil(t, err)
|
||||
assert.True(t, reflect.DeepEqual(changedFiles, expectFiles))
|
||||
}
|
||||
228
pkg/prow/job.go
Normal file
228
pkg/prow/job.go
Normal file
@ -0,0 +1,228 @@
|
||||
/*
|
||||
Copyright 2020 Qiniu Cloud (qiniu.com)
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package prow
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/qiniu/goc/pkg/cover"
|
||||
"github.com/qiniu/goc/pkg/github"
|
||||
"github.com/qiniu/goc/pkg/qiniu"
|
||||
)
|
||||
|
||||
// IProwAction defines the normal action in prow system
|
||||
type IProwAction interface {
|
||||
Fetch(BuildID, name string) []byte
|
||||
RunPresubmit() error
|
||||
RunPostsubmit() error
|
||||
RunPeriodic() error
|
||||
}
|
||||
|
||||
// Job is a prowjob in prow
|
||||
type Job struct {
|
||||
JobName string
|
||||
Org string
|
||||
RepoName string
|
||||
PRNumStr string
|
||||
BuildId string //prow job build number
|
||||
PostSubmitJob string
|
||||
PostSubmitCoverProfile string
|
||||
CovThreshold int
|
||||
LocalProfilePath string
|
||||
QiniuClient *qiniu.Client
|
||||
LocalArtifacts *qiniu.Artifacts
|
||||
GithubComment *github.PrComment
|
||||
FullDiff bool
|
||||
}
|
||||
|
||||
// Fetch the file from cloud
|
||||
func (j *Job) Fetch(BuildID, name string) []byte {
|
||||
return []byte{}
|
||||
}
|
||||
|
||||
// RunPresubmit run a presubmit job
|
||||
func (j *Job) RunPresubmit() error {
|
||||
var changedFiles []string
|
||||
var deltaCovList cover.DeltaCovList
|
||||
|
||||
// step1: get github pull request changed files' name
|
||||
if !j.FullDiff {
|
||||
var ghChangedFiles, err = j.GithubComment.GetPrChangedFiles()
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatalf("Get pull request changed file failed.")
|
||||
}
|
||||
if len(ghChangedFiles) == 0 {
|
||||
logrus.Printf("0 files changed in github pull request, don't need to run coverage profile in presubmit.\n")
|
||||
return nil
|
||||
}
|
||||
changedFiles = trimGhFileToProfile(ghChangedFiles)
|
||||
}
|
||||
|
||||
// step2: get local profile cov
|
||||
localP, err := cover.ReadFileToCoverList(j.LocalProfilePath)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatalf("failed to get remote cover profile")
|
||||
}
|
||||
|
||||
//step3: find the remote healthy cover profile from qiniu bucket
|
||||
remoteProfile, err := qiniu.FindBaseProfileFromQiniu(j.QiniuClient, j.PostSubmitJob, j.PostSubmitCoverProfile)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatalf("failed to get remote cover profile")
|
||||
}
|
||||
if remoteProfile == nil {
|
||||
logrus.Infof("get non healthy remoteProfile, do nothing")
|
||||
return nil
|
||||
}
|
||||
baseP, err := cover.CovList(bytes.NewReader(remoteProfile))
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatalf("failed to get remote cover profile")
|
||||
}
|
||||
|
||||
// step4: calculate diff cov between local and remote profile
|
||||
if !j.FullDiff {
|
||||
deltaCovList = cover.GetChFileDeltaCov(localP, baseP, changedFiles)
|
||||
} else {
|
||||
deltaCovList = cover.GetDeltaCov(localP, baseP)
|
||||
logrus.Infof("get delta file name is:")
|
||||
for _, d := range deltaCovList {
|
||||
logrus.Infof("%s", d.FileName)
|
||||
changedFiles = append(changedFiles, d.FileName)
|
||||
}
|
||||
}
|
||||
|
||||
// step5: generate changed file html coverage
|
||||
err = j.WriteChangedCov(changedFiles)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatalf("filter local profile to %s with changed files failed", j.LocalArtifacts.ChangedProfileName)
|
||||
}
|
||||
err = j.CreateChangedCovHtml()
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatalf("create changed file related coverage html failed")
|
||||
}
|
||||
j.SetDeltaCovLinks(deltaCovList)
|
||||
|
||||
// step6: post comment to github
|
||||
commentPrefix := github.CommentsPrefix
|
||||
if j.GithubComment.CommentFlag != "" {
|
||||
commentPrefix = fmt.Sprintf("**%s** ", j.GithubComment.CommentFlag) + commentPrefix
|
||||
}
|
||||
err = j.GithubComment.CreateGithubComment(commentPrefix, deltaCovList)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatalf("Post comment to github failed.")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RunPostsubmit run a postsubmit job
|
||||
func (j *Job) RunPostsubmit() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// RunPeriodic run a periodic job
|
||||
func (j *Job) RunPeriodic() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
//trim github filename to profile format:
|
||||
// src/qiniu.com/kodo/io/io/io_svr.go -> qiniu.com/kodo/io/io/io_svr.go
|
||||
func trimGhFileToProfile(ghFiles []string) (pFiles []string) {
|
||||
//TODO: need compatible other situation
|
||||
logrus.Infof("trim PR changed file name to:")
|
||||
for _, f := range ghFiles {
|
||||
file := strings.TrimPrefix(f, "src/")
|
||||
logrus.Infof("%s", file)
|
||||
pFiles = append(pFiles, file)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// filter local profile with changed files and save to j.LocalArtifacts.ChangedProfileName
|
||||
func (j *Job) WriteChangedCov(changedFiles []string) error {
|
||||
p, err := ioutil.ReadFile(j.LocalProfilePath)
|
||||
if err != nil {
|
||||
logrus.Printf("Open file %s failed", j.LocalProfilePath)
|
||||
return err
|
||||
}
|
||||
cp := j.LocalArtifacts.CreateChangedProfile()
|
||||
defer cp.Close()
|
||||
s := bufio.NewScanner(bytes.NewReader(p))
|
||||
s.Scan()
|
||||
writeLine(cp, s.Text())
|
||||
|
||||
for s.Scan() {
|
||||
for _, file := range changedFiles {
|
||||
if strings.HasPrefix(s.Text(), file) {
|
||||
writeLine(cp, s.Text())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeLine writes a line in the given file, if the file pointer is not nil
|
||||
func writeLine(file *os.File, content string) {
|
||||
if file != nil {
|
||||
fmt.Fprintln(file, content)
|
||||
}
|
||||
}
|
||||
|
||||
func (j *Job) JobPrefixOnQiniu() string {
|
||||
return path.Join("pr-logs", "pull", j.Org+"_"+j.RepoName, j.PRNumStr, j.JobName, j.BuildId)
|
||||
}
|
||||
|
||||
func (j *Job) HtmlProfile() string {
|
||||
return fmt.Sprintf("%s-%s-pr%s-coverage.html", j.Org, j.RepoName, j.PRNumStr)
|
||||
}
|
||||
|
||||
func (j *Job) SetDeltaCovLinks(c cover.DeltaCovList) {
|
||||
c.Sort()
|
||||
for i := 0; i < len(c); i++ {
|
||||
qnKey := path.Join(j.JobPrefixOnQiniu(), "artifacts", j.HtmlProfile())
|
||||
authQnKey := j.QiniuClient.GetAccessURL(qnKey, time.Hour*24*7)
|
||||
c[i].SetLineCovLink(authQnKey + "#file" + strconv.Itoa(i))
|
||||
logrus.Printf("file %s html coverage link is: %s\n", c[i].FileName, c[i].GetLineCovLink())
|
||||
}
|
||||
}
|
||||
|
||||
// CreateChangedCovHtml create changed file related coverage html base on the local artifact
|
||||
func (j *Job) CreateChangedCovHtml() error {
|
||||
if j.LocalArtifacts.ChangedProfileName == "" {
|
||||
logrus.Errorf("param LocalArtifacts.ChangedProfileName is empty")
|
||||
}
|
||||
pathProfileCov := j.LocalArtifacts.ChangedProfileName
|
||||
pathHtmlCov := path.Join(os.Getenv("ARTIFACTS"), j.HtmlProfile())
|
||||
cmdTxt := fmt.Sprintf("go tool cover -html=%s -o %s", pathProfileCov, pathHtmlCov)
|
||||
logrus.Printf("Running command '%s'\n", cmdTxt)
|
||||
cmd := exec.Command("go", "tool", "cover", "-html="+pathProfileCov, "-o", pathHtmlCov)
|
||||
stdOut, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
logrus.Printf("Error executing cmd: %v; combinedOutput=%s", err, stdOut)
|
||||
}
|
||||
return err
|
||||
}
|
||||
17
pkg/prow/job_test.go
Normal file
17
pkg/prow/job_test.go
Normal file
@ -0,0 +1,17 @@
|
||||
/*
|
||||
Copyright 2020 Qiniu Cloud (qiniu.com)
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package prow
|
||||
241
pkg/qiniu/client.go
Normal file
241
pkg/qiniu/client.go
Normal file
@ -0,0 +1,241 @@
|
||||
/*
|
||||
Copyright 2020 Qiniu Cloud (qiniu.com)
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package qiniu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/qiniu/api.v7/v7/auth/qbox"
|
||||
"github.com/qiniu/api.v7/v7/client"
|
||||
"github.com/qiniu/api.v7/v7/storage"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Config store the credentials to connect with qiniu cloud
|
||||
type Config struct {
|
||||
Bucket string `json:"bucket"`
|
||||
AccessKey string `json:"accessKey"`
|
||||
SecretKey string `json:"secretKey"`
|
||||
|
||||
// domain used to download files from qiniu cloud
|
||||
Domain string `json:"domain"`
|
||||
}
|
||||
|
||||
// Client for the operation with qiniu cloud
|
||||
type Client struct {
|
||||
cfg *Config
|
||||
BucketManager *storage.BucketManager
|
||||
}
|
||||
|
||||
// NewClient creates a new client to work with qiniu cloud
|
||||
func NewClient(cfg *Config) *Client {
|
||||
return &Client{
|
||||
cfg: cfg,
|
||||
BucketManager: storage.NewBucketManager(qbox.NewMac(cfg.AccessKey, cfg.SecretKey), nil),
|
||||
}
|
||||
}
|
||||
|
||||
// QiniuObjectHandle construct a object hanle to access file in qiniu
|
||||
func (q *Client) QiniuObjectHandle(key string) *ObjectHandle {
|
||||
return &ObjectHandle{
|
||||
key: key,
|
||||
cfg: q.cfg,
|
||||
bm: q.BucketManager,
|
||||
mac: qbox.NewMac(q.cfg.AccessKey, q.cfg.SecretKey),
|
||||
client: &client.Client{Client: http.DefaultClient},
|
||||
}
|
||||
}
|
||||
|
||||
// ReadObject to read all the content of key
|
||||
func (q *Client) ReadObject(key string) ([]byte, error) {
|
||||
objectHandle := q.QiniuObjectHandle(key)
|
||||
reader, err := objectHandle.NewReader(context.Background())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error getting qiniu artifact reader: %v", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
return ioutil.ReadAll(reader)
|
||||
}
|
||||
|
||||
// ListAll to list all the files with contains the expected prefix
|
||||
func (q *Client) ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error) {
|
||||
var files []string
|
||||
artifacts, err := q.listEntries(prefix, delimiter)
|
||||
if err != nil {
|
||||
return files, err
|
||||
}
|
||||
|
||||
for _, item := range artifacts {
|
||||
files = append(files, item.Key)
|
||||
}
|
||||
|
||||
return files, nil
|
||||
}
|
||||
|
||||
// ListAll to list all the entries with contains the expected prefix
|
||||
func (q *Client) listEntries(prefix string, delimiter string) ([]storage.ListItem, error) {
|
||||
var marker string
|
||||
var artifacts []storage.ListItem
|
||||
|
||||
wait := []time.Duration{16, 32, 64, 128, 256, 256, 512, 512}
|
||||
for i := 0; ; {
|
||||
entries, _, nextMarker, hashNext, err := q.BucketManager.ListFiles(q.cfg.Bucket, prefix, delimiter, marker, 500)
|
||||
if err != nil {
|
||||
logrus.WithField("prefix", prefix).WithError(err).Error("Error accessing QINIU artifact.")
|
||||
if i >= len(wait) {
|
||||
return artifacts, fmt.Errorf("timed out: error accessing QINIU artifact: %v", err)
|
||||
}
|
||||
time.Sleep((wait[i] + time.Duration(rand.Intn(10))) * time.Millisecond)
|
||||
i++
|
||||
continue
|
||||
}
|
||||
artifacts = append(artifacts, entries...)
|
||||
|
||||
if hashNext {
|
||||
marker = nextMarker
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return artifacts, nil
|
||||
}
|
||||
|
||||
// GetAccessURL return a url which can access artifact directly in qiniu
|
||||
func (q *Client) GetAccessURL(key string, timeout time.Duration) string {
|
||||
deadline := time.Now().Add(timeout).Unix()
|
||||
return storage.MakePrivateURL(qbox.NewMac(q.cfg.AccessKey, q.cfg.SecretKey), q.cfg.Domain, key, deadline)
|
||||
}
|
||||
|
||||
type LogHistoryTemplate struct {
|
||||
BucketName string
|
||||
KeyPath string
|
||||
Items []logHistoryItem
|
||||
}
|
||||
|
||||
type logHistoryItem struct {
|
||||
Name string
|
||||
Size string
|
||||
Time string
|
||||
Url string
|
||||
}
|
||||
|
||||
// Artifacts lists all artifacts available for the given job source
|
||||
func (q *Client) GetArtifactDetails(key string) (*LogHistoryTemplate, error) {
|
||||
tmpl := new(LogHistoryTemplate)
|
||||
item := logHistoryItem{}
|
||||
listStart := time.Now()
|
||||
artifacts, err := q.listEntries(key, "")
|
||||
if err != nil {
|
||||
return tmpl, err
|
||||
}
|
||||
|
||||
for _, entry := range artifacts {
|
||||
item.Name = splitKey(entry.Key, key)
|
||||
item.Size = size(entry.Fsize)
|
||||
item.Time = timeConv(entry.PutTime)
|
||||
item.Url = q.GetAccessURL(entry.Key, time.Duration(time.Second*60*60))
|
||||
tmpl.Items = append(tmpl.Items, item)
|
||||
}
|
||||
|
||||
logrus.WithField("duration", time.Since(listStart).String()).Infof("Listed %d artifacts.", len(tmpl.Items))
|
||||
return tmpl, nil
|
||||
}
|
||||
|
||||
func splitKey(item, key string) string {
|
||||
return strings.TrimPrefix(item, key)
|
||||
}
|
||||
|
||||
func size(fsize int64) string {
|
||||
return strings.Join([]string{strconv.FormatInt(fsize, 10), "bytes"}, " ")
|
||||
}
|
||||
|
||||
func timeConv(ptime int64) string {
|
||||
s := strconv.FormatInt(ptime, 10)[0:10]
|
||||
t, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil {
|
||||
logrus.Errorf("time string parse int error : %v", err)
|
||||
return ""
|
||||
}
|
||||
tm := time.Unix(t, 0)
|
||||
return tm.Format("2006-01-02 03:04:05 PM")
|
||||
}
|
||||
|
||||
func (q *Client) ListSubDirs(prefix string) ([]string, error) {
|
||||
var dirs []string
|
||||
var marker string
|
||||
|
||||
wait := []time.Duration{16, 32, 64, 128, 256, 256, 512, 512}
|
||||
for i := 0; ; {
|
||||
// use rsf list v2 interface to get the sub folder based on the delimiter
|
||||
entries, err := q.BucketManager.ListBucketContext(context.Background(), q.cfg.Bucket, prefix, "/", marker)
|
||||
if err != nil {
|
||||
logrus.WithField("prefix", prefix).WithError(err).Error("Error accessing QINIU artifact.")
|
||||
if i >= len(wait) {
|
||||
return dirs, fmt.Errorf("timed out: error accessing QINIU artifact: %v", err)
|
||||
}
|
||||
time.Sleep((wait[i] + time.Duration(rand.Intn(10))) * time.Millisecond)
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
for entry := range entries {
|
||||
if entry.Dir != "" {
|
||||
// entry.Dir should be like "logs/kodo-periodics-integration-test/1181915661132107776/"
|
||||
// the sub folder is 1181915661132107776, also known as prowjob buildid.
|
||||
buildId := getBuildId(entry.Dir)
|
||||
if buildId != "" {
|
||||
dirs = append(dirs, buildId)
|
||||
} else {
|
||||
logrus.Warnf("invalid dir format: %v", entry.Dir)
|
||||
}
|
||||
}
|
||||
|
||||
marker = entry.Marker
|
||||
}
|
||||
|
||||
if marker != "" {
|
||||
i = 0
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return dirs, nil
|
||||
}
|
||||
|
||||
var nonPRLogsBuildIdSubffixRe = regexp.MustCompile("([0-9]+)/$")
|
||||
|
||||
// extract the build number from dir path
|
||||
// expect the dir as the following formats:
|
||||
// 1. logs/kodo-periodics-integration-test/1181915661132107776/
|
||||
func getBuildId(dir string) string {
|
||||
matches := nonPRLogsBuildIdSubffixRe.FindStringSubmatch(dir)
|
||||
if len(matches) == 2 {
|
||||
return matches[1]
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
40
pkg/qiniu/client_test.go
Normal file
40
pkg/qiniu/client_test.go
Normal file
@ -0,0 +1,40 @@
|
||||
/*
|
||||
Copyright 2020 Qiniu Cloud (qiniu.com)
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package qiniu
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestGetBuildId(t *testing.T) {
|
||||
type tc struct {
|
||||
dir string
|
||||
expected string
|
||||
}
|
||||
|
||||
tcs := []tc{
|
||||
{dir: "logs/kodo-periodics-integration-test/1181915661132107776/", expected: "1181915661132107776"},
|
||||
{dir: "logs/kodo-periodics-integration-test/1181915661132107776", expected: ""},
|
||||
{dir: "pr-logs/directory/WIP-qtest-pull-request-kodo-test/1181915661132107776/", expected: "1181915661132107776"},
|
||||
{dir: "pr-logs/directory/WIP-qtest-pull-request-kodo-test/1181915661132107776.txt", expected: ""},
|
||||
}
|
||||
|
||||
for _, tc := range tcs {
|
||||
got := getBuildId(tc.dir)
|
||||
if tc.expected != got {
|
||||
t.Errorf("getBuildId error, dir: %s, expect: %s, but got: %s", tc.dir, tc.expected, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
124
pkg/qiniu/object.go
Normal file
124
pkg/qiniu/object.go
Normal file
@ -0,0 +1,124 @@
|
||||
/*
|
||||
Copyright 2020 Qiniu Cloud (qiniu.com)
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package qiniu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/qiniu/api.v7/v7/auth/qbox"
|
||||
"github.com/qiniu/api.v7/v7/client"
|
||||
"github.com/qiniu/api.v7/v7/storage"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// ObjectHandle provides operations on an object in a qiniu cloud bucket
|
||||
type ObjectHandle struct {
|
||||
key string
|
||||
cfg *Config
|
||||
bm *storage.BucketManager
|
||||
mac *qbox.Mac
|
||||
client *client.Client
|
||||
}
|
||||
|
||||
func (o *ObjectHandle) Attrs(ctx context.Context) (storage.FileInfo, error) {
|
||||
//TODO(CarlJi): need retry when errors
|
||||
return o.bm.Stat(o.cfg.Bucket, o.key)
|
||||
}
|
||||
|
||||
// NewReader creates a reader to read the contents of the object.
|
||||
// ErrObjectNotExist will be returned if the object is not found.
|
||||
// The caller must call Close on the returned Reader when done reading.
|
||||
func (o *ObjectHandle) NewReader(ctx context.Context) (io.ReadCloser, error) {
|
||||
return o.NewRangeReader(ctx, 0, -1)
|
||||
}
|
||||
|
||||
// NewRangeReader reads parts of an object, reading at most length bytes starting
|
||||
// from the given offset. If length is negative, the object is read until the end.
|
||||
func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) (io.ReadCloser, error) {
|
||||
verb := "GET"
|
||||
if length == 0 {
|
||||
verb = "HEAD"
|
||||
}
|
||||
|
||||
var res *http.Response
|
||||
var err error
|
||||
|
||||
err = runWithRetry(3, func() (bool, error) {
|
||||
headers := http.Header{}
|
||||
start := offset
|
||||
if length < 0 && start >= 0 {
|
||||
headers.Set("Range", fmt.Sprintf("bytes=%d-", start))
|
||||
} else if length > 0 {
|
||||
// The end character isn't affected by how many bytes we have seen.
|
||||
headers.Set("Range", fmt.Sprintf("bytes=%d-%d", start, offset+length-1))
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(time.Second * 60 * 10).Unix()
|
||||
accessURL := storage.MakePrivateURL(o.mac, o.cfg.Domain, o.key, deadline)
|
||||
res, err = o.client.DoRequest(ctx, verb, accessURL, headers)
|
||||
if err != nil {
|
||||
time.Sleep(time.Second) //TODO enhance
|
||||
return true, err
|
||||
}
|
||||
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
res.Body.Close()
|
||||
return true, fmt.Errorf("qiniu storage: object not exists")
|
||||
}
|
||||
|
||||
return shouldRetry(res), nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return res.Body, nil
|
||||
}
|
||||
|
||||
func runWithRetry(maxTry int, f func() (bool, error)) error {
|
||||
var err error
|
||||
for maxTry > 0 {
|
||||
needRetry, err := f()
|
||||
if err != nil {
|
||||
logrus.Warnf("err occurred: %v. try again", err)
|
||||
} else if needRetry {
|
||||
logrus.Warn("results do not meet the expectation. try again")
|
||||
} else {
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Millisecond * 100)
|
||||
maxTry = maxTry - 1
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func shouldRetry(res *http.Response) bool {
|
||||
|
||||
// 571 and 573 mean the request was limited by cloud storage because of concurrency count exceed
|
||||
// so it's better to retry after a while
|
||||
if res.StatusCode == 571 || res.StatusCode == 573 {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
130
pkg/qiniu/qnPresubmit.go
Normal file
130
pkg/qiniu/qnPresubmit.go
Normal file
@ -0,0 +1,130 @@
|
||||
/*
|
||||
Copyright 2020 Qiniu Cloud (qiniu.com)
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package qiniu
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
const (
|
||||
//statusJSON is the JSON file that stores build success info
|
||||
statusJSON = "finished.json"
|
||||
|
||||
// ArtifactsDirName is the name of directory defined in prow to store test artifacts
|
||||
ArtifactsDirName = "artifacts"
|
||||
|
||||
//default prow coverage file
|
||||
PostSubmitCoverProfile = "filtered.cov"
|
||||
|
||||
//default to save changed file related coverage profile
|
||||
ChangedProfileName = "changed-file-profile.cov"
|
||||
)
|
||||
|
||||
// sortBuilds converts all build from str to int and sorts all builds in descending order and
|
||||
// returns the sorted slice
|
||||
func sortBuilds(strBuilds []string) []int {
|
||||
var res []int
|
||||
for _, buildStr := range strBuilds {
|
||||
num, err := strconv.Atoi(buildStr)
|
||||
if err != nil {
|
||||
log.Printf("Non-int build number found: '%s'", buildStr)
|
||||
} else {
|
||||
res = append(res, num)
|
||||
}
|
||||
}
|
||||
sort.Sort(sort.Reverse(sort.IntSlice(res)))
|
||||
return res
|
||||
}
|
||||
|
||||
type finishedStatus struct {
|
||||
Timestamp int
|
||||
Passed bool
|
||||
}
|
||||
|
||||
func isBuildSucceeded(jsonText []byte) bool {
|
||||
var status finishedStatus
|
||||
err := json.Unmarshal(jsonText, &status)
|
||||
return err == nil && status.Passed
|
||||
}
|
||||
|
||||
// FindBaseProfileFromQiniu finds the coverage profile file from the latest healthy build
|
||||
// stored in given gcs directory
|
||||
func FindBaseProfileFromQiniu(qc *Client, prowJobName, covProfileName string) ([]byte, error) {
|
||||
dirOfJob := path.Join("logs", prowJobName)
|
||||
prefix := dirOfJob + "/"
|
||||
strBuilds, err := qc.ListSubDirs(prefix)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error listing qiniu objects, prowjob:%v, err:%v", prowJobName, err)
|
||||
}
|
||||
if len(strBuilds) == 0 {
|
||||
log.Printf("no cover profiles found from remote, do nothing")
|
||||
return nil, nil
|
||||
}
|
||||
log.Printf("total sub dirs: %d", len(strBuilds))
|
||||
|
||||
builds := sortBuilds(strBuilds)
|
||||
profilePath := ""
|
||||
for _, build := range builds {
|
||||
buildDirPath := path.Join(dirOfJob, strconv.Itoa(build))
|
||||
dirOfStatusJSON := path.Join(buildDirPath, statusJSON)
|
||||
|
||||
statusText, err := qc.ReadObject(dirOfStatusJSON)
|
||||
if err != nil {
|
||||
log.Printf("Cannot read finished.json (%s) ", dirOfStatusJSON)
|
||||
} else if isBuildSucceeded(statusText) {
|
||||
artifactsDirPath := path.Join(buildDirPath, ArtifactsDirName)
|
||||
profilePath = path.Join(artifactsDirPath, covProfileName)
|
||||
break
|
||||
}
|
||||
}
|
||||
if profilePath == "" {
|
||||
log.Printf("no cover profiles found from remote job %s, do nothing", prowJobName)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
log.Printf("base cover profile path: %s", profilePath)
|
||||
return qc.ReadObject(profilePath)
|
||||
}
|
||||
|
||||
type Artifacts struct {
|
||||
Directory string
|
||||
ProfileName string
|
||||
ChangedProfileName string // create temporary to save changed file related coverage profile
|
||||
}
|
||||
|
||||
func (a *Artifacts) ProfilePath() string {
|
||||
return path.Join(a.Directory, a.ProfileName)
|
||||
}
|
||||
|
||||
func (a *Artifacts) CreateChangedProfile() *os.File {
|
||||
if a.ChangedProfileName == "" {
|
||||
log.Fatalf("param Artifacts.ChangedProfileName should not be empty")
|
||||
}
|
||||
p, err := os.Create(a.ChangedProfileName)
|
||||
log.Printf("os create: %s", a.ChangedProfileName)
|
||||
if err != nil {
|
||||
log.Fatalf("file(%s) create failed: %v", a.ChangedProfileName, err)
|
||||
}
|
||||
|
||||
return p
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user