ar0c/goc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

mock qiniu, gihub, artifacts client

Browse Source
This commit is contained in:
tongjingran 2020-07-24 19:24:07 +08:00
parent 8c719effc8
commit dc72708d64
9 changed files with 265 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/diff.go
View File

@ -180,7 +180,7 @@ func doDiffUnderProw(cmd *cobra.Command, args []string) {
}
qc = qiniu.NewClient(&conf)
localArtifacts := qiniu.Artifacts{
localArtifacts := qiniu.ProfileArtifacts{
Directory: artifacts,
ProfileName: newProfile,
ChangedProfileName: qiniu.ChangedProfileName,

14
pkg/github/github_test.go
View File

@ -135,6 +135,12 @@ func TestCreateGithubComment(t *testing.T) {
p.CreateGithubComment("", coverList)
}
func TestCreateGithubCommentError(t *testing.T) {
p := &GithubPrComment{}
err := p.CreateGithubComment("", cover.DeltaCovList{})
assert.NoError(t, err)
}
func TestGetPrChangedFiles(t *testing.T) {
client, router, _, teardown := setup()
defer teardown()
@ -160,3 +166,11 @@ func TestGetPrChangedFiles(t *testing.T) {
assert.Equal(t, err, nil)
assert.Equal(t, changedFiles, expectFiles)
}
func TestGetCommentFlag(t *testing.T) {
p := GithubPrComment{
CommentFlag: "flag",
}
flag := p.GetCommentFlag()
assert.Equal(t, flag, p.CommentFlag)
}

22
pkg/mock/clients/github.go
View File

@ -1,24 +1,34 @@
package clients
import "github.com/qiniu/goc/pkg/cover"
import (
"github.com/qiniu/goc/pkg/cover"
)
type MockPrComment struct {
GetPrChangedFilesRes []string
GetPrChangedFilesErr error
PostCommentErr error
EraseHistoryCommentErr error
CreateGithubCommentErr error
CommentFlag string
}
func (s *MockPrComment) GetPrChangedFiles() (files []string, err error) {
return nil, nil
return s.GetPrChangedFilesRes, s.GetPrChangedFilesErr
}
func (s *MockPrComment) PostComment(content, commentPrefix string) error {
return nil
return s.PostCommentErr
}
func (s *MockPrComment) EraseHistoryComment(commentPrefix string) error {
return nil
return s.EraseHistoryCommentErr
}
func (s *MockPrComment) CreateGithubComment(commentPrefix string, diffCovList cover.DeltaCovList) (err error) {
return nil
return s.CreateGithubCommentErr
}
func (s *MockPrComment) GetCommentFlag() string {
return ""
return s.CommentFlag
}

39
pkg/mock/clients/qiniu.go
View File

@ -3,32 +3,59 @@ package clients
import (
"context"
"github.com/qiniu/goc/pkg/qiniu"
"os"
"time"
)
type MockQnClient struct {
QiniuObjectHandleRes qiniu.ObjectHandle
ReadObjectRes []byte
ReadObjectErr error
ListAllRes []string
ListAllErr error
GetAccessURLRes string
GetArtifactDetailsRes *qiniu.LogHistoryTemplate
GetArtifactDetailsErr error
ListSubDirsRes []string
ListSubDirsErr error
}
func (s *MockQnClient) QiniuObjectHandle(key string) qiniu.ObjectHandle {
return nil
return s.QiniuObjectHandleRes
}
func (s *MockQnClient) ReadObject(key string) ([]byte, error) {
return nil, nil
return s.ReadObjectRes, s.ReadObjectErr
}
func (s *MockQnClient) ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error) {
return nil, nil
return s.ListAllRes, s.ListAllErr
}
func (s *MockQnClient) GetAccessURL(key string, timeout time.Duration) string {
return ""
return s.GetAccessURLRes
}
func (s *MockQnClient) GetArtifactDetails(key string) (*qiniu.LogHistoryTemplate, error) {
return nil, nil
return s.GetArtifactDetailsRes, s.GetArtifactDetailsErr
}
func (s *MockQnClient) ListSubDirs(prefix string) ([]string, error) {
return nil, nil
return s.ListSubDirsRes, s.ListSubDirsErr
}
type MockArtifacts struct {
ProfilePathRes string
CreateChangedProfileRes *os.File
GetChangedProfileNameRes string
}
func (s *MockArtifacts) ProfilePath() string {
return s.ProfilePathRes
}
func (s *MockArtifacts) CreateChangedProfile() *os.File {
return s.CreateChangedProfileRes
}
func (s *MockArtifacts) GetChangedProfileName() string {
return s.GetChangedProfileNameRes
}

29
pkg/prow/job.go
View File

@ -55,7 +55,7 @@ type Job struct {
CovThreshold int
LocalProfilePath string
QiniuClient qiniu.Client
LocalArtifacts *qiniu.Artifacts
LocalArtifacts qiniu.Artifacts
GithubComment github.PrComment
FullDiff bool
}
@ -70,16 +70,13 @@ func (j *Job) RunPresubmit() error {
// step1: get local profile cov
localP, err := cover.ReadFileToCoverList(j.LocalProfilePath)
if err != nil {
logrus.Errorf("failed to get remote cover profile: %s", err.Error())
return err
return fmt.Errorf("failed to get remote cover profile: %s", err.Error())
}
logrus.Warnf("localP: [%+v]", localP)
//step2: find the remote healthy cover profile from qiniu bucket
remoteProfile, err := qiniu.FindBaseProfileFromQiniu(j.QiniuClient, j.PostSubmitJob, j.PostSubmitCoverProfile)
if err != nil {
logrus.Errorf("failed to get remote cover profile: %s", err.Error())
return err
return fmt.Errorf("failed to get remote cover profile: %s", err.Error())
}
if remoteProfile == nil {
logrus.Infof("get non healthy remoteProfile, do nothing")
@ -87,31 +84,27 @@ func (j *Job) RunPresubmit() error {
}
baseP, err := cover.CovList(bytes.NewReader(remoteProfile))
if err != nil {
logrus.Errorf("failed to get remote cover profile: %s", err.Error())
return err
return fmt.Errorf("failed to get remote cover profile: %s", err.Error())
}
logrus.Warnf("baseP: [%+v]", baseP)
// step3: get github pull request changed files' name and calculate diff cov between local and remote profile
changedFiles, deltaCovList, err := getFilesAndCovList(j.FullDiff, j.GithubComment, localP, baseP)
if err != nil {
logrus.Errorf("Get files and covlist failed: %s", err.Error())
return err
return fmt.Errorf("Get files and covlist failed: %s", err.Error())
}
// step4: generate changed file html coverage
err = j.WriteChangedCov(changedFiles)
if err != nil {
logrus.WithError(err).Fatalf("filter local profile to %s with changed files failed", j.LocalArtifacts.ChangedProfileName)
return fmt.Errorf("filter local profile to %s with changed files failed: %s", j.LocalArtifacts.GetChangedProfileName(), err.Error())
}
err = j.CreateChangedCovHtml()
if err != nil {
logrus.WithError(err).Fatalf("create changed file related coverage html failed")
return fmt.Errorf("create changed file related coverage html failed: %s", err.Error())
}
j.SetDeltaCovLinks(deltaCovList)
// step6: post comment to github
// step5: post comment to github
commentPrefix := github.CommentsPrefix
if j.GithubComment.GetCommentFlag() != "" {
commentPrefix = fmt.Sprintf("**%s** ", j.GithubComment.GetCommentFlag()) + commentPrefix
@ -122,7 +115,7 @@ func (j *Job) RunPresubmit() error {
}
err = j.GithubComment.CreateGithubComment(commentPrefix, deltaCovList)
if err != nil {
logrus.WithError(err).Fatalf("Post comment to github failed.")
return fmt.Errorf("Post comment to github failed: %s", err.Error())
}
return nil
@ -202,10 +195,10 @@ func (j *Job) SetDeltaCovLinks(c cover.DeltaCovList) {
// CreateChangedCovHtml create changed file related coverage html base on the local artifact
func (j *Job) CreateChangedCovHtml() error {
if j.LocalArtifacts.ChangedProfileName == "" {
if j.LocalArtifacts.GetChangedProfileName() == "" {
logrus.Errorf("param LocalArtifacts.ChangedProfileName is empty")
}
pathProfileCov := j.LocalArtifacts.ChangedProfileName
pathProfileCov := j.LocalArtifacts.GetChangedProfileName()
pathHtmlCov := path.Join(os.Getenv("ARTIFACTS"), j.HtmlProfile())
cmdTxt := fmt.Sprintf("go tool cover -html=%s -o %s", pathProfileCov, pathHtmlCov)
logrus.Printf("Running command '%s'\n", cmdTxt)

217
pkg/prow/job_test.go
View File

@ -18,6 +18,8 @@ package prow
import (
"errors"
"fmt"
"github.com/qiniu/goc/pkg/cover"
"github.com/qiniu/goc/pkg/github"
"github.com/qiniu/goc/pkg/mock/clients"
"github.com/qiniu/goc/pkg/qiniu"
@ -36,6 +38,7 @@ qiniu.com/kodo/bd/bdgetter/source.go:37.34,39.2 1 0
qiniu.com/kodo/bd/pfd/locker/app/qboxbdlocker/main.go:50.2,53.52 4 1
qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go:33.51,35.2 1 0`
defaultLocalPath = "local.cov"
defaultChangedPath = "changed.cov"
)
func TestTrimGhFileToProfile(t *testing.T) {
@ -76,7 +79,7 @@ qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go:33.51,35.2 1 0
defer os.Remove(savePath)
j := &Job{
LocalProfilePath: path,
LocalArtifacts: &qiniu.Artifacts{ChangedProfileName: savePath},
LocalArtifacts: &qiniu.ProfileArtifacts{ChangedProfileName: savePath},
}
j.WriteChangedCov(changedFiles)
@ -133,7 +136,7 @@ func TestRunPresubmitFulldiff(t *testing.T) {
PostSubmitJob: "kodo-postsubmits-go-st-coverage",
PostSubmitCoverProfile: "filterd.cov",
LocalProfilePath: localPath,
LocalArtifacts: &qiniu.Artifacts{ChangedProfileName: ChangedProfilePath},
LocalArtifacts: &qiniu.ProfileArtifacts{ChangedProfileName: ChangedProfilePath},
QiniuClient: qc,
GithubComment: prClient,
FullDiff: true,
@ -144,44 +147,60 @@ func TestRunPresubmitFulldiff(t *testing.T) {
assert.NoError(t, err)
}
func TestRunPresubmitErrorGetRemoteCoverProfile(t *testing.T) {
j := &Job{
func TestRunPresubmitError(t *testing.T) {
items := []struct {
prepare bool // 是否需要准备本地cov
j Job
err string
}{
{
prepare: false,
j: Job{
LocalProfilePath: "unkown",
}
err := j.RunPresubmit()
assert.Contains(t, err.Error(), "no such file or directory")
}
func TestRunPresubmitGetEmptyProfile(t *testing.T) {
path := "local.cov"
setup(path, defaultContent)
defer os.Remove(path)
j := &Job{
LocalProfilePath: path,
},
err: "no such file or directory",
},
{
prepare: true,
j: Job{
LocalProfilePath: defaultLocalPath,
QiniuClient: &clients.MockQnClient{},
},
},
{
prepare: true,
j: Job{
LocalProfilePath: defaultLocalPath,
QiniuClient: &clients.MockQnClient{ListSubDirsErr: errors.New("mock error")},
},
err: "mock error",
},
{
prepare: true,
j: Job{
LocalProfilePath: defaultLocalPath,
QiniuClient: &MockProfileQnClient{},
GithubComment: &clients.MockPrComment{GetPrChangedFilesRes: []string{"qiniu.com/kodo/apiserver/server/main.go"}},
FullDiff: true,
LocalArtifacts: &qiniu.ProfileArtifacts{ChangedProfileName: defaultChangedPath},
},
err: "",
},
}
err := j.RunPresubmit()
assert.NoError(t, err)
}
type MockErrorListSubDirsQnClient struct {
*clients.MockQnClient
}
func (s *MockErrorListSubDirsQnClient) ListSubDirs(prefix string) ([]string, error) {
return nil, errors.New("mock error")
}
func TestRunPresubmitErrorGetBaseProfile(t *testing.T) {
path := "local.cov"
for _, tc := range items {
if tc.prepare {
path := defaultLocalPath
setup(path, defaultContent)
defer os.Remove(path)
j := &Job{
LocalProfilePath: path,
QiniuClient: &MockErrorListSubDirsQnClient{},
defer os.Remove(defaultChangedPath)
}
err := tc.j.RunPresubmit()
if tc.err == "" {
assert.NoError(t, err)
} else {
assert.Contains(t, err.Error(), tc.err)
}
}
err := j.RunPresubmit()
assert.Contains(t, err.Error(), "mock error")
}
type MockProfileQnClient struct {
@ -189,7 +208,7 @@ type MockProfileQnClient struct {
}
func (s *MockProfileQnClient) ListSubDirs(prefix string) ([]string, error) {
return []string{"1"}, nil
return []string{defaultContent}, nil
}
func (s *MockProfileQnClient) ReadObject(key string) ([]byte, error) {
@ -200,43 +219,93 @@ func (s *MockProfileQnClient) ReadObject(key string) ([]byte, error) {
return []byte(""), nil
}
// 无法实现coverList报错
//func TestRunPresubmitErrorReadBaseProfile(t *testing.T) {
// path := "local.cov"
// setup(path, defaultContent)
// defer os.Remove(path)
// j := &Job{
// LocalProfilePath: path,
// QiniuClient: &MockProfileQnClient{},
// }
// err := j.RunPresubmit()
// assert.Contains(t, err.Error(), "mock error")
//}
func TestGetFilesAndCovList(t *testing.T) {
items := []struct {
fullDiff bool
prComment github.PrComment
localP cover.CoverageList
baseP cover.CoverageList
err string
lenFiles int
lenCovList int
}{
{
fullDiff: true,
prComment: &clients.MockPrComment{},
localP: cover.CoverageList{
{FileName: "qiniu.com/kodo/apiserver/server/main.go", NCoveredStmts: 2, NAllStmts: 2},
{FileName: "qiniu.com/kodo/apiserver/server/test.go", NCoveredStmts: 2, NAllStmts: 2},
},
baseP: cover.CoverageList{
{FileName: "qiniu.com/kodo/apiserver/server/main.go", NCoveredStmts: 1, NAllStmts: 2},
{FileName: "qiniu.com/kodo/apiserver/server/test.go", NCoveredStmts: 1, NAllStmts: 2},
},
lenFiles: 2,
lenCovList: 2,
},
{
fullDiff: false,
prComment: &clients.MockPrComment{GetPrChangedFilesErr: errors.New("mock error")},
err: "mock error",
},
{
fullDiff: false,
prComment: &clients.MockPrComment{},
lenFiles: 0,
lenCovList: 0,
},
{
fullDiff: false,
prComment: &clients.MockPrComment{GetPrChangedFilesRes: []string{"qiniu.com/kodo/apiserver/server/main.go"}},
localP: cover.CoverageList{
{FileName: "qiniu.com/kodo/apiserver/server/main.go", NCoveredStmts: 2, NAllStmts: 2},
{FileName: "qiniu.com/kodo/apiserver/server/test.go", NCoveredStmts: 2, NAllStmts: 2},
},
baseP: cover.CoverageList{
{FileName: "qiniu.com/kodo/apiserver/server/main.go", NCoveredStmts: 1, NAllStmts: 2},
{FileName: "qiniu.com/kodo/apiserver/server/test.go", NCoveredStmts: 1, NAllStmts: 2},
},
lenFiles: 1,
lenCovList: 1,
},
}
//type MockErrorPrComment struct {
// *clients.MockPrComment
//}
//
//func (s *MockErrorPrComment) GetPrChangedFiles() (files []string, err error) {
// return []string{"aaa"}, nil
//}
//
//func TestGetFilesAndCovList(t *testing.T){
// items := []struct {
// fullDiff bool
// prComment github.PrComment
// localP cover.CoverageList
// baseP cover.CoverageList
// }{
// {
// inputFiles: []string{"src/qiniu.com/kodo/io/io/io_svr.go", "README.md"},
// expectFiles: []string{"qiniu.com/kodo/io/io/io_svr.go", "README.md"},
// },
// }
//
// for _, tc := range items {
// f := trimGhFileToProfile(tc.inputFiles)
// assert.Equal(t, f, tc.expectFiles)
// }
// getFilesAndCovList
//}
for i, tc := range items {
fmt.Println(i)
files, covList, err := getFilesAndCovList(tc.fullDiff, tc.prComment, tc.localP, tc.baseP)
if err != nil {
assert.Equal(t, err.Error(), tc.err)
} else {
assert.Equal(t, len(files), tc.lenFiles)
assert.Equal(t, len(covList), tc.lenCovList)
}
}
}
func TestSetDeltaCovLinks(t *testing.T) {
covList := cover.DeltaCovList{{FileName: "file1", BasePer: "5%", NewPer: "5%", DeltaPer: "0"}}
j := &Job{
QiniuClient: &clients.MockQnClient{},
}
j.SetDeltaCovLinks(covList)
}
// functions to be done
func TestRunPostsubmit(t *testing.T) {
j := &Job{}
err := j.RunPostsubmit()
assert.NoError(t, err)
}
func TestRunPeriodic(t *testing.T) {
j := &Job{}
err := j.RunPeriodic()
assert.NoError(t, err)
}
func TestFetch(t *testing.T) {
j := &Job{}
res := j.Fetch("buidID", "name")
assert.Equal(t, res, []byte{})
}

2
pkg/qiniu/client.go
View File

@ -58,7 +58,7 @@ type QnClient struct {
BucketManager *storage.BucketManager
}
// NewQnClient creates a new QnClient to work with qiniu cloud
// NewClient creates a new QnClient to work with qiniu cloud
func NewClient(cfg *Config) *QnClient {
return &QnClient{
cfg: cfg,

18
pkg/qiniu/qnPresubmit.go
View File

@ -107,20 +107,26 @@ func FindBaseProfileFromQiniu(qc Client, prowJobName, covProfileName string) ([]
return qc.ReadObject(profilePath)
}
// Artifacts prepresents the rule to store test artifacts in prow
type Artifacts struct {
type Artifacts interface {
ProfilePath() string
CreateChangedProfile() *os.File
GetChangedProfileName() string
}
// ProfileArtifacts prepresents the rule to store test artifacts in prow
type ProfileArtifacts struct {
Directory string
ProfileName string
ChangedProfileName string // create temporary to save changed file related coverage profile
}
// ProfilePath returns a full path for profile
func (a *Artifacts) ProfilePath() string {
func (a *ProfileArtifacts) ProfilePath() string {
return path.Join(a.Directory, a.ProfileName)
}
// CreateChangedProfile creates a profile in order to store the most related files based on Github Pull Request
func (a *Artifacts) CreateChangedProfile() *os.File {
func (a *ProfileArtifacts) CreateChangedProfile() *os.File {
if a.ChangedProfileName == "" {
log.Fatalf("param Artifacts.ChangedProfileName should not be empty")
}
@ -132,3 +138,7 @@ func (a *Artifacts) CreateChangedProfile() *os.File {
return p
}
func (a *ProfileArtifacts) GetChangedProfileName() string {
return a.ChangedProfileName
}

27
pkg/qiniu/qnPresubmit_test.go
View File

@ -20,6 +20,7 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"os"
)
func TestFindBaseProfileFromQiniu(t *testing.T) {
@ -39,3 +40,29 @@ func TestFindBaseProfileFromQiniu(t *testing.T) {
assert.Equal(t, err, nil)
assert.Equal(t, string(getProfile), mockProfileContent)
}
func TestArtifacts_ProfilePath(t *testing.T) {
p := &ProfileArtifacts{
Directory: "directory/",
ProfileName: "profile",
}
profilePath := p.ProfilePath()
assert.Equal(t, profilePath, "directory/profile")
}
func TestProfileArtifacts_CreateChangedProfile(t *testing.T) {
p := &ProfileArtifacts{
ChangedProfileName: "test.cov",
}
file := p.CreateChangedProfile()
defer file.Close()
defer os.Remove(p.ChangedProfileName)
}
func TestProfileArtifacts_GetChangedProfileName(t *testing.T) {
p := &ProfileArtifacts{
ChangedProfileName: "change.cov",
}
name := p.GetChangedProfileName()
assert.Equal(t, name, "change.cov")
}