ar0c/goc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge pull request #81 from tongjingran/diffReset

Browse Source 
Add test case for goc diff
This commit is contained in:
qiniu-bot 2020-07-30 19:57:45 +08:00 committed by GitHub
commit 89dd3585d1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 430 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/diff.go
View File

@ -160,7 +160,7 @@ func doDiffUnderProw(cmd *cobra.Command, args []string) {
if qiniuCredential == "" {
logrus.Fatalf("qiniu credential not provided")
}
var qc *qiniu.Client
var qc qiniu.Client
var conf qiniu.Config
files, err := ioutil.ReadFile(*&qiniuCredential)
if err != nil {
@ -180,7 +180,7 @@ func doDiffUnderProw(cmd *cobra.Command, args []string) {
}
qc = qiniu.NewClient(&conf)
localArtifacts := qiniu.Artifacts{
localArtifacts := qiniu.ProfileArtifacts{
Directory: artifacts,
ProfileName: newProfile,
ChangedProfileName: qiniu.ChangedProfileName,

6
pkg/cover/cover_test.go
View File

@ -93,6 +93,12 @@ func TestCovList(t *testing.T) {
}
}
func TestReadFileToCoverList(t *testing.T) {
path := "unknown"
_, err := ReadFileToCoverList(path)
assert.Equal(t, err.Error(), "open unknown: no such file or directory")
}
func TestTotalPercentage(t *testing.T) {
items := []struct {
list CoverageList

32
pkg/github/github.go
View File

@ -38,8 +38,17 @@ import (
// It is also the flag when checking whether the target comment exists or not to avoid duplicate
const CommentsPrefix = "The following is the coverage report on the affected files."
// PrComment is the entry which is able to comment on Github Pull Requests
type PrComment struct {
// PrComment is the interface of the entry which is able to comment on Github Pull Requests
type PrComment interface {
CreateGithubComment(commentPrefix string, diffCovList cover.DeltaCovList) (err error)
PostComment(content, commentPrefix string) error
EraseHistoryComment(commentPrefix string) error
GetPrChangedFiles() (files []string, err error)
GetCommentFlag() string
}
// GithubPrComment is the entry which is able to comment on Github Pull Requests
type GithubPrComment struct {
RobotUserName string
RepoOwner string
RepoName string
@ -51,7 +60,7 @@ type PrComment struct {
}
// NewPrClient creates an Client which be able to comment on Github Pull Request
func NewPrClient(githubTokenPath, repoOwner, repoName, prNumStr, botUserName, commentFlag string) *PrComment {
func NewPrClient(githubTokenPath, repoOwner, repoName, prNumStr, botUserName, commentFlag string) *GithubPrComment {
var client *github.Client
// performs automatic retries when connection error occurs or a 500-range response code received (except 501)
@ -72,7 +81,7 @@ func NewPrClient(githubTokenPath, repoOwner, repoName, prNumStr, botUserName, co
tc := oauth2.NewClient(ctx, ts)
client = github.NewClient(tc)
return &PrComment{
return &GithubPrComment{
RobotUserName: botUserName,
RepoOwner: repoOwner,
RepoName: repoName,
@ -85,7 +94,7 @@ func NewPrClient(githubTokenPath, repoOwner, repoName, prNumStr, botUserName, co
}
// CreateGithubComment post github comment of diff coverage
func (c *PrComment) CreateGithubComment(commentPrefix string, diffCovList cover.DeltaCovList) (err error) {
func (c *GithubPrComment) CreateGithubComment(commentPrefix string, diffCovList cover.DeltaCovList) (err error) {
if len(diffCovList) == 0 {
logrus.Printf("Detect 0 files coverage diff, will not comment to github.")
return nil
@ -101,7 +110,7 @@ func (c *PrComment) CreateGithubComment(commentPrefix string, diffCovList cover.
}
// PostComment post comment on github. It erased the old one if existed to avoid duplicate
func (c *PrComment) PostComment(content, commentPrefix string) error {
func (c *GithubPrComment) PostComment(content, commentPrefix string) error {
//step1: erase history similar comment to avoid too many comment for same job
err := c.EraseHistoryComment(commentPrefix)
if err != nil {
@ -121,7 +130,7 @@ func (c *PrComment) PostComment(content, commentPrefix string) error {
}
// EraseHistoryComment erase history similar comment before post again
func (c *PrComment) EraseHistoryComment(commentPrefix string) error {
func (c *GithubPrComment) EraseHistoryComment(commentPrefix string) error {
comments, _, err := c.GithubClient.Issues.ListComments(c.Ctx, c.RepoOwner, c.RepoName, c.PrNumber, nil)
if err != nil {
logrus.Errorf("list PR comments failed.")
@ -142,8 +151,8 @@ func (c *PrComment) EraseHistoryComment(commentPrefix string) error {
return nil
}
//GetPrChangedFiles get github pull request changes file list
func (c *PrComment) GetPrChangedFiles() (files []string, err error) {
// GetPrChangedFiles get github pull request changes file list
func (c *GithubPrComment) GetPrChangedFiles() (files []string, err error) {
var commitFiles []*github.CommitFile
for {
f, resp, err := c.GithubClient.PullRequests.ListFiles(c.Ctx, c.RepoOwner, c.RepoName, c.PrNumber, c.opt)
@ -165,6 +174,11 @@ func (c *PrComment) GetPrChangedFiles() (files []string, err error) {
return
}
// GetCommentFlag get CommentFlag from the GithubPrComment
func (c *GithubPrComment) GetCommentFlag() string {
return c.CommentFlag
}
// GenCommentContent generate github comment content based on diff coverage and commentFlag
func GenCommentContent(commentPrefix string, delta cover.DeltaCovList) string {
var buf bytes.Buffer

18
pkg/github/github_test.go
View File

@ -121,7 +121,7 @@ func TestCreateGithubComment(t *testing.T) {
router.HandlerFunc("DELETE", "/repos/qiniu/goc/issues/comments/1", func(w http.ResponseWriter, r *http.Request) {
})
p := PrComment{
p := GithubPrComment{
RobotUserName: "qiniu-bot",
RepoOwner: "qiniu",
RepoName: "goc",
@ -135,6 +135,12 @@ func TestCreateGithubComment(t *testing.T) {
p.CreateGithubComment("", coverList)
}
func TestCreateGithubCommentError(t *testing.T) {
p := &GithubPrComment{}
err := p.CreateGithubComment("", cover.DeltaCovList{})
assert.NoError(t, err)
}
func TestGetPrChangedFiles(t *testing.T) {
client, router, _, teardown := setup()
defer teardown()
@ -146,7 +152,7 @@ func TestGetPrChangedFiles(t *testing.T) {
fmt.Fprint(w, `[{"filename":"src/qiniu.com/kodo/s3apiv2/bucket/bucket.go"}]`)
})
p := PrComment{
p := GithubPrComment{
RobotUserName: "qiniu-bot",
RepoOwner: "qiniu",
RepoName: "goc",
@ -160,3 +166,11 @@ func TestGetPrChangedFiles(t *testing.T) {
assert.Equal(t, err, nil)
assert.Equal(t, changedFiles, expectFiles)
}
func TestGetCommentFlag(t *testing.T) {
p := GithubPrComment{
CommentFlag: "flag",
}
flag := p.GetCommentFlag()
assert.Equal(t, flag, p.CommentFlag)
}

90
pkg/prow/job.go
View File

@ -54,9 +54,9 @@ type Job struct {
PostSubmitCoverProfile string
CovThreshold int
LocalProfilePath string
QiniuClient *qiniu.Client
LocalArtifacts *qiniu.Artifacts
GithubComment *github.PrComment
QiniuClient qiniu.Client
LocalArtifacts qiniu.Artifacts
GithubComment github.PrComment
FullDiff bool
}
@ -67,32 +67,16 @@ func (j *Job) Fetch(BuildID, name string) []byte {
// RunPresubmit run a presubmit job
func (j *Job) RunPresubmit() error {
var changedFiles []string
var deltaCovList cover.DeltaCovList
// step1: get github pull request changed files' name
if !j.FullDiff {
var ghChangedFiles, err = j.GithubComment.GetPrChangedFiles()
if err != nil {
logrus.WithError(err).Fatalf("Get pull request changed file failed.")
}
if len(ghChangedFiles) == 0 {
logrus.Printf("0 files changed in github pull request, don't need to run coverage profile in presubmit.\n")
return nil
}
changedFiles = trimGhFileToProfile(ghChangedFiles)
}
// step2: get local profile cov
// step1: get local profile cov
localP, err := cover.ReadFileToCoverList(j.LocalProfilePath)
if err != nil {
logrus.WithError(err).Fatalf("failed to get remote cover profile")
return fmt.Errorf("failed to get remote cover profile: %s", err.Error())
}
//step3: find the remote healthy cover profile from qiniu bucket
//step2: find the remote healthy cover profile from qiniu bucket
remoteProfile, err := qiniu.FindBaseProfileFromQiniu(j.QiniuClient, j.PostSubmitJob, j.PostSubmitCoverProfile)
if err != nil {
logrus.WithError(err).Fatalf("failed to get remote cover profile")
return fmt.Errorf("failed to get remote cover profile: %s", err.Error())
}
if remoteProfile == nil {
logrus.Infof("get non healthy remoteProfile, do nothing")
@ -100,36 +84,30 @@ func (j *Job) RunPresubmit() error {
}
baseP, err := cover.CovList(bytes.NewReader(remoteProfile))
if err != nil {
logrus.WithError(err).Fatalf("failed to get remote cover profile")
return fmt.Errorf("failed to get remote cover profile: %s", err.Error())
}
// step4: calculate diff cov between local and remote profile
if !j.FullDiff {
deltaCovList = cover.GetChFileDeltaCov(localP, baseP, changedFiles)
} else {
deltaCovList = cover.GetDeltaCov(localP, baseP)
logrus.Infof("get delta file name is:")
for _, d := range deltaCovList {
logrus.Infof("%s", d.FileName)
changedFiles = append(changedFiles, d.FileName)
}
// step3: get github pull request changed files' name and calculate diff cov between local and remote profile
changedFiles, deltaCovList, err := getFilesAndCovList(j.FullDiff, j.GithubComment, localP, baseP)
if err != nil {
return fmt.Errorf("Get files and covlist failed: %s", err.Error())
}
// step5: generate changed file html coverage
// step4: generate changed file html coverage
err = j.WriteChangedCov(changedFiles)
if err != nil {
logrus.WithError(err).Fatalf("filter local profile to %s with changed files failed", j.LocalArtifacts.ChangedProfileName)
return fmt.Errorf("filter local profile to %s with changed files failed: %s", j.LocalArtifacts.GetChangedProfileName(), err.Error())
}
err = j.CreateChangedCovHtml()
if err != nil {
logrus.WithError(err).Fatalf("create changed file related coverage html failed")
return fmt.Errorf("create changed file related coverage html failed: %s", err.Error())
}
j.SetDeltaCovLinks(deltaCovList)
// step6: post comment to github
// step5: post comment to github
commentPrefix := github.CommentsPrefix
if j.GithubComment.CommentFlag != "" {
commentPrefix = fmt.Sprintf("**%s** ", j.GithubComment.CommentFlag) + commentPrefix
if j.GithubComment.GetCommentFlag() != "" {
commentPrefix = fmt.Sprintf("**%s** ", j.GithubComment.GetCommentFlag()) + commentPrefix
}
if len(deltaCovList) > 0 {
totalDelta := cover.PercentStr(cover.TotalDelta(localP, baseP))
@ -137,7 +115,7 @@ func (j *Job) RunPresubmit() error {
}
err = j.GithubComment.CreateGithubComment(commentPrefix, deltaCovList)
if err != nil {
logrus.WithError(err).Fatalf("Post comment to github failed.")
return fmt.Errorf("Post comment to github failed: %s", err.Error())
}
return nil
@ -217,10 +195,10 @@ func (j *Job) SetDeltaCovLinks(c cover.DeltaCovList) {
// CreateChangedCovHtml create changed file related coverage html base on the local artifact
func (j *Job) CreateChangedCovHtml() error {
if j.LocalArtifacts.ChangedProfileName == "" {
if j.LocalArtifacts.GetChangedProfileName() == "" {
logrus.Errorf("param LocalArtifacts.ChangedProfileName is empty")
}
pathProfileCov := j.LocalArtifacts.ChangedProfileName
pathProfileCov := j.LocalArtifacts.GetChangedProfileName()
pathHtmlCov := path.Join(os.Getenv("ARTIFACTS"), j.HtmlProfile())
cmdTxt := fmt.Sprintf("go tool cover -html=%s -o %s", pathProfileCov, pathHtmlCov)
logrus.Printf("Running command '%s'\n", cmdTxt)
@ -231,3 +209,29 @@ func (j *Job) CreateChangedCovHtml() error {
}
return err
}
func getFilesAndCovList(fullDiff bool, prComment github.PrComment, localP, baseP cover.CoverageList) (changedFiles []string, deltaCovList cover.DeltaCovList, err error) {
if !fullDiff {
// get github pull request changed files' name
var ghChangedFiles, err = prComment.GetPrChangedFiles()
if err != nil {
return nil, nil, fmt.Errorf("Get pull request changed file failed: %s", err.Error())
}
if len(ghChangedFiles) == 0 {
logrus.Printf("0 files changed in github pull request, don't need to run coverage profile in presubmit.\n")
return nil, nil, nil
}
changedFiles = trimGhFileToProfile(ghChangedFiles)
// calculate diff cov between local and remote profile
deltaCovList = cover.GetChFileDeltaCov(localP, baseP, changedFiles)
logrus.Printf("Get changed files and delta cover list success. ChangedFiles: [%+v], DeltaCovList: [%+v]", changedFiles, deltaCovList)
return changedFiles, deltaCovList, nil
}
deltaCovList = cover.GetDeltaCov(localP, baseP)
for _, d := range deltaCovList {
changedFiles = append(changedFiles, d.FileName)
}
logrus.Printf("Get all files and delta cover list success. Files: [%+v], DeltaCovList: [%+v]", changedFiles, deltaCovList)
return changedFiles, deltaCovList, nil
}

265
pkg/prow/job_test.go
View File

@ -17,18 +17,99 @@
package prow
import (
"context"
"errors"
"fmt"
"io/ioutil"
"os"
"path"
"testing"
"time"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/qiniu/goc/pkg/cover"
"github.com/qiniu/goc/pkg/github"
"github.com/qiniu/goc/pkg/qiniu"
)
var (
defaultContent = `mode: atomic
qiniu.com/kodo/bd/bdgetter/source.go:19.118,22.2 2 0
qiniu.com/kodo/bd/bdgetter/source.go:37.34,39.2 1 0
qiniu.com/kodo/bd/pfd/locker/app/qboxbdlocker/main.go:50.2,53.52 4 1
qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go:33.51,35.2 1 0`
defaultLocalPath = "local.cov"
defaultChangedPath = "changed.cov"
)
type MockQnClient struct {
QiniuObjectHandleRes qiniu.ObjectHandle
ReadObjectRes []byte
ReadObjectErr error
ListAllRes []string
ListAllErr error
GetAccessURLRes string
GetArtifactDetailsRes *qiniu.LogHistoryTemplate
GetArtifactDetailsErr error
ListSubDirsRes []string
ListSubDirsErr error
}
func (s *MockQnClient) QiniuObjectHandle(key string) qiniu.ObjectHandle {
return s.QiniuObjectHandleRes
}
func (s *MockQnClient) ReadObject(key string) ([]byte, error) {
return s.ReadObjectRes, s.ReadObjectErr
}
func (s *MockQnClient) ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error) {
return s.ListAllRes, s.ListAllErr
}
func (s *MockQnClient) GetAccessURL(key string, timeout time.Duration) string {
return s.GetAccessURLRes
}
func (s *MockQnClient) GetArtifactDetails(key string) (*qiniu.LogHistoryTemplate, error) {
return s.GetArtifactDetailsRes, s.GetArtifactDetailsErr
}
func (s *MockQnClient) ListSubDirs(prefix string) ([]string, error) {
return s.ListSubDirsRes, s.ListSubDirsErr
}
type MockPrComment struct {
GetPrChangedFilesRes []string
GetPrChangedFilesErr error
PostCommentErr error
EraseHistoryCommentErr error
CreateGithubCommentErr error
CommentFlag string
}
func (s *MockPrComment) GetPrChangedFiles() (files []string, err error) {
return s.GetPrChangedFilesRes, s.GetPrChangedFilesErr
}
func (s *MockPrComment) PostComment(content, commentPrefix string) error {
return s.PostCommentErr
}
func (s *MockPrComment) EraseHistoryComment(commentPrefix string) error {
return s.EraseHistoryCommentErr
}
func (s *MockPrComment) CreateGithubComment(commentPrefix string, diffCovList cover.DeltaCovList) (err error) {
return s.CreateGithubCommentErr
}
func (s *MockPrComment) GetCommentFlag() string {
return s.CommentFlag
}
func TestTrimGhFileToProfile(t *testing.T) {
items := []struct {
inputFiles []string
@ -54,13 +135,9 @@ func setup(path, content string) {
}
func TestWriteChangedCov(t *testing.T) {
path := "local.cov"
path := defaultLocalPath
savePath := qiniu.ChangedProfileName
content := `mode: atomic
qiniu.com/kodo/bd/bdgetter/source.go:19.118,22.2 2 0
qiniu.com/kodo/bd/bdgetter/source.go:37.34,39.2 1 0
qiniu.com/kodo/bd/pfd/locker/app/qboxbdlocker/main.go:50.2,53.52 4 1
qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go:33.51,35.2 1 0`
content := defaultContent
changedFiles := []string{"qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go"}
expectContent := `mode: atomic
qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go:33.51,35.2 1 0
@ -71,7 +148,7 @@ qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go:33.51,35.2 1 0
defer os.Remove(savePath)
j := &Job{
LocalProfilePath: path,
LocalArtifacts: &qiniu.Artifacts{ChangedProfileName: savePath},
LocalArtifacts: &qiniu.ProfileArtifacts{ChangedProfileName: savePath},
}
j.WriteChangedCov(changedFiles)
@ -95,10 +172,8 @@ func TestRunPresubmitFulldiff(t *testing.T) {
//mock local profile
pwd, err := os.Getwd()
if err != nil {
logrus.WithError(err).Fatalf("get pwd failed")
}
localPath := "local.cov"
assert.NoError(t, err)
localPath := defaultLocalPath
localProfileContent := `mode: atomic
"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30
"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0`
@ -130,12 +205,176 @@ func TestRunPresubmitFulldiff(t *testing.T) {
PostSubmitJob: "kodo-postsubmits-go-st-coverage",
PostSubmitCoverProfile: "filterd.cov",
LocalProfilePath: localPath,
LocalArtifacts: &qiniu.Artifacts{ChangedProfileName: ChangedProfilePath},
LocalArtifacts: &qiniu.ProfileArtifacts{ChangedProfileName: ChangedProfilePath},
QiniuClient: qc,
GithubComment: prClient,
FullDiff: true,
}
defer os.Remove(path.Join(os.Getenv("ARTIFACTS"), j.HtmlProfile()))
j.RunPresubmit()
err = j.RunPresubmit()
assert.NoError(t, err)
}
func TestRunPresubmitError(t *testing.T) {
items := []struct {
prepare bool // prepare local profile
j Job
err string
}{
{
prepare: false,
j: Job{
LocalProfilePath: "unkown",
},
err: "no such file or directory",
},
{
prepare: true,
j: Job{
LocalProfilePath: defaultLocalPath,
QiniuClient: &MockQnClient{},
},
},
{
prepare: true,
j: Job{
LocalProfilePath: defaultLocalPath,
QiniuClient: &MockQnClient{ListSubDirsErr: errors.New("mock error")},
},
err: "mock error",
},
{
prepare: true,
j: Job{
LocalProfilePath: defaultLocalPath,
QiniuClient: &MockProfileQnClient{},
GithubComment: &MockPrComment{GetPrChangedFilesRes: []string{"qiniu.com/kodo/apiserver/server/main.go"}},
FullDiff: true,
LocalArtifacts: &qiniu.ProfileArtifacts{ChangedProfileName: defaultChangedPath},
},
err: "",
},
}
for _, tc := range items {
if tc.prepare {
path := defaultLocalPath
setup(path, defaultContent)
defer os.Remove(path)
defer os.Remove(defaultChangedPath)
}
err := tc.j.RunPresubmit()
if tc.err == "" {
assert.NoError(t, err)
} else {
assert.Contains(t, err.Error(), tc.err)
}
}
}
type MockProfileQnClient struct {
*MockQnClient
}
func (s *MockProfileQnClient) ListSubDirs(prefix string) ([]string, error) {
return []string{defaultContent}, nil
}
func (s *MockProfileQnClient) ReadObject(key string) ([]byte, error) {
logrus.Info(key)
if key == "logs/1/finished.json" {
return []byte(`{"timestamp":1590750306,"passed":true,"result":"SUCCESS","repo-version":"76433418ea48aae57af028f9cb2fa3735ce08c7d"}`), nil
}
return []byte(""), nil
}
func TestGetFilesAndCovList(t *testing.T) {
items := []struct {
fullDiff bool
prComment github.PrComment
localP cover.CoverageList
baseP cover.CoverageList
err string
lenFiles int
lenCovList int
}{
{
fullDiff: true,
prComment: &MockPrComment{},
localP: cover.CoverageList{
{FileName: "qiniu.com/kodo/apiserver/server/main.go", NCoveredStmts: 2, NAllStmts: 2},
{FileName: "qiniu.com/kodo/apiserver/server/test.go", NCoveredStmts: 2, NAllStmts: 2},
},
baseP: cover.CoverageList{
{FileName: "qiniu.com/kodo/apiserver/server/main.go", NCoveredStmts: 1, NAllStmts: 2},
{FileName: "qiniu.com/kodo/apiserver/server/test.go", NCoveredStmts: 1, NAllStmts: 2},
},
lenFiles: 2,
lenCovList: 2,
},
{
fullDiff: false,
prComment: &MockPrComment{GetPrChangedFilesErr: errors.New("mock error")},
err: "mock error",
},
{
fullDiff: false,
prComment: &MockPrComment{},
lenFiles: 0,
lenCovList: 0,
},
{
fullDiff: false,
prComment: &MockPrComment{GetPrChangedFilesRes: []string{"qiniu.com/kodo/apiserver/server/main.go"}},
localP: cover.CoverageList{
{FileName: "qiniu.com/kodo/apiserver/server/main.go", NCoveredStmts: 2, NAllStmts: 2},
{FileName: "qiniu.com/kodo/apiserver/server/test.go", NCoveredStmts: 2, NAllStmts: 2},
},
baseP: cover.CoverageList{
{FileName: "qiniu.com/kodo/apiserver/server/main.go", NCoveredStmts: 1, NAllStmts: 2},
{FileName: "qiniu.com/kodo/apiserver/server/test.go", NCoveredStmts: 1, NAllStmts: 2},
},
lenFiles: 1,
lenCovList: 1,
},
}
for i, tc := range items {
fmt.Println(i)
files, covList, err := getFilesAndCovList(tc.fullDiff, tc.prComment, tc.localP, tc.baseP)
if err != nil {
assert.Contains(t, err.Error(), tc.err)
} else {
assert.Equal(t, len(files), tc.lenFiles)
assert.Equal(t, len(covList), tc.lenCovList)
}
}
}
func TestSetDeltaCovLinks(t *testing.T) {
covList := cover.DeltaCovList{{FileName: "file1", BasePer: "5%", NewPer: "5%", DeltaPer: "0"}}
j := &Job{
QiniuClient: &MockQnClient{},
}
j.SetDeltaCovLinks(covList)
}
// functions to be done
func TestRunPostsubmit(t *testing.T) {
j := &Job{}
err := j.RunPostsubmit()
assert.NoError(t, err)
}
func TestRunPeriodic(t *testing.T) {
j := &Job{}
err := j.RunPeriodic()
assert.NoError(t, err)
}
func TestFetch(t *testing.T) {
j := &Job{}
res := j.Fetch("buidID", "name")
assert.Equal(t, res, []byte{})
}

38
pkg/qiniu/client.go
View File

@ -43,23 +43,33 @@ type Config struct {
Domain string `json:"domain"`
}
// Client for the operation with qiniu cloud
type Client struct {
// Client is the interface contains the operation with qiniu cloud
type Client interface {
QiniuObjectHandle(key string) ObjectHandle
ReadObject(key string) ([]byte, error)
ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error)
GetAccessURL(key string, timeout time.Duration) string
GetArtifactDetails(key string) (*LogHistoryTemplate, error)
ListSubDirs(prefix string) ([]string, error)
}
// QnClient for the operation with qiniu cloud
type QnClient struct {
cfg *Config
BucketManager *storage.BucketManager
}
// NewClient creates a new client to work with qiniu cloud
func NewClient(cfg *Config) *Client {
return &Client{
// NewClient creates a new QnClient to work with qiniu cloud
func NewClient(cfg *Config) *QnClient {
return &QnClient{
cfg: cfg,
BucketManager: storage.NewBucketManager(qbox.NewMac(cfg.AccessKey, cfg.SecretKey), nil),
}
}
// QiniuObjectHandle construct a object hanle to access file in qiniu
func (q *Client) QiniuObjectHandle(key string) *ObjectHandle {
return &ObjectHandle{
func (q *QnClient) QiniuObjectHandle(key string) ObjectHandle {
return &QnObjectHandle{
key: key,
cfg: q.cfg,
bm: q.BucketManager,
@ -69,7 +79,7 @@ func (q *Client) QiniuObjectHandle(key string) *ObjectHandle {
}
// ReadObject to read all the content of key
func (q *Client) ReadObject(key string) ([]byte, error) {
func (q *QnClient) ReadObject(key string) ([]byte, error) {
objectHandle := q.QiniuObjectHandle(key)
reader, err := objectHandle.NewReader(context.Background())
if err != nil {
@ -80,7 +90,7 @@ func (q *Client) ReadObject(key string) ([]byte, error) {
}
// ListAll to list all the files with contains the expected prefix
func (q *Client) ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error) {
func (q *QnClient) ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error) {
var files []string
artifacts, err := q.listEntries(prefix, delimiter)
if err != nil {
@ -94,8 +104,8 @@ func (q *Client) ListAll(ctx context.Context, prefix string, delimiter string) (
return files, nil
}
// ListAll to list all the entries with contains the expected prefix
func (q *Client) listEntries(prefix string, delimiter string) ([]storage.ListItem, error) {
// listEntries to list all the entries with contains the expected prefix
func (q *QnClient) listEntries(prefix string, delimiter string) ([]storage.ListItem, error) {
var marker string
var artifacts []storage.ListItem
@ -124,7 +134,7 @@ func (q *Client) listEntries(prefix string, delimiter string) ([]storage.ListIte
}
// GetAccessURL return a url which can access artifact directly in qiniu
func (q *Client) GetAccessURL(key string, timeout time.Duration) string {
func (q *QnClient) GetAccessURL(key string, timeout time.Duration) string {
deadline := time.Now().Add(timeout).Unix()
return storage.MakePrivateURL(qbox.NewMac(q.cfg.AccessKey, q.cfg.SecretKey), q.cfg.Domain, key, deadline)
}
@ -143,7 +153,7 @@ type logHistoryItem struct {
}
// Artifacts lists all artifacts available for the given job source
func (q *Client) GetArtifactDetails(key string) (*LogHistoryTemplate, error) {
func (q *QnClient) GetArtifactDetails(key string) (*LogHistoryTemplate, error) {
tmpl := new(LogHistoryTemplate)
item := logHistoryItem{}
listStart := time.Now()
@ -183,7 +193,7 @@ func timeConv(ptime int64) string {
return tm.Format("2006-01-02 03:04:05 PM")
}
func (q *Client) ListSubDirs(prefix string) ([]string, error) {
func (q *QnClient) ListSubDirs(prefix string) ([]string, error) {
var dirs []string
var marker string

2
pkg/qiniu/mock.go
View File

@ -27,7 +27,7 @@ import (
)
// MockQiniuServer simulate qiniu cloud for testing
func MockQiniuServer(config *Config) (client *Client, router *httprouter.Router, serverURL string, teardown func()) {
func MockQiniuServer(config *Config) (client *QnClient, router *httprouter.Router, serverURL string, teardown func()) {
// router is the HTTP request multiplexer used with the test server.
router = httprouter.New()

20
pkg/qiniu/object.go
View File

@ -29,8 +29,14 @@ import (
"github.com/sirupsen/logrus"
)
// ObjectHandle provides operations on an object in a qiniu cloud bucket
type ObjectHandle struct {
// ObjectHandle is the interface contains the operations on an object in a qiniu cloud bucket
type ObjectHandle interface {
NewReader(ctx context.Context) (io.ReadCloser, error)
NewRangeReader(ctx context.Context, offset, length int64) (io.ReadCloser, error)
}
// QnObjectHandle provides operations on an object in a qiniu cloud bucket
type QnObjectHandle struct {
key string
cfg *Config
bm *storage.BucketManager
@ -38,22 +44,16 @@ type ObjectHandle struct {
client *client.Client
}
// Attrs get the object's metainfo
func (o *ObjectHandle) Attrs(ctx context.Context) (storage.FileInfo, error) {
//TODO(CarlJi): need retry when errors
return o.bm.Stat(o.cfg.Bucket, o.key)
}
// NewReader creates a reader to read the contents of the object.
// ErrObjectNotExist will be returned if the object is not found.
// The caller must call Close on the returned Reader when done reading.
func (o *ObjectHandle) NewReader(ctx context.Context) (io.ReadCloser, error) {
func (o *QnObjectHandle) NewReader(ctx context.Context) (io.ReadCloser, error) {
return o.NewRangeReader(ctx, 0, -1)
}
// NewRangeReader reads parts of an object, reading at most length bytes starting
// from the given offset. If length is negative, the object is read until the end.
func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) (io.ReadCloser, error) {
func (o *QnObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) (io.ReadCloser, error) {
verb := "GET"
if length == 0 {
verb = "HEAD"

6
pkg/qiniu/object_test.go
View File

@ -41,7 +41,7 @@ func TestNewRangeReader(t *testing.T) {
MockPrivateDomainUrl(router, 0)
oh := &ObjectHandle{
oh := &QnObjectHandle{
key: "key",
cfg: cfg,
bm: nil,
@ -78,7 +78,7 @@ func TestNewRangeReaderWithTimeoutAndRecover(t *testing.T) {
MockPrivateDomainUrl(router, 2)
oh := &ObjectHandle{
oh := &QnObjectHandle{
key: "key",
cfg: cfg,
bm: nil,
@ -116,7 +116,7 @@ func TestNewRangeReaderWithTimeoutNoRecover(t *testing.T) {
MockPrivateDomainUrl(router, 12)
oh := &ObjectHandle{
oh := &QnObjectHandle{
key: "key",
cfg: cfg,
bm: nil,

22
pkg/qiniu/qnPresubmit.go
View File

@ -70,7 +70,7 @@ func isBuildSucceeded(jsonText []byte) bool {
// FindBaseProfileFromQiniu finds the coverage profile file from the latest healthy build
// stored in given gcs directory
func FindBaseProfileFromQiniu(qc *Client, prowJobName, covProfileName string) ([]byte, error) {
func FindBaseProfileFromQiniu(qc Client, prowJobName, covProfileName string) ([]byte, error) {
dirOfJob := path.Join("logs", prowJobName)
prefix := dirOfJob + "/"
strBuilds, err := qc.ListSubDirs(prefix)
@ -107,20 +107,27 @@ func FindBaseProfileFromQiniu(qc *Client, prowJobName, covProfileName string) ([
return qc.ReadObject(profilePath)
}
// Artifacts prepresents the rule to store test artifacts in prow
type Artifacts struct {
// Artifacts is the interface of the rule to store test artifacts in prow
type Artifacts interface {
ProfilePath() string
CreateChangedProfile() *os.File
GetChangedProfileName() string
}
// ProfileArtifacts presents the rule to store test artifacts in prow
type ProfileArtifacts struct {
Directory string
ProfileName string
ChangedProfileName string // create temporary to save changed file related coverage profile
}
// ProfilePath returns a full path for profile
func (a *Artifacts) ProfilePath() string {
func (a *ProfileArtifacts) ProfilePath() string {
return path.Join(a.Directory, a.ProfileName)
}
// CreateChangedProfile creates a profile in order to store the most related files based on Github Pull Request
func (a *Artifacts) CreateChangedProfile() *os.File {
func (a *ProfileArtifacts) CreateChangedProfile() *os.File {
if a.ChangedProfileName == "" {
log.Fatalf("param Artifacts.ChangedProfileName should not be empty")
}
@ -132,3 +139,8 @@ func (a *Artifacts) CreateChangedProfile() *os.File {
return p
}
// GetChangedProfileName get ChangedProfileName of the ProfileArtifacts
func (a *ProfileArtifacts) GetChangedProfileName() string {
return a.ChangedProfileName
}

29
pkg/qiniu/qnPresubmit_test.go
View File

@ -17,6 +17,7 @@
package qiniu
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
@ -39,3 +40,31 @@ func TestFindBaseProfileFromQiniu(t *testing.T) {
assert.Equal(t, err, nil)
assert.Equal(t, string(getProfile), mockProfileContent)
}
func TestArtifacts_ProfilePath(t *testing.T) {
p := &ProfileArtifacts{
Directory: "directory/",
ProfileName: "profile",
}
profilePath := p.ProfilePath()
assert.Equal(t, profilePath, "directory/profile")
}
func TestProfileArtifacts_CreateChangedProfile(t *testing.T) {
p := &ProfileArtifacts{
ChangedProfileName: "test.cov",
}
file := p.CreateChangedProfile()
file.Close()
defer os.Remove(p.ChangedProfileName)
_, err := os.Stat(p.ChangedProfileName)
assert.NoError(t, err)
}
func TestProfileArtifacts_GetChangedProfileName(t *testing.T) {
p := &ProfileArtifacts{
ChangedProfileName: "change.cov",
}
name := p.GetChangedProfileName()
assert.Equal(t, name, "change.cov")
}