ut case add

This commit is contained in:
tongjingran 2020-07-24 17:27:03 +08:00
parent 1917e39977
commit 8c719effc8
13 changed files with 292 additions and 86 deletions

View File

@ -160,7 +160,7 @@ func doDiffUnderProw(cmd *cobra.Command, args []string) {
if qiniuCredential == "" { if qiniuCredential == "" {
logrus.Fatalf("qiniu credential not provided") logrus.Fatalf("qiniu credential not provided")
} }
var qc *qiniu.Client var qc qiniu.Client
var conf qiniu.Config var conf qiniu.Config
files, err := ioutil.ReadFile(*&qiniuCredential) files, err := ioutil.ReadFile(*&qiniuCredential)
if err != nil { if err != nil {

View File

@ -93,6 +93,12 @@ func TestCovList(t *testing.T) {
} }
} }
func TestReadFileToCoverList(t *testing.T) {
path := "unknown"
_, err := ReadFileToCoverList(path)
assert.Equal(t, err.Error(), "open unknown: no such file or directory")
}
func TestTotalPercentage(t *testing.T) { func TestTotalPercentage(t *testing.T) {
items := []struct { items := []struct {
list CoverageList list CoverageList

View File

@ -38,8 +38,16 @@ import (
// It is also the flag when checking whether the target comment exists or not to avoid duplicate // It is also the flag when checking whether the target comment exists or not to avoid duplicate
const CommentsPrefix = "The following is the coverage report on the affected files." const CommentsPrefix = "The following is the coverage report on the affected files."
// PrComment is the entry which is able to comment on Github Pull Requests type PrComment interface {
type PrComment struct { CreateGithubComment(commentPrefix string, diffCovList cover.DeltaCovList) (err error)
PostComment(content, commentPrefix string) error
EraseHistoryComment(commentPrefix string) error
GetPrChangedFiles() (files []string, err error)
GetCommentFlag() string
}
// GithubPrComment is the entry which is able to comment on Github Pull Requests
type GithubPrComment struct {
RobotUserName string RobotUserName string
RepoOwner string RepoOwner string
RepoName string RepoName string
@ -51,7 +59,7 @@ type PrComment struct {
} }
// NewPrClient creates an Client which be able to comment on Github Pull Request // NewPrClient creates an Client which be able to comment on Github Pull Request
func NewPrClient(githubTokenPath, repoOwner, repoName, prNumStr, botUserName, commentFlag string) *PrComment { func NewPrClient(githubTokenPath, repoOwner, repoName, prNumStr, botUserName, commentFlag string) *GithubPrComment {
var client *github.Client var client *github.Client
// performs automatic retries when connection error occurs or a 500-range response code received (except 501) // performs automatic retries when connection error occurs or a 500-range response code received (except 501)
@ -72,7 +80,7 @@ func NewPrClient(githubTokenPath, repoOwner, repoName, prNumStr, botUserName, co
tc := oauth2.NewClient(ctx, ts) tc := oauth2.NewClient(ctx, ts)
client = github.NewClient(tc) client = github.NewClient(tc)
return &PrComment{ return &GithubPrComment{
RobotUserName: botUserName, RobotUserName: botUserName,
RepoOwner: repoOwner, RepoOwner: repoOwner,
RepoName: repoName, RepoName: repoName,
@ -85,7 +93,7 @@ func NewPrClient(githubTokenPath, repoOwner, repoName, prNumStr, botUserName, co
} }
// CreateGithubComment post github comment of diff coverage // CreateGithubComment post github comment of diff coverage
func (c *PrComment) CreateGithubComment(commentPrefix string, diffCovList cover.DeltaCovList) (err error) { func (c *GithubPrComment) CreateGithubComment(commentPrefix string, diffCovList cover.DeltaCovList) (err error) {
if len(diffCovList) == 0 { if len(diffCovList) == 0 {
logrus.Printf("Detect 0 files coverage diff, will not comment to github.") logrus.Printf("Detect 0 files coverage diff, will not comment to github.")
return nil return nil
@ -101,7 +109,7 @@ func (c *PrComment) CreateGithubComment(commentPrefix string, diffCovList cover.
} }
// PostComment post comment on github. It erased the old one if existed to avoid duplicate // PostComment post comment on github. It erased the old one if existed to avoid duplicate
func (c *PrComment) PostComment(content, commentPrefix string) error { func (c *GithubPrComment) PostComment(content, commentPrefix string) error {
//step1: erase history similar comment to avoid too many comment for same job //step1: erase history similar comment to avoid too many comment for same job
err := c.EraseHistoryComment(commentPrefix) err := c.EraseHistoryComment(commentPrefix)
if err != nil { if err != nil {
@ -121,7 +129,7 @@ func (c *PrComment) PostComment(content, commentPrefix string) error {
} }
// EraseHistoryComment erase history similar comment before post again // EraseHistoryComment erase history similar comment before post again
func (c *PrComment) EraseHistoryComment(commentPrefix string) error { func (c *GithubPrComment) EraseHistoryComment(commentPrefix string) error {
comments, _, err := c.GithubClient.Issues.ListComments(c.Ctx, c.RepoOwner, c.RepoName, c.PrNumber, nil) comments, _, err := c.GithubClient.Issues.ListComments(c.Ctx, c.RepoOwner, c.RepoName, c.PrNumber, nil)
if err != nil { if err != nil {
logrus.Errorf("list PR comments failed.") logrus.Errorf("list PR comments failed.")
@ -143,7 +151,7 @@ func (c *PrComment) EraseHistoryComment(commentPrefix string) error {
} }
//GetPrChangedFiles get github pull request changes file list //GetPrChangedFiles get github pull request changes file list
func (c *PrComment) GetPrChangedFiles() (files []string, err error) { func (c *GithubPrComment) GetPrChangedFiles() (files []string, err error) {
var commitFiles []*github.CommitFile var commitFiles []*github.CommitFile
for { for {
f, resp, err := c.GithubClient.PullRequests.ListFiles(c.Ctx, c.RepoOwner, c.RepoName, c.PrNumber, c.opt) f, resp, err := c.GithubClient.PullRequests.ListFiles(c.Ctx, c.RepoOwner, c.RepoName, c.PrNumber, c.opt)
@ -165,6 +173,11 @@ func (c *PrComment) GetPrChangedFiles() (files []string, err error) {
return return
} }
// GetCommentFlag get CommentFlag from the GithubPrComment
func (c *GithubPrComment) GetCommentFlag() string {
return c.CommentFlag
}
// GenCommentContent generate github comment content based on diff coverage and commentFlag // GenCommentContent generate github comment content based on diff coverage and commentFlag
func GenCommentContent(commentPrefix string, delta cover.DeltaCovList) string { func GenCommentContent(commentPrefix string, delta cover.DeltaCovList) string {
var buf bytes.Buffer var buf bytes.Buffer

View File

@ -121,7 +121,7 @@ func TestCreateGithubComment(t *testing.T) {
router.HandlerFunc("DELETE", "/repos/qiniu/goc/issues/comments/1", func(w http.ResponseWriter, r *http.Request) { router.HandlerFunc("DELETE", "/repos/qiniu/goc/issues/comments/1", func(w http.ResponseWriter, r *http.Request) {
}) })
p := PrComment{ p := GithubPrComment{
RobotUserName: "qiniu-bot", RobotUserName: "qiniu-bot",
RepoOwner: "qiniu", RepoOwner: "qiniu",
RepoName: "goc", RepoName: "goc",
@ -146,7 +146,7 @@ func TestGetPrChangedFiles(t *testing.T) {
fmt.Fprint(w, `[{"filename":"src/qiniu.com/kodo/s3apiv2/bucket/bucket.go"}]`) fmt.Fprint(w, `[{"filename":"src/qiniu.com/kodo/s3apiv2/bucket/bucket.go"}]`)
}) })
p := PrComment{ p := GithubPrComment{
RobotUserName: "qiniu-bot", RobotUserName: "qiniu-bot",
RepoOwner: "qiniu", RepoOwner: "qiniu",
RepoName: "goc", RepoName: "goc",

View File

@ -0,0 +1,24 @@
package clients
import "github.com/qiniu/goc/pkg/cover"
type MockPrComment struct {
}
func (s *MockPrComment) GetPrChangedFiles() (files []string, err error) {
return nil, nil
}
func (s *MockPrComment) PostComment(content, commentPrefix string) error {
return nil
}
func (s *MockPrComment) EraseHistoryComment(commentPrefix string) error {
return nil
}
func (s *MockPrComment) CreateGithubComment(commentPrefix string, diffCovList cover.DeltaCovList) (err error) {
return nil
}
func (s *MockPrComment) GetCommentFlag() string {
return ""
}

34
pkg/mock/clients/qiniu.go Normal file
View File

@ -0,0 +1,34 @@
package clients
import (
"context"
"github.com/qiniu/goc/pkg/qiniu"
"time"
)
type MockQnClient struct {
}
func (s *MockQnClient) QiniuObjectHandle(key string) qiniu.ObjectHandle {
return nil
}
func (s *MockQnClient) ReadObject(key string) ([]byte, error) {
return nil, nil
}
func (s *MockQnClient) ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error) {
return nil, nil
}
func (s *MockQnClient) GetAccessURL(key string, timeout time.Duration) string {
return ""
}
func (s *MockQnClient) GetArtifactDetails(key string) (*qiniu.LogHistoryTemplate, error) {
return nil, nil
}
func (s *MockQnClient) ListSubDirs(prefix string) ([]string, error) {
return nil, nil
}

View File

@ -54,9 +54,9 @@ type Job struct {
PostSubmitCoverProfile string PostSubmitCoverProfile string
CovThreshold int CovThreshold int
LocalProfilePath string LocalProfilePath string
QiniuClient *qiniu.Client QiniuClient qiniu.Client
LocalArtifacts *qiniu.Artifacts LocalArtifacts *qiniu.Artifacts
GithubComment *github.PrComment GithubComment github.PrComment
FullDiff bool FullDiff bool
} }
@ -67,32 +67,19 @@ func (j *Job) Fetch(BuildID, name string) []byte {
// RunPresubmit run a presubmit job // RunPresubmit run a presubmit job
func (j *Job) RunPresubmit() error { func (j *Job) RunPresubmit() error {
var changedFiles []string // step1: get local profile cov
var deltaCovList cover.DeltaCovList
// step1: get github pull request changed files' name
if !j.FullDiff {
var ghChangedFiles, err = j.GithubComment.GetPrChangedFiles()
if err != nil {
logrus.WithError(err).Fatalf("Get pull request changed file failed.")
}
if len(ghChangedFiles) == 0 {
logrus.Printf("0 files changed in github pull request, don't need to run coverage profile in presubmit.\n")
return nil
}
changedFiles = trimGhFileToProfile(ghChangedFiles)
}
// step2: get local profile cov
localP, err := cover.ReadFileToCoverList(j.LocalProfilePath) localP, err := cover.ReadFileToCoverList(j.LocalProfilePath)
if err != nil { if err != nil {
logrus.WithError(err).Fatalf("failed to get remote cover profile") logrus.Errorf("failed to get remote cover profile: %s", err.Error())
return err
} }
logrus.Warnf("localP: [%+v]", localP)
//step3: find the remote healthy cover profile from qiniu bucket //step2: find the remote healthy cover profile from qiniu bucket
remoteProfile, err := qiniu.FindBaseProfileFromQiniu(j.QiniuClient, j.PostSubmitJob, j.PostSubmitCoverProfile) remoteProfile, err := qiniu.FindBaseProfileFromQiniu(j.QiniuClient, j.PostSubmitJob, j.PostSubmitCoverProfile)
if err != nil { if err != nil {
logrus.WithError(err).Fatalf("failed to get remote cover profile") logrus.Errorf("failed to get remote cover profile: %s", err.Error())
return err
} }
if remoteProfile == nil { if remoteProfile == nil {
logrus.Infof("get non healthy remoteProfile, do nothing") logrus.Infof("get non healthy remoteProfile, do nothing")
@ -100,22 +87,20 @@ func (j *Job) RunPresubmit() error {
} }
baseP, err := cover.CovList(bytes.NewReader(remoteProfile)) baseP, err := cover.CovList(bytes.NewReader(remoteProfile))
if err != nil { if err != nil {
logrus.WithError(err).Fatalf("failed to get remote cover profile") logrus.Errorf("failed to get remote cover profile: %s", err.Error())
return err
} }
// step4: calculate diff cov between local and remote profile logrus.Warnf("baseP: [%+v]", baseP)
if !j.FullDiff {
deltaCovList = cover.GetChFileDeltaCov(localP, baseP, changedFiles) // step3: get github pull request changed files' name and calculate diff cov between local and remote profile
} else { changedFiles, deltaCovList, err := getFilesAndCovList(j.FullDiff, j.GithubComment, localP, baseP)
deltaCovList = cover.GetDeltaCov(localP, baseP) if err != nil {
logrus.Infof("get delta file name is:") logrus.Errorf("Get files and covlist failed: %s", err.Error())
for _, d := range deltaCovList { return err
logrus.Infof("%s", d.FileName)
changedFiles = append(changedFiles, d.FileName)
}
} }
// step5: generate changed file html coverage // step4: generate changed file html coverage
err = j.WriteChangedCov(changedFiles) err = j.WriteChangedCov(changedFiles)
if err != nil { if err != nil {
logrus.WithError(err).Fatalf("filter local profile to %s with changed files failed", j.LocalArtifacts.ChangedProfileName) logrus.WithError(err).Fatalf("filter local profile to %s with changed files failed", j.LocalArtifacts.ChangedProfileName)
@ -128,8 +113,8 @@ func (j *Job) RunPresubmit() error {
// step6: post comment to github // step6: post comment to github
commentPrefix := github.CommentsPrefix commentPrefix := github.CommentsPrefix
if j.GithubComment.CommentFlag != "" { if j.GithubComment.GetCommentFlag() != "" {
commentPrefix = fmt.Sprintf("**%s** ", j.GithubComment.CommentFlag) + commentPrefix commentPrefix = fmt.Sprintf("**%s** ", j.GithubComment.GetCommentFlag()) + commentPrefix
} }
if len(deltaCovList) > 0 { if len(deltaCovList) > 0 {
totalDelta := cover.PercentStr(cover.TotalDelta(localP, baseP)) totalDelta := cover.PercentStr(cover.TotalDelta(localP, baseP))
@ -231,3 +216,31 @@ func (j *Job) CreateChangedCovHtml() error {
} }
return err return err
} }
func getFilesAndCovList(fullDiff bool, prComment github.PrComment, localP, baseP cover.CoverageList) (changedFiles []string, deltaCovList cover.DeltaCovList, err error) {
if !fullDiff {
// get github pull request changed files' name
var ghChangedFiles, err = prComment.GetPrChangedFiles()
if err != nil {
logrus.Errorf("Get pull request changed file failed.")
return nil, nil, err
}
if len(ghChangedFiles) == 0 {
logrus.Printf("0 files changed in github pull request, don't need to run coverage profile in presubmit.\n")
return nil, nil, nil
}
changedFiles = trimGhFileToProfile(ghChangedFiles)
// calculate diff cov between local and remote profile
deltaCovList = cover.GetChFileDeltaCov(localP, baseP, changedFiles)
return changedFiles, deltaCovList, nil
}
deltaCovList = cover.GetDeltaCov(localP, baseP)
logrus.Infof("get delta file name is:")
for _, d := range deltaCovList {
logrus.Infof("%s", d.FileName)
changedFiles = append(changedFiles, d.FileName)
}
return changedFiles, deltaCovList, nil
}

View File

@ -17,16 +17,25 @@
package prow package prow
import ( import (
"errors"
"github.com/qiniu/goc/pkg/github"
"github.com/qiniu/goc/pkg/mock/clients"
"github.com/qiniu/goc/pkg/qiniu"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"io/ioutil" "io/ioutil"
"os" "os"
"path" "path"
"testing" "testing"
)
"github.com/sirupsen/logrus" var (
"github.com/stretchr/testify/assert" defaultContent = `mode: atomic
qiniu.com/kodo/bd/bdgetter/source.go:19.118,22.2 2 0
"github.com/qiniu/goc/pkg/github" qiniu.com/kodo/bd/bdgetter/source.go:37.34,39.2 1 0
"github.com/qiniu/goc/pkg/qiniu" qiniu.com/kodo/bd/pfd/locker/app/qboxbdlocker/main.go:50.2,53.52 4 1
qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go:33.51,35.2 1 0`
defaultLocalPath = "local.cov"
) )
func TestTrimGhFileToProfile(t *testing.T) { func TestTrimGhFileToProfile(t *testing.T) {
@ -54,13 +63,9 @@ func setup(path, content string) {
} }
func TestWriteChangedCov(t *testing.T) { func TestWriteChangedCov(t *testing.T) {
path := "local.cov" path := defaultLocalPath
savePath := qiniu.ChangedProfileName savePath := qiniu.ChangedProfileName
content := `mode: atomic content := defaultContent
qiniu.com/kodo/bd/bdgetter/source.go:19.118,22.2 2 0
qiniu.com/kodo/bd/bdgetter/source.go:37.34,39.2 1 0
qiniu.com/kodo/bd/pfd/locker/app/qboxbdlocker/main.go:50.2,53.52 4 1
qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go:33.51,35.2 1 0`
changedFiles := []string{"qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go"} changedFiles := []string{"qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go"}
expectContent := `mode: atomic expectContent := `mode: atomic
qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go:33.51,35.2 1 0 qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go:33.51,35.2 1 0
@ -95,10 +100,8 @@ func TestRunPresubmitFulldiff(t *testing.T) {
//mock local profile //mock local profile
pwd, err := os.Getwd() pwd, err := os.Getwd()
if err != nil { assert.NoError(t, err)
logrus.WithError(err).Fatalf("get pwd failed") localPath := defaultLocalPath
}
localPath := "local.cov"
localProfileContent := `mode: atomic localProfileContent := `mode: atomic
"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30 "qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30
"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0` "qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0`
@ -137,5 +140,103 @@ func TestRunPresubmitFulldiff(t *testing.T) {
} }
defer os.Remove(path.Join(os.Getenv("ARTIFACTS"), j.HtmlProfile())) defer os.Remove(path.Join(os.Getenv("ARTIFACTS"), j.HtmlProfile()))
j.RunPresubmit() err = j.RunPresubmit()
assert.NoError(t, err)
} }
func TestRunPresubmitErrorGetRemoteCoverProfile(t *testing.T) {
j := &Job{
LocalProfilePath: "unkown",
}
err := j.RunPresubmit()
assert.Contains(t, err.Error(), "no such file or directory")
}
func TestRunPresubmitGetEmptyProfile(t *testing.T) {
path := "local.cov"
setup(path, defaultContent)
defer os.Remove(path)
j := &Job{
LocalProfilePath: path,
QiniuClient: &clients.MockQnClient{},
}
err := j.RunPresubmit()
assert.NoError(t, err)
}
type MockErrorListSubDirsQnClient struct {
*clients.MockQnClient
}
func (s *MockErrorListSubDirsQnClient) ListSubDirs(prefix string) ([]string, error) {
return nil, errors.New("mock error")
}
func TestRunPresubmitErrorGetBaseProfile(t *testing.T) {
path := "local.cov"
setup(path, defaultContent)
defer os.Remove(path)
j := &Job{
LocalProfilePath: path,
QiniuClient: &MockErrorListSubDirsQnClient{},
}
err := j.RunPresubmit()
assert.Contains(t, err.Error(), "mock error")
}
type MockProfileQnClient struct {
*clients.MockQnClient
}
func (s *MockProfileQnClient) ListSubDirs(prefix string) ([]string, error) {
return []string{"1"}, nil
}
func (s *MockProfileQnClient) ReadObject(key string) ([]byte, error) {
logrus.Info(key)
if key == "logs/1/finished.json" {
return []byte(`{"timestamp":1590750306,"passed":true,"result":"SUCCESS","repo-version":"76433418ea48aae57af028f9cb2fa3735ce08c7d"}`), nil
}
return []byte(""), nil
}
// 无法实现coverList报错
//func TestRunPresubmitErrorReadBaseProfile(t *testing.T) {
// path := "local.cov"
// setup(path, defaultContent)
// defer os.Remove(path)
// j := &Job{
// LocalProfilePath: path,
// QiniuClient: &MockProfileQnClient{},
// }
// err := j.RunPresubmit()
// assert.Contains(t, err.Error(), "mock error")
//}
//type MockErrorPrComment struct {
// *clients.MockPrComment
//}
//
//func (s *MockErrorPrComment) GetPrChangedFiles() (files []string, err error) {
// return []string{"aaa"}, nil
//}
//
//func TestGetFilesAndCovList(t *testing.T){
// items := []struct {
// fullDiff bool
// prComment github.PrComment
// localP cover.CoverageList
// baseP cover.CoverageList
// }{
// {
// inputFiles: []string{"src/qiniu.com/kodo/io/io/io_svr.go", "README.md"},
// expectFiles: []string{"qiniu.com/kodo/io/io/io_svr.go", "README.md"},
// },
// }
//
// for _, tc := range items {
// f := trimGhFileToProfile(tc.inputFiles)
// assert.Equal(t, f, tc.expectFiles)
// }
// getFilesAndCovList
//}

View File

@ -43,23 +43,32 @@ type Config struct {
Domain string `json:"domain"` Domain string `json:"domain"`
} }
// Client for the operation with qiniu cloud type Client interface {
type Client struct { QiniuObjectHandle(key string) ObjectHandle
ReadObject(key string) ([]byte, error)
ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error)
GetAccessURL(key string, timeout time.Duration) string
GetArtifactDetails(key string) (*LogHistoryTemplate, error)
ListSubDirs(prefix string) ([]string, error)
}
// QnClient for the operation with qiniu cloud
type QnClient struct {
cfg *Config cfg *Config
BucketManager *storage.BucketManager BucketManager *storage.BucketManager
} }
// NewClient creates a new client to work with qiniu cloud // NewQnClient creates a new QnClient to work with qiniu cloud
func NewClient(cfg *Config) *Client { func NewClient(cfg *Config) *QnClient {
return &Client{ return &QnClient{
cfg: cfg, cfg: cfg,
BucketManager: storage.NewBucketManager(qbox.NewMac(cfg.AccessKey, cfg.SecretKey), nil), BucketManager: storage.NewBucketManager(qbox.NewMac(cfg.AccessKey, cfg.SecretKey), nil),
} }
} }
// QiniuObjectHandle construct a object hanle to access file in qiniu // QiniuObjectHandle construct a object hanle to access file in qiniu
func (q *Client) QiniuObjectHandle(key string) *ObjectHandle { func (q *QnClient) QiniuObjectHandle(key string) ObjectHandle {
return &ObjectHandle{ return &QnObjectHandle{
key: key, key: key,
cfg: q.cfg, cfg: q.cfg,
bm: q.BucketManager, bm: q.BucketManager,
@ -69,7 +78,7 @@ func (q *Client) QiniuObjectHandle(key string) *ObjectHandle {
} }
// ReadObject to read all the content of key // ReadObject to read all the content of key
func (q *Client) ReadObject(key string) ([]byte, error) { func (q *QnClient) ReadObject(key string) ([]byte, error) {
objectHandle := q.QiniuObjectHandle(key) objectHandle := q.QiniuObjectHandle(key)
reader, err := objectHandle.NewReader(context.Background()) reader, err := objectHandle.NewReader(context.Background())
if err != nil { if err != nil {
@ -80,7 +89,7 @@ func (q *Client) ReadObject(key string) ([]byte, error) {
} }
// ListAll to list all the files with contains the expected prefix // ListAll to list all the files with contains the expected prefix
func (q *Client) ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error) { func (q *QnClient) ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error) {
var files []string var files []string
artifacts, err := q.listEntries(prefix, delimiter) artifacts, err := q.listEntries(prefix, delimiter)
if err != nil { if err != nil {
@ -95,7 +104,7 @@ func (q *Client) ListAll(ctx context.Context, prefix string, delimiter string) (
} }
// ListAll to list all the entries with contains the expected prefix // ListAll to list all the entries with contains the expected prefix
func (q *Client) listEntries(prefix string, delimiter string) ([]storage.ListItem, error) { func (q *QnClient) listEntries(prefix string, delimiter string) ([]storage.ListItem, error) {
var marker string var marker string
var artifacts []storage.ListItem var artifacts []storage.ListItem
@ -124,7 +133,7 @@ func (q *Client) listEntries(prefix string, delimiter string) ([]storage.ListIte
} }
// GetAccessURL return a url which can access artifact directly in qiniu // GetAccessURL return a url which can access artifact directly in qiniu
func (q *Client) GetAccessURL(key string, timeout time.Duration) string { func (q *QnClient) GetAccessURL(key string, timeout time.Duration) string {
deadline := time.Now().Add(timeout).Unix() deadline := time.Now().Add(timeout).Unix()
return storage.MakePrivateURL(qbox.NewMac(q.cfg.AccessKey, q.cfg.SecretKey), q.cfg.Domain, key, deadline) return storage.MakePrivateURL(qbox.NewMac(q.cfg.AccessKey, q.cfg.SecretKey), q.cfg.Domain, key, deadline)
} }
@ -143,7 +152,7 @@ type logHistoryItem struct {
} }
// Artifacts lists all artifacts available for the given job source // Artifacts lists all artifacts available for the given job source
func (q *Client) GetArtifactDetails(key string) (*LogHistoryTemplate, error) { func (q *QnClient) GetArtifactDetails(key string) (*LogHistoryTemplate, error) {
tmpl := new(LogHistoryTemplate) tmpl := new(LogHistoryTemplate)
item := logHistoryItem{} item := logHistoryItem{}
listStart := time.Now() listStart := time.Now()
@ -183,7 +192,7 @@ func timeConv(ptime int64) string {
return tm.Format("2006-01-02 03:04:05 PM") return tm.Format("2006-01-02 03:04:05 PM")
} }
func (q *Client) ListSubDirs(prefix string) ([]string, error) { func (q *QnClient) ListSubDirs(prefix string) ([]string, error) {
var dirs []string var dirs []string
var marker string var marker string

View File

@ -27,7 +27,7 @@ import (
) )
// MockQiniuServer simulate qiniu cloud for testing // MockQiniuServer simulate qiniu cloud for testing
func MockQiniuServer(config *Config) (client *Client, router *httprouter.Router, serverURL string, teardown func()) { func MockQiniuServer(config *Config) (client *QnClient, router *httprouter.Router, serverURL string, teardown func()) {
// router is the HTTP request multiplexer used with the test server. // router is the HTTP request multiplexer used with the test server.
router = httprouter.New() router = httprouter.New()

View File

@ -29,8 +29,14 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
// ObjectHandle provides operations on an object in a qiniu cloud bucket type ObjectHandle interface {
type ObjectHandle struct { Attrs(ctx context.Context) (storage.FileInfo, error)
NewReader(ctx context.Context) (io.ReadCloser, error)
NewRangeReader(ctx context.Context, offset, length int64) (io.ReadCloser, error)
}
// QnObjectHandle provides operations on an object in a qiniu cloud bucket
type QnObjectHandle struct {
key string key string
cfg *Config cfg *Config
bm *storage.BucketManager bm *storage.BucketManager
@ -39,7 +45,7 @@ type ObjectHandle struct {
} }
// Attrs get the object's metainfo // Attrs get the object's metainfo
func (o *ObjectHandle) Attrs(ctx context.Context) (storage.FileInfo, error) { func (o *QnObjectHandle) Attrs(ctx context.Context) (storage.FileInfo, error) {
//TODO(CarlJi): need retry when errors //TODO(CarlJi): need retry when errors
return o.bm.Stat(o.cfg.Bucket, o.key) return o.bm.Stat(o.cfg.Bucket, o.key)
} }
@ -47,13 +53,13 @@ func (o *ObjectHandle) Attrs(ctx context.Context) (storage.FileInfo, error) {
// NewReader creates a reader to read the contents of the object. // NewReader creates a reader to read the contents of the object.
// ErrObjectNotExist will be returned if the object is not found. // ErrObjectNotExist will be returned if the object is not found.
// The caller must call Close on the returned Reader when done reading. // The caller must call Close on the returned Reader when done reading.
func (o *ObjectHandle) NewReader(ctx context.Context) (io.ReadCloser, error) { func (o *QnObjectHandle) NewReader(ctx context.Context) (io.ReadCloser, error) {
return o.NewRangeReader(ctx, 0, -1) return o.NewRangeReader(ctx, 0, -1)
} }
// NewRangeReader reads parts of an object, reading at most length bytes starting // NewRangeReader reads parts of an object, reading at most length bytes starting
// from the given offset. If length is negative, the object is read until the end. // from the given offset. If length is negative, the object is read until the end.
func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) (io.ReadCloser, error) { func (o *QnObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) (io.ReadCloser, error) {
verb := "GET" verb := "GET"
if length == 0 { if length == 0 {
verb = "HEAD" verb = "HEAD"

View File

@ -41,7 +41,7 @@ func TestNewRangeReader(t *testing.T) {
MockPrivateDomainUrl(router, 0) MockPrivateDomainUrl(router, 0)
oh := &ObjectHandle{ oh := &QnObjectHandle{
key: "key", key: "key",
cfg: cfg, cfg: cfg,
bm: nil, bm: nil,
@ -78,7 +78,7 @@ func TestNewRangeReaderWithTimeoutAndRecover(t *testing.T) {
MockPrivateDomainUrl(router, 2) MockPrivateDomainUrl(router, 2)
oh := &ObjectHandle{ oh := &QnObjectHandle{
key: "key", key: "key",
cfg: cfg, cfg: cfg,
bm: nil, bm: nil,
@ -116,7 +116,7 @@ func TestNewRangeReaderWithTimeoutNoRecover(t *testing.T) {
MockPrivateDomainUrl(router, 12) MockPrivateDomainUrl(router, 12)
oh := &ObjectHandle{ oh := &QnObjectHandle{
key: "key", key: "key",
cfg: cfg, cfg: cfg,
bm: nil, bm: nil,

View File

@ -70,7 +70,7 @@ func isBuildSucceeded(jsonText []byte) bool {
// FindBaseProfileFromQiniu finds the coverage profile file from the latest healthy build // FindBaseProfileFromQiniu finds the coverage profile file from the latest healthy build
// stored in given gcs directory // stored in given gcs directory
func FindBaseProfileFromQiniu(qc *Client, prowJobName, covProfileName string) ([]byte, error) { func FindBaseProfileFromQiniu(qc Client, prowJobName, covProfileName string) ([]byte, error) {
dirOfJob := path.Join("logs", prowJobName) dirOfJob := path.Join("logs", prowJobName)
prefix := dirOfJob + "/" prefix := dirOfJob + "/"
strBuilds, err := qc.ListSubDirs(prefix) strBuilds, err := qc.ListSubDirs(prefix)