ut case add
This commit is contained in:
parent
1917e39977
commit
8c719effc8
@ -160,7 +160,7 @@ func doDiffUnderProw(cmd *cobra.Command, args []string) {
|
||||
if qiniuCredential == "" {
|
||||
logrus.Fatalf("qiniu credential not provided")
|
||||
}
|
||||
var qc *qiniu.Client
|
||||
var qc qiniu.Client
|
||||
var conf qiniu.Config
|
||||
files, err := ioutil.ReadFile(*&qiniuCredential)
|
||||
if err != nil {
|
||||
|
@ -93,6 +93,12 @@ func TestCovList(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadFileToCoverList(t *testing.T) {
|
||||
path := "unknown"
|
||||
_, err := ReadFileToCoverList(path)
|
||||
assert.Equal(t, err.Error(), "open unknown: no such file or directory")
|
||||
}
|
||||
|
||||
func TestTotalPercentage(t *testing.T) {
|
||||
items := []struct {
|
||||
list CoverageList
|
||||
|
@ -38,8 +38,16 @@ import (
|
||||
// It is also the flag when checking whether the target comment exists or not to avoid duplicate
|
||||
const CommentsPrefix = "The following is the coverage report on the affected files."
|
||||
|
||||
// PrComment is the entry which is able to comment on Github Pull Requests
|
||||
type PrComment struct {
|
||||
type PrComment interface {
|
||||
CreateGithubComment(commentPrefix string, diffCovList cover.DeltaCovList) (err error)
|
||||
PostComment(content, commentPrefix string) error
|
||||
EraseHistoryComment(commentPrefix string) error
|
||||
GetPrChangedFiles() (files []string, err error)
|
||||
GetCommentFlag() string
|
||||
}
|
||||
|
||||
// GithubPrComment is the entry which is able to comment on Github Pull Requests
|
||||
type GithubPrComment struct {
|
||||
RobotUserName string
|
||||
RepoOwner string
|
||||
RepoName string
|
||||
@ -51,7 +59,7 @@ type PrComment struct {
|
||||
}
|
||||
|
||||
// NewPrClient creates an Client which be able to comment on Github Pull Request
|
||||
func NewPrClient(githubTokenPath, repoOwner, repoName, prNumStr, botUserName, commentFlag string) *PrComment {
|
||||
func NewPrClient(githubTokenPath, repoOwner, repoName, prNumStr, botUserName, commentFlag string) *GithubPrComment {
|
||||
var client *github.Client
|
||||
|
||||
// performs automatic retries when connection error occurs or a 500-range response code received (except 501)
|
||||
@ -72,7 +80,7 @@ func NewPrClient(githubTokenPath, repoOwner, repoName, prNumStr, botUserName, co
|
||||
tc := oauth2.NewClient(ctx, ts)
|
||||
client = github.NewClient(tc)
|
||||
|
||||
return &PrComment{
|
||||
return &GithubPrComment{
|
||||
RobotUserName: botUserName,
|
||||
RepoOwner: repoOwner,
|
||||
RepoName: repoName,
|
||||
@ -85,7 +93,7 @@ func NewPrClient(githubTokenPath, repoOwner, repoName, prNumStr, botUserName, co
|
||||
}
|
||||
|
||||
// CreateGithubComment post github comment of diff coverage
|
||||
func (c *PrComment) CreateGithubComment(commentPrefix string, diffCovList cover.DeltaCovList) (err error) {
|
||||
func (c *GithubPrComment) CreateGithubComment(commentPrefix string, diffCovList cover.DeltaCovList) (err error) {
|
||||
if len(diffCovList) == 0 {
|
||||
logrus.Printf("Detect 0 files coverage diff, will not comment to github.")
|
||||
return nil
|
||||
@ -101,7 +109,7 @@ func (c *PrComment) CreateGithubComment(commentPrefix string, diffCovList cover.
|
||||
}
|
||||
|
||||
// PostComment post comment on github. It erased the old one if existed to avoid duplicate
|
||||
func (c *PrComment) PostComment(content, commentPrefix string) error {
|
||||
func (c *GithubPrComment) PostComment(content, commentPrefix string) error {
|
||||
//step1: erase history similar comment to avoid too many comment for same job
|
||||
err := c.EraseHistoryComment(commentPrefix)
|
||||
if err != nil {
|
||||
@ -121,7 +129,7 @@ func (c *PrComment) PostComment(content, commentPrefix string) error {
|
||||
}
|
||||
|
||||
// EraseHistoryComment erase history similar comment before post again
|
||||
func (c *PrComment) EraseHistoryComment(commentPrefix string) error {
|
||||
func (c *GithubPrComment) EraseHistoryComment(commentPrefix string) error {
|
||||
comments, _, err := c.GithubClient.Issues.ListComments(c.Ctx, c.RepoOwner, c.RepoName, c.PrNumber, nil)
|
||||
if err != nil {
|
||||
logrus.Errorf("list PR comments failed.")
|
||||
@ -143,7 +151,7 @@ func (c *PrComment) EraseHistoryComment(commentPrefix string) error {
|
||||
}
|
||||
|
||||
//GetPrChangedFiles get github pull request changes file list
|
||||
func (c *PrComment) GetPrChangedFiles() (files []string, err error) {
|
||||
func (c *GithubPrComment) GetPrChangedFiles() (files []string, err error) {
|
||||
var commitFiles []*github.CommitFile
|
||||
for {
|
||||
f, resp, err := c.GithubClient.PullRequests.ListFiles(c.Ctx, c.RepoOwner, c.RepoName, c.PrNumber, c.opt)
|
||||
@ -165,6 +173,11 @@ func (c *PrComment) GetPrChangedFiles() (files []string, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
// GetCommentFlag get CommentFlag from the GithubPrComment
|
||||
func (c *GithubPrComment) GetCommentFlag() string {
|
||||
return c.CommentFlag
|
||||
}
|
||||
|
||||
// GenCommentContent generate github comment content based on diff coverage and commentFlag
|
||||
func GenCommentContent(commentPrefix string, delta cover.DeltaCovList) string {
|
||||
var buf bytes.Buffer
|
||||
|
@ -121,7 +121,7 @@ func TestCreateGithubComment(t *testing.T) {
|
||||
router.HandlerFunc("DELETE", "/repos/qiniu/goc/issues/comments/1", func(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
|
||||
p := PrComment{
|
||||
p := GithubPrComment{
|
||||
RobotUserName: "qiniu-bot",
|
||||
RepoOwner: "qiniu",
|
||||
RepoName: "goc",
|
||||
@ -146,7 +146,7 @@ func TestGetPrChangedFiles(t *testing.T) {
|
||||
fmt.Fprint(w, `[{"filename":"src/qiniu.com/kodo/s3apiv2/bucket/bucket.go"}]`)
|
||||
})
|
||||
|
||||
p := PrComment{
|
||||
p := GithubPrComment{
|
||||
RobotUserName: "qiniu-bot",
|
||||
RepoOwner: "qiniu",
|
||||
RepoName: "goc",
|
||||
|
24
pkg/mock/clients/github.go
Normal file
24
pkg/mock/clients/github.go
Normal file
@ -0,0 +1,24 @@
|
||||
package clients
|
||||
|
||||
import "github.com/qiniu/goc/pkg/cover"
|
||||
|
||||
type MockPrComment struct {
|
||||
}
|
||||
|
||||
func (s *MockPrComment) GetPrChangedFiles() (files []string, err error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (s *MockPrComment) PostComment(content, commentPrefix string) error {
|
||||
return nil
|
||||
}
|
||||
func (s *MockPrComment) EraseHistoryComment(commentPrefix string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MockPrComment) CreateGithubComment(commentPrefix string, diffCovList cover.DeltaCovList) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MockPrComment) GetCommentFlag() string {
|
||||
return ""
|
||||
}
|
34
pkg/mock/clients/qiniu.go
Normal file
34
pkg/mock/clients/qiniu.go
Normal file
@ -0,0 +1,34 @@
|
||||
package clients
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/qiniu/goc/pkg/qiniu"
|
||||
"time"
|
||||
)
|
||||
|
||||
type MockQnClient struct {
|
||||
}
|
||||
|
||||
func (s *MockQnClient) QiniuObjectHandle(key string) qiniu.ObjectHandle {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MockQnClient) ReadObject(key string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *MockQnClient) ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *MockQnClient) GetAccessURL(key string, timeout time.Duration) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s *MockQnClient) GetArtifactDetails(key string) (*qiniu.LogHistoryTemplate, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *MockQnClient) ListSubDirs(prefix string) ([]string, error) {
|
||||
return nil, nil
|
||||
}
|
@ -54,9 +54,9 @@ type Job struct {
|
||||
PostSubmitCoverProfile string
|
||||
CovThreshold int
|
||||
LocalProfilePath string
|
||||
QiniuClient *qiniu.Client
|
||||
QiniuClient qiniu.Client
|
||||
LocalArtifacts *qiniu.Artifacts
|
||||
GithubComment *github.PrComment
|
||||
GithubComment github.PrComment
|
||||
FullDiff bool
|
||||
}
|
||||
|
||||
@ -67,32 +67,19 @@ func (j *Job) Fetch(BuildID, name string) []byte {
|
||||
|
||||
// RunPresubmit run a presubmit job
|
||||
func (j *Job) RunPresubmit() error {
|
||||
var changedFiles []string
|
||||
var deltaCovList cover.DeltaCovList
|
||||
|
||||
// step1: get github pull request changed files' name
|
||||
if !j.FullDiff {
|
||||
var ghChangedFiles, err = j.GithubComment.GetPrChangedFiles()
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatalf("Get pull request changed file failed.")
|
||||
}
|
||||
if len(ghChangedFiles) == 0 {
|
||||
logrus.Printf("0 files changed in github pull request, don't need to run coverage profile in presubmit.\n")
|
||||
return nil
|
||||
}
|
||||
changedFiles = trimGhFileToProfile(ghChangedFiles)
|
||||
}
|
||||
|
||||
// step2: get local profile cov
|
||||
// step1: get local profile cov
|
||||
localP, err := cover.ReadFileToCoverList(j.LocalProfilePath)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatalf("failed to get remote cover profile")
|
||||
logrus.Errorf("failed to get remote cover profile: %s", err.Error())
|
||||
return err
|
||||
}
|
||||
logrus.Warnf("localP: [%+v]", localP)
|
||||
|
||||
//step3: find the remote healthy cover profile from qiniu bucket
|
||||
//step2: find the remote healthy cover profile from qiniu bucket
|
||||
remoteProfile, err := qiniu.FindBaseProfileFromQiniu(j.QiniuClient, j.PostSubmitJob, j.PostSubmitCoverProfile)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatalf("failed to get remote cover profile")
|
||||
logrus.Errorf("failed to get remote cover profile: %s", err.Error())
|
||||
return err
|
||||
}
|
||||
if remoteProfile == nil {
|
||||
logrus.Infof("get non healthy remoteProfile, do nothing")
|
||||
@ -100,22 +87,20 @@ func (j *Job) RunPresubmit() error {
|
||||
}
|
||||
baseP, err := cover.CovList(bytes.NewReader(remoteProfile))
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatalf("failed to get remote cover profile")
|
||||
logrus.Errorf("failed to get remote cover profile: %s", err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// step4: calculate diff cov between local and remote profile
|
||||
if !j.FullDiff {
|
||||
deltaCovList = cover.GetChFileDeltaCov(localP, baseP, changedFiles)
|
||||
} else {
|
||||
deltaCovList = cover.GetDeltaCov(localP, baseP)
|
||||
logrus.Infof("get delta file name is:")
|
||||
for _, d := range deltaCovList {
|
||||
logrus.Infof("%s", d.FileName)
|
||||
changedFiles = append(changedFiles, d.FileName)
|
||||
}
|
||||
logrus.Warnf("baseP: [%+v]", baseP)
|
||||
|
||||
// step3: get github pull request changed files' name and calculate diff cov between local and remote profile
|
||||
changedFiles, deltaCovList, err := getFilesAndCovList(j.FullDiff, j.GithubComment, localP, baseP)
|
||||
if err != nil {
|
||||
logrus.Errorf("Get files and covlist failed: %s", err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// step5: generate changed file html coverage
|
||||
// step4: generate changed file html coverage
|
||||
err = j.WriteChangedCov(changedFiles)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatalf("filter local profile to %s with changed files failed", j.LocalArtifacts.ChangedProfileName)
|
||||
@ -128,8 +113,8 @@ func (j *Job) RunPresubmit() error {
|
||||
|
||||
// step6: post comment to github
|
||||
commentPrefix := github.CommentsPrefix
|
||||
if j.GithubComment.CommentFlag != "" {
|
||||
commentPrefix = fmt.Sprintf("**%s** ", j.GithubComment.CommentFlag) + commentPrefix
|
||||
if j.GithubComment.GetCommentFlag() != "" {
|
||||
commentPrefix = fmt.Sprintf("**%s** ", j.GithubComment.GetCommentFlag()) + commentPrefix
|
||||
}
|
||||
if len(deltaCovList) > 0 {
|
||||
totalDelta := cover.PercentStr(cover.TotalDelta(localP, baseP))
|
||||
@ -231,3 +216,31 @@ func (j *Job) CreateChangedCovHtml() error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func getFilesAndCovList(fullDiff bool, prComment github.PrComment, localP, baseP cover.CoverageList) (changedFiles []string, deltaCovList cover.DeltaCovList, err error) {
|
||||
if !fullDiff {
|
||||
// get github pull request changed files' name
|
||||
var ghChangedFiles, err = prComment.GetPrChangedFiles()
|
||||
if err != nil {
|
||||
logrus.Errorf("Get pull request changed file failed.")
|
||||
return nil, nil, err
|
||||
}
|
||||
if len(ghChangedFiles) == 0 {
|
||||
logrus.Printf("0 files changed in github pull request, don't need to run coverage profile in presubmit.\n")
|
||||
return nil, nil, nil
|
||||
}
|
||||
changedFiles = trimGhFileToProfile(ghChangedFiles)
|
||||
|
||||
// calculate diff cov between local and remote profile
|
||||
deltaCovList = cover.GetChFileDeltaCov(localP, baseP, changedFiles)
|
||||
return changedFiles, deltaCovList, nil
|
||||
}
|
||||
deltaCovList = cover.GetDeltaCov(localP, baseP)
|
||||
logrus.Infof("get delta file name is:")
|
||||
for _, d := range deltaCovList {
|
||||
logrus.Infof("%s", d.FileName)
|
||||
changedFiles = append(changedFiles, d.FileName)
|
||||
}
|
||||
|
||||
return changedFiles, deltaCovList, nil
|
||||
}
|
||||
|
@ -17,16 +17,25 @@
|
||||
package prow
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"github.com/qiniu/goc/pkg/github"
|
||||
"github.com/qiniu/goc/pkg/mock/clients"
|
||||
"github.com/qiniu/goc/pkg/qiniu"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"testing"
|
||||
)
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/qiniu/goc/pkg/github"
|
||||
"github.com/qiniu/goc/pkg/qiniu"
|
||||
var (
|
||||
defaultContent = `mode: atomic
|
||||
qiniu.com/kodo/bd/bdgetter/source.go:19.118,22.2 2 0
|
||||
qiniu.com/kodo/bd/bdgetter/source.go:37.34,39.2 1 0
|
||||
qiniu.com/kodo/bd/pfd/locker/app/qboxbdlocker/main.go:50.2,53.52 4 1
|
||||
qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go:33.51,35.2 1 0`
|
||||
defaultLocalPath = "local.cov"
|
||||
)
|
||||
|
||||
func TestTrimGhFileToProfile(t *testing.T) {
|
||||
@ -54,13 +63,9 @@ func setup(path, content string) {
|
||||
}
|
||||
|
||||
func TestWriteChangedCov(t *testing.T) {
|
||||
path := "local.cov"
|
||||
path := defaultLocalPath
|
||||
savePath := qiniu.ChangedProfileName
|
||||
content := `mode: atomic
|
||||
qiniu.com/kodo/bd/bdgetter/source.go:19.118,22.2 2 0
|
||||
qiniu.com/kodo/bd/bdgetter/source.go:37.34,39.2 1 0
|
||||
qiniu.com/kodo/bd/pfd/locker/app/qboxbdlocker/main.go:50.2,53.52 4 1
|
||||
qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go:33.51,35.2 1 0`
|
||||
content := defaultContent
|
||||
changedFiles := []string{"qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go"}
|
||||
expectContent := `mode: atomic
|
||||
qiniu.com/kodo/bd/pfd/locker/bdlocker/locker.go:33.51,35.2 1 0
|
||||
@ -95,10 +100,8 @@ func TestRunPresubmitFulldiff(t *testing.T) {
|
||||
|
||||
//mock local profile
|
||||
pwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
logrus.WithError(err).Fatalf("get pwd failed")
|
||||
}
|
||||
localPath := "local.cov"
|
||||
assert.NoError(t, err)
|
||||
localPath := defaultLocalPath
|
||||
localProfileContent := `mode: atomic
|
||||
"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30
|
||||
"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0`
|
||||
@ -137,5 +140,103 @@ func TestRunPresubmitFulldiff(t *testing.T) {
|
||||
}
|
||||
defer os.Remove(path.Join(os.Getenv("ARTIFACTS"), j.HtmlProfile()))
|
||||
|
||||
j.RunPresubmit()
|
||||
err = j.RunPresubmit()
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestRunPresubmitErrorGetRemoteCoverProfile(t *testing.T) {
|
||||
j := &Job{
|
||||
LocalProfilePath: "unkown",
|
||||
}
|
||||
err := j.RunPresubmit()
|
||||
assert.Contains(t, err.Error(), "no such file or directory")
|
||||
}
|
||||
|
||||
func TestRunPresubmitGetEmptyProfile(t *testing.T) {
|
||||
path := "local.cov"
|
||||
setup(path, defaultContent)
|
||||
defer os.Remove(path)
|
||||
j := &Job{
|
||||
LocalProfilePath: path,
|
||||
QiniuClient: &clients.MockQnClient{},
|
||||
}
|
||||
err := j.RunPresubmit()
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
type MockErrorListSubDirsQnClient struct {
|
||||
*clients.MockQnClient
|
||||
}
|
||||
|
||||
func (s *MockErrorListSubDirsQnClient) ListSubDirs(prefix string) ([]string, error) {
|
||||
return nil, errors.New("mock error")
|
||||
}
|
||||
|
||||
func TestRunPresubmitErrorGetBaseProfile(t *testing.T) {
|
||||
path := "local.cov"
|
||||
setup(path, defaultContent)
|
||||
defer os.Remove(path)
|
||||
j := &Job{
|
||||
LocalProfilePath: path,
|
||||
QiniuClient: &MockErrorListSubDirsQnClient{},
|
||||
}
|
||||
err := j.RunPresubmit()
|
||||
assert.Contains(t, err.Error(), "mock error")
|
||||
}
|
||||
|
||||
type MockProfileQnClient struct {
|
||||
*clients.MockQnClient
|
||||
}
|
||||
|
||||
func (s *MockProfileQnClient) ListSubDirs(prefix string) ([]string, error) {
|
||||
return []string{"1"}, nil
|
||||
}
|
||||
|
||||
func (s *MockProfileQnClient) ReadObject(key string) ([]byte, error) {
|
||||
logrus.Info(key)
|
||||
if key == "logs/1/finished.json" {
|
||||
return []byte(`{"timestamp":1590750306,"passed":true,"result":"SUCCESS","repo-version":"76433418ea48aae57af028f9cb2fa3735ce08c7d"}`), nil
|
||||
}
|
||||
return []byte(""), nil
|
||||
}
|
||||
|
||||
// 无法实现coverList报错
|
||||
//func TestRunPresubmitErrorReadBaseProfile(t *testing.T) {
|
||||
// path := "local.cov"
|
||||
// setup(path, defaultContent)
|
||||
// defer os.Remove(path)
|
||||
// j := &Job{
|
||||
// LocalProfilePath: path,
|
||||
// QiniuClient: &MockProfileQnClient{},
|
||||
// }
|
||||
// err := j.RunPresubmit()
|
||||
// assert.Contains(t, err.Error(), "mock error")
|
||||
//}
|
||||
|
||||
//type MockErrorPrComment struct {
|
||||
// *clients.MockPrComment
|
||||
//}
|
||||
//
|
||||
//func (s *MockErrorPrComment) GetPrChangedFiles() (files []string, err error) {
|
||||
// return []string{"aaa"}, nil
|
||||
//}
|
||||
//
|
||||
//func TestGetFilesAndCovList(t *testing.T){
|
||||
// items := []struct {
|
||||
// fullDiff bool
|
||||
// prComment github.PrComment
|
||||
// localP cover.CoverageList
|
||||
// baseP cover.CoverageList
|
||||
// }{
|
||||
// {
|
||||
// inputFiles: []string{"src/qiniu.com/kodo/io/io/io_svr.go", "README.md"},
|
||||
// expectFiles: []string{"qiniu.com/kodo/io/io/io_svr.go", "README.md"},
|
||||
// },
|
||||
// }
|
||||
//
|
||||
// for _, tc := range items {
|
||||
// f := trimGhFileToProfile(tc.inputFiles)
|
||||
// assert.Equal(t, f, tc.expectFiles)
|
||||
// }
|
||||
// getFilesAndCovList
|
||||
//}
|
||||
|
@ -43,23 +43,32 @@ type Config struct {
|
||||
Domain string `json:"domain"`
|
||||
}
|
||||
|
||||
// Client for the operation with qiniu cloud
|
||||
type Client struct {
|
||||
type Client interface {
|
||||
QiniuObjectHandle(key string) ObjectHandle
|
||||
ReadObject(key string) ([]byte, error)
|
||||
ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error)
|
||||
GetAccessURL(key string, timeout time.Duration) string
|
||||
GetArtifactDetails(key string) (*LogHistoryTemplate, error)
|
||||
ListSubDirs(prefix string) ([]string, error)
|
||||
}
|
||||
|
||||
// QnClient for the operation with qiniu cloud
|
||||
type QnClient struct {
|
||||
cfg *Config
|
||||
BucketManager *storage.BucketManager
|
||||
}
|
||||
|
||||
// NewClient creates a new client to work with qiniu cloud
|
||||
func NewClient(cfg *Config) *Client {
|
||||
return &Client{
|
||||
// NewQnClient creates a new QnClient to work with qiniu cloud
|
||||
func NewClient(cfg *Config) *QnClient {
|
||||
return &QnClient{
|
||||
cfg: cfg,
|
||||
BucketManager: storage.NewBucketManager(qbox.NewMac(cfg.AccessKey, cfg.SecretKey), nil),
|
||||
}
|
||||
}
|
||||
|
||||
// QiniuObjectHandle construct a object hanle to access file in qiniu
|
||||
func (q *Client) QiniuObjectHandle(key string) *ObjectHandle {
|
||||
return &ObjectHandle{
|
||||
func (q *QnClient) QiniuObjectHandle(key string) ObjectHandle {
|
||||
return &QnObjectHandle{
|
||||
key: key,
|
||||
cfg: q.cfg,
|
||||
bm: q.BucketManager,
|
||||
@ -69,7 +78,7 @@ func (q *Client) QiniuObjectHandle(key string) *ObjectHandle {
|
||||
}
|
||||
|
||||
// ReadObject to read all the content of key
|
||||
func (q *Client) ReadObject(key string) ([]byte, error) {
|
||||
func (q *QnClient) ReadObject(key string) ([]byte, error) {
|
||||
objectHandle := q.QiniuObjectHandle(key)
|
||||
reader, err := objectHandle.NewReader(context.Background())
|
||||
if err != nil {
|
||||
@ -80,7 +89,7 @@ func (q *Client) ReadObject(key string) ([]byte, error) {
|
||||
}
|
||||
|
||||
// ListAll to list all the files with contains the expected prefix
|
||||
func (q *Client) ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error) {
|
||||
func (q *QnClient) ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error) {
|
||||
var files []string
|
||||
artifacts, err := q.listEntries(prefix, delimiter)
|
||||
if err != nil {
|
||||
@ -95,7 +104,7 @@ func (q *Client) ListAll(ctx context.Context, prefix string, delimiter string) (
|
||||
}
|
||||
|
||||
// ListAll to list all the entries with contains the expected prefix
|
||||
func (q *Client) listEntries(prefix string, delimiter string) ([]storage.ListItem, error) {
|
||||
func (q *QnClient) listEntries(prefix string, delimiter string) ([]storage.ListItem, error) {
|
||||
var marker string
|
||||
var artifacts []storage.ListItem
|
||||
|
||||
@ -124,7 +133,7 @@ func (q *Client) listEntries(prefix string, delimiter string) ([]storage.ListIte
|
||||
}
|
||||
|
||||
// GetAccessURL return a url which can access artifact directly in qiniu
|
||||
func (q *Client) GetAccessURL(key string, timeout time.Duration) string {
|
||||
func (q *QnClient) GetAccessURL(key string, timeout time.Duration) string {
|
||||
deadline := time.Now().Add(timeout).Unix()
|
||||
return storage.MakePrivateURL(qbox.NewMac(q.cfg.AccessKey, q.cfg.SecretKey), q.cfg.Domain, key, deadline)
|
||||
}
|
||||
@ -143,7 +152,7 @@ type logHistoryItem struct {
|
||||
}
|
||||
|
||||
// Artifacts lists all artifacts available for the given job source
|
||||
func (q *Client) GetArtifactDetails(key string) (*LogHistoryTemplate, error) {
|
||||
func (q *QnClient) GetArtifactDetails(key string) (*LogHistoryTemplate, error) {
|
||||
tmpl := new(LogHistoryTemplate)
|
||||
item := logHistoryItem{}
|
||||
listStart := time.Now()
|
||||
@ -183,7 +192,7 @@ func timeConv(ptime int64) string {
|
||||
return tm.Format("2006-01-02 03:04:05 PM")
|
||||
}
|
||||
|
||||
func (q *Client) ListSubDirs(prefix string) ([]string, error) {
|
||||
func (q *QnClient) ListSubDirs(prefix string) ([]string, error) {
|
||||
var dirs []string
|
||||
var marker string
|
||||
|
||||
|
@ -27,7 +27,7 @@ import (
|
||||
)
|
||||
|
||||
// MockQiniuServer simulate qiniu cloud for testing
|
||||
func MockQiniuServer(config *Config) (client *Client, router *httprouter.Router, serverURL string, teardown func()) {
|
||||
func MockQiniuServer(config *Config) (client *QnClient, router *httprouter.Router, serverURL string, teardown func()) {
|
||||
// router is the HTTP request multiplexer used with the test server.
|
||||
router = httprouter.New()
|
||||
|
||||
|
@ -29,8 +29,14 @@ import (
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// ObjectHandle provides operations on an object in a qiniu cloud bucket
|
||||
type ObjectHandle struct {
|
||||
type ObjectHandle interface {
|
||||
Attrs(ctx context.Context) (storage.FileInfo, error)
|
||||
NewReader(ctx context.Context) (io.ReadCloser, error)
|
||||
NewRangeReader(ctx context.Context, offset, length int64) (io.ReadCloser, error)
|
||||
}
|
||||
|
||||
// QnObjectHandle provides operations on an object in a qiniu cloud bucket
|
||||
type QnObjectHandle struct {
|
||||
key string
|
||||
cfg *Config
|
||||
bm *storage.BucketManager
|
||||
@ -39,7 +45,7 @@ type ObjectHandle struct {
|
||||
}
|
||||
|
||||
// Attrs get the object's metainfo
|
||||
func (o *ObjectHandle) Attrs(ctx context.Context) (storage.FileInfo, error) {
|
||||
func (o *QnObjectHandle) Attrs(ctx context.Context) (storage.FileInfo, error) {
|
||||
//TODO(CarlJi): need retry when errors
|
||||
return o.bm.Stat(o.cfg.Bucket, o.key)
|
||||
}
|
||||
@ -47,13 +53,13 @@ func (o *ObjectHandle) Attrs(ctx context.Context) (storage.FileInfo, error) {
|
||||
// NewReader creates a reader to read the contents of the object.
|
||||
// ErrObjectNotExist will be returned if the object is not found.
|
||||
// The caller must call Close on the returned Reader when done reading.
|
||||
func (o *ObjectHandle) NewReader(ctx context.Context) (io.ReadCloser, error) {
|
||||
func (o *QnObjectHandle) NewReader(ctx context.Context) (io.ReadCloser, error) {
|
||||
return o.NewRangeReader(ctx, 0, -1)
|
||||
}
|
||||
|
||||
// NewRangeReader reads parts of an object, reading at most length bytes starting
|
||||
// from the given offset. If length is negative, the object is read until the end.
|
||||
func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) (io.ReadCloser, error) {
|
||||
func (o *QnObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) (io.ReadCloser, error) {
|
||||
verb := "GET"
|
||||
if length == 0 {
|
||||
verb = "HEAD"
|
||||
|
@ -41,7 +41,7 @@ func TestNewRangeReader(t *testing.T) {
|
||||
|
||||
MockPrivateDomainUrl(router, 0)
|
||||
|
||||
oh := &ObjectHandle{
|
||||
oh := &QnObjectHandle{
|
||||
key: "key",
|
||||
cfg: cfg,
|
||||
bm: nil,
|
||||
@ -78,7 +78,7 @@ func TestNewRangeReaderWithTimeoutAndRecover(t *testing.T) {
|
||||
|
||||
MockPrivateDomainUrl(router, 2)
|
||||
|
||||
oh := &ObjectHandle{
|
||||
oh := &QnObjectHandle{
|
||||
key: "key",
|
||||
cfg: cfg,
|
||||
bm: nil,
|
||||
@ -116,7 +116,7 @@ func TestNewRangeReaderWithTimeoutNoRecover(t *testing.T) {
|
||||
|
||||
MockPrivateDomainUrl(router, 12)
|
||||
|
||||
oh := &ObjectHandle{
|
||||
oh := &QnObjectHandle{
|
||||
key: "key",
|
||||
cfg: cfg,
|
||||
bm: nil,
|
||||
|
@ -70,7 +70,7 @@ func isBuildSucceeded(jsonText []byte) bool {
|
||||
|
||||
// FindBaseProfileFromQiniu finds the coverage profile file from the latest healthy build
|
||||
// stored in given gcs directory
|
||||
func FindBaseProfileFromQiniu(qc *Client, prowJobName, covProfileName string) ([]byte, error) {
|
||||
func FindBaseProfileFromQiniu(qc Client, prowJobName, covProfileName string) ([]byte, error) {
|
||||
dirOfJob := path.Join("logs", prowJobName)
|
||||
prefix := dirOfJob + "/"
|
||||
strBuilds, err := qc.ListSubDirs(prefix)
|
||||
|
Loading…
Reference in New Issue
Block a user