/* Copyright 2020 Qiniu Cloud (qiniu.com) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package qiniu import ( "context" "fmt" "io/ioutil" "math/rand" "net/http" "regexp" "strconv" "strings" "time" "github.com/qiniu/api.v7/v7/auth/qbox" "github.com/qiniu/api.v7/v7/client" "github.com/qiniu/api.v7/v7/storage" "github.com/sirupsen/logrus" ) // Config store the credentials to connect with qiniu cloud type Config struct { Bucket string `json:"bucket"` AccessKey string `json:"accessKey"` SecretKey string `json:"secretKey"` // domain used to download files from qiniu cloud Domain string `json:"domain"` } // Client is the interface contains the operation with qiniu cloud type Client interface { QiniuObjectHandle(key string) ObjectHandle ReadObject(key string) ([]byte, error) ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error) GetAccessURL(key string, timeout time.Duration) string GetArtifactDetails(key string) (*LogHistoryTemplate, error) ListSubDirs(prefix string) ([]string, error) } // QnClient for the operation with qiniu cloud type QnClient struct { cfg *Config BucketManager *storage.BucketManager } // NewClient creates a new QnClient to work with qiniu cloud func NewClient(cfg *Config) *QnClient { return &QnClient{ cfg: cfg, BucketManager: storage.NewBucketManager(qbox.NewMac(cfg.AccessKey, cfg.SecretKey), nil), } } // QiniuObjectHandle construct a object hanle to access file in qiniu func (q *QnClient) QiniuObjectHandle(key string) ObjectHandle { return &QnObjectHandle{ key: key, cfg: q.cfg, bm: q.BucketManager, mac: qbox.NewMac(q.cfg.AccessKey, q.cfg.SecretKey), client: &client.Client{Client: http.DefaultClient}, } } // ReadObject to read all the content of key func (q *QnClient) ReadObject(key string) ([]byte, error) { objectHandle := q.QiniuObjectHandle(key) reader, err := objectHandle.NewReader(context.Background()) if err != nil { return nil, fmt.Errorf("error getting qiniu artifact reader: %v", err) } defer reader.Close() return ioutil.ReadAll(reader) } // ListAll to list all the files with contains the expected prefix func (q *QnClient) ListAll(ctx context.Context, prefix string, delimiter string) ([]string, error) { var files []string artifacts, err := q.listEntries(prefix, delimiter) if err != nil { return files, err } for _, item := range artifacts { files = append(files, item.Key) } return files, nil } // listEntries to list all the entries with contains the expected prefix func (q *QnClient) listEntries(prefix string, delimiter string) ([]storage.ListItem, error) { var marker string var artifacts []storage.ListItem wait := []time.Duration{16, 32, 64, 128, 256, 256, 512, 512} for i := 0; ; { entries, _, nextMarker, hashNext, err := q.BucketManager.ListFiles(q.cfg.Bucket, prefix, delimiter, marker, 500) if err != nil { logrus.WithField("prefix", prefix).WithError(err).Error("Error accessing QINIU artifact.") if i >= len(wait) { return artifacts, fmt.Errorf("timed out: error accessing QINIU artifact: %v", err) } time.Sleep((wait[i] + time.Duration(rand.Intn(10))) * time.Millisecond) i++ continue } artifacts = append(artifacts, entries...) if hashNext { marker = nextMarker } else { break } } return artifacts, nil } // GetAccessURL return a url which can access artifact directly in qiniu func (q *QnClient) GetAccessURL(key string, timeout time.Duration) string { deadline := time.Now().Add(timeout).Unix() return storage.MakePrivateURL(qbox.NewMac(q.cfg.AccessKey, q.cfg.SecretKey), q.cfg.Domain, key, deadline) } // LogHistoryTemplate is the template of the log history type LogHistoryTemplate struct { BucketName string KeyPath string Items []logHistoryItem } // logHistoryItem represents a log history item type logHistoryItem struct { Name string Size string Time string Url string } // GetArtifactDetails lists all artifacts available for the given job source func (q *QnClient) GetArtifactDetails(key string) (*LogHistoryTemplate, error) { tmpl := new(LogHistoryTemplate) item := logHistoryItem{} listStart := time.Now() artifacts, err := q.listEntries(key, "") if err != nil { return tmpl, err } for _, entry := range artifacts { item.Name = splitKey(entry.Key, key) item.Size = size(entry.Fsize) item.Time = timeConv(entry.PutTime) item.Url = q.GetAccessURL(entry.Key, time.Duration(time.Second*60*60)) tmpl.Items = append(tmpl.Items, item) } logrus.WithField("duration", time.Since(listStart).String()).Infof("Listed %d artifacts.", len(tmpl.Items)) return tmpl, nil } func splitKey(item, key string) string { return strings.TrimPrefix(item, key) } func size(fsize int64) string { return strings.Join([]string{strconv.FormatInt(fsize, 10), "bytes"}, " ") } func timeConv(ptime int64) string { s := strconv.FormatInt(ptime, 10)[0:10] t, err := strconv.ParseInt(s, 10, 64) if err != nil { logrus.Errorf("time string parse int error : %v", err) return "" } tm := time.Unix(t, 0) return tm.Format("2006-01-02 03:04:05 PM") } // ListSubDirs list all the sub directions of the prefix string in qiniu client func (q *QnClient) ListSubDirs(prefix string) ([]string, error) { var dirs []string var marker string wait := []time.Duration{16, 32, 64, 128, 256, 256, 512, 512} for i := 0; ; { // use rsf list v2 interface to get the sub folder based on the delimiter entries, err := q.BucketManager.ListBucketContext(context.Background(), q.cfg.Bucket, prefix, "/", marker) if err != nil { logrus.WithField("prefix", prefix).WithError(err).Error("Error accessing QINIU artifact.") if i >= len(wait) { return dirs, fmt.Errorf("timed out: error accessing QINIU artifact: %v", err) } time.Sleep((wait[i] + time.Duration(rand.Intn(10))) * time.Millisecond) i++ continue } for entry := range entries { if entry.Dir != "" { // entry.Dir should be like "logs/kodo-periodics-integration-test/1181915661132107776/" // the sub folder is 1181915661132107776, also known as prowjob buildid. buildId := getBuildId(entry.Dir) if buildId != "" { dirs = append(dirs, buildId) } else { logrus.Warnf("invalid dir format: %v", entry.Dir) } } marker = entry.Marker } if marker != "" { i = 0 } else { break } } return dirs, nil } var nonPRLogsBuildIdSubffixRe = regexp.MustCompile("([0-9]+)/$") // extract the build number from dir path // expect the dir as the following formats: // 1. logs/kodo-periodics-integration-test/1181915661132107776/ func getBuildId(dir string) string { matches := nonPRLogsBuildIdSubffixRe.FindStringSubmatch(dir) if len(matches) == 2 { return matches[1] } return "" }