add unittes case & add e2e sample

.github/workflows/e2e_test_check.yml

@@ -60,6 +60,8 @@ jobs:
           cd bats-core
           sudo ./install.sh /usr/local
       - name: Do test
+        env:
+          GOVERSION: ${{ matrix.go-version }}
         run: |
           chmod +x /home/runner/tools/goc/goc
           export PATH=/home/runner/tools/goc:$PATH

.github/workflows/ut_check.yml

@@ -24,7 +24,9 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v2
       - name: Go test
+        env:
+          GOVERSION: ${{ matrix.go-version }}
         run: |
           export DEFAULT_EXCEPT_PKGS=e2e
           go test -p 1 -coverprofile=coverage.txt $(go list ./... | grep -v -E $DEFAULT_EXCEPT_PKGS)
-          bash <(curl -s https://codecov.io/bash) -F unittest
+          bash <(curl -s https://codecov.io/bash) -F unittest-$GOVERSION

.gitignore

@@ -11,4 +11,5 @@ goc
 _svrs_address.txt
 
 # other
-*.iml
+*.iml
+.DS_Store

go.sum

@@ -696,6 +696,7 @@ github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jW
 github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=

pkg/build/build_test.go

@@ -90,3 +90,12 @@ func TestInvalidPackageNameForBuild(t *testing.T) {
 		assert.FailNow(t, "should not success with non . or ./... package")
 	}
 }
+
+// test NewBuild with wrong parameters
+func TestNewBuildWithWrongParameters(t *testing.T) {
+	_, err := NewBuild("", []string{"a.go", "b.go"}, "cur", "cur")
+	assert.Equal(t, err, ErrTooManyArgs)
+
+	_, err = NewBuild("", []string{"a.go"}, "", "cur")
+	assert.Equal(t, err, ErrInvalidWorkingDir)
+}

pkg/build/gomodules_test.go

@@ -0,0 +1,55 @@
+/*
+ Copyright 2020 Qiniu Cloud (qiniu.com)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package build
+
+import (
+	"bytes"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/qiniu/goc/pkg/cover"
+	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+func captureOutput(f func()) string {
+	var buf bytes.Buffer
+	log.SetOutput(&buf)
+	f()
+	log.SetOutput(os.Stderr)
+	return buf.String()
+}
+
+// copy in cpGoModulesProject of invalid src, dst name
+func TestModProjectCopyWithUnexistedDir(t *testing.T) {
+	pkgs := make(map[string]*cover.Package)
+	pkgs["main"] = &cover.Package{
+		Name: "main",
+		Module: &cover.ModulePublic{
+			Dir: "not exied, ia mas duser", // not real one, should fail copy
+		},
+	}
+	pkgs["another"] = &cover.Package{}
+	b := &Build{
+		TmpDir: "sdfsfev2234444", // not real one, should fail copy
+		Pkgs:   pkgs,
+	}
+
+	output := captureOutput(b.cpGoModulesProject)
+	assert.Equal(t, strings.Contains(output, "Failed to Copy"), true)
+}

pkg/build/legacy_test.go

@@ -0,0 +1,71 @@
+/*
+ Copyright 2020 Qiniu Cloud (qiniu.com)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package build
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/qiniu/goc/pkg/cover"
+	"github.com/stretchr/testify/assert"
+)
+
+// copy in cpLegacyProject/cpNonStandardLegacy of invalid src, dst name
+func TestLegacyProjectCopyWithUnexistedDir(t *testing.T) {
+	pkgs := make(map[string]*cover.Package)
+	pkgs["main"] = &cover.Package{
+		Module: &cover.ModulePublic{
+			Dir: "not exied, ia mas duser", // not real one, should fail copy
+		},
+		Dir:  "not exit, iasdfs",
+		Name: "main",
+	}
+	pkgs["another"] = &cover.Package{}
+	b := &Build{
+		TmpDir: "sdfsfev2234444", // not real one, should fail copy
+		Pkgs:   pkgs,
+	}
+
+	output := captureOutput(b.cpLegacyProject)
+	assert.Equal(t, strings.Contains(output, "Failed to Copy"), true)
+
+	output = captureOutput(b.cpNonStandardLegacy)
+	assert.Equal(t, strings.Contains(output, "Failed to Copy"), true)
+}
+
+// copy in cpDepPackages of invalid dst name
+func TestDepPackagesCopyWithInvalidDir(t *testing.T) {
+	gopath := filepath.Join(baseDir, "../../tests/samples/simple_gopath_project")
+	pkg := &cover.Package{
+		Module: &cover.ModulePublic{
+			Dir: "not exied, ia mas duser",
+		},
+		Root: gopath,
+		Deps: []string{"qiniu.com", "ddfee 2344234"},
+	}
+	b := &Build{
+		TmpDir: "/", // "/" is invalid dst in Linux, it should fail
+	}
+
+	output := captureOutput(func() {
+		visited := make(map[string]bool)
+
+		b.cpDepPackages(pkg, visited)
+	})
+	assert.Equal(t, strings.Contains(output, "Failed to Copy"), true)
+}

pkg/build/tmpfolder_test.go

@@ -22,6 +22,8 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 var baseDir string
@@ -102,3 +104,46 @@ func TestLegacyProjectNotInGoPATH(t *testing.T) {
 		t.Fatalf("There should be a main.go in temporary directory directly, the error: %v", err)
 	}
 }
+
+// test traversePkgsList error case
+func TestTraversePkgsList(t *testing.T) {
+	b := &Build{
+		Pkgs: nil,
+	}
+	_, _, err := b.traversePkgsList()
+	assert.EqualError(t, err, ErrShouldNotReached.Error())
+}
+
+// test getTmpwd error case
+func TestGetTmpwd(t *testing.T) {
+	b := &Build{
+		Pkgs: nil,
+	}
+	_, err := b.getTmpwd()
+	assert.EqualError(t, err, ErrShouldNotReached.Error())
+}
+
+// test findWhereToInstall
+func TestFindWhereToInstall(t *testing.T) {
+	// if a legacy project without project root find
+	// should find no plcae to install
+	b := &Build{
+		Pkgs:  nil,
+		IsMod: false,
+		Root:  "",
+	}
+	_, err := b.findWhereToInstall()
+	assert.EqualError(t, err, ErrNoplaceToInstall.Error())
+
+	// if $GOBIN not found
+	// and if $GOPATH not found
+	// should install to $HOME/go/bin
+	b = &Build{
+		Pkgs:      nil,
+		IsMod:     true,
+		OriGOPATH: "",
+	}
+	placeToInstall, err := b.findWhereToInstall()
+	expectedPlace := filepath.Join(os.Getenv("HOME"), "go", "bin")
+	assert.Equal(t, placeToInstall, expectedPlace)
+}

pkg/cover/client_test.go

@@ -21,8 +21,9 @@ import (
 	"os"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
 	"net/http"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestClientAction(t *testing.T) {
@@ -128,9 +129,31 @@ func TestClientAction(t *testing.T) {
 	assert.Equal(t, "{}", string(res))
 }
 
-func TestE2E(t *testing.T) {
-	// FIXME: start goc server
-	// FIXME: call goc build to cover goc server
-	// FIXME: do some tests again goc server
-	// FIXME: goc profile and checkout coverage
+func TestClientRegisterService(t *testing.T) {
+	c := &client{}
+
+	// client register with empty address
+	testService1 := Service{
+		Address: "",
+		Name:    "abc",
+	}
+	_, err := c.RegisterService(testService1)
+	assert.Contains(t, err.Error(), "empty url")
+
+	// client register with empty name
+	testService2 := Service{
+		Address: "http://127.0.0.1:444",
+		Name:    "",
+	}
+	_, err = c.RegisterService(testService2)
+	assert.EqualError(t, err, "invalid service name")
+}
+
+func TestClientListServices(t *testing.T) {
+	c := &client{
+		Host:   "http://127.0.0.1:64445", // a invalid host
+		client: http.DefaultClient,
+	}
+	_, err := c.ListServices()
+	assert.Contains(t, err.Error(), "connect: connection refused")
 }

pkg/cover/server.go

@@ -25,12 +25,12 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"strconv"
 
 	"github.com/gin-gonic/gin"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/tools/cover"
 	"k8s.io/test-infra/gopherage/pkg/cov"
-	"strconv"
 )
 
 // DefaultStore implements the IPersistence interface

pkg/cover/server_test.go

@@ -1,10 +1,46 @@
 package cover
 
 import (
-	"github.com/stretchr/testify/assert"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"strings"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 )
 
+// MockStore is mock store mainly for unittest
+type MockStore struct {
+	mock.Mock
+}
+
+func (m *MockStore) Add(s Service) error {
+	args := m.Called(s)
+	return args.Error(0)
+}
+
+func (m *MockStore) Get(name string) []string {
+	args := m.Called(name)
+	return args.Get(0).([]string)
+}
+
+func (m *MockStore) GetAll() map[string][]string {
+	args := m.Called()
+	return args.Get(0).(map[string][]string)
+}
+
+func (m *MockStore) Init() error {
+	args := m.Called()
+	return args.Error(0)
+}
+
+func (m *MockStore) Set(services map[string][]string) {
+}
+
 func TestContains(t *testing.T) {
 	assert.Equal(t, contains([]string{"a", "b"}, "a"), true)
 	assert.Equal(t, contains([]string{"a", "b"}, "c"), false)
@@ -76,3 +112,105 @@ func TestRemoveDuplicateElement(t *testing.T) {
 	strArr := []string{"a", "a", "b"}
 	assert.Equal(t, removeDuplicateElement(strArr), []string{"a", "b"})
 }
+
+func TestRegisterService(t *testing.T) {
+	router := GocServer(os.Stdout)
+
+	// register with empty service struct
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("POST", "/v1/cover/register", nil)
+	router.ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusBadRequest, w.Code)
+
+	// register with invalid service.Address
+	data := url.Values{}
+	data.Set("name", "aaa")
+	data.Set("address", "&%%")
+	w = httptest.NewRecorder()
+	req, _ = http.NewRequest("POST", "/v1/cover/register", strings.NewReader(data.Encode()))
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	router.ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusBadRequest, w.Code)
+	assert.Contains(t, w.Body.String(), "invalid URL escape")
+
+	// register with host but no port
+	data = url.Values{}
+	data.Set("name", "aaa")
+	data.Set("address", "http://127.0.0.1")
+	w = httptest.NewRecorder()
+	req, _ = http.NewRequest("POST", "/v1/cover/register", strings.NewReader(data.Encode()))
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	router.ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusBadRequest, w.Code)
+	assert.Contains(t, w.Body.String(), "missing port in address")
+
+	// register with store failure
+	expectedS := Service{
+		Name:    "foo",
+		Address: "http://:64444", // the real IP is empty in unittest, so server will get a empty one
+	}
+	testObj := new(MockStore)
+	testObj.On("Get", "foo").Return([]string{"http://127.0.0.1:66666"})
+	testObj.On("Add", expectedS).Return(fmt.Errorf("lala error"))
+
+	DefaultStore = testObj
+
+	w = httptest.NewRecorder()
+	data.Set("name", expectedS.Name)
+	data.Set("address", expectedS.Address)
+	req, _ = http.NewRequest("POST", "/v1/cover/register", strings.NewReader(data.Encode()))
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	router.ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusInternalServerError, w.Code)
+	assert.Contains(t, w.Body.String(), "lala error")
+}
+
+func TestProfileService(t *testing.T) {
+	router := GocServer(os.Stdout)
+
+	// get profile with invalid force parameter
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "/v1/cover/profile?force=11", nil)
+	router.ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusExpectationFailed, w.Code)
+	assert.Contains(t, w.Body.String(), "invalid param")
+}
+
+func TestClearService(t *testing.T) {
+	testObj := new(MockStore)
+	testObj.On("GetAll").Return(map[string][]string{"foo": {"http://127.0.0.1:66666"}})
+
+	DefaultStore = testObj
+
+	router := GocServer(os.Stdout)
+
+	// get profile with invalid force parameter
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("POST", "/v1/cover/clear", nil)
+	router.ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusExpectationFailed, w.Code)
+	assert.Contains(t, w.Body.String(), "invalid port")
+}
+
+func TestInitService(t *testing.T) {
+	testObj := new(MockStore)
+	testObj.On("Init").Return(fmt.Errorf("lala error"))
+
+	DefaultStore = testObj
+
+	router := GocServer(os.Stdout)
+
+	// get profile with invalid force parameter
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("POST", "/v1/cover/init", nil)
+	router.ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusInternalServerError, w.Code)
+	assert.Contains(t, w.Body.String(), "lala error")
+}

pkg/prow/job_test.go

@@ -111,7 +111,7 @@ func TestRunPresubmitFulldiff(t *testing.T) {
 	}
 	qc, router, _, teardown := qiniu.MockQiniuServer(&conf)
 	defer teardown()
-	qiniu.MockRouterAPI(router, localProfileContent)
+	qiniu.MockRouterAPI(router, localProfileContent, 0)
 
 	ChangedProfilePath := "changed.cov"
 	defer os.Remove(path.Join(pwd, ChangedProfilePath))

pkg/qiniu/client_test.go

@@ -16,7 +16,14 @@
 
 package qiniu
 
-import "testing"
+import (
+	"context"
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
 
 func TestGetBuildId(t *testing.T) {
 	type tc struct {
@@ -38,3 +45,140 @@ func TestGetBuildId(t *testing.T) {
 		}
 	}
 }
+
+// test basic listEntries function
+func TestListAllFiles(t *testing.T) {
+	conf := Config{
+		Bucket: "artifacts",
+	}
+	qc, router, _, teardown := MockQiniuServer(&conf)
+	defer teardown()
+	prowJobName := "kodo-postsubmits-go-st-coverage"
+	dirOfJob := path.Join("logs", prowJobName)
+	prefix := dirOfJob + "/"
+
+	MockRouterListAllAPI(router, 0)
+	listItems, err := qc.listEntries(prefix, "/")
+
+	assert.Equal(t, err, nil)
+	assert.Equal(t, len(listItems), 1)
+	assert.Equal(t, listItems[0].Key, "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json")
+}
+
+// test basic listEntries function, recover after 3 times
+func TestListAllFilesWithServerTimeoutAndRecover(t *testing.T) {
+	conf := Config{
+		Bucket: "artifacts",
+	}
+	qc, router, _, teardown := MockQiniuServer(&conf)
+	defer teardown()
+	prowJobName := "kodo-postsubmits-go-st-coverage"
+	dirOfJob := path.Join("logs", prowJobName)
+	prefix := dirOfJob + "/"
+
+	// recover after 3 times
+	MockRouterListAllAPI(router, 3)
+	listItems, err := qc.listEntries(prefix, "/")
+
+	assert.Equal(t, err, nil)
+	assert.Equal(t, len(listItems), 1)
+	assert.Equal(t, listItems[0].Key, "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json")
+}
+
+// test basic listEntries function, never recover
+func TestListAllFilesWithServerTimeout(t *testing.T) {
+	conf := Config{
+		Bucket: "artifacts",
+	}
+	qc, router, _, teardown := MockQiniuServer(&conf)
+	defer teardown()
+	prowJobName := "kodo-postsubmits-go-st-coverage"
+	dirOfJob := path.Join("logs", prowJobName)
+	prefix := dirOfJob + "/"
+
+	// never recover
+	MockRouterListAllAPI(router, 13)
+	_, err := qc.listEntries(prefix, "/")
+
+	assert.Equal(t, strings.Contains(err.Error(), "timed out: error accessing QINIU artifact"), true)
+}
+
+// test ListAll function
+func TestListAllFilesWithContext(t *testing.T) {
+	conf := Config{
+		Bucket: "artifacts",
+	}
+	qc, router, _, teardown := MockQiniuServer(&conf)
+	defer teardown()
+	prowJobName := "kodo-postsubmits-go-st-coverage"
+	dirOfJob := path.Join("logs", prowJobName)
+	prefix := dirOfJob + "/"
+
+	MockRouterListAllAPI(router, 0)
+	listItems, err := qc.ListAll(context.Background(), prefix, "/")
+
+	assert.Equal(t, err, nil)
+	assert.Equal(t, len(listItems), 1)
+	assert.Equal(t, listItems[0], "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json")
+}
+
+// test GetArtifactDetails function
+func TestGetArtifactDetails(t *testing.T) {
+	conf := Config{
+		Bucket: "artifacts",
+	}
+	qc, router, _, teardown := MockQiniuServer(&conf)
+	defer teardown()
+	prowJobName := "kodo-postsubmits-go-st-coverage"
+	dirOfJob := path.Join("logs", prowJobName)
+	prefix := dirOfJob + "/"
+
+	MockRouterListAllAPI(router, 0)
+	tmpl, err := qc.GetArtifactDetails(prefix)
+	assert.Equal(t, err, nil)
+	assert.Equal(t, len(tmpl.Items), 1)
+	assert.Equal(t, tmpl.Items[0].Name, "1181915661132107776/finished.json")
+	assert.Equal(t, strings.Contains(tmpl.Items[0].Url, prowJobName), true)
+}
+
+// test ListSubDirs function, recover after 3 times
+func TestListSubDirsWithServerTimeoutAndRecover(t *testing.T) {
+	conf := Config{
+		Bucket: "artifacts",
+	}
+	qc, router, _, teardown := MockQiniuServer(&conf)
+	defer teardown()
+	prowJobName := "kodo-postsubmits-go-st-coverage"
+	dirOfJob := path.Join("logs", prowJobName)
+	prefix := dirOfJob + "/"
+	localProfileContent := `mode: atomic
+"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30
+"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0`
+	// recover after 3 times
+	MockRouterAPI(router, localProfileContent, 3)
+	listItems, err := qc.ListSubDirs(prefix)
+
+	assert.Equal(t, err, nil)
+	assert.Equal(t, len(listItems), 1)
+	assert.Equal(t, listItems[0], "1181915661132107776")
+}
+
+// test ListSubDirs function, never recover
+func TestListSubDirsWithServerTimeout(t *testing.T) {
+	conf := Config{
+		Bucket: "artifacts",
+	}
+	qc, router, _, teardown := MockQiniuServer(&conf)
+	defer teardown()
+	prowJobName := "kodo-postsubmits-go-st-coverage"
+	dirOfJob := path.Join("logs", prowJobName)
+	prefix := dirOfJob + "/"
+	localProfileContent := `mode: atomic
+"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30
+"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0`
+	// never recover
+	MockRouterAPI(router, localProfileContent, 13)
+	_, err := qc.ListSubDirs(prefix)
+
+	assert.Equal(t, strings.Contains(err.Error(), "timed out: error accessing QINIU artifact"), true)
+}

pkg/qiniu/mock.go

@@ -43,11 +43,22 @@ func MockQiniuServer(config *Config) (client *Client, router *httprouter.Router,
 	return client, router, server.URL, server.Close
 }
 
-func MockRouterAPI(router *httprouter.Router, profile string) {
+// MockRouterAPI mocks qiniu /v2/list API.
+// You need to provide a expected profile content.
+// count controls the mocks qiniu server to error before 'count' times request.
+func MockRouterAPI(router *httprouter.Router, profile string, count int) {
+	timeout := count
+
 	// mock rsf /v2/list
 	router.HandlerFunc("POST", "/v2/list", func(w http.ResponseWriter, r *http.Request) {
 		logrus.Infof("request url is: %s", r.URL.String())
 
+		if timeout > 0 {
+			timeout -= 1
+			http.Error(w, "not found", http.StatusNotFound)
+			return
+		}
+
 		fmt.Fprint(w, `{
 	"item": {
 		"key": "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json",
@@ -75,3 +86,76 @@ func MockRouterAPI(router *httprouter.Router, profile string) {
 	})
 
 }
+
+// MockRouterListAllAPI mocks qiniu /list API.
+// count controls the mocks qiniu server to error before 'count' times request.
+func MockRouterListAllAPI(router *httprouter.Router, count int) {
+	timeout := count
+	// mock rsf /v2/list
+	router.HandlerFunc("POST", "/list", func(w http.ResponseWriter, r *http.Request) {
+		logrus.Infof("will respond after %v times", timeout)
+		logrus.Infof("request url is: %s", r.URL.String())
+
+		if timeout > 0 {
+			timeout -= 1
+			http.Error(w, "not found", http.StatusNotFound)
+			return
+		}
+
+		fmt.Fprint(w, `{
+	"items": [{
+		"key": "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json",
+		"hash": "FkBhdo9odL2Xjvu-YdwtDIw79fIL",
+		"fsize": 51523,
+		"mimeType": "application/octet-stream",
+		"putTime": 15909068578047958,
+		"type": 0,
+		"status": 0,
+		"md5": "e0bd20e97ea1c6a5e2480192ee3ae884"
+	}],
+	"marker": "",
+	"commonPrefixes": ["logs/kodo-postsubmits-go-st-coverage/1181915661132107776/"]
+}`)
+	})
+}
+
+// MockPrivateDomainUrl mocks bucket domain /key, /timeout, /retry API.
+// count controls the mocks qiniu server to error before 'count' times request.
+func MockPrivateDomainUrl(router *httprouter.Router, count int) {
+	timeout1 := count
+	timeout2 := count
+
+	router.HandlerFunc("GET", "/key", func(w http.ResponseWriter, r *http.Request) {
+		logrus.Infof("request url is: %s", r.URL.String())
+
+		fmt.Fprint(w, "mock server ok")
+	})
+
+	router.HandlerFunc("GET", "/timeout", func(w http.ResponseWriter, r *http.Request) {
+		logrus.Infof("request url is: %s", r.URL.String())
+
+		if timeout1 > 0 {
+			timeout1 -= 1
+			http.Error(w, "not found", http.StatusNotFound)
+			return
+		}
+
+		fmt.Fprint(w, "mock server ok")
+	})
+
+	router.HandlerFunc("GET", "/retry", func(w http.ResponseWriter, r *http.Request) {
+		logrus.Infof("request url is: %s", r.URL.String())
+
+		if timeout2 > 0 {
+			timeout2 -= 1
+			if timeout2%2 == 0 {
+				http.Error(w, "not found", 571)
+			} else {
+				http.Error(w, "not found", 573)
+			}
+			return
+		}
+
+		fmt.Fprint(w, "mock server ok")
+	})
+}

pkg/qiniu/object.go

@@ -97,7 +97,8 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64)
 func runWithRetry(maxTry int, f func() (bool, error)) error {
 	var err error
 	for maxTry > 0 {
-		needRetry, err := f()
+		var needRetry bool
+		needRetry, err = f() // fix -  needRetry, err := f(), err hides the outside error
 		if err != nil {
 			logrus.Warnf("err occurred: %v. try again", err)
 		} else if needRetry {

pkg/qiniu/object_test.go

@@ -0,0 +1,134 @@
+/*
+ Copyright 2020 Qiniu Cloud (qiniu.com)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package qiniu
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"testing"
+
+	"github.com/qiniu/api.v7/v7/auth/qbox"
+	"github.com/qiniu/api.v7/v7/client"
+	"github.com/stretchr/testify/assert"
+)
+
+// test NewRangeReader logic
+func TestNewRangeReader(t *testing.T) {
+	cfg := &Config{
+		Bucket:    "artifacts",
+		AccessKey: "ak",
+		SecretKey: "sk",
+	}
+	_, router, serverUrl, teardown := MockQiniuServer(cfg)
+	defer teardown()
+	cfg.Domain = serverUrl
+
+	MockPrivateDomainUrl(router, 0)
+
+	oh := &ObjectHandle{
+		key:    "key",
+		cfg:    cfg,
+		bm:     nil,
+		mac:    qbox.NewMac(cfg.AccessKey, cfg.SecretKey),
+		client: &client.Client{Client: http.DefaultClient},
+	}
+
+	// test read unlimited
+	body, err := oh.NewRangeReader(context.Background(), 0, -1)
+	assert.Equal(t, err, nil)
+
+	bodyBytes, err := ioutil.ReadAll(body)
+	assert.Equal(t, string(bodyBytes), "mock server ok")
+
+	// test with HEAD method
+	body, err = oh.NewRangeReader(context.Background(), 0, 0)
+	assert.Equal(t, err, nil)
+
+	bodyBytes, err = ioutil.ReadAll(body)
+	assert.Equal(t, string(bodyBytes), "")
+
+}
+
+// test retry logic
+func TestNewRangeReaderWithTimeoutAndRecover(t *testing.T) {
+	cfg := &Config{
+		Bucket:    "artifacts",
+		AccessKey: "ak",
+		SecretKey: "sk",
+	}
+	_, router, serverUrl, teardown := MockQiniuServer(cfg)
+	defer teardown()
+	cfg.Domain = serverUrl
+
+	MockPrivateDomainUrl(router, 2)
+
+	oh := &ObjectHandle{
+		key:    "key",
+		cfg:    cfg,
+		bm:     nil,
+		mac:    qbox.NewMac(cfg.AccessKey, cfg.SecretKey),
+		client: &client.Client{Client: http.DefaultClient},
+	}
+
+	// test with timeout
+	oh.key = "timeout"
+	body, err := oh.NewRangeReader(context.Background(), 0, 10)
+	assert.Equal(t, err, nil)
+
+	bodyBytes, err := ioutil.ReadAll(body)
+	assert.Equal(t, string(bodyBytes), "mock server ok")
+
+	// test with retry with statuscode=571, 573
+	oh.key = "retry"
+	body, err = oh.NewRangeReader(context.Background(), 0, 10)
+	assert.Equal(t, err, nil)
+
+	bodyBytes, err = ioutil.ReadAll(body)
+	assert.Equal(t, string(bodyBytes), "mock server ok")
+}
+
+// test retry logic
+func TestNewRangeReaderWithTimeoutNoRecover(t *testing.T) {
+	cfg := &Config{
+		Bucket:    "artifacts",
+		AccessKey: "ak",
+		SecretKey: "sk",
+	}
+	_, router, serverUrl, teardown := MockQiniuServer(cfg)
+	defer teardown()
+	cfg.Domain = serverUrl
+
+	MockPrivateDomainUrl(router, 12)
+
+	oh := &ObjectHandle{
+		key:    "key",
+		cfg:    cfg,
+		bm:     nil,
+		mac:    qbox.NewMac(cfg.AccessKey, cfg.SecretKey),
+		client: &client.Client{Client: http.DefaultClient},
+	}
+
+	// test with timeout
+	oh.key = "timeout"
+	_, err := oh.NewRangeReader(context.Background(), 0, -1)
+	assert.Equal(t, err, fmt.Errorf("qiniu storage: object not exists"))
+
+	// bodyBytes, err := ioutil.ReadAll(body)
+	// assert.Equal(t, string(bodyBytes), "mock server ok")
+}

pkg/qiniu/qnPresubmit_test.go

@@ -34,7 +34,7 @@ func TestFindBaseProfileFromQiniu(t *testing.T) {
 "qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30
 "qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0`
 
-	MockRouterAPI(router, mockProfileContent)
+	MockRouterAPI(router, mockProfileContent, 0)
 	getProfile, err := FindBaseProfileFromQiniu(qc, prowJobName, covProfileName)
 	assert.Equal(t, err, nil)
 	assert.Equal(t, string(getProfile), mockProfileContent)

tests/install.bats

@@ -39,12 +39,51 @@ setup() {
     export GO111MODULE=off
     cd samples/simple_gopath_project/src/qiniu.com/simple_gopath_project
 
-    wait_profile_backend "install" &
+    wait_profile_backend "install1" &
     profile_pid=$!
 
     run gocc install --debug --debugcisyncfile ci-sync.bak;
-    info install output: $output
+    info install1 output: $output
     [ "$status" -eq 0 ]
 
     wait $profile_pid
 }
+
+@test "test basic goc install command with GOBIN set" {
+    info $PWD
+    export GOPATH=$PWD/samples/simple_gopath_project
+    export GOBIN=$PWD
+    export GO111MODULE=off
+    cd samples/simple_gopath_project/src/qiniu.com/simple_gopath_project
+
+    wait_profile_backend "install2" &
+    profile_pid=$!
+
+    run gocc install --debug --debugcisyncfile ci-sync.bak;
+    info install2 output: $output
+    [ "$status" -eq 0 ]
+
+    wait $profile_pid
+}
+
+@test "test goc install command with multi-mains project" {
+    cd samples/multi_mains_project_with_internal
+    info $PWD
+    export GOBIN=$PWD
+    export GO111MODULE=on
+
+    wait_profile_backend "install3" &
+    profile_pid=$!
+
+    run gocc install ./... --debug --debugcisyncfile ci-sync.bak;
+    info install3 output: $output
+    [ "$status" -eq 0 ]
+
+    run ls -al 
+    info install3 ls output: $output
+
+    [[ -f main1 ]]
+    [[ -f main2 ]]
+
+    wait $profile_pid
+}

tests/run-ci-actions.sh

@@ -41,4 +41,4 @@ bats -t diff.bats
 
 bats -t cover.bats
 
-bash <(curl -s https://codecov.io/bash) -f 'filtered*' -F e2e
+bash <(curl -s https://codecov.io/bash) -f 'filtered*' -F e2e-$GOVERSION

tests/samples/multi_mains_project_with_internal/cmd/main1/main.go

@@ -0,0 +1,12 @@
+package main
+
+import (
+	"example.com/multi-mains-project/foo"
+	"example.com/multi-mains-project/internal"
+)
+
+func main() {
+	foo.Bar1()
+	foo.Bar2()
+	internal.Hello()
+}

tests/samples/multi_mains_project_with_internal/cmd/main2/main.go

@@ -0,0 +1,12 @@
+package main
+
+import (
+	"example.com/multi-mains-project/foo"
+	"example.com/multi-mains-project/internal"
+)
+
+func main() {
+	foo.Bar1()
+	foo.Bar2()
+	internal.Hello()
+}

tests/samples/multi_mains_project_with_internal/foo/bar1.go

@@ -0,0 +1,11 @@
+package foo
+
+import (
+	"example.com/multi-mains-project/foo/internal"
+	"example.com/multi-mains-project/foo/internal/qiniu"
+)
+
+func Bar1() {
+	internal.Xiaohong()
+	qiniu.BB()
+}

tests/samples/multi_mains_project_with_internal/foo/bar2.go

@@ -0,0 +1,11 @@
+package foo
+
+import (
+	"example.com/multi-mains-project/foo/internal"
+	"example.com/multi-mains-project/foo/internal/qiniu"
+)
+
+func Bar2() {
+	internal.Xiaohong()
+	qiniu.BB()
+}

tests/samples/multi_mains_project_with_internal/foo/internal/qiniu/xiaoda.go

@@ -0,0 +1,5 @@
+package qiniu
+
+func BB() {
+
+}

tests/samples/multi_mains_project_with_internal/foo/internal/xiaoming.go

@@ -0,0 +1,5 @@
+package internal
+
+func Xiaohong() {
+
+}

tests/samples/multi_mains_project_with_internal/go.mod

@@ -0,0 +1,3 @@
+module example.com/multi-mains-project
+
+go 1.11

tests/samples/multi_mains_project_with_internal/internal/foo.go

@@ -0,0 +1,7 @@
+package internal
+
+import "fmt"
+
+func Hello() {
+	fmt.Println("hello, world.")
+}

tests/samples/multi_mains_project_with_internal/main.go

@@ -0,0 +1,12 @@
+package main
+
+import (
+	"example.com/multi-mains-project/foo"
+	"example.com/multi-mains-project/internal"
+)
+
+func main() {
+	foo.Bar1()
+	foo.Bar2()
+	internal.Hello()
+}

tests/samples/simple_gopath_project/src/qiniu.com/simple_gopath_project/main.go

@@ -2,8 +2,10 @@ package main
 
 import (
 	"fmt"
+	"qiniu.com/simple_gopath_project/modulea"
 )
 
 func main() {
+	modulea.Bar()
 	fmt.Println("hello, world.")
 }

tests/samples/simple_gopath_project/src/qiniu.com/simple_gopath_project/modulea/foo.go

@@ -0,0 +1,5 @@
+package modulea
+
+func Bar() {
+
+}

tests/samples/simple_project_with_internal/foo/bar1.go

@@ -0,0 +1,11 @@
+package foo
+
+import (
+	"example.com/simple-project/foo/internal"
+	"example.com/simple-project/foo/internal/qiniu"
+)
+
+func Bar1() {
+	internal.Xiaohong()
+	qiniu.BB()
+}

tests/samples/simple_project_with_internal/foo/bar2.go

@@ -0,0 +1,11 @@
+package foo
+
+import (
+	"example.com/simple-project/foo/internal"
+	"example.com/simple-project/foo/internal/qiniu"
+)
+
+func Bar2() {
+	internal.Xiaohong()
+	qiniu.BB()
+}

tests/samples/simple_project_with_internal/foo/internal/qiniu/xiaoda.go

@@ -0,0 +1,5 @@
+package qiniu
+
+func BB() {
+
+}

tests/samples/simple_project_with_internal/foo/internal/xiaoming.go

@@ -0,0 +1,5 @@
+package internal
+
+func Xiaohong() {
+
+}

tests/samples/simple_project_with_internal/main.go

@@ -1,7 +1,12 @@
 package main
 
-import "example.com/simple-project/internal"
+import (
+	"example.com/simple-project/foo"
+	"example.com/simple-project/internal"
+)
 
 func main() {
+	foo.Bar1()
+	foo.Bar2()
 	internal.Hello()
 }