From 628629cbe7c2ba0a954d5eab64c1d6d0d275cc31 Mon Sep 17 00:00:00 2001 From: lyyyuna Date: Tue, 21 Jul 2020 11:49:57 +0800 Subject: [PATCH] add unittes case & add e2e sample --- .github/workflows/e2e_test_check.yml | 2 + .github/workflows/ut_check.yml | 4 +- .gitignore | 3 +- go.sum | 1 + pkg/build/build_test.go | 9 ++ pkg/build/gomodules_test.go | 55 +++++++ pkg/build/legacy_test.go | 71 +++++++++ pkg/build/tmpfolder_test.go | 45 ++++++ pkg/cover/client_test.go | 35 ++++- pkg/cover/server.go | 2 +- pkg/cover/server_test.go | 140 ++++++++++++++++- pkg/prow/job_test.go | 2 +- pkg/qiniu/client_test.go | 146 +++++++++++++++++- pkg/qiniu/mock.go | 86 ++++++++++- pkg/qiniu/object.go | 3 +- pkg/qiniu/object_test.go | 134 ++++++++++++++++ pkg/qiniu/qnPresubmit_test.go | 2 +- tests/install.bats | 43 +++++- tests/run-ci-actions.sh | 2 +- .../cmd/main1/main.go | 12 ++ .../cmd/main2/main.go | 12 ++ .../foo/bar1.go | 11 ++ .../foo/bar2.go | 11 ++ .../foo/internal/qiniu/xiaoda.go | 5 + .../foo/internal/xiaoming.go | 5 + .../multi_mains_project_with_internal/go.mod | 3 + .../internal/foo.go | 7 + .../multi_mains_project_with_internal/main.go | 12 ++ .../qiniu.com/simple_gopath_project/main.go | 2 + .../simple_gopath_project/modulea/foo.go | 5 + .../simple_project_with_internal/foo/bar1.go | 11 ++ .../simple_project_with_internal/foo/bar2.go | 11 ++ .../foo/internal/qiniu/xiaoda.go | 5 + .../foo/internal/xiaoming.go | 5 + .../simple_project_with_internal/main.go | 7 +- 35 files changed, 890 insertions(+), 19 deletions(-) create mode 100644 pkg/build/gomodules_test.go create mode 100644 pkg/build/legacy_test.go create mode 100644 pkg/qiniu/object_test.go create mode 100644 tests/samples/multi_mains_project_with_internal/cmd/main1/main.go create mode 100644 tests/samples/multi_mains_project_with_internal/cmd/main2/main.go create mode 100644 tests/samples/multi_mains_project_with_internal/foo/bar1.go create mode 100644 tests/samples/multi_mains_project_with_internal/foo/bar2.go create mode 100644 tests/samples/multi_mains_project_with_internal/foo/internal/qiniu/xiaoda.go create mode 100644 tests/samples/multi_mains_project_with_internal/foo/internal/xiaoming.go create mode 100644 tests/samples/multi_mains_project_with_internal/go.mod create mode 100644 tests/samples/multi_mains_project_with_internal/internal/foo.go create mode 100644 tests/samples/multi_mains_project_with_internal/main.go create mode 100644 tests/samples/simple_gopath_project/src/qiniu.com/simple_gopath_project/modulea/foo.go create mode 100644 tests/samples/simple_project_with_internal/foo/bar1.go create mode 100644 tests/samples/simple_project_with_internal/foo/bar2.go create mode 100644 tests/samples/simple_project_with_internal/foo/internal/qiniu/xiaoda.go create mode 100644 tests/samples/simple_project_with_internal/foo/internal/xiaoming.go diff --git a/.github/workflows/e2e_test_check.yml b/.github/workflows/e2e_test_check.yml index c27f5ed..4739346 100644 --- a/.github/workflows/e2e_test_check.yml +++ b/.github/workflows/e2e_test_check.yml @@ -60,6 +60,8 @@ jobs: cd bats-core sudo ./install.sh /usr/local - name: Do test + env: + GOVERSION: ${{ matrix.go-version }} run: | chmod +x /home/runner/tools/goc/goc export PATH=/home/runner/tools/goc:$PATH diff --git a/.github/workflows/ut_check.yml b/.github/workflows/ut_check.yml index 86d7600..0b9d311 100644 --- a/.github/workflows/ut_check.yml +++ b/.github/workflows/ut_check.yml @@ -24,7 +24,9 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - name: Go test + env: + GOVERSION: ${{ matrix.go-version }} run: | export DEFAULT_EXCEPT_PKGS=e2e go test -p 1 -coverprofile=coverage.txt $(go list ./... | grep -v -E $DEFAULT_EXCEPT_PKGS) - bash <(curl -s https://codecov.io/bash) -F unittest + bash <(curl -s https://codecov.io/bash) -F unittest-$GOVERSION diff --git a/.gitignore b/.gitignore index 70cf02d..f5c9ba6 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ goc _svrs_address.txt # other -*.iml \ No newline at end of file +*.iml +.DS_Store \ No newline at end of file diff --git a/go.sum b/go.sum index d6bec85..3f22cde 100644 --- a/go.sum +++ b/go.sum @@ -696,6 +696,7 @@ github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jW github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/pkg/build/build_test.go b/pkg/build/build_test.go index 82df7ea..7255730 100644 --- a/pkg/build/build_test.go +++ b/pkg/build/build_test.go @@ -90,3 +90,12 @@ func TestInvalidPackageNameForBuild(t *testing.T) { assert.FailNow(t, "should not success with non . or ./... package") } } + +// test NewBuild with wrong parameters +func TestNewBuildWithWrongParameters(t *testing.T) { + _, err := NewBuild("", []string{"a.go", "b.go"}, "cur", "cur") + assert.Equal(t, err, ErrTooManyArgs) + + _, err = NewBuild("", []string{"a.go"}, "", "cur") + assert.Equal(t, err, ErrInvalidWorkingDir) +} diff --git a/pkg/build/gomodules_test.go b/pkg/build/gomodules_test.go new file mode 100644 index 0000000..12276bd --- /dev/null +++ b/pkg/build/gomodules_test.go @@ -0,0 +1,55 @@ +/* + Copyright 2020 Qiniu Cloud (qiniu.com) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package build + +import ( + "bytes" + "os" + "strings" + "testing" + + "github.com/qiniu/goc/pkg/cover" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +func captureOutput(f func()) string { + var buf bytes.Buffer + log.SetOutput(&buf) + f() + log.SetOutput(os.Stderr) + return buf.String() +} + +// copy in cpGoModulesProject of invalid src, dst name +func TestModProjectCopyWithUnexistedDir(t *testing.T) { + pkgs := make(map[string]*cover.Package) + pkgs["main"] = &cover.Package{ + Name: "main", + Module: &cover.ModulePublic{ + Dir: "not exied, ia mas duser", // not real one, should fail copy + }, + } + pkgs["another"] = &cover.Package{} + b := &Build{ + TmpDir: "sdfsfev2234444", // not real one, should fail copy + Pkgs: pkgs, + } + + output := captureOutput(b.cpGoModulesProject) + assert.Equal(t, strings.Contains(output, "Failed to Copy"), true) +} diff --git a/pkg/build/legacy_test.go b/pkg/build/legacy_test.go new file mode 100644 index 0000000..53f9cc1 --- /dev/null +++ b/pkg/build/legacy_test.go @@ -0,0 +1,71 @@ +/* + Copyright 2020 Qiniu Cloud (qiniu.com) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package build + +import ( + "path/filepath" + "strings" + "testing" + + "github.com/qiniu/goc/pkg/cover" + "github.com/stretchr/testify/assert" +) + +// copy in cpLegacyProject/cpNonStandardLegacy of invalid src, dst name +func TestLegacyProjectCopyWithUnexistedDir(t *testing.T) { + pkgs := make(map[string]*cover.Package) + pkgs["main"] = &cover.Package{ + Module: &cover.ModulePublic{ + Dir: "not exied, ia mas duser", // not real one, should fail copy + }, + Dir: "not exit, iasdfs", + Name: "main", + } + pkgs["another"] = &cover.Package{} + b := &Build{ + TmpDir: "sdfsfev2234444", // not real one, should fail copy + Pkgs: pkgs, + } + + output := captureOutput(b.cpLegacyProject) + assert.Equal(t, strings.Contains(output, "Failed to Copy"), true) + + output = captureOutput(b.cpNonStandardLegacy) + assert.Equal(t, strings.Contains(output, "Failed to Copy"), true) +} + +// copy in cpDepPackages of invalid dst name +func TestDepPackagesCopyWithInvalidDir(t *testing.T) { + gopath := filepath.Join(baseDir, "../../tests/samples/simple_gopath_project") + pkg := &cover.Package{ + Module: &cover.ModulePublic{ + Dir: "not exied, ia mas duser", + }, + Root: gopath, + Deps: []string{"qiniu.com", "ddfee 2344234"}, + } + b := &Build{ + TmpDir: "/", // "/" is invalid dst in Linux, it should fail + } + + output := captureOutput(func() { + visited := make(map[string]bool) + + b.cpDepPackages(pkg, visited) + }) + assert.Equal(t, strings.Contains(output, "Failed to Copy"), true) +} diff --git a/pkg/build/tmpfolder_test.go b/pkg/build/tmpfolder_test.go index 9373ac8..c730068 100644 --- a/pkg/build/tmpfolder_test.go +++ b/pkg/build/tmpfolder_test.go @@ -22,6 +22,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/stretchr/testify/assert" ) var baseDir string @@ -102,3 +104,46 @@ func TestLegacyProjectNotInGoPATH(t *testing.T) { t.Fatalf("There should be a main.go in temporary directory directly, the error: %v", err) } } + +// test traversePkgsList error case +func TestTraversePkgsList(t *testing.T) { + b := &Build{ + Pkgs: nil, + } + _, _, err := b.traversePkgsList() + assert.EqualError(t, err, ErrShouldNotReached.Error()) +} + +// test getTmpwd error case +func TestGetTmpwd(t *testing.T) { + b := &Build{ + Pkgs: nil, + } + _, err := b.getTmpwd() + assert.EqualError(t, err, ErrShouldNotReached.Error()) +} + +// test findWhereToInstall +func TestFindWhereToInstall(t *testing.T) { + // if a legacy project without project root find + // should find no plcae to install + b := &Build{ + Pkgs: nil, + IsMod: false, + Root: "", + } + _, err := b.findWhereToInstall() + assert.EqualError(t, err, ErrNoplaceToInstall.Error()) + + // if $GOBIN not found + // and if $GOPATH not found + // should install to $HOME/go/bin + b = &Build{ + Pkgs: nil, + IsMod: true, + OriGOPATH: "", + } + placeToInstall, err := b.findWhereToInstall() + expectedPlace := filepath.Join(os.Getenv("HOME"), "go", "bin") + assert.Equal(t, placeToInstall, expectedPlace) +} diff --git a/pkg/cover/client_test.go b/pkg/cover/client_test.go index 687eb1c..23ea4bd 100644 --- a/pkg/cover/client_test.go +++ b/pkg/cover/client_test.go @@ -21,8 +21,9 @@ import ( "os" "testing" - "github.com/stretchr/testify/assert" "net/http" + + "github.com/stretchr/testify/assert" ) func TestClientAction(t *testing.T) { @@ -128,9 +129,31 @@ func TestClientAction(t *testing.T) { assert.Equal(t, "{}", string(res)) } -func TestE2E(t *testing.T) { - // FIXME: start goc server - // FIXME: call goc build to cover goc server - // FIXME: do some tests again goc server - // FIXME: goc profile and checkout coverage +func TestClientRegisterService(t *testing.T) { + c := &client{} + + // client register with empty address + testService1 := Service{ + Address: "", + Name: "abc", + } + _, err := c.RegisterService(testService1) + assert.Contains(t, err.Error(), "empty url") + + // client register with empty name + testService2 := Service{ + Address: "http://127.0.0.1:444", + Name: "", + } + _, err = c.RegisterService(testService2) + assert.EqualError(t, err, "invalid service name") +} + +func TestClientListServices(t *testing.T) { + c := &client{ + Host: "http://127.0.0.1:64445", // a invalid host + client: http.DefaultClient, + } + _, err := c.ListServices() + assert.Contains(t, err.Error(), "connect: connection refused") } diff --git a/pkg/cover/server.go b/pkg/cover/server.go index 3b36122..05d0efe 100644 --- a/pkg/cover/server.go +++ b/pkg/cover/server.go @@ -25,12 +25,12 @@ import ( "net/http" "net/url" "os" + "strconv" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" "golang.org/x/tools/cover" "k8s.io/test-infra/gopherage/pkg/cov" - "strconv" ) // DefaultStore implements the IPersistence interface diff --git a/pkg/cover/server_test.go b/pkg/cover/server_test.go index 61f65ed..806c63a 100644 --- a/pkg/cover/server_test.go +++ b/pkg/cover/server_test.go @@ -1,10 +1,46 @@ package cover import ( - "github.com/stretchr/testify/assert" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" ) +// MockStore is mock store mainly for unittest +type MockStore struct { + mock.Mock +} + +func (m *MockStore) Add(s Service) error { + args := m.Called(s) + return args.Error(0) +} + +func (m *MockStore) Get(name string) []string { + args := m.Called(name) + return args.Get(0).([]string) +} + +func (m *MockStore) GetAll() map[string][]string { + args := m.Called() + return args.Get(0).(map[string][]string) +} + +func (m *MockStore) Init() error { + args := m.Called() + return args.Error(0) +} + +func (m *MockStore) Set(services map[string][]string) { +} + func TestContains(t *testing.T) { assert.Equal(t, contains([]string{"a", "b"}, "a"), true) assert.Equal(t, contains([]string{"a", "b"}, "c"), false) @@ -76,3 +112,105 @@ func TestRemoveDuplicateElement(t *testing.T) { strArr := []string{"a", "a", "b"} assert.Equal(t, removeDuplicateElement(strArr), []string{"a", "b"}) } + +func TestRegisterService(t *testing.T) { + router := GocServer(os.Stdout) + + // register with empty service struct + w := httptest.NewRecorder() + req, _ := http.NewRequest("POST", "/v1/cover/register", nil) + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Code) + + // register with invalid service.Address + data := url.Values{} + data.Set("name", "aaa") + data.Set("address", "&%%") + w = httptest.NewRecorder() + req, _ = http.NewRequest("POST", "/v1/cover/register", strings.NewReader(data.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.Contains(t, w.Body.String(), "invalid URL escape") + + // register with host but no port + data = url.Values{} + data.Set("name", "aaa") + data.Set("address", "http://127.0.0.1") + w = httptest.NewRecorder() + req, _ = http.NewRequest("POST", "/v1/cover/register", strings.NewReader(data.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.Contains(t, w.Body.String(), "missing port in address") + + // register with store failure + expectedS := Service{ + Name: "foo", + Address: "http://:64444", // the real IP is empty in unittest, so server will get a empty one + } + testObj := new(MockStore) + testObj.On("Get", "foo").Return([]string{"http://127.0.0.1:66666"}) + testObj.On("Add", expectedS).Return(fmt.Errorf("lala error")) + + DefaultStore = testObj + + w = httptest.NewRecorder() + data.Set("name", expectedS.Name) + data.Set("address", expectedS.Address) + req, _ = http.NewRequest("POST", "/v1/cover/register", strings.NewReader(data.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusInternalServerError, w.Code) + assert.Contains(t, w.Body.String(), "lala error") +} + +func TestProfileService(t *testing.T) { + router := GocServer(os.Stdout) + + // get profile with invalid force parameter + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/v1/cover/profile?force=11", nil) + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusExpectationFailed, w.Code) + assert.Contains(t, w.Body.String(), "invalid param") +} + +func TestClearService(t *testing.T) { + testObj := new(MockStore) + testObj.On("GetAll").Return(map[string][]string{"foo": {"http://127.0.0.1:66666"}}) + + DefaultStore = testObj + + router := GocServer(os.Stdout) + + // get profile with invalid force parameter + w := httptest.NewRecorder() + req, _ := http.NewRequest("POST", "/v1/cover/clear", nil) + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusExpectationFailed, w.Code) + assert.Contains(t, w.Body.String(), "invalid port") +} + +func TestInitService(t *testing.T) { + testObj := new(MockStore) + testObj.On("Init").Return(fmt.Errorf("lala error")) + + DefaultStore = testObj + + router := GocServer(os.Stdout) + + // get profile with invalid force parameter + w := httptest.NewRecorder() + req, _ := http.NewRequest("POST", "/v1/cover/init", nil) + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusInternalServerError, w.Code) + assert.Contains(t, w.Body.String(), "lala error") +} diff --git a/pkg/prow/job_test.go b/pkg/prow/job_test.go index 048ba11..85899d4 100644 --- a/pkg/prow/job_test.go +++ b/pkg/prow/job_test.go @@ -111,7 +111,7 @@ func TestRunPresubmitFulldiff(t *testing.T) { } qc, router, _, teardown := qiniu.MockQiniuServer(&conf) defer teardown() - qiniu.MockRouterAPI(router, localProfileContent) + qiniu.MockRouterAPI(router, localProfileContent, 0) ChangedProfilePath := "changed.cov" defer os.Remove(path.Join(pwd, ChangedProfilePath)) diff --git a/pkg/qiniu/client_test.go b/pkg/qiniu/client_test.go index 0c1b624..a8367a1 100644 --- a/pkg/qiniu/client_test.go +++ b/pkg/qiniu/client_test.go @@ -16,7 +16,14 @@ package qiniu -import "testing" +import ( + "context" + "path" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) func TestGetBuildId(t *testing.T) { type tc struct { @@ -38,3 +45,140 @@ func TestGetBuildId(t *testing.T) { } } } + +// test basic listEntries function +func TestListAllFiles(t *testing.T) { + conf := Config{ + Bucket: "artifacts", + } + qc, router, _, teardown := MockQiniuServer(&conf) + defer teardown() + prowJobName := "kodo-postsubmits-go-st-coverage" + dirOfJob := path.Join("logs", prowJobName) + prefix := dirOfJob + "/" + + MockRouterListAllAPI(router, 0) + listItems, err := qc.listEntries(prefix, "/") + + assert.Equal(t, err, nil) + assert.Equal(t, len(listItems), 1) + assert.Equal(t, listItems[0].Key, "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json") +} + +// test basic listEntries function, recover after 3 times +func TestListAllFilesWithServerTimeoutAndRecover(t *testing.T) { + conf := Config{ + Bucket: "artifacts", + } + qc, router, _, teardown := MockQiniuServer(&conf) + defer teardown() + prowJobName := "kodo-postsubmits-go-st-coverage" + dirOfJob := path.Join("logs", prowJobName) + prefix := dirOfJob + "/" + + // recover after 3 times + MockRouterListAllAPI(router, 3) + listItems, err := qc.listEntries(prefix, "/") + + assert.Equal(t, err, nil) + assert.Equal(t, len(listItems), 1) + assert.Equal(t, listItems[0].Key, "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json") +} + +// test basic listEntries function, never recover +func TestListAllFilesWithServerTimeout(t *testing.T) { + conf := Config{ + Bucket: "artifacts", + } + qc, router, _, teardown := MockQiniuServer(&conf) + defer teardown() + prowJobName := "kodo-postsubmits-go-st-coverage" + dirOfJob := path.Join("logs", prowJobName) + prefix := dirOfJob + "/" + + // never recover + MockRouterListAllAPI(router, 13) + _, err := qc.listEntries(prefix, "/") + + assert.Equal(t, strings.Contains(err.Error(), "timed out: error accessing QINIU artifact"), true) +} + +// test ListAll function +func TestListAllFilesWithContext(t *testing.T) { + conf := Config{ + Bucket: "artifacts", + } + qc, router, _, teardown := MockQiniuServer(&conf) + defer teardown() + prowJobName := "kodo-postsubmits-go-st-coverage" + dirOfJob := path.Join("logs", prowJobName) + prefix := dirOfJob + "/" + + MockRouterListAllAPI(router, 0) + listItems, err := qc.ListAll(context.Background(), prefix, "/") + + assert.Equal(t, err, nil) + assert.Equal(t, len(listItems), 1) + assert.Equal(t, listItems[0], "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json") +} + +// test GetArtifactDetails function +func TestGetArtifactDetails(t *testing.T) { + conf := Config{ + Bucket: "artifacts", + } + qc, router, _, teardown := MockQiniuServer(&conf) + defer teardown() + prowJobName := "kodo-postsubmits-go-st-coverage" + dirOfJob := path.Join("logs", prowJobName) + prefix := dirOfJob + "/" + + MockRouterListAllAPI(router, 0) + tmpl, err := qc.GetArtifactDetails(prefix) + assert.Equal(t, err, nil) + assert.Equal(t, len(tmpl.Items), 1) + assert.Equal(t, tmpl.Items[0].Name, "1181915661132107776/finished.json") + assert.Equal(t, strings.Contains(tmpl.Items[0].Url, prowJobName), true) +} + +// test ListSubDirs function, recover after 3 times +func TestListSubDirsWithServerTimeoutAndRecover(t *testing.T) { + conf := Config{ + Bucket: "artifacts", + } + qc, router, _, teardown := MockQiniuServer(&conf) + defer teardown() + prowJobName := "kodo-postsubmits-go-st-coverage" + dirOfJob := path.Join("logs", prowJobName) + prefix := dirOfJob + "/" + localProfileContent := `mode: atomic +"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30 +"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0` + // recover after 3 times + MockRouterAPI(router, localProfileContent, 3) + listItems, err := qc.ListSubDirs(prefix) + + assert.Equal(t, err, nil) + assert.Equal(t, len(listItems), 1) + assert.Equal(t, listItems[0], "1181915661132107776") +} + +// test ListSubDirs function, never recover +func TestListSubDirsWithServerTimeout(t *testing.T) { + conf := Config{ + Bucket: "artifacts", + } + qc, router, _, teardown := MockQiniuServer(&conf) + defer teardown() + prowJobName := "kodo-postsubmits-go-st-coverage" + dirOfJob := path.Join("logs", prowJobName) + prefix := dirOfJob + "/" + localProfileContent := `mode: atomic +"qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30 +"qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0` + // never recover + MockRouterAPI(router, localProfileContent, 13) + _, err := qc.ListSubDirs(prefix) + + assert.Equal(t, strings.Contains(err.Error(), "timed out: error accessing QINIU artifact"), true) +} diff --git a/pkg/qiniu/mock.go b/pkg/qiniu/mock.go index f60abae..59e56ad 100644 --- a/pkg/qiniu/mock.go +++ b/pkg/qiniu/mock.go @@ -43,11 +43,22 @@ func MockQiniuServer(config *Config) (client *Client, router *httprouter.Router, return client, router, server.URL, server.Close } -func MockRouterAPI(router *httprouter.Router, profile string) { +// MockRouterAPI mocks qiniu /v2/list API. +// You need to provide a expected profile content. +// count controls the mocks qiniu server to error before 'count' times request. +func MockRouterAPI(router *httprouter.Router, profile string, count int) { + timeout := count + // mock rsf /v2/list router.HandlerFunc("POST", "/v2/list", func(w http.ResponseWriter, r *http.Request) { logrus.Infof("request url is: %s", r.URL.String()) + if timeout > 0 { + timeout -= 1 + http.Error(w, "not found", http.StatusNotFound) + return + } + fmt.Fprint(w, `{ "item": { "key": "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json", @@ -75,3 +86,76 @@ func MockRouterAPI(router *httprouter.Router, profile string) { }) } + +// MockRouterListAllAPI mocks qiniu /list API. +// count controls the mocks qiniu server to error before 'count' times request. +func MockRouterListAllAPI(router *httprouter.Router, count int) { + timeout := count + // mock rsf /v2/list + router.HandlerFunc("POST", "/list", func(w http.ResponseWriter, r *http.Request) { + logrus.Infof("will respond after %v times", timeout) + logrus.Infof("request url is: %s", r.URL.String()) + + if timeout > 0 { + timeout -= 1 + http.Error(w, "not found", http.StatusNotFound) + return + } + + fmt.Fprint(w, `{ + "items": [{ + "key": "logs/kodo-postsubmits-go-st-coverage/1181915661132107776/finished.json", + "hash": "FkBhdo9odL2Xjvu-YdwtDIw79fIL", + "fsize": 51523, + "mimeType": "application/octet-stream", + "putTime": 15909068578047958, + "type": 0, + "status": 0, + "md5": "e0bd20e97ea1c6a5e2480192ee3ae884" + }], + "marker": "", + "commonPrefixes": ["logs/kodo-postsubmits-go-st-coverage/1181915661132107776/"] +}`) + }) +} + +// MockPrivateDomainUrl mocks bucket domain /key, /timeout, /retry API. +// count controls the mocks qiniu server to error before 'count' times request. +func MockPrivateDomainUrl(router *httprouter.Router, count int) { + timeout1 := count + timeout2 := count + + router.HandlerFunc("GET", "/key", func(w http.ResponseWriter, r *http.Request) { + logrus.Infof("request url is: %s", r.URL.String()) + + fmt.Fprint(w, "mock server ok") + }) + + router.HandlerFunc("GET", "/timeout", func(w http.ResponseWriter, r *http.Request) { + logrus.Infof("request url is: %s", r.URL.String()) + + if timeout1 > 0 { + timeout1 -= 1 + http.Error(w, "not found", http.StatusNotFound) + return + } + + fmt.Fprint(w, "mock server ok") + }) + + router.HandlerFunc("GET", "/retry", func(w http.ResponseWriter, r *http.Request) { + logrus.Infof("request url is: %s", r.URL.String()) + + if timeout2 > 0 { + timeout2 -= 1 + if timeout2%2 == 0 { + http.Error(w, "not found", 571) + } else { + http.Error(w, "not found", 573) + } + return + } + + fmt.Fprint(w, "mock server ok") + }) +} diff --git a/pkg/qiniu/object.go b/pkg/qiniu/object.go index 940bb86..e374c60 100644 --- a/pkg/qiniu/object.go +++ b/pkg/qiniu/object.go @@ -97,7 +97,8 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) func runWithRetry(maxTry int, f func() (bool, error)) error { var err error for maxTry > 0 { - needRetry, err := f() + var needRetry bool + needRetry, err = f() // fix - needRetry, err := f(), err hides the outside error if err != nil { logrus.Warnf("err occurred: %v. try again", err) } else if needRetry { diff --git a/pkg/qiniu/object_test.go b/pkg/qiniu/object_test.go new file mode 100644 index 0000000..cfd5424 --- /dev/null +++ b/pkg/qiniu/object_test.go @@ -0,0 +1,134 @@ +/* + Copyright 2020 Qiniu Cloud (qiniu.com) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package qiniu + +import ( + "context" + "fmt" + "io/ioutil" + "net/http" + "testing" + + "github.com/qiniu/api.v7/v7/auth/qbox" + "github.com/qiniu/api.v7/v7/client" + "github.com/stretchr/testify/assert" +) + +// test NewRangeReader logic +func TestNewRangeReader(t *testing.T) { + cfg := &Config{ + Bucket: "artifacts", + AccessKey: "ak", + SecretKey: "sk", + } + _, router, serverUrl, teardown := MockQiniuServer(cfg) + defer teardown() + cfg.Domain = serverUrl + + MockPrivateDomainUrl(router, 0) + + oh := &ObjectHandle{ + key: "key", + cfg: cfg, + bm: nil, + mac: qbox.NewMac(cfg.AccessKey, cfg.SecretKey), + client: &client.Client{Client: http.DefaultClient}, + } + + // test read unlimited + body, err := oh.NewRangeReader(context.Background(), 0, -1) + assert.Equal(t, err, nil) + + bodyBytes, err := ioutil.ReadAll(body) + assert.Equal(t, string(bodyBytes), "mock server ok") + + // test with HEAD method + body, err = oh.NewRangeReader(context.Background(), 0, 0) + assert.Equal(t, err, nil) + + bodyBytes, err = ioutil.ReadAll(body) + assert.Equal(t, string(bodyBytes), "") + +} + +// test retry logic +func TestNewRangeReaderWithTimeoutAndRecover(t *testing.T) { + cfg := &Config{ + Bucket: "artifacts", + AccessKey: "ak", + SecretKey: "sk", + } + _, router, serverUrl, teardown := MockQiniuServer(cfg) + defer teardown() + cfg.Domain = serverUrl + + MockPrivateDomainUrl(router, 2) + + oh := &ObjectHandle{ + key: "key", + cfg: cfg, + bm: nil, + mac: qbox.NewMac(cfg.AccessKey, cfg.SecretKey), + client: &client.Client{Client: http.DefaultClient}, + } + + // test with timeout + oh.key = "timeout" + body, err := oh.NewRangeReader(context.Background(), 0, 10) + assert.Equal(t, err, nil) + + bodyBytes, err := ioutil.ReadAll(body) + assert.Equal(t, string(bodyBytes), "mock server ok") + + // test with retry with statuscode=571, 573 + oh.key = "retry" + body, err = oh.NewRangeReader(context.Background(), 0, 10) + assert.Equal(t, err, nil) + + bodyBytes, err = ioutil.ReadAll(body) + assert.Equal(t, string(bodyBytes), "mock server ok") +} + +// test retry logic +func TestNewRangeReaderWithTimeoutNoRecover(t *testing.T) { + cfg := &Config{ + Bucket: "artifacts", + AccessKey: "ak", + SecretKey: "sk", + } + _, router, serverUrl, teardown := MockQiniuServer(cfg) + defer teardown() + cfg.Domain = serverUrl + + MockPrivateDomainUrl(router, 12) + + oh := &ObjectHandle{ + key: "key", + cfg: cfg, + bm: nil, + mac: qbox.NewMac(cfg.AccessKey, cfg.SecretKey), + client: &client.Client{Client: http.DefaultClient}, + } + + // test with timeout + oh.key = "timeout" + _, err := oh.NewRangeReader(context.Background(), 0, -1) + assert.Equal(t, err, fmt.Errorf("qiniu storage: object not exists")) + + // bodyBytes, err := ioutil.ReadAll(body) + // assert.Equal(t, string(bodyBytes), "mock server ok") +} diff --git a/pkg/qiniu/qnPresubmit_test.go b/pkg/qiniu/qnPresubmit_test.go index 3f5a29a..dfb0d94 100644 --- a/pkg/qiniu/qnPresubmit_test.go +++ b/pkg/qiniu/qnPresubmit_test.go @@ -34,7 +34,7 @@ func TestFindBaseProfileFromQiniu(t *testing.T) { "qiniu.com/kodo/apiserver/server/main.go:32.49,33.13 1 30 "qiniu.com/kodo/apiserver/server/main.go:42.49,43.13 1 0` - MockRouterAPI(router, mockProfileContent) + MockRouterAPI(router, mockProfileContent, 0) getProfile, err := FindBaseProfileFromQiniu(qc, prowJobName, covProfileName) assert.Equal(t, err, nil) assert.Equal(t, string(getProfile), mockProfileContent) diff --git a/tests/install.bats b/tests/install.bats index dc4252d..40e0a7a 100755 --- a/tests/install.bats +++ b/tests/install.bats @@ -39,12 +39,51 @@ setup() { export GO111MODULE=off cd samples/simple_gopath_project/src/qiniu.com/simple_gopath_project - wait_profile_backend "install" & + wait_profile_backend "install1" & profile_pid=$! run gocc install --debug --debugcisyncfile ci-sync.bak; - info install output: $output + info install1 output: $output [ "$status" -eq 0 ] wait $profile_pid } + +@test "test basic goc install command with GOBIN set" { + info $PWD + export GOPATH=$PWD/samples/simple_gopath_project + export GOBIN=$PWD + export GO111MODULE=off + cd samples/simple_gopath_project/src/qiniu.com/simple_gopath_project + + wait_profile_backend "install2" & + profile_pid=$! + + run gocc install --debug --debugcisyncfile ci-sync.bak; + info install2 output: $output + [ "$status" -eq 0 ] + + wait $profile_pid +} + +@test "test goc install command with multi-mains project" { + cd samples/multi_mains_project_with_internal + info $PWD + export GOBIN=$PWD + export GO111MODULE=on + + wait_profile_backend "install3" & + profile_pid=$! + + run gocc install ./... --debug --debugcisyncfile ci-sync.bak; + info install3 output: $output + [ "$status" -eq 0 ] + + run ls -al + info install3 ls output: $output + + [[ -f main1 ]] + [[ -f main2 ]] + + wait $profile_pid +} diff --git a/tests/run-ci-actions.sh b/tests/run-ci-actions.sh index c8840d2..792b68d 100755 --- a/tests/run-ci-actions.sh +++ b/tests/run-ci-actions.sh @@ -41,4 +41,4 @@ bats -t diff.bats bats -t cover.bats -bash <(curl -s https://codecov.io/bash) -f 'filtered*' -F e2e \ No newline at end of file +bash <(curl -s https://codecov.io/bash) -f 'filtered*' -F e2e-$GOVERSION \ No newline at end of file diff --git a/tests/samples/multi_mains_project_with_internal/cmd/main1/main.go b/tests/samples/multi_mains_project_with_internal/cmd/main1/main.go new file mode 100644 index 0000000..bb80c64 --- /dev/null +++ b/tests/samples/multi_mains_project_with_internal/cmd/main1/main.go @@ -0,0 +1,12 @@ +package main + +import ( + "example.com/multi-mains-project/foo" + "example.com/multi-mains-project/internal" +) + +func main() { + foo.Bar1() + foo.Bar2() + internal.Hello() +} diff --git a/tests/samples/multi_mains_project_with_internal/cmd/main2/main.go b/tests/samples/multi_mains_project_with_internal/cmd/main2/main.go new file mode 100644 index 0000000..bb80c64 --- /dev/null +++ b/tests/samples/multi_mains_project_with_internal/cmd/main2/main.go @@ -0,0 +1,12 @@ +package main + +import ( + "example.com/multi-mains-project/foo" + "example.com/multi-mains-project/internal" +) + +func main() { + foo.Bar1() + foo.Bar2() + internal.Hello() +} diff --git a/tests/samples/multi_mains_project_with_internal/foo/bar1.go b/tests/samples/multi_mains_project_with_internal/foo/bar1.go new file mode 100644 index 0000000..f0bef31 --- /dev/null +++ b/tests/samples/multi_mains_project_with_internal/foo/bar1.go @@ -0,0 +1,11 @@ +package foo + +import ( + "example.com/multi-mains-project/foo/internal" + "example.com/multi-mains-project/foo/internal/qiniu" +) + +func Bar1() { + internal.Xiaohong() + qiniu.BB() +} diff --git a/tests/samples/multi_mains_project_with_internal/foo/bar2.go b/tests/samples/multi_mains_project_with_internal/foo/bar2.go new file mode 100644 index 0000000..40dfd43 --- /dev/null +++ b/tests/samples/multi_mains_project_with_internal/foo/bar2.go @@ -0,0 +1,11 @@ +package foo + +import ( + "example.com/multi-mains-project/foo/internal" + "example.com/multi-mains-project/foo/internal/qiniu" +) + +func Bar2() { + internal.Xiaohong() + qiniu.BB() +} diff --git a/tests/samples/multi_mains_project_with_internal/foo/internal/qiniu/xiaoda.go b/tests/samples/multi_mains_project_with_internal/foo/internal/qiniu/xiaoda.go new file mode 100644 index 0000000..482e5bc --- /dev/null +++ b/tests/samples/multi_mains_project_with_internal/foo/internal/qiniu/xiaoda.go @@ -0,0 +1,5 @@ +package qiniu + +func BB() { + +} diff --git a/tests/samples/multi_mains_project_with_internal/foo/internal/xiaoming.go b/tests/samples/multi_mains_project_with_internal/foo/internal/xiaoming.go new file mode 100644 index 0000000..bf0d355 --- /dev/null +++ b/tests/samples/multi_mains_project_with_internal/foo/internal/xiaoming.go @@ -0,0 +1,5 @@ +package internal + +func Xiaohong() { + +} diff --git a/tests/samples/multi_mains_project_with_internal/go.mod b/tests/samples/multi_mains_project_with_internal/go.mod new file mode 100644 index 0000000..fa221ed --- /dev/null +++ b/tests/samples/multi_mains_project_with_internal/go.mod @@ -0,0 +1,3 @@ +module example.com/multi-mains-project + +go 1.11 diff --git a/tests/samples/multi_mains_project_with_internal/internal/foo.go b/tests/samples/multi_mains_project_with_internal/internal/foo.go new file mode 100644 index 0000000..90912fc --- /dev/null +++ b/tests/samples/multi_mains_project_with_internal/internal/foo.go @@ -0,0 +1,7 @@ +package internal + +import "fmt" + +func Hello() { + fmt.Println("hello, world.") +} diff --git a/tests/samples/multi_mains_project_with_internal/main.go b/tests/samples/multi_mains_project_with_internal/main.go new file mode 100644 index 0000000..bb80c64 --- /dev/null +++ b/tests/samples/multi_mains_project_with_internal/main.go @@ -0,0 +1,12 @@ +package main + +import ( + "example.com/multi-mains-project/foo" + "example.com/multi-mains-project/internal" +) + +func main() { + foo.Bar1() + foo.Bar2() + internal.Hello() +} diff --git a/tests/samples/simple_gopath_project/src/qiniu.com/simple_gopath_project/main.go b/tests/samples/simple_gopath_project/src/qiniu.com/simple_gopath_project/main.go index 01a2bca..99c0515 100644 --- a/tests/samples/simple_gopath_project/src/qiniu.com/simple_gopath_project/main.go +++ b/tests/samples/simple_gopath_project/src/qiniu.com/simple_gopath_project/main.go @@ -2,8 +2,10 @@ package main import ( "fmt" + "qiniu.com/simple_gopath_project/modulea" ) func main() { + modulea.Bar() fmt.Println("hello, world.") } diff --git a/tests/samples/simple_gopath_project/src/qiniu.com/simple_gopath_project/modulea/foo.go b/tests/samples/simple_gopath_project/src/qiniu.com/simple_gopath_project/modulea/foo.go new file mode 100644 index 0000000..7b69ca4 --- /dev/null +++ b/tests/samples/simple_gopath_project/src/qiniu.com/simple_gopath_project/modulea/foo.go @@ -0,0 +1,5 @@ +package modulea + +func Bar() { + +} diff --git a/tests/samples/simple_project_with_internal/foo/bar1.go b/tests/samples/simple_project_with_internal/foo/bar1.go new file mode 100644 index 0000000..e0eeaee --- /dev/null +++ b/tests/samples/simple_project_with_internal/foo/bar1.go @@ -0,0 +1,11 @@ +package foo + +import ( + "example.com/simple-project/foo/internal" + "example.com/simple-project/foo/internal/qiniu" +) + +func Bar1() { + internal.Xiaohong() + qiniu.BB() +} diff --git a/tests/samples/simple_project_with_internal/foo/bar2.go b/tests/samples/simple_project_with_internal/foo/bar2.go new file mode 100644 index 0000000..7ec60a1 --- /dev/null +++ b/tests/samples/simple_project_with_internal/foo/bar2.go @@ -0,0 +1,11 @@ +package foo + +import ( + "example.com/simple-project/foo/internal" + "example.com/simple-project/foo/internal/qiniu" +) + +func Bar2() { + internal.Xiaohong() + qiniu.BB() +} diff --git a/tests/samples/simple_project_with_internal/foo/internal/qiniu/xiaoda.go b/tests/samples/simple_project_with_internal/foo/internal/qiniu/xiaoda.go new file mode 100644 index 0000000..482e5bc --- /dev/null +++ b/tests/samples/simple_project_with_internal/foo/internal/qiniu/xiaoda.go @@ -0,0 +1,5 @@ +package qiniu + +func BB() { + +} diff --git a/tests/samples/simple_project_with_internal/foo/internal/xiaoming.go b/tests/samples/simple_project_with_internal/foo/internal/xiaoming.go new file mode 100644 index 0000000..bf0d355 --- /dev/null +++ b/tests/samples/simple_project_with_internal/foo/internal/xiaoming.go @@ -0,0 +1,5 @@ +package internal + +func Xiaohong() { + +} diff --git a/tests/samples/simple_project_with_internal/main.go b/tests/samples/simple_project_with_internal/main.go index 5bd41fa..ba23ed8 100644 --- a/tests/samples/simple_project_with_internal/main.go +++ b/tests/samples/simple_project_with_internal/main.go @@ -1,7 +1,12 @@ package main -import "example.com/simple-project/internal" +import ( + "example.com/simple-project/foo" + "example.com/simple-project/internal" +) func main() { + foo.Bar1() + foo.Bar2() internal.Hello() }