ar0c/goc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge pull request #120 from tongjingran/1082

Browse Source 
Support configuring the persistence file path
This commit is contained in:
qiniu-bot 2020-09-25 21:12:14 +08:00 committed by GitHub
commit e831a002be
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 160 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/register.go
View File

@ -33,7 +33,7 @@ var registerCmd = &cobra.Command{
goc register [flags]
`,
Run: func(cmd *cobra.Command, args []string) {
s := cover.Service{
s := cover.ServiceUnderTest{
Name: name,
Address: address,
}

10
cmd/run.go
View File

@ -50,12 +50,16 @@ goc run . [--buildflags] [--exec] [--arguments]
gocBuild.GoRunArguments = goRunArguments
defer gocBuild.Clean()
// only save services in memory
cover.DefaultStore = cover.NewMemoryStore()
server := cover.NewMemoryBasedServer() // only save services in memory
// start goc server
var l = newLocalListener()
go cover.GocServer(ioutil.Discard).RunListener(l)
go func() {
err = server.Route(ioutil.Discard).RunListener(l)
if err != nil {
log.Fatalf("Start goc server failed: %v", err)
}
}()
gocServer := fmt.Sprintf("http://%s", l.Addr().String())
fmt.Printf("[goc] goc server started: %s \n", gocServer)

10
cmd/server.go
View File

@ -19,6 +19,7 @@ package cmd
import (
"github.com/qiniu/goc/pkg/cover"
"github.com/spf13/cobra"
"log"
)
var serverCmd = &cobra.Command{
@ -36,13 +37,18 @@ goc server --port=:8080
goc server --port=localhost:8080
`,
Run: func(cmd *cobra.Command, args []string) {
cover.Run(port)
server, err := cover.NewFileBasedServer(localPersistence)
if err != nil {
log.Fatalf("New file based server failed, err: %v", err)
}
server.Run(port)
},
}
var port string
var port, localPersistence string
func init() {
serverCmd.Flags().StringVarP(&port, "port", "", ":7777", "listen port to start a coverage host center")
serverCmd.Flags().StringVarP(&localPersistence, "local-persistence", "", "_svrs_address.txt", "the file to save services address information")
rootCmd.AddCommand(serverCmd)
}

2
go.mod
View File

@ -20,6 +20,6 @@ require (
golang.org/x/net v0.0.0-20200625001655-4c5254603344
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
golang.org/x/tools v0.0.0-20200730221956-1ac65761fe2c
k8s.io/kubernetes v1.13.0
k8s.io/kubernetes v1.13.0 // indirect
k8s.io/test-infra v0.0.0-20200511080351-8ac9dbfab055
)

1
go.sum
View File

@ -464,6 +464,7 @@ github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQ
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=

4
pkg/cover/client.go
View File

@ -36,7 +36,7 @@ type Action interface {
Clear(param ProfileParam) ([]byte, error)
InitSystem() ([]byte, error)
ListServices() ([]byte, error)
RegisterService(svr Service) ([]byte, error)
RegisterService(svr ServiceUnderTest) ([]byte, error)
}
const (
@ -69,7 +69,7 @@ func NewWorker(host string) Action {
}
}
func (c *client) RegisterService(srv Service) ([]byte, error) {
func (c *client) RegisterService(srv ServiceUnderTest) ([]byte, error) {
if _, err := url.ParseRequestURI(srv.Address); err != nil {
return nil, err
}

50
pkg/cover/client_test.go
View File

@ -28,7 +28,9 @@ import (
func TestClientAction(t *testing.T) {
// mock goc server
ts := httptest.NewServer(GocServer(os.Stdout))
server, err := NewFileBasedServer("_svrs_address.txt")
assert.NoError(t, err)
ts := httptest.NewServer(server.Route(os.Stdout))
defer ts.Close()
var client = NewWorker(ts.URL)
@ -46,15 +48,15 @@ func TestClientAction(t *testing.T) {
}))
defer profileErrMockSvr.Close()
// regsiter service into goc server
var src Service
// register service into goc server
var src ServiceUnderTest
src.Name = "serviceSuccess"
src.Address = profileSuccessMockSvr.URL
res, err := client.RegisterService(src)
assert.NoError(t, err)
assert.Contains(t, string(res), "success")
// do list and check service
// do list and check server
res, err = client.ListServices()
assert.NoError(t, err)
assert.Contains(t, string(res), src.Address)
@ -63,64 +65,64 @@ func TestClientAction(t *testing.T) {
// get profile from goc server
tcs := []struct {
name string
service Service
service ServiceUnderTest
param ProfileParam
expected string
expectedErr bool
}{
{
name: "both service and address existed",
service: Service{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
name: "both server and address existed",
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{Force: false, Service: []string{"serviceOK"}, Address: []string{profileSuccessMockSvr.URL}},
expectedErr: true,
},
{
name: "valid test with no service flag provide",
service: Service{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
name: "valid test with no server flag provide",
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{},
expected: "mockService/main.go:30.13,48.33 13 1",
},
{
name: "valid test with service flag provide",
service: Service{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
name: "valid test with server flag provide",
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{Service: []string{"serviceOK"}},
expected: "mockService/main.go:30.13,48.33 13 1",
},
{
name: "valid test with address flag provide",
service: Service{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{Address: []string{profileSuccessMockSvr.URL}},
expected: "mockService/main.go:30.13,48.33 13 1",
},
{
name: "invalid test with invalid service flag provide",
service: Service{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
name: "invalid test with invalid server flag provide",
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{Service: []string{"unknown"}},
expected: "service [unknown] not found",
expected: "server [unknown] not found",
expectedErr: true,
},
{
name: "invalid test with invalid profile got by service",
service: Service{Name: "serviceErr", Address: profileErrMockSvr.URL},
name: "invalid test with invalid profile got by server",
service: ServiceUnderTest{Name: "serviceErr", Address: profileErrMockSvr.URL},
expected: "bad mode line: error",
expectedErr: true,
},
{
name: "invalid test with disconnected service",
service: Service{Name: "serviceNotExist", Address: "http://172.0.0.2:7777"},
name: "invalid test with disconnected server",
service: ServiceUnderTest{Name: "serviceNotExist", Address: "http://172.0.0.2:7777"},
expected: "connection refused",
expectedErr: true,
},
{
name: "invalid test with empty profile",
service: Service{Name: "serviceNotExist", Address: "http://172.0.0.2:7777"},
service: ServiceUnderTest{Name: "serviceNotExist", Address: "http://172.0.0.2:7777"},
param: ProfileParam{Force: true},
expectedErr: true,
expected: "no profiles",
},
{
name: "valid test with coverfile flag provide",
service: Service{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
service: ServiceUnderTest{Name: "serviceOK", Address: profileSuccessMockSvr.URL},
param: ProfileParam{CoverFilePatterns: []string{"b.go$"}},
expected: "b/b.go",
},
@ -150,7 +152,7 @@ func TestClientAction(t *testing.T) {
})
}
// init system and check service again
// init system and check server again
_, err = client.InitSystem()
assert.NoError(t, err)
res, err = client.ListServices()
@ -162,7 +164,7 @@ func TestClientRegisterService(t *testing.T) {
c := &client{}
// client register with empty address
testService1 := Service{
testService1 := ServiceUnderTest{
Address: "",
Name: "abc",
}
@ -170,7 +172,7 @@ func TestClientRegisterService(t *testing.T) {
assert.Contains(t, err.Error(), "empty url")
// client register with empty name
testService2 := Service{
testService2 := ServiceUnderTest{
Address: "http://127.0.0.1:444",
Name: "",
}

79
pkg/cover/server.go
View File

@ -33,18 +33,35 @@ import (
"k8s.io/test-infra/gopherage/pkg/cov"
)
// DefaultStore implements the IPersistence interface
var DefaultStore Store
// LogFile a file to save log.
const LogFile = "goc.log"
func init() {
DefaultStore = NewFileStore()
type server struct {
PersistenceFile string
Store Store
}
// NewFileBasedServer new a file based server with persistenceFile
func NewFileBasedServer(persistenceFile string) (*server, error) {
store, err := NewFileStore(persistenceFile)
if err != nil {
return nil, err
}
return &server{
PersistenceFile: persistenceFile,
Store: store,
}, nil
}
// NewMemoryBasedServer new a memory based server without persistenceFile
func NewMemoryBasedServer() *server {
return &server{
Store: NewMemoryStore(),
}
}
// Run starts coverage host center
func Run(port string) {
func (s *server) Run(port string) {
f, err := os.Create(LogFile)
if err != nil {
log.Fatalf("failed to create log file %s, err: %v", LogFile, err)
@ -52,34 +69,34 @@ func Run(port string) {
// both log to stdout and file by default
mw := io.MultiWriter(f, os.Stdout)
r := GocServer(mw)
r := s.Route(mw)
log.Fatal(r.Run(port))
}
// GocServer init goc server engine
func GocServer(w io.Writer) *gin.Engine {
// Router init goc server engine
func (s *server) Route(w io.Writer) *gin.Engine {
if w != nil {
gin.DefaultWriter = w
}
r := gin.Default()
// api to show the registered services
r.StaticFile(PersistenceFile, "./"+PersistenceFile)
r.StaticFile("static", "./"+s.PersistenceFile)
v1 := r.Group("/v1")
{
v1.POST("/cover/register", registerService)
v1.GET("/cover/profile", profile)
v1.POST("/cover/profile", profile)
v1.POST("/cover/clear", clear)
v1.POST("/cover/init", initSystem)
v1.GET("/cover/list", listServices)
v1.POST("/cover/register", s.registerService)
v1.GET("/cover/profile", s.profile)
v1.POST("/cover/profile", s.profile)
v1.POST("/cover/clear", s.clear)
v1.POST("/cover/init", s.initSystem)
v1.GET("/cover/list", s.listServices)
}
return r
}
// Service is a entry under being tested
type Service struct {
// ServiceUnderTest is a entry under being tested
type ServiceUnderTest struct {
Name string `form:"name" json:"name" binding:"required"`
Address string `form:"address" json:"address" binding:"required"`
}
@ -93,13 +110,13 @@ type ProfileParam struct {
}
//listServices list all the registered services
func listServices(c *gin.Context) {
services := DefaultStore.GetAll()
func (s *server) listServices(c *gin.Context) {
services := s.Store.GetAll()
c.JSON(http.StatusOK, services)
}
func registerService(c *gin.Context) {
var service Service
func (s *server) registerService(c *gin.Context) {
var service ServiceUnderTest
if err := c.ShouldBind(&service); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
@ -122,9 +139,9 @@ func registerService(c *gin.Context) {
service.Address = fmt.Sprintf("http://%s:%s", realIP, port)
}
address := DefaultStore.Get(service.Name)
address := s.Store.Get(service.Name)
if !contains(address, service.Address) {
if err := DefaultStore.Add(service); err != nil {
if err := s.Store.Add(service); err != nil && err != ErrServiceAlreadyRegistered {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
@ -137,14 +154,14 @@ func registerService(c *gin.Context) {
// profile API examples:
// POST /v1/cover/profile
// { "force": "true", "service":["a","b"], "address":["c","d"],"coverfile":["e","f"] }
func profile(c *gin.Context) {
func (s *server) profile(c *gin.Context) {
var body ProfileParam
if err := c.ShouldBind(&body); err != nil {
c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
return
}
allInfos := DefaultStore.GetAll()
allInfos := s.Store.GetAll()
filterAddrList, err := filterAddrs(body.Service, body.Address, body.Force, allInfos)
if err != nil {
c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
@ -216,13 +233,13 @@ func filterProfile(coverFile []string, profiles []*cover.Profile) ([]*cover.Prof
return out, nil
}
func clear(c *gin.Context) {
func (s *server) clear(c *gin.Context) {
var body ProfileParam
if err := c.ShouldBind(&body); err != nil {
c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
return
}
svrsUnderTest := DefaultStore.GetAll()
svrsUnderTest := s.Store.GetAll()
filterAddrList, err := filterAddrs(body.Service, body.Address, true, svrsUnderTest)
if err != nil {
c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
@ -239,8 +256,8 @@ func clear(c *gin.Context) {
}
func initSystem(c *gin.Context) {
if err := DefaultStore.Init(); err != nil {
func (s *server) initSystem(c *gin.Context) {
if err := s.Store.Init(); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
@ -313,6 +330,6 @@ func filterAddrs(serviceList, addressList []string, force bool, allInfos map[str
filterAddrList = addressAll
}
// Return all servers when all param is nil
// Return all services when all param is nil
return filterAddrList, nil
}

28
pkg/cover/server_test.go
View File

@ -22,7 +22,7 @@ type MockStore struct {
mock.Mock
}
func (m *MockStore) Add(s Service) error {
func (m *MockStore) Add(s ServiceUnderTest) error {
args := m.Called(s)
return args.Error(0)
}
@ -113,7 +113,9 @@ func TestFilterAddrs(t *testing.T) {
}
func TestRegisterService(t *testing.T) {
router := GocServer(os.Stdout)
server, err := NewFileBasedServer("_svrs_address.txt")
assert.NoError(t, err)
router := server.Route(os.Stdout)
// register with empty service struct
w := httptest.NewRecorder()
@ -147,7 +149,7 @@ func TestRegisterService(t *testing.T) {
assert.Contains(t, w.Body.String(), "missing port in address")
// register with store failure
expectedS := Service{
expectedS := ServiceUnderTest{
Name: "foo",
Address: "http://:64444", // the real IP is empty in unittest, so server will get a empty one
}
@ -155,7 +157,7 @@ func TestRegisterService(t *testing.T) {
testObj.On("Get", "foo").Return([]string{"http://127.0.0.1:66666"})
testObj.On("Add", expectedS).Return(fmt.Errorf("lala error"))
DefaultStore = testObj
server.Store = testObj
w = httptest.NewRecorder()
data.Set("name", expectedS.Name)
@ -169,7 +171,9 @@ func TestRegisterService(t *testing.T) {
}
func TestProfileService(t *testing.T) {
router := GocServer(os.Stdout)
server, err := NewFileBasedServer("_svrs_address.txt")
assert.NoError(t, err)
router := server.Route(os.Stdout)
// get profile with invalid force parameter
w := httptest.NewRecorder()
@ -184,9 +188,10 @@ func TestClearService(t *testing.T) {
testObj := new(MockStore)
testObj.On("GetAll").Return(map[string][]string{"foo": {"http://127.0.0.1:66666"}})
DefaultStore = testObj
router := GocServer(os.Stdout)
server := &server{
Store: testObj,
}
router := server.Route(os.Stdout)
// clear profile with non-exist port
w := httptest.NewRecorder()
@ -224,9 +229,10 @@ func TestInitService(t *testing.T) {
testObj := new(MockStore)
testObj.On("Init").Return(fmt.Errorf("lala error"))
DefaultStore = testObj
router := GocServer(os.Stdout)
server := &server{
Store: testObj,
}
router := server.Route(os.Stdout)
// get profile with invalid force parameter
w := httptest.NewRecorder()

40
pkg/cover/store.go
View File

@ -18,18 +18,21 @@ package cover
import (
"bufio"
"errors"
"fmt"
log "github.com/sirupsen/logrus"
"os"
"path/filepath"
"strings"
"sync"
log "github.com/sirupsen/logrus"
)
var ErrServiceAlreadyRegistered = errors.New("service already registered")
// Store persistents the registered service information
type Store interface {
// Add adds the given service to store
Add(s Service) error
Add(s ServiceUnderTest) error
// Get returns the registered service information with the given service's name
Get(name string) []string
@ -44,9 +47,6 @@ type Store interface {
Set(services map[string][]string)
}
// PersistenceFile is the file to save services address information
const PersistenceFile = "_svrs_address.txt"
// fileStore holds the registered services into memory and persistent to a local file
type fileStore struct {
mu sync.RWMutex
@ -56,9 +56,17 @@ type fileStore struct {
}
// NewFileStore creates a store using local file
func NewFileStore() Store {
func NewFileStore(persistenceFile string) (store Store, err error) {
path, err := filepath.Abs(persistenceFile)
if err != nil {
return nil, err
}
err = os.MkdirAll(filepath.Dir(path), os.ModePerm)
if err != nil {
return nil, err
}
l := &fileStore{
persistentFile: PersistenceFile,
persistentFile: path,
memoryStore: NewMemoryStore(),
}
@ -66,12 +74,14 @@ func NewFileStore() Store {
log.Fatalf("load failed, file: %s, err: %v", l.persistentFile, err)
}
return l
return l, nil
}
// Add adds the given service to file Store
func (l *fileStore) Add(s Service) error {
l.memoryStore.Add(s)
func (l *fileStore) Add(s ServiceUnderTest) error {
if err := l.memoryStore.Add(s); err != nil {
return err
}
// persistent to local store
l.mu.Lock()
@ -147,7 +157,7 @@ func (l *fileStore) Set(services map[string][]string) {
panic("TO BE IMPLEMENTED")
}
func (l *fileStore) appendToFile(s Service) error {
func (l *fileStore) appendToFile(s ServiceUnderTest) error {
f, err := os.OpenFile(l.persistentFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return err
@ -163,7 +173,7 @@ func (l *fileStore) appendToFile(s Service) error {
return nil
}
func format(s Service) string {
func format(s ServiceUnderTest) string {
return fmt.Sprintf("%s&%s", s.Name, s.Address)
}
@ -185,7 +195,7 @@ func NewMemoryStore() Store {
}
// Add adds the given service to MemoryStore
func (l *memoryStore) Add(s Service) error {
func (l *memoryStore) Add(s ServiceUnderTest) error {
l.mu.Lock()
defer l.mu.Unlock()
// load to memory
@ -193,7 +203,7 @@ func (l *memoryStore) Add(s Service) error {
for _, addr := range addrs {
if addr == s.Address {
log.Printf("service registered already, name: %s, address: %s", s.Name, s.Address)
return nil
return ErrServiceAlreadyRegistered
}
}
addrs = append(addrs, s.Address)

25
pkg/cover/store_test.go
View File

@ -17,31 +17,34 @@
package cover
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestLocalStore(t *testing.T) {
localStore := NewFileStore()
var tc1 = Service{
localStore, err := NewFileStore("_svrs_address.txt")
assert.NoError(t, err)
var tc1 = ServiceUnderTest{
Name: "a",
Address: "http://127.0.0.1",
}
var tc2 = Service{
var tc2 = ServiceUnderTest{
Name: "b",
Address: "http://127.0.0.2",
}
var tc3 = Service{
var tc3 = ServiceUnderTest{
Name: "c",
Address: "http://127.0.0.3",
}
var tc4 = Service{
var tc4 = ServiceUnderTest{
Name: "a",
Address: "http://127.0.0.4",
}
localStore.Add(tc1)
localStore.Add(tc2)
localStore.Add(tc3)
localStore.Add(tc4)
assert.NoError(t, localStore.Add(tc1))
assert.Equal(t, localStore.Add(tc1), ErrServiceAlreadyRegistered)
assert.NoError(t, localStore.Add(tc2))
assert.NoError(t, localStore.Add(tc3))
assert.NoError(t, localStore.Add(tc4))
addrs := localStore.Get(tc1.Name)
if len(addrs) != 2 {
t.Error("unexpected result")
@ -57,6 +60,10 @@ func TestLocalStore(t *testing.T) {
t.Error("local store check failed")
}
localStoreNew, err := NewFileStore("_svrs_address.txt")
assert.NoError(t, err)
assert.Equal(t, localStore.GetAll(), localStoreNew.GetAll())
localStore.Init()
if len(localStore.GetAll()) != 0 {
t.Error("local store init failed")

9
tests/server.bats
View File

@ -22,7 +22,7 @@ setup_file() {
sleep 2
goc init
# run covered goc
gocc server --port=:60001 --debug 3>&- &
gocc server --port=:60001 --local-persistence="persistence/servicesAll.txt" --debug 3>&- &
GOCC_PID=$!
sleep 2
info "goc gocc server started"
@ -57,4 +57,11 @@ teardown_file() {
# connect to covered goc
run goc profile --center=http://127.0.0.1:60001
[ "$status" -eq 0 ]
# verify the persistence file exist
[ -f "$WORKDIR/persistence/servicesAll.txt" ]
# remove goc persistence file
run goc init --center=http://127.0.0.1:60001
[ "$status" -eq 0 ]
[ ! -f "$WORKDIR/persistence/servicesAll.txt" ]
}