add coverage server (#4)

* add coverage server

* remove noise comments

.gitignore

@@ -1,2 +1,5 @@
 # Vscode files
 .vscode
+
+# ignore log file
+**/goc.log

README.md

@@ -1,5 +1,5 @@
 # goc
-A Comprehensive Coverage Testing Tool for The Go Programming Language
+A Comprehensive Coverage Testing System for The Go Programming Language
 
 > **Note:**
 >

cmd/goc/app/server.go

@@ -16,14 +16,22 @@
 
 package app
 
-import "github.com/spf13/cobra"
+import (
+	"github.com/qiniu/goc/pkg/cover"
+	"github.com/spf13/cobra"
+)
 
 var serverCmd = &cobra.Command{
 	Use:   "server",
 	Short: "start a server to host all services",
-	Run:   func(cmd *cobra.Command, args []string) {},
+	Run: func(cmd *cobra.Command, args []string) {
+		cover.StartServer(port)
+	},
 }
 
+var port string
+
 func init() {
+	serverCmd.Flags().StringVarP(&port, "port", "", ":7777", "listen port to start a coverage host center")
 	rootCmd.AddCommand(serverCmd)
 }

go.mod

@@ -2,4 +2,9 @@ module github.com/qiniu/goc
 
 go 1.13
 
-require github.com/spf13/cobra v1.0.0
+require (
+	github.com/gin-gonic/gin v1.6.3
+	github.com/spf13/cobra v1.0.0
+	golang.org/x/tools v0.0.0-20200303214625-2b0b585e22fe
+	k8s.io/test-infra v0.0.0-20200511080351-8ac9dbfab055
+)

pkg/cover/server.go

@@ -0,0 +1,181 @@
+/*
+ Copyright 2020 Qiniu Cloud (七牛云)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package cover
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+
+	"github.com/gin-gonic/gin"
+	"golang.org/x/tools/cover"
+	"k8s.io/test-infra/gopherage/pkg/cov"
+)
+
+// LocalStore implements the IPersistence interface
+var LocalStore Store
+
+// Client implements the Action interface
+var Client Action
+
+// LogFile a file to save log.
+const LogFile = "goc.log"
+
+// StartServer starts coverage host center
+func StartServer(port string) {
+	LocalStore = NewStore()
+	Client = NewWorker()
+
+	f, err := os.Create(LogFile)
+	if err != nil {
+		log.Fatalf("failed to create log file %s, err: %v", LogFile, err)
+	}
+
+	gin.DefaultWriter = io.MultiWriter(f, os.Stdout)
+	r := gin.Default()
+
+	// api to show the registerd services
+	r.StaticFile(PersistenceFile, "./"+PersistenceFile)
+
+	v1 := r.Group("/v1")
+	{
+		v1.POST("/cover/register", registerService)
+		v1.GET("/cover/profile", profile)
+		v1.POST("/cover/clear", clear)
+		v1.POST("/cover/init", initSystem)
+	}
+
+	log.Fatal(r.Run(port))
+}
+
+// Service is a entry under being tested
+type Service struct {
+	Name    string `form:"name" json:"name" binding:"required"`
+	Address string `form:"address" json:"address" binding:"required"`
+}
+
+func registerService(c *gin.Context) {
+	var service Service
+	if err := c.ShouldBind(&service); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+
+	u, err := url.Parse(service.Address)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+	host, port, err := net.SplitHostPort(u.Host)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+
+	realIP := c.ClientIP()
+	if host != realIP {
+		log.Printf("the registed host %s of service %s is different with the real one %s, here we choose the real one", service.Name, host, realIP)
+		service.Address = fmt.Sprintf("http://%s:%s", realIP, port)
+	}
+	if err := LocalStore.Add(service); err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{"name": service.Name, "address": service.Address})
+}
+
+func profile(c *gin.Context) {
+	svrsUnderTest := LocalStore.GetAll()
+	var mergedProfiles = make([][]*cover.Profile, len(svrsUnderTest))
+	for _, addrs := range svrsUnderTest {
+		for _, addr := range addrs {
+			pp, err := Client.Profile(addr)
+			if err != nil {
+				c.JSON(http.StatusExpectationFailed, gin.H{"error": err.Error()})
+				return
+			}
+			profile, err := convertProfile(pp)
+			if err != nil {
+				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+				return
+			}
+			mergedProfiles = append(mergedProfiles, profile)
+		}
+	}
+
+	if len(mergedProfiles) == 0 {
+		c.JSON(http.StatusOK, "no profiles")
+		return
+	}
+
+	merged, err := cov.MergeMultipleProfiles(mergedProfiles)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+
+	if err := cov.DumpProfile(merged, c.Writer); err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+}
+
+func clear(c *gin.Context) {
+	if err := LocalStore.Init(); err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+
+	if err := Client.Clear(); err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+	c.JSON(http.StatusOK, "TO BE IMPLEMENTED")
+}
+
+func initSystem(c *gin.Context) {
+	if err := LocalStore.Init(); err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+
+	c.JSON(http.StatusOK, "")
+}
+
+func convertProfile(p []byte) ([]*cover.Profile, error) {
+	// Annoyingly, ParseProfiles only accepts a filename, so we have to write the bytes to disk
+	// so it can read them back.
+	// We could probably also just give it /dev/stdin, but that'll break on Windows.
+	tf, err := ioutil.TempFile("", "")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create temp file, err: %v", err)
+	}
+	defer tf.Close()
+	defer os.Remove(tf.Name())
+	if _, err := io.Copy(tf, bytes.NewReader(p)); err != nil {
+		return nil, fmt.Errorf("failed to copy data to temp file, err: %v", err)
+	}
+
+	return cover.ParseProfiles(tf.Name())
+}

pkg/cover/store.go

@@ -0,0 +1,175 @@
+/*
+ Copyright 2020 Qiniu Cloud (七牛云)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package cover
+
+import (
+	"bufio"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+	"sync"
+)
+
+// Store persistents the registered service information
+type Store interface {
+	// Add adds the given service to store
+	Add(s Service) error
+
+	// Get returns the registered service informations with the given service's name
+	Get(name string) []string
+
+	// Get returns all the registered service informations as a map
+	GetAll() map[string][]string
+
+	// Init cleanup all the registered service informations
+	Init() error
+}
+
+// PersistenceFile is the file to save services address information
+const PersistenceFile = "_svrs_address.txt"
+
+// localStore holds the registered services into memory and persistent to a local file
+type localStore struct {
+	mu             sync.RWMutex
+	servicesMap    map[string][]string
+	persistentFile string
+}
+
+// Add adds the given service to localStore
+func (l *localStore) Add(s Service) error {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	// load to memory
+	if addrs, ok := l.servicesMap[s.Name]; ok {
+		for _, addr := range addrs {
+			if addr == s.Address {
+				log.Printf("service registered already, name: %s, address: %s", s.Name, s.Address)
+				return nil
+			}
+		}
+		addrs = append(addrs, s.Address)
+		l.servicesMap[s.Name] = addrs
+	} else {
+		l.servicesMap[s.Name] = []string{s.Address}
+	}
+
+	// persistent to local sotre
+	return l.appendToFile(s)
+}
+
+// Get returns the registered service informations with the given name
+func (l *localStore) Get(name string) []string {
+	l.mu.RLock()
+	defer l.mu.RUnlock()
+	return l.servicesMap[name]
+}
+
+// Get returns all the registered service informations
+func (l *localStore) GetAll() map[string][]string {
+	l.mu.RLock()
+	defer l.mu.RUnlock()
+	return l.servicesMap
+}
+
+// Init cleanup all the registered service informations
+// and the local persistent file
+func (l *localStore) Init() error {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if err := os.Remove(l.persistentFile); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to delete file %s, err: %v", l.persistentFile, err)
+	}
+
+	l.servicesMap = make(map[string][]string, 0)
+	return nil
+}
+
+// load all registered servcie from file to memory
+func (l *localStore) load() (map[string][]string, error) {
+	var svrsMap = make(map[string][]string, 0)
+
+	f, err := os.Open(l.persistentFile)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return svrsMap, nil
+		}
+		return svrsMap, fmt.Errorf("failed to open file, path: %s, err: %v", l.persistentFile, err)
+	}
+	defer f.Close()
+
+	ns := bufio.NewScanner(f)
+	for ns.Scan() {
+		line := ns.Text()
+		ss := strings.FieldsFunc(line, split)
+
+		// TODO: use regex
+		if len(ss) == 2 {
+			if urls, ok := svrsMap[ss[0]]; ok {
+				urls = append(urls, ss[1])
+				svrsMap[ss[0]] = urls
+			} else {
+				svrsMap[ss[0]] = []string{ss[1]}
+			}
+		}
+	}
+
+	if err := ns.Err(); err != nil {
+		return svrsMap, fmt.Errorf("read file failed, file: %s, err: %v", l.persistentFile, err)
+	}
+
+	return svrsMap, nil
+}
+
+func (l *localStore) appendToFile(s Service) error {
+	f, err := os.OpenFile(l.persistentFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	_, err = f.WriteString(format(s) + "\n")
+	if err != nil {
+		return err
+	}
+
+	f.Sync()
+	return nil
+}
+
+func format(s Service) string {
+	return fmt.Sprintf("%s&%s", s.Name, s.Address)
+}
+
+func split(r rune) bool {
+	return r == '&'
+}
+
+// NewStore creates a store using local file
+func NewStore() Store {
+	l := &localStore{
+		persistentFile: PersistenceFile,
+		servicesMap:    make(map[string][]string, 0),
+	}
+
+	services, err := l.load()
+	if err != nil {
+		log.Fatalf("load failed, file: %s, err: %v", l.persistentFile, err)
+	}
+	l.servicesMap = services
+	return l
+}

pkg/cover/store_test.go

@@ -0,0 +1,64 @@
+/*
+ Copyright 2020 Qiniu Cloud (七牛云)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package cover
+
+import (
+	"testing"
+)
+
+func TestLocalStore(t *testing.T) {
+	localStore := NewStore()
+	var tc1 = Service{
+		Name:    "a",
+		Address: "http://127.0.0.1",
+	}
+	var tc2 = Service{
+		Name:    "b",
+		Address: "http://127.0.0.2",
+	}
+	var tc3 = Service{
+		Name:    "c",
+		Address: "http://127.0.0.3",
+	}
+	var tc4 = Service{
+		Name:    "a",
+		Address: "http://127.0.0.4",
+	}
+	localStore.Add(tc1)
+	localStore.Add(tc2)
+	localStore.Add(tc3)
+	localStore.Add(tc4)
+	addrs := localStore.Get(tc1.Name)
+	if len(addrs) != 2 {
+		t.Error("unexpect result")
+	}
+
+	for _, addr := range addrs {
+		if addr != tc1.Address && addr != tc4.Address {
+			t.Error("get address failed")
+		}
+	}
+
+	if len(localStore.GetAll()) != 3 {
+		t.Error("local store check failed")
+	}
+
+	localStore.Init()
+	if len(localStore.GetAll()) != 0 {
+		t.Error("local store init failed")
+	}
+}

pkg/cover/worker.go

@@ -0,0 +1,95 @@
+/*
+ Copyright 2020 Qiniu Cloud (七牛云)
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package cover
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+)
+
+// Action provides methods to contact with the coverd service under test
+type Action interface {
+	Profile(host string) ([]byte, error)
+	Clear() error
+	InitSystem(host string) ([]byte, error)
+}
+
+// CoverProfileAPI is provided by the covered service to get profiles
+const CoverProfileAPI = "/v1/cover/profile"
+
+// CoverProfileClearAPI is provided by the covered service to clear profiles
+const CoverProfileClearAPI = "/v1/cover/clear"
+
+// CoverInitSystemAPI prepare a new round of testing
+const CoverInitSystemAPI = "/v1/cover/init"
+
+type client struct {
+	client *http.Client
+}
+
+// NewWorker creates a worker to contact with service
+func NewWorker() Action {
+	return &client{
+		client: http.DefaultClient,
+	}
+}
+
+func (w *client) Profile(host string) ([]byte, error) {
+	u := fmt.Sprintf("%s%s", host, CoverProfileAPI)
+	profile, err := w.do("GET", u, nil)
+	if err != nil && isNetworkError(err) {
+		profile, err = w.do("GET", u, nil)
+	}
+
+	return profile, err
+}
+
+func (w *client) Clear() error { return nil }
+
+func (w *client) InitSystem(host string) ([]byte, error) {
+	u := fmt.Sprintf("%s%s", host, CoverInitSystemAPI)
+	return w.do("POST", u, nil)
+}
+
+func (w *client) do(method, url string, body io.Reader) ([]byte, error) {
+	req, err := http.NewRequest(method, url, body)
+	if err != nil {
+		return nil, err
+	}
+	res, err := w.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	responseBody, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return responseBody, nil
+}
+
+func isNetworkError(err error) bool {
+	if err == io.EOF {
+		return true
+	}
+	_, ok := err.(net.Error)
+	return ok
+}