ar0c/goc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

✨ feat: 新增 profile get/delete, agent get/delete 接口

Browse Source
This commit is contained in:
lyyyuna 2021-09-08 10:54:18 +08:00 committed by Li Yiyang
parent 547016fc9b
commit 51a137411b
13 changed files with 553 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
cmd/profile.go
View File

@ -16,6 +16,7 @@ package cmd
import (
"github.com/qiniu/goc/v2/pkg/client"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
var profileCmd = &cobra.Command{
@ -28,20 +29,54 @@ goc profile
# Get coverage counter from specified register center, the result output to specified file.
goc profile --host=http://192.168.1.1:8080 --output=./coverage.cov
`,
Run: profile,
//Run: profile,
}
var (
profileHost string
profileoutput string // --output flag
profileHost string
profileOutput string // --output flag
profileIds []string
profilePackages string
profileExtra string
)
func init() {
profileCmd.Flags().StringVar(&profileHost, "host", "127.0.0.1:7777", "specify the host of the goc server")
profileCmd.Flags().StringVarP(&profileoutput, "output", "o", "", "download cover profile")
add1Flags := func(f *pflag.FlagSet) {
f.StringVar(&profileHost, "host", "127.0.0.1:7777", "specify the host of the goc server")
f.StringSliceVar(&profileIds, "id", nil, "specify the ids of the services")
f.StringVar(&profileExtra, "extra", "", "specify the regex expression of extra, only profile with extra information will be downloaded")
}
add2Flags := func(f *pflag.FlagSet) {
f.StringVarP(&profileOutput, "output", "o", "", "download cover profile")
f.StringVar(&profilePackages, "packages", "", "specify the regex expression of packages, only profile of these packages will be downloaded")
}
add1Flags(getProfileCmd.Flags())
add2Flags(getProfileCmd.Flags())
add1Flags(clearProfileCmd.Flags())
profileCmd.AddCommand(getProfileCmd)
profileCmd.AddCommand(clearProfileCmd)
rootCmd.AddCommand(profileCmd)
}
func profile(cmd *cobra.Command, args []string) {
client.NewWorker("http://" + profileHost).Profile(profileoutput)
var getProfileCmd = &cobra.Command{
Use: "get",
Run: getProfile,
}
func getProfile(cmd *cobra.Command, args []string) {
client.GetProfile(profileHost, profileIds, profilePackages, profileExtra, profileOutput)
}
var clearProfileCmd = &cobra.Command{
Use: "clear",
Run: clearProfile,
}
func clearProfile(cmd *cobra.Command, args []string) {
client.ClearProfile(profileHost, profileIds, profileExtra)
}

36
cmd/list.go → cmd/service.go
View File

@ -19,27 +19,45 @@ import (
)
var listCmd = &cobra.Command{
Use: "list",
Use: "service",
Short: "Lists all the registered services",
Long: "Lists all the registered services",
Example: `
goc list [flags]
`,
Run: list,
}
var (
listHost string
listWide bool
listIds []string
)
func init() {
listCmd.Flags().StringVar(&listHost, "host", "127.0.0.1:7777", "specify the host of the goc server")
listCmd.Flags().BoolVar(&listWide, "wide", false, "list all services with more information (such as pid)")
listCmd.PersistentFlags().StringVar(&listHost, "host", "127.0.0.1:7777", "specify the host of the goc server")
listCmd.PersistentFlags().BoolVar(&listWide, "wide", false, "list all services with more information (such as pid)")
listCmd.PersistentFlags().StringSliceVar(&listIds, "id", nil, "specify the ids of the services")
listCmd.AddCommand(getServiceCmd)
listCmd.AddCommand(deleteServiceCmd)
rootCmd.AddCommand(listCmd)
}
func list(cmd *cobra.Command, args []string) {
client.NewWorker("http://" + listHost).ListAgents(listWide)
client.ListAgents(listHost, listIds, listWide)
}
var getServiceCmd = &cobra.Command{
Use: "get",
Run: getAgents,
}
func getAgents(cmd *cobra.Command, args []string) {
client.ListAgents(listHost, listIds, listWide)
}
var deleteServiceCmd = &cobra.Command{
Use: "delete",
Run: deleteAgents,
}
func deleteAgents(cmd *cobra.Command, args []string) {
client.DeleteAgents(listHost, listIds)
}

1
go.mod
View File

@ -4,6 +4,7 @@ go 1.16
require (
github.com/gin-gonic/gin v1.7.2
github.com/go-resty/resty/v2 v2.6.0
github.com/gofrs/flock v0.8.1
github.com/gorilla/websocket v1.4.2
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213

2
go.sum
View File

@ -472,6 +472,8 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/go-resty/resty/v2 v2.6.0 h1:joIR5PNLM2EFqqESUjCMGXrWmXNHEU9CEiK813oKYS4=
github.com/go-resty/resty/v2 v2.6.0/go.mod h1:PwvJS6hvaPkjtjNg9ph+VrSD92bi5Zq73w/BIH7cC3Q=
github.com/go-sql-driver/mysql v0.0.0-20160411075031-7ebe0a500653/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=

82
pkg/client/agent.go Normal file
View File

@ -0,0 +1,82 @@
/*
Copyright 2021 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package client
import (
"os"
"github.com/olekukonko/tablewriter"
"github.com/qiniu/goc/v2/pkg/client/rest"
"github.com/qiniu/goc/v2/pkg/log"
)
const (
DISCONNECT = 1 << iota
RPCCONNECT = 1 << iota
WATCHCONNECT = 1 << iota
)
func ListAgents(host string, ids []string, wide bool) {
gocClient := rest.NewV2Client(host)
agents, err := gocClient.Agent().Get(ids)
if err != nil {
log.Fatalf("cannot get agent list from goc server: %v", err)
}
table := tablewriter.NewWriter(os.Stdout)
table.SetCenterSeparator("")
table.SetColumnSeparator("")
table.SetRowSeparator("")
table.SetHeaderLine(false)
table.SetBorder(false)
table.SetTablePadding(" ") // pad with 3 blank spaces
table.SetNoWhiteSpace(true)
table.SetReflowDuringAutoWrap(false)
table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
table.SetAutoWrapText(false)
if wide {
table.SetHeader([]string{"ID", "STATUS", "REMOTEIP", "HOSTNAME", "PID", "CMD", "EXTRA"})
table.SetColumnAlignment([]int{tablewriter.ALIGN_LEFT, tablewriter.ALIGN_LEFT, tablewriter.ALIGN_LEFT, tablewriter.ALIGN_LEFT, tablewriter.ALIGN_LEFT, tablewriter.ALIGN_LEFT, tablewriter.ALIGN_LEFT, tablewriter.ALIGN_LEFT})
} else {
table.SetHeader([]string{"ID", "STATUS", "REMOTEIP", "CMD"})
table.SetColumnAlignment([]int{tablewriter.ALIGN_LEFT, tablewriter.ALIGN_LEFT, tablewriter.ALIGN_LEFT, tablewriter.ALIGN_LEFT})
}
for _, agent := range agents {
var status string
if agent.Status == DISCONNECT {
status = "DISCONNECT"
} else if agent.Status&(RPCCONNECT|WATCHCONNECT) > 0 {
status = "CONNECT"
}
if wide {
table.Append([]string{agent.Id, status, agent.RemoteIP, agent.Hostname, agent.Pid, agent.CmdLine, agent.Extra})
} else {
preLen := len(agent.Id) + len(agent.RemoteIP) + 9
table.Append([]string{agent.Id, status, agent.RemoteIP, getSimpleCmdLine(preLen, agent.CmdLine)})
}
}
table.Render()
}
func DeleteAgents(host string, ids []string) {
gocClient := rest.NewV2Client(host)
err := gocClient.Agent().Delete(ids)
if err != nil {
log.Fatalf("cannot delete agents from goc server: %v", err)
}
}

73
pkg/client/profie.go Normal file
View File

@ -0,0 +1,73 @@
/*
Copyright 2021 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package client
import (
"bytes"
"fmt"
"io"
"os"
"path/filepath"
"github.com/qiniu/goc/v2/pkg/client/rest"
"github.com/qiniu/goc/v2/pkg/client/rest/profile"
"github.com/qiniu/goc/v2/pkg/log"
)
func GetProfile(host string, ids []string, packages string, extra string, output string) {
gocClient := rest.NewV2Client(host)
profiles, err := gocClient.Profile().Get(ids,
profile.WithPackagePattern(packages),
profile.WithExtraPattern(extra))
if err != nil {
log.Fatalf("fail to get profile from the goc server: %v, response: %v", err, profiles)
}
if output == "" {
fmt.Fprint(os.Stdout, profiles)
} else {
var dir, filename string = filepath.Split(output)
if dir != "" {
err = os.MkdirAll(dir, os.ModePerm)
if err != nil {
log.Fatalf("failed to create directory %s, err:%v", dir, err)
}
}
if filename == "" {
output += "coverage.cov"
}
f, err := os.Create(output)
if err != nil {
log.Fatalf("failed to create file %s, err:%v", output, err)
}
defer f.Close()
_, err = io.Copy(f, bytes.NewReader([]byte(profiles)))
if err != nil {
log.Fatalf("failed to write file: %v, err: %v", output, err)
}
}
}
func ClearProfile(host string, ids []string, extra string) {
gocClient := rest.NewV2Client(host)
err := gocClient.Profile().Delete(ids,
profile.WithExtraPattern(extra))
if err != nil {
log.Fatalf("fail to clear the profile: %v", err)
}
}

95
pkg/client/rest/agent/agent.go Normal file
View File

@ -0,0 +1,95 @@
/*
Copyright 2021 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package agent
import (
"encoding/json"
"strings"
"github.com/go-resty/resty/v2"
)
type Agent struct {
Id string `json:"id"`
RemoteIP string `json:"rpc_remoteip"`
Hostname string `json:"hostname"`
CmdLine string `json:"cmdline"`
Pid string `json:"pid"`
Status int `json:"status"`
Extra string `json:"extra"`
}
const (
agentsAPI = "/v2/agents"
)
type AgentInterface interface {
Get(ids []string) ([]Agent, error)
Delete(ids []string) error
}
type agentsClient struct {
c *resty.Client
}
func NewAgentsClient(c *resty.Client) *agentsClient {
return &agentsClient{
c: c,
}
}
type agentOption func(*agentsClient)
func (a *agentsClient) Get(ids []string) ([]Agent, error) {
req := a.c.R()
idQuery := strings.Join(ids, ",")
req.QueryParam.Add("id", idQuery)
res := struct {
Items []Agent `json:"items"`
}{}
resp, err := req.
Get(agentsAPI)
if err != nil {
return nil, err
}
err = json.Unmarshal(resp.Body(), &res)
if err != nil {
return nil, err
}
return res.Items, nil
}
func (a *agentsClient) Delete(ids []string) error {
req := a.c.R()
idQuery := strings.Join(ids, ",")
req.QueryParam.Add("id", idQuery)
_, err := req.
Delete(agentsAPI)
if err != nil {
return err
}
return nil
}

39
pkg/client/rest/client.go Normal file
View File

@ -0,0 +1,39 @@
/*
Copyright 2021 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package rest
import (
"github.com/go-resty/resty/v2"
"github.com/qiniu/goc/v2/pkg/client/rest/agent"
"github.com/qiniu/goc/v2/pkg/client/rest/profile"
)
// V2Client provides methods contact with the covered agent under test
type V2Client struct {
rest *resty.Client
}
func NewV2Client(host string) *V2Client {
return &V2Client{
rest: resty.New().SetHostURL("http://" + host),
}
}
func (c *V2Client) Agent() agent.AgentInterface {
return agent.NewAgentsClient(c.rest)
}
func (c *V2Client) Profile() profile.ProfileInterface {
return profile.NewProfileClient(c.rest)
}

120
pkg/client/rest/profile/profile.go Normal file
View File

@ -0,0 +1,120 @@
/*
Copyright 2021 Qiniu Cloud (qiniu.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package profile
import (
"encoding/json"
"fmt"
"net/http"
"strings"
"github.com/go-resty/resty/v2"
)
type Profile struct {
Profile string `json:"profile"`
}
const (
profileAPI = "/v2/cover/profile"
)
type ProfileInterface interface {
Get(ids []string, opts ...profileOption) (string, error)
Delete(ids []string, opts ...profileOption) error
}
type profileClient struct {
c *resty.Client
packagePattern string
extraPattern string
}
func NewProfileClient(c *resty.Client) *profileClient {
return &profileClient{
c: c,
}
}
type profileOption func(*profileClient)
func WithPackagePattern(pattern string) profileOption {
return func(pc *profileClient) {
pc.packagePattern = pattern
}
}
func WithExtraPattern(pattern string) profileOption {
return func(pc *profileClient) {
pc.extraPattern = pattern
}
}
func (p *profileClient) Get(ids []string, opts ...profileOption) (string, error) {
for _, opt := range opts {
opt(p)
}
req := p.c.R()
idQuery := strings.Join(ids, ",")
req.QueryParam.Add("id", idQuery)
req.QueryParam.Add("pattern", p.packagePattern)
req.QueryParam.Add("extra", p.extraPattern)
res := struct {
Data string `json:"profile,omitempty"`
Msg string `jaon:"msg,omitempty"`
}{}
resp, err := req.
Get(profileAPI)
if err != nil {
return "", err
}
err = json.Unmarshal(resp.Body(), &res)
if err != nil {
return "", err
}
if resp.StatusCode() != http.StatusOK {
return res.Msg, fmt.Errorf("status code not 200")
}
return res.Data, nil
}
func (p *profileClient) Delete(ids []string, opts ...profileOption) error {
for _, opt := range opts {
opt(p)
}
req := p.c.R()
idQuery := strings.Join(ids, ",")
req.QueryParam.Add("id", idQuery)
req.QueryParam.Add("pattern", p.packagePattern)
req.QueryParam.Add("extra", p.extraPattern)
_, err := req.
Delete(profileAPI)
if err != nil {
return err
}
return nil
}

65
pkg/server/api.go
View File

@ -16,6 +16,7 @@ package server
import (
"bytes"
"net/http"
"strings"
"sync"
"time"
@ -25,11 +26,46 @@ import (
"k8s.io/test-infra/gopherage/pkg/cov"
)
func idMaps(idQuery string) func(key string) bool {
idMap := make(map[string]bool)
if strings.Contains(idQuery, ",") == false {
} else {
ids := strings.Split(idQuery, ",")
for _, id := range ids {
idMap[id] = true
}
}
inIdMaps := func(key string) bool {
// if no id in query, then all id agent will be return
if len(idMap) == 0 {
return true
}
// other
_, ok := idMap[key]
if !ok {
return false
} else {
return true
}
}
return inIdMaps
}
// listAgents return all service informations
func (gs *gocServer) listAgents(c *gin.Context) {
idQuery := c.Query("id")
ifInIdMap := idMaps(idQuery)
agents := make([]*gocCoveredAgent, 0)
gs.agents.Range(func(key, value interface{}) bool {
// check if id is in the query ids
if !ifInIdMap(key.(string)) {
return true
}
agent, ok := value.(*gocCoveredAgent)
if !ok {
return false
@ -53,6 +89,7 @@ func (gs *gocServer) getProfiles(c *gin.Context) {
mergedProfiles := make([][]*cover.Profile, 0)
gs.agents.Range(func(key, value interface{}) bool {
agent, ok := value.(*gocCoveredAgent)
if !ok {
return false
@ -69,6 +106,12 @@ func (gs *gocServer) getProfiles(c *gin.Context) {
var req ProfileReq = "getprofile"
var res ProfileRes
go func() {
// lock-free
rpc := agent.rpc
if rpc == nil || agent.Status == DISCONNECT {
done <- nil
return
}
err := agent.rpc.Call("GocAgent.GetProfile", req, &res)
if err != nil {
log.Errorf("fail to get profile from: %v, reasson: %v. let's close the connection", agent.Id, err)
@ -134,8 +177,10 @@ func (gs *gocServer) getProfiles(c *gin.Context) {
//
// it is async, the function will return immediately
func (gs *gocServer) resetProfiles(c *gin.Context) {
gs.agents.Range(func(key, value interface{}) bool {
agent, ok := value.(gocCoveredAgent)
agent, ok := value.(*gocCoveredAgent)
if !ok {
return false
}
@ -143,7 +188,12 @@ func (gs *gocServer) resetProfiles(c *gin.Context) {
var req ProfileReq = "resetprofile"
var res ProfileRes
go func() {
err := agent.rpc.Call("GocAgent.ResetProfile", req, &res)
// lock-free
rpc := agent.rpc
if rpc == nil || agent.Status == DISCONNECT {
return
}
err := rpc.Call("GocAgent.ResetProfile", req, &res)
if err != nil {
log.Errorf("fail to reset profile from: %v, reasson: %v. let's close the connection", agent.Id, err)
// 关闭链接
@ -207,7 +257,7 @@ func (gs *gocServer) watchProfileUpdate(c *gin.Context) {
}
func (gs *gocServer) removeAgentById(c *gin.Context) {
id := c.Param("id")
id := c.Query("id")
rawagent, ok := gs.agents.Load(id)
if !ok {
@ -234,7 +284,16 @@ func (gs *gocServer) removeAgentById(c *gin.Context) {
}
func (gs *gocServer) removeAgents(c *gin.Context) {
idQuery := c.Query("id")
ifInIdMap := idMaps(idQuery)
gs.agents.Range(func(key, value interface{}) bool {
// check if id is in the query ids
if !ifInIdMap(key.(string)) {
return true
}
agent, ok := value.(*gocCoveredAgent)
if !ok {
return false

2
pkg/server/rpcstream.go
View File

@ -92,6 +92,8 @@ func (gs *gocServer) serveRpcStream(c *gin.Context) {
ws.Close()
log.Infof("close rpc connection, %v", agent.Hostname)
// reset rpc client
agent.rpc = nil
}()
// set pong handler

1
pkg/server/server.go
View File

@ -123,7 +123,6 @@ func RunGocServerUntilExit(host string, path string) {
v2.GET("/cover/profile", gs.getProfiles)
v2.DELETE("/cover/profile", gs.resetProfiles)
v2.GET("/agents", gs.listAgents)
v2.DELETE("/agents/:id", gs.removeAgentById)
v2.DELETE("/agents", gs.removeAgents)
v2.GET("/cover/ws/watch", gs.watchProfileUpdate)

16
tests/e2e/e2e_test.go
View File

@ -64,19 +64,19 @@ var _ = Describe("1 [基础测试]", func() {
basicC.Run()
defer basicC.Stop()
By("使用 goc list 获取服务列表")
output, err = RunShortRunCmd([]string{"goc", "list"}, dir, nil)
Expect(err).To(BeNil(), "goc list 运行错误")
Expect(output).To(ContainSubstring("127.0.0.1 ./basic2"), "goc list 输出应该包含 basic 服务")
By("使用 goc service get 获取服务列表")
output, err = RunShortRunCmd([]string{"goc", "service", "get"}, dir, nil)
Expect(err).To(BeNil(), "goc servive get 运行错误")
Expect(output).To(ContainSubstring("127.0.0.1 ./basic2"), "goc service get 输出应该包含 basic 服务")
By("使用 goc profile 获取覆盖率")
By("使用 goc profile get 获取覆盖率")
profileStr := `mode: count
basic2/main.go:8.13,9.6 1 1
basic2/main.go:9.6,12.3 2 2`
time.Sleep(time.Second)
output, err = RunShortRunCmd([]string{"goc", "profile"}, dir, nil)
Expect(err).To(BeNil(), "goc profile 运行错误")
Expect(output).To(ContainSubstring(profileStr), "goc profile 获取的覆盖率有误")
output, err = RunShortRunCmd([]string{"goc", "profile", "get"}, dir, nil)
Expect(err).To(BeNil(), "goc profile get运行错误")
Expect(output).To(ContainSubstring(profileStr), "goc profile get 获取的覆盖率有误")
})
})
})