Skip to content

Commit

Permalink
#79 record polling history (#94)
Browse files Browse the repository at this point in the history
* record polling history

* fix bugs and use chan get body

* add unit test

* rename by comment

* use context instead of chan

* use const instead string

* modify as comment
  • Loading branch information
GuoYL123 authored Feb 17, 2020
1 parent f10641e commit fc67b31
Show file tree
Hide file tree
Showing 10 changed files with 195 additions and 24 deletions.
42 changes: 42 additions & 0 deletions deployments/db.js
Original file line number Diff line number Diff line change
Expand Up @@ -96,11 +96,53 @@ db.createCollection( "view", {
}
} }
} );

db.createCollection( "polling_detail", {
validator: { $jsonSchema: {
bsonType: "object",
required: [ "id","params","session_id","url_path" ],
properties: {
id: {
bsonType: "string",
},
session_id: {
bsonType: "string",
},
domain: {
bsonType: "string",
},
params: {
bsonType: "string"
},
ip: {
bsonType: "string"
},
user_agent: {
bsonType: "string"
},
url_path: {
bsonType: "string"
},
response_body: {
bsonType: "object"
},
response_header: {
bsonType: "object"
},
response_code: {
bsonType: "string"
}
}
} }
} );

//index
db.kv.createIndex({"id": 1}, { unique: true } );
db.kv.createIndex({key: 1, label_id: 1,domain:1,project:1},{ unique: true });
db.label.createIndex({"id": 1}, { unique: true } );
db.label.createIndex({format: 1,domain:1,project:1},{ unique: true });
db.polling_detail.createIndex({"id": 1}, { unique: true } );
db.polling_detail.createIndex({session:1,domain:1}, { unique: true } );
db.view.createIndex({"id": 1}, { unique: true } );
db.view.createIndex({display:1,domain:1,project:1},{ unique: true });
//db config
Expand Down
1 change: 1 addition & 0 deletions pkg/common/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ const (
MsgInvalidWait = "wait param should be formed with number and time unit like 5s,100ms, and less than 5m"
MsgInvalidRev = "revision param should be formed with number greater than 0"
ErrKvIDMustNotEmpty = "must supply kv id if you want to remove key"
RespBodyContextKey = "responseBody"

MaxWait = 5 * time.Minute
)
24 changes: 24 additions & 0 deletions pkg/iputil/ip_util.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package iputil

import (
"net"
"net/http"
"strings"
)

//ClientIP try to get ip from http header
func ClientIP(r *http.Request) string {
xForwardedFor := r.Header.Get("X-Forwarded-For")
ip := strings.TrimSpace(strings.Split(xForwardedFor, ",")[0])
if ip != "" {
return ip
}
ip = strings.TrimSpace(r.Header.Get("X-Real-Ip"))
if ip != "" {
return ip
}
if ip, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr)); err == nil {
return ip
}
return ""
}
14 changes: 14 additions & 0 deletions pkg/model/kv.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,3 +52,17 @@ type ViewResponse struct {
Total int `json:"total,omitempty"`
Data []*ViewDoc `json:"data,omitempty"`
}

//PollingDetail record operation history
type PollingDetail struct {
ID string `json:"id,omitempty" yaml:"id,omitempty"`
SessionID string `json:"session_id,omitempty" yaml:"session_id,omitempty"`
Domain string `json:"domain,omitempty" yaml:"domain,omitempty"`
PollingData map[string]interface{} `json:"params,omitempty" yaml:"params,omitempty"`
IP string `json:"ip,omitempty" yaml:"ip,omitempty"`
UserAgent string `json:"user_agent,omitempty" yaml:"user_agent,omitempty"`
URLPath string `json:"url_path,omitempty" yaml:"url_path,omitempty"`
ResponseBody interface{} `json:"response_body,omitempty" yaml:"response_body,omitempty"`
ResponseHeader map[string][]string `json:"response_header,omitempty" yaml:"response_header,omitempty"`
ResponseCode int `json:"response_code,omitempty" yaml:"response_code,omitempty"`
}
1 change: 1 addition & 0 deletions server/resource/v1/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,7 @@ func queryAndResponse(rctx *restful.Context,
}
rctx.ReadResponseWriter().Header().Set(common.HeaderRevision, strconv.FormatInt(rev, 10))
err = writeResponse(rctx, kv)
rctx.Ctx = context.WithValue(rctx.Ctx, common.RespBodyContextKey, kv)
if err != nil {
openlogging.Error(err.Error())
}
Expand Down
43 changes: 40 additions & 3 deletions server/resource/v1/kv_resource.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,15 @@ package v1
import (
"fmt"
"github.com/apache/servicecomb-kie/pkg/common"
"github.com/apache/servicecomb-kie/pkg/iputil"
"github.com/apache/servicecomb-kie/pkg/model"
"github.com/apache/servicecomb-kie/server/pubsub"
"github.com/apache/servicecomb-kie/server/service"
"github.com/apache/servicecomb-kie/server/service/mongo/record"
goRestful "github.com/emicklei/go-restful"
"github.com/go-chassis/go-chassis/server/restful"
"github.com/go-mesh/openlogging"
uuid "github.com/satori/go.uuid"
"net/http"
)

Expand Down Expand Up @@ -108,13 +111,14 @@ func (r *KVResource) GetByKey(rctx *restful.Context) {
WriteErrResponse(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText)
return
}
insID := rctx.ReadHeader("sessionID")
statusStr := rctx.ReadQueryParameter("status")
status, err := checkStatus(statusStr)
if err != nil {
WriteErrResponse(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText)
return
}
returnData(rctx, domain, project, labels, pageNum, pageSize, status)
returnData(rctx, domain, project, labels, pageNum, pageSize, status, insID)
}

//List response kv list
Expand All @@ -138,18 +142,22 @@ func (r *KVResource) List(rctx *restful.Context) {
WriteErrResponse(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText)
return
}
sessionID := rctx.ReadHeader("sessionID")
statusStr := rctx.ReadQueryParameter("status")
status, err := checkStatus(statusStr)
if err != nil {
WriteErrResponse(rctx, http.StatusBadRequest, err.Error(), common.ContentTypeText)
return
}
returnData(rctx, domain, project, labels, pageNum, pageSize, status)
returnData(rctx, domain, project, labels, pageNum, pageSize, status, sessionID)
}

func returnData(rctx *restful.Context, domain interface{}, project string, labels map[string]string, pageNum, pageSize int64, status string) {
func returnData(rctx *restful.Context, domain interface{}, project string, labels map[string]string, pageNum, pageSize int64, status, sessionID string) {
revStr := rctx.ReadQueryParameter(common.QueryParamRev)
wait := rctx.ReadQueryParameter(common.QueryParamWait)
if sessionID != "" {
defer RecordPollingDetail(rctx, revStr, wait, domain.(string), project, labels, pageNum, pageSize, sessionID)
}
if revStr == "" {
if wait == "" {
queryAndResponse(rctx, domain, project, "", labels, pageNum, pageSize, status)
Expand Down Expand Up @@ -206,6 +214,35 @@ func returnData(rctx *restful.Context, domain interface{}, project string, label
}
}

//RecordPollingDetail to record data after get or list
func RecordPollingDetail(context *restful.Context, revStr, wait, domain, project string, labels map[string]string, limit, offset int64, sessionID string) {
data := &model.PollingDetail{}
data.ID = uuid.NewV4().String()
data.SessionID = sessionID
data.Domain = domain
data.IP = iputil.ClientIP(context.Req.Request)
dataMap := map[string]interface{}{
"revStr": revStr,
"wait": wait,
"domain": domain,
"project": project,
"labels": labels,
"limit": limit,
"offset": offset,
}
data.PollingData = dataMap
data.UserAgent = context.Req.HeaderParameter("User-Agent")
data.URLPath = context.ReadRequest().Method + " " + context.ReadRequest().URL.Path
data.ResponseHeader = context.Resp.Header()
data.ResponseCode = context.Resp.StatusCode()
data.ResponseBody = context.Ctx.Value(common.RespBodyContextKey)
_, err := record.CreateOrUpdate(context.Ctx, data)
if err != nil {
openlogging.Warn("record polling detail failed" + err.Error())
return
}
}

//Search search key only by label
func (r *KVResource) Search(context *restful.Context) {
var err error
Expand Down
2 changes: 2 additions & 0 deletions server/resource/v1/kv_resource_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ func TestKVResource_Put(t *testing.T) {
noopH := &handler2.NoopAuthHandler{}
chain, _ := handler.CreateChain(common.Provider, "testchain1", noopH.Name())
r.Header.Set("Content-Type", "application/json")
r.Header.Set("sessionID", "test")
kvr := &v1.KVResource{}
c, _ := restfultest.New(kvr, chain)
resp := httptest.NewRecorder()
Expand All @@ -101,6 +102,7 @@ func TestKVResource_Put(t *testing.T) {
noopH := &handler2.NoopAuthHandler{}
chain, _ := handler.CreateChain(common.Provider, "testchain1", noopH.Name())
r.Header.Set("Content-Type", "application/json")
r.Header.Set("sessionID", "test")
kvr := &v1.KVResource{}
c, _ := restfultest.New(kvr, chain)
resp := httptest.NewRecorder()
Expand Down
29 changes: 15 additions & 14 deletions server/service/mongo/kv/kv_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"testing"
)

//
func TestService_CreateOrUpdate(t *testing.T) {
var err error
config.Configurations = &config.Config{DB: config.DB{URI: "mongodb://kie:[email protected]:27017/kie"}}
Expand All @@ -43,7 +44,7 @@ func TestService_CreateOrUpdate(t *testing.T) {
"service": "cart",
},
Domain: "default",
Project: "test",
Project: "kv-test",
})
assert.NoError(t, err)
assert.NotEmpty(t, kv.ID)
Expand All @@ -58,9 +59,9 @@ func TestService_CreateOrUpdate(t *testing.T) {
"version": "1.0.0",
},
Domain: "default",
Project: "test",
Project: "kv-test",
})
oid, err := kvsvc.Exist(context.TODO(), "default", "timeout", "test", service.WithLabels(map[string]string{
oid, err := kvsvc.Exist(context.TODO(), "default", "timeout", "kv-test", service.WithLabels(map[string]string{
"app": "mall",
"service": "cart",
"version": "1.0.0",
Expand All @@ -78,10 +79,10 @@ func TestService_CreateOrUpdate(t *testing.T) {
"app": "mall",
},
Domain: "default",
Project: "test",
Project: "kv-test",
})
assert.NoError(t, err)
kvs1, err := kvsvc.FindKV(context.Background(), "default", "test",
kvs1, err := kvsvc.FindKV(context.Background(), "default", "kv-test",
service.WithKey("timeout"),
service.WithLabels(map[string]string{
"app": "mall",
Expand All @@ -95,15 +96,15 @@ func TestService_CreateOrUpdate(t *testing.T) {
"app": "mall",
},
Domain: "default",
Project: "test",
Project: "kv-test",
})
assert.Equal(t, "3s", afterKV.Value)
savedKV, err := kvsvc.Exist(context.Background(), "default", "timeout", "test", service.WithLabels(map[string]string{
savedKV, err := kvsvc.Exist(context.Background(), "default", "timeout", "kv-test", service.WithLabels(map[string]string{
"app": "mall",
}))
assert.NoError(t, err)
assert.Equal(t, afterKV.Value, savedKV.Value)
kvs, err := kvsvc.FindKV(context.Background(), "default", "test",
kvs, err := kvsvc.FindKV(context.Background(), "default", "kv-test",
service.WithKey("timeout"),
service.WithLabels(map[string]string{
"app": "mall",
Expand All @@ -117,7 +118,7 @@ func TestService_CreateOrUpdate(t *testing.T) {
func TestService_FindKV(t *testing.T) {
kvsvc := &kv.Service{}
t.Run("exact find by kv and labels with label app", func(t *testing.T) {
kvs, err := kvsvc.FindKV(context.Background(), "default", "test",
kvs, err := kvsvc.FindKV(context.Background(), "default", "kv-test",
service.WithKey("timeout"),
service.WithLabels(map[string]string{
"app": "mall",
Expand All @@ -127,7 +128,7 @@ func TestService_FindKV(t *testing.T) {
assert.Equal(t, 1, len(kvs))
})
t.Run("greedy find by labels,with labels app ans service ", func(t *testing.T) {
kvs, err := kvsvc.FindKV(context.Background(), "default", "test",
kvs, err := kvsvc.FindKV(context.Background(), "default", "kv-test",
service.WithLabels(map[string]string{
"app": "mall",
"service": "cart",
Expand All @@ -146,20 +147,20 @@ func TestService_Delete(t *testing.T) {
"env": "test",
},
Domain: "default",
Project: "test",
Project: "kv-test",
})
assert.NoError(t, err)

err = kvsvc.Delete(context.TODO(), kv1.ID, "default", "test")
err = kvsvc.Delete(context.TODO(), kv1.ID, "default", "kv-test")
assert.NoError(t, err)

})
t.Run("miss id", func(t *testing.T) {
err := kvsvc.Delete(context.TODO(), "", "default", "test")
err := kvsvc.Delete(context.TODO(), "", "default", "kv-test")
assert.Error(t, err)
})
t.Run("miss domain", func(t *testing.T) {
err := kvsvc.Delete(context.TODO(), "2", "", "test")
err := kvsvc.Delete(context.TODO(), "2", "", "kv-test")
assert.Equal(t, session.ErrMissingDomain, err)
})
}
48 changes: 48 additions & 0 deletions server/service/mongo/record/polling_detail_dao.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package record

import (
"context"
"github.com/apache/servicecomb-kie/pkg/model"
"github.com/apache/servicecomb-kie/server/service/mongo/session"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
)

//CreateOrUpdate create a record or update exist record
func CreateOrUpdate(ctx context.Context, detail *model.PollingDetail) (*model.PollingDetail, error) {
collection := session.GetDB().Collection(session.CollectionPollingDetail)
queryFilter := bson.M{"domain": detail.Domain, "session_id": detail.SessionID}
res := collection.FindOne(ctx, queryFilter)
if res.Err() != nil {
if res.Err() == mongo.ErrNoDocuments {
_, err := collection.InsertOne(ctx, detail)
if err != nil {
return nil, err
}
return detail, nil
}
return nil, res.Err()
}
_, err := collection.UpdateOne(ctx, queryFilter, detail)
if err != nil {
return nil, err
}
return detail, nil
}
15 changes: 8 additions & 7 deletions server/service/mongo/session/session.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,14 @@ import (
const (
DBName = "kie"

CollectionLabel = "label"
CollectionKV = "kv"
CollectionKVRevision = "kv_revision"
CollectionCounter = "counter"
CollectionView = "view"
DefaultTimeout = 5 * time.Second
DefaultValueType = "text"
CollectionLabel = "label"
CollectionKV = "kv"
CollectionKVRevision = "kv_revision"
CollectionPollingDetail = "polling_detail"
CollectionCounter = "counter"
CollectionView = "view"
DefaultTimeout = 5 * time.Second
DefaultValueType = "text"
)

//db errors
Expand Down

0 comments on commit fc67b31

Please sign in to comment.