diff --git a/deployments/db.js b/deployments/db.js index 42b919fb..99f2c710 100644 --- a/deployments/db.js +++ b/deployments/db.js @@ -83,7 +83,7 @@ db.createCollection( "polling_detail", { max: 100, validator: { $jsonSchema: { bsonType: "object", - required: [ "id","session_id","domain","url_path" ], + required: [ "id","session_id","revision","url_path" ], properties: { id: { bsonType: "string", @@ -91,7 +91,7 @@ db.createCollection( "polling_detail", { domain: { bsonType: "string", }, - params: { + polling_data: { bsonType: "object" }, ip: { @@ -103,8 +103,8 @@ db.createCollection( "polling_detail", { response_body: { bsonType: "object" }, - response_header: { - bsonType: "object" + revision: { + bsonType: "string" }, response_code: { bsonType: "number" @@ -114,15 +114,14 @@ db.createCollection( "polling_detail", { } ); //index -db.kv.createIndex({"id": 1}, { unique: true } ); -db.kv.createIndex({key: 1, label_format: 1,domain:1,project:1},{ unique: true }); -db.kv_revision.createIndex( { "delete_time": 1 }, { expireAfterSeconds: 7 * 24 * 3600 } ); -db.label.createIndex({"id": 1}, { unique: true } ); -db.label.createIndex({format: 1,domain:1,project:1},{ unique: true }); -db.polling_detail.createIndex({"id": 1}, { unique: true } ); -db.polling_detail.createIndex({session_id:1,domain:1}, { unique: true } ); -db.counter.createIndex({name: 1,domain:1},{ unique: true }); -db.view.createIndex({"id": 1}, { unique: true } ); -db.view.createIndex({display:1,domain:1,project:1},{ unique: true }); +db.kv.createIndex({"id": 1}, {unique: true}); +db.kv.createIndex({key: 1, label_format: 1, domain: 1, project: 1}, {unique: true}); +db.kv_revision.createIndex({"delete_time": 1}, {expireAfterSeconds: 7 * 24 * 3600}); +db.label.createIndex({"id": 1}, {unique: true}); +db.label.createIndex({format: 1, domain: 1, project: 1}, {unique: true}); +db.polling_detail.createIndex({timestamp: 1}, {expireAfterSeconds: 7 * 24 * 3600}); +db.counter.createIndex({name: 1, domain: 1}, {unique: true}); +db.view.createIndex({"id": 1}, {unique: true}); +db.view.createIndex({display: 1, domain: 1, project: 1}, {unique: true}); //db config -db.setProfilingLevel(1, {slowms: 80, sampleRate: 1} ); \ No newline at end of file +db.setProfilingLevel(1, {slowms: 80, sampleRate: 1}); \ No newline at end of file diff --git a/go.sum b/go.sum index 309b9e1c..613b0c63 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,7 @@ github.com/go-chassis/foundation v0.1.1-0.20191113114104-2b05871e9ec4 h1:wx8JXvg github.com/go-chassis/foundation v0.1.1-0.20191113114104-2b05871e9ec4/go.mod h1:21/ajGtgJlWTCeM0TxGJdRhO8bJkKirWyV8Stlh6g6c= github.com/go-chassis/foundation v0.1.1-0.20200825060850-b16bf420f7b3 h1:c+bwT0qLY69jSU8TmzuNcb9UL/QFAiU96kjuX5TMiQc= github.com/go-chassis/foundation v0.1.1-0.20200825060850-b16bf420f7b3/go.mod h1:21/ajGtgJlWTCeM0TxGJdRhO8bJkKirWyV8Stlh6g6c= +github.com/go-chassis/foundation v0.2.2-0.20201208060254-d5e8e5beb1f0 h1:6oazFyZKryG3r7K20UuWx+bclHMFGXjKBSMJpCefmMI= github.com/go-chassis/foundation v0.2.2-0.20201208060254-d5e8e5beb1f0/go.mod h1:2PjwqpVwYEVaAldl5A58a08viH8p27pNeYaiE3ZxOBA= github.com/go-chassis/go-archaius v1.2.1-0.20200309104817-8c3d4e87d33c h1:pimEM4Oy/Uf4xG4G7TrRUQbIRFAfHiarxDQQS2gmKaM= github.com/go-chassis/go-archaius v1.2.1-0.20200309104817-8c3d4e87d33c/go.mod h1:gVP52u/jCU0fgUjXdUW1VLp5YLLJ+Yl2zoOPrLM/WOM= @@ -134,6 +135,7 @@ github.com/go-chassis/go-chassis/v2 v2.0.2 h1:FMiZgAa6IjDobtZN5oql+eSgMIlDq5+12o github.com/go-chassis/go-chassis/v2 v2.0.2/go.mod h1:xgXXrRUgn5sybtecTDn60ypepHKIV9QpSqR3CIEBNt0= github.com/go-chassis/go-chassis/v2 v2.1.0 h1:NnVUBj3U/pPijpZGooH+3QIRlOB2nAuZtx9b71aiaig= github.com/go-chassis/go-chassis/v2 v2.1.0/go.mod h1:DHA83QKVmuOtdkaWCP5yCDOh+GIRH+YpX3uZ2BlGMWw= +github.com/go-chassis/go-chassis/v2 v2.1.1-0.20201208062518-9c2e86bd7a6c h1:k+D2w1oZ/fQjingJc1KzuwGv0cJKfaGDgaOW/4T4Q1c= github.com/go-chassis/go-chassis/v2 v2.1.1-0.20201208062518-9c2e86bd7a6c/go.mod h1:C5j2kuBAdch4DsSjKQxiYJr3Xy0PoZu2Bxx9QIJIArI= github.com/go-chassis/go-restful-swagger20 v1.0.3-0.20200310030431-17d80f34264f h1:5QmmNpVcGqIc6tuKNe5EAI4PA8Yn2EL9Oee7YdcJ4PE= github.com/go-chassis/go-restful-swagger20 v1.0.3-0.20200310030431-17d80f34264f/go.mod h1:eW62fYuzlNFDvIacB6AV8bhUDCTy4myfTCv0bT9Gbb0= diff --git a/pkg/model/db_schema.go b/pkg/model/db_schema.go index 4d6fe506..8875c242 100644 --- a/pkg/model/db_schema.go +++ b/pkg/model/db_schema.go @@ -17,6 +17,8 @@ package model +import "time" + //LabelDoc is database struct to store labels type LabelDoc struct { ID string `json:"id,omitempty" bson:"id,omitempty" yaml:"id,omitempty" swag:"string"` @@ -57,16 +59,18 @@ type ViewDoc struct { //PollingDetail is db struct, it record operation history type PollingDetail struct { - ID string `json:"id,omitempty" yaml:"id,omitempty"` - SessionID string `json:"session_id,omitempty" bson:"session_id," yaml:"session_id,omitempty"` - Domain string `json:"domain,omitempty" yaml:"domain,omitempty"` - PollingData map[string]interface{} `json:"polling_data,omitempty" yaml:"polling_data,omitempty"` - IP string `json:"ip,omitempty" yaml:"ip,omitempty"` - UserAgent string `json:"user_agent,omitempty" bson:"user_agent," yaml:"user_agent,omitempty"` - URLPath string `json:"url_path,omitempty" bson:"url_path," yaml:"url_path,omitempty"` - ResponseBody []*KVDoc `json:"kv,omitempty" bson:"kv," yaml:"kv,omitempty"` - ResponseHeader map[string][]string `json:"response_header,omitempty" bson:"response_header," yaml:"response_header,omitempty"` - ResponseCode int `json:"response_code,omitempty" bson:"response_code," yaml:"response_code,omitempty"` + ID string `json:"id,omitempty" yaml:"id,omitempty"` + SessionID string `json:"session_id,omitempty" bson:"session_id," yaml:"session_id,omitempty"` + SessionGroup string `json:"session_group,omitempty" bson:"session_group," yaml:"session_group,omitempty"` + Domain string `json:"domain,omitempty" yaml:"domain,omitempty"` + PollingData map[string]interface{} `json:"polling_data,omitempty" yaml:"polling_data,omitempty"` + Revision string `json:"revision,omitempty" yaml:"revision,omitempty"` + IP string `json:"ip,omitempty" yaml:"ip,omitempty"` + UserAgent string `json:"user_agent,omitempty" bson:"user_agent," yaml:"user_agent,omitempty"` + URLPath string `json:"url_path,omitempty" bson:"url_path," yaml:"url_path,omitempty"` + ResponseBody []*KVDoc `json:"kv,omitempty" bson:"kv," yaml:"kv,omitempty"` + ResponseCode int `json:"response_code,omitempty" bson:"response_code," yaml:"response_code,omitempty"` + Timestamp time.Time `json:"timestamp,omitempty" yaml:"timestamp,omitempty"` } // UpdateKVRequest is db struct, it contains kv update request params diff --git a/server/handler/track_handler.go b/server/handler/track_handler.go index bddd7a8b..4228c984 100644 --- a/server/handler/track_handler.go +++ b/server/handler/track_handler.go @@ -29,6 +29,7 @@ import ( "github.com/go-chassis/openlog" "net/http" "strings" + "time" ) //const of noop auth handler @@ -66,13 +67,15 @@ func (h *TrackHandler) Handle(chain *handler.Chain, inv *invocation.Invocation, data := &model.PollingDetail{} data.URLPath = req.Request.Method + " " + req.Request.URL.Path data.SessionID = sessionID + data.SessionGroup = req.HeaderParameter(v1.HeaderSessionGroup) data.UserAgent = req.HeaderParameter(v1.HeaderUserAgent) data.Domain = v1.ReadDomain(req.Request.Context()) data.IP = iputil.ClientIP(req.Request) data.ResponseBody = req.Attribute(common.RespBodyContextKey).([]*model.KVDoc) data.ResponseCode = ir.Status + data.Timestamp = time.Now() if resp != nil { - data.ResponseHeader = resp.Header() + data.Revision = resp.Header().Get(common.HeaderRevision) } data.PollingData = map[string]interface{}{ "revision": revStr, diff --git a/server/resource/v1/common.go b/server/resource/v1/common.go index 21b80c1e..8500e415 100644 --- a/server/resource/v1/common.go +++ b/server/resource/v1/common.go @@ -43,6 +43,7 @@ import ( const ( HeaderUserAgent = "User-Agent" HeaderSessionID = "X-Session-Id" + HeaderSessionGroup = "X-Session-Group" AttributeDomainKey = "domain" FmtReadRequestError = "decode request body failed: %v" diff --git a/server/service/mongo/session/session.go b/server/service/mongo/session/session.go index 16e3d6b2..e55e16b7 100644 --- a/server/service/mongo/session/session.go +++ b/server/service/mongo/session/session.go @@ -222,19 +222,14 @@ func ensureKVLongPolling(session *mgo.Session) { c := session.DB(DBName).C(CollectionPollingDetail) err := c.Create(&mgo.CollectionInfo{Validator: bson.M{ "id": bson.M{"$exists": true}, - "params": bson.M{"$exists": true}, + "revision": bson.M{"$exists": true}, "session_id": bson.M{"$exists": true}, "url_path": bson.M{"$exists": true}, }}) wrapError(err, MsgDBExists) err = c.EnsureIndex(mgo.Index{ - Key: []string{"id"}, - Unique: true, - }) - wrapError(err) - err = c.EnsureIndex(mgo.Index{ - Key: []string{"session_id", "domain"}, - Unique: true, + Key: []string{"timestamp"}, + ExpireAfter: 7 * 24 * time.Hour, }) wrapError(err) } diff --git a/server/service/mongo/track/polling_detail_dao.go b/server/service/mongo/track/polling_detail_dao.go index bb3bcae6..f9cc8c5b 100644 --- a/server/service/mongo/track/polling_detail_dao.go +++ b/server/service/mongo/track/polling_detail_dao.go @@ -29,9 +29,10 @@ import ( ) //CreateOrUpdate create a record or update exist record +//If revision and session_id is exist: update else:insert func CreateOrUpdate(ctx context.Context, detail *model.PollingDetail) (*model.PollingDetail, error) { collection := session.GetDB().Collection(session.CollectionPollingDetail) - queryFilter := bson.M{"domain": detail.Domain, "session_id": detail.SessionID} + queryFilter := bson.M{"revision": detail.Domain, "session_id": detail.SessionID} res := collection.FindOne(ctx, queryFilter) if res.Err() != nil { if res.Err() == mongo.ErrNoDocuments { @@ -67,6 +68,9 @@ func Get(ctx context.Context, detail *model.PollingDetail) ([]*model.PollingDeta if detail.URLPath != "" { queryFilter["url_path"] = detail.URLPath } + if detail.Revision != "" { + queryFilter["revision"] = detail.Revision + } cur, err := collection.Find(ctx, queryFilter) if err != nil { return nil, err