Cherry pick fix for Revised Data Paging/Totals #426 & #427

gateway/field_presence.go

@@ -151,7 +151,7 @@ type presenceInterceptorOptionsDecorator struct {
 
 type presenceInterceptorOption func(*presenceInterceptorOptionsDecorator)
 
-//WithOverrideFieldMask represent an option to override field mask generated by grpc-gateway
+// WithOverrideFieldMask represent an option to override field mask generated by grpc-gateway
 func WithOverrideFieldMask(d *presenceInterceptorOptionsDecorator) {
 	d.overrideFieldMask = true
 }

gateway/fields.go

@@ -7,10 +7,10 @@ import (
 	"github.com/infobloxopen/atlas-app-toolkit/query"
 )
 
-//retainFields function extracts the configuration for fields that
-//need to be ratained either from gRPC response or from original testRequest
-//(in case when gRPC side didn't set any preferences) and retains only
-//this fields on outgoing response (dynmap).
+// retainFields function extracts the configuration for fields that
+// need to be ratained either from gRPC response or from original testRequest
+// (in case when gRPC side didn't set any preferences) and retains only
+// this fields on outgoing response (dynmap).
 func retainFields(ctx context.Context, req *http.Request, dynmap map[string]interface{}) {
 	fieldsStr := ""
 	if req != nil {

gateway/gateway.go

@@ -88,8 +88,9 @@ func ClientUnaryInterceptor(parentCtx context.Context, method string, req, reply
 		l := vals.Get(limitQueryKey)
 		o := vals.Get(offsetQueryKey)
 		pt := vals.Get(pageTokenQueryKey)
+		t := vals.Get(isTotalSizeNeededQueryKey)
 
-		p, err = query.ParsePagination(l, o, pt)
+		p, err = query.ParsePagination(l, o, pt, t)
 		if err != nil {
 			return status.Error(codes.InvalidArgument, err.Error())
 		}

gateway/operator.go

@@ -11,16 +11,18 @@ import (
 )
 
 const (
-	filterQueryKey           = "_filter"
-	sortQueryKey             = "_order_by"
-	fieldsQueryKey           = "_fields"
-	limitQueryKey            = "_limit"
-	offsetQueryKey           = "_offset"
-	pageTokenQueryKey        = "_page_token"
-	searchQueryKey           = "_fts"
-	pageInfoSizeMetaKey      = "status-page-info-size"
-	pageInfoOffsetMetaKey    = "status-page-info-offset"
-	pageInfoPageTokenMetaKey = "status-page-info-page_token"
+	filterQueryKey               = "_filter"
+	sortQueryKey                 = "_order_by"
+	fieldsQueryKey               = "_fields"
+	limitQueryKey                = "_limit"
+	offsetQueryKey               = "_offset"
+	pageTokenQueryKey            = "_page_token"
+	searchQueryKey               = "_fts"
+	isTotalSizeNeededQueryKey    = "_is_total_size_needed"
+	pageInfoSizeMetaKey          = "status-page-info-size"
+	pageInfoOffsetMetaKey        = "status-page-info-offset"
+	pageInfoPageTokenMetaKey     = "status-page-info-page_token"
+	pageInfoPageTotalSizeMetaKey = "status-page-info-total_size"
 
 	query_url = "query_url"
 )
@@ -55,5 +57,9 @@ func SetPageInfo(ctx context.Context, p *query.PageInfo) error {
 		m[pageInfoSizeMetaKey] = strconv.FormatUint(uint64(s), 10)
 	}
 
+	if t := p.GetTotalSize(); t != 0 {
+		m[pageInfoPageTotalSizeMetaKey] = strconv.FormatUint(uint64(t), 10)
+	}
+
 	return grpc.SetHeader(ctx, metadata.New(m))
 }

gorm/filtering.go

@@ -57,7 +57,7 @@ func FilterStringToGorm(ctx context.Context, filter string, obj interface{}, pb
 	return FilteringToGormEx(ctx, f, obj, c)
 }
 
-//Deprecated: Use FilteringToGormEx instead
+// Deprecated: Use FilteringToGormEx instead
 // FilteringToGorm returns GORM Plain SQL representation of the filtering expression.
 func FilteringToGorm(ctx context.Context, m *query.Filtering, obj interface{}, pb proto.Message) (string, []interface{}, map[string]struct{}, error) {
 	c := &DefaultFilteringConditionConverter{&DefaultFilteringConditionProcessor{pb}}

gorm/utilities.go

@@ -41,7 +41,7 @@ func HandleFieldPath(ctx context.Context, fieldPath []string, obj interface{}) (
 	return dbPath, "", nil
 }
 
-//HandleJSONFiledPath translate field path to JSONB path for postgres jsonb
+// HandleJSONFiledPath translate field path to JSONB path for postgres jsonb
 func HandleJSONFieldPath(ctx context.Context, fieldPath []string, obj interface{}, values ...string) (string, string, error) {
 	operator := "#>>"
 	if isRawJSON(values...) {
@@ -82,7 +82,7 @@ func isRawJSON(values ...string) bool {
 	return true
 }
 
-//TODO: add supprt for embeded objects
+// TODO: add supprt for embeded objects
 func IsJSONCondition(ctx context.Context, fieldPath []string, obj interface{}) bool {
 	fieldName := generator.CamelCase(fieldPath[0])
 	objType := indirectType(reflect.TypeOf(obj))

query/collection_operators.proto

@@ -202,6 +202,9 @@ message Pagination {
     // The service may impose maximum value.
     // If omitted the service may impose a default value.
     int32 limit = 3;
+    // The bool value enables/disables the record count.
+    // The default setting disables the record count.
+    bool is_total_size_needed = 4;
 }
 
 // PageInfo represents both server-driven and client-driven pagination response.
@@ -220,6 +223,8 @@ message PageInfo {
     // The service may optionally include the offset of the next page of resources.
     // A null value indicates no more pages.
     int32 offset = 3;
+    // total_size indicates the total records present.
+    int64 total_size = 4;
 }
 
 // Searching represents search by.

query/fields.go

@@ -9,8 +9,8 @@ const (
 	opCommonInnerDelimiter = "."
 )
 
-//FieldSelectionMap is a convenience type that represents map[string]*Field
-//used in FieldSelection and Field structs
+// FieldSelectionMap is a convenience type that represents map[string]*Field
+// used in FieldSelection and Field structs
 type FieldSelectionMap map[string]*Field
 
 func innerDelimiter(delimiter ...string) string {
@@ -26,9 +26,9 @@ func toParts(input string, delimiter ...string) []string {
 	return strings.Split(input, split)
 }
 
-//ParseFieldSelection transforms a string with comma-separated fields that comes
-//from client to FieldSelection struct. For complex fields dot is used as a delimeter by
-//default, but it is also possible to specify a different delimiter.
+// ParseFieldSelection transforms a string with comma-separated fields that comes
+// from client to FieldSelection struct. For complex fields dot is used as a delimeter by
+// default, but it is also possible to specify a different delimiter.
 func ParseFieldSelection(input string, delimiter ...string) *FieldSelection {
 	if len(input) == 0 {
 		return nil
@@ -44,8 +44,8 @@ func ParseFieldSelection(input string, delimiter ...string) *FieldSelection {
 	return result
 }
 
-//GoString converts FieldSelection to a string representation
-//It implements fmt.GoStringer interface and returns dot-notated fields separated by commas
+// GoString converts FieldSelection to a string representation
+// It implements fmt.GoStringer interface and returns dot-notated fields separated by commas
 func (f *FieldSelection) GoString() string {
 	return strings.Join(f.AllFieldStrings(), opCommonDelimiter)
 }
@@ -70,7 +70,7 @@ func addChildFieldString(result *[]string, parent string, field *Field) {
 	}
 }
 
-//Add allows to add new fields to FieldSelection
+// Add allows to add new fields to FieldSelection
 func (f *FieldSelection) Add(field string, delimiter ...string) {
 	if len(field) == 0 {
 		return
@@ -97,7 +97,7 @@ func (f *FieldSelection) Add(field string, delimiter ...string) {
 	}
 }
 
-//Delete allows to remove fields from FieldSelection
+// Delete allows to remove fields from FieldSelection
 func (f *FieldSelection) Delete(field string, delimiter ...string) bool {
 	if len(field) == 0 || f.Fields == nil {
 		return false
@@ -122,7 +122,7 @@ func (f *FieldSelection) Delete(field string, delimiter ...string) bool {
 	return true
 }
 
-//Get allows to get specified field from FieldSelection
+// Get allows to get specified field from FieldSelection
 func (f *FieldSelection) Get(field string, delimiter ...string) *Field {
 	if len(field) == 0 || f.Fields == nil {
 		return nil

query/filtering_lexer.go

@@ -221,7 +221,7 @@ func (t InToken) String() string {
 	return "in"
 }
 
-//NumberArrayToken represent number array e.g. [1,2,5]
+// NumberArrayToken represent number array e.g. [1,2,5]
 type StringArrayToken struct {
 	TokenBase
 	Values []string
@@ -231,7 +231,7 @@ func (t StringArrayToken) String() string {
 	return fmt.Sprintf("%v", t.Values)
 }
 
-//NumberArrayToken represent number array e.g. [1,2,5]
+// NumberArrayToken represent number array e.g. [1,2,5]
 type NumberArrayToken struct {
 	TokenBase
 	Values []float64

query/pagination.go

@@ -14,7 +14,7 @@ const (
 
 // Pagination parses string representation of pagination limit, offset.
 // Returns error if limit or offset has invalid syntax or out of range.
-func ParsePagination(limit, offset, ptoken string) (*Pagination, error) {
+func ParsePagination(limit, offset, ptoken, isTotalSizeNeeded string) (*Pagination, error) {
 	p := new(Pagination)
 
 	if limit != "" {
@@ -43,6 +43,14 @@ func ParsePagination(limit, offset, ptoken string) (*Pagination, error) {
 		p.PageToken = ptoken
 	}
 
+	if isTotalSizeNeeded != "" {
+		u, err := strconv.ParseBool(isTotalSizeNeeded)
+		if err != nil {
+			return nil, fmt.Errorf("pagination: is_total_size_needed - %s", err.(*strconv.NumError).Err)
+		}
+		p.IsTotalSizeNeeded = u
+	}
+
 	return p, nil
 }
 

query/pagination_test.go

@@ -2,11 +2,13 @@ package query
 
 import (
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestParsePagination(t *testing.T) {
 	// invalid limit
-	_, err := ParsePagination("1s", "0", "ptoken")
+	_, err := ParsePagination("1s", "0", "ptoken", "")
 	if err == nil {
 		t.Error("unexpected nil error - expected: pagination: limit - invalid syntax")
 	}
@@ -15,7 +17,7 @@ func TestParsePagination(t *testing.T) {
 	}
 
 	// negative limit
-	_, err = ParsePagination("-1", "0", "ptoken")
+	_, err = ParsePagination("-1", "0", "ptoken", "")
 	if err == nil {
 		t.Error("unexpected nil error - expected: pagination: limit must be a positive value")
 	}
@@ -24,7 +26,7 @@ func TestParsePagination(t *testing.T) {
 	}
 
 	// zero limit
-	_, err = ParsePagination("0", "0", "ptoken")
+	_, err = ParsePagination("0", "0", "ptoken", "")
 	if err == nil {
 		t.Error("unexpected nil error - expected: pagination: limit must be a positive value")
 	}
@@ -33,7 +35,7 @@ func TestParsePagination(t *testing.T) {
 	}
 
 	// invalid offset
-	_, err = ParsePagination("", "0w", "ptoken")
+	_, err = ParsePagination("", "0w", "ptoken", "")
 	if err == nil {
 		t.Error("unexpected nil error - expected: pagination: offset - invalid syntax")
 	}
@@ -42,7 +44,7 @@ func TestParsePagination(t *testing.T) {
 	}
 
 	// negative offset
-	_, err = ParsePagination("", "-1", "ptoken")
+	_, err = ParsePagination("", "-1", "ptoken", "")
 	if err == nil {
 		t.Error("unexpected nil error - expected: pagination: offset - negative value")
 	}
@@ -51,7 +53,7 @@ func TestParsePagination(t *testing.T) {
 	}
 
 	// null offset
-	p, err := ParsePagination("", "null", "ptoken")
+	p, err := ParsePagination("", "null", "ptoken", "")
 	if err != nil {
 		t.Errorf("unexpected error: %v", err)
 	}
@@ -60,14 +62,14 @@ func TestParsePagination(t *testing.T) {
 	}
 
 	// first page
-	p, err = ParsePagination("", "0", "ptoken")
+	p, err = ParsePagination("", "0", "ptoken", "")
 	if err != nil {
 		t.Errorf("unexpected error: %s", err)
 	}
 	if !p.FirstPage() {
 		t.Errorf("invalid value of first page: %v - expected: true", p.FirstPage())
 	}
-	p, err = ParsePagination("", "100", "null")
+	p, err = ParsePagination("", "100", "null", "")
 	if err != nil {
 		t.Errorf("unexpected error: %s", err)
 	}
@@ -81,7 +83,7 @@ func TestParsePagination(t *testing.T) {
 	}
 
 	// valid pagination
-	p, err = ParsePagination("1000", "100", "ptoken")
+	p, err = ParsePagination("1000", "100", "ptoken", "")
 	if err != nil {
 		t.Errorf("unexpected error: %s", err)
 	}
@@ -94,6 +96,30 @@ func TestParsePagination(t *testing.T) {
 	if p.GetPageToken() != "ptoken" {
 		t.Errorf("invalid page token: %q - expected: ptoken", p.GetPageToken())
 	}
+
+	// valid pagination with isTotalSizeNeeded=true
+	p, err = ParsePagination("1000", "100", "ptoken", "true")
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
+	assert.Equal(t, true, p.GetIsTotalSizeNeeded())
+
+	// valid pagination with isTotalSizeNeeded=false
+	p, err = ParsePagination("1000", "100", "ptoken", "false")
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
+	assert.Equal(t, false, p.GetIsTotalSizeNeeded())
+
+	// valid pagination with isTotalSizeNeeded=null
+	_, err = ParsePagination("1000", "100", "ptoken", "null")
+	if err == nil {
+		t.Error("unexpected nil error - expected: pagination: is_total_size_needed - invalid syntax")
+	}
+	if err.Error() != "pagination: is_total_size_needed - invalid syntax" {
+		t.Errorf("invalid error: %s - expected: pagination: is_total_size_needed - invalid syntax", err)
+	}
+
 }
 
 func TestPageInfo(t *testing.T) {

requestid/interceptor.go

@@ -11,7 +11,6 @@ import (
 // that should be used as a middleware to generate/include Request-Id in headers and context
 // for tracing and tracking user's request.
 //
-//
 // Returned middleware populates Request-Id from gRPC metadata if
 // they defined in a testRequest message else creates a new one.
 func UnaryServerInterceptor() grpc.UnaryServerInterceptor {

rpc/resource/jsonpb.go

@@ -9,7 +9,9 @@ import (
 
 // MarshalJSONPB implements jsonpb.JSONPBMarshaler interface by marshal
 // Identifier from a JSON string in accordance with Atlas Reference format
-// 		<application_name>/<resource_type>/<resource_id>
+//
+//	<application_name>/<resource_type>/<resource_id>
+//
 // Support "null" value.
 func (m Identifier) MarshalJSONPB(*jsonpb.Marshaler) ([]byte, error) {
 	v := BuildString(m.GetApplicationName(), m.GetResourceType(), m.GetResourceId())
@@ -28,7 +30,9 @@ var _ json.Marshaler = &Identifier{}
 
 // UnmarshalJSONPB implements jsonpb.JSONPBUnmarshaler interface by unmarshal
 // Identifier to a JSON string in accordance with Atlas Reference format
-// 		<application_name>/<resource_type>/<resource_id>
+//
+//	<application_name>/<resource_type>/<resource_id>
+//
 // Support "null" value.
 func (m *Identifier) UnmarshalJSONPB(_ *jsonpb.Unmarshaler, data []byte) error {
 	v := strings.Trim(string(data), "\"")

rpc/resource/string.go

@@ -8,7 +8,8 @@ const (
 )
 
 // BuildString builds string id according to Atlas Reference format:
-//  <application_name>/<resource_type>/<resource_id>
+//
+//	<application_name>/<resource_type>/<resource_id>
 func BuildString(aname, rtype, rid string) string {
 	var l []string
 
@@ -26,7 +27,9 @@ func BuildString(aname, rtype, rid string) string {
 }
 
 // ParseString parses id according to Atlas Reference format:
+//
 //	<application_name>/<resource_type>/<resource_id>
+//
 // All leading and trailing Delimiter are removed.
 // The resource_id is parsed first, then resource type and last application name.
 // The id "/a/b/c/" will be converted to "a/b/c" and returned as (a, b, c).

server/server.go

@@ -43,7 +43,7 @@ type Server struct {
 	isAutomaticStop bool
 }
 
-//Middleware wrapper
+// Middleware wrapper
 type Middleware func(handler http.Handler) http.Handler
 
 // Option is a functional option for creating a Server