From 6939198ea0de4f84bda3c456c4c5969447239f99 Mon Sep 17 00:00:00 2001 From: saurabh-narkhede Date: Mon, 23 Dec 2024 22:30:00 +0530 Subject: [PATCH 1/6] changes for geo endpoint --- modules/pubmatic/openwrap/cache/cache.go | 1 + .../openwrap/cache/gocache/gdpr_country.go | 21 ++++++++++ modules/pubmatic/openwrap/cache/mock/mock.go | 15 +++++++ modules/pubmatic/openwrap/config/config.go | 1 + .../pubmatic/openwrap/database/database.go | 1 + .../pubmatic/openwrap/database/mock/mock.go | 15 +++++++ .../openwrap/database/mysql/gdpr_country.go | 36 ++++++++++++++++ modules/pubmatic/openwrap/geo.go | 42 +++++++++++++++++++ modules/pubmatic/openwrap/models/constants.go | 1 + .../openwrap/publisherfeature/feature.go | 1 + .../openwrap/publisherfeature/gdpr_country.go | 9 ++++ .../openwrap/publisherfeature/mock/mock.go | 14 +++++++ .../openwrap/publisherfeature/reloader.go | 24 +++++++++++ 13 files changed, 181 insertions(+) create mode 100644 modules/pubmatic/openwrap/cache/gocache/gdpr_country.go create mode 100644 modules/pubmatic/openwrap/database/mysql/gdpr_country.go create mode 100644 modules/pubmatic/openwrap/geo.go create mode 100644 modules/pubmatic/openwrap/publisherfeature/gdpr_country.go diff --git a/modules/pubmatic/openwrap/cache/cache.go b/modules/pubmatic/openwrap/cache/cache.go index e42d9c775c9..f983be49ed7 100644 --- a/modules/pubmatic/openwrap/cache/cache.go +++ b/modules/pubmatic/openwrap/cache/cache.go @@ -20,6 +20,7 @@ type Cache interface { GetProfileTypePlatforms() (map[string]int, error) GetAppIntegrationPaths() (map[string]int, error) GetAppSubIntegrationPaths() (map[string]int, error) + GetGDPRCountryCodes() (map[string]struct{}, error) Set(key string, value interface{}) Get(key string) (interface{}, bool) diff --git a/modules/pubmatic/openwrap/cache/gocache/gdpr_country.go b/modules/pubmatic/openwrap/cache/gocache/gdpr_country.go new file mode 100644 index 00000000000..c1247b038db --- /dev/null +++ b/modules/pubmatic/openwrap/cache/gocache/gdpr_country.go @@ -0,0 +1,21 @@ +package gocache + +import ( + "fmt" + + "github.com/golang/glog" + "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/models" +) + +var errorGDPRCountryUpdate = "[ErrorGDPRCountryUpdate]:%w" + +// We are not saving data in cache here +func (c *cache) GetGDPRCountryCodes() (map[string]struct{}, error) { + gdprCountryCodes, err := c.db.GetGDPRCountryCodes() + if err != nil { + c.metricEngine.RecordDBQueryFailure(models.GDPRCountryCodesQuery, "", "") + glog.Errorf(models.ErrDBQueryFailed, models.GDPRCountryCodesQuery, "", "", err) + return gdprCountryCodes, fmt.Errorf(errorGDPRCountryUpdate, err) + } + return gdprCountryCodes, nil +} diff --git a/modules/pubmatic/openwrap/cache/mock/mock.go b/modules/pubmatic/openwrap/cache/mock/mock.go index 5cefc290ae3..cd936ada641 100644 --- a/modules/pubmatic/openwrap/cache/mock/mock.go +++ b/modules/pubmatic/openwrap/cache/mock/mock.go @@ -125,6 +125,21 @@ func (mr *MockCacheMockRecorder) GetFSCThresholdPerDSP() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFSCThresholdPerDSP", reflect.TypeOf((*MockCache)(nil).GetFSCThresholdPerDSP)) } +// GetGDPRCountryCodes mocks base method +func (m *MockCache) GetGDPRCountryCodes() (map[string]struct{}, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetGDPRCountryCodes") + ret0, _ := ret[0].(map[string]struct{}) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetGDPRCountryCodes indicates an expected call of GetGDPRCountryCodes +func (mr *MockCacheMockRecorder) GetGDPRCountryCodes() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetGDPRCountryCodes", reflect.TypeOf((*MockCache)(nil).GetGDPRCountryCodes)) +} + // GetMappingsFromCacheV25 mocks base method func (m *MockCache) GetMappingsFromCacheV25(arg0 models.RequestCtx, arg1 int) map[string]models.SlotMapping { m.ctrl.T.Helper() diff --git a/modules/pubmatic/openwrap/config/config.go b/modules/pubmatic/openwrap/config/config.go index 112a50a2770..5141c50c747 100755 --- a/modules/pubmatic/openwrap/config/config.go +++ b/modules/pubmatic/openwrap/config/config.go @@ -74,6 +74,7 @@ type Queries struct { GetProfileTypePlatformMapQuery string GetAppIntegrationPathMapQuery string GetAppSubIntegrationPathMapQuery string + GetGDPRCountryCodes string } type Cache struct { diff --git a/modules/pubmatic/openwrap/database/database.go b/modules/pubmatic/openwrap/database/database.go index 7cb41fa9de7..615e7bd80cd 100644 --- a/modules/pubmatic/openwrap/database/database.go +++ b/modules/pubmatic/openwrap/database/database.go @@ -19,4 +19,5 @@ type Database interface { GetProfileTypePlatforms() (map[string]int, error) GetAppIntegrationPaths() (map[string]int, error) GetAppSubIntegrationPaths() (map[string]int, error) + GetGDPRCountryCodes() (map[string]struct{}, error) } diff --git a/modules/pubmatic/openwrap/database/mock/mock.go b/modules/pubmatic/openwrap/database/mock/mock.go index 81f395c2ce5..5b134650eda 100644 --- a/modules/pubmatic/openwrap/database/mock/mock.go +++ b/modules/pubmatic/openwrap/database/mock/mock.go @@ -125,6 +125,21 @@ func (mr *MockDatabaseMockRecorder) GetFSCThresholdPerDSP() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFSCThresholdPerDSP", reflect.TypeOf((*MockDatabase)(nil).GetFSCThresholdPerDSP)) } +// GetGDPRCountryCodes mocks base method +func (m *MockDatabase) GetGDPRCountryCodes() (map[string]struct{}, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetGDPRCountryCodes") + ret0, _ := ret[0].(map[string]struct{}) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetGDPRCountryCodes indicates an expected call of GetGDPRCountryCodes +func (mr *MockDatabaseMockRecorder) GetGDPRCountryCodes() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetGDPRCountryCodes", reflect.TypeOf((*MockDatabase)(nil).GetGDPRCountryCodes)) +} + // GetMappings mocks base method func (m *MockDatabase) GetMappings(arg0 string, arg1 map[string]models.SlotMapping) (map[string]interface{}, error) { m.ctrl.T.Helper() diff --git a/modules/pubmatic/openwrap/database/mysql/gdpr_country.go b/modules/pubmatic/openwrap/database/mysql/gdpr_country.go new file mode 100644 index 00000000000..4d9a2b1030d --- /dev/null +++ b/modules/pubmatic/openwrap/database/mysql/gdpr_country.go @@ -0,0 +1,36 @@ +package mysql + +import ( + "context" + "time" + + "github.com/golang/glog" +) + +func (db *mySqlDB) GetGDPRCountryCodes() (map[string]struct{}, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(time.Millisecond*time.Duration(db.cfg.MaxDbContextTimeout))) + defer cancel() + + rows, err := db.conn.QueryContext(ctx, db.cfg.Queries.GetGDPRCountryCodes) + if err != nil { + return nil, err + } + defer rows.Close() + + // Map to store the country codes + countryCodes := make(map[string]struct{}) + for rows.Next() { + var countryCode string + if err := rows.Scan(&countryCode); err != nil { + glog.Error("ErrRowScanFailed GetGDPRCountryCodes Err: ", err.Error()) + continue + } + countryCodes[countryCode] = struct{}{} + } + + //TO-DO - partial error is allowed? + if err = rows.Err(); err != nil { + return nil, err + } + return countryCodes, nil +} diff --git a/modules/pubmatic/openwrap/geo.go b/modules/pubmatic/openwrap/geo.go new file mode 100644 index 00000000000..e65c4acaf72 --- /dev/null +++ b/modules/pubmatic/openwrap/geo.go @@ -0,0 +1,42 @@ +package openwrap + +import ( + "net/http" + "time" + + "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/geodb/netacuity" +) + +// geo provides geo metadata from ip +type geo struct { + countryCode string `json:"cc,omitempty"` + stateCode string `json:"sc,omitempty"` + compliance string `json:"compliance,omitempty"` + sectionID string `json:"sectionId,omitempty"` +} + +const ( + cacheTimeout = time.Duration(48) * time.Hour + headerContentType = "Content-Type" + headerAccessControlAllowOrigin = "Access-Control-Allow-Origin" + headerCacheControl = "Cache-Control" + headerContentTypeValue = "application/json" + headerAccessControlAllowOriginValue = "*" +) + +// geoHandler provides a handler for geo lookups. +type geoHandler struct { + geoService netacuity.NetAcuity +} + +// NewGeoHandler initializes and returns a new GeoHandler. +func NewGeoHandler() *geoHandler { + return &geoHandler{ + geoService: netacuity.NetAcuity{}, + } +} + +// Handler for /geo endpoint +func (handler *geoHandler) Handle(w http.ResponseWriter, r *http.Request) { + +} diff --git a/modules/pubmatic/openwrap/models/constants.go b/modules/pubmatic/openwrap/models/constants.go index beacaffec93..2d47338960c 100755 --- a/modules/pubmatic/openwrap/models/constants.go +++ b/modules/pubmatic/openwrap/models/constants.go @@ -549,6 +549,7 @@ const ( ProfileTypePlatformMapQuery = "GetProfileTypePlatformMapQuery" AppIntegrationPathMapQuery = "GetAppIntegrationPathMapQuery" AppSubIntegrationPathMapQuery = "GetAppSubIntegrationPathMapQuery" + GDPRCountryCodesQuery = "GetGDPRCountryCodes" ) // constants for owlogger Integration Type diff --git a/modules/pubmatic/openwrap/publisherfeature/feature.go b/modules/pubmatic/openwrap/publisherfeature/feature.go index 1c4e785cc86..6c0528ebc61 100644 --- a/modules/pubmatic/openwrap/publisherfeature/feature.go +++ b/modules/pubmatic/openwrap/publisherfeature/feature.go @@ -12,4 +12,5 @@ type Feature interface { IsApplovinMultiFloorsEnabled(pubID int, profileID string) bool GetApplovinMultiFloors(pubID int, profileID string) models.ApplovinAdUnitFloors GetImpCountingMethodEnabledBidders() map[string]struct{} + IsCountryGDPREnabled(countryCode string) bool } diff --git a/modules/pubmatic/openwrap/publisherfeature/gdpr_country.go b/modules/pubmatic/openwrap/publisherfeature/gdpr_country.go new file mode 100644 index 00000000000..706e3305555 --- /dev/null +++ b/modules/pubmatic/openwrap/publisherfeature/gdpr_country.go @@ -0,0 +1,9 @@ +package publisherfeature + +// IsCountryGDPREnabled returns true if country is gdpr enabled +func (fe *feature) IsCountryGDPREnabled(countryCode string) bool { + fe.RLock() + defer fe.RUnlock() + _, enabled := fe.gdprCountryCodes[countryCode] + return enabled +} diff --git a/modules/pubmatic/openwrap/publisherfeature/mock/mock.go b/modules/pubmatic/openwrap/publisherfeature/mock/mock.go index d22eb6b28ff..2781ba8e0dd 100644 --- a/modules/pubmatic/openwrap/publisherfeature/mock/mock.go +++ b/modules/pubmatic/openwrap/publisherfeature/mock/mock.go @@ -118,6 +118,20 @@ func (mr *MockFeatureMockRecorder) IsBidRecoveryEnabled(arg0, arg1 interface{}) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsBidRecoveryEnabled", reflect.TypeOf((*MockFeature)(nil).IsBidRecoveryEnabled), arg0, arg1) } +// IsCountryGDPREnabled mocks base method +func (m *MockFeature) IsCountryGDPREnabled(arg0 string) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsCountryGDPREnabled", arg0) + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsCountryGDPREnabled indicates an expected call of IsCountryGDPREnabled +func (mr *MockFeatureMockRecorder) IsCountryGDPREnabled(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsCountryGDPREnabled", reflect.TypeOf((*MockFeature)(nil).IsCountryGDPREnabled), arg0) +} + // IsFscApplicable mocks base method func (m *MockFeature) IsFscApplicable(arg0 int, arg1 string, arg2 int) bool { m.ctrl.T.Helper() diff --git a/modules/pubmatic/openwrap/publisherfeature/reloader.go b/modules/pubmatic/openwrap/publisherfeature/reloader.go index 6aafe9beee2..9b39c952d44 100644 --- a/modules/pubmatic/openwrap/publisherfeature/reloader.go +++ b/modules/pubmatic/openwrap/publisherfeature/reloader.go @@ -29,6 +29,7 @@ type feature struct { bidRecovery bidRecovery appLovinMultiFloors appLovinMultiFloors impCountingMethod impCountingMethod + gdprCountryCodes map[string]struct{} } var fe *feature @@ -62,6 +63,7 @@ func New(config Config) *feature { enabledPublisherProfile: make(map[int]map[string]models.ApplovinAdUnitFloors), }, impCountingMethod: newImpCountingMethod(), + gdprCountryCodes: make(map[string]struct{}), } }) return fe @@ -87,6 +89,8 @@ var initReloader = func(fe *feature) { for { //Populating feature config maps from cache fe.updateFeatureConfigMaps() + //update gdprCountryCodes + fe.updateGDPRCountryCodes() select { case t := <-ticker.C: glog.Info("Feature Reloader loads cache @", t) @@ -125,3 +129,23 @@ func (fe *feature) updateFeatureConfigMaps() { glog.Error(err.Error()) } } + +func (fe *feature) updateGDPRCountryCodes() { + var err error + //fetch gdpr countrycodes + gdprCountryCodes, errorGDPRCountryUpdate := fe.cache.GetGDPRCountryCodes() + if errorGDPRCountryUpdate != nil { + err = models.ErrorWrap(err, errorGDPRCountryUpdate) + } + + //set updated countrycodes in map + fe.Lock() + if gdprCountryCodes != nil { + fe.gdprCountryCodes = gdprCountryCodes + } + fe.Unlock() + + if err != nil { + glog.Error(err.Error()) + } +} From 4d036e593dc5a5bddb59dcc84fd1e639b6ef2542 Mon Sep 17 00:00:00 2001 From: saurabh-narkhede Date: Tue, 24 Dec 2024 02:00:27 +0530 Subject: [PATCH 2/6] add geohandler code --- modules/pubmatic/openwrap/geo.go | 76 ++++++++++++++++++- modules/pubmatic/openwrap/models/constants.go | 1 + 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/modules/pubmatic/openwrap/geo.go b/modules/pubmatic/openwrap/geo.go index e65c4acaf72..6e9fa9ac1b7 100644 --- a/modules/pubmatic/openwrap/geo.go +++ b/modules/pubmatic/openwrap/geo.go @@ -1,18 +1,27 @@ package openwrap import ( + "encoding/json" + "fmt" "net/http" + "runtime/debug" + "strconv" "time" + "git.pubmatic.com/PubMatic/go-common/util" + "github.com/golang/glog" "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/geodb/netacuity" + "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/metrics" + "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/metrics/prometheus" + "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/models" ) // geo provides geo metadata from ip type geo struct { - countryCode string `json:"cc,omitempty"` - stateCode string `json:"sc,omitempty"` - compliance string `json:"compliance,omitempty"` - sectionID string `json:"sectionId,omitempty"` + CountryCode string `json:"cc,omitempty"` + StateCode string `json:"sc,omitempty"` + Compliance string `json:"compliance,omitempty"` + SectionID string `json:"sectionId,omitempty"` } const ( @@ -36,7 +45,66 @@ func NewGeoHandler() *geoHandler { } } +const ( + OriginHeaderKey = "Origin" + RefererKey = "Referer" + GDPRCompliance = "GDPR" + USPCompliance = "USP" + StateCodeCalifornia = "ca" + CountryCodeUS = "US" +) + // Handler for /geo endpoint func (handler *geoHandler) Handle(w http.ResponseWriter, r *http.Request) { + var pubIdStr string + metricEngine := ow.GetMetricEngine() + defer func() { + if r := recover(); r != nil { + metricEngine.RecordOpenWrapServerPanicStats(ow.cfg.Server.HostName, "HandleGeoEndpoint") + glog.Errorf("stacktrace:[%s], error:[%v], pubid:[%s]", string(debug.Stack()), r, pubIdStr) + return + } + }() + metricEngine.RecordRequest(metrics.Labels{RType: models.EndpointGeo, RequestStatus: prometheus.RequestStatusOK}) + pubIdStr = r.URL.Query().Get(models.PublisherID) + _, err := strconv.Atoi(pubIdStr) + if err != nil { + glog.Errorf("[geo] error:[invalid pubid passed:%s], [requestType]:%v [url]:%v [origin]:%v [referer]:%v", err.Error(), models.EndpointGeo, + r.URL.RequestURI(), r.Header.Get(OriginHeaderKey), r.Header.Get(RefererKey)) + + //TO-Do keep this stat? + metricEngine.RecordBadRequests(models.EndpointGeo, pubIdStr, 0) + w.WriteHeader(http.StatusBadRequest) + return + } + metricEngine.RecordPublisherRequests(models.EndpointGeo, pubIdStr, "") + + ip := util.GetIP(r) + w.Header().Set(headerContentType, headerContentTypeValue) + w.Header().Set(headerAccessControlAllowOrigin, "*") + success := false + geoInfo, _ := handler.geoService.LookUp(ip) + if geoInfo != nil { + if geoInfo.ISOCountryCode != "" { + success = true + geo := geo{ + CountryCode: geoInfo.ISOCountryCode, + StateCode: geoInfo.RegionCode, + } + + if ow.GetFeature().IsCountryGDPREnabled(geo.CountryCode) { + geo.Compliance = GDPRCompliance + } else if geo.CountryCode == CountryCodeUS && geo.StateCode == StateCodeCalifornia { + geo.Compliance = USPCompliance + } else { + //check if GPP country and set sectionID + } + w.Header().Set(headerCacheControl, "max-age="+fmt.Sprint(cacheTimeout.Seconds())) + json.NewEncoder(w).Encode(geo) + } + } + if !success { + //globals.GetMetricsEngine().RecordGeoLookupFailure(globals.GeoEndpoint) + } } diff --git a/modules/pubmatic/openwrap/models/constants.go b/modules/pubmatic/openwrap/models/constants.go index 2d47338960c..3694b54e5b4 100755 --- a/modules/pubmatic/openwrap/models/constants.go +++ b/modules/pubmatic/openwrap/models/constants.go @@ -479,6 +479,7 @@ const ( EndPointCTV = "ctv" EndpointHybrid = "hybrid" EndpointAppLovinMax = "applovinmax" + EndpointGeo = "geo" Openwrap = "openwrap" ImpTypeBanner = "banner" From 94fa1355bd81e6c46826933fc6837068852750a8 Mon Sep 17 00:00:00 2001 From: saurabh-narkhede Date: Tue, 24 Dec 2024 14:44:10 +0530 Subject: [PATCH 3/6] add gppsectionids and stat for geolookupfailure --- modules/pubmatic/openwrap/geo.go | 21 ++++-- .../openwrap/metrics/config/multimetrics.go | 7 ++ .../metrics/config/multimetrics_test.go | 2 + modules/pubmatic/openwrap/metrics/metrics.go | 1 + .../pubmatic/openwrap/metrics/mock/mock.go | 74 +++++++++++-------- .../openwrap/metrics/prometheus/prometheus.go | 14 ++++ .../metrics/prometheus/prometheus_test.go | 12 +++ .../openwrap/metrics/stats/tcp_stats.go | 1 + .../publisherfeature/reloader_test.go | 1 + 9 files changed, 97 insertions(+), 36 deletions(-) diff --git a/modules/pubmatic/openwrap/geo.go b/modules/pubmatic/openwrap/geo.go index 6e9fa9ac1b7..10788456d8a 100644 --- a/modules/pubmatic/openwrap/geo.go +++ b/modules/pubmatic/openwrap/geo.go @@ -21,7 +21,7 @@ type geo struct { CountryCode string `json:"cc,omitempty"` StateCode string `json:"sc,omitempty"` Compliance string `json:"compliance,omitempty"` - SectionID string `json:"sectionId,omitempty"` + SectionID int `json:"sectionId,omitempty"` } const ( @@ -45,11 +45,20 @@ func NewGeoHandler() *geoHandler { } } +var gppSectionIDs = map[string]int{ + "ca": 8, + "va": 9, + "co": 10, + "ut": 11, + "ct": 12, +} + const ( OriginHeaderKey = "Origin" RefererKey = "Referer" GDPRCompliance = "GDPR" USPCompliance = "USP" + GPPCompliance = "GPP" StateCodeCalifornia = "ca" CountryCodeUS = "US" ) @@ -74,7 +83,7 @@ func (handler *geoHandler) Handle(w http.ResponseWriter, r *http.Request) { r.URL.RequestURI(), r.Header.Get(OriginHeaderKey), r.Header.Get(RefererKey)) //TO-Do keep this stat? - metricEngine.RecordBadRequests(models.EndpointGeo, pubIdStr, 0) + metricEngine.RecordBadRequests(models.EndpointGeo, pubIdStr, -1) w.WriteHeader(http.StatusBadRequest) return } @@ -97,14 +106,16 @@ func (handler *geoHandler) Handle(w http.ResponseWriter, r *http.Request) { geo.Compliance = GDPRCompliance } else if geo.CountryCode == CountryCodeUS && geo.StateCode == StateCodeCalifornia { geo.Compliance = USPCompliance - } else { - //check if GPP country and set sectionID + } else if sectionid, ok := gppSectionIDs[geo.StateCode]; ok { + geo.Compliance = GPPCompliance + geo.SectionID = sectionid } + w.Header().Set(headerCacheControl, "max-age="+fmt.Sprint(cacheTimeout.Seconds())) json.NewEncoder(w).Encode(geo) } } if !success { - //globals.GetMetricsEngine().RecordGeoLookupFailure(globals.GeoEndpoint) + metricEngine.RecordGeoLookupFailure(models.EndpointGeo) } } diff --git a/modules/pubmatic/openwrap/metrics/config/multimetrics.go b/modules/pubmatic/openwrap/metrics/config/multimetrics.go index a179b695930..94e0d6091f9 100644 --- a/modules/pubmatic/openwrap/metrics/config/multimetrics.go +++ b/modules/pubmatic/openwrap/metrics/config/multimetrics.go @@ -557,3 +557,10 @@ func (me *MultiMetricsEngine) RecordBidRecoveryResponseTime(publisher, profile s thisME.RecordBidRecoveryResponseTime(publisher, profile, responseTime) } } + +// RecordGeoLookupFailure across all engines +func (me *MultiMetricsEngine) RecordGeoLookupFailure(endpoint string) { + for _, thisME := range *me { + thisME.RecordGeoLookupFailure(endpoint) + } +} diff --git a/modules/pubmatic/openwrap/metrics/config/multimetrics_test.go b/modules/pubmatic/openwrap/metrics/config/multimetrics_test.go index 038f17706a3..b61e8744923 100644 --- a/modules/pubmatic/openwrap/metrics/config/multimetrics_test.go +++ b/modules/pubmatic/openwrap/metrics/config/multimetrics_test.go @@ -224,6 +224,7 @@ func TestRecordFunctionForMultiMetricsEngine(t *testing.T) { mockEngine.EXPECT().RecordIBVRequest("pubid", "profileid") mockEngine.EXPECT().RecordBidRecoveryStatus(publisher, profile, true) mockEngine.EXPECT().RecordBidRecoveryResponseTime(publisher, profile, time.Duration(200)) + mockEngine.EXPECT().RecordGeoLookupFailure("geo") // create the multi-metric engine multiMetricEngine := MultiMetricsEngine{} @@ -294,4 +295,5 @@ func TestRecordFunctionForMultiMetricsEngine(t *testing.T) { multiMetricEngine.RecordIBVRequest("pubid", "profileid") multiMetricEngine.RecordBidRecoveryStatus(publisher, profile, true) multiMetricEngine.RecordBidRecoveryResponseTime(publisher, profile, time.Duration(200)) + multiMetricEngine.RecordGeoLookupFailure("geo") } diff --git a/modules/pubmatic/openwrap/metrics/metrics.go b/modules/pubmatic/openwrap/metrics/metrics.go index df8969caaad..2a411f1a728 100644 --- a/modules/pubmatic/openwrap/metrics/metrics.go +++ b/modules/pubmatic/openwrap/metrics/metrics.go @@ -97,6 +97,7 @@ type MetricsEngine interface { //AppLovinMax metrics RecordFailedParsingItuneID(pubId, profId string) RecordEndpointResponseSize(endpoint string, bodySize float64) + RecordGeoLookupFailure(endpoint string) //IBV metric RecordIBVRequest(pubId, profId string) diff --git a/modules/pubmatic/openwrap/metrics/mock/mock.go b/modules/pubmatic/openwrap/metrics/mock/mock.go index bcf90c528fe..8eb9abe31d1 100644 --- a/modules/pubmatic/openwrap/metrics/mock/mock.go +++ b/modules/pubmatic/openwrap/metrics/mock/mock.go @@ -130,6 +130,30 @@ func (mr *MockMetricsEngineMockRecorder) RecordBadRequests(arg0, arg1, arg2 inte return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordBadRequests", reflect.TypeOf((*MockMetricsEngine)(nil).RecordBadRequests), arg0, arg1, arg2) } +// RecordBidRecoveryResponseTime mocks base method +func (m *MockMetricsEngine) RecordBidRecoveryResponseTime(arg0, arg1 string, arg2 time.Duration) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "RecordBidRecoveryResponseTime", arg0, arg1, arg2) +} + +// RecordBidRecoveryResponseTime indicates an expected call of RecordBidRecoveryResponseTime +func (mr *MockMetricsEngineMockRecorder) RecordBidRecoveryResponseTime(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordBidRecoveryResponseTime", reflect.TypeOf((*MockMetricsEngine)(nil).RecordBidRecoveryResponseTime), arg0, arg1, arg2) +} + +// RecordBidRecoveryStatus mocks base method +func (m *MockMetricsEngine) RecordBidRecoveryStatus(arg0, arg1 string, arg2 bool) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "RecordBidRecoveryStatus", arg0, arg1, arg2) +} + +// RecordBidRecoveryStatus indicates an expected call of RecordBidRecoveryStatus +func (mr *MockMetricsEngineMockRecorder) RecordBidRecoveryStatus(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordBidRecoveryStatus", reflect.TypeOf((*MockMetricsEngine)(nil).RecordBidRecoveryStatus), arg0, arg1, arg2) +} + // RecordBidResponseByDealCountInHB mocks base method func (m *MockMetricsEngine) RecordBidResponseByDealCountInHB(arg0, arg1, arg2, arg3 string) { m.ctrl.T.Helper() @@ -298,16 +322,16 @@ func (mr *MockMetricsEngineMockRecorder) RecordFailedParsingItuneID(arg0, arg1 i return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordFailedParsingItuneID", reflect.TypeOf((*MockMetricsEngine)(nil).RecordFailedParsingItuneID), arg0, arg1) } -// RecordIBVRequest mocks base method -func (m *MockMetricsEngine) RecordIBVRequest(arg0, arg1 string) { +// RecordGeoLookupFailure mocks base method +func (m *MockMetricsEngine) RecordGeoLookupFailure(arg0 string) { m.ctrl.T.Helper() - m.ctrl.Call(m, "RecordIBVRequest", arg0, arg1) + m.ctrl.Call(m, "RecordGeoLookupFailure", arg0) } -// RecordIBVRequest indicates an expected call of RecordIBVRequest -func (mr *MockMetricsEngineMockRecorder) RecordIBVRequest(arg0, arg1 interface{}) *gomock.Call { +// RecordGeoLookupFailure indicates an expected call of RecordGeoLookupFailure +func (mr *MockMetricsEngineMockRecorder) RecordGeoLookupFailure(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordIBVRequest", reflect.TypeOf((*MockMetricsEngine)(nil).RecordIBVRequest), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordGeoLookupFailure", reflect.TypeOf((*MockMetricsEngine)(nil).RecordGeoLookupFailure), arg0) } // RecordGetProfileDataTime mocks base method @@ -322,6 +346,18 @@ func (mr *MockMetricsEngineMockRecorder) RecordGetProfileDataTime(arg0 interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordGetProfileDataTime", reflect.TypeOf((*MockMetricsEngine)(nil).RecordGetProfileDataTime), arg0) } +// RecordIBVRequest mocks base method +func (m *MockMetricsEngine) RecordIBVRequest(arg0, arg1 string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "RecordIBVRequest", arg0, arg1) +} + +// RecordIBVRequest indicates an expected call of RecordIBVRequest +func (mr *MockMetricsEngineMockRecorder) RecordIBVRequest(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordIBVRequest", reflect.TypeOf((*MockMetricsEngine)(nil).RecordIBVRequest), arg0, arg1) +} + // RecordImpDisabledViaConfigStats mocks base method func (m *MockMetricsEngine) RecordImpDisabledViaConfigStats(arg0, arg1, arg2 string) { m.ctrl.T.Helper() @@ -838,31 +874,7 @@ func (mr *MockMetricsEngineMockRecorder) RecordVideoInstlImpsStats(arg0, arg1 in return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordVideoInstlImpsStats", reflect.TypeOf((*MockMetricsEngine)(nil).RecordVideoInstlImpsStats), arg0, arg1) } -// RecordBidRecoveryStatus mocks base method. -func (m *MockMetricsEngine) RecordBidRecoveryStatus(arg0 string, arg1 string, arg2 bool) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "RecordBidRecoveryStatus", arg0, arg1, arg2) -} - -// RecordBidRecoveryStatus indicates an expected call of RecordBidRecoveryStatus. -func (mr *MockMetricsEngineMockRecorder) RecordBidRecoveryStatus(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordBidRecoveryStatus", reflect.TypeOf((*MockMetricsEngine)(nil).RecordBidRecoveryStatus), arg0, arg1, arg2) -} - -// RecordBidRecoveryResponseTime mocks base method. -func (m *MockMetricsEngine) RecordBidRecoveryResponseTime(arg0 string, arg1 string, arg2 time.Duration) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "RecordBidRecoveryResponseTime", arg0, arg1, arg2) -} - -// RecordBidRecoveryResponseTime indicates an expected call of RecordBidRecoveryStatus. -func (mr *MockMetricsEngineMockRecorder) RecordBidRecoveryResponseTime(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordBidRecoveryResponseTime", reflect.TypeOf((*MockMetricsEngine)(nil).RecordBidRecoveryResponseTime), arg0, arg1, arg2) -} - -// Shutdown mocks base method. +// Shutdown mocks base method func (m *MockMetricsEngine) Shutdown() { m.ctrl.T.Helper() m.ctrl.Call(m, "Shutdown") diff --git a/modules/pubmatic/openwrap/metrics/prometheus/prometheus.go b/modules/pubmatic/openwrap/metrics/prometheus/prometheus.go index 66348544787..4d7132cdeb7 100644 --- a/modules/pubmatic/openwrap/metrics/prometheus/prometheus.go +++ b/modules/pubmatic/openwrap/metrics/prometheus/prometheus.go @@ -101,6 +101,9 @@ type Metrics struct { //IBV request ibvRequests *prometheus.CounterVec + + //geo lookup + geoLookUpFailure *prometheus.CounterVec } const ( @@ -400,6 +403,11 @@ func newMetrics(cfg *config.PrometheusMetrics, promRegistry *prometheus.Registry []string{pubIDLabel, profileIDLabel, successLabel}, ) + metrics.geoLookUpFailure = newCounter(cfg, promRegistry, + "geo_lookup_fail", + "Count of geo lookup failures", + []string{endpointLabel}) + newSSHBMetrics(&metrics, cfg, promRegistry) return &metrics @@ -750,3 +758,9 @@ func (m *Metrics) RecordEndpointResponseSize(endpoint string, bodySize float64) endpointLabel: endpoint, }).Observe(float64(bodySize) / 1024) } + +func (m *Metrics) RecordGeoLookupFailure(endpoint string) { + m.geoLookUpFailure.With(prometheus.Labels{ + endpointLabel: endpoint, + }).Inc() +} diff --git a/modules/pubmatic/openwrap/metrics/prometheus/prometheus_test.go b/modules/pubmatic/openwrap/metrics/prometheus/prometheus_test.go index 9f6525371b9..3506ea327ca 100644 --- a/modules/pubmatic/openwrap/metrics/prometheus/prometheus_test.go +++ b/modules/pubmatic/openwrap/metrics/prometheus/prometheus_test.go @@ -399,6 +399,18 @@ func TestRecordBidRecoveryResponseTime(t *testing.T) { assertHistogram(t, "bid_recovery_response_time", resultingHistogram, 2, 200) } +func TestRecordGeoLookUpFailure(t *testing.T) { + m := createMetricsForTesting() + + m.RecordGeoLookupFailure("geo") + + expectedCount := float64(1) + assertCounterVecValue(t, "", "geo_lookup_fail", m.geoLookUpFailure, + expectedCount, prometheus.Labels{ + endpointLabel: "geo", + }) +} + func getHistogramFromHistogramVec(histogram *prometheus.HistogramVec, labelKey, labelValue string) dto.Histogram { var result dto.Histogram processMetrics(histogram, func(m dto.Metric) { diff --git a/modules/pubmatic/openwrap/metrics/stats/tcp_stats.go b/modules/pubmatic/openwrap/metrics/stats/tcp_stats.go index ebccaa848b1..69cfd7e3b40 100644 --- a/modules/pubmatic/openwrap/metrics/stats/tcp_stats.go +++ b/modules/pubmatic/openwrap/metrics/stats/tcp_stats.go @@ -353,3 +353,4 @@ func (st *StatsTCP) RecordPrebidAuctionBidResponse(publisher string, partnerName func (st *StatsTCP) RecordFailedParsingItuneID(pubId, profId string) {} func (st *StatsTCP) RecordEndpointResponseSize(endpoint string, bodySize float64) {} func (st *StatsTCP) RecordIBVRequest(pubId, profId string) {} +func (st *StatsTCP) RecordGeoLookupFailure(endpoint string) {} diff --git a/modules/pubmatic/openwrap/publisherfeature/reloader_test.go b/modules/pubmatic/openwrap/publisherfeature/reloader_test.go index 2c351c5dce9..85a10b9d1a2 100644 --- a/modules/pubmatic/openwrap/publisherfeature/reloader_test.go +++ b/modules/pubmatic/openwrap/publisherfeature/reloader_test.go @@ -66,6 +66,7 @@ func TestInitiateReloader(t *testing.T) { setup: func() { mockCache.EXPECT().GetPublisherFeatureMap().Return(map[int]map[int]models.FeatureData{}, nil) mockCache.EXPECT().GetFSCThresholdPerDSP().Return(map[int]int{}, nil) + mockCache.EXPECT().GetGDPRCountryCodes().Return(map[string]struct{}{}, nil) }, }, } From 645232736bcf684af7d04ab3f2ee5aeeebbd3800 Mon Sep 17 00:00:00 2001 From: saurabh-narkhede Date: Fri, 3 Jan 2025 20:17:45 +0530 Subject: [PATCH 4/6] use double buffer method --- .../{gdpr_country.go => compliance.go} | 0 .../mysql/{gdpr_country.go => compliance.go} | 3 +- modules/pubmatic/openwrap/geo.go | 14 +++--- .../openwrap/publisherfeature/compliance.go | 47 +++++++++++++++++++ .../openwrap/publisherfeature/gdpr_country.go | 9 ---- .../openwrap/publisherfeature/reloader.go | 24 +--------- 6 files changed, 56 insertions(+), 41 deletions(-) rename modules/pubmatic/openwrap/cache/gocache/{gdpr_country.go => compliance.go} (100%) rename modules/pubmatic/openwrap/database/mysql/{gdpr_country.go => compliance.go} (91%) create mode 100644 modules/pubmatic/openwrap/publisherfeature/compliance.go delete mode 100644 modules/pubmatic/openwrap/publisherfeature/gdpr_country.go diff --git a/modules/pubmatic/openwrap/cache/gocache/gdpr_country.go b/modules/pubmatic/openwrap/cache/gocache/compliance.go similarity index 100% rename from modules/pubmatic/openwrap/cache/gocache/gdpr_country.go rename to modules/pubmatic/openwrap/cache/gocache/compliance.go diff --git a/modules/pubmatic/openwrap/database/mysql/gdpr_country.go b/modules/pubmatic/openwrap/database/mysql/compliance.go similarity index 91% rename from modules/pubmatic/openwrap/database/mysql/gdpr_country.go rename to modules/pubmatic/openwrap/database/mysql/compliance.go index 4d9a2b1030d..775f66e9d33 100644 --- a/modules/pubmatic/openwrap/database/mysql/gdpr_country.go +++ b/modules/pubmatic/openwrap/database/mysql/compliance.go @@ -17,7 +17,7 @@ func (db *mySqlDB) GetGDPRCountryCodes() (map[string]struct{}, error) { } defer rows.Close() - // Map to store the country codes + // map to store the country codes countryCodes := make(map[string]struct{}) for rows.Next() { var countryCode string @@ -28,7 +28,6 @@ func (db *mySqlDB) GetGDPRCountryCodes() (map[string]struct{}, error) { countryCodes[countryCode] = struct{}{} } - //TO-DO - partial error is allowed? if err = rows.Err(); err != nil { return nil, err } diff --git a/modules/pubmatic/openwrap/geo.go b/modules/pubmatic/openwrap/geo.go index 10788456d8a..13f97f1c335 100644 --- a/modules/pubmatic/openwrap/geo.go +++ b/modules/pubmatic/openwrap/geo.go @@ -38,8 +38,8 @@ type geoHandler struct { geoService netacuity.NetAcuity } -// NewGeoHandler initializes and returns a new GeoHandler. -func NewGeoHandler() *geoHandler { +// NewGeo initializes and returns a new GeoHandler. +func NewGeo() *geoHandler { return &geoHandler{ geoService: netacuity.NetAcuity{}, } @@ -64,30 +64,28 @@ const ( ) // Handler for /geo endpoint -func (handler *geoHandler) Handle(w http.ResponseWriter, r *http.Request) { +func (handler *geoHandler) Handler(w http.ResponseWriter, r *http.Request) { var pubIdStr string metricEngine := ow.GetMetricEngine() + metricLabels := metrics.Labels{RType: models.EndpointGeo, RequestStatus: prometheus.RequestStatusOK} defer func() { + metricEngine.RecordRequest(metricLabels) if r := recover(); r != nil { metricEngine.RecordOpenWrapServerPanicStats(ow.cfg.Server.HostName, "HandleGeoEndpoint") glog.Errorf("stacktrace:[%s], error:[%v], pubid:[%s]", string(debug.Stack()), r, pubIdStr) return } }() - metricEngine.RecordRequest(metrics.Labels{RType: models.EndpointGeo, RequestStatus: prometheus.RequestStatusOK}) pubIdStr = r.URL.Query().Get(models.PublisherID) _, err := strconv.Atoi(pubIdStr) if err != nil { glog.Errorf("[geo] error:[invalid pubid passed:%s], [requestType]:%v [url]:%v [origin]:%v [referer]:%v", err.Error(), models.EndpointGeo, r.URL.RequestURI(), r.Header.Get(OriginHeaderKey), r.Header.Get(RefererKey)) - - //TO-Do keep this stat? - metricEngine.RecordBadRequests(models.EndpointGeo, pubIdStr, -1) w.WriteHeader(http.StatusBadRequest) + metricLabels.RequestStatus = prometheus.RequestStatusBadInput return } - metricEngine.RecordPublisherRequests(models.EndpointGeo, pubIdStr, "") ip := util.GetIP(r) w.Header().Set(headerContentType, headerContentTypeValue) diff --git a/modules/pubmatic/openwrap/publisherfeature/compliance.go b/modules/pubmatic/openwrap/publisherfeature/compliance.go new file mode 100644 index 00000000000..4a898c99042 --- /dev/null +++ b/modules/pubmatic/openwrap/publisherfeature/compliance.go @@ -0,0 +1,47 @@ +package publisherfeature + +import ( + "github.com/golang/glog" + "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/models" +) + +type gdprCountryCodes struct { + codes [2]map[string]struct{} + index int +} + +func newGDPRCountryCodes() gdprCountryCodes { + return gdprCountryCodes{ + codes: [2]map[string]struct{}{ + make(map[string]struct{}), + make(map[string]struct{}), + }, + index: 0, + } +} + +func (fe *feature) updateGDPRCountryCodes() { + var err error + //fetch gdpr countrycodes + gdprCountryCodes, errorGDPRCountryUpdate := fe.cache.GetGDPRCountryCodes() + if errorGDPRCountryUpdate != nil { + err = models.ErrorWrap(err, errorGDPRCountryUpdate) + } + // assign fetched codes to the inactive map + if gdprCountryCodes != nil { + fe.gdprCountryCodes.codes[fe.gdprCountryCodes.index^1] = gdprCountryCodes + } + + // toggle the index to make the updated map active + fe.gdprCountryCodes.index ^= 1 + if err != nil { + glog.Error(err.Error()) + } +} + +// IsCountryGDPREnabled returns true if country is gdpr enabled +func (fe *feature) IsCountryGDPREnabled(countryCode string) bool { + codes := fe.gdprCountryCodes.codes[fe.gdprCountryCodes.index] + _, enabled := codes[countryCode] + return enabled +} diff --git a/modules/pubmatic/openwrap/publisherfeature/gdpr_country.go b/modules/pubmatic/openwrap/publisherfeature/gdpr_country.go deleted file mode 100644 index 706e3305555..00000000000 --- a/modules/pubmatic/openwrap/publisherfeature/gdpr_country.go +++ /dev/null @@ -1,9 +0,0 @@ -package publisherfeature - -// IsCountryGDPREnabled returns true if country is gdpr enabled -func (fe *feature) IsCountryGDPREnabled(countryCode string) bool { - fe.RLock() - defer fe.RUnlock() - _, enabled := fe.gdprCountryCodes[countryCode] - return enabled -} diff --git a/modules/pubmatic/openwrap/publisherfeature/reloader.go b/modules/pubmatic/openwrap/publisherfeature/reloader.go index 9b39c952d44..ce92f32a19d 100644 --- a/modules/pubmatic/openwrap/publisherfeature/reloader.go +++ b/modules/pubmatic/openwrap/publisherfeature/reloader.go @@ -29,7 +29,7 @@ type feature struct { bidRecovery bidRecovery appLovinMultiFloors appLovinMultiFloors impCountingMethod impCountingMethod - gdprCountryCodes map[string]struct{} + gdprCountryCodes gdprCountryCodes } var fe *feature @@ -63,7 +63,7 @@ func New(config Config) *feature { enabledPublisherProfile: make(map[int]map[string]models.ApplovinAdUnitFloors), }, impCountingMethod: newImpCountingMethod(), - gdprCountryCodes: make(map[string]struct{}), + gdprCountryCodes: newGDPRCountryCodes(), } }) return fe @@ -129,23 +129,3 @@ func (fe *feature) updateFeatureConfigMaps() { glog.Error(err.Error()) } } - -func (fe *feature) updateGDPRCountryCodes() { - var err error - //fetch gdpr countrycodes - gdprCountryCodes, errorGDPRCountryUpdate := fe.cache.GetGDPRCountryCodes() - if errorGDPRCountryUpdate != nil { - err = models.ErrorWrap(err, errorGDPRCountryUpdate) - } - - //set updated countrycodes in map - fe.Lock() - if gdprCountryCodes != nil { - fe.gdprCountryCodes = gdprCountryCodes - } - fe.Unlock() - - if err != nil { - glog.Error(err.Error()) - } -} From 1ec697bdb97a34c378b85188009d3f2181774526 Mon Sep 17 00:00:00 2001 From: saurabh-narkhede Date: Wed, 8 Jan 2025 00:59:57 +0530 Subject: [PATCH 5/6] add test cases --- .../openwrap/cache/gocache/compliance_test.go | 95 +++++ .../openwrap/database/mysql/compliance.go | 1 + .../database/mysql/compliance_test.go | 156 ++++++++ modules/pubmatic/openwrap/geo.go | 22 +- modules/pubmatic/openwrap/geo_test.go | 345 ++++++++++++++++++ .../publisherfeature/compliance_test.go | 181 +++++++++ 6 files changed, 785 insertions(+), 15 deletions(-) create mode 100644 modules/pubmatic/openwrap/cache/gocache/compliance_test.go create mode 100644 modules/pubmatic/openwrap/database/mysql/compliance_test.go create mode 100644 modules/pubmatic/openwrap/geo_test.go create mode 100644 modules/pubmatic/openwrap/publisherfeature/compliance_test.go diff --git a/modules/pubmatic/openwrap/cache/gocache/compliance_test.go b/modules/pubmatic/openwrap/cache/gocache/compliance_test.go new file mode 100644 index 00000000000..63ed3214c8e --- /dev/null +++ b/modules/pubmatic/openwrap/cache/gocache/compliance_test.go @@ -0,0 +1,95 @@ +package gocache + +import ( + "errors" + "testing" + + "github.com/golang/mock/gomock" + gocache "github.com/patrickmn/go-cache" + + "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/config" + "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/database" + mock_database "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/database/mock" + "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/metrics" + mock_metrics "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/metrics/mock" + "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/models" + "github.com/stretchr/testify/assert" +) + +func TestCache_GetGDPRCountryCodes(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockDatabase := mock_database.NewMockDatabase(ctrl) + mockEngine := mock_metrics.NewMockMetricsEngine(ctrl) + type fields struct { + cache *gocache.Cache + cfg config.Cache + db database.Database + metricEngine metrics.MetricsEngine + } + tests := []struct { + name string + fields fields + want map[string]struct{} + setup func() + wantErr bool + }{ + { + name: "Valid Data present in DB, return same", + want: map[string]struct{}{ + "US": {}, + "LV": {}, + "DE": {}, + }, + setup: func() { + mockDatabase.EXPECT().GetGDPRCountryCodes().Return(map[string]struct{}{ + "US": {}, + "LV": {}, + "DE": {}, + }, nil) + }, + fields: fields{ + cache: gocache.New(100, 100), + db: mockDatabase, + cfg: config.Cache{ + CacheDefaultExpiry: 1000, + }, + }, + wantErr: false, + }, + { + name: "Error In DB, Set Empty", + want: nil, + setup: func() { + mockDatabase.EXPECT().GetGDPRCountryCodes().Return(nil, errors.New("QUERY FAILD")) + mockEngine.EXPECT().RecordDBQueryFailure(models.GDPRCountryCodesQuery, "", "").Return() + }, + fields: fields{ + cache: gocache.New(100, 100), + db: mockDatabase, + cfg: config.Cache{ + CacheDefaultExpiry: 1000, + }, + metricEngine: mockEngine, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.setup() + c := &cache{ + cache: tt.fields.cache, + cfg: tt.fields.cfg, + db: tt.fields.db, + metricEngine: tt.fields.metricEngine, + } + got, err := c.GetGDPRCountryCodes() + if (err != nil) != tt.wantErr { + t.Errorf("cache.GetGDPRCountryCodes() error = %v, wantErr %v", err, tt.wantErr) + return + } + assert.Equal(t, tt.want, got, tt.name) + }) + } +} diff --git a/modules/pubmatic/openwrap/database/mysql/compliance.go b/modules/pubmatic/openwrap/database/mysql/compliance.go index 775f66e9d33..ed2ef72e36d 100644 --- a/modules/pubmatic/openwrap/database/mysql/compliance.go +++ b/modules/pubmatic/openwrap/database/mysql/compliance.go @@ -25,6 +25,7 @@ func (db *mySqlDB) GetGDPRCountryCodes() (map[string]struct{}, error) { glog.Error("ErrRowScanFailed GetGDPRCountryCodes Err: ", err.Error()) continue } + //TO-DO keeping case-sensitive? countryCodes[countryCode] = struct{}{} } diff --git a/modules/pubmatic/openwrap/database/mysql/compliance_test.go b/modules/pubmatic/openwrap/database/mysql/compliance_test.go new file mode 100644 index 00000000000..df411ed3bf2 --- /dev/null +++ b/modules/pubmatic/openwrap/database/mysql/compliance_test.go @@ -0,0 +1,156 @@ +package mysql + +import ( + "database/sql" + "errors" + "regexp" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/config" + "github.com/stretchr/testify/assert" +) + +func Test_mySqlDB_GetGDPRCountryCodes(t *testing.T) { + type fields struct { + cfg config.Database + } + tests := []struct { + name string + fields fields + want map[string]struct{} + wantErr error + setup func() *sql.DB + }{ + { + name: "empty query in config file", + fields: fields{ + cfg: config.Database{ + MaxDbContextTimeout: 100, + }, + }, + want: nil, + wantErr: errors.New("all expectations were already fulfilled, call to Query '' with args [] was not expected"), + setup: func() *sql.DB { + db, _, err := sqlmock.New() + if err != nil { + t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) + } + return db + }, + }, + { + name: "valid rows returned from DB", + fields: fields{ + cfg: config.Database{ + MaxDbContextTimeout: 100, + Queries: config.Queries{ + GetGDPRCountryCodes: "^SELECT (.+) FROM KomliAdServer.geo (.+)", + }, + }, + }, + want: map[string]struct{}{ + "DE": {}, + "LV": {}, + }, + wantErr: nil, + setup: func() *sql.DB { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) + } + rows := sqlmock.NewRows([]string{"countrycode"}). + AddRow(`DE`). + AddRow(`LV`) + mock.ExpectQuery(regexp.QuoteMeta("^SELECT (.+) FROM KomliAdServer.geo (.+)")).WillReturnRows(rows) + return db + }, + }, + { + name: "no rows returned from DB", + fields: fields{ + cfg: config.Database{ + MaxDbContextTimeout: 100, + Queries: config.Queries{ + GetGDPRCountryCodes: "^SELECT (.+) FROM KomliAdServer.geo (.+)", + }, + }, + }, + want: map[string]struct{}{}, + wantErr: nil, + setup: func() *sql.DB { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) + } + rows := sqlmock.NewRows([]string{"countrycode"}) + mock.ExpectQuery(regexp.QuoteMeta("^SELECT (.+) FROM KomliAdServer.geo (.+)")).WillReturnRows(rows) + return db + }, + }, + { + name: "partial row scan error", + fields: fields{ + cfg: config.Database{ + MaxDbContextTimeout: 1000000, + Queries: config.Queries{ + GetGDPRCountryCodes: "^SELECT (.+) FROM KomliAdServer.geo (.+)", + }, + }, + }, + want: map[string]struct{}{}, + wantErr: nil, + setup: func() *sql.DB { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) + } + rows := sqlmock.NewRows([]string{"countrycode", "extra_column"}). + AddRow(`DE`, `12`) + rows = rows.RowError(1, errors.New("error in row scan")) + mock.ExpectQuery(regexp.QuoteMeta("^SELECT (.+) FROM KomliAdServer.geo (.+)")).WillReturnRows(rows) + return db + }, + }, + { + name: "error in row scan", + fields: fields{ + cfg: config.Database{ + MaxDbContextTimeout: 100, + Queries: config.Queries{ + GetGDPRCountryCodes: "^SELECT (.+) FROM KomliAdServer.geo (.+)", + }, + }, + }, + want: nil, + wantErr: errors.New("error in row scan"), + setup: func() *sql.DB { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) + } + rows := sqlmock.NewRows([]string{"countrycode"}). + AddRow(`DE`). + AddRow(`LV`) + rows = rows.RowError(1, errors.New("error in row scan")) + mock.ExpectQuery(regexp.QuoteMeta("^SELECT (.+) FROM KomliAdServer.geo (.+)")).WillReturnRows(rows) + return db + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + db := &mySqlDB{ + conn: tt.setup(), + cfg: tt.fields.cfg, + } + got, err := db.GetGDPRCountryCodes() + if tt.wantErr == nil { + assert.NoError(t, err, tt.name) + } else { + assert.EqualError(t, err, tt.wantErr.Error(), tt.name) + } + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/modules/pubmatic/openwrap/geo.go b/modules/pubmatic/openwrap/geo.go index 13f97f1c335..b84063ec18f 100644 --- a/modules/pubmatic/openwrap/geo.go +++ b/modules/pubmatic/openwrap/geo.go @@ -10,7 +10,6 @@ import ( "git.pubmatic.com/PubMatic/go-common/util" "github.com/golang/glog" - "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/geodb/netacuity" "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/metrics" "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/metrics/prometheus" "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/models" @@ -33,18 +32,6 @@ const ( headerAccessControlAllowOriginValue = "*" ) -// geoHandler provides a handler for geo lookups. -type geoHandler struct { - geoService netacuity.NetAcuity -} - -// NewGeo initializes and returns a new GeoHandler. -func NewGeo() *geoHandler { - return &geoHandler{ - geoService: netacuity.NetAcuity{}, - } -} - var gppSectionIDs = map[string]int{ "ca": 8, "va": 9, @@ -64,7 +51,7 @@ const ( ) // Handler for /geo endpoint -func (handler *geoHandler) Handler(w http.ResponseWriter, r *http.Request) { +func Handler(w http.ResponseWriter, r *http.Request) { var pubIdStr string metricEngine := ow.GetMetricEngine() metricLabels := metrics.Labels{RType: models.EndpointGeo, RequestStatus: prometheus.RequestStatusOK} @@ -77,6 +64,11 @@ func (handler *geoHandler) Handler(w http.ResponseWriter, r *http.Request) { } }() + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + metricLabels.RequestStatus = prometheus.RequestStatusBadInput + return + } pubIdStr = r.URL.Query().Get(models.PublisherID) _, err := strconv.Atoi(pubIdStr) if err != nil { @@ -91,7 +83,7 @@ func (handler *geoHandler) Handler(w http.ResponseWriter, r *http.Request) { w.Header().Set(headerContentType, headerContentTypeValue) w.Header().Set(headerAccessControlAllowOrigin, "*") success := false - geoInfo, _ := handler.geoService.LookUp(ip) + geoInfo, _ := ow.geoInfoFetcher.LookUp(ip) if geoInfo != nil { if geoInfo.ISOCountryCode != "" { success = true diff --git a/modules/pubmatic/openwrap/geo_test.go b/modules/pubmatic/openwrap/geo_test.go new file mode 100644 index 00000000000..208cf3ede16 --- /dev/null +++ b/modules/pubmatic/openwrap/geo_test.go @@ -0,0 +1,345 @@ +package openwrap + +import ( + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/golang/mock/gomock" + "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/geodb" + mock_geodb "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/geodb/mock" + metrics "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/metrics" + mock_metrics "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/metrics/mock" + "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/metrics/prometheus" + "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/models" + mock_feature "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/publisherfeature/mock" + "github.com/stretchr/testify/assert" +) + +const ( + geoWithPubid = `http://localhost:8001/geo?pubid=23105` + geoWithoutPubid = `http://localhost:8001/geo` + geoWithWrongPubid = `http://localhost:8001/geo?pubid=bad` +) + +type headerType int + +const ( + NoHeaders headerType = iota + NonEURequest + EURequest + USPRequest + GPPCountryRequest + InvalidIP + InvalidUID + NilGeo + GeoLookupFail + EmptyGeo +) + +func getTestHTTPRequest(url string, headers http.Header) *http.Request { + r, err := http.NewRequest("GET", url, nil) + if err != nil { + panic("error: creating an http request") + } + + for k, v := range headers { + r.Header.Add(k, v[0]) + } + + if err := r.ParseForm(); err != nil { + panic("error: parsing http request") + } + return r +} + +func getHeaders(headerType headerType) http.Header { + switch headerType { + case NoHeaders: + return nil + case NonEURequest: + return http.Header{ + "User-Agent": []string{"golang sample request"}, + "Cookie": []string{"KADUSERCOOKIE=pmuserid"}, + "SOURCE_IP": []string{"115.114.134.174"}, + } + case EURequest: + return http.Header{ + "User-Agent": []string{"golang sample request"}, + "Cookie": []string{"KADUSERCOOKIE=pmuserid"}, + "SOURCE_IP": []string{"2.16.1.255"}, + } + case USPRequest: + return http.Header{ + "User-Agent": []string{"golang sample request"}, + "Cookie": []string{"KADUSERCOOKIE=pmuserid"}, + "SOURCE_IP": []string{"43.135.143.132"}, + } + case GPPCountryRequest: + return http.Header{ + "User-Agent": []string{"golang sample request"}, + "Cookie": []string{"KADUSERCOOKIE=pmuserid"}, + "SOURCE_IP": []string{"208.253.114.165"}, + } + case InvalidIP: + return http.Header{ + "User-Agent": []string{"golang sample request"}, + "Cookie": []string{"KADUSERCOOKIE=pmuserid"}, + "SOURCE_IP": []string{"115.114.134"}, + } + case InvalidUID: + return http.Header{ + "User-Agent": []string{"golang sample request"}, + "Cookie": []string{"KADUSERCOOKIE=pmuserid}"}, + } + case EmptyGeo: + return http.Header{ + "SOURCE_IP": []string{"30.30.30.30"}, + } + case GeoLookupFail: + return http.Header{ + "SOURCE_IP": []string{"20.20.20.20"}, + } + case NilGeo: + return http.Header{ + "SOURCE_IP": []string{"10.10.10.10"}, + } + } + return nil +} + +func TestHandler(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockMetrics := mock_metrics.NewMockMetricsEngine(ctrl) + mockgeodb := mock_geodb.NewMockGeography(ctrl) + mockFeature := mock_feature.NewMockFeature(ctrl) + + originalOw := ow + defer func() { ow = originalOw }() + ow = &OpenWrap{ + metricEngine: mockMetrics, + geoInfoFetcher: mockgeodb, + pubFeatures: mockFeature, + } + + type args struct { + r *http.Request + setup func() + } + type want struct { + geo string + statuCode int + header []string + } + tests := []struct { + name string + args args + setup geodb.Geography + want want + }{ + { + name: "POST request", + args: args{ + r: func() *http.Request { + r, err := http.NewRequest("POST", geoWithoutPubid, nil) + if err != nil { + panic("error: creating an http request") + } + return r + }(), + setup: func() { + labels := metrics.Labels{RType: models.EndpointGeo, RequestStatus: prometheus.RequestStatusBadInput} + mockMetrics.EXPECT().RecordRequest(labels) + }, + }, + want: want{ + geo: "", + statuCode: http.StatusMethodNotAllowed, + header: nil, + }, + }, + { + name: "invalid pubid", + args: args{ + r: getTestHTTPRequest(geoWithWrongPubid, getHeaders(NoHeaders)), + setup: func() { + labels := metrics.Labels{RType: models.EndpointGeo, RequestStatus: prometheus.RequestStatusBadInput} + mockMetrics.EXPECT().RecordRequest(labels) + }, + }, + want: want{ + geo: "", + statuCode: http.StatusBadRequest, + header: nil, + }, + }, + { + name: "pubid not present", + args: args{ + r: getTestHTTPRequest(geoWithoutPubid, getHeaders(NoHeaders)), + setup: func() { + labels := metrics.Labels{RType: models.EndpointGeo, RequestStatus: prometheus.RequestStatusBadInput} + mockMetrics.EXPECT().RecordRequest(labels) + }, + }, + want: want{ + geo: "", + statuCode: http.StatusBadRequest, + header: nil, + }, + }, + { + name: "valid pubid with no ip in headers", + args: args{ + r: getTestHTTPRequest(geoWithPubid, getHeaders(NoHeaders)), + setup: func() { + labels := metrics.Labels{RType: models.EndpointGeo, RequestStatus: prometheus.RequestStatusOK} + mockMetrics.EXPECT().RecordRequest(labels) + mockMetrics.EXPECT().RecordGeoLookupFailure(models.EndpointGeo) + mockgeodb.EXPECT().LookUp(gomock.Any()).Return(&geodb.GeoInfo{}, nil) + }, + }, + want: want{ + geo: "", + statuCode: http.StatusOK, + header: nil, + }, + }, + { + name: "ip Lookup fail", + args: args{ + r: getTestHTTPRequest(geoWithPubid, getHeaders(GeoLookupFail)), + setup: func() { + labels := metrics.Labels{RType: models.EndpointGeo, RequestStatus: prometheus.RequestStatusOK} + mockMetrics.EXPECT().RecordRequest(labels) + mockMetrics.EXPECT().RecordGeoLookupFailure(models.EndpointGeo) + mockgeodb.EXPECT().LookUp(gomock.Any()).Return(nil, errors.New("ErrDummy")) + }, + }, + want: want{ + geo: "", + statuCode: http.StatusOK, + header: nil, + }, + }, + { + name: "ip lookup returns nil", + args: args{ + r: getTestHTTPRequest(geoWithPubid, getHeaders(NilGeo)), + setup: func() { + labels := metrics.Labels{RType: models.EndpointGeo, RequestStatus: prometheus.RequestStatusOK} + mockMetrics.EXPECT().RecordRequest(labels) + mockMetrics.EXPECT().RecordGeoLookupFailure(models.EndpointGeo) + mockgeodb.EXPECT().LookUp(gomock.Any()).Return(nil, nil) + }, + }, + want: want{ + geo: "", + statuCode: http.StatusOK, + header: nil, + }, + }, + { + name: "empty countrycode", + args: args{ + r: getTestHTTPRequest(geoWithPubid, getHeaders(NonEURequest)), + setup: func() { + labels := metrics.Labels{RType: models.EndpointGeo, RequestStatus: prometheus.RequestStatusOK} + mockMetrics.EXPECT().RecordRequest(labels) + mockMetrics.EXPECT().RecordGeoLookupFailure(models.EndpointGeo) + mockgeodb.EXPECT().LookUp(gomock.Any()).Return(&geodb.GeoInfo{ISOCountryCode: ""}, nil) + }, + }, + want: want{ + geo: "", + statuCode: http.StatusOK, + header: nil, + }, + }, + { + name: "EU region request", + args: args{ + r: getTestHTTPRequest(geoWithPubid, getHeaders(EURequest)), + setup: func() { + labels := metrics.Labels{RType: models.EndpointGeo, RequestStatus: prometheus.RequestStatusOK} + mockMetrics.EXPECT().RecordRequest(labels) + mockFeature.EXPECT().IsCountryGDPREnabled(gomock.Any()).Return(true) + mockgeodb.EXPECT().LookUp(gomock.Any()). + Return(&geodb.GeoInfo{ISOCountryCode: "UK", CountryCode: "uk", RegionCode: "lnd"}, nil) + }, + }, + want: want{ + geo: "{\"cc\":\"UK\",\"sc\":\"lnd\",\"compliance\":\"GDPR\"}\n", + statuCode: http.StatusOK, + header: []string{"max-age=172800"}, + }, + }, + { + name: "non-EU region request", + args: args{ + r: getTestHTTPRequest(geoWithPubid, getHeaders(NonEURequest)), + setup: func() { + labels := metrics.Labels{RType: models.EndpointGeo, RequestStatus: prometheus.RequestStatusOK} + mockMetrics.EXPECT().RecordRequest(labels) + mockFeature.EXPECT().IsCountryGDPREnabled(gomock.Any()).Return(false) + mockgeodb.EXPECT().LookUp(gomock.Any()). + Return(&geodb.GeoInfo{ISOCountryCode: "IN", CountryCode: "in", RegionCode: "mh"}, nil) + }, + }, + want: want{ + geo: "{\"cc\":\"IN\",\"sc\":\"mh\"}\n", + statuCode: http.StatusOK, + header: []string{"max-age=172800"}, + }, + }, + { + name: "statecode is california(USP compliance)", + args: args{ + r: getTestHTTPRequest(geoWithPubid, getHeaders(EURequest)), + setup: func() { + labels := metrics.Labels{RType: models.EndpointGeo, RequestStatus: prometheus.RequestStatusOK} + mockMetrics.EXPECT().RecordRequest(labels) + mockFeature.EXPECT().IsCountryGDPREnabled(gomock.Any()).Return(false) + mockgeodb.EXPECT().LookUp(gomock.Any()). + Return(&geodb.GeoInfo{ISOCountryCode: "US", CountryCode: "us", RegionCode: "ca"}, nil) + }, + }, + want: want{ + geo: "{\"cc\":\"US\",\"sc\":\"ca\",\"compliance\":\"USP\"}\n", + statuCode: http.StatusOK, + header: []string{"max-age=172800"}, + }, + }, + { + name: "gpp country request", + args: args{ + r: getTestHTTPRequest(geoWithPubid, getHeaders(EURequest)), + setup: func() { + labels := metrics.Labels{RType: models.EndpointGeo, RequestStatus: prometheus.RequestStatusOK} + mockMetrics.EXPECT().RecordRequest(labels) + mockFeature.EXPECT().IsCountryGDPREnabled(gomock.Any()).Return(false) + mockgeodb.EXPECT().LookUp(gomock.Any()). + Return(&geodb.GeoInfo{ISOCountryCode: "US", CountryCode: "us", RegionCode: "va"}, nil) + }, + }, + want: want{ + geo: "{\"cc\":\"US\",\"sc\":\"va\",\"compliance\":\"GPP\",\"sectionId\":9}\n", + statuCode: http.StatusOK, + header: []string{"max-age=172800"}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.args.setup() + w := httptest.NewRecorder() + Handler(w, tt.args.r) + assert.Equal(t, tt.want.statuCode, w.Result().StatusCode) + assert.Equal(t, tt.want.geo, w.Body.String()) + assert.Equal(t, tt.want.header, w.Header().Values("Cache-Control")) + }) + } +} diff --git a/modules/pubmatic/openwrap/publisherfeature/compliance_test.go b/modules/pubmatic/openwrap/publisherfeature/compliance_test.go new file mode 100644 index 00000000000..7910407f36c --- /dev/null +++ b/modules/pubmatic/openwrap/publisherfeature/compliance_test.go @@ -0,0 +1,181 @@ +package publisherfeature + +import ( + "errors" + "testing" + + "github.com/golang/mock/gomock" + "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/cache" + mock_cache "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/cache/mock" + "github.com/stretchr/testify/assert" +) + +func TestFeatureUpdateGDPRCountryCodes(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockCache := mock_cache.NewMockCache(ctrl) + + type fields struct { + cache cache.Cache + gdprCountryCodes gdprCountryCodes + } + tests := []struct { + name string + fields fields + setup func() + expectedGDPRCountryCodes gdprCountryCodes + }{ + { + name: "query failed", + fields: fields{ + cache: mockCache, + gdprCountryCodes: newGDPRCountryCodes(), + }, + setup: func() { + mockCache.EXPECT().GetGDPRCountryCodes().Return(nil, errors.New("QUERY FAILED")) + }, + expectedGDPRCountryCodes: gdprCountryCodes{ + codes: [2]map[string]struct{}{ + make(map[string]struct{}), + make(map[string]struct{}), + }, + index: 1, + }, + }, + { + name: "query success", + fields: fields{ + cache: mockCache, + gdprCountryCodes: newGDPRCountryCodes(), + }, + setup: func() { + mockCache.EXPECT().GetGDPRCountryCodes().Return(map[string]struct{}{ + "US": {}, + "DE": {}, + }, nil) + }, + expectedGDPRCountryCodes: gdprCountryCodes{ + codes: [2]map[string]struct{}{ + {}, + { + "US": {}, + "DE": {}, + }, + }, + index: 1, + }, + }, + { + name: "query success toggled", + fields: fields{ + cache: mockCache, + gdprCountryCodes: gdprCountryCodes{ + codes: [2]map[string]struct{}{ + {}, + { + "US": {}, + "DE": {}, + }, + }, + index: 1, + }, + }, + setup: func() { + mockCache.EXPECT().GetGDPRCountryCodes().Return(map[string]struct{}{ + "US": {}, + "DE": {}, + }, nil) + }, + expectedGDPRCountryCodes: gdprCountryCodes{ + codes: [2]map[string]struct{}{ + { + "US": {}, + "DE": {}, + }, + { + "US": {}, + "DE": {}, + }, + }, + index: 0, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.setup() + fe := &feature{ + cache: tt.fields.cache, + gdprCountryCodes: tt.fields.gdprCountryCodes, + } + defer func() { + fe = nil + }() + fe.updateGDPRCountryCodes() + assert.Equal(t, tt.expectedGDPRCountryCodes, fe.gdprCountryCodes, tt.name) + }) + } +} + +func TestFeature_IsCountryGDPREnabled(t *testing.T) { + type fields struct { + gdprCountryCodes gdprCountryCodes + } + type args struct { + countryCode string + } + tests := []struct { + name string + fields fields + args args + want bool + }{ + { + name: "gdpr enabled for countrycode", + args: args{ + countryCode: "LV", + }, + fields: fields{ + gdprCountryCodes: gdprCountryCodes{ + codes: [2]map[string]struct{}{ + {}, + { + "LV": {}, + "DE": {}, + }, + }, + index: 1, + }, + }, + want: true, + }, + { + name: "gdpr disabled for countrycode", + args: args{ + countryCode: "IN", + }, + fields: fields{ + gdprCountryCodes: gdprCountryCodes{ + codes: [2]map[string]struct{}{ + {}, + { + "LV": {}, + "DE": {}, + }, + }, + index: 1, + }, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fe := &feature{ + gdprCountryCodes: tt.fields.gdprCountryCodes, + } + got := fe.IsCountryGDPREnabled(tt.args.countryCode) + assert.Equal(t, tt.want, got, tt.name) + }) + } +} From 75eaf847c6753a04077207ee9c3ec7cca566b430 Mon Sep 17 00:00:00 2001 From: saurabh-narkhede Date: Wed, 8 Jan 2025 12:05:05 +0530 Subject: [PATCH 6/6] minor changes --- modules/pubmatic/openwrap/geo.go | 66 ++++++++----------------- modules/pubmatic/openwrap/models/geo.go | 24 +++++++++ 2 files changed, 45 insertions(+), 45 deletions(-) create mode 100644 modules/pubmatic/openwrap/models/geo.go diff --git a/modules/pubmatic/openwrap/geo.go b/modules/pubmatic/openwrap/geo.go index b84063ec18f..72136b0064d 100644 --- a/modules/pubmatic/openwrap/geo.go +++ b/modules/pubmatic/openwrap/geo.go @@ -6,7 +6,6 @@ import ( "net/http" "runtime/debug" "strconv" - "time" "git.pubmatic.com/PubMatic/go-common/util" "github.com/golang/glog" @@ -23,15 +22,6 @@ type geo struct { SectionID int `json:"sectionId,omitempty"` } -const ( - cacheTimeout = time.Duration(48) * time.Hour - headerContentType = "Content-Type" - headerAccessControlAllowOrigin = "Access-Control-Allow-Origin" - headerCacheControl = "Cache-Control" - headerContentTypeValue = "application/json" - headerAccessControlAllowOriginValue = "*" -) - var gppSectionIDs = map[string]int{ "ca": 8, "va": 9, @@ -40,16 +30,6 @@ var gppSectionIDs = map[string]int{ "ct": 12, } -const ( - OriginHeaderKey = "Origin" - RefererKey = "Referer" - GDPRCompliance = "GDPR" - USPCompliance = "USP" - GPPCompliance = "GPP" - StateCodeCalifornia = "ca" - CountryCodeUS = "US" -) - // Handler for /geo endpoint func Handler(w http.ResponseWriter, r *http.Request) { var pubIdStr string @@ -73,39 +53,35 @@ func Handler(w http.ResponseWriter, r *http.Request) { _, err := strconv.Atoi(pubIdStr) if err != nil { glog.Errorf("[geo] error:[invalid pubid passed:%s], [requestType]:%v [url]:%v [origin]:%v [referer]:%v", err.Error(), models.EndpointGeo, - r.URL.RequestURI(), r.Header.Get(OriginHeaderKey), r.Header.Get(RefererKey)) + r.URL.RequestURI(), r.Header.Get(models.HeaderOriginKey), r.Header.Get(models.HeaderRefererKey)) w.WriteHeader(http.StatusBadRequest) metricLabels.RequestStatus = prometheus.RequestStatusBadInput return } ip := util.GetIP(r) - w.Header().Set(headerContentType, headerContentTypeValue) - w.Header().Set(headerAccessControlAllowOrigin, "*") - success := false - geoInfo, _ := ow.geoInfoFetcher.LookUp(ip) - if geoInfo != nil { - if geoInfo.ISOCountryCode != "" { - success = true - geo := geo{ - CountryCode: geoInfo.ISOCountryCode, - StateCode: geoInfo.RegionCode, - } + w.Header().Set(models.HeaderContentType, models.HeaderContentTypeValue) + w.Header().Set(models.HeaderAccessControlAllowOrigin, "*") - if ow.GetFeature().IsCountryGDPREnabled(geo.CountryCode) { - geo.Compliance = GDPRCompliance - } else if geo.CountryCode == CountryCodeUS && geo.StateCode == StateCodeCalifornia { - geo.Compliance = USPCompliance - } else if sectionid, ok := gppSectionIDs[geo.StateCode]; ok { - geo.Compliance = GPPCompliance - geo.SectionID = sectionid - } + geoInfo, _ := ow.geoInfoFetcher.LookUp(ip) + if geoInfo == nil || geoInfo.ISOCountryCode == "" { + metricEngine.RecordGeoLookupFailure(models.EndpointGeo) + return + } - w.Header().Set(headerCacheControl, "max-age="+fmt.Sprint(cacheTimeout.Seconds())) - json.NewEncoder(w).Encode(geo) - } + geo := geo{ + CountryCode: geoInfo.ISOCountryCode, + StateCode: geoInfo.RegionCode, } - if !success { - metricEngine.RecordGeoLookupFailure(models.EndpointGeo) + if ow.GetFeature().IsCountryGDPREnabled(geo.CountryCode) { + geo.Compliance = models.GDPRCompliance + } else if geo.CountryCode == models.CountryCodeUS && geo.StateCode == models.StateCodeCalifornia { + geo.Compliance = models.USPCompliance + } else if sectionid, ok := gppSectionIDs[geo.StateCode]; ok { + geo.Compliance = models.GPPCompliance + geo.SectionID = sectionid } + + w.Header().Set(models.HeaderCacheControl, "max-age="+fmt.Sprint(models.CacheTimeout.Seconds())) + json.NewEncoder(w).Encode(geo) } diff --git a/modules/pubmatic/openwrap/models/geo.go b/modules/pubmatic/openwrap/models/geo.go new file mode 100644 index 00000000000..c1d8dac4a28 --- /dev/null +++ b/modules/pubmatic/openwrap/models/geo.go @@ -0,0 +1,24 @@ +package models + +import "time" + +// compliance consts +const ( + GDPRCompliance = "GDPR" + USPCompliance = "USP" + GPPCompliance = "GPP" + CountryCodeUS = "US" + StateCodeCalifornia = "ca" +) + +// headers const +const ( + CacheTimeout = time.Duration(48) * time.Hour + HeaderContentType = "Content-Type" + HeaderAccessControlAllowOrigin = "Access-Control-Allow-Origin" + HeaderCacheControl = "Cache-Control" + HeaderContentTypeValue = "application/json" + HeaderAccessControlAllowOriginValue = "*" + HeaderOriginKey = "Origin" + HeaderRefererKey = "Referer" +)