Merge pull request #1450 from pelias/blackmad-higher-levels-es-debug

Add new debugging levels

README.md

@@ -70,8 +70,7 @@ The API recognizes the following properties under the top-level `api` key in you
 |`attributionURL`|no| (autodetected)|The full URL to use for the attribution link returned in all Pelias responses. Pelias will attempt to autodetect this host, but it will often be incorrect if, for example, there is a proxy between Pelias and its users. This parameter allows setting a specific URL to avoid any such issues|
 |`accessLog`|*no*||name of the format to use for access logs; may be any one of the [predefined values](https://github.com/expressjs/morgan#predefined-formats) in the `morgan` package. Defaults to `"common"`; if set to `false`, or an otherwise falsy value, disables access-logging entirely.|
 |`relativeScores`|*no*|true|if set to true, confidence scores will be normalized, realistically at this point setting this to false is not tested or desirable
-|`serveCompareFrontend`|*no*|true|if set to true, the [pelias compare frontend](https://github.com/pelias/compare) will be served from /frontend, pointing at the local instance, to aid in on-server debugging. [example](https://pelias.github.io/compare/)
-
+|`exposeInternalDebugTools`|*no*|true|Exposes several debugging tools, such as the ability to enable Elasticsearch explain mode, that may come at a performance cost or expose sensitive infrastructure details. Not recommended if the Pelias API is open to the public.
 
 A good starting configuration file includes this section (fill in the service and Elasticsearch hosts as needed):
 

controller/search.js

@@ -53,6 +53,10 @@ function setup( apiConfig, esclient, query, should_execute ){
       body: renderedQuery.body
     };
 
+    if (req.clean.enableElasticExplain) {
+      cmd.explain = true;
+    }
+
     logger.debug( '[ES req]', cmd );
     debugLog.push(req, {ES_req: cmd});
 
@@ -126,6 +130,9 @@ function setup( apiConfig, esclient, query, should_execute ){
           }});
         }
         logger.debug('[ES response]', docs);
+        if (req.clean.enableElasticDebug) {
+          debugLog.push(req, {ES_response: {docs, meta, data}});
+        }
         next();
       });
       debugLog.stopTimer(req, initialTime);

routes/v1.js

@@ -213,7 +213,7 @@ function addRoutes(app, peliasConfig) {
       // try 3 different query types: address search using ids, cascading fallback, pelias parser
       controllers.search(peliasConfig.api, esclient, queries.address_search_using_ids, searchWithIdsShouldExecute),
       controllers.search(peliasConfig.api, esclient, queries.search, fallbackQueryShouldExecute),
-      sanitizers.defer_to_pelias_parser(shouldDeferToPeliasParser), //run additional sanitizers needed for pelias parser
+      sanitizers.defer_to_pelias_parser(peliasConfig.api, shouldDeferToPeliasParser), //run additional sanitizers needed for pelias parser
       controllers.search(peliasConfig.api, esclient, queries.search_pelias_parser, searchPeliasParserShouldExecute),
       middleware.trimByGranularity(),
       middleware.distance('focus.point.'),
@@ -273,7 +273,7 @@ function addRoutes(app, peliasConfig) {
       middleware.sendJSON
     ]),
     reverse: createRouter([
-      sanitizers.reverse.middleware,
+      sanitizers.reverse.middleware(peliasConfig.api),
       middleware.requestLanguage,
       middleware.sizeCalculator(2),
       controllers.search(peliasConfig.api, esclient, queries.reverse, nonCoarseReverseShouldExecute),
@@ -294,7 +294,7 @@ function addRoutes(app, peliasConfig) {
       middleware.sendJSON
     ]),
     nearby: createRouter([
-      sanitizers.nearby.middleware,
+      sanitizers.nearby.middleware(peliasConfig.api),
       middleware.requestLanguage,
       middleware.sizeCalculator(),
       controllers.search(peliasConfig.api, esclient, queries.reverse, not(hasResponseDataOrRequestErrors)),
@@ -314,7 +314,7 @@ function addRoutes(app, peliasConfig) {
       middleware.sendJSON
     ]),
     place: createRouter([
-      sanitizers.place.middleware,
+      sanitizers.place.middleware(peliasConfig.api),
       middleware.requestLanguage,
       controllers.place(peliasConfig.api, esclient),
       middleware.accuracy(),
@@ -348,7 +348,7 @@ function addRoutes(app, peliasConfig) {
   app.get ( base + 'reverse',              routers.reverse );
   app.get ( base + 'nearby',               routers.nearby );
 
-  if (peliasConfig.api.serveCompareFrontend) {
+  if (peliasConfig.api.exposeInternalDebugTools) {
     app.use ( '/frontend',                   express.static('node_modules/pelias-compare/dist-api/'));
   }
 }

sanitizer/_debug.js

@@ -1,23 +1,61 @@
-var _ = require('lodash');
+const _ = require('lodash');
 
-function _sanitize(raw, clean){
-  const messages = {errors: [], warnings: []};
+/**
+* @param {object} exposeInternalDebugTools property of pelias config
+*/
+function _setup(exposeInternalDebugTools) {
+  return {
+    sanitize: (raw, clean) => {
+      const messages = { errors: [], warnings: [] };
 
-  if(!_.isUndefined(raw.debug) ){
-    clean.enableDebug = (typeof raw.debug === 'string') ? isTruthy(raw.debug.toLowerCase()) : isTruthy( raw.debug );
-  }
-  return messages;
-}
+      if (_.isUndefined(raw.debug)) {
+        return messages;
+      }
 
-function _expected() {
-  return [{ name: 'debug' }];
-}
+      clean.enableDebug = false;
+
+      if (_.isEqual(raw.debug, {})) {
+        return messages;
+      }
+
+      const debugStr = raw.debug.toString().toLowerCase();
+
+      const debugLevelMapping = {
+        'false': 0,
+        'true': 1,
+        'elastic': 2,
+        'explain': 3
+      };
+
+      const numericDebugStr = Number(debugStr);
+
+      const debugLevel = isNaN(numericDebugStr) ? debugLevelMapping[debugStr] : numericDebugStr;
+
+      if (_.isNil(debugLevel)) {
+        messages.errors.push('Unknown debug value: ' + debugStr);
+      }
+
+      if (debugLevel >= 2 && !exposeInternalDebugTools) {
+        messages.errors.push('Debug level not enabled: ' + debugStr);
+      } else {
+        if (debugLevel >= 1) {
+          clean.enableDebug = true;
+        }
+        if (debugLevel >= 2) {
+          clean.enableElasticDebug = true;
+        }
+        if (debugLevel >= 3) {
+          clean.enableElasticExplain = true;
+        }
+      }
+
+      return messages;
+    },
 
-function isTruthy(val) {
-  return _.includes( ['true', '1', 1, true], val );
+    expected: () => {
+      return [{ name: 'debug' }];
+    },
+  };
 }
 
-module.exports = () => ({
-  sanitize: _sanitize,
-  expected: _expected
-});
+module.exports = _setup;

sanitizer/autocomplete.js

@@ -5,7 +5,7 @@ var sanitizeAll = require('../sanitizer/sanitizeAll');
 module.exports.middleware = (_api_pelias_config) => {
   var sanitizers = {
       singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(),
-      debug: require('../sanitizer/_debug')(),
+      debug: require('../sanitizer/_debug')(_api_pelias_config.exposeInternalDebugTools),
       text: require('../sanitizer/_text_pelias_parser')(),
       tokenizer: require('../sanitizer/_tokenizer')(),
       size: require('../sanitizer/_size')(/* use defaults*/),

sanitizer/defer_to_pelias_parser.js

@@ -1,14 +1,12 @@
-const sanitizeAll = require('../sanitizer/sanitizeAll'),
-    sanitizers = {
-      debug: require('../sanitizer/_debug')(),
-      text: require('../sanitizer/_text_pelias_parser')()
-    };
+// middleware
+module.exports = (_api_pelias_config, should_execute) => {
+  const sanitizeAll = require('../sanitizer/sanitizeAll'),
+  sanitizers = {
+    debug: require('../sanitizer/_debug')(_api_pelias_config.exposeInternalDebugTools),
+    text: require('../sanitizer/_text_pelias_parser')()
+  };
 
-const logger = require('pelias-logger').get('api');
-const logging = require( '../helper/logging' );
 
-// middleware
-module.exports = (should_execute) => {
   return function(req, res, next) {
     // if res.data already has results then don't call the _text_autocomplete sanitizer
     // this has been put into place for when the libpostal integration way of querying

sanitizer/nearby.js

@@ -1,25 +1,26 @@
 var sanitizeAll = require('../sanitizer/sanitizeAll');
 var type_mapping = require('../helper/type_mapping');
 
-// add categories to the sanitizer list
-var sanitizers = {
-  singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(),
-  debug: require('../sanitizer/_debug')(),
-  layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping),
-  sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping),
-  // depends on the layers and sources sanitizers, must be run after them
-  sources_and_layers: require('../sanitizer/_sources_and_layers')(),
-  geonames_deprecation: require('../sanitizer/_geonames_deprecation')(),
-  size: require('../sanitizer/_size')(/* use defaults*/),
-  private: require('../sanitizer/_flag_bool')('private', false),
-  geo_reverse: require('../sanitizer/_geo_reverse')(),
-  boundary_country: require('../sanitizer/_boundary_country')(),
-  categories: require('../sanitizer/_categories')(),
-  request_language: require('../sanitizer/_request_language')()
-};
+module.exports.middleware = (_api_pelias_config) => {
+  // add categories to the sanitizer list
+  var sanitizers = {
+    singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(),
+    debug: require('../sanitizer/_debug')(_api_pelias_config.exposeInternalDebugTools),
+    layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping),
+    sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping),
+    // depends on the layers and sources sanitizers, must be run after them
+    sources_and_layers: require('../sanitizer/_sources_and_layers')(),
+    geonames_deprecation: require('../sanitizer/_geonames_deprecation')(),
+    size: require('../sanitizer/_size')(/* use defaults*/),
+    private: require('../sanitizer/_flag_bool')('private', false),
+    geo_reverse: require('../sanitizer/_geo_reverse')(),
+    boundary_country: require('../sanitizer/_boundary_country')(),
+    categories: require('../sanitizer/_categories')(),
+    request_language: require('../sanitizer/_request_language')()
+  };
 
-// middleware
-module.exports.middleware = function( req, res, next ){
-  sanitizeAll.runAllChecks(req, sanitizers);
-  next();
-};
+  return function( req, res, next ){
+    sanitizeAll.runAllChecks(req, sanitizers);
+    next();
+  };
+};

sanitizer/place.js

@@ -1,15 +1,16 @@
-var sanitizeAll = require('../sanitizer/sanitizeAll'),
-    sanitizers = {
-      singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(),
-      debug: require('../sanitizer/_debug')(),
-      ids: require('../sanitizer/_ids')(),
-      private: require('../sanitizer/_flag_bool')('private', false),
-      categories: require('../sanitizer/_categories')(true),
-      request_language: require('../sanitizer/_request_language')()
-    };
+module.exports.middleware = (_api_pelias_config) => {
+  var sanitizeAll = require('../sanitizer/sanitizeAll'),
+      sanitizers = {
+        singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(),
+        debug: require('../sanitizer/_debug')(_api_pelias_config.exposeInternalDebugTools),
+        ids: require('../sanitizer/_ids')(),
+        private: require('../sanitizer/_flag_bool')('private', false),
+        categories: require('../sanitizer/_categories')(true),
+        request_language: require('../sanitizer/_request_language')()
+      };
 
-// middleware
-module.exports.middleware = function(req, res, next){
-  sanitizeAll.runAllChecks(req, sanitizers);
-  next();
-};
+  return function(req, res, next){
+    sanitizeAll.runAllChecks(req, sanitizers);
+    next();
+  };
+};

sanitizer/reverse.js

@@ -1,23 +1,25 @@
-var type_mapping = require('../helper/type_mapping');
-var sanitizeAll = require('../sanitizer/sanitizeAll'),
-    sanitizers = {
-      singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(),
-      debug: require('../sanitizer/_debug')(),
-      layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping),
-      sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping),
-      // depends on the layers and sources sanitizers, must be run after them
-      sources_and_layers: require('../sanitizer/_sources_and_layers')(),
-      geonames_deprecation: require('../sanitizer/_geonames_deprecation')(),
-      size: require('../sanitizer/_size')(/* use defaults*/),
-      private: require('../sanitizer/_flag_bool')('private', false),
-      geo_reverse: require('../sanitizer/_geo_reverse')(),
-      boundary_country: require('../sanitizer/_boundary_country')(),
-      request_language: require('../sanitizer/_request_language')(),
-      boundary_gid: require('../sanitizer/_boundary_gid')()
-    };
+module.exports.middleware = (_api_pelias_config) => {
+  var type_mapping = require('../helper/type_mapping');
+  var sanitizeAll = require('../sanitizer/sanitizeAll'),
+      sanitizers = {
+        singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(),
+        debug: require('../sanitizer/_debug')(_api_pelias_config.exposeInternalDebugTools),
+        layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping),
+        sources: require('../sanitizer/_targets')('sources', type_mapping.source_mapping),
+        // depends on the layers and sources sanitizers, must be run after them
+        sources_and_layers: require('../sanitizer/_sources_and_layers')(),
+        geonames_deprecation: require('../sanitizer/_geonames_deprecation')(),
+        size: require('../sanitizer/_size')(/* use defaults*/),
+        private: require('../sanitizer/_flag_bool')('private', false),
+        geo_reverse: require('../sanitizer/_geo_reverse')(),
+        boundary_country: require('../sanitizer/_boundary_country')(),
+        request_language: require('../sanitizer/_request_language')(),
+        boundary_gid: require('../sanitizer/_boundary_gid')()
+      };
 
-// middleware
-module.exports.middleware = function( req, res, next ){
-  sanitizeAll.runAllChecks(req, sanitizers);
-  next();
-};
+  // middleware
+  return function( req, res, next ){
+    sanitizeAll.runAllChecks(req, sanitizers);
+    next();
+  };
+};

sanitizer/search.js

@@ -4,7 +4,7 @@ var sanitizeAll = require('../sanitizer/sanitizeAll');
 module.exports.middleware = (_api_pelias_config) => {
   var sanitizers = {
         singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(),
-        debug: require('../sanitizer/_debug')(),
+        debug: require('../sanitizer/_debug')(_api_pelias_config.exposeInternalDebugTools),
         text: require('../sanitizer/_text')(),
         size: require('../sanitizer/_size')(/* use defaults*/),
         layers: require('../sanitizer/_targets')('layers', type_mapping.layer_mapping),

sanitizer/structured_geocoding.js

@@ -5,7 +5,7 @@ var sanitizeAll = require('../sanitizer/sanitizeAll');
 module.exports.middleware = (_api_pelias_config) => {
   var sanitizers = {
         singleScalarParameters: require('../sanitizer/_single_scalar_parameters')(),
-        debug: require('../sanitizer/_debug')(),
+        debug: require('../sanitizer/_debug')(_api_pelias_config.exposeInternalDebugTools),
         synthesize_analysis: require('../sanitizer/_synthesize_analysis')(),
         iso2_to_iso3: require('../sanitizer/_iso2_to_iso3')(),
         city_name_standardizer: require('../sanitizer/_city_name_standardizer')(),

schema.js

@@ -11,7 +11,7 @@ const Joi = require('@hapi/joi');
 // optional:
 // * api.accessLog (string)
 // * api.relativeScores (boolean)
-// * api.serveCompareFrontend (boolean)
+// * api.exposeInternalDebugTools (boolean)
 // * api.localization (flipNumberAndStreetCountries is array of 3 character strings)
 module.exports = Joi.object().keys({
   api: Joi.object().required().keys({
@@ -20,7 +20,7 @@ module.exports = Joi.object().keys({
     host: Joi.string(),
     accessLog: Joi.string().allow(''),
     relativeScores: Joi.boolean(),
-    serveCompareFrontend: Joi.boolean(),
+    exposeInternalDebugTools: Joi.boolean(),
     requestRetries: Joi.number().integer().min(0),
     customBoosts: Joi.object().keys({
       layer: Joi.object(),