Merge pull request #247 from NYPL/SCC-3099

Scc 3099 Include Checkin cards

README.md

@@ -98,7 +98,7 @@ npm test
 
 ### Adding fixtures
 
-Almost all HTTP dependencies are rerouted to fixtures (except for requesting nypl-core mapping files). All fixtures can be updated dynamically (using creds in `./.env`) via the following:
+Almost all HTTP dependencies are rerouted to fixtures (except for requesting nypl-core mapping files). All fixtures can be updated dynamically (using creds in `./config/production.env`) via the following:
 
 Run tests and automatically build any missing Elasticsearch or SCSB fixtures:
 
@@ -231,4 +231,4 @@ Note that `page=` is not supported for aggregations (ES doesn't seem to offer a
 
 ### Features
 
-There is currently one feature flag in this app, which is 'no-on-site-edd'. When it is set, all onsite items have an eddRequestable property of false. 
+There is currently one feature flag in this app, which is 'no-on-site-edd'. When it is set, all onsite items have an eddRequestable property of false.

config/production.env

@@ -1,5 +1,5 @@
 SCSB_URL=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAH0wewYJKoZIhvcNAQcGoG4wbAIBADBnBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDPYxpIZG6Hz0ZHQlIQIBEIA6uaSyPyQSmODwefw9OLGpxwG/87z94WCcyGQhc9SPjx3fkKKJsZFVvXAxdGAPNu5hDICTWimWu50V2A==
-SCSCB_API_KEY=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAIMwgYAGCSqGSIb3DQEHBqBzMHECAQAwbAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAw8tglwVzGKBduDD9wCARCAP4biSz13FvZVHyQ8LKCb0+uLcKUKmzWqC5abVJI0kTmQJvjr9ViHsuP9/qj94Y8E7K96sb+fn0+HZk8So6CssA==
+SCSB_API_KEY=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAIMwgYAGCSqGSIb3DQEHBqBzMHECAQAwbAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAw8tglwVzGKBduDD9wCARCAP4biSz13FvZVHyQ8LKCb0+uLcKUKmzWqC5abVJI0kTmQJvjr9ViHsuP9/qj94Y8E7K96sb+fn0+HZk8So6CssA==
 ELASTICSEARCH_HOST=search-discovery-api-production-wio7hqrai645zhzi2cvpitpu6q.us-east-1.es.amazonaws.com
 RESOURCES_INDEX=resources-2018-04-09
 NYPL_OAUTH_URL=https://isso.nypl.org/

lib/kms-helper.js

@@ -0,0 +1,15 @@
+const KMS = require('aws-sdk/clients/kms')
+
+function decrypt (encrypted) {
+  return new Promise((resolve, reject) => {
+    const kms = new KMS({ region: 'us-east-1' })
+    kms.decrypt({ CiphertextBlob: Buffer.from(encrypted, 'base64') }, (err, data) => {
+      if (err) return reject(err)
+
+      const decrypted = data.Plaintext.toString('ascii')
+      resolve(decrypted)
+    })
+  })
+}
+
+module.exports = { decrypt }

lib/resources.js

@@ -52,8 +52,7 @@ const SORT_FIELDS = {
 // Configure search scopes:
 const SEARCH_SCOPES = {
   all: {
-    fields:
-    [
+    fields: [
       'title^5',
       'title.folded^2',
       'description.folded',
@@ -78,8 +77,7 @@ const SEARCH_SCOPES = {
     ]
   },
   title: {
-    fields:
-    [
+    fields: [
       'title^5',
       'title.folded^2',
       'titleAlt.folded',
@@ -171,7 +169,8 @@ var parseSearchParams = function (params) {
     isbn: { type: 'string' },
     issn: { type: 'string' },
     lccn: { type: 'string' },
-    oclc: { type: 'string' }
+    oclc: { type: 'string' },
+    merge_checkin_card_items: { type: 'boolean', default: false }
   })
 }
 
@@ -195,11 +194,11 @@ module.exports = function (app, _private = null) {
         }
       },
       _source: {
-        excludes: EXCLUDE_FIELDS.concat('items')
+        excludes: EXCLUDE_FIELDS.concat(util.itemsAndOrHoldings(params.merge_checkin_card_items))
       }
     }
 
-    body = params.itemUri ? addItemQuery(body, params.itemUri) : addInnerHits(body, { size: params.items_size, from: params.items_from })
+    body = params.itemUri ? addItemQuery(body, params.itemUri) : addInnerHits(body, { size: params.items_size, from: params.items_from, merge_checkin_card_items: params.merge_checkin_card_items })
 
     app.logger.debug('Resources#findByUri', body)
     return app.esClient.search({
@@ -274,7 +273,7 @@ module.exports = function (app, _private = null) {
       index: RESOURCES_INDEX,
       body: body
 
-    // In lieu of indexing items as `nested` so that we can do proper nested queries, let's flatten down to items here:
+      // In lieu of indexing items as `nested` so that we can do proper nested queries, let's flatten down to items here:
     }).then((resp) => {
       if (!resp || !resp.hits || resp.hits.total === 0) return Promise.reject('No matching items')
       resp = new LocationLabelUpdater(resp).responseWithUpdatedLabels()
@@ -313,15 +312,15 @@ module.exports = function (app, _private = null) {
       { terms: { 'items.identifier': identifierValues } },
       { _source: ['uri', 'type', 'items.uri', 'items.type', 'items.identifier', 'items.holdingLocation', 'items.status', 'items.catalogItemType', 'items.accessMessage'] }
 
-    // Filter out any items (multi item bib) that don't match one of the queriered barcodes:
+      // Filter out any items (multi item bib) that don't match one of the queriered barcodes:
     ).then((items) => {
       return items.filter((item) => {
         return item.identifier.filter((i) => identifierValues.indexOf(i) >= 0).length > 0
       })
     })
 
     // Run both item fetch and patron fetch in parallel:
-    return Promise.all([ fetchItems, lookupPatronType ])
+    return Promise.all([fetchItems, lookupPatronType])
       .then((resp) => {
         // The resolved values of Promise.all are strictly ordered based on original array of promises
         let items = resp[0]
@@ -351,7 +350,8 @@ module.exports = function (app, _private = null) {
   const addInnerHits = (body, _options = {}) => {
     const options = Object.assign({
       size: process.env.SEARCH_ITEMS_SIZE || 200,
-      from: 0
+      from: 0,
+      merge_checkin_card_items: false
     }, _options)
 
     // The place to add the filter depends on the query built to this point:
@@ -361,6 +361,17 @@ module.exports = function (app, _private = null) {
     // If filter object already exists, convert it to array:
     if (!Array.isArray(placeToAddFilter.filter)) placeToAddFilter.filter = [placeToAddFilter.filter]
 
+    // If merge_checkin_card_items is true, those will be included with regular items. Otherwise they are excluded (default behavior)
+    const query = options.merge_checkin_card_items ? { match_all: {} } : {
+      'bool': {
+        'must_not': {
+          'term': {
+            'items.type': 'nypl:CheckinCardItem'
+          }
+        }
+      }
+    }
+
     placeToAddFilter.filter.push({
       bool: {
         // Add a boolean OR matching on either:
@@ -371,7 +382,7 @@ module.exports = function (app, _private = null) {
           {
             nested: {
               path: 'items',
-              query: { match_all: {} },
+              query,
               inner_hits: {
                 sort: [{ 'items.shelfMark_sort': 'asc' }],
                 size: options.size,
@@ -415,11 +426,11 @@ module.exports = function (app, _private = null) {
       )
     // Strip unnecessary _source fields
     body._source = {
-      excludes: EXCLUDE_FIELDS.concat(shouldAddInnerHits ? ['items'] : [])
+      excludes: EXCLUDE_FIELDS.concat(shouldAddInnerHits ? util.itemsAndOrHoldings(params.merge_checkin_card_items) : [])
     }
 
     if (shouldAddInnerHits) {
-      body = addInnerHits(body)
+      body = addInnerHits(body, { merge_checkin_card_items: params['merge_checkin_card_items'] })
     }
 
     app.logger.debug('Resources#search', RESOURCES_INDEX, body)
@@ -529,7 +540,7 @@ module.exports = function (app, _private = null) {
   app.resources.byTerm = function (id, cb) {
     app.db.returnCollectionTripleStore('resources', function (err, resources) {
       if (err) throw err
-      resources.find({allTerms: parseInt(id)}).limit(100).toArray(function (err, results) {
+      resources.find({ allTerms: parseInt(id) }).limit(100).toArray(function (err, results) {
         if (err) throw err
         if (results.length === 0) {
           cb(false)
@@ -584,7 +595,7 @@ const buildElasticBody = function (params) {
       body.min_score = 0.65
       body.sort = ['_score']
 
-    // just filters: no score
+      // just filters: no score
     } else {
       body.query = query
     }
@@ -871,10 +882,14 @@ const buildElasticQueryForFilters = function (params) {
           // Figure out the base property (e.g. 'owner')
           var baseField = config.field.replace(/_packed$/, '')
           // Allow supplied val to match against either id or value:
-          return { bool: { should: [
-            { term: { [`${baseField}.id`]: value } },
-            { term: { [`${baseField}.label`]: value } }
-          ] } }
+          return {
+            bool: {
+              should: [
+                { term: { [`${baseField}.id`]: value } },
+                { term: { [`${baseField}.label`]: value } }
+              ]
+            }
+          }
         } else if (config.operator === 'match') return { term: { [config.field]: value } }
       }
 
@@ -931,18 +946,18 @@ const buildElasticQuery = function (params) {
   // Build ES query:
   var query = {}
 
-  // clean up params
-  ;['q'].forEach(function (param) {
-    if (params[param]) {
-      params[param] = params[param].replace(/date:/g, 'dateStartYear:')
-      params[param] = params[param].replace(/location:/g, 'locations:')
-      params[param] = params[param].replace(/subject:/g, 'subjectLiteral:')
-    }
-  })
+    // clean up params
+    ;['q'].forEach(function (param) {
+      if (params[param]) {
+        params[param] = params[param].replace(/date:/g, 'dateStartYear:')
+        params[param] = params[param].replace(/location:/g, 'locations:')
+        params[param] = params[param].replace(/subject:/g, 'subjectLiteral:')
+      }
+    })
 
   if (params.q) {
     // Merge keyword-specific ES query into the query we're building:
-    query.bool = { must: [ buildElasticQueryForKeywords(params) ] }
+    query.bool = { must: [buildElasticQueryForKeywords(params)] }
   }
 
   const advancedParamQueries = ADVANCED_SEARCH_PARAMS

lib/util.js

@@ -21,8 +21,8 @@ exports.context = (which) => {
   var contexts = all.indexOf(which) < 0 ? all : [which]
   return Promise.all(
     contexts
-    .map((f) => `./data/contexts/${f}.json`)
-    .map(readJson)
+      .map((f) => `./data/contexts/${f}.json`)
+      .map(readJson)
   ).then((contexts) => {
     contexts = contexts.map((c) => c['@context'])
     var statements = Object.assign.apply(null, [{}].concat(contexts))
@@ -95,7 +95,7 @@ exports.eachValue = function (a, cb) {
 }
 
 exports.flatenTriples = function (object) {
-  var flat = {objectLiteral: {}, objectUri: {}}
+  var flat = { objectLiteral: {}, objectUri: {} }
   for (var key in object) {
     // is this a triple
     if (config['predicatesAgents'].indexOf(key) > -1 || config['predicatesResources'].indexOf(key) > -1) {
@@ -109,7 +109,7 @@ exports.flatenTriples = function (object) {
           flat.objectUri[key].push(value.objectUri)
           if (value.label) {
             if (!flat.objectUri[key + ':label']) flat.objectUri[key + ':label'] = []
-            flat.objectUri[key + ':label'].push({uri: value.objectUri, label: value.label})
+            flat.objectUri[key + ':label'].push({ uri: value.objectUri, label: value.label })
           }
         }
       })
@@ -184,7 +184,7 @@ exports.returnNtTriples = function (obj, type) {
 exports.returnNtJsonLd = function (obj, type, cb) {
   var triples = exports.returnNtTriples(obj, type).join('\n')
   // console.log(triples)
-  jsonld.fromRDF(triples, {format: 'application/nquads'}, function (err, doc) {
+  jsonld.fromRDF(triples, { format: 'application/nquads' }, function (err, doc) {
     if (err) console.log(JSON.stringify(err, null, 2))
     jsonld.compact(doc, exports.context, function (err, compacted) {
       if (err) console.log(err)
@@ -209,11 +209,11 @@ exports.parseParams = function (params, spec) {
       if ((typeof parsed) !== 'undefined') {
         ret[param] = parsed
 
-      // If not valid, fall back on default, if specified:
+        // If not valid, fall back on default, if specified:
       } else if (spec[param].default) {
         ret[param] = spec[param].default
       }
-    } else if (spec[param].default) {
+    } else if (spec[param].default !== undefined) {
       ret[param] = spec[param].default
     }
   })
@@ -222,17 +222,17 @@ exports.parseParams = function (params, spec) {
 
 // Given raw query param value `val`
 // returns value validated against supplied spec:
-//   `type`: (int, string, date, object) - Type to validate (and cast) against
+//   `type`: (int, string, date, object, boolean) - Type to validate (and cast) against
 //   `range`: Array - Constrain allowed values to range
 //   `default`: (mixed) - Return this if value missing
 //   `fields`: Hash - When `type` is 'hash', this property provides field spec to validate internal fields against
 //   `repeatable`: Boolean - If true, array of values may be returned. Otherwise will select last. Default false
 exports.parseParam = function (val, spec) {
   if (spec.fields &&
-      spec.fields.subjectLiteral &&
-      spec.fields.subjectLiteral.field === 'subjectLiteral_exploded' &&
-      val.subjectLiteral
-    ) {
+    spec.fields.subjectLiteral &&
+    spec.fields.subjectLiteral.field === 'subjectLiteral_exploded' &&
+    val.subjectLiteral
+  ) {
     if (typeof val.subjectLiteral === 'string' && val.subjectLiteral.slice(-1) === '.') {
       val.subjectLiteral = val.subjectLiteral.slice(0, -1)
       logger.debug('Removing terminal period', JSON.stringify(val, null, 4))
@@ -266,6 +266,9 @@ exports.parseParam = function (val, spec) {
     case 'hash':
       val = exports.parseParams(val, spec.fields || {})
       break
+    case 'boolean':
+      if (val === 'true' || val === 'false') return val === 'true'
+      break
   }
 
   if (spec.range) {
@@ -293,13 +296,13 @@ exports.arrayIntersection = (a1, a2) => {
  */
 exports.objectEntries = (obj) => {
   return Object.keys(obj)
-    .map((key) => [ key, obj[key] ])
+    .map((key) => [key, obj[key]])
 }
 
 exports.gatherParams = function (req, acceptedParams) {
   // If specific params configured, pass those to handler
   // otherwise just pass `value` param (i.e. keyword search)
-  acceptedParams = (typeof acceptedParams === 'undefined') ? ['page', 'per_page', 'value', 'q', 'filters', 'contributor', 'subject', 'title', 'isbn', 'issn', 'lccn', 'oclc'] : acceptedParams
+  acceptedParams = (typeof acceptedParams === 'undefined') ? ['page', 'per_page', 'value', 'q', 'filters', 'contributor', 'subject', 'title', 'isbn', 'issn', 'lccn', 'oclc', 'merge_checkin_card_items'] : acceptedParams
 
   var params = {}
   acceptedParams.forEach((k) => {
@@ -335,3 +338,7 @@ exports.backslashes = (phrase, count = 1) => {
       .join('')
   }
 }
+
+exports.itemsAndOrHoldings = (merge_checkin_card_items) => {
+  return merge_checkin_card_items ? ['items', 'holdings'] : ['items']
+}