Merge pull request nightscout#1 from mddub/v1.0.0

Represent pump status as 'device status' entry

README.md

@@ -19,7 +19,7 @@ The easiest installation mode is to set up an instance of Nightscout [cgm-remote
 
 ## Installation on Heroku
 
-Another turnkey installation option is to run this on a Heroku worker dyno. You may find this more affordable than Azure. Follow the [Share2 Bridge instructions for Heroku], substituting this repo for `share2nightscout-bridge`.
+Another turnkey installation option is to run this on a Heroku worker dyno. You may find this more reliable than Azure. Follow the [Share2 Bridge instructions for Heroku], substituting this repo for `share2nightscout-bridge`.
 
 [![Deploy](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy)
 
@@ -66,8 +66,10 @@ var client = mmcns.carelink.Client({username: 'username', password: 'password'})
 client.fetch(function(err, data) {
   if (!err) {
     var transformed = mmcns.transform(data);
-    mmcns.nightscout.upload(transformed, 'https://your.ns.host/api/v1/entries.json', 'api-secret', callback);
-    // ...or use `transformed` directly
+    mmcns.nightscout.upload(transformed.entries, 'https://your.ns.host/api/v1/entries.json', 'api-secret', callback);
+    // ...or:
+    mmcns.nightscout.upload(transformed.devicestatus, 'https://your.ns.host/api/v1/devicestatus.json', 'api-secret', callback);
+    // ...or use `transformed.entries` and `transformed.devicestatus` directly
   }
 });
 ```

carelink.js

@@ -3,8 +3,7 @@
 
 var _ = require('lodash'),
   common = require('common'),
-  request = require('request'),
-  zlib = require('zlib');
+  request = require('request');
 
 var logger = require('./logger');
 
@@ -129,11 +128,13 @@ var Client = exports.Client = function (options) {
   }
 
   function parseData(response, next) {
+    var parsed;
     try {
-      next(null, JSON.parse(response.body));
+      parsed = JSON.parse(response.body);
     } catch (e) {
       next(e);
     }
+    next(null, parsed);
   }
 
   function firstFetch(callback) {

filter.js

@@ -0,0 +1,24 @@
+/* jshint node: true */
+"use strict";
+
+// Returns a stateful filter which remembers the time of the last-seen entry to
+// prevent uploading duplicates.
+function makeRecencyFilter(timeFn) {
+  var lastTime = 0;
+
+  return function(items) {
+    var out = [];
+    items.forEach(function(item) {
+      if (timeFn(item) > lastTime) {
+        out.push(item);
+      }
+    });
+    out.forEach(function(item) {
+      lastTime = Math.max(lastTime, timeFn(item));
+    });
+
+    return out;
+  };
+}
+
+module.exports.makeRecencyFilter = makeRecencyFilter;

index.js

@@ -1,5 +1,6 @@
 module.exports = {
   carelink: require('./carelink'),
+  filter: require('./filter'),
   logger: require('./logger'),
   nightscout: require('./nightscout'),
   transform: require('./transform')

package.json

@@ -1,6 +1,6 @@
 {
   "name": "minimed-connect-to-nightscout",
-  "version": "0.2.4",
+  "version": "1.0.0",
   "description": "Send Medtronic pump and CGM data to Nightscout.",
   "author": "Mark Wilson <[email protected]>",
   "repository": {

run.js

@@ -2,6 +2,7 @@
 "use strict";
 
 var carelink = require('./carelink'),
+  filter = require('./filter'),
   logger = require('./logger'),
   nightscout = require('./nightscout'),
   transform = require('./transform');
@@ -38,52 +39,55 @@ var client = carelink.Client({
   password: config.password,
   maxRetryDuration: config.maxRetryDuration
 });
-var endpoint = (config.nsBaseUrl ? config.nsBaseUrl : 'https://' + config.nsHost) + '/api/v1/entries.json';
+var entriesUrl = (config.nsBaseUrl ? config.nsBaseUrl : 'https://' + config.nsHost) + '/api/v1/entries.json';
+var devicestatusUrl = (config.nsBaseUrl ? config.nsBaseUrl : 'https://' + config.nsHost) + '/api/v1/devicestatus.json';
 
 logger.setVerbose(config.verbose);
 
-var filterRecentSgvs = (function() {
-  var lastSgvDate = 0;
+var filterSgvs = filter.makeRecencyFilter(function(item) {
+  return item['date'];
+});
+
+var filterDeviceStatus = filter.makeRecencyFilter(function(item) {
+  return new Date(item['created_at']).getTime();
+});
 
-  return function(entries) {
-    var out = [];
-    entries.forEach(function(entry) {
-      if (entry['type'] !== 'sgv' || entry['date'] > lastSgvDate) {
-        out.push(entry);
+function uploadMaybe(items, endpoint, callback) {
+  if (items.length === 0) {
+    logger.log('No new items for ' + endpoint);
+    callback();
+  } else {
+    nightscout.upload(items, endpoint, config.nsSecret, function(err, response) {
+      if (err) {
+        // Continue gathering data from CareLink even if Nightscout can't be reached
+        console.log(err);
       }
+      callback();
     });
-    out.filter(function(e) { return e['type'] === 'sgv'; })
-      .forEach(function(e) {
-        lastSgvDate = Math.max(lastSgvDate, e['date']);
-      });
-
-    return out;
-  };
-})();
+  }
+}
 
 (function requestLoop() {
   client.fetch(function(err, data) {
     if (err) {
       throw new Error(err);
     } else {
-      var allEntries = transform(data, config.sgvLimit);
+      var transformed = transform(data, config.sgvLimit);
 
       // Because of Nightscout's upsert semantics and the fact that CareLink provides trend
       // data only for the most recent sgv, we need to filter out sgvs we've already sent.
       // Otherwise we'll overwrite existing sgv entries and remove their trend data.
-      var newEntries = filterRecentSgvs(allEntries);
+      var newSgvs = filterSgvs(transformed.entries);
+
+      // Nightscout's entries collection upserts based on date, but the devicestatus collection
+      // does not do the same for created_at, so we need to de-dupe them here.
+      var newDeviceStatuses = filterDeviceStatus(transformed.devicestatus);
 
-      if (newEntries.length > 0) {
-        nightscout.upload(newEntries, endpoint, config.nsSecret, function(err, response) {
-          if (err) {
-            // Continue gathering data from CareLink even if Nightscout can't be reached
-            console.log(err);
-          }
+      uploadMaybe(newSgvs, entriesUrl, function() {
+        uploadMaybe(newDeviceStatuses, devicestatusUrl, function() {
           setTimeout(requestLoop, config.interval);
         });
-      } else {
-        setTimeout(requestLoop, config.interval);
-      }
+      });
     }
   });
 })();

test/filter.js

@@ -0,0 +1,30 @@
+/* jshint node: true */
+/* globals describe, it */
+"use strict";
+
+var _ = require('lodash'),
+  expect = require('expect.js');
+
+var makeRecencyFilter = require('../filter.js').makeRecencyFilter;
+
+describe('makeRecencyFilter()', function() {
+  it('should return a stateful filter which discards items older than the most recent one seen', function() {
+    function sgv(date) {
+      return {type: 'sgv', someDateKey: date};
+    }
+
+    var filter = makeRecencyFilter(function(item) {
+      return item['someDateKey'];
+    });
+
+    expect(filter([2, 3, 4].map(sgv))).to.have.length(3);
+
+    expect(filter([2, 3, 4].map(sgv))).to.have.length(0);
+
+    var filtered = filter([2, 3, 4, 8, 6, 7, 5].map(sgv));
+    expect(filtered).to.have.length(4);
+    [5, 6, 7, 8].forEach(function(val) {
+      expect(_.pluck(filtered, 'someDateKey')).to.contain(val);
+    });
+  });
+});

test/integration.js

@@ -11,12 +11,11 @@ var samples = require('./_samples'),
 describe('integration test: missingLastSgv', function() {
   var sample = samples.missingLastSgv;
   var transformed = transform(sample);
-  var pumpStatuses = _.filter(transformed, {type: 'pump_status'});
-  var sgvs = _.filter(transformed, {type: 'sgv'});
+  var pumpStatuses = transformed.devicestatus;
+  var sgvs = transformed.entries;
 
   it('should set the pump status time based on the "last device data update time"', function() {
-    expect(pumpStatuses[0]['date']).to.equal(sample['lastMedicalDeviceDataUpdateServerTime']);
-    expect(pumpStatuses[0]['dateString']).to.equal(new Date(sample['lastMedicalDeviceDataUpdateServerTime']).toISOString());
+    expect(pumpStatuses[0]['created_at']).to.equal(new Date(sample['lastMedicalDeviceDataUpdateServerTime']).toISOString());
   });
 
   it('should have one pump_status and 5 sgv entries', function() {
@@ -40,31 +39,30 @@ describe('integration test: missingLastSgv', function() {
 
   it('should include pump status data, including active insulin', function() {
     _.forEach({
-      activeInsulin: 4.85,
-      calibStatus: 'LESS_THAN_TWELVE_HRS',
-      conduitBatteryLevel: 29,
-      conduitInRange: true,
-      conduitMedicalDeviceInRange: true,
-      conduitSensorInRange: true,
-      device: 'connect://paradigm',
-      medicalDeviceBatteryLevelPercent: 75,
-      reservoirAmount: 60,
-      reservoirLevelPercent: 25,
-      sensorDurationHours: 73,
-      sensorState: 'NORMAL',
-      timeToNextCalibHours: 10
+      'uploader.battery': 29,
+      'pump.battery.percent': 75,
+      'pump.reservoir': 60,
+      'pump.iob.bolusiob': 4.85,
+      'pump.iob.timestamp': new Date(sample['lastMedicalDeviceDataUpdateServerTime']).toISOString(),
+      'connect.calibStatus': 'LESS_THAN_TWELVE_HRS',
+      'connect.conduitInRange': true,
+      'connect.conduitMedicalDeviceInRange': true,
+      'connect.conduitSensorInRange': true,
+      'connect.sensorDurationHours': 73,
+      'connect.sensorState': 'NORMAL',
+      'connect.timeToNextCalibHours': 10,
+      'device': 'connect://paradigm',
     }, function(val, key) {
-      expect(pumpStatuses[0][key]).to.be(val);
+      expect(_.get(pumpStatuses[0], key)).to.be(val);
     });
   });
 });
 
 describe('integration test: withTrend', function() {
   var sample = samples.withTrend;
   var transformed = transform(sample);
-  var pumpStatuses = _.filter(transformed, {type: 'pump_status'});
-
-  var sgvs = _.filter(transformed, {type: 'sgv'});
+  var pumpStatuses = transformed.devicestatus;
+  var sgvs = transformed.entries;
 
   it('should have one pump_status and 6 sgv entries', function() {
     expect(pumpStatuses.length).to.be(1);
@@ -97,21 +95,21 @@ describe('integration test: withTrend', function() {
 
   it('should include pump status data, including active insulin', function() {
     _.forEach({
-      activeInsulin: 1.35,
-      calibStatus: 'LESS_THAN_NINE_HRS',
-      conduitBatteryLevel: 86,
-      conduitInRange: true,
-      conduitMedicalDeviceInRange: true,
-      conduitSensorInRange: true,
-      device: 'connect://paradigm',
-      medicalDeviceBatteryLevelPercent: 50,
-      reservoirAmount: 67,
-      reservoirLevelPercent: 50,
-      sensorDurationHours: 137,
-      sensorState: 'NORMAL',
-      timeToNextCalibHours: 6
+      'uploader.battery': 86,
+      'pump.battery.percent': 50,
+      'pump.reservoir': 67,
+      'pump.iob.bolusiob': 1.35,
+      'pump.iob.timestamp': new Date(sample['lastMedicalDeviceDataUpdateServerTime']).toISOString(),
+      'connect.calibStatus': 'LESS_THAN_NINE_HRS',
+      'connect.conduitInRange': true,
+      'connect.conduitMedicalDeviceInRange': true,
+      'connect.conduitSensorInRange': true,
+      'connect.sensorDurationHours': 137,
+      'connect.sensorState': 'NORMAL',
+      'connect.timeToNextCalibHours': 6,
+      'device': 'connect://paradigm',
     }, function(val, key) {
-      expect(pumpStatuses[0][key]).to.be(val);
+      expect(_.get(pumpStatuses[0], key)).to.be(val);
     });
   });
 });