[GH75] Remove anomalies from data set

* Modified the algorithm to exclude data points outside of the 5 times standard deviation range. * Excluding data points that took fewer than 5 minutes. * Made the longest and shortest more readble
Qualtrax · Nov 18, 2016 · 2a427e8 · 2a427e8
1 parent b56e076
commit 2a427e8
Show file tree

Hide file tree

Showing 4 changed files with 64 additions and 20 deletions.
diff --git a/lib/static/static/js/controllers/metricsCtrl.js b/lib/static/static/js/controllers/metricsCtrl.js
@@ -3,9 +3,9 @@
         .module('huburn')
         .controller('metricsCtrl', metricsCtrl);
 
-    metricsCtrl.$inject = ['$routeParams', '$q', 'metricService', 'gitHubService', 'cycleTimeService'];
+    metricsCtrl.$inject = ['$routeParams', '$q', 'metricService', 'gitHubService', 'cycleTimeService', 'dateService'];
 
-    function metricsCtrl($routeParams, $q, metricService, gitHubService, cycleTimeService) {
+    function metricsCtrl($routeParams, $q, metricService, gitHubService, cycleTimeService, dateService) {
         var vm = this;
         vm.numberOfMilestones = 6;
         vm.currentMilestoneIssues = [];
@@ -188,11 +188,18 @@
         vm.showCycleTimes = function showCycleTimes() {
             vm.displayCycleTimes = 'flex';
             vm.displayDistributions = 'none';
-        }
+        };
 
         vm.showDistributions = function showDistributions() {
             vm.displayCycleTimes = 'none';
             vm.displayDistributions = 'flex';
-        }
+        };
+
+        vm.getTimeInReadbleForm = function getTimeInReadbleForm(timeInDays) {
+            if (timeInDays < 0.25)
+                return dateService.daysToHours(timeInDays).toFixed(1) + " hours";
+
+            return timeInDays.toFixed(1) + " days";
+        };
     }
 }());
diff --git a/lib/static/static/js/services/cycleTimeService.js b/lib/static/static/js/services/cycleTimeService.js
@@ -117,7 +117,7 @@
                 var historicalInProgressCycles = getCycleTimesForColumn(issuesWithLabel, '2 - In Progress')
                     .filter(hasValidCycleTime)
                     .map(getCycleInDays)
-                    .sort(sortCycles);
+                    .sort(sortCycles);                    
 
                 if (historicalInProgressCycles.length < 2)
                     continue;
@@ -127,16 +127,21 @@
                     .filter(hasValidCycleTime)
                     .map(getCycleInDays);
 
-                var historicalInProgressCycleTimes = historicalInProgressCycles.map(getTimeFromCycle);
-                var mean = jStat.mean(historicalInProgressCycleTimes);
-                var standardDeviation = jStat.stdev(historicalInProgressCycleTimes);
-                maxValues.push(mean + (3 * standardDeviation));
+                var mean = jStat.mean(historicalInProgressCycles.map(getTimeFromCycle));
+                var standardDeviation = jStat.stdev(historicalInProgressCycles.map(getTimeFromCycle));
+                var maxValue = mean + (3 * standardDeviation);                
+
+                historicalInProgressCycles = removeAnomalies(historicalInProgressCycles, standardDeviation, mean);
+                mean = jStat.mean(historicalInProgressCycles.map(getTimeFromCycle));
+                standardDeviation = jStat.stdev(historicalInProgressCycles.map(getTimeFromCycle));
+                maxValue = mean + (3 * standardDeviation);
+                maxValues.push(maxValue);
 
                 statistics.push({
                     pointsLabel: pointsLabelsToRetrieve[i],
-                    numberOfHistoricalItems: historicalInProgressCycleTimes.length,
+                    numberOfHistoricalItems: historicalInProgressCycles.length,
                     shortestDataPoint: historicalInProgressCycles[0],
-                    longestDataPoint: historicalInProgressCycles[historicalInProgressCycleTimes.length - 1],
+                    longestDataPoint: historicalInProgressCycles[historicalInProgressCycles.length - 1],
                     dataPoints: currentInProgressTimes,
                     mean: mean,
                     standardDeviation: standardDeviation
@@ -158,10 +163,18 @@
         }
 
         function getCycleInDays(cycleTime) {
-            cycleTime.time = dateService.toDays(cycleTime.time);
+            cycleTime.time = dateService.toDaysNotRounded(cycleTime.time);
             return cycleTime;
         }
 
+        function removeAnomalies(cycles, standardDeviation, mean) {
+            return cycles.filter(function (cycle) {
+                return cycle.time > (mean - (5 * standardDeviation)) &&
+                    cycle.time < (mean + (5 * standardDeviation)) &&
+                    dateService.daysToMinutes(cycle.time) > 5;
+            });
+        }
+
         function sortCycles(cycleOne, cycleTwo) {
             return cycleOne.time - cycleTwo.time;
         }

diff --git a/lib/static/static/js/services/dateService.js b/lib/static/static/js/services/dateService.js
@@ -5,25 +5,49 @@
     .factory('dateService', dateService);
 
     function dateService() {
-
+        var hoursInADay = 24;
+        var minutesInAnHour = 60;
+        var secondsInAMinute = 60;
+        var millisecondsInASecond = 1000;
+
         return {
+            daysToMinutes: daysToMinutes,
+            daysToHours: daysToHours,
             getTimeBetween: getTimeBetween,
             toHours: toHours,
-            toDays: toDays
+            toDays: toDays,
+            toDaysNotRounded: toDaysNotRounded
         };
 
         function getTimeBetween(firstDate, secondDate) {
             return secondDate.getTime() - firstDate.getTime();
         }
 
         function toDays(timeInMilliseconds) {
-            var oneDay = 24*60*60*1000; // days*minutes*seconds*milliseconds
-            return round(timeInMilliseconds/oneDay);
+            var millisecondsInADay = hoursInADay * getMillisecondsInAnHour();
+            return round(timeInMilliseconds / millisecondsInADay);
+        }
+
+        function toDaysNotRounded(timeInMilliseconds) {
+            var millisecondsInADay = hoursInADay * getMillisecondsInAnHour();
+            return timeInMilliseconds / millisecondsInADay;
         }
 
         function toHours(timeInMilliseconds) {
-            var oneHour = 60*60*1000; // hours*minutes*seconds*milliseconds
-            return round(timeInMilliseconds/oneHour);
+            var millisecondsInAnHour = minutesInAnHour * secondsInAMinute * millisecondsInASecond;
+            return round(timeInMilliseconds / millisecondsInAnHour);
+        }
+
+        function daysToMinutes(timeInDays) {
+            return timeInDays * hoursInADay * minutesInAnHour;
+        }
+
+        function daysToHours(timeInDays) {
+            return timeInDays * hoursInADay;
+        }
+
+        function getMillisecondsInAnHour() {
+            return minutesInAnHour * secondsInAMinute * millisecondsInASecond;
         }
 
         function round(value) {

diff --git a/lib/static/static/templates/metrics.html b/lib/static/static/templates/metrics.html
@@ -103,8 +103,8 @@ <h1><a ng-class="{'selected-header':vm.selectedHeader === 'distributions'}"
         </div> 
         <div class="statistic-textual-summary">
             {{ statistic.numberOfHistoricalItems }} historical issues, on average, spending {{ statistic.mean.toFixed(1) }} days in progress.
-            Longest: <a href="https://github.com/{{ vm.repo }}/issues/{{ statistic.longestDataPoint.issue.number }}" target="_blank">{{ statistic.longestDataPoint.issue.number }}</a> - taking {{ statistic.longestDataPoint.time }} days,
-            Shortest: <a href="https://github.com/{{ vm.repo }}/issues/{{ statistic.shortestDataPoint.issue.number }}" target="_blank">{{ statistic.shortestDataPoint.issue.number }}</a>  - taking {{ statistic.shortestDataPoint.time }} days
+            Longest: <a href="https://github.com/{{ vm.repo }}/issues/{{ statistic.longestDataPoint.issue.number }}" target="_blank">{{ statistic.longestDataPoint.issue.number }}</a> - {{ vm.getTimeInReadbleForm(statistic.longestDataPoint.time) }},
+            Shortest: <a href="https://github.com/{{ vm.repo }}/issues/{{ statistic.shortestDataPoint.issue.number }}" target="_blank">{{ statistic.shortestDataPoint.issue.number }}</a>  - {{ vm.getTimeInReadbleForm(statistic.shortestDataPoint.time) }}
         </div>     
     </div>
   </div>