Updated to version 2.2.0.

MyGeotabAPIAdapter.Configuration/AdapterConfiguration.cs

@@ -1,5 +1,6 @@
 using NLog;
 using System;
+using System.Linq;
 using System.Reflection;
 
 namespace MyGeotabAPIAdapter.Configuration
@@ -9,6 +10,8 @@ namespace MyGeotabAPIAdapter.Configuration
     /// </summary>
     public class AdapterConfiguration : IAdapterConfiguration
     {
+        const string WildcardString = "*";
+
         // Argument Names for appsettings:
         // > OverrideSetings
         const string ArgNameDisableMachineNameValidation = "OverrideSetings:DisableMachineNameValidation";
@@ -78,9 +81,13 @@ public class AdapterConfiguration : IAdapterConfiguration
         // > AppSettings:GeneralFeedSettings
         const string ArgNameFeedStartOption = "AppSettings:GeneralFeedSettings:FeedStartOption";
         const string ArgNameFeedStartSpecificTimeUTC = "AppSettings:GeneralFeedSettings:FeedStartSpecificTimeUTC";
-        public readonly string ArgNameDevicesToTrack = "AppSettings:GeneralFeedSettings:DevicesToTrack";
-        public readonly string ArgNameDiagnosticsToTrack = "AppSettings:GeneralFeedSettings:DiagnosticsToTrack";
+        const string ArgNameDevicesToTrack = "AppSettings:GeneralFeedSettings:DevicesToTrack";
+        const string ArgNameDiagnosticsToTrack = "AppSettings:GeneralFeedSettings:DiagnosticsToTrack";
         const string ArgNameExcludeDiagnosticsToTrack = "AppSettings:GeneralFeedSettings:ExcludeDiagnosticsToTrack";
+        const string ArgNameEnableMinimunIntervalSamplingForLogRecords = "AppSettings:GeneralFeedSettings:EnableMinimunIntervalSamplingForLogRecords";
+        const string ArgNameEnableMinimunIntervalSamplingForStatusData = "AppSettings:GeneralFeedSettings:EnableMinimunIntervalSamplingForStatusData";
+        const string ArgNameMinimumIntervalSamplingDiagnostics = "AppSettings:GeneralFeedSettings:MinimumIntervalSamplingDiagnostics";
+        const string ArgNameMinimumIntervalSamplingIntervalSeconds = "AppSettings:GeneralFeedSettings:MinimumIntervalSamplingIntervalSeconds";
         // > AppSettings:Feeds:BinaryData
         const string ArgNameEnableBinaryDataFeed = "AppSettings:Feeds:BinaryData:EnableBinaryDataFeed";
         const string ArgNameBinaryDataFeedIntervalSeconds = "AppSettings:Feeds:BinaryData:BinaryDataFeedIntervalSeconds";
@@ -142,6 +149,9 @@ public class AdapterConfiguration : IAdapterConfiguration
         const int MinDutyStatusAvailabilityFeedLastAccessDateCutoffDays = 14;
         const int MaxDutyStatusAvailabilityFeedLastAccessDateCutoffDays = 60;
         const int DefaultDutyStatusAvailabilityFeedLastAccessDateCutoffDays = 30;
+        const int MinSamplingIntervalSeconds = 1;
+        const int MaxSamplingIntervalSeconds = 3600; // 3600 sec = 1 hr
+        const int DefaultSamplingIntervalSeconds = 300;
 
         // Arbitrary timeout limits:
         const int DefaultTimeoutSeconds = 30;
@@ -280,6 +290,12 @@ public class AdapterConfiguration : IAdapterConfiguration
         /// <inheritdoc/>
         public bool EnableLogRecordFeed { get; private set; }
 
+        /// <inheritdoc/>
+        public bool EnableMinimunIntervalSamplingForLogRecords { get; private set; }
+
+        /// <inheritdoc/>
+        public bool EnableMinimunIntervalSamplingForStatusData { get; private set; }
+
         /// <inheritdoc/>
         public bool EnableRuleCache { get; private set; }
 
@@ -340,6 +356,12 @@ public class AdapterConfiguration : IAdapterConfiguration
         /// <inheritdoc/>
         public int LogRecordFeedIntervalSeconds { get; private set; }
 
+        /// <inheritdoc/>
+        public string MinimumIntervalSamplingDiagnosticsList { get; private set; }
+
+        /// <inheritdoc/>
+        public int MinimumIntervalSamplingIntervalSeconds { get; private set; }
+
         /// <inheritdoc/>
         public string MyGeotabDatabase { get; private set; }
 
@@ -556,6 +578,11 @@ public void ProcessConfigItems()
             DevicesToTrackList = configurationHelper.GetConfigKeyValueString(ArgNameDevicesToTrack);
             DiagnosticsToTrackList = configurationHelper.GetConfigKeyValueString(ArgNameDiagnosticsToTrack);
             ExcludeDiagnosticsToTrack = configurationHelper.GetConfigKeyValueBoolean(ArgNameExcludeDiagnosticsToTrack);
+            EnableMinimunIntervalSamplingForLogRecords = configurationHelper.GetConfigKeyValueBoolean(ArgNameEnableMinimunIntervalSamplingForLogRecords);
+            EnableMinimunIntervalSamplingForStatusData = configurationHelper.GetConfigKeyValueBoolean(ArgNameEnableMinimunIntervalSamplingForStatusData);
+            MinimumIntervalSamplingDiagnosticsList = configurationHelper.GetConfigKeyValueString(ArgNameMinimumIntervalSamplingDiagnostics);
+            MinimumIntervalSamplingIntervalSeconds = configurationHelper.GetConfigKeyValueInt(ArgNameMinimumIntervalSamplingIntervalSeconds, null, false, MinSamplingIntervalSeconds, MaxSamplingIntervalSeconds, DefaultSamplingIntervalSeconds);
+            ValidateMinimumIntervalSamplingDiagnosticsList();
 
             // AppSettings:Feeds:BinaryData:
             EnableBinaryDataFeed = configurationHelper.GetConfigKeyValueBoolean(ArgNameEnableBinaryDataFeed);
@@ -617,5 +644,54 @@ public void ProcessConfigItems()
 
             logger.Trace($"End {methodBase.ReflectedType.Name}.{methodBase.Name}");
         }
+
+        /// <summary>
+        /// Validated the MinimumIntervalSamplingDiagnosticsList configuration item.
+        /// </summary>
+        void ValidateMinimumIntervalSamplingDiagnosticsList()
+        {
+            string errorMessage;
+
+            // If both EnableMinimunIntervalSamplingForLogRecords and EnableMinimunIntervalSamplingForStatusData are false, then there is no need to validate the MinimumIntervalSamplingDiagnosticsList.
+            if (EnableMinimunIntervalSamplingForLogRecords == false && EnableMinimunIntervalSamplingForStatusData == false)
+            {
+                return;
+            }
+
+            // If EnableMinimunIntervalSamplingForStatusData is true, then ExcludeDiagnosticsToTrack must be false.
+            if (EnableMinimunIntervalSamplingForStatusData == true && ExcludeDiagnosticsToTrack == true)
+            {
+                errorMessage = $"'{ArgNameEnableMinimunIntervalSamplingForStatusData}' is set to true, but so is '{ArgNameExcludeDiagnosticsToTrack}'. Minimum interval sanpling cannot be enabled for StatusData unless '{ArgNameExcludeDiagnosticsToTrack}' is set to false.";
+                logger.Error(errorMessage);
+                throw new Exception(errorMessage);
+            }
+
+            // If EnableMinimunIntervalSamplingForStatusData is true, then DiagnosticsToTrack must be populated with a list of specific Diagnostic Ids.
+            if (EnableMinimunIntervalSamplingForStatusData == true && (DiagnosticsToTrackList == string.Empty || DiagnosticsToTrackList == WildcardString))
+            {
+                errorMessage = $"'{ArgNameEnableMinimunIntervalSamplingForStatusData}' is set to true, but '{ArgNameDiagnosticsToTrack}' is set to '{DiagnosticsToTrackList}'. Minimum interval sanpling cannot be enabled for StatusData unless '{ArgNameDiagnosticsToTrack}' is populated with a list of specific Diagnostic Ids.";
+                logger.Error(errorMessage);
+                throw new Exception(errorMessage);
+            }
+
+            // If EnableMinimunIntervalSamplingForStatusData is true, then MinimumIntervalSamplingDiagnostics must be populated with a list of specific Diagnostic Ids that is the same as that provided in DiagnosticsToTrack or a subset thereof.
+            if (EnableMinimunIntervalSamplingForStatusData == true && (MinimumIntervalSamplingDiagnosticsList == string.Empty || (MinimumIntervalSamplingDiagnosticsList == WildcardString)))
+            {
+                errorMessage = $"'{ArgNameEnableMinimunIntervalSamplingForStatusData}' is set to true. As such, '{ArgNameMinimumIntervalSamplingDiagnostics}' must be populated with the same list of specific Diagnostic Ids specified in '{ArgNameDiagnosticsToTrack}' or a subset thereof. The wildcard ('{WildcardString}' cannot be used.)";
+                logger.Error(errorMessage);
+                throw new Exception(errorMessage);
+            }
+
+            // If EnableMinimunIntervalSamplingForStatusData is true, then MinimumIntervalSamplingDiagnostics must be populated with a list of specific Diagnostic Ids that is the same as that provided in DiagnosticsToTrack or a subset thereof.
+            string[] diagnosticsToTrackArray = DiagnosticsToTrackList.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
+            string[] minimumIntervalSamplingDiagnosticsArray = MinimumIntervalSamplingDiagnosticsList.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
+            bool allMinimumIntervalSamplingDiagnosticsContainedInDiagnosticsToTrack = minimumIntervalSamplingDiagnosticsArray.All(minimumIntervalSamplingDiagnostic => diagnosticsToTrackArray.Contains(minimumIntervalSamplingDiagnostic));
+            if (EnableMinimunIntervalSamplingForStatusData == true && allMinimumIntervalSamplingDiagnosticsContainedInDiagnosticsToTrack == false)
+            {
+                errorMessage = $"'{ArgNameEnableMinimunIntervalSamplingForStatusData}' is set to true. As such, '{ArgNameMinimumIntervalSamplingDiagnostics}' must be populated with the same list of specific Diagnostic Ids specified in '{ArgNameDiagnosticsToTrack}' or a subset thereof. The list provided for '{ArgNameMinimumIntervalSamplingDiagnostics}' contains at least one item that is not in the list provided for '{ArgNameDiagnosticsToTrack}'.";
+                logger.Error(errorMessage);
+                throw new Exception(errorMessage);
+            }
+        }
     }
 }

MyGeotabAPIAdapter.Configuration/IAdapterConfiguration.cs

@@ -227,6 +227,16 @@ public interface IAdapterConfiguration
         /// </summary>
         bool EnableLogRecordFeed { get; }
 
+        /// <summary>
+        /// Indicates whether minimum interval sampling should be enabled for the <see cref="LogRecord"/> feed.
+        /// </summary>
+        bool EnableMinimunIntervalSamplingForLogRecords { get; }
+
+        /// <summary>
+        /// Indicates whether minimum interval sampling should be enabled for the <see cref="StatusData"/> feed. Applies only to StatusData with Diagnostics in the <see cref="MinimumIntervalSamplingDiagnosticsList"/> list.
+        /// </summary>
+        bool EnableMinimunIntervalSamplingForStatusData { get; }
+
         /// <summary>
         /// Indicates whether a <see cref="Rule"/> cache should be enabled. 
         /// </summary>
@@ -327,6 +337,16 @@ public interface IAdapterConfiguration
         /// </summary>
         int LogRecordFeedIntervalSeconds { get; }
 
+        /// <summary>
+        /// The comma-separated list of <see cref="Id"/>s of of specific <see cref="Diagnostic"/> entities for which minimum interval sampling of <see cref="StatusData"/> and/or <see cref="LogRecord"/> entities should be applied. <see cref="ExcludeDiagnosticsToTrack"/> must be false. Items in this list must also be included in the <see cref="DiagnosticsToTrackList"/> list and the wildcard value (*) cannot be used. There can be more Diagnostic Ids listed in the <see cref="DiagnosticsToTrackList"/> list than the <see cref="MinimumIntervalSamplingDiagnosticsList"/> list (i.e. minimum interval sampling can be applied to a subset of the list of Diagnostics being tracked with all StatusData being pulled for the rest of the items in the <see cref="DiagnosticsToTrackList"/> list.
+        /// </summary>
+        string MinimumIntervalSamplingDiagnosticsList {  get; }
+
+        /// <summary>
+        /// The minimum number of seconds that must have elapsed since the DateTime of the last captured record for the subject <see cref="Device"/> (in the case of <see cref="LogRecord"/>s) or <see cref="Device"/> + <see cref="Diagnostic"/> combination (in the case of <see cref="StatusData"/>) before the next record is captured. Note that data points are not captured or interpolated at this exact interval. Rather, the next actual data point at or after the elapsed interval is captured. 
+        /// </summary>
+        int MinimumIntervalSamplingIntervalSeconds { get; }
+
         /// <summary>
         /// The value to use for the <c>Database</c> parameter when authenticating the <see cref="MyGeotabAPIAdapter.Checkmate.API"/>.
         /// </summary>

MyGeotabAPIAdapter.DataOptimizer/MyGeotabAPIAdapter.DataOptimizer.csproj

@@ -6,8 +6,8 @@
     <Authors>Geotab Inc.</Authors>
     <Company>Geotab Inc.</Company>
     <Product>MyGeotab API Adapter - DataOptimizer</Product>
-    <AssemblyVersion>2.1.4</AssemblyVersion>
-    <FileVersion>2.1.4</FileVersion>
+    <AssemblyVersion>2.2.0</AssemblyVersion>
+    <FileVersion>2.2.0</FileVersion>
     <RuntimeIdentifiers>win-x64;linux-x64</RuntimeIdentifiers>
     <Description>A worker service designed to migrate data from the MyGeotab API Adapter database into another set of tables that are optimized for use by applications and data analysis tools. Additional columns are added to some of the tables and these are populated via interpolation or other query-based procedues.</Description>
   </PropertyGroup>

MyGeotabAPIAdapter.Database/MyGeotabAPIAdapter.Database.csproj

@@ -13,7 +13,7 @@
     <PackageReference Include="Microsoft.VisualStudio.Threading" Version="17.9.28" />
     <PackageReference Include="NLog.Extensions.Logging" Version="5.3.8" />
     <PackageReference Include="Npgsql" Version="8.0.2" />
-    <PackageReference Include="Oracle.ManagedDataAccess.Core" Version="3.21.130" />
+    <PackageReference Include="Oracle.ManagedDataAccess.Core" Version="3.21.140" />
     <PackageReference Include="Polly" Version="8.3.1" />
   </ItemGroup>
 

MyGeotabAPIAdapter.Tests/MyGeotabAPIAdapter.Tests.csproj

@@ -31,7 +31,7 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.9.0" />
     <PackageReference Include="Moq" Version="4.18.4" />
-    <PackageReference Include="Oracle.ManagedDataAccess.Core" Version="3.21.130" />
+    <PackageReference Include="Oracle.ManagedDataAccess.Core" Version="3.21.140" />
     <PackageReference Include="xunit" Version="2.7.0" />
     <PackageReference Include="xunit.runner.console" Version="2.7.0">
       <PrivateAssets>all</PrivateAssets>

MyGeotabAPIAdapter.Tests/appsettingsTest.json

@@ -93,7 +93,11 @@
       "FeedStartSpecificTimeUTC": "2020-06-23T08:00:00Z",
       "DevicesToTrack": "*",
       "DiagnosticsToTrack": "*",
-      "ExcludeDiagnosticsToTrack": false
+      "ExcludeDiagnosticsToTrack": false,
+      "EnableMinimunIntervalSamplingForLogRecords": false,
+      "EnableMinimunIntervalSamplingForStatusData": false,
+      "MinimumIntervalSamplingDiagnostics": "*",
+      "MinimumIntervalSamplingIntervalSeconds": 300
     },
     "Feeds": {
       "BinaryData": {

MyGeotabAPIAdapter/IMinimumIntervalSampler.cs

@@ -0,0 +1,29 @@
+using Geotab.Checkmate.ObjectModel;
+using Geotab.Checkmate.ObjectModel.Engine;
+using MyGeotabAPIAdapter.Configuration;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace MyGeotabAPIAdapter
+{
+    /// <summary>
+    /// Interface for a data reduction class that is used to reduce lists of <see cref="Entity"/>s by applying the configured <see cref="AdapterConfiguration.MinimumIntervalSamplingIntervalSeconds"/> setting to ensure that a minimum of the specified interval exists between each entity that is kept for later downstream persistence for the subject <see cref="Device"/> + <see cref="Entity"/> type (+ <see cref="Diagnostic"/> if applicable) combination."/>
+    /// </summary>
+    /// <typeparam name="T">The type of Geotab <see cref="Entity"/> to be processed by this <see cref="IMinimumIntervalSampler{T}"/> instance.</typeparam>
+    internal interface IMinimumIntervalSampler<T> where T : Entity
+    {
+        /// <summary>
+        /// A unique identifier assigned during instantiation. Intended for debugging purposes.
+        /// </summary>
+        string Id { get; }
+
+        /// <summary>
+        /// Evaluates the <paramref name="entitiesToBeFiltered"/> and returns a filtered list of entities that have a minimum interval of <see cref="AdapterConfiguration.MinimumIntervalSamplingIntervalSeconds"/> between each entity that is kept for later downstream persistence for the subject <see cref="Device"/> + <see cref="Entity"/> type (+ <see cref="Diagnostic"/> if applicable) combination.
+        /// </summary>
+        /// <param name="cancellationTokenSource">The <see cref="CancellationTokenSource"/>.</param>
+        /// <param name="entitiesToBeFiltered">The list of entities to be filtered.</param>
+        /// <returns></returns>
+        Task<List<T>> ApplyMinimumIntervalAsync(CancellationTokenSource cancellationTokenSource, List<T> entitiesToBeFiltered);
+    }
+}