add data refresher, remote storage database toggle

.github/workflows/openactive-test-suite.yml

@@ -84,6 +84,8 @@ jobs:
         dotnet run --no-launch-profile --project ./server/Examples/BookingSystem.AspNetCore/BookingSystem.AspNetCore.csproj --configuration Release --no-build &
       env:
         ASPNETCORE_ENVIRONMENT: ${{ matrix.profile }}
+        USE_REMOTE_STORAGE: false
+        DROP_TABLES_ON_RESTART: true
     - name: Install OpenActive Test Suite
       run: npm install
       working-directory: tests

Examples/BookingSystem.AspNetCore.IdentityServer/BookingSystem.AspNetCore.IdentityServer.csproj

@@ -11,6 +11,9 @@
     <PackageReference Include="Serilog.AspNetCore" Version="3.1.0" />
     <PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" />
     <PackageReference Include="Serilog.Sinks.File" Version="4.0.0" />
+    <PackageReference Include="Microsoft.AspNetCore.AzureAppServicesIntegration" Version="3.1.16" />
+    <PackageReference Include="Microsoft.AspNetCore.AzureAppServices.HostingStartup" Version="3.1.16" />
+    <PackageReference Include="Microsoft.Extensions.Logging.AzureAppServices" Version="5.0.8" />
   </ItemGroup>
 
   <ItemGroup>

Examples/BookingSystem.AspNetCore.IdentityServer/Custom/BookingPartners/BookingPartnersController.cs

@@ -109,7 +109,7 @@ public async Task<IActionResult> Remove([FromForm] string clientId)
         public async Task<IActionResult> Delete([FromForm] string clientId)
         {
             await _interaction.RevokeUserConsentAsync(clientId);
-            await FakeBookingSystem.Database.DeleteBookingPartner(clientId);
+            await FakeBookingSystem.FakeDatabase.DeleteBookingPartner(clientId);
             await _events.RaiseAsync(new GrantsRevokedEvent(User.GetSubjectId(), clientId));
 
             return Redirect("/booking-partners/sys-admin");

Examples/BookingSystem.AspNetCore.IdentityServer/Custom/Grants/PersistedGrantStore.cs

@@ -22,7 +22,7 @@ public async Task<IEnumerable<PersistedGrant>> GetAllAsync(PersistedGrantFilter
         {
             filter.Validate();
 
-            var grants = await FakeBookingSystem.Database.GetAllGrants(filter.SubjectId, filter.SessionId, filter.ClientId, filter.Type);
+            var grants = await FakeBookingSystem.FakeDatabase.GetAllGrants(filter.SubjectId, filter.SessionId, filter.ClientId, filter.Type);
 
             _logger.LogDebug("{persistedGrantCount} persisted grants found for {@filter}", grants.Count, filter);
 
@@ -44,7 +44,7 @@ public async Task<IEnumerable<PersistedGrant>> GetAllAsync(PersistedGrantFilter
 
         public async Task<PersistedGrant> GetAsync(string key)
         {
-            var grant = await FakeBookingSystem.Database.GetGrant(key);
+            var grant = await FakeBookingSystem.FakeDatabase.GetGrant(key);
 
             _logger.LogDebug("{persistedGrantKey} found in database: {persistedGrantKeyFound}", key, grant != null);
 
@@ -68,19 +68,19 @@ public async Task RemoveAllAsync(PersistedGrantFilter filter)
 
             _logger.LogDebug("removing all persisted grants from database for {@filter}", filter);
 
-            await FakeBookingSystem.Database.RemoveAllGrants(filter.SubjectId, filter.SessionId, filter.ClientId, filter.Type);
+            await FakeBookingSystem.FakeDatabase.RemoveAllGrants(filter.SubjectId, filter.SessionId, filter.ClientId, filter.Type);
         }
 
         public async Task RemoveAsync(string key)
         {
             _logger.LogDebug("removing {persistedGrantKey} persisted grant from database", key);
 
-            await FakeBookingSystem.Database.RemoveGrant(key);
+            await FakeBookingSystem.FakeDatabase.RemoveGrant(key);
         }
 
         public async Task StoreAsync(PersistedGrant grant)
         {
-            if (await FakeBookingSystem.Database.AddGrant(grant.Key, grant.Type, grant.SubjectId, grant.SessionId, grant.ClientId, grant.CreationTime, grant.ConsumedTime, grant.Expiration, grant.Data))
+            if (await FakeBookingSystem.FakeDatabase.AddGrant(grant.Key, grant.Type, grant.SubjectId, grant.SessionId, grant.ClientId, grant.CreationTime, grant.ConsumedTime, grant.Expiration, grant.Data))
             {
                 _logger.LogDebug("{persistedGrantKey} not found in database, and so was inserted", grant.Key);
             }

Examples/BookingSystem.AspNetCore.IdentityServer/Custom/Users/UserRepository.cs

@@ -16,19 +16,19 @@ public UserRepository(string jsonLdIdBaseUrl)
 
         public Task<bool> ValidateCredentials(string username, string password)
         {
-            return FakeBookingSystem.Database.ValidateSellerUserCredentials(username, password);
+            return FakeBookingSystem.FakeDatabase.ValidateSellerUserCredentials(username, password);
         }
 
         public async Task<UserWithClaims> FindBySubjectId(string subjectId)
         {
             return long.TryParse(subjectId, out var longSubjectId)
-                ? GetUserFromSellerUserWithClaims(await FakeBookingSystem.Database.GetSellerUserById(longSubjectId))
+                ? GetUserFromSellerUserWithClaims(await FakeBookingSystem.FakeDatabase.GetSellerUserById(longSubjectId))
                 : null;
         }
 
         public async Task<User> FindByUsername(string username)
         {
-            return GetUserFromSellerUser(await FakeBookingSystem.Database.GetSellerUser(username));
+            return GetUserFromSellerUser(await FakeBookingSystem.FakeDatabase.GetSellerUser(username));
         }
 
         // TODO: Make this an extension method

Examples/BookingSystem.AspNetCore/BackgroundServices/FakeDataRefresherService.cs

@@ -0,0 +1,44 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using OpenActive.FakeDatabase.NET;
+
+
+namespace BookingSystem
+{
+    // Background task
+    // More information: https://docs.microsoft.com/en-us/dotnet/architecture/microservices/multi-container-microservice-net-applications/background-tasks-with-ihostedservice#implementing-ihostedservice-with-a-custom-hosted-service-class-deriving-from-the-backgroundservice-base-class
+    public class FakeDataRefresherService : BackgroundService
+    {
+        private readonly ILogger<FakeDataRefresherService> _logger;
+        private readonly AppSettings _settings;
+
+        public FakeDataRefresherService(AppSettings settings, ILogger<FakeDataRefresherService> logger)
+        {
+            _settings = settings;
+            _logger = logger;
+        }
+
+        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+        {
+            _logger.LogDebug($"FakeDataRefresherService is starting..");
+
+            stoppingToken.Register(() =>
+                _logger.LogDebug($"FakeDataRefresherService background task is stopping."));
+
+            while (!stoppingToken.IsCancellationRequested)
+            {
+                await FakeBookingSystem.FakeDatabase.HardDeletedOldSoftDeletedOccurrencesAndSlots();
+                _logger.LogDebug($"FakeDataRefresherService hard deleted opportunities that were previously old and soft deleted");
+
+                await FakeBookingSystem.FakeDatabase.SoftDeletedPastOpportunitiesAndInsertNewAtEdgeOfWindow();
+                _logger.LogDebug($"FakeDataRefresherService soft deleted opportunities and inserted new ones at edge of window.");
+
+                _logger.LogDebug($"FakeDataRefresherService is finished");
+                await Task.Delay(_settings.DataRefresherInterval, stoppingToken);
+            }
+        }
+    }
+}

Examples/BookingSystem.AspNetCore/BookingSystem.AspNetCore.csproj

@@ -3,9 +3,21 @@
   <PropertyGroup>
     <TargetFramework>netcoreapp3.1</TargetFramework>
     <UserSecretsId>aspnet-BookingSystem.AspNetCore-443B4F82-A20C-41CE-9924-329A0BCF0D14</UserSecretsId>
+    <!-- <AspNetCoreHostingModel>OutOfProcess</AspNetCoreHostingModel>
+    <AspNetCoreModuleName>AspNetCoreModule</AspNetCoreModuleName> -->
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.AspNetCore.App">
+      <PrivateAssets Condition="'%(PackageReference.Version)' == ''">all</PrivateAssets>
+      <Publish Condition="'%(PackageReference.Version)' == ''">true</Publish>
+    </PackageReference>
+    <PackageReference Include="Microsoft.AspNetCore.Authentication.AzureADB2C.UI" Version="2.1.1" />
+    <PackageReference Include="Microsoft.AspNetCore.Razor.Design" Version="2.1.2" PrivateAssets="All" />
+    <PackageReference Include="OpenActive.NET" Version="15.2.21" />
+    <PackageReference Include="Microsoft.AspNetCore.AzureAppServicesIntegration" Version="3.1.16" />
+    <PackageReference Include="Microsoft.AspNetCore.AzureAppServices.HostingStartup" Version="3.1.16" />
+    <PackageReference Include="Microsoft.Extensions.Logging.AzureAppServices" Version="5.0.8" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="2.1.2" />
   </ItemGroup>
 
@@ -16,6 +28,7 @@
 
   <ItemGroup>
     <Folder Include="Authentication\" />
+    <Folder Include="BackgroundServices\" />
   </ItemGroup>
   <PropertyGroup>
     <NoWarn>1701;1702;1591</NoWarn>

Examples/BookingSystem.AspNetCore/Feeds/FacilitiesFeeds.cs

@@ -24,7 +24,10 @@ public AcmeFacilityUseRpdeGenerator(AppSettings appSettings)
 
         protected override async Task<List<RpdeItem<FacilityUse>>> GetRpdeItems(long? afterTimestamp, long? afterId)
         {
-            using (var db = FakeBookingSystem.Database.Mem.Database.Open())
+            var facilityTypeId = Environment.GetEnvironmentVariable("FACILITY_TYPE_ID") ?? "https://openactive.io/facility-types#a1f82b7a-1258-4d9a-8dc5-bfc2ae961651";
+            var facilityTypePrefLabel = Environment.GetEnvironmentVariable("FACILITY_TYPE_PREF_LABEL") ?? "Squash Court";
+
+            using (var db = FakeBookingSystem.FakeDatabase.DatabaseWrapper.Database.Open())
             {
                 var q = db.From<FacilityUseTable>()
                 .Join<SellerTable>()
@@ -107,8 +110,8 @@ protected override async Task<List<RpdeItem<FacilityUse>>> GetRpdeItems(long? af
                             FacilityType = new List<Concept> {
                                 new Concept
                                 {
-                                    Id = new Uri("https://openactive.io/facility-types#a1f82b7a-1258-4d9a-8dc5-bfc2ae961651"),
-                                    PrefLabel = "Squash Court",
+                                    Id = new Uri(facilityTypeId),
+                                    PrefLabel = facilityTypePrefLabel,
                                     InScheme = new Uri("https://openactive.io/facility-types")
                                 }
                             }
@@ -133,7 +136,7 @@ public AcmeFacilityUseSlotRpdeGenerator(AppSettings appSettings)
 
         protected override async Task<List<RpdeItem<Slot>>> GetRpdeItems(long? afterTimestamp, long? afterId)
         {
-            using (var db = FakeBookingSystem.Database.Mem.Database.Open())
+            using (var db = FakeBookingSystem.FakeDatabase.DatabaseWrapper.Database.Open())
             {
                 var query = db.Select<SlotTable>()
                 .OrderBy(x => x.Modified)

Examples/BookingSystem.AspNetCore/Feeds/OrdersFeed.cs

@@ -27,7 +27,7 @@ protected override async Task<List<RpdeItem>> GetRPDEItems(string clientId, long
             // and update this class to inherit from OrdersRPDEFeedIncrementingUniqueChangeNumber
             // (to use afterChangeNumber, instead of afterTimestamp and afterId)
 
-            using (var db = FakeBookingSystem.Database.Mem.Database.Open())
+            using (var db = FakeBookingSystem.FakeDatabase.DatabaseWrapper.Database.Open())
             {
                 long afterTimestampLong = afterTimestamp ?? 0;
                 var q = db.From<OrderTable>()
@@ -126,7 +126,7 @@ public AcmeOrderProposalsFeedRpdeGenerator(AppSettings appSettings)
 
         protected override async Task<List<RpdeItem>> GetRPDEItems(string clientId, long? afterTimestamp, string afterId)
         {
-            using (var db = FakeBookingSystem.Database.Mem.Database.Open())
+            using (var db = FakeBookingSystem.FakeDatabase.DatabaseWrapper.Database.Open())
             {
                 long afterTimestampLong = afterTimestamp ?? 0;
                 var q = db.From<OrderTable>()

Examples/BookingSystem.AspNetCore/Feeds/SessionsFeeds.cs

@@ -17,7 +17,7 @@ public class AcmeScheduledSessionRpdeGenerator : RpdeFeedModifiedTimestampAndIdL
 
         protected override async Task<List<RpdeItem<ScheduledSession>>> GetRpdeItems(long? afterTimestamp, long? afterId)
         {
-            using (var db = FakeBookingSystem.Database.Mem.Database.Open())
+            using (var db = FakeBookingSystem.FakeDatabase.DatabaseWrapper.Database.Open())
             {
                 var query = db.Select<OccurrenceTable>()
                 .OrderBy(x => x.Modified)
@@ -74,7 +74,10 @@ public AcmeSessionSeriesRpdeGenerator(AppSettings appSettings)
 
         protected override async Task<List<RpdeItem<SessionSeries>>> GetRpdeItems(long? afterTimestamp, long? afterId)
         {
-            using (var db = FakeBookingSystem.Database.Mem.Database.Open())
+            var activityId = Environment.GetEnvironmentVariable("ACTIVITY_ID") ?? "https://openactive.io/activity-list#c07d63a0-8eb9-4602-8bcc-23be6deb8f83";
+            var activityPrefLabel = Environment.GetEnvironmentVariable("ACTIVITY_PREF_LABEL") ?? "Jet Skiing";
+
+            using (var db = FakeBookingSystem.FakeDatabase.DatabaseWrapper.Database.Open())
             {
                 var q = db.From<ClassTable>()
                 .Join<SellerTable>()
@@ -199,8 +202,8 @@ protected override async Task<List<RpdeItem<SessionSeries>>> GetRpdeItems(long?
                             {
                                 new Concept
                                 {
-                                    Id = new Uri("https://openactive.io/activity-list#c07d63a0-8eb9-4602-8bcc-23be6deb8f83"),
-                                    PrefLabel = "Jet Skiing",
+                                    Id = new Uri(activityId),
+                                    PrefLabel = activityPrefLabel,
                                     InScheme = new Uri("https://openactive.io/activity-list")
                                 }
                             }

Examples/BookingSystem.AspNetCore/Program.cs

@@ -8,13 +8,18 @@ public class Program
     {
         public static void Main(string[] args)
         {
-            // Initialising fake database (shared with IdentityServer)
+            var host = CreateWebHostBuilder(args)
+                .Build();
+
             FakeBookingSystem.Initialise();
-            CreateWebHostBuilder(args).Build().Run();
+
+            host.Run();
         }
 
         public static IWebHostBuilder CreateWebHostBuilder(string[] args) =>
             WebHost.CreateDefaultBuilder(args)
-                .UseStartup<Startup>();
+            .CaptureStartupErrors(true)
+            .UseSetting("detailedErrors", "true")
+            .UseStartup<Startup>();
     }
 }

Examples/BookingSystem.AspNetCore/Properties/launchSettings.json

@@ -11,7 +11,11 @@
       "applicationUrl": "https://localhost:5001",
       "environmentVariables": {
         "ASPNETCORE_ENVIRONMENT": "Development",
-        "ApplicationHostBaseUrl": "https://localhost:5001"
+        "ApplicationHostBaseUrl": "https://localhost:5001",
+        "REMOTE_STORAGE_CONNECTION_STRING": "",
+        "USE_REMOTE_STORAGE": "true",
+        "DROP_TABLES_ON_RESTART": "true",
+        "OPPORTUNITY_COUNT": "20"
       }
     }
   }

Examples/BookingSystem.AspNetCore/Settings/AppSettings.cs

@@ -1,3 +1,5 @@
+using System;
+
 namespace BookingSystem
 {
     public class AppSettings
@@ -6,6 +8,8 @@ public class AppSettings
         public string OpenIdIssuerUrl { get; set; }
         public FeatureSettings FeatureFlags { get; set; }
         public PaymentSettings Payment { get; set; }
+        public TimeSpan DataRefresherInterval = TimeSpan.FromHours(6);
+
     }
 
     /**

Examples/BookingSystem.AspNetCore/Startup.cs

@@ -60,7 +60,8 @@ public void ConfigureServices(IServiceCollection services)
                 services.AddAuthorization(options =>
                 {
                     // No authorization checks are performed, this just ensures that the required claims are supplied
-                    options.AddPolicy(OpenActiveScopes.OpenBooking, policy => {
+                    options.AddPolicy(OpenActiveScopes.OpenBooking, policy =>
+                    {
                         policy.RequireClaim(OpenActiveCustomClaimNames.ClientId);
                         policy.RequireClaim(OpenActiveCustomClaimNames.SellerId);
                     });
@@ -72,7 +73,13 @@ public void ConfigureServices(IServiceCollection services)
                 .AddControllers()
                 .AddMvcOptions(options => options.InputFormatters.Insert(0, new OpenBookingInputFormatter()));
 
+            // Add config as a singleton to pipe it through DI to the booking engine and stores
+            services.AddSingleton(x => AppSettings);
+
             services.AddSingleton<IBookingEngine>(sp => EngineConfig.CreateStoreBookingEngine(AppSettings));
+
+            // Add background OrderItem polling
+            services.AddHostedService<FakeDataRefresherService>();
         }
 
         // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
@@ -93,7 +100,6 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env)
             app.UseHttpsRedirection();
             app.UseStaticFiles();
             app.UseRouting();
-
             app.UseAuthentication();
             app.UseAuthorization();
             app.UseEndpoints(endpoints =>