add data refresher, remote storage database toggle

.github/workflows/openactive-test-suite.yml

@@ -27,14 +27,10 @@ jobs:
         repository: openactive/openactive-test-suite
         ref: ${{ steps.refs.outputs.mirror_ref }}
         path: tests
-    - name: Setup .NET Core 2.1.808 for Booking Server Reference Implementation
+    - name: Setup .NET Core SDK 3.1.411
       uses: actions/setup-dotnet@v1
       with:
-        dotnet-version: 2.1.808
-    - name: Setup .NET Core 3.0.103 for Authentication Authority Reference Implementation
-      uses: actions/setup-dotnet@v1
-      with:
-        dotnet-version: 3.0.103
+        dotnet-version: 3.1.411
     - name: Setup Node.js 14.x
       uses: actions/setup-node@v1
       with:
@@ -150,14 +146,10 @@ jobs:
       # Checkout the repo
       - uses: actions/checkout@master
       # Setup .NET Core SDK
-      - name: Setup .NET Core 2.1.808
-        uses: actions/setup-dotnet@v1
-        with:
-          dotnet-version: 2.1.808
-      - name: Setup .NET Core 3.0.103 for Authentication Authority Reference Implementation
+      - name: Setup .NET Core SDK 3.1.411
         uses: actions/setup-dotnet@v1
         with:
-          dotnet-version: 3.0.103
+          dotnet-version: 3.1.411
       # Run dotnet build and publish
       - name: Install OpenActive.Server.NET dependencies
         run: dotnet restore

Examples/BookingSystem.AspNetCore.IdentityServer/BookingSystem.AspNetCore.IdentityServer.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.0</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
@@ -11,6 +11,9 @@
     <PackageReference Include="Serilog.AspNetCore" Version="3.1.0" />
     <PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" />
     <PackageReference Include="Serilog.Sinks.File" Version="4.0.0" />
+    <PackageReference Include="Microsoft.AspNetCore.AzureAppServicesIntegration" Version="3.1.16" />
+    <PackageReference Include="Microsoft.AspNetCore.AzureAppServices.HostingStartup" Version="3.1.16" />
+    <PackageReference Include="Microsoft.Extensions.Logging.AzureAppServices" Version="5.0.8" />
   </ItemGroup>
 
   <ItemGroup>

Examples/BookingSystem.AspNetCore.IdentityServer/Custom/Grants/BookingPartnersController.cs

@@ -107,7 +107,7 @@ public async Task<IActionResult> Remove([FromForm] string clientId)
         public async Task<IActionResult> Delete([FromForm] string clientId)
         {
             await _interaction.RevokeUserConsentAsync(clientId);
-            await FakeBookingSystem.Database.DeleteBookingPartner(clientId);
+            await FakeBookingSystem.FakeDatabase.DeleteBookingPartner(clientId);
             await _events.RaiseAsync(new GrantsRevokedEvent(User.GetSubjectId(), clientId));
 
             return Redirect("/booking-partners/sys-admin");

Examples/BookingSystem.AspNetCore.IdentityServer/Custom/Grants/PersistedGrantStore.cs

@@ -11,7 +11,7 @@ public class AcmePersistedGrantStore : IPersistedGrantStore
     {
         public async Task<IEnumerable<PersistedGrant>> GetAllAsync(string subjectId)
         {
-            var grants = await FakeBookingSystem.Database.GetAllGrants(subjectId);
+            var grants = await FakeBookingSystem.FakeDatabase.GetAllGrants(subjectId);
             var persistedGrants = grants.Select(grant => new PersistedGrant
             {
                 Key = grant.Key,
@@ -28,7 +28,7 @@ public async Task<IEnumerable<PersistedGrant>> GetAllAsync(string subjectId)
 
         public async Task<PersistedGrant> GetAsync(string key)
         {
-            var grant = await FakeBookingSystem.Database.GetGrant(key);
+            var grant = await FakeBookingSystem.FakeDatabase.GetGrant(key);
 
             return grant != null ? new PersistedGrant
             {
@@ -44,25 +44,25 @@ public async Task<PersistedGrant> GetAsync(string key)
 
         public Task RemoveAllAsync(string subjectId, string clientId)
         {
-            FakeBookingSystem.Database.RemoveGrant(subjectId, clientId);
+            FakeBookingSystem.FakeDatabase.RemoveGrant(subjectId, clientId);
             return Task.CompletedTask;
         }
 
         public Task RemoveAllAsync(string subjectId, string clientId, string type)
         {
-            FakeBookingSystem.Database.RemoveGrant(subjectId, clientId, type);
+            FakeBookingSystem.FakeDatabase.RemoveGrant(subjectId, clientId, type);
             return Task.CompletedTask;
         }
 
         public Task RemoveAsync(string key)
         {
-            FakeBookingSystem.Database.RemoveGrant(key);
+            FakeBookingSystem.FakeDatabase.RemoveGrant(key);
             return Task.CompletedTask;
         }
 
         public Task StoreAsync(PersistedGrant grant)
         {
-            FakeBookingSystem.Database.AddGrant(grant.Key, grant.Type, grant.SubjectId, grant.ClientId, grant.CreationTime, grant.Expiration, grant.Data);
+            FakeBookingSystem.FakeDatabase.AddGrant(grant.Key, grant.Type, grant.SubjectId, grant.ClientId, grant.CreationTime, grant.Expiration, grant.Data);
             return Task.CompletedTask;
         }
     }

Examples/BookingSystem.AspNetCore.IdentityServer/Custom/Users/UserRepository.cs

@@ -16,19 +16,19 @@ public UserRepository(string jsonLdIdBaseUrl)
 
         public Task<bool> ValidateCredentials(string username, string password)
         {
-            return FakeBookingSystem.Database.ValidateSellerUserCredentials(username, password);
+            return FakeBookingSystem.FakeDatabase.ValidateSellerUserCredentials(username, password);
         }
 
         public async Task<UserWithClaims> FindBySubjectId(string subjectId)
         {
             return long.TryParse(subjectId, out var longSubjectId)
-                ? GetUserFromSellerUserWithClaims(await FakeBookingSystem.Database.GetSellerUserById(longSubjectId))
+                ? GetUserFromSellerUserWithClaims(await FakeBookingSystem.FakeDatabase.GetSellerUserById(longSubjectId))
                 : null;
         }
 
         public async Task<User> FindByUsername(string username)
         {
-            return GetUserFromSellerUser(await FakeBookingSystem.Database.GetSellerUser(username));
+            return GetUserFromSellerUser(await FakeBookingSystem.FakeDatabase.GetSellerUser(username));
         }
 
         // TODO: Make this an extension method

Examples/BookingSystem.AspNetCore/BackgroundServices/FakeDataRefresherService.cs

@@ -0,0 +1,44 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using OpenActive.FakeDatabase.NET;
+
+
+namespace BookingSystem
+{
+    // Background task
+    // More information: https://docs.microsoft.com/en-us/dotnet/architecture/microservices/multi-container-microservice-net-applications/background-tasks-with-ihostedservice#implementing-ihostedservice-with-a-custom-hosted-service-class-deriving-from-the-backgroundservice-base-class
+    public class FakeDataRefresherService : BackgroundService
+    {
+        private readonly ILogger<FakeDataRefresherService> _logger;
+        private readonly AppSettings _settings;
+
+        public FakeDataRefresherService(AppSettings settings, ILogger<FakeDataRefresherService> logger)
+        {
+            _settings = settings;
+            _logger = logger;
+        }
+
+        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+        {
+            _logger.LogDebug($"FakeDataRefresherService is starting..");
+
+            stoppingToken.Register(() =>
+                _logger.LogDebug($"FakeDataRefresherService background task is stopping."));
+
+            while (!stoppingToken.IsCancellationRequested)
+            {
+                await FakeBookingSystem.FakeDatabase.HardDeletedOldSoftDeletedOccurrencesAndSlots();
+                _logger.LogDebug($"FakeDataRefresherService hard deleted opportunities that were previously old and soft deleted");
+
+                await FakeBookingSystem.FakeDatabase.SoftDeletedPastOpportunitiesAndInsertNewAtEdgeOfWindow();
+                _logger.LogDebug($"FakeDataRefresherService soft deleted opportunities and inserted new ones at edge of window.");
+
+                _logger.LogDebug($"FakeDataRefresherService is finished");
+                await Task.Delay(_settings.DataRefresherInterval, stoppingToken);
+            }
+        }
+    }
+}

Examples/BookingSystem.AspNetCore/BookingSystem.AspNetCore.csproj

@@ -1,8 +1,10 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
     <UserSecretsId>aspnet-BookingSystem.AspNetCore-443B4F82-A20C-41CE-9924-329A0BCF0D14</UserSecretsId>
+    <!-- <AspNetCoreHostingModel>OutOfProcess</AspNetCoreHostingModel>
+    <AspNetCoreModuleName>AspNetCoreModule</AspNetCoreModuleName> -->
   </PropertyGroup>
 
   <ItemGroup>
@@ -13,6 +15,9 @@
     <PackageReference Include="Microsoft.AspNetCore.Authentication.AzureADB2C.UI" Version="2.1.1" />
     <PackageReference Include="Microsoft.AspNetCore.Razor.Design" Version="2.1.2" PrivateAssets="All" />
     <PackageReference Include="OpenActive.NET" Version="15.2.5" />
+    <PackageReference Include="Microsoft.AspNetCore.AzureAppServicesIntegration" Version="3.1.16" />
+    <PackageReference Include="Microsoft.AspNetCore.AzureAppServices.HostingStartup" Version="3.1.16" />
+    <PackageReference Include="Microsoft.Extensions.Logging.AzureAppServices" Version="5.0.8" />
   </ItemGroup>
 
   <ItemGroup>
@@ -22,6 +27,7 @@
 
   <ItemGroup>
     <Folder Include="Authentication\" />
+    <Folder Include="BackgroundServices\" />
   </ItemGroup>
   <PropertyGroup>
     <NoWarn>1701;1702;1591</NoWarn>

Examples/BookingSystem.AspNetCore/Feeds/FacilitiesFeeds.cs

@@ -24,7 +24,7 @@ public AcmeFacilityUseRpdeGenerator(bool useSingleSellerMode)
 
         protected override async Task<List<RpdeItem<FacilityUse>>> GetRpdeItems(long? afterTimestamp, long? afterId)
         {
-            using (var db = FakeBookingSystem.Database.Mem.Database.Open())
+            using (var db = FakeBookingSystem.FakeDatabase.DatabaseWrapper.Database.Open())
             {
                 var q = db.From<FacilityUseTable>()
                 .Join<SellerTable>()
@@ -126,7 +126,7 @@ public class AcmeFacilityUseSlotRpdeGenerator : RpdeFeedModifiedTimestampAndIdLo
 
         protected override async Task<List<RpdeItem<Slot>>> GetRpdeItems(long? afterTimestamp, long? afterId)
         {
-            using (var db = FakeBookingSystem.Database.Mem.Database.Open())
+            using (var db = FakeBookingSystem.FakeDatabase.DatabaseWrapper.Database.Open())
             {
                 var query = db.Select<SlotTable>()
                 .OrderBy(x => x.Modified)

Examples/BookingSystem.AspNetCore/Feeds/OrdersFeed.cs

@@ -20,7 +20,7 @@ protected override async Task<List<RpdeItem>> GetRPDEItems(string clientId, long
             // and update this class to inherit from OrdersRPDEFeedIncrementingUniqueChangeNumber
             // (to use afterChangeNumber, instead of afterTimestamp and afterId)
 
-            using (var db = FakeBookingSystem.Database.Mem.Database.Open())
+            using (var db = FakeBookingSystem.FakeDatabase.DatabaseWrapper.Database.Open())
             {
                 long afterTimestampLong = afterTimestamp ?? 0;
                 var q = db.From<OrderTable>()
@@ -112,7 +112,7 @@ public class AcmeOrderProposalsFeedRpdeGenerator : OrdersRPDEFeedModifiedTimesta
 
         protected override async Task<List<RpdeItem>> GetRPDEItems(string clientId, long? afterTimestamp, string afterId)
         {
-            using (var db = FakeBookingSystem.Database.Mem.Database.Open())
+            using (var db = FakeBookingSystem.FakeDatabase.DatabaseWrapper.Database.Open())
             {
                 long afterTimestampLong = afterTimestamp ?? 0;
                 var q = db.From<OrderTable>()

Examples/BookingSystem.AspNetCore/Feeds/SessionsFeeds.cs

@@ -17,7 +17,7 @@ public class AcmeScheduledSessionRpdeGenerator : RpdeFeedModifiedTimestampAndIdL
 
         protected override async Task<List<RpdeItem<ScheduledSession>>> GetRpdeItems(long? afterTimestamp, long? afterId)
         {
-            using (var db = FakeBookingSystem.Database.Mem.Database.Open())
+            using (var db = FakeBookingSystem.FakeDatabase.DatabaseWrapper.Database.Open())
             {
                 var query = db.Select<OccurrenceTable>()
                 .OrderBy(x => x.Modified)
@@ -74,7 +74,7 @@ public AcmeSessionSeriesRpdeGenerator(bool useSingleSellerMode)
 
         protected override async Task<List<RpdeItem<SessionSeries>>> GetRpdeItems(long? afterTimestamp, long? afterId)
         {
-            using (var db = FakeBookingSystem.Database.Mem.Database.Open())
+            using (var db = FakeBookingSystem.FakeDatabase.DatabaseWrapper.Database.Open())
             {
                 var q = db.From<ClassTable>()
                 .Join<SellerTable>()

Examples/BookingSystem.AspNetCore/Program.cs

@@ -8,13 +8,18 @@ public class Program
     {
         public static void Main(string[] args)
         {
-            // Initialising fake database (shared with IdentityServer)
+            var host = CreateWebHostBuilder(args)
+                .Build();
+
             FakeBookingSystem.Initialise();
-            CreateWebHostBuilder(args).Build().Run();
+
+            host.Run();
         }
 
         public static IWebHostBuilder CreateWebHostBuilder(string[] args) =>
             WebHost.CreateDefaultBuilder(args)
-                .UseStartup<Startup>();
+            .CaptureStartupErrors(true)
+            .UseSetting("detailedErrors", "true")
+            .UseStartup<Startup>();
     }
 }

Examples/BookingSystem.AspNetCore/Properties/launchSettings.json

@@ -11,7 +11,10 @@
       "applicationUrl": "https://localhost:5001",
       "environmentVariables": {
         "ASPNETCORE_ENVIRONMENT": "Development",
-        "ApplicationHostBaseUrl": "https://localhost:5001"
+        "ApplicationHostBaseUrl": "https://localhost:5001",
+        "REMOTE_STORAGE_CONNECTION_STRING": "",
+        "USE_REMOTE_STORAGE": "true",
+        "DROP_TABLES_ON_RESTART": "true"
       }
     }
   }

Examples/BookingSystem.AspNetCore/Settings/AppSettings.cs

@@ -1,3 +1,5 @@
+using System;
+
 namespace BookingSystem
 {
     public class AppSettings
@@ -6,6 +8,8 @@ public class AppSettings
         public string OpenIdIssuerUrl { get; set; }
         public FeatureSettings FeatureFlags { get; set; }
         public PaymentSettings Payment { get; set; }
+        public TimeSpan DataRefresherInterval = TimeSpan.FromHours(6);
+
     }
 
     /**

Examples/BookingSystem.AspNetCore/Startup.cs

@@ -60,7 +60,8 @@ public void ConfigureServices(IServiceCollection services)
                 services.AddAuthorization(options =>
                 {
                     // No authorization checks are performed, this just ensures that the required claims are supplied
-                    options.AddPolicy(OpenActiveScopes.OpenBooking, policy => {
+                    options.AddPolicy(OpenActiveScopes.OpenBooking, policy =>
+                    {
                         policy.RequireClaim(OpenActiveCustomClaimNames.ClientId);
                         policy.RequireClaim(OpenActiveCustomClaimNames.SellerId);
                     });
@@ -69,11 +70,16 @@ public void ConfigureServices(IServiceCollection services)
             }
 
             services
-                .AddMvc()
-                .AddMvcOptions(options => options.InputFormatters.Insert(0, new OpenBookingInputFormatter()))
-                .SetCompatibilityVersion(CompatibilityVersion.Version_2_1);
+                .AddControllers()
+                .AddMvcOptions(options => options.InputFormatters.Insert(0, new OpenBookingInputFormatter()));
+
+            // Add config as a singleton to pipe it through DI to the booking engine and stores
+            services.AddSingleton(x => AppSettings);
 
             services.AddSingleton<IBookingEngine>(sp => EngineConfig.CreateStoreBookingEngine(AppSettings));
+
+            // Add background OrderItem polling
+            services.AddHostedService<FakeDataRefresherService>();
         }
 
         // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
@@ -93,8 +99,13 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env)
 
             app.UseHttpsRedirection();
             app.UseStaticFiles();
+            app.UseRouting();
             app.UseAuthentication();
-            app.UseMvc();
+            app.UseAuthorization();
+            app.UseEndpoints(endpoints =>
+            {
+                endpoints.MapControllers();
+            });
         }
     }
 }