Prepare release

CHANGELOG.md

@@ -1,3 +1,7 @@
+## v2.0.0-beta.28 - 2024-08-26
+
+- Improved test coverage and fixed the core file resolution algorithm
+
 ## v2.0.0-beta.27 - 2024-08-26
 
 - Added support for file named after the end of the period of the data they contain

README.md

@@ -23,4 +23,6 @@ These tests cover all cases for file/folder structures which contain files with
 | M    | datetime file with random start time                                                                      | 00:00:00           | 01:00:00             | x     |
 | N    | date folder, time file                                                                                    | 12:00:00           | 06:00:00             |       |
 | O    | datetime file with random start time and interval exceeding end                                           | 00:00:00           | 01:00:00             | x     |
+| O2   | datetime file with random start time and interval exceeding end, different time period for test           | 00:00:00           | 01:00:00             | x     |
 | P    | datetime file with file name offset                                                                       | 00:00:00           | 00:10:00             |       |
+| Q    | datetime file with file period larger than UTC offset                                                     | 01:00:00           | 1.00:00:00           |       |

src/Nexus.Sources.StructuredFile/StructuredFileDataSource.cs

@@ -380,15 +380,15 @@ protected virtual async Task ReadAsync(
                     // get next file source
                     var nextFileSource = fileSourceGroup.FirstOrDefault(current => current.Begin > fileSourceCompensatedBegin);
 
-                    var fileSourceEnd = nextFileSource is null
+                    var fileSourceCompensatedEnd = nextFileSource is null
                         ? end
                         : new DateTime(Math.Min(end.Ticks, nextFileSource.Begin.Ticks), DateTimeKind.Utc);
 
                     // go!
                     var regularFileLength = fileSource.FilePeriod.Ticks / samplePeriod.Ticks;
                     var bufferOffset = (int)((fileSourceCompensatedBegin - begin).Ticks / samplePeriod.Ticks);
                     var currentBegin = fileSourceCompensatedBegin;
-                    var totalPeriod = fileSourceEnd - fileSourceCompensatedBegin;
+                    var totalPeriod = fileSourceCompensatedEnd - fileSourceCompensatedBegin;
                     var consumedPeriod = TimeSpan.Zero;
                     var remainingPeriod = totalPeriod;
 
@@ -406,7 +406,6 @@ protected virtual async Task ReadAsync(
                         Logger.LogTrace("Process period {CurrentBegin} to {CurrentEnd}", currentBegin, currentBegin + currentPeriod);
 
                         var fileBlock = (int)(currentPeriod.Ticks / samplePeriod.Ticks);
-                        var fileOffset = consumedFilePeriod.Ticks / samplePeriod.Ticks;
 
                         foreach (var (filePath, fileBeginOffset) in fileInfos)
                         {
@@ -422,41 +421,64 @@ protected virtual async Task ReadAsync(
 
                             if (File.Exists(filePath))
                             {
-                                // compensate offsets and lengths in case of incomplete or irregular file
                                 var fileCompensation = (int)(fileBeginOffset.Ticks / samplePeriod.Ticks);
 
+                                /* Actual buffer offset */
                                 var actualBufferOffset = bufferOffset +
-                                    (
-                                        fileCompensation < 0 /* = irregular file */
+                                (
+                                    fileCompensation < 0 /* = irregular file */
 
-                                            /* The irregular file has data for the current buffer position, no action required */
-                                            ? 0
+                                        /* The irregular file has data for the current buffer position, no action required */
+                                        ? 0
 
-                                            /* The irregular or incomplete file contains no data for the current buffer position, so compensate for it */
-                                            : +fileCompensation
-                                    );
+                                        /* The irregular or incomplete file contains no data for the current buffer position, so compensate for it */
+                                        : fileCompensation
+                                );
 
-                                /* The irregular or incomplete file contains not enough data, so make the file block smaller */
-                                var actualFileBlock = fileBlock - Math.Abs(fileCompensation);
+                                /* Actual file block */
+                                var actualFileBegin = currentBegin + fileBeginOffset;
 
-                                /* Irregular or incomplete file: Compensate the file offset */
-                                var actualFileOffset = fileOffset +
-                                    (
-                                        fileCompensation < 0 /* = irregular file */
+                                var actualBegin = actualFileBegin > currentBegin
+                                    ? actualFileBegin
+                                    : currentBegin;
 
-                                            /* The irregular file starts earlier than expected, so compensate for it */
-                                            ? -fileCompensation
+                                Span<long> potentialEndTicks = [
 
-                                            /* The irregular or incomplete file starts later than expected */
-                                            : -fileCompensation
-                                    );
+                                    /* The end of the regular time interval */
+                                    (regularUtcFileBegin + fileSource.FilePeriod).Ticks,
+
+                                    /* The end of the irregular time interval */
+                                    fileSource.IrregularTimeInterval
+                                        ? (actualFileBegin + fileSource.FilePeriod).Ticks
+                                        : long.MaxValue,
+
+                                    /* The end of the overall time interval */
+                                    fileSourceCompensatedEnd.Ticks
+
+                                ];
+
+                                var actualEnd = new DateTime(Math.Min(Math.Min(potentialEndTicks[0], potentialEndTicks[1]), potentialEndTicks[2]));
+                                var actualPeriod = actualEnd - actualBegin;
+
+                                if (actualPeriod <= TimeSpan.Zero)
+                                    continue;
+
+                                var actualFileBlock = (int)(actualPeriod.Ticks / samplePeriod.Ticks);
+
+                                /* Actual file offset */
+                                var actualFileOffset = fileCompensation < 0
+
+                                    /* The irregular file has data available even before the currently requested period,
+                                     * i.e. a file offset is required 
+                                     */
+                                    ? Math.Abs(fileCompensation)
 
-                                /* The maximum value for fileCompensation is MaxFileBlock = FilePeriod / SamplePeriod
-                                * so there is no need to check for actualFileOffset >= MaxFileBlock.
-                                * However, it might happen that actualFileOffset < 0. This must be compensated. */
-                                if (actualFileOffset < 0)
-                                    actualFileOffset = 0;
+                                    /* The irregular or incomplete file contains no data for the currently requested period, 
+                                     * i.e. we provide data right from the start of the file
+                                     */
+                                    : 0;
 
+                                /* Go! */
                                 Logger.LogTrace("Process file {FilePath}", filePath);
 
                                 try
@@ -582,20 +604,20 @@ protected abstract Task ReadAsync(
             ? DateTime.SpecifyKind(begin.Add(fileSource.UtcOffset), DateTimeKind.Local)
             : throw new ArgumentException("The begin parameter must of kind UTC.");
 
-        var localFileBegin = localBegin
+        var regularLocalFileBegin = localBegin
             .RoundDown(fileSource.FilePeriod);
 
         var folderNames = fileSource
             .PathSegments
-            .Select(localFileBegin.ToString);
+            .Select(regularLocalFileBegin.ToString);
 
         var folderNameArray = new List<string>() { Root }
             .Concat(folderNames)
             .ToArray();   
 
         var regularUtcFileBegin = new CustomDateTimeOffset
         (
-            DateTime.SpecifyKind(localFileBegin, DateTimeKind.Unspecified),
+            DateTime.SpecifyKind(regularLocalFileBegin, DateTimeKind.Unspecified),
             fileSource.UtcOffset
         ).UtcDateTime;
 
@@ -636,7 +658,7 @@ protected abstract Task ReadAsync(
                         current.DateTimeOffset.UtcDateTime < regularUtcFileEnd;
                 }
             })
-            .Select(current => (current.FilePath, current.DateTimeOffset.UtcDateTime - regularUtcFileBegin))
+            .Select(current => (current.FilePath, current.DateTimeOffset.UtcDateTime - begin))
             .ToArray();
 
             // no files left for current time period - try to find next file after this period
@@ -645,7 +667,7 @@ protected abstract Task ReadAsync(
                 fileInfos = tmpFileInfos
                     .OrderBy(x => x.DateTimeOffset.DateTime)
                     .Where(current => regularUtcFileEnd <= current.DateTimeOffset.UtcDateTime)
-                    .Select(current => (current.FilePath, current.DateTimeOffset.UtcDateTime - regularUtcFileBegin))
+                    .Select(current => (current.FilePath, current.DateTimeOffset.UtcDateTime - begin))
                     .Take(1)
                     .ToArray();
             }
@@ -655,12 +677,12 @@ protected abstract Task ReadAsync(
         {
             /* correct local file begin */
             if (fileSource.FileNameOffset != TimeSpan.Zero)
-                localFileBegin = localFileBegin.Add(fileSource.FileNameOffset);
+                regularLocalFileBegin = regularLocalFileBegin.Add(fileSource.FileNameOffset);
 
             var folderPath = Path.Combine(folderNameArray);
-            var fileName = localFileBegin.ToString(fileSource.FileTemplate);
+            var fileName = regularLocalFileBegin.ToString(fileSource.FileTemplate);
 
-            fileInfos = [(Path.Combine(folderPath, fileName), TimeSpan.Zero)];
+            fileInfos = [(Path.Combine(folderPath, fileName), regularUtcFileBegin - begin)];
         }
 
         return Task.FromResult((regularUtcFileBegin, fileInfos));

tests/Nexus.Sources.StructuredFile.Tests/DATABASES/Q/config.json

@@ -0,0 +1,14 @@
+{
+  "/A/B/C": {
+    "default": [
+      {
+        "PathSegments": [
+          "'DATA'"
+        ],
+        "FileTemplate": "yyyy-MM-ddTHH-mm-ss'.dat'",
+        "FilePeriod": "1.00:00:00",
+        "UtcOffset": "01:00:00"
+      }
+    ]
+  }
+}

tests/Nexus.Sources.StructuredFile.Tests/StructuredFileDataSourceTests.cs

@@ -116,6 +116,7 @@ public async Task CanProvideFirstFile()
     [InlineData("DATABASES/N", "2019-12-31T12-00-00Z", "2020-01-03T12-00-00Z")]
     [InlineData("DATABASES/O", "2020-01-01T00-35-23Z", "2020-01-01T01-00-10Z")]
     [InlineData("DATABASES/P", "2020-01-01T00-00-00Z", "2020-01-01T00-20-00Z")]
+    [InlineData("DATABASES/Q", "2019-12-31T23-00-00Z", "2020-01-02T23-00-00Z")]
     public async Task CanProvideTimeRange(string root, string expectedBeginString, string expectedEndString)
     {
         var expectedBegin = DateTime.ParseExact(expectedBeginString, "yyyy-MM-ddTHH-mm-ssZ", null, DateTimeStyles.AdjustToUniversal);
@@ -154,6 +155,7 @@ public async Task CanProvideTimeRange(string root, string expectedBeginString, s
     [InlineData("DATABASES/N", "2020-01-01T00-00-00Z", "2020-01-03T00-00-00Z", 7 / 8.0, 2)]
     [InlineData("DATABASES/O", "2020-01-01T00-00-00Z", "2020-01-02T00-00-00Z", 3 / 288.0, 4)]
     [InlineData("DATABASES/P", "2020-01-01T00-00-00Z", "2020-01-02T00-00-00Z", 2 / 144.0, 4)]
+    [InlineData("DATABASES/Q", "2020-01-01T00-00-00Z", "2020-01-03T00-00-00Z", 0.5, 1)]
     public async Task CanProvideAvailability(string root, string beginString, string endString, double expected, int precision)
     {
         // Arrange
@@ -299,15 +301,17 @@ await Assert.ThrowsAsync<ArgumentException>(() =>
     [InlineData("M", "2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z")]
     [InlineData("N", "2020-01-01T00:00:00Z", "2020-01-04T00:00:00Z")]
     [InlineData("O", "2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z")]
+    [InlineData("O2", "2020-01-01T00:35:23Z", "2020-01-01T00:35:24Z")]
     [InlineData("P", "2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z")]
+    [InlineData("Q", "2020-01-01T00:00:00Z", "2020-01-03T00:00:00Z")]
     public async Task CanRead_ReadInfos(string database, string beginString, string endString)
     {
         // Arrange
         var readInfos = new List<ReadInfo>();
         var dataSource = new StructuredFileDataSourceTester(readInfos.Add) as IDataSource;
 
         var context = new DataSourceContext(
-            ResourceLocator: new Uri(Path.Combine(Directory.GetCurrentDirectory(), $"DATABASES/{database}")),
+            ResourceLocator: new Uri(Path.Combine(Directory.GetCurrentDirectory(), $"DATABASES/{database[0]}")),
             SystemConfiguration: default!,
             SourceConfiguration: default!,
             RequestConfiguration: default!);
@@ -345,7 +349,7 @@ await dataSource.ReadAsync(
             .OrderBy(x => x.FilePath);
 
         var actualJsonString = JsonSerializer.Serialize(preparedReadInfos, new JsonSerializerOptions { WriteIndented = true });
-        // File.WriteAllText($"{database}.json", actualJsonString);
+        File.WriteAllText($"{database}.json", actualJsonString);
 
         var expectedJsonString = File.ReadAllText($"expected/{database}.json");
 

tests/Nexus.Sources.StructuredFile.Tests/expected/O2.json

@@ -0,0 +1,11 @@
+[
+  {
+    "BufferOffset": 0,
+    "FilePath": "O/DATA/2020-01-01T00-35-23Z.dat",
+    "FileSource": null,
+    "RegularFileBegin": "2020-01-01T00:35:00Z",
+    "FileOffset": 0,
+    "FileBlock": 1,
+    "FileLength": 300
+  }
+]

tests/Nexus.Sources.StructuredFile.Tests/expected/Q.json

@@ -0,0 +1,20 @@
+[
+  {
+    "BufferOffset": 0,
+    "FilePath": "Q/DATA/2020-01-01T00-00-00.dat",
+    "FileSource": null,
+    "RegularFileBegin": "2019-12-31T23:00:00Z",
+    "FileOffset": 3600,
+    "FileBlock": 82800,
+    "FileLength": 86400
+  },
+  {
+    "BufferOffset": 82800,
+    "FilePath": "Q/DATA/2020-01-02T00-00-00.dat",
+    "FileSource": null,
+    "RegularFileBegin": "2020-01-01T23:00:00Z",
+    "FileOffset": 0,
+    "FileBlock": 86400,
+    "FileLength": 86400
+  }
+]

version.json

@@ -1,4 +1,4 @@
 {
     "version": "2.0.0",
-    "suffix": "beta.27"
+    "suffix": "beta.28"
 }