From c6e58f410969985b03906987e2710bb35faf626f Mon Sep 17 00:00:00 2001 From: Apollo3zehn Date: Mon, 26 Aug 2024 22:05:01 +0200 Subject: [PATCH] Prepare release --- CHANGELOG.md | 4 + README.md | 2 + .../StructuredFileDataSource.cs | 92 ++++++++++++------- .../DATABASES/Q/DATA/2020-01-01T00-00-00.dat | 0 .../DATABASES/Q/DATA/2020-01-02T00-00-00.dat | 0 .../DATABASES/Q/config.json | 14 +++ .../StructuredFileDataSourceTests.cs | 8 +- .../expected/O2.json | 11 +++ .../expected/Q.json | 20 ++++ version.json | 2 +- 10 files changed, 115 insertions(+), 38 deletions(-) create mode 100644 tests/Nexus.Sources.StructuredFile.Tests/DATABASES/Q/DATA/2020-01-01T00-00-00.dat create mode 100644 tests/Nexus.Sources.StructuredFile.Tests/DATABASES/Q/DATA/2020-01-02T00-00-00.dat create mode 100644 tests/Nexus.Sources.StructuredFile.Tests/DATABASES/Q/config.json create mode 100644 tests/Nexus.Sources.StructuredFile.Tests/expected/O2.json create mode 100644 tests/Nexus.Sources.StructuredFile.Tests/expected/Q.json diff --git a/CHANGELOG.md b/CHANGELOG.md index ceaf2dd..30012f8 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.0.0-beta.28 - 2024-08-26 + +- Improved test coverage and fixed the core file resolution algorithm + ## v2.0.0-beta.27 - 2024-08-26 - Added support for file named after the end of the period of the data they contain diff --git a/README.md b/README.md index 54e840f..b76911c 100644 --- a/README.md +++ b/README.md @@ -23,4 +23,6 @@ These tests cover all cases for file/folder structures which contain files with | M | datetime file with random start time | 00:00:00 | 01:00:00 | x | | N | date folder, time file | 12:00:00 | 06:00:00 | | | O | datetime file with random start time and interval exceeding end | 00:00:00 | 01:00:00 | x | +| O2 | datetime file with random start time and interval exceeding end, different time period for test | 00:00:00 | 01:00:00 | x | | P | datetime file with file name offset | 00:00:00 | 00:10:00 | | +| Q | datetime file with file period larger than UTC offset | 01:00:00 | 1.00:00:00 | | diff --git a/src/Nexus.Sources.StructuredFile/StructuredFileDataSource.cs b/src/Nexus.Sources.StructuredFile/StructuredFileDataSource.cs index 1ed5893..258b09e 100644 --- a/src/Nexus.Sources.StructuredFile/StructuredFileDataSource.cs +++ b/src/Nexus.Sources.StructuredFile/StructuredFileDataSource.cs @@ -380,7 +380,7 @@ protected virtual async Task ReadAsync( // get next file source var nextFileSource = fileSourceGroup.FirstOrDefault(current => current.Begin > fileSourceCompensatedBegin); - var fileSourceEnd = nextFileSource is null + var fileSourceCompensatedEnd = nextFileSource is null ? end : new DateTime(Math.Min(end.Ticks, nextFileSource.Begin.Ticks), DateTimeKind.Utc); @@ -388,7 +388,7 @@ protected virtual async Task ReadAsync( var regularFileLength = fileSource.FilePeriod.Ticks / samplePeriod.Ticks; var bufferOffset = (int)((fileSourceCompensatedBegin - begin).Ticks / samplePeriod.Ticks); var currentBegin = fileSourceCompensatedBegin; - var totalPeriod = fileSourceEnd - fileSourceCompensatedBegin; + var totalPeriod = fileSourceCompensatedEnd - fileSourceCompensatedBegin; var consumedPeriod = TimeSpan.Zero; var remainingPeriod = totalPeriod; @@ -406,7 +406,6 @@ protected virtual async Task ReadAsync( Logger.LogTrace("Process period {CurrentBegin} to {CurrentEnd}", currentBegin, currentBegin + currentPeriod); var fileBlock = (int)(currentPeriod.Ticks / samplePeriod.Ticks); - var fileOffset = consumedFilePeriod.Ticks / samplePeriod.Ticks; foreach (var (filePath, fileBeginOffset) in fileInfos) { @@ -422,41 +421,64 @@ protected virtual async Task ReadAsync( if (File.Exists(filePath)) { - // compensate offsets and lengths in case of incomplete or irregular file var fileCompensation = (int)(fileBeginOffset.Ticks / samplePeriod.Ticks); + /* Actual buffer offset */ var actualBufferOffset = bufferOffset + - ( - fileCompensation < 0 /* = irregular file */ + ( + fileCompensation < 0 /* = irregular file */ - /* The irregular file has data for the current buffer position, no action required */ - ? 0 + /* The irregular file has data for the current buffer position, no action required */ + ? 0 - /* The irregular or incomplete file contains no data for the current buffer position, so compensate for it */ - : +fileCompensation - ); + /* The irregular or incomplete file contains no data for the current buffer position, so compensate for it */ + : fileCompensation + ); - /* The irregular or incomplete file contains not enough data, so make the file block smaller */ - var actualFileBlock = fileBlock - Math.Abs(fileCompensation); + /* Actual file block */ + var actualFileBegin = currentBegin + fileBeginOffset; - /* Irregular or incomplete file: Compensate the file offset */ - var actualFileOffset = fileOffset + - ( - fileCompensation < 0 /* = irregular file */ + var actualBegin = actualFileBegin > currentBegin + ? actualFileBegin + : currentBegin; - /* The irregular file starts earlier than expected, so compensate for it */ - ? -fileCompensation + Span potentialEndTicks = [ - /* The irregular or incomplete file starts later than expected */ - : -fileCompensation - ); + /* The end of the regular time interval */ + (regularUtcFileBegin + fileSource.FilePeriod).Ticks, + + /* The end of the irregular time interval */ + fileSource.IrregularTimeInterval + ? (actualFileBegin + fileSource.FilePeriod).Ticks + : long.MaxValue, + + /* The end of the overall time interval */ + fileSourceCompensatedEnd.Ticks + + ]; + + var actualEnd = new DateTime(Math.Min(Math.Min(potentialEndTicks[0], potentialEndTicks[1]), potentialEndTicks[2])); + var actualPeriod = actualEnd - actualBegin; + + if (actualPeriod <= TimeSpan.Zero) + continue; + + var actualFileBlock = (int)(actualPeriod.Ticks / samplePeriod.Ticks); + + /* Actual file offset */ + var actualFileOffset = fileCompensation < 0 + + /* The irregular file has data available even before the currently requested period, + * i.e. a file offset is required + */ + ? Math.Abs(fileCompensation) - /* The maximum value for fileCompensation is MaxFileBlock = FilePeriod / SamplePeriod - * so there is no need to check for actualFileOffset >= MaxFileBlock. - * However, it might happen that actualFileOffset < 0. This must be compensated. */ - if (actualFileOffset < 0) - actualFileOffset = 0; + /* The irregular or incomplete file contains no data for the currently requested period, + * i.e. we provide data right from the start of the file + */ + : 0; + /* Go! */ Logger.LogTrace("Process file {FilePath}", filePath); try @@ -582,12 +604,12 @@ protected abstract Task ReadAsync( ? DateTime.SpecifyKind(begin.Add(fileSource.UtcOffset), DateTimeKind.Local) : throw new ArgumentException("The begin parameter must of kind UTC."); - var localFileBegin = localBegin + var regularLocalFileBegin = localBegin .RoundDown(fileSource.FilePeriod); var folderNames = fileSource .PathSegments - .Select(localFileBegin.ToString); + .Select(regularLocalFileBegin.ToString); var folderNameArray = new List() { Root } .Concat(folderNames) @@ -595,7 +617,7 @@ protected abstract Task ReadAsync( var regularUtcFileBegin = new CustomDateTimeOffset ( - DateTime.SpecifyKind(localFileBegin, DateTimeKind.Unspecified), + DateTime.SpecifyKind(regularLocalFileBegin, DateTimeKind.Unspecified), fileSource.UtcOffset ).UtcDateTime; @@ -636,7 +658,7 @@ protected abstract Task ReadAsync( current.DateTimeOffset.UtcDateTime < regularUtcFileEnd; } }) - .Select(current => (current.FilePath, current.DateTimeOffset.UtcDateTime - regularUtcFileBegin)) + .Select(current => (current.FilePath, current.DateTimeOffset.UtcDateTime - begin)) .ToArray(); // no files left for current time period - try to find next file after this period @@ -645,7 +667,7 @@ protected abstract Task ReadAsync( fileInfos = tmpFileInfos .OrderBy(x => x.DateTimeOffset.DateTime) .Where(current => regularUtcFileEnd <= current.DateTimeOffset.UtcDateTime) - .Select(current => (current.FilePath, current.DateTimeOffset.UtcDateTime - regularUtcFileBegin)) + .Select(current => (current.FilePath, current.DateTimeOffset.UtcDateTime - begin)) .Take(1) .ToArray(); } @@ -655,12 +677,12 @@ protected abstract Task ReadAsync( { /* correct local file begin */ if (fileSource.FileNameOffset != TimeSpan.Zero) - localFileBegin = localFileBegin.Add(fileSource.FileNameOffset); + regularLocalFileBegin = regularLocalFileBegin.Add(fileSource.FileNameOffset); var folderPath = Path.Combine(folderNameArray); - var fileName = localFileBegin.ToString(fileSource.FileTemplate); + var fileName = regularLocalFileBegin.ToString(fileSource.FileTemplate); - fileInfos = [(Path.Combine(folderPath, fileName), TimeSpan.Zero)]; + fileInfos = [(Path.Combine(folderPath, fileName), regularUtcFileBegin - begin)]; } return Task.FromResult((regularUtcFileBegin, fileInfos)); diff --git a/tests/Nexus.Sources.StructuredFile.Tests/DATABASES/Q/DATA/2020-01-01T00-00-00.dat b/tests/Nexus.Sources.StructuredFile.Tests/DATABASES/Q/DATA/2020-01-01T00-00-00.dat new file mode 100644 index 0000000..e69de29 diff --git a/tests/Nexus.Sources.StructuredFile.Tests/DATABASES/Q/DATA/2020-01-02T00-00-00.dat b/tests/Nexus.Sources.StructuredFile.Tests/DATABASES/Q/DATA/2020-01-02T00-00-00.dat new file mode 100644 index 0000000..e69de29 diff --git a/tests/Nexus.Sources.StructuredFile.Tests/DATABASES/Q/config.json b/tests/Nexus.Sources.StructuredFile.Tests/DATABASES/Q/config.json new file mode 100644 index 0000000..aac9f9c --- /dev/null +++ b/tests/Nexus.Sources.StructuredFile.Tests/DATABASES/Q/config.json @@ -0,0 +1,14 @@ +{ + "/A/B/C": { + "default": [ + { + "PathSegments": [ + "'DATA'" + ], + "FileTemplate": "yyyy-MM-ddTHH-mm-ss'.dat'", + "FilePeriod": "1.00:00:00", + "UtcOffset": "01:00:00" + } + ] + } +} \ No newline at end of file diff --git a/tests/Nexus.Sources.StructuredFile.Tests/StructuredFileDataSourceTests.cs b/tests/Nexus.Sources.StructuredFile.Tests/StructuredFileDataSourceTests.cs index a266215..a47fff3 100644 --- a/tests/Nexus.Sources.StructuredFile.Tests/StructuredFileDataSourceTests.cs +++ b/tests/Nexus.Sources.StructuredFile.Tests/StructuredFileDataSourceTests.cs @@ -116,6 +116,7 @@ public async Task CanProvideFirstFile() [InlineData("DATABASES/N", "2019-12-31T12-00-00Z", "2020-01-03T12-00-00Z")] [InlineData("DATABASES/O", "2020-01-01T00-35-23Z", "2020-01-01T01-00-10Z")] [InlineData("DATABASES/P", "2020-01-01T00-00-00Z", "2020-01-01T00-20-00Z")] + [InlineData("DATABASES/Q", "2019-12-31T23-00-00Z", "2020-01-02T23-00-00Z")] public async Task CanProvideTimeRange(string root, string expectedBeginString, string expectedEndString) { var expectedBegin = DateTime.ParseExact(expectedBeginString, "yyyy-MM-ddTHH-mm-ssZ", null, DateTimeStyles.AdjustToUniversal); @@ -154,6 +155,7 @@ public async Task CanProvideTimeRange(string root, string expectedBeginString, s [InlineData("DATABASES/N", "2020-01-01T00-00-00Z", "2020-01-03T00-00-00Z", 7 / 8.0, 2)] [InlineData("DATABASES/O", "2020-01-01T00-00-00Z", "2020-01-02T00-00-00Z", 3 / 288.0, 4)] [InlineData("DATABASES/P", "2020-01-01T00-00-00Z", "2020-01-02T00-00-00Z", 2 / 144.0, 4)] + [InlineData("DATABASES/Q", "2020-01-01T00-00-00Z", "2020-01-03T00-00-00Z", 0.5, 1)] public async Task CanProvideAvailability(string root, string beginString, string endString, double expected, int precision) { // Arrange @@ -299,7 +301,9 @@ await Assert.ThrowsAsync(() => [InlineData("M", "2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z")] [InlineData("N", "2020-01-01T00:00:00Z", "2020-01-04T00:00:00Z")] [InlineData("O", "2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z")] + [InlineData("O2", "2020-01-01T00:35:23Z", "2020-01-01T00:35:24Z")] [InlineData("P", "2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z")] + [InlineData("Q", "2020-01-01T00:00:00Z", "2020-01-03T00:00:00Z")] public async Task CanRead_ReadInfos(string database, string beginString, string endString) { // Arrange @@ -307,7 +311,7 @@ public async Task CanRead_ReadInfos(string database, string beginString, string var dataSource = new StructuredFileDataSourceTester(readInfos.Add) as IDataSource; var context = new DataSourceContext( - ResourceLocator: new Uri(Path.Combine(Directory.GetCurrentDirectory(), $"DATABASES/{database}")), + ResourceLocator: new Uri(Path.Combine(Directory.GetCurrentDirectory(), $"DATABASES/{database[0]}")), SystemConfiguration: default!, SourceConfiguration: default!, RequestConfiguration: default!); @@ -345,7 +349,7 @@ await dataSource.ReadAsync( .OrderBy(x => x.FilePath); var actualJsonString = JsonSerializer.Serialize(preparedReadInfos, new JsonSerializerOptions { WriteIndented = true }); - // File.WriteAllText($"{database}.json", actualJsonString); + File.WriteAllText($"{database}.json", actualJsonString); var expectedJsonString = File.ReadAllText($"expected/{database}.json"); diff --git a/tests/Nexus.Sources.StructuredFile.Tests/expected/O2.json b/tests/Nexus.Sources.StructuredFile.Tests/expected/O2.json new file mode 100644 index 0000000..88889ee --- /dev/null +++ b/tests/Nexus.Sources.StructuredFile.Tests/expected/O2.json @@ -0,0 +1,11 @@ +[ + { + "BufferOffset": 0, + "FilePath": "O/DATA/2020-01-01T00-35-23Z.dat", + "FileSource": null, + "RegularFileBegin": "2020-01-01T00:35:00Z", + "FileOffset": 0, + "FileBlock": 1, + "FileLength": 300 + } +] \ No newline at end of file diff --git a/tests/Nexus.Sources.StructuredFile.Tests/expected/Q.json b/tests/Nexus.Sources.StructuredFile.Tests/expected/Q.json new file mode 100644 index 0000000..cef5ab7 --- /dev/null +++ b/tests/Nexus.Sources.StructuredFile.Tests/expected/Q.json @@ -0,0 +1,20 @@ +[ + { + "BufferOffset": 0, + "FilePath": "Q/DATA/2020-01-01T00-00-00.dat", + "FileSource": null, + "RegularFileBegin": "2019-12-31T23:00:00Z", + "FileOffset": 3600, + "FileBlock": 82800, + "FileLength": 86400 + }, + { + "BufferOffset": 82800, + "FilePath": "Q/DATA/2020-01-02T00-00-00.dat", + "FileSource": null, + "RegularFileBegin": "2020-01-01T23:00:00Z", + "FileOffset": 0, + "FileBlock": 86400, + "FileLength": 86400 + } +] \ No newline at end of file diff --git a/version.json b/version.json index 8b0eb77..bad48eb 100755 --- a/version.json +++ b/version.json @@ -1,4 +1,4 @@ { "version": "2.0.0", - "suffix": "beta.27" + "suffix": "beta.28" } \ No newline at end of file