diff --git a/BattleNetPrefill.Integration.Test/LogFileUpToDateTests.cs b/BattleNetPrefill.Integration.Test/LogFileUpToDateTests.cs index 2eb65e1b..5e18d648 100644 --- a/BattleNetPrefill.Integration.Test/LogFileUpToDateTests.cs +++ b/BattleNetPrefill.Integration.Test/LogFileUpToDateTests.cs @@ -24,7 +24,7 @@ public void LogFilesAreUpToDate(string productCode) private static VersionsEntry GetLatestCdnVersion(TactProduct product) { // Finding the latest version of the game - ConfigFileHandler configFileHandler = new ConfigFileHandler(new CdnRequestManager(AppConfig.BattleNetPatchUri, new TestConsole())); + ConfigFileHandler configFileHandler = new ConfigFileHandler(new CdnRequestManager(new TestConsole())); VersionsEntry cdnVersion = configFileHandler.GetLatestVersionEntryAsync(product).Result; return cdnVersion; } diff --git a/BattleNetPrefill.Integration.Test/Parsers/BuildConfigParserTests.cs b/BattleNetPrefill.Integration.Test/Parsers/BuildConfigParserTests.cs index 7f169bf8..ae257b06 100644 --- a/BattleNetPrefill.Integration.Test/Parsers/BuildConfigParserTests.cs +++ b/BattleNetPrefill.Integration.Test/Parsers/BuildConfigParserTests.cs @@ -33,7 +33,7 @@ public async Task BuildConfig_ShouldHaveNoUnknownKeyPairs(string productCode) // Setting up required classes AppConfig.SkipDownloads = true; - CdnRequestManager cdnRequestManager = new CdnRequestManager(AppConfig.BattleNetPatchUri, new TestConsole()); + CdnRequestManager cdnRequestManager = new CdnRequestManager(new TestConsole()); await cdnRequestManager.InitializeAsync(tactProduct); var configFileHandler = new ConfigFileHandler(cdnRequestManager); VersionsEntry targetVersion = await configFileHandler.GetLatestVersionEntryAsync(tactProduct); diff --git a/BattleNetPrefill/AppConfig.cs b/BattleNetPrefill/AppConfig.cs index 29e22584..9fef931b 100644 --- a/BattleNetPrefill/AppConfig.cs +++ b/BattleNetPrefill/AppConfig.cs @@ -27,16 +27,10 @@ static AppConfig() public static readonly string UserSelectedAppsPath = Path.Combine(ConfigDir, "selectedAppsToPrefill.json"); - //TODO comment - private static bool _verboseLogs; public static bool VerboseLogs { - get => _verboseLogs; - set - { - _verboseLogs = value; - AnsiConsoleExtensions.WriteVerboseLogs = value; - } + get => AnsiConsoleExtensions.WriteVerboseLogs; + set => AnsiConsoleExtensions.WriteVerboseLogs = value; } /// @@ -56,7 +50,7 @@ public static bool VerboseLogs public static readonly string LogFileBasePath = @$"{DirectorySearch.TryGetSolutionDirectory()}/Logs"; /// - /// /// When enabled, will skip using any locally cached index files from disk. + /// When enabled, will skip using any locally cached index files from disk. /// The disk cache can speed up repeated runs, however it can use up a non-trivial amount /// of storage in some cases (Wow uses several hundred mb of index files). Intended for debugging. /// diff --git a/BattleNetPrefill/Program.cs b/BattleNetPrefill/Program.cs index f33498f6..d153117c 100644 --- a/BattleNetPrefill/Program.cs +++ b/BattleNetPrefill/Program.cs @@ -43,7 +43,6 @@ private static List ParseHiddenFlags() // Have to skip the first argument, since its the path to the executable var args = Environment.GetCommandLineArgs().Skip(1).ToList(); - // TODO comment if (args.Any(e => e.Contains("--compare-requests"))) { AnsiConsole.Console.LogMarkupLine($"Using {LightYellow("--compare-requests")} flag. Running comparison logic..."); diff --git a/BattleNetPrefill/Properties/launchSettings.json b/BattleNetPrefill/Properties/launchSettings.json index b24bef6e..59f0f6b2 100644 --- a/BattleNetPrefill/Properties/launchSettings.json +++ b/BattleNetPrefill/Properties/launchSettings.json @@ -4,9 +4,9 @@ "commandName": "Project", "commandLineArgs": "prefill --all --force" }, - "Prefill All - No Download": { + "Prefill Test Products - No Download": { "commandName": "Project", - "commandLineArgs": "prefill --all --no-download --force" + "commandLineArgs": "prefill -p s1 s2 pro wow wow_classic --no-download --force" }, "Prefill All - No Download - No Cache": { "commandName": "Project", @@ -14,7 +14,7 @@ }, "Prefill Starcraft 1": { "commandName": "Project", - "commandLineArgs": "prefill -p s1 s2 pro wow --no-download --force" + "commandLineArgs": "prefill -p s1 --force" }, "Prefill Overwatch": { "commandName": "Project", @@ -22,7 +22,7 @@ }, "Prefill WOW - Compare Requests": { "commandName": "Project", - "commandLineArgs": "prefill -p wow_classic --compare-requests --no-download --force" + "commandLineArgs": "prefill -p wow --compare-requests --no-download --force" }, "Select Apps": { "commandName": "Project", diff --git a/BattleNetPrefill/TactProductHandler.cs b/BattleNetPrefill/TactProductHandler.cs index 12564e2f..f0ec1459 100644 --- a/BattleNetPrefill/TactProductHandler.cs +++ b/BattleNetPrefill/TactProductHandler.cs @@ -43,12 +43,13 @@ public async Task ProcessMultipleProductsAsync(List productsToProce /// public async Task ProcessProductAsync(TactProduct product) { + //TODO it would be nice if this was only displayed when there is something to download _ansiConsole.LogMarkupLine($"Starting {Cyan(product.DisplayName)}"); var metadataTimer = Stopwatch.StartNew(); // Initializing classes, now that we have our CDN info loaded - using var cdnRequestManager = new CdnRequestManager(AppConfig.BattleNetPatchUri, _ansiConsole, AppConfig.NoLocalCache); + using var cdnRequestManager = new CdnRequestManager(_ansiConsole); var downloadFileHandler = new DownloadFileHandler(cdnRequestManager); var configFileHandler = new ConfigFileHandler(cdnRequestManager); var installFileHandler = new InstallFileHandler(cdnRequestManager); @@ -87,6 +88,13 @@ await Task.WhenAll(archiveIndexHandler.BuildArchiveIndexesAsync(cdnConfig, ctx), _ansiConsole.LogMarkupLine("Retrieved product metadata", metadataTimer); + //TODO this breaks things so that it no longer correctly shows the total amount of bytes downloaded + if (AppConfig.SkipDownloads) + { + _ansiConsole.MarkupLine(""); + return new ComparisonResult(); + } + // Actually start the download of any deferred requests var downloadSuccessful = await cdnRequestManager.DownloadQueuedRequestsAsync(_prefillSummaryResult); if (downloadSuccessful) diff --git a/BattleNetPrefill/Utils/Debug/Models/Request.cs b/BattleNetPrefill/Utils/Debug/Models/Request.cs index 169b2e39..e9f27582 100644 --- a/BattleNetPrefill/Utils/Debug/Models/Request.cs +++ b/BattleNetPrefill/Utils/Debug/Models/Request.cs @@ -2,7 +2,7 @@ { //TODO probably shouldn't be in debug namespace /// - /// Model that represents a request that could be made to a CDN. + /// Model that represents a request that could be made to a CDN. /// public sealed class Request { @@ -18,7 +18,6 @@ public Request(string productRootUri, RootFolder rootFolder, MD5Hash cdnKey, lon RootFolder = rootFolder; CdnKey = cdnKey; IsIndex = isIndex; - WriteToDevNull = writeToDevNull; if (startBytes != null && endBytes != null) { @@ -47,32 +46,26 @@ public Request(string productRootUri, RootFolder rootFolder, MD5Hash cdnKey, lon public long LowerByteRange { get; set; } public long UpperByteRange { get; set; } - //TODO this name kind of sucks - public bool WriteToDevNull { get; set; } - // Bytes are an inclusive range. Ex bytes 0->9 == 10 bytes public long TotalBytes => (UpperByteRange - LowerByteRange) + 1; /// /// Request path, without a host name. Agnostic towards host name, since we will combine with it later if needed to make a real request. /// Example : - /// tpr/sc1live/data/b5/20/b520b25e5d4b5627025aeba235d60708 + /// tpr/sc1live/data/b5/20/b520b25e5d4b5627025a6ba235d60708 /// - private string _uri; public string Uri { get { - if (_uri == null) + var hashId = CdnKey.ToStringLower(); + var uri = $"{ProductRootUri}/{RootFolder.Name}/{hashId.Substring(0, 2)}/{hashId.Substring(2, 2)}/{hashId}"; + + if (IsIndex) { - var hashId = CdnKey.ToStringLower(); - _uri = $"{ProductRootUri}/{RootFolder.Name}/{hashId.Substring(0, 2)}/{hashId.Substring(2, 2)}/{hashId}"; - if (IsIndex) - { - _uri = $"{_uri}.index"; - } + return $"{uri}.index"; } - return _uri; + return uri; } } @@ -92,7 +85,7 @@ public bool Overlaps(Request request2, bool isBattleNetClient) int overlap = 1; if (isBattleNetClient) { - // For some reason, the real Battle.Net client seems to combine requests if their ranges are within 4kb of each other. + // For some reason, the real Battle.Net client seems to combine requests if their ranges are within 4kb of each other. // This does not make intuitive sense, as looking at the entries in the Archive Index shows that the range should not be requested, // ex. only bytes 0-340 and bytes 1400-2650 should be individually requested. However these two individual requests get combined into 0-2650 overlap = 4096; diff --git a/BattleNetPrefill/Utils/Debug/RequestUtils.cs b/BattleNetPrefill/Utils/Debug/RequestUtils.cs index 9cacc4c9..35f69cf7 100644 --- a/BattleNetPrefill/Utils/Debug/RequestUtils.cs +++ b/BattleNetPrefill/Utils/Debug/RequestUtils.cs @@ -14,7 +14,7 @@ public static List CoalesceRequests(List initialRequests, bool { var coalesced = new List(); - // Coalescing any requests to the same URI that have sequential/overlapping byte ranges. + // Coalescing any requests to the same URI that have sequential/overlapping byte ranges. var requestsGroupedByUri = initialRequests.GroupBy(e => new { e.RootFolder, e.CdnKey, e.IsIndex }).ToList(); foreach (var grouping in requestsGroupedByUri) { @@ -32,7 +32,7 @@ public static List CoalesceRequests(Dictionary> { var coalesced = new List(); - // Coalescing any requests to the same URI that have sequential/overlapping byte ranges. + // Coalescing any requests to the same URI that have sequential/overlapping byte ranges. foreach (var grouping in initialRequests.Values) { grouping.Sort((x, y) => x.LowerByteRange.CompareTo(y.LowerByteRange)); diff --git a/BattleNetPrefill/Web/CdnRequestManager.cs b/BattleNetPrefill/Web/CdnRequestManager.cs index 65bdce23..4b20f538 100644 --- a/BattleNetPrefill/Web/CdnRequestManager.cs +++ b/BattleNetPrefill/Web/CdnRequestManager.cs @@ -3,6 +3,7 @@ public sealed class CdnRequestManager : IDisposable { private readonly HttpClient _client; + private readonly IAnsiConsole _ansiConsole; private readonly List _cdnList = new List { @@ -24,18 +25,9 @@ public sealed class CdnRequestManager : IDisposable /// private string _productBasePath; - private readonly Uri _battleNetPatchUri; - + //TODO why is this a dictionary of requests? private readonly Dictionary> _queuedRequests = new Dictionary>(); - /// - /// When enabled, will skip using any cached files from disk. The disk cache can speed up repeated runs, however it can use up a non-trivial amount - /// of storage in some cases (Wow uses several hundred mb of index files). - /// - private bool SkipDiskCache; - - private readonly IAnsiConsole _ansiConsole; - #region Debugging /// @@ -43,16 +35,13 @@ public sealed class CdnRequestManager : IDisposable /// /// Must always be a ConcurrentBag, otherwise odd issues with unit tests can pop up due to concurrency /// - public ConcurrentBag allRequestsMade = new ConcurrentBag(); + public readonly ConcurrentBag allRequestsMade = new ConcurrentBag(); #endregion - public CdnRequestManager(Uri battleNetPatchUri, IAnsiConsole ansiConsole, bool skipDiskCache = false) + public CdnRequestManager(IAnsiConsole ansiConsole) { - _battleNetPatchUri = battleNetPatchUri; - SkipDiskCache = skipDiskCache; _ansiConsole = ansiConsole; - _client = new HttpClient(); } @@ -83,6 +72,7 @@ public void QueueRequest(RootFolder rootPath, in MD5Hash hash, in long? startByt { Request request = new Request(_productBasePath, rootPath, hash, startBytes, endBytes, writeToDevNull: true, isIndex); + //TODO why is this a dictionary? _queuedRequests.TryGetValue(hash, out List requests); if (requests == null) { @@ -106,16 +96,17 @@ public async Task DownloadQueuedRequestsAsync(PrefillSummaryResult prefill ByteSize totalDownloadSize = coalescedRequests.SumTotalBytes(); _queuedRequests.Clear(); - _ansiConsole.LogMarkup($"Downloading {Magenta(totalDownloadSize.ToDecimalString())}"); -#if DEBUG - _ansiConsole.Markup($" from {LightYellow(coalescedRequests.Count)} queued requests"); -#endif - _ansiConsole.MarkupLine(""); + _ansiConsole.LogMarkupVerbose($"Downloading {Magenta(totalDownloadSize.ToDecimalString())} from {LightYellow(coalescedRequests.Count)} queued requests"); + var downloadTimer = Stopwatch.StartNew(); var failedRequests = new ConcurrentBag(); await _ansiConsole.CreateSpectreProgress(AppConfig.TransferSpeedUnit).StartAsync(async ctx => { + if (AppConfig.SkipDownloads) + { + return; + } //TODO can probably cleanup this attempt 3 times logic since there is the polly stuff in place now. // Run the initial download failedRequests = await AttemptDownloadAsync(ctx, "Downloading..", coalescedRequests); @@ -159,37 +150,41 @@ private async Task> AttemptDownloadAsync(ProgressContext var progressTask = ctx.AddTask(taskTitle, new ProgressTaskSettings { MaxValue = requests.SumTotalBytes().Bytes }); var failedRequests = new ConcurrentBag(); - var downloadRequest = async (Request request, CancellationToken _) => - { - try - { - await GetRequestAsBytesAsync(request, progressTask, forceRecache); - } - catch - { - failedRequests.Add(request); - } - }; // Splitting up small/large requests into two batches. Splitting into two batches with different # of parallel requests will prevent the small // requests from choking out overall throughput. var byteThreshold = (long)ByteSize.FromMegaBytes(1).Bytes; + var smallRequests = requests.Where(e => e.TotalBytes < byteThreshold).ToList(); - var smallDownloadTask = Parallel.ForEachAsync(smallRequests, new ParallelOptions { MaxDegreeOfParallelism = 15 }, async (request, _) => await downloadRequest(request, _)); + var smallDownloadTask = Parallel.ForEachAsync(smallRequests, new ParallelOptions { MaxDegreeOfParallelism = 15 }, async (request, _) => await DownloadRequestWrapper(request, _)); var largeRequests = requests.Where(e => e.TotalBytes >= byteThreshold).ToList(); - var largeDownloadTask = Parallel.ForEachAsync(largeRequests, new ParallelOptions { MaxDegreeOfParallelism = 10 }, async (request, _) => await downloadRequest(request, _)); + var largeDownloadTask = Parallel.ForEachAsync(largeRequests, new ParallelOptions { MaxDegreeOfParallelism = 10 }, async (request, _) => await DownloadRequestWrapper(request, _)); await Task.WhenAll(smallDownloadTask, largeDownloadTask); // Making sure the progress bar is always set to its max value, some files don't have a size, so the progress bar will appear as unfinished. return failedRequests; + + async Task DownloadRequestWrapper(Request request, CancellationToken _) + { + try + { + await DownloadRequestAsync(request, forceRecache); + } + catch + { + failedRequests.Add(request); + } + + progressTask.Increment(request.TotalBytes); + }; } #endregion /// - /// Requests data from Blizzard's CDN, and returns the raw response. + /// Requests data from Blizzard's CDN, and returns the raw response. Will retry automatically. /// /// If true, will attempt to download a .index file for the specified hash /// If true, the response data will be ignored, and dumped to a null stream. @@ -206,21 +201,11 @@ public async Task GetRequestAsBytesAsync(RootFolder rootPath, MD5Hash ha }); } - - public async Task GetRequestAsBytesAsync(Request request, ProgressTask task = null, bool forceRecache = false) + //TODO not a fan of how many times forceRecache is passed down + private async Task GetRequestAsBytesAsync(Request request, bool forceRecache = false) { - var writeToDevNull = request.WriteToDevNull; - var startBytes = request.LowerByteRange; - var endBytes = request.UpperByteRange; - allRequestsMade.Add(request); - // When we are running in debug mode, we can skip any requests that will end up written to dev/null. Will speed up debugging. - if (AppConfig.SkipDownloads && writeToDevNull) - { - return null; - } - var uri = new Uri($"http://{_lancacheAddress}/{request.Uri}"); if (forceRecache) { @@ -228,9 +213,9 @@ public async Task GetRequestAsBytesAsync(Request request, ProgressTask t } // Try to return a cached copy from the disk first, before making an actual request - if (!writeToDevNull && !SkipDiskCache) + if (!AppConfig.NoLocalCache) { - string outputFilePath = Path.Combine(AppConfig.CacheDir + uri.AbsolutePath); + string outputFilePath = AppConfig.CacheDir + uri.AbsolutePath; if (File.Exists(outputFilePath)) { return await File.ReadAllBytesAsync(outputFilePath); @@ -241,57 +226,62 @@ public async Task GetRequestAsBytesAsync(Request request, ProgressTask t requestMessage.Headers.Host = _currentCdn; if (!request.DownloadWholeFile) { - requestMessage.Headers.Range = new RangeHeaderValue(startBytes, endBytes); + requestMessage.Headers.Range = new RangeHeaderValue(request.LowerByteRange, request.UpperByteRange); } using var cts = new CancellationTokenSource(); using var responseMessage = await _client.SendAsync(requestMessage, HttpCompletionOption.ResponseHeadersRead, cts.Token); await using Stream responseStream = await responseMessage.Content.ReadAsStreamAsync(cts.Token); - responseMessage.EnsureSuccessStatusCode(); - if (writeToDevNull) - { - byte[] buffer = new byte[4096]; - var totalBytesRead = 0; - try - { - while (true) - { - // Dump the received data, so we don't have to waste time writing it to disk. - var read = await responseStream.ReadAsync(buffer, cts.Token); - if (read == 0) - { - return null; - } - task.Increment(read); - totalBytesRead += read; - } - } - catch (Exception) - { - // Making sure that the current request is marked as "complete" in the progress bar, otherwise the progress bar will never hit 100% - task.Increment(request.TotalBytes - totalBytesRead); - throw; - } - } await using var memoryStream = new MemoryStream(); await responseStream.CopyToAsync(memoryStream, cts.Token); var byteArray = memoryStream.ToArray(); - if (SkipDiskCache) + // Prevents the response from being cached to disk when --no-cache is specified + if (AppConfig.NoLocalCache) { return await Task.FromResult(byteArray); } // Cache to disk - FileInfo file = new FileInfo(Path.Combine(AppConfig.CacheDir + uri.AbsolutePath)); + FileInfo file = new FileInfo(AppConfig.CacheDir + uri.AbsolutePath); file.Directory.Create(); await File.WriteAllBytesAsync(file.FullName, byteArray, cts.Token); return await Task.FromResult(byteArray); } + //TODO comment the difference + private async Task DownloadRequestAsync(Request request, bool forceRecache = false) + { + allRequestsMade.Add(request); + + var uri = new Uri($"http://{_lancacheAddress}/{request.Uri}"); + if (forceRecache) + { + uri = new Uri($"http://{_lancacheAddress}/{request.Uri}?nocache=1"); + } + + using var requestMessage = new HttpRequestMessage(HttpMethod.Get, uri); + requestMessage.Headers.Host = _currentCdn; + if (!request.DownloadWholeFile) + { + requestMessage.Headers.Range = new RangeHeaderValue(request.LowerByteRange, request.UpperByteRange); + } + + using var cts = new CancellationTokenSource(); + using var responseMessage = await _client.SendAsync(requestMessage, HttpCompletionOption.ResponseHeadersRead, cts.Token); + await using Stream responseStream = await responseMessage.Content.ReadAsStreamAsync(cts.Token); + responseMessage.EnsureSuccessStatusCode(); + + // Don't save the data anywhere, so we don't have to waste time writing it to disk. + var buffer = new byte[4096]; + while (await responseStream.ReadAsync(buffer, cts.Token) != 0) + { + } + } + /// /// Makes a request to the Patch API. Caches the response if possible. /// @@ -303,12 +293,12 @@ public async Task MakePatchRequestAsync(TactProduct tactProduct, PatchRe var cacheFile = $"{AppConfig.CacheDir}/{endpoint.Name}-{tactProduct.ProductCode}.txt"; // Load cached version, only valid for 30 minutes so that updated versions don't get accidentally ignored - if (!SkipDiskCache && File.Exists(cacheFile) && DateTime.Now < File.GetLastWriteTime(cacheFile).AddMinutes(30)) + if (!AppConfig.NoLocalCache && File.Exists(cacheFile) && DateTime.Now < File.GetLastWriteTime(cacheFile).AddMinutes(30)) { return await File.ReadAllTextAsync(cacheFile); } - using HttpResponseMessage response = await _client.GetAsync(new Uri($"{_battleNetPatchUri}{tactProduct.ProductCode}/{endpoint.Name}")); + using HttpResponseMessage response = await _client.GetAsync(new Uri($"{AppConfig.BattleNetPatchUri}{tactProduct.ProductCode}/{endpoint.Name}")); if (!response.IsSuccessStatusCode) { throw new Exception("Error during retrieving HTTP cdns: Received bad HTTP code " + response.StatusCode); @@ -316,7 +306,7 @@ public async Task MakePatchRequestAsync(TactProduct tactProduct, PatchRe using HttpContent res = response.Content; string content = await res.ReadAsStringAsync(); - if (SkipDiskCache) + if (AppConfig.NoLocalCache) { return content; } diff --git a/LogFileGenerator/Program.cs b/LogFileGenerator/Program.cs index 2f44c5f6..a68d2654 100644 --- a/LogFileGenerator/Program.cs +++ b/LogFileGenerator/Program.cs @@ -3,7 +3,7 @@ //TODO battlenet needs to be running in order for this to work public static class Program { - private static readonly CdnRequestManager CdnRequestManager = new CdnRequestManager(AppConfig.BattleNetPatchUri, AnsiConsole.Console); + private static readonly CdnRequestManager CdnRequestManager = new CdnRequestManager(AnsiConsole.Console); private static readonly ConfigFileHandler ConfigFileHandler = new ConfigFileHandler(CdnRequestManager); private static string RootInstallDir = @"C:\BattleNet";