From cf1e1634aa6ec128335be02e646e2d1d486dbe5c Mon Sep 17 00:00:00 2001 From: Bulat Farrakhov Date: Fri, 4 Sep 2020 16:11:10 +0300 Subject: [PATCH] GUARD-784: change retry time back to initial value; bad page will be skipped after retries; V1 can handle all skipped orders now; --- .../V2/IShipStationService.cs | 3 +- .../V2/Misc/ActionPolicies.cs | 8 +- .../V2/Services/PaginatedResponse.cs | 17 ++++- .../V2/ShipStationService.cs | 76 +++++++++++++------ 4 files changed, 70 insertions(+), 34 deletions(-) diff --git a/src/ShipStationAccess/V2/IShipStationService.cs b/src/ShipStationAccess/V2/IShipStationService.cs index 93db8cf..ee98c3e 100644 --- a/src/ShipStationAccess/V2/IShipStationService.cs +++ b/src/ShipStationAccess/V2/IShipStationService.cs @@ -7,13 +7,14 @@ using ShipStationAccess.V2.Models.Store; using ShipStationAccess.V2.Models.TagList; using ShipStationAccess.V2.Models.WarehouseLocation; +using ShipStationAccess.V2.Services; namespace ShipStationAccess.V2 { public interface IShipStationService { IEnumerable< ShipStationOrder > GetOrders( DateTime dateFrom, DateTime dateTo, Func< ShipStationOrder, ShipStationOrder > processOrder = null ); - Task< IEnumerable< ShipStationOrder > > GetOrdersAsync( DateTime dateFrom, DateTime dateTo, bool getShipmentsAndFulfillments = false, Func< ShipStationOrder, Task< ShipStationOrder > > processOrder = null ); + Task< IEnumerable< ShipStationOrder > > GetOrdersAsync( DateTime dateFrom, DateTime dateTo, bool getShipmentsAndFulfillments = false, Func< ShipStationOrder, Task< ShipStationOrder > > processOrder = null, Action< IEnumerable< ReadError > > handleSkippedOrders = null ); IEnumerable< ShipStationOrder > GetOrders( string storeId, string orderNumber ); ShipStationOrder GetOrderById( string orderId ); diff --git a/src/ShipStationAccess/V2/Misc/ActionPolicies.cs b/src/ShipStationAccess/V2/Misc/ActionPolicies.cs index 9be9982..bc30606 100644 --- a/src/ShipStationAccess/V2/Misc/ActionPolicies.cs +++ b/src/ShipStationAccess/V2/Misc/ActionPolicies.cs @@ -26,7 +26,7 @@ public static ActionPolicy Submit private static readonly ActionPolicy _shipStationSubmitPolicy = ActionPolicy.With( _exceptionHandler ).Retry( 10, ( ex, i ) => { - var delay = TimeSpan.FromSeconds( Math.Pow( 2, i ) ); + var delay = TimeSpan.FromSeconds( 0.5 + i ); ShipStationLogger.Log.Error( ex, "Retrying ShipStation API submit call for the {retryCounter} time, delay {delayInSeconds} seconds", i, delay.TotalSeconds ); SystemUtil.Sleep( delay ); } ); @@ -38,7 +38,7 @@ public static ActionPolicyAsync SubmitAsync private static readonly ActionPolicyAsync _shipStationSubmitAsyncPolicy = ActionPolicyAsync.With( _exceptionHandler ).RetryAsync( 10, async ( ex, i ) => { - var delay = TimeSpan.FromSeconds( Math.Pow( 2, i ) ); + var delay = TimeSpan.FromSeconds( 0.5 + i ); ShipStationLogger.Log.Error( ex, "Retrying ShipStation API submit call for the {retryCounter} time, delay {delayInSeconds} seconds", i, delay.TotalSeconds ); await Task.Delay( delay ); } ); @@ -50,7 +50,7 @@ public static ActionPolicy Get private static readonly ActionPolicy _shipStationGetPolicy = ActionPolicy.With( _exceptionHandler ).Retry( 10, ( ex, i ) => { - var delay = TimeSpan.FromSeconds( Math.Pow( 2, i ) ); + var delay = TimeSpan.FromSeconds( 0.5 + i ); ShipStationLogger.Log.Error( ex, "Retrying ShipStation API get call for the {retryCounter} time, delay {delayInSeconds} seconds", i, delay.TotalSeconds ); SystemUtil.Sleep( delay ); } ); @@ -62,7 +62,7 @@ public static ActionPolicyAsync GetAsync private static readonly ActionPolicyAsync _shipStationGetAsyncPolicy = ActionPolicyAsync.With( _exceptionHandler ).RetryAsync( 10, async ( ex, i ) => { - var delay = TimeSpan.FromSeconds( Math.Pow( 2, i ) ); + var delay = TimeSpan.FromSeconds( 0.5 + i ); ShipStationLogger.Log.Error( ex, "Retrying ShipStation API get call for the {retryCounter} time, delay {delayInSeconds} seconds", i, delay.TotalSeconds ); await Task.Delay( delay ); } ); diff --git a/src/ShipStationAccess/V2/Services/PaginatedResponse.cs b/src/ShipStationAccess/V2/Services/PaginatedResponse.cs index 24445a7..d0cd07f 100644 --- a/src/ShipStationAccess/V2/Services/PaginatedResponse.cs +++ b/src/ShipStationAccess/V2/Services/PaginatedResponse.cs @@ -6,13 +6,22 @@ namespace ShipStationAccess.V2.Services { public int? TotalPagesExpected { get; set; } public int? TotalEntitiesExpected { get; set; } - public int? TotalPagesReceived { get; set; } + public int TotalPagesReceived { get; set; } - public IEnumerable< T > Data { get; private set; } + public List< T > Data { get; private set; } + public List< ReadError > ReadErrors { get; private set; } - public PaginatedResponse( IEnumerable< T > data ) + public PaginatedResponse() { - this.Data = data ?? new List< T >(); + this.Data = new List< T >(); + this.ReadErrors = new List< ReadError >(); } } + + public sealed class ReadError + { + public string Url { get; set; } + public int PageSize { get; set; } + public int Page { get; set; } + } } \ No newline at end of file diff --git a/src/ShipStationAccess/V2/ShipStationService.cs b/src/ShipStationAccess/V2/ShipStationService.cs index e9bdf32..55bf6de 100644 --- a/src/ShipStationAccess/V2/ShipStationService.cs +++ b/src/ShipStationAccess/V2/ShipStationService.cs @@ -136,14 +136,14 @@ public IEnumerable< ShipStationOrder > GetOrders( DateTime dateFrom, DateTime da return orders; } - public async Task< IEnumerable< ShipStationOrder > > GetOrdersAsync( DateTime dateFrom, DateTime dateTo, bool getShipmentsAndFulfillments = false, Func< ShipStationOrder, Task< ShipStationOrder > > processOrder = null ) + public async Task< IEnumerable< ShipStationOrder > > GetOrdersAsync( DateTime dateFrom, DateTime dateTo, bool getShipmentsAndFulfillments = false, Func< ShipStationOrder, Task< ShipStationOrder > > processOrder = null, Action< IEnumerable< ReadError > > handleSkippedOrders = null ) { var allOrders = new List< ShipStationOrder >(); var createdOrders = await this.GetCreatedOrdersAsync( dateFrom, dateTo ).ConfigureAwait( false ); - allOrders.AddRange( createdOrders ); + allOrders.AddRange( createdOrders.Data ); var modifiedOrders = await this.GetModifiedOrdersAsync( dateFrom, dateTo ).ConfigureAwait( false ); - allOrders.AddRange( modifiedOrders ); + allOrders.AddRange( modifiedOrders.Data ); var uniqueOrders = allOrders.GroupBy( o => o.OrderId ).Select( gr => gr.First() ).ToList(); var processedOrders = await uniqueOrders.ProcessInBatchAsync( 5, async order => @@ -159,10 +159,20 @@ public async Task< IEnumerable< ShipStationOrder > > GetOrdersAsync( DateTime da if ( getShipmentsAndFulfillments ) await this.FindShipmentsAndFulfillments( processedOrders ).ConfigureAwait( false ); + if ( handleSkippedOrders != null ) + { + var allSkippedOrders = new List< ReadError >(); + allSkippedOrders.AddRange( createdOrders.ReadErrors ); + allSkippedOrders.AddRange( modifiedOrders.ReadErrors ); + + if ( allSkippedOrders.Any() ) + handleSkippedOrders( allSkippedOrders ); + } + return processedOrders; } - public async Task< IEnumerable< ShipStationOrder > > GetCreatedOrdersAsync( DateTime dateFrom, DateTime dateTo ) + public async Task< PaginatedResponse< ShipStationOrder > > GetCreatedOrdersAsync( DateTime dateFrom, DateTime dateTo ) { var createdOrdersEndpoint = ParamsBuilder.CreateNewOrdersParams( dateFrom, dateTo ); var createdOrdersResponse = await this.DownloadOrdersAsync( createdOrdersEndpoint ).ConfigureAwait( false ); @@ -172,15 +182,15 @@ public async Task< IEnumerable< ShipStationOrder > > GetCreatedOrdersAsync( Date _webRequestServices.GetApiKey(), createdOrdersResponse.Data.Count(), createdOrdersResponse.TotalEntitiesExpected ?? 0, - createdOrdersResponse.TotalPagesReceived ?? 0, + createdOrdersResponse.TotalPagesReceived, createdOrdersResponse.TotalPagesExpected ?? 0, createdOrdersResponse ); } - return createdOrdersResponse.Data; + return createdOrdersResponse; } - public async Task< IEnumerable< ShipStationOrder > > GetModifiedOrdersAsync( DateTime dateFrom, DateTime dateTo ) + public async Task< PaginatedResponse< ShipStationOrder > > GetModifiedOrdersAsync( DateTime dateFrom, DateTime dateTo ) { var modifiedOrdersEndpoint = ParamsBuilder.CreateModifiedOrdersParams( dateFrom, dateTo ); var modifiedOrdersResponse = await this.DownloadOrdersAsync( modifiedOrdersEndpoint ).ConfigureAwait( false ); @@ -190,19 +200,17 @@ public async Task< IEnumerable< ShipStationOrder > > GetModifiedOrdersAsync( Dat _webRequestServices.GetApiKey(), modifiedOrdersResponse.Data.Count(), modifiedOrdersResponse.TotalEntitiesExpected ?? 0, - modifiedOrdersResponse.TotalPagesReceived ?? 0, + modifiedOrdersResponse.TotalPagesReceived, modifiedOrdersResponse.TotalPagesExpected ?? 0, modifiedOrdersResponse ); } - return modifiedOrdersResponse.Data; + return modifiedOrdersResponse; } public async Task< PaginatedResponse< ShipStationOrder > > DownloadOrdersAsync( string endPoint ) { - var orders = new List< ShipStationOrder >(); - int? totalShipStationPages = null; - int? totalShipStationOrders = null; + var response = new PaginatedResponse< ShipStationOrder >(); var currentPage = 1; do @@ -211,29 +219,47 @@ public async Task< PaginatedResponse< ShipStationOrder > > DownloadOrdersAsync( var ordersEndPoint = endPoint.ConcatParams( nextPageParams ); ShipStationOrders ordersPage = null; - await ActionPolicies.GetAsync.Do( async () => + try { - ordersPage = await this._webRequestServices.GetResponseAsync< ShipStationOrders >( ShipStationCommand.GetOrders, ordersEndPoint ).ConfigureAwait( false ); - } ); + await ActionPolicies.GetAsync.Do( async () => + { + ordersPage = await this._webRequestServices.GetResponseAsync< ShipStationOrders >( ShipStationCommand.GetOrders, ordersEndPoint ).ConfigureAwait( false ); + } ); + } + catch( WebException e ) + { + if( WebRequestServices.CanSkipException( e ) ) + { + response.ReadErrors.Add( new ReadError() + { + Url = endPoint, + Page = currentPage, + PageSize = RequestMaxLimit + } ); + + ShipStationLogger.Log.Warn( e, "Skipped get orders request page {pageNumber} of request {request} due to internal error on ShipStation's side", currentPage, ordersEndPoint ); + currentPage++; + continue; + } + else + throw; + } currentPage++; if ( ordersPage?.Orders == null || !ordersPage.Orders.Any() ) break; - totalShipStationPages = ordersPage.TotalPages; - totalShipStationOrders = ordersPage.TotalOrders; + response.TotalPagesExpected = ordersPage.TotalPages; + response.TotalEntitiesExpected = ordersPage.TotalOrders; - orders.AddRange( ordersPage.Orders ); + response.Data.AddRange( ordersPage.Orders ); - } while( currentPage <= totalShipStationPages ); + } while( currentPage <= response.TotalPagesExpected ); - return new PaginatedResponse< ShipStationOrder >( orders ) - { - TotalPagesExpected = totalShipStationPages, - TotalEntitiesExpected = totalShipStationOrders, - TotalPagesReceived = currentPage - 1 - }; + response.TotalPagesReceived = currentPage - 1; + + return response; } public IEnumerable< ShipStationOrder > GetOrders( string storeId, string orderNumber )