From ef05473fa490ec256dcae59f0a1b5e5526463c78 Mon Sep 17 00:00:00 2001 From: Bulat Farrakhov Date: Mon, 31 Aug 2020 15:01:38 +0300 Subject: [PATCH 1/4] GUARD-784: retry time now has exponential law; refactored code; --- src/Global/GlobalAssemblyInfo.cs | 2 +- .../V2/Misc/ActionPolicies.cs | 28 ++++---- .../V2/ShipStationService.cs | 64 ++++++++----------- .../Orders/OrderTests.cs | 2 +- 4 files changed, 44 insertions(+), 52 deletions(-) diff --git a/src/Global/GlobalAssemblyInfo.cs b/src/Global/GlobalAssemblyInfo.cs index 0cdcb95..2e5b9b2 100644 --- a/src/Global/GlobalAssemblyInfo.cs +++ b/src/Global/GlobalAssemblyInfo.cs @@ -23,4 +23,4 @@ // [assembly: AssemblyVersion("1.0.*")] // Keep in track with CA API version -[ assembly : AssemblyVersion( "1.4.2.0" ) ] \ No newline at end of file +[ assembly : AssemblyVersion( "1.4.3.0" ) ] \ No newline at end of file diff --git a/src/ShipStationAccess/V2/Misc/ActionPolicies.cs b/src/ShipStationAccess/V2/Misc/ActionPolicies.cs index 7db4ba0..9be9982 100644 --- a/src/ShipStationAccess/V2/Misc/ActionPolicies.cs +++ b/src/ShipStationAccess/V2/Misc/ActionPolicies.cs @@ -11,7 +11,7 @@ public static class ActionPolicies { public static ActionPolicy Submit { - get { return _shipStationSumbitPolicy; } + get { return _shipStationSubmitPolicy; } } private static readonly ExceptionHandler _exceptionHandler = delegate( Exception x ) @@ -24,21 +24,23 @@ public static ActionPolicy Submit return webX.Response.GetHttpStatusCode() != HttpStatusCode.Unauthorized; }; - private static readonly ActionPolicy _shipStationSumbitPolicy = ActionPolicy.With( _exceptionHandler ).Retry( 10, ( ex, i ) => + private static readonly ActionPolicy _shipStationSubmitPolicy = ActionPolicy.With( _exceptionHandler ).Retry( 10, ( ex, i ) => { - ShipStationLogger.Log.Error( ex, "Retrying ShipStation API submit call for the {retryCounter} time", i ); - SystemUtil.Sleep( TimeSpan.FromSeconds( 0.5 + i ) ); + var delay = TimeSpan.FromSeconds( Math.Pow( 2, i ) ); + ShipStationLogger.Log.Error( ex, "Retrying ShipStation API submit call for the {retryCounter} time, delay {delayInSeconds} seconds", i, delay.TotalSeconds ); + SystemUtil.Sleep( delay ); } ); public static ActionPolicyAsync SubmitAsync { - get { return _shipStationSumbitAsyncPolicy; } + get { return _shipStationSubmitAsyncPolicy; } } - private static readonly ActionPolicyAsync _shipStationSumbitAsyncPolicy = ActionPolicyAsync.With( _exceptionHandler ).RetryAsync( 10, async ( ex, i ) => + private static readonly ActionPolicyAsync _shipStationSubmitAsyncPolicy = ActionPolicyAsync.With( _exceptionHandler ).RetryAsync( 10, async ( ex, i ) => { - ShipStationLogger.Log.Error( ex, "Retrying ShipStation API submit call for the {retryCounter} time", i ); - await Task.Delay( TimeSpan.FromSeconds( 0.5 + i ) ); + var delay = TimeSpan.FromSeconds( Math.Pow( 2, i ) ); + ShipStationLogger.Log.Error( ex, "Retrying ShipStation API submit call for the {retryCounter} time, delay {delayInSeconds} seconds", i, delay.TotalSeconds ); + await Task.Delay( delay ); } ); public static ActionPolicy Get @@ -48,8 +50,9 @@ public static ActionPolicy Get private static readonly ActionPolicy _shipStationGetPolicy = ActionPolicy.With( _exceptionHandler ).Retry( 10, ( ex, i ) => { - ShipStationLogger.Log.Error( ex, "Retrying ShipStation API get call for the {retryCounter} time", i ); - SystemUtil.Sleep( TimeSpan.FromSeconds( 0.5 + i ) ); + var delay = TimeSpan.FromSeconds( Math.Pow( 2, i ) ); + ShipStationLogger.Log.Error( ex, "Retrying ShipStation API get call for the {retryCounter} time, delay {delayInSeconds} seconds", i, delay.TotalSeconds ); + SystemUtil.Sleep( delay ); } ); public static ActionPolicyAsync GetAsync @@ -59,8 +62,9 @@ public static ActionPolicyAsync GetAsync private static readonly ActionPolicyAsync _shipStationGetAsyncPolicy = ActionPolicyAsync.With( _exceptionHandler ).RetryAsync( 10, async ( ex, i ) => { - ShipStationLogger.Log.Error( ex, "Retrying ShipStation API get call for the {retryCounter} time", i ); - await Task.Delay( TimeSpan.FromSeconds( 0.5 + i ) ); + var delay = TimeSpan.FromSeconds( Math.Pow( 2, i ) ); + ShipStationLogger.Log.Error( ex, "Retrying ShipStation API get call for the {retryCounter} time, delay {delayInSeconds} seconds", i, delay.TotalSeconds ); + await Task.Delay( delay ); } ); } } \ No newline at end of file diff --git a/src/ShipStationAccess/V2/ShipStationService.cs b/src/ShipStationAccess/V2/ShipStationService.cs index 50a5c1a..60e37b0 100644 --- a/src/ShipStationAccess/V2/ShipStationService.cs +++ b/src/ShipStationAccess/V2/ShipStationService.cs @@ -146,19 +146,19 @@ public async Task< IEnumerable< ShipStationOrder > > GetOrdersAsync( DateTime da var orders = new List< ShipStationOrder >(); var processedOrderIds = new HashSet< long >(); - Func< ShipStationOrders, Task > processOrders = async sorders => + Func< IList< ShipStationOrder >, Task > processOrders = async incomingOrders => { - var processedOrders = await sorders.Orders.ProcessInBatchAsync( 5, async o => + var processedOrders = await incomingOrders.ProcessInBatchAsync( 5, async incomingOrder => { - var curOrder = o; + var curOrder = incomingOrder; if( processedOrderIds.Contains( curOrder.OrderId ) ) return null; if( processOrder != null ) - curOrder = await processOrder( curOrder ); + curOrder = await processOrder( curOrder ).ConfigureAwait( false ); return curOrder; - } ); + } ).ConfigureAwait( false ); foreach( var order in processedOrders ) { @@ -175,60 +175,48 @@ public async Task< IEnumerable< ShipStationOrder > > GetOrdersAsync( DateTime da Func< string, Task > downloadOrders = async endPoint => { - var pagesCount = int.MaxValue; + int? totalShipStationPages = null; + int? totalShipStationOrders = null; var currentPage = 1; - var ordersCount = 0; - var ordersExpected = -1; + var ordersDownloaded = 0; do { var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, RequestMaxLimit ) ); var ordersEndPoint = endPoint.ConcatParams( nextPageParams ); - ShipStationOrders ordersWithinPage = null; - try - { - await ActionPolicies.GetAsync.Do( async () => - { - ordersWithinPage = await this._webRequestServices.GetResponseAsync< ShipStationOrders >( ShipStationCommand.GetOrders, ordersEndPoint ); - } ); - } - catch( WebException e ) + ShipStationOrders ordersPage = null; + await ActionPolicies.GetAsync.Do( async () => { - if( WebRequestServices.CanSkipException( e ) ) - { - ShipStationLogger.Log.Warn( e, "Skipped get orders request page {pageNumber} of request {request} due to internal error on ShipStation's side", currentPage, ordersEndPoint ); - } - else - throw; - } + ordersPage = await this._webRequestServices.GetResponseAsync< ShipStationOrders >( ShipStationCommand.GetOrders, ordersEndPoint ).ConfigureAwait( false ); + } ).ConfigureAwait( false ); currentPage++; - if( ordersWithinPage != null ) + if ( ordersPage?.Orders == null || !ordersPage.Orders.Any() ) + break; + + if( totalShipStationPages == null ) { - if( pagesCount == int.MaxValue ) - { - pagesCount = ordersWithinPage.TotalPages; - ordersExpected = ordersWithinPage.TotalOrders; - } + totalShipStationPages = ordersPage.TotalPages; + totalShipStationOrders = ordersPage.TotalOrders; + } - ordersCount += ordersWithinPage.Orders.Count; + ordersDownloaded += ordersPage.Orders.Count; + await processOrders( ordersPage.Orders ).ConfigureAwait( false ); - await processOrders( ordersWithinPage ); - } - } while( currentPage <= pagesCount ); + } while( currentPage <= totalShipStationPages ); - ShipStationLogger.Log.Info( "Orders dowloaded API '{apiKey}' - {orders}/{expectedOrders} orders in {pages}/{expectedPages} from {endpoint}", _webRequestServices.GetApiKey(), ordersCount, ordersExpected, currentPage - 1, pagesCount, endPoint ); + ShipStationLogger.Log.Info( "Orders downloaded API key '{apiKey}' - {orders}/{expectedOrders} orders in {pages}/{expectedPages} from {endpoint}", _webRequestServices.GetApiKey(), ordersDownloaded, totalShipStationOrders ?? 0, currentPage - 1, totalShipStationPages ?? 0, endPoint ); }; var newOrdersEndpoint = ParamsBuilder.CreateNewOrdersParams( dateFrom, dateTo ); - await downloadOrders( newOrdersEndpoint ); + await downloadOrders( newOrdersEndpoint ).ConfigureAwait( false ); var modifiedOrdersEndpoint = ParamsBuilder.CreateModifiedOrdersParams( dateFrom, dateTo ); - await downloadOrders( modifiedOrdersEndpoint ); + await downloadOrders( modifiedOrdersEndpoint ).ConfigureAwait( false ); - await this.FindMarketplaceIdsAsync( orders ); + await this.FindMarketplaceIdsAsync( orders ).ConfigureAwait( false ); return orders; } diff --git a/src/ShipStationAccessTests/Orders/OrderTests.cs b/src/ShipStationAccessTests/Orders/OrderTests.cs index 1da25d8..240ef71 100644 --- a/src/ShipStationAccessTests/Orders/OrderTests.cs +++ b/src/ShipStationAccessTests/Orders/OrderTests.cs @@ -53,7 +53,7 @@ [ Test ] public async Task GetOrdersAsync() { var service = this.ShipStationFactory.CreateServiceV2( this._credentials ); - var orders = await service.GetOrdersAsync( DateTime.UtcNow.AddDays( -3 ), DateTime.UtcNow ); + var orders = await service.GetOrdersAsync( DateTime.UtcNow.AddDays( -30 ), DateTime.UtcNow, false ); orders.Count().Should().BeGreaterThan( 0 ); } From c67fc94ba3810593eca520e8b31a408655cc55ca Mon Sep 17 00:00:00 2001 From: Bulat Farrakhov Date: Tue, 1 Sep 2020 09:49:30 +0300 Subject: [PATCH 2/4] GUARD-784: completely refactored orders sync service code; --- .../ShipStationAccess.csproj | 1 + .../V2/IShipStationService.cs | 2 +- .../V2/Services/PaginatedResponse.cs | 18 ++ .../V2/ShipStationService.cs | 198 ++++++++++-------- .../Orders/OrderTests.cs | 4 +- 5 files changed, 133 insertions(+), 90 deletions(-) create mode 100644 src/ShipStationAccess/V2/Services/PaginatedResponse.cs diff --git a/src/ShipStationAccess/ShipStationAccess.csproj b/src/ShipStationAccess/ShipStationAccess.csproj index 209fa36..37ef95c 100644 --- a/src/ShipStationAccess/ShipStationAccess.csproj +++ b/src/ShipStationAccess/ShipStationAccess.csproj @@ -115,6 +115,7 @@ + diff --git a/src/ShipStationAccess/V2/IShipStationService.cs b/src/ShipStationAccess/V2/IShipStationService.cs index 758088b..93db8cf 100644 --- a/src/ShipStationAccess/V2/IShipStationService.cs +++ b/src/ShipStationAccess/V2/IShipStationService.cs @@ -13,7 +13,7 @@ namespace ShipStationAccess.V2 public interface IShipStationService { IEnumerable< ShipStationOrder > GetOrders( DateTime dateFrom, DateTime dateTo, Func< ShipStationOrder, ShipStationOrder > processOrder = null ); - Task< IEnumerable< ShipStationOrder > > GetOrdersAsync( DateTime dateFrom, DateTime dateTo, bool getShipmentsAndFulfillments = true, Func< ShipStationOrder, Task< ShipStationOrder > > processOrder = null ); + Task< IEnumerable< ShipStationOrder > > GetOrdersAsync( DateTime dateFrom, DateTime dateTo, bool getShipmentsAndFulfillments = false, Func< ShipStationOrder, Task< ShipStationOrder > > processOrder = null ); IEnumerable< ShipStationOrder > GetOrders( string storeId, string orderNumber ); ShipStationOrder GetOrderById( string orderId ); diff --git a/src/ShipStationAccess/V2/Services/PaginatedResponse.cs b/src/ShipStationAccess/V2/Services/PaginatedResponse.cs new file mode 100644 index 0000000..24445a7 --- /dev/null +++ b/src/ShipStationAccess/V2/Services/PaginatedResponse.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; + +namespace ShipStationAccess.V2.Services +{ + public sealed class PaginatedResponse< T > where T : class, new() + { + public int? TotalPagesExpected { get; set; } + public int? TotalEntitiesExpected { get; set; } + public int? TotalPagesReceived { get; set; } + + public IEnumerable< T > Data { get; private set; } + + public PaginatedResponse( IEnumerable< T > data ) + { + this.Data = data ?? new List< T >(); + } + } +} \ No newline at end of file diff --git a/src/ShipStationAccess/V2/ShipStationService.cs b/src/ShipStationAccess/V2/ShipStationService.cs index 60e37b0..e9bdf32 100644 --- a/src/ShipStationAccess/V2/ShipStationService.cs +++ b/src/ShipStationAccess/V2/ShipStationService.cs @@ -1,14 +1,9 @@ using System; using System.Collections.Generic; -using System.Diagnostics; -using System.IO; using System.Linq; using System.Net; -using System.Reflection; using System.Threading.Tasks; -using Microsoft.CSharp.RuntimeBinder; using Netco.Extensions; -using Newtonsoft.Json; using ShipStationAccess.V2.Exceptions; using ShipStationAccess.V2.Misc; using ShipStationAccess.V2.Models; @@ -141,86 +136,106 @@ public IEnumerable< ShipStationOrder > GetOrders( DateTime dateFrom, DateTime da return orders; } - public async Task< IEnumerable< ShipStationOrder > > GetOrdersAsync( DateTime dateFrom, DateTime dateTo, bool getShipmentsAndFulfillments = true, Func< ShipStationOrder, Task< ShipStationOrder > > processOrder = null ) + public async Task< IEnumerable< ShipStationOrder > > GetOrdersAsync( DateTime dateFrom, DateTime dateTo, bool getShipmentsAndFulfillments = false, Func< ShipStationOrder, Task< ShipStationOrder > > processOrder = null ) { - var orders = new List< ShipStationOrder >(); - var processedOrderIds = new HashSet< long >(); + var allOrders = new List< ShipStationOrder >(); + var createdOrders = await this.GetCreatedOrdersAsync( dateFrom, dateTo ).ConfigureAwait( false ); + allOrders.AddRange( createdOrders ); - Func< IList< ShipStationOrder >, Task > processOrders = async incomingOrders => + var modifiedOrders = await this.GetModifiedOrdersAsync( dateFrom, dateTo ).ConfigureAwait( false ); + allOrders.AddRange( modifiedOrders ); + + var uniqueOrders = allOrders.GroupBy( o => o.OrderId ).Select( gr => gr.First() ).ToList(); + var processedOrders = await uniqueOrders.ProcessInBatchAsync( 5, async order => { - var processedOrders = await incomingOrders.ProcessInBatchAsync( 5, async incomingOrder => - { - var curOrder = incomingOrder; - if( processedOrderIds.Contains( curOrder.OrderId ) ) - return null; + if( processOrder != null ) + order = await processOrder( order ).ConfigureAwait( false ); - if( processOrder != null ) - curOrder = await processOrder( curOrder ).ConfigureAwait( false ); + return order; + } ); - return curOrder; - } ).ConfigureAwait( false ); + await this.FindMarketplaceIdsAsync( processedOrders ).ConfigureAwait( false ); - foreach( var order in processedOrders ) - { - if ( getShipmentsAndFulfillments ) - { - order.Shipments = await GetOrderShipmentsByIdAsync( order.OrderId.ToString() ).ConfigureAwait( false ); - order.Fulfillments = await GetOrderFulfillmentsByIdAsync( order.OrderId.ToString() ).ConfigureAwait( false ); - } + if ( getShipmentsAndFulfillments ) + await this.FindShipmentsAndFulfillments( processedOrders ).ConfigureAwait( false ); - orders.Add( order ); - processedOrderIds.Add( order.OrderId ); - } - }; + return processedOrders; + } - Func< string, Task > downloadOrders = async endPoint => + public async Task< IEnumerable< ShipStationOrder > > GetCreatedOrdersAsync( DateTime dateFrom, DateTime dateTo ) + { + var createdOrdersEndpoint = ParamsBuilder.CreateNewOrdersParams( dateFrom, dateTo ); + var createdOrdersResponse = await this.DownloadOrdersAsync( createdOrdersEndpoint ).ConfigureAwait( false ); + if ( createdOrdersResponse.Data.Any() ) { - int? totalShipStationPages = null; - int? totalShipStationOrders = null; - var currentPage = 1; - var ordersDownloaded = 0; - - do - { - var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, RequestMaxLimit ) ); - var ordersEndPoint = endPoint.ConcatParams( nextPageParams ); + ShipStationLogger.Log.Info( "Created orders downloaded using tenant's API key '{apiKey}' - {orders}/{expectedOrders} orders in {pages}/{expectedPages} from {endpoint}", + _webRequestServices.GetApiKey(), + createdOrdersResponse.Data.Count(), + createdOrdersResponse.TotalEntitiesExpected ?? 0, + createdOrdersResponse.TotalPagesReceived ?? 0, + createdOrdersResponse.TotalPagesExpected ?? 0, + createdOrdersResponse ); + } - ShipStationOrders ordersPage = null; - await ActionPolicies.GetAsync.Do( async () => - { - ordersPage = await this._webRequestServices.GetResponseAsync< ShipStationOrders >( ShipStationCommand.GetOrders, ordersEndPoint ).ConfigureAwait( false ); - } ).ConfigureAwait( false ); + return createdOrdersResponse.Data; + } - currentPage++; + public async Task< IEnumerable< ShipStationOrder > > GetModifiedOrdersAsync( DateTime dateFrom, DateTime dateTo ) + { + var modifiedOrdersEndpoint = ParamsBuilder.CreateModifiedOrdersParams( dateFrom, dateTo ); + var modifiedOrdersResponse = await this.DownloadOrdersAsync( modifiedOrdersEndpoint ).ConfigureAwait( false ); + if ( modifiedOrdersResponse.Data.Any() ) + { + ShipStationLogger.Log.Info( "Modified orders downloaded using tenant's API key '{apiKey}' - {orders}/{expectedOrders} orders in {pages}/{expectedPages} from {endpoint}", + _webRequestServices.GetApiKey(), + modifiedOrdersResponse.Data.Count(), + modifiedOrdersResponse.TotalEntitiesExpected ?? 0, + modifiedOrdersResponse.TotalPagesReceived ?? 0, + modifiedOrdersResponse.TotalPagesExpected ?? 0, + modifiedOrdersResponse ); + } - if ( ordersPage?.Orders == null || !ordersPage.Orders.Any() ) - break; + return modifiedOrdersResponse.Data; + } - if( totalShipStationPages == null ) - { - totalShipStationPages = ordersPage.TotalPages; - totalShipStationOrders = ordersPage.TotalOrders; - } + public async Task< PaginatedResponse< ShipStationOrder > > DownloadOrdersAsync( string endPoint ) + { + var orders = new List< ShipStationOrder >(); + int? totalShipStationPages = null; + int? totalShipStationOrders = null; + var currentPage = 1; - ordersDownloaded += ordersPage.Orders.Count; - await processOrders( ordersPage.Orders ).ConfigureAwait( false ); + do + { + var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, RequestMaxLimit ) ); + var ordersEndPoint = endPoint.ConcatParams( nextPageParams ); - } while( currentPage <= totalShipStationPages ); + ShipStationOrders ordersPage = null; + await ActionPolicies.GetAsync.Do( async () => + { + ordersPage = await this._webRequestServices.GetResponseAsync< ShipStationOrders >( ShipStationCommand.GetOrders, ordersEndPoint ).ConfigureAwait( false ); + } ); - ShipStationLogger.Log.Info( "Orders downloaded API key '{apiKey}' - {orders}/{expectedOrders} orders in {pages}/{expectedPages} from {endpoint}", _webRequestServices.GetApiKey(), ordersDownloaded, totalShipStationOrders ?? 0, currentPage - 1, totalShipStationPages ?? 0, endPoint ); - }; + currentPage++; - var newOrdersEndpoint = ParamsBuilder.CreateNewOrdersParams( dateFrom, dateTo ); - await downloadOrders( newOrdersEndpoint ).ConfigureAwait( false ); + if ( ordersPage?.Orders == null || !ordersPage.Orders.Any() ) + break; - var modifiedOrdersEndpoint = ParamsBuilder.CreateModifiedOrdersParams( dateFrom, dateTo ); - await downloadOrders( modifiedOrdersEndpoint ).ConfigureAwait( false ); + totalShipStationPages = ordersPage.TotalPages; + totalShipStationOrders = ordersPage.TotalOrders; - await this.FindMarketplaceIdsAsync( orders ).ConfigureAwait( false ); + orders.AddRange( ordersPage.Orders ); - return orders; + } while( currentPage <= totalShipStationPages ); + + return new PaginatedResponse< ShipStationOrder >( orders ) + { + TotalPagesExpected = totalShipStationPages, + TotalEntitiesExpected = totalShipStationOrders, + TotalPagesReceived = currentPage - 1 + }; } - + public IEnumerable< ShipStationOrder > GetOrders( string storeId, string orderNumber ) { var orders = new List< ShipStationOrder >(); @@ -295,12 +310,21 @@ await ActionPolicies.GetAsync.Do( async () => return order; } + private async Task FindShipmentsAndFulfillments( IEnumerable< ShipStationOrder > orders ) + { + foreach( var order in orders ) + { + order.Shipments = await this.GetOrderShipmentsByIdAsync( order.OrderId.ToString() ).ConfigureAwait( false ); + order.Fulfillments = await this.GetOrderFulfillmentsByIdAsync( order.OrderId.ToString() ).ConfigureAwait( false ); + } + } + public async Task< IEnumerable< ShipStationOrderShipment > > GetOrderShipmentsByIdAsync( string orderId ) { var orderShipments = new List< ShipStationOrderShipment >(); var currentPage = 1; - var pagesCount = int.MaxValue; + int? totalShipStationShipmentsPages; do { @@ -308,21 +332,21 @@ public async Task< IEnumerable< ShipStationOrderShipment > > GetOrderShipmentsBy var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, RequestMaxLimit ) ); var orderShipmentsByPageEndPoint = getOrderShipmentsEndpoint.ConcatParams( nextPageParams ); + ShipStationOrderShipments ordersShipmentsPage = null; await ActionPolicies.GetAsync.Do( async () => { - var orderShipmentsPage = await this._webRequestServices.GetResponseAsync< ShipStationOrderShipments >( ShipStationCommand.GetOrderShipments, orderShipmentsByPageEndPoint ); - - ++currentPage; - if ( pagesCount == int.MaxValue ) - { - pagesCount = orderShipmentsPage.Pages + 1; - } + ordersShipmentsPage = await this._webRequestServices.GetResponseAsync< ShipStationOrderShipments >( ShipStationCommand.GetOrderShipments, orderShipmentsByPageEndPoint ).ConfigureAwait( false ); + } ); - orderShipments.AddRange( orderShipmentsPage.Shipments ); + if ( ordersShipmentsPage?.Shipments == null || !ordersShipmentsPage.Shipments.Any() ) + break; - } ); + ++currentPage; + totalShipStationShipmentsPages = ordersShipmentsPage.Pages + 1; + + orderShipments.AddRange( ordersShipmentsPage.Shipments ); } - while( currentPage <= pagesCount ); + while( currentPage <= totalShipStationShipmentsPages ); return orderShipments; } @@ -332,7 +356,7 @@ public async Task< IEnumerable< ShipStationOrderFulfillment > > GetOrderFulfillm var orderFulfillments = new List< ShipStationOrderFulfillment >(); var currentPage = 1; - var pagesCount = int.MaxValue; + int? totalShipStationFulfillmentsPages; do { @@ -340,21 +364,21 @@ public async Task< IEnumerable< ShipStationOrderFulfillment > > GetOrderFulfillm var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, RequestMaxLimit ) ); var orderFulfillmentsByPageEndPoint = getOrderFulfillmentsEndpoint.ConcatParams( nextPageParams ); + ShipStationOrderFulfillments orderFulfillmentsPage = null; await ActionPolicies.GetAsync.Do( async () => { - var orderFulfillmentsPage = await this._webRequestServices.GetResponseAsync< ShipStationOrderFulfillments >( ShipStationCommand.GetOrderFulfillments, orderFulfillmentsByPageEndPoint ); - - ++currentPage; - if ( pagesCount == int.MaxValue ) - { - pagesCount = orderFulfillmentsPage.Pages + 1; - } + orderFulfillmentsPage = await this._webRequestServices.GetResponseAsync< ShipStationOrderFulfillments >( ShipStationCommand.GetOrderFulfillments, orderFulfillmentsByPageEndPoint ).ConfigureAwait( false ); + } ); - orderFulfillments.AddRange( orderFulfillmentsPage.Fulfillments ); + if ( orderFulfillmentsPage?.Fulfillments == null || !orderFulfillmentsPage.Fulfillments.Any() ) + break; - } ); + ++currentPage; + totalShipStationFulfillmentsPages = orderFulfillmentsPage.Pages + 1; + + orderFulfillments.AddRange( orderFulfillmentsPage.Fulfillments ); } - while( currentPage <= pagesCount ); + while( currentPage <= totalShipStationFulfillmentsPages ); return orderFulfillments; } @@ -508,7 +532,7 @@ private void FindMarketplaceIds( IEnumerable< ShipStationOrder > orders ) private async Task FindMarketplaceIdsAsync( IEnumerable< ShipStationOrder > orders ) { - var stores = await this.GetStoresAsync(); + var stores = await this.GetStoresAsync().ConfigureAwait( false ); foreach( var order in orders ) { diff --git a/src/ShipStationAccessTests/Orders/OrderTests.cs b/src/ShipStationAccessTests/Orders/OrderTests.cs index 240ef71..f1d66aa 100644 --- a/src/ShipStationAccessTests/Orders/OrderTests.cs +++ b/src/ShipStationAccessTests/Orders/OrderTests.cs @@ -53,7 +53,7 @@ [ Test ] public async Task GetOrdersAsync() { var service = this.ShipStationFactory.CreateServiceV2( this._credentials ); - var orders = await service.GetOrdersAsync( DateTime.UtcNow.AddDays( -30 ), DateTime.UtcNow, false ); + var orders = await service.GetOrdersAsync( DateTime.UtcNow.AddDays( -30 ), DateTime.UtcNow, getShipmentsAndFulfillments: true ); orders.Count().Should().BeGreaterThan( 0 ); } @@ -62,7 +62,7 @@ [ Test ] public async Task GetOrdersWithoutShipmentsAndFulfillmentsAsync() { var service = this.ShipStationFactory.CreateServiceV2( this._credentials ); - var orders = await service.GetOrdersAsync( DateTime.UtcNow.AddDays( -3 ), DateTime.UtcNow, false ); + var orders = await service.GetOrdersAsync( DateTime.UtcNow.AddDays( -30 ), DateTime.UtcNow, false ); orders.Count().Should().BeGreaterThan( 0 ); orders.Any( o => o.Shipments != null ).Should().Be( false ); From a9b45bf98227ddeb3edf64bc0ddeaf3b34d56c70 Mon Sep 17 00:00:00 2001 From: Bulat Farrakhov Date: Tue, 1 Sep 2020 16:46:17 +0300 Subject: [PATCH 3/4] GUARD-784: changed test methods parameters; --- src/ShipStationAccessTests/Orders/OrderTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShipStationAccessTests/Orders/OrderTests.cs b/src/ShipStationAccessTests/Orders/OrderTests.cs index f1d66aa..e3c5a20 100644 --- a/src/ShipStationAccessTests/Orders/OrderTests.cs +++ b/src/ShipStationAccessTests/Orders/OrderTests.cs @@ -53,7 +53,7 @@ [ Test ] public async Task GetOrdersAsync() { var service = this.ShipStationFactory.CreateServiceV2( this._credentials ); - var orders = await service.GetOrdersAsync( DateTime.UtcNow.AddDays( -30 ), DateTime.UtcNow, getShipmentsAndFulfillments: true ); + var orders = await service.GetOrdersAsync( DateTime.UtcNow.AddDays( -1 ), DateTime.UtcNow, getShipmentsAndFulfillments: true ); orders.Count().Should().BeGreaterThan( 0 ); } @@ -62,7 +62,7 @@ [ Test ] public async Task GetOrdersWithoutShipmentsAndFulfillmentsAsync() { var service = this.ShipStationFactory.CreateServiceV2( this._credentials ); - var orders = await service.GetOrdersAsync( DateTime.UtcNow.AddDays( -30 ), DateTime.UtcNow, false ); + var orders = await service.GetOrdersAsync( DateTime.UtcNow.AddDays( -3 ), DateTime.UtcNow, getShipmentsAndFulfillments: false ); orders.Count().Should().BeGreaterThan( 0 ); orders.Any( o => o.Shipments != null ).Should().Be( false ); From cf1e1634aa6ec128335be02e646e2d1d486dbe5c Mon Sep 17 00:00:00 2001 From: Bulat Farrakhov Date: Fri, 4 Sep 2020 16:11:10 +0300 Subject: [PATCH 4/4] GUARD-784: change retry time back to initial value; bad page will be skipped after retries; V1 can handle all skipped orders now; --- .../V2/IShipStationService.cs | 3 +- .../V2/Misc/ActionPolicies.cs | 8 +- .../V2/Services/PaginatedResponse.cs | 17 ++++- .../V2/ShipStationService.cs | 76 +++++++++++++------ 4 files changed, 70 insertions(+), 34 deletions(-) diff --git a/src/ShipStationAccess/V2/IShipStationService.cs b/src/ShipStationAccess/V2/IShipStationService.cs index 93db8cf..ee98c3e 100644 --- a/src/ShipStationAccess/V2/IShipStationService.cs +++ b/src/ShipStationAccess/V2/IShipStationService.cs @@ -7,13 +7,14 @@ using ShipStationAccess.V2.Models.Store; using ShipStationAccess.V2.Models.TagList; using ShipStationAccess.V2.Models.WarehouseLocation; +using ShipStationAccess.V2.Services; namespace ShipStationAccess.V2 { public interface IShipStationService { IEnumerable< ShipStationOrder > GetOrders( DateTime dateFrom, DateTime dateTo, Func< ShipStationOrder, ShipStationOrder > processOrder = null ); - Task< IEnumerable< ShipStationOrder > > GetOrdersAsync( DateTime dateFrom, DateTime dateTo, bool getShipmentsAndFulfillments = false, Func< ShipStationOrder, Task< ShipStationOrder > > processOrder = null ); + Task< IEnumerable< ShipStationOrder > > GetOrdersAsync( DateTime dateFrom, DateTime dateTo, bool getShipmentsAndFulfillments = false, Func< ShipStationOrder, Task< ShipStationOrder > > processOrder = null, Action< IEnumerable< ReadError > > handleSkippedOrders = null ); IEnumerable< ShipStationOrder > GetOrders( string storeId, string orderNumber ); ShipStationOrder GetOrderById( string orderId ); diff --git a/src/ShipStationAccess/V2/Misc/ActionPolicies.cs b/src/ShipStationAccess/V2/Misc/ActionPolicies.cs index 9be9982..bc30606 100644 --- a/src/ShipStationAccess/V2/Misc/ActionPolicies.cs +++ b/src/ShipStationAccess/V2/Misc/ActionPolicies.cs @@ -26,7 +26,7 @@ public static ActionPolicy Submit private static readonly ActionPolicy _shipStationSubmitPolicy = ActionPolicy.With( _exceptionHandler ).Retry( 10, ( ex, i ) => { - var delay = TimeSpan.FromSeconds( Math.Pow( 2, i ) ); + var delay = TimeSpan.FromSeconds( 0.5 + i ); ShipStationLogger.Log.Error( ex, "Retrying ShipStation API submit call for the {retryCounter} time, delay {delayInSeconds} seconds", i, delay.TotalSeconds ); SystemUtil.Sleep( delay ); } ); @@ -38,7 +38,7 @@ public static ActionPolicyAsync SubmitAsync private static readonly ActionPolicyAsync _shipStationSubmitAsyncPolicy = ActionPolicyAsync.With( _exceptionHandler ).RetryAsync( 10, async ( ex, i ) => { - var delay = TimeSpan.FromSeconds( Math.Pow( 2, i ) ); + var delay = TimeSpan.FromSeconds( 0.5 + i ); ShipStationLogger.Log.Error( ex, "Retrying ShipStation API submit call for the {retryCounter} time, delay {delayInSeconds} seconds", i, delay.TotalSeconds ); await Task.Delay( delay ); } ); @@ -50,7 +50,7 @@ public static ActionPolicy Get private static readonly ActionPolicy _shipStationGetPolicy = ActionPolicy.With( _exceptionHandler ).Retry( 10, ( ex, i ) => { - var delay = TimeSpan.FromSeconds( Math.Pow( 2, i ) ); + var delay = TimeSpan.FromSeconds( 0.5 + i ); ShipStationLogger.Log.Error( ex, "Retrying ShipStation API get call for the {retryCounter} time, delay {delayInSeconds} seconds", i, delay.TotalSeconds ); SystemUtil.Sleep( delay ); } ); @@ -62,7 +62,7 @@ public static ActionPolicyAsync GetAsync private static readonly ActionPolicyAsync _shipStationGetAsyncPolicy = ActionPolicyAsync.With( _exceptionHandler ).RetryAsync( 10, async ( ex, i ) => { - var delay = TimeSpan.FromSeconds( Math.Pow( 2, i ) ); + var delay = TimeSpan.FromSeconds( 0.5 + i ); ShipStationLogger.Log.Error( ex, "Retrying ShipStation API get call for the {retryCounter} time, delay {delayInSeconds} seconds", i, delay.TotalSeconds ); await Task.Delay( delay ); } ); diff --git a/src/ShipStationAccess/V2/Services/PaginatedResponse.cs b/src/ShipStationAccess/V2/Services/PaginatedResponse.cs index 24445a7..d0cd07f 100644 --- a/src/ShipStationAccess/V2/Services/PaginatedResponse.cs +++ b/src/ShipStationAccess/V2/Services/PaginatedResponse.cs @@ -6,13 +6,22 @@ namespace ShipStationAccess.V2.Services { public int? TotalPagesExpected { get; set; } public int? TotalEntitiesExpected { get; set; } - public int? TotalPagesReceived { get; set; } + public int TotalPagesReceived { get; set; } - public IEnumerable< T > Data { get; private set; } + public List< T > Data { get; private set; } + public List< ReadError > ReadErrors { get; private set; } - public PaginatedResponse( IEnumerable< T > data ) + public PaginatedResponse() { - this.Data = data ?? new List< T >(); + this.Data = new List< T >(); + this.ReadErrors = new List< ReadError >(); } } + + public sealed class ReadError + { + public string Url { get; set; } + public int PageSize { get; set; } + public int Page { get; set; } + } } \ No newline at end of file diff --git a/src/ShipStationAccess/V2/ShipStationService.cs b/src/ShipStationAccess/V2/ShipStationService.cs index e9bdf32..55bf6de 100644 --- a/src/ShipStationAccess/V2/ShipStationService.cs +++ b/src/ShipStationAccess/V2/ShipStationService.cs @@ -136,14 +136,14 @@ public IEnumerable< ShipStationOrder > GetOrders( DateTime dateFrom, DateTime da return orders; } - public async Task< IEnumerable< ShipStationOrder > > GetOrdersAsync( DateTime dateFrom, DateTime dateTo, bool getShipmentsAndFulfillments = false, Func< ShipStationOrder, Task< ShipStationOrder > > processOrder = null ) + public async Task< IEnumerable< ShipStationOrder > > GetOrdersAsync( DateTime dateFrom, DateTime dateTo, bool getShipmentsAndFulfillments = false, Func< ShipStationOrder, Task< ShipStationOrder > > processOrder = null, Action< IEnumerable< ReadError > > handleSkippedOrders = null ) { var allOrders = new List< ShipStationOrder >(); var createdOrders = await this.GetCreatedOrdersAsync( dateFrom, dateTo ).ConfigureAwait( false ); - allOrders.AddRange( createdOrders ); + allOrders.AddRange( createdOrders.Data ); var modifiedOrders = await this.GetModifiedOrdersAsync( dateFrom, dateTo ).ConfigureAwait( false ); - allOrders.AddRange( modifiedOrders ); + allOrders.AddRange( modifiedOrders.Data ); var uniqueOrders = allOrders.GroupBy( o => o.OrderId ).Select( gr => gr.First() ).ToList(); var processedOrders = await uniqueOrders.ProcessInBatchAsync( 5, async order => @@ -159,10 +159,20 @@ public async Task< IEnumerable< ShipStationOrder > > GetOrdersAsync( DateTime da if ( getShipmentsAndFulfillments ) await this.FindShipmentsAndFulfillments( processedOrders ).ConfigureAwait( false ); + if ( handleSkippedOrders != null ) + { + var allSkippedOrders = new List< ReadError >(); + allSkippedOrders.AddRange( createdOrders.ReadErrors ); + allSkippedOrders.AddRange( modifiedOrders.ReadErrors ); + + if ( allSkippedOrders.Any() ) + handleSkippedOrders( allSkippedOrders ); + } + return processedOrders; } - public async Task< IEnumerable< ShipStationOrder > > GetCreatedOrdersAsync( DateTime dateFrom, DateTime dateTo ) + public async Task< PaginatedResponse< ShipStationOrder > > GetCreatedOrdersAsync( DateTime dateFrom, DateTime dateTo ) { var createdOrdersEndpoint = ParamsBuilder.CreateNewOrdersParams( dateFrom, dateTo ); var createdOrdersResponse = await this.DownloadOrdersAsync( createdOrdersEndpoint ).ConfigureAwait( false ); @@ -172,15 +182,15 @@ public async Task< IEnumerable< ShipStationOrder > > GetCreatedOrdersAsync( Date _webRequestServices.GetApiKey(), createdOrdersResponse.Data.Count(), createdOrdersResponse.TotalEntitiesExpected ?? 0, - createdOrdersResponse.TotalPagesReceived ?? 0, + createdOrdersResponse.TotalPagesReceived, createdOrdersResponse.TotalPagesExpected ?? 0, createdOrdersResponse ); } - return createdOrdersResponse.Data; + return createdOrdersResponse; } - public async Task< IEnumerable< ShipStationOrder > > GetModifiedOrdersAsync( DateTime dateFrom, DateTime dateTo ) + public async Task< PaginatedResponse< ShipStationOrder > > GetModifiedOrdersAsync( DateTime dateFrom, DateTime dateTo ) { var modifiedOrdersEndpoint = ParamsBuilder.CreateModifiedOrdersParams( dateFrom, dateTo ); var modifiedOrdersResponse = await this.DownloadOrdersAsync( modifiedOrdersEndpoint ).ConfigureAwait( false ); @@ -190,19 +200,17 @@ public async Task< IEnumerable< ShipStationOrder > > GetModifiedOrdersAsync( Dat _webRequestServices.GetApiKey(), modifiedOrdersResponse.Data.Count(), modifiedOrdersResponse.TotalEntitiesExpected ?? 0, - modifiedOrdersResponse.TotalPagesReceived ?? 0, + modifiedOrdersResponse.TotalPagesReceived, modifiedOrdersResponse.TotalPagesExpected ?? 0, modifiedOrdersResponse ); } - return modifiedOrdersResponse.Data; + return modifiedOrdersResponse; } public async Task< PaginatedResponse< ShipStationOrder > > DownloadOrdersAsync( string endPoint ) { - var orders = new List< ShipStationOrder >(); - int? totalShipStationPages = null; - int? totalShipStationOrders = null; + var response = new PaginatedResponse< ShipStationOrder >(); var currentPage = 1; do @@ -211,29 +219,47 @@ public async Task< PaginatedResponse< ShipStationOrder > > DownloadOrdersAsync( var ordersEndPoint = endPoint.ConcatParams( nextPageParams ); ShipStationOrders ordersPage = null; - await ActionPolicies.GetAsync.Do( async () => + try { - ordersPage = await this._webRequestServices.GetResponseAsync< ShipStationOrders >( ShipStationCommand.GetOrders, ordersEndPoint ).ConfigureAwait( false ); - } ); + await ActionPolicies.GetAsync.Do( async () => + { + ordersPage = await this._webRequestServices.GetResponseAsync< ShipStationOrders >( ShipStationCommand.GetOrders, ordersEndPoint ).ConfigureAwait( false ); + } ); + } + catch( WebException e ) + { + if( WebRequestServices.CanSkipException( e ) ) + { + response.ReadErrors.Add( new ReadError() + { + Url = endPoint, + Page = currentPage, + PageSize = RequestMaxLimit + } ); + + ShipStationLogger.Log.Warn( e, "Skipped get orders request page {pageNumber} of request {request} due to internal error on ShipStation's side", currentPage, ordersEndPoint ); + currentPage++; + continue; + } + else + throw; + } currentPage++; if ( ordersPage?.Orders == null || !ordersPage.Orders.Any() ) break; - totalShipStationPages = ordersPage.TotalPages; - totalShipStationOrders = ordersPage.TotalOrders; + response.TotalPagesExpected = ordersPage.TotalPages; + response.TotalEntitiesExpected = ordersPage.TotalOrders; - orders.AddRange( ordersPage.Orders ); + response.Data.AddRange( ordersPage.Orders ); - } while( currentPage <= totalShipStationPages ); + } while( currentPage <= response.TotalPagesExpected ); - return new PaginatedResponse< ShipStationOrder >( orders ) - { - TotalPagesExpected = totalShipStationPages, - TotalEntitiesExpected = totalShipStationOrders, - TotalPagesReceived = currentPage - 1 - }; + response.TotalPagesReceived = currentPage - 1; + + return response; } public IEnumerable< ShipStationOrder > GetOrders( string storeId, string orderNumber )