diff --git a/src/Global/GlobalAssemblyInfo.cs b/src/Global/GlobalAssemblyInfo.cs index 0cdcb95..2e5b9b2 100644 --- a/src/Global/GlobalAssemblyInfo.cs +++ b/src/Global/GlobalAssemblyInfo.cs @@ -23,4 +23,4 @@ // [assembly: AssemblyVersion("1.0.*")] // Keep in track with CA API version -[ assembly : AssemblyVersion( "1.4.2.0" ) ] \ No newline at end of file +[ assembly : AssemblyVersion( "1.4.3.0" ) ] \ No newline at end of file diff --git a/src/ShipStationAccess/ShipStationAccess.csproj b/src/ShipStationAccess/ShipStationAccess.csproj index 209fa36..37ef95c 100644 --- a/src/ShipStationAccess/ShipStationAccess.csproj +++ b/src/ShipStationAccess/ShipStationAccess.csproj @@ -115,6 +115,7 @@ + diff --git a/src/ShipStationAccess/V2/IShipStationService.cs b/src/ShipStationAccess/V2/IShipStationService.cs index 758088b..ee98c3e 100644 --- a/src/ShipStationAccess/V2/IShipStationService.cs +++ b/src/ShipStationAccess/V2/IShipStationService.cs @@ -7,13 +7,14 @@ using ShipStationAccess.V2.Models.Store; using ShipStationAccess.V2.Models.TagList; using ShipStationAccess.V2.Models.WarehouseLocation; +using ShipStationAccess.V2.Services; namespace ShipStationAccess.V2 { public interface IShipStationService { IEnumerable< ShipStationOrder > GetOrders( DateTime dateFrom, DateTime dateTo, Func< ShipStationOrder, ShipStationOrder > processOrder = null ); - Task< IEnumerable< ShipStationOrder > > GetOrdersAsync( DateTime dateFrom, DateTime dateTo, bool getShipmentsAndFulfillments = true, Func< ShipStationOrder, Task< ShipStationOrder > > processOrder = null ); + Task< IEnumerable< ShipStationOrder > > GetOrdersAsync( DateTime dateFrom, DateTime dateTo, bool getShipmentsAndFulfillments = false, Func< ShipStationOrder, Task< ShipStationOrder > > processOrder = null, Action< IEnumerable< ReadError > > handleSkippedOrders = null ); IEnumerable< ShipStationOrder > GetOrders( string storeId, string orderNumber ); ShipStationOrder GetOrderById( string orderId ); diff --git a/src/ShipStationAccess/V2/Misc/ActionPolicies.cs b/src/ShipStationAccess/V2/Misc/ActionPolicies.cs index 7db4ba0..bc30606 100644 --- a/src/ShipStationAccess/V2/Misc/ActionPolicies.cs +++ b/src/ShipStationAccess/V2/Misc/ActionPolicies.cs @@ -11,7 +11,7 @@ public static class ActionPolicies { public static ActionPolicy Submit { - get { return _shipStationSumbitPolicy; } + get { return _shipStationSubmitPolicy; } } private static readonly ExceptionHandler _exceptionHandler = delegate( Exception x ) @@ -24,21 +24,23 @@ public static ActionPolicy Submit return webX.Response.GetHttpStatusCode() != HttpStatusCode.Unauthorized; }; - private static readonly ActionPolicy _shipStationSumbitPolicy = ActionPolicy.With( _exceptionHandler ).Retry( 10, ( ex, i ) => + private static readonly ActionPolicy _shipStationSubmitPolicy = ActionPolicy.With( _exceptionHandler ).Retry( 10, ( ex, i ) => { - ShipStationLogger.Log.Error( ex, "Retrying ShipStation API submit call for the {retryCounter} time", i ); - SystemUtil.Sleep( TimeSpan.FromSeconds( 0.5 + i ) ); + var delay = TimeSpan.FromSeconds( 0.5 + i ); + ShipStationLogger.Log.Error( ex, "Retrying ShipStation API submit call for the {retryCounter} time, delay {delayInSeconds} seconds", i, delay.TotalSeconds ); + SystemUtil.Sleep( delay ); } ); public static ActionPolicyAsync SubmitAsync { - get { return _shipStationSumbitAsyncPolicy; } + get { return _shipStationSubmitAsyncPolicy; } } - private static readonly ActionPolicyAsync _shipStationSumbitAsyncPolicy = ActionPolicyAsync.With( _exceptionHandler ).RetryAsync( 10, async ( ex, i ) => + private static readonly ActionPolicyAsync _shipStationSubmitAsyncPolicy = ActionPolicyAsync.With( _exceptionHandler ).RetryAsync( 10, async ( ex, i ) => { - ShipStationLogger.Log.Error( ex, "Retrying ShipStation API submit call for the {retryCounter} time", i ); - await Task.Delay( TimeSpan.FromSeconds( 0.5 + i ) ); + var delay = TimeSpan.FromSeconds( 0.5 + i ); + ShipStationLogger.Log.Error( ex, "Retrying ShipStation API submit call for the {retryCounter} time, delay {delayInSeconds} seconds", i, delay.TotalSeconds ); + await Task.Delay( delay ); } ); public static ActionPolicy Get @@ -48,8 +50,9 @@ public static ActionPolicy Get private static readonly ActionPolicy _shipStationGetPolicy = ActionPolicy.With( _exceptionHandler ).Retry( 10, ( ex, i ) => { - ShipStationLogger.Log.Error( ex, "Retrying ShipStation API get call for the {retryCounter} time", i ); - SystemUtil.Sleep( TimeSpan.FromSeconds( 0.5 + i ) ); + var delay = TimeSpan.FromSeconds( 0.5 + i ); + ShipStationLogger.Log.Error( ex, "Retrying ShipStation API get call for the {retryCounter} time, delay {delayInSeconds} seconds", i, delay.TotalSeconds ); + SystemUtil.Sleep( delay ); } ); public static ActionPolicyAsync GetAsync @@ -59,8 +62,9 @@ public static ActionPolicyAsync GetAsync private static readonly ActionPolicyAsync _shipStationGetAsyncPolicy = ActionPolicyAsync.With( _exceptionHandler ).RetryAsync( 10, async ( ex, i ) => { - ShipStationLogger.Log.Error( ex, "Retrying ShipStation API get call for the {retryCounter} time", i ); - await Task.Delay( TimeSpan.FromSeconds( 0.5 + i ) ); + var delay = TimeSpan.FromSeconds( 0.5 + i ); + ShipStationLogger.Log.Error( ex, "Retrying ShipStation API get call for the {retryCounter} time, delay {delayInSeconds} seconds", i, delay.TotalSeconds ); + await Task.Delay( delay ); } ); } } \ No newline at end of file diff --git a/src/ShipStationAccess/V2/Services/PaginatedResponse.cs b/src/ShipStationAccess/V2/Services/PaginatedResponse.cs new file mode 100644 index 0000000..d0cd07f --- /dev/null +++ b/src/ShipStationAccess/V2/Services/PaginatedResponse.cs @@ -0,0 +1,27 @@ +using System.Collections.Generic; + +namespace ShipStationAccess.V2.Services +{ + public sealed class PaginatedResponse< T > where T : class, new() + { + public int? TotalPagesExpected { get; set; } + public int? TotalEntitiesExpected { get; set; } + public int TotalPagesReceived { get; set; } + + public List< T > Data { get; private set; } + public List< ReadError > ReadErrors { get; private set; } + + public PaginatedResponse() + { + this.Data = new List< T >(); + this.ReadErrors = new List< ReadError >(); + } + } + + public sealed class ReadError + { + public string Url { get; set; } + public int PageSize { get; set; } + public int Page { get; set; } + } +} \ No newline at end of file diff --git a/src/ShipStationAccess/V2/ShipStationService.cs b/src/ShipStationAccess/V2/ShipStationService.cs index 50a5c1a..55bf6de 100644 --- a/src/ShipStationAccess/V2/ShipStationService.cs +++ b/src/ShipStationAccess/V2/ShipStationService.cs @@ -1,14 +1,9 @@ using System; using System.Collections.Generic; -using System.Diagnostics; -using System.IO; using System.Linq; using System.Net; -using System.Reflection; using System.Threading.Tasks; -using Microsoft.CSharp.RuntimeBinder; using Netco.Extensions; -using Newtonsoft.Json; using ShipStationAccess.V2.Exceptions; using ShipStationAccess.V2.Misc; using ShipStationAccess.V2.Models; @@ -141,98 +136,132 @@ public IEnumerable< ShipStationOrder > GetOrders( DateTime dateFrom, DateTime da return orders; } - public async Task< IEnumerable< ShipStationOrder > > GetOrdersAsync( DateTime dateFrom, DateTime dateTo, bool getShipmentsAndFulfillments = true, Func< ShipStationOrder, Task< ShipStationOrder > > processOrder = null ) + public async Task< IEnumerable< ShipStationOrder > > GetOrdersAsync( DateTime dateFrom, DateTime dateTo, bool getShipmentsAndFulfillments = false, Func< ShipStationOrder, Task< ShipStationOrder > > processOrder = null, Action< IEnumerable< ReadError > > handleSkippedOrders = null ) { - var orders = new List< ShipStationOrder >(); - var processedOrderIds = new HashSet< long >(); + var allOrders = new List< ShipStationOrder >(); + var createdOrders = await this.GetCreatedOrdersAsync( dateFrom, dateTo ).ConfigureAwait( false ); + allOrders.AddRange( createdOrders.Data ); + + var modifiedOrders = await this.GetModifiedOrdersAsync( dateFrom, dateTo ).ConfigureAwait( false ); + allOrders.AddRange( modifiedOrders.Data ); - Func< ShipStationOrders, Task > processOrders = async sorders => + var uniqueOrders = allOrders.GroupBy( o => o.OrderId ).Select( gr => gr.First() ).ToList(); + var processedOrders = await uniqueOrders.ProcessInBatchAsync( 5, async order => { - var processedOrders = await sorders.Orders.ProcessInBatchAsync( 5, async o => - { - var curOrder = o; - if( processedOrderIds.Contains( curOrder.OrderId ) ) - return null; + if( processOrder != null ) + order = await processOrder( order ).ConfigureAwait( false ); - if( processOrder != null ) - curOrder = await processOrder( curOrder ); + return order; + } ); - return curOrder; - } ); + await this.FindMarketplaceIdsAsync( processedOrders ).ConfigureAwait( false ); - foreach( var order in processedOrders ) - { - if ( getShipmentsAndFulfillments ) - { - order.Shipments = await GetOrderShipmentsByIdAsync( order.OrderId.ToString() ).ConfigureAwait( false ); - order.Fulfillments = await GetOrderFulfillmentsByIdAsync( order.OrderId.ToString() ).ConfigureAwait( false ); - } + if ( getShipmentsAndFulfillments ) + await this.FindShipmentsAndFulfillments( processedOrders ).ConfigureAwait( false ); - orders.Add( order ); - processedOrderIds.Add( order.OrderId ); - } - }; + if ( handleSkippedOrders != null ) + { + var allSkippedOrders = new List< ReadError >(); + allSkippedOrders.AddRange( createdOrders.ReadErrors ); + allSkippedOrders.AddRange( modifiedOrders.ReadErrors ); + + if ( allSkippedOrders.Any() ) + handleSkippedOrders( allSkippedOrders ); + } + + return processedOrders; + } - Func< string, Task > downloadOrders = async endPoint => + public async Task< PaginatedResponse< ShipStationOrder > > GetCreatedOrdersAsync( DateTime dateFrom, DateTime dateTo ) + { + var createdOrdersEndpoint = ParamsBuilder.CreateNewOrdersParams( dateFrom, dateTo ); + var createdOrdersResponse = await this.DownloadOrdersAsync( createdOrdersEndpoint ).ConfigureAwait( false ); + if ( createdOrdersResponse.Data.Any() ) { - var pagesCount = int.MaxValue; - var currentPage = 1; - var ordersCount = 0; - var ordersExpected = -1; + ShipStationLogger.Log.Info( "Created orders downloaded using tenant's API key '{apiKey}' - {orders}/{expectedOrders} orders in {pages}/{expectedPages} from {endpoint}", + _webRequestServices.GetApiKey(), + createdOrdersResponse.Data.Count(), + createdOrdersResponse.TotalEntitiesExpected ?? 0, + createdOrdersResponse.TotalPagesReceived, + createdOrdersResponse.TotalPagesExpected ?? 0, + createdOrdersResponse ); + } - do - { - var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, RequestMaxLimit ) ); - var ordersEndPoint = endPoint.ConcatParams( nextPageParams ); + return createdOrdersResponse; + } - ShipStationOrders ordersWithinPage = null; - try - { - await ActionPolicies.GetAsync.Do( async () => - { - ordersWithinPage = await this._webRequestServices.GetResponseAsync< ShipStationOrders >( ShipStationCommand.GetOrders, ordersEndPoint ); - } ); - } - catch( WebException e ) - { - if( WebRequestServices.CanSkipException( e ) ) - { - ShipStationLogger.Log.Warn( e, "Skipped get orders request page {pageNumber} of request {request} due to internal error on ShipStation's side", currentPage, ordersEndPoint ); - } - else - throw; - } + public async Task< PaginatedResponse< ShipStationOrder > > GetModifiedOrdersAsync( DateTime dateFrom, DateTime dateTo ) + { + var modifiedOrdersEndpoint = ParamsBuilder.CreateModifiedOrdersParams( dateFrom, dateTo ); + var modifiedOrdersResponse = await this.DownloadOrdersAsync( modifiedOrdersEndpoint ).ConfigureAwait( false ); + if ( modifiedOrdersResponse.Data.Any() ) + { + ShipStationLogger.Log.Info( "Modified orders downloaded using tenant's API key '{apiKey}' - {orders}/{expectedOrders} orders in {pages}/{expectedPages} from {endpoint}", + _webRequestServices.GetApiKey(), + modifiedOrdersResponse.Data.Count(), + modifiedOrdersResponse.TotalEntitiesExpected ?? 0, + modifiedOrdersResponse.TotalPagesReceived, + modifiedOrdersResponse.TotalPagesExpected ?? 0, + modifiedOrdersResponse ); + } - currentPage++; + return modifiedOrdersResponse; + } - if( ordersWithinPage != null ) + public async Task< PaginatedResponse< ShipStationOrder > > DownloadOrdersAsync( string endPoint ) + { + var response = new PaginatedResponse< ShipStationOrder >(); + var currentPage = 1; + + do + { + var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, RequestMaxLimit ) ); + var ordersEndPoint = endPoint.ConcatParams( nextPageParams ); + + ShipStationOrders ordersPage = null; + try + { + await ActionPolicies.GetAsync.Do( async () => { - if( pagesCount == int.MaxValue ) + ordersPage = await this._webRequestServices.GetResponseAsync< ShipStationOrders >( ShipStationCommand.GetOrders, ordersEndPoint ).ConfigureAwait( false ); + } ); + } + catch( WebException e ) + { + if( WebRequestServices.CanSkipException( e ) ) + { + response.ReadErrors.Add( new ReadError() { - pagesCount = ordersWithinPage.TotalPages; - ordersExpected = ordersWithinPage.TotalOrders; - } - - ordersCount += ordersWithinPage.Orders.Count; + Url = endPoint, + Page = currentPage, + PageSize = RequestMaxLimit + } ); - await processOrders( ordersWithinPage ); + ShipStationLogger.Log.Warn( e, "Skipped get orders request page {pageNumber} of request {request} due to internal error on ShipStation's side", currentPage, ordersEndPoint ); + currentPage++; + continue; } - } while( currentPage <= pagesCount ); + else + throw; + } - ShipStationLogger.Log.Info( "Orders dowloaded API '{apiKey}' - {orders}/{expectedOrders} orders in {pages}/{expectedPages} from {endpoint}", _webRequestServices.GetApiKey(), ordersCount, ordersExpected, currentPage - 1, pagesCount, endPoint ); - }; + currentPage++; - var newOrdersEndpoint = ParamsBuilder.CreateNewOrdersParams( dateFrom, dateTo ); - await downloadOrders( newOrdersEndpoint ); + if ( ordersPage?.Orders == null || !ordersPage.Orders.Any() ) + break; - var modifiedOrdersEndpoint = ParamsBuilder.CreateModifiedOrdersParams( dateFrom, dateTo ); - await downloadOrders( modifiedOrdersEndpoint ); + response.TotalPagesExpected = ordersPage.TotalPages; + response.TotalEntitiesExpected = ordersPage.TotalOrders; - await this.FindMarketplaceIdsAsync( orders ); + response.Data.AddRange( ordersPage.Orders ); - return orders; + } while( currentPage <= response.TotalPagesExpected ); + + response.TotalPagesReceived = currentPage - 1; + + return response; } - + public IEnumerable< ShipStationOrder > GetOrders( string storeId, string orderNumber ) { var orders = new List< ShipStationOrder >(); @@ -307,12 +336,21 @@ await ActionPolicies.GetAsync.Do( async () => return order; } + private async Task FindShipmentsAndFulfillments( IEnumerable< ShipStationOrder > orders ) + { + foreach( var order in orders ) + { + order.Shipments = await this.GetOrderShipmentsByIdAsync( order.OrderId.ToString() ).ConfigureAwait( false ); + order.Fulfillments = await this.GetOrderFulfillmentsByIdAsync( order.OrderId.ToString() ).ConfigureAwait( false ); + } + } + public async Task< IEnumerable< ShipStationOrderShipment > > GetOrderShipmentsByIdAsync( string orderId ) { var orderShipments = new List< ShipStationOrderShipment >(); var currentPage = 1; - var pagesCount = int.MaxValue; + int? totalShipStationShipmentsPages; do { @@ -320,21 +358,21 @@ public async Task< IEnumerable< ShipStationOrderShipment > > GetOrderShipmentsBy var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, RequestMaxLimit ) ); var orderShipmentsByPageEndPoint = getOrderShipmentsEndpoint.ConcatParams( nextPageParams ); + ShipStationOrderShipments ordersShipmentsPage = null; await ActionPolicies.GetAsync.Do( async () => { - var orderShipmentsPage = await this._webRequestServices.GetResponseAsync< ShipStationOrderShipments >( ShipStationCommand.GetOrderShipments, orderShipmentsByPageEndPoint ); - - ++currentPage; - if ( pagesCount == int.MaxValue ) - { - pagesCount = orderShipmentsPage.Pages + 1; - } + ordersShipmentsPage = await this._webRequestServices.GetResponseAsync< ShipStationOrderShipments >( ShipStationCommand.GetOrderShipments, orderShipmentsByPageEndPoint ).ConfigureAwait( false ); + } ); - orderShipments.AddRange( orderShipmentsPage.Shipments ); + if ( ordersShipmentsPage?.Shipments == null || !ordersShipmentsPage.Shipments.Any() ) + break; - } ); + ++currentPage; + totalShipStationShipmentsPages = ordersShipmentsPage.Pages + 1; + + orderShipments.AddRange( ordersShipmentsPage.Shipments ); } - while( currentPage <= pagesCount ); + while( currentPage <= totalShipStationShipmentsPages ); return orderShipments; } @@ -344,7 +382,7 @@ public async Task< IEnumerable< ShipStationOrderFulfillment > > GetOrderFulfillm var orderFulfillments = new List< ShipStationOrderFulfillment >(); var currentPage = 1; - var pagesCount = int.MaxValue; + int? totalShipStationFulfillmentsPages; do { @@ -352,21 +390,21 @@ public async Task< IEnumerable< ShipStationOrderFulfillment > > GetOrderFulfillm var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, RequestMaxLimit ) ); var orderFulfillmentsByPageEndPoint = getOrderFulfillmentsEndpoint.ConcatParams( nextPageParams ); + ShipStationOrderFulfillments orderFulfillmentsPage = null; await ActionPolicies.GetAsync.Do( async () => { - var orderFulfillmentsPage = await this._webRequestServices.GetResponseAsync< ShipStationOrderFulfillments >( ShipStationCommand.GetOrderFulfillments, orderFulfillmentsByPageEndPoint ); - - ++currentPage; - if ( pagesCount == int.MaxValue ) - { - pagesCount = orderFulfillmentsPage.Pages + 1; - } + orderFulfillmentsPage = await this._webRequestServices.GetResponseAsync< ShipStationOrderFulfillments >( ShipStationCommand.GetOrderFulfillments, orderFulfillmentsByPageEndPoint ).ConfigureAwait( false ); + } ); - orderFulfillments.AddRange( orderFulfillmentsPage.Fulfillments ); + if ( orderFulfillmentsPage?.Fulfillments == null || !orderFulfillmentsPage.Fulfillments.Any() ) + break; - } ); + ++currentPage; + totalShipStationFulfillmentsPages = orderFulfillmentsPage.Pages + 1; + + orderFulfillments.AddRange( orderFulfillmentsPage.Fulfillments ); } - while( currentPage <= pagesCount ); + while( currentPage <= totalShipStationFulfillmentsPages ); return orderFulfillments; } @@ -520,7 +558,7 @@ private void FindMarketplaceIds( IEnumerable< ShipStationOrder > orders ) private async Task FindMarketplaceIdsAsync( IEnumerable< ShipStationOrder > orders ) { - var stores = await this.GetStoresAsync(); + var stores = await this.GetStoresAsync().ConfigureAwait( false ); foreach( var order in orders ) { diff --git a/src/ShipStationAccessTests/Orders/OrderTests.cs b/src/ShipStationAccessTests/Orders/OrderTests.cs index 1da25d8..e3c5a20 100644 --- a/src/ShipStationAccessTests/Orders/OrderTests.cs +++ b/src/ShipStationAccessTests/Orders/OrderTests.cs @@ -53,7 +53,7 @@ [ Test ] public async Task GetOrdersAsync() { var service = this.ShipStationFactory.CreateServiceV2( this._credentials ); - var orders = await service.GetOrdersAsync( DateTime.UtcNow.AddDays( -3 ), DateTime.UtcNow ); + var orders = await service.GetOrdersAsync( DateTime.UtcNow.AddDays( -1 ), DateTime.UtcNow, getShipmentsAndFulfillments: true ); orders.Count().Should().BeGreaterThan( 0 ); } @@ -62,7 +62,7 @@ [ Test ] public async Task GetOrdersWithoutShipmentsAndFulfillmentsAsync() { var service = this.ShipStationFactory.CreateServiceV2( this._credentials ); - var orders = await service.GetOrdersAsync( DateTime.UtcNow.AddDays( -3 ), DateTime.UtcNow, false ); + var orders = await service.GetOrdersAsync( DateTime.UtcNow.AddDays( -3 ), DateTime.UtcNow, getShipmentsAndFulfillments: false ); orders.Count().Should().BeGreaterThan( 0 ); orders.Any( o => o.Shipments != null ).Should().Be( false );