From 0cc7876cfa5716f49513d3a74a24c04053d836ef Mon Sep 17 00:00:00 2001 From: christian-bejarano-skuvault <103965678+christian-bejarano-skuvault@users.noreply.github.com> Date: Mon, 21 Aug 2023 15:40:15 -0500 Subject: [PATCH] GUARD-3049 Improve Orders Sync (#13) * GUARD-3049 Add new endpoints to obtain shipments and fullfilment data by creation date * GUARD-3049 Extracted logic into its own methods, renamed variables, added comments. * GUARD-3049 Minor changes * GUARD-3049 Rename orders properties * GUARD-3049 Update package version --- .build.ps1 | 2 +- src/Global/GlobalAssemblyInfo.cs | 2 +- .../V2/IShipStationService.cs | 2 + .../Order/ShipStationOrderFulfillment.cs | 4 +- .../Models/Order/ShipStationOrderShipment.cs | 4 +- .../V2/Services/ParamsBuilder.cs | 13 ++ .../V2/ShipStationService.cs | 124 ++++++++++++++++-- .../Orders/OrderTests.cs | 20 +++ 8 files changed, 151 insertions(+), 20 deletions(-) diff --git a/.build.ps1 b/.build.ps1 index 0e353e8..4c95347 100644 --- a/.build.ps1 +++ b/.build.ps1 @@ -76,7 +76,7 @@ task NuGet Package, Version, { $project_name - $Version + $Version-alpha Agile Harbor Agile Harbor https://github.com/agileharbor/$project_name diff --git a/src/Global/GlobalAssemblyInfo.cs b/src/Global/GlobalAssemblyInfo.cs index fa48d8b..590fa38 100644 --- a/src/Global/GlobalAssemblyInfo.cs +++ b/src/Global/GlobalAssemblyInfo.cs @@ -22,4 +22,4 @@ // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[ assembly : AssemblyVersion( "1.6.1" ) ] \ No newline at end of file +[ assembly : AssemblyVersion( "1.7.1" ) ] \ No newline at end of file diff --git a/src/ShipStationAccess/V2/IShipStationService.cs b/src/ShipStationAccess/V2/IShipStationService.cs index e0070bf..0afd7d6 100644 --- a/src/ShipStationAccess/V2/IShipStationService.cs +++ b/src/ShipStationAccess/V2/IShipStationService.cs @@ -21,7 +21,9 @@ public interface IShipStationService ShipStationOrder GetOrderById( string orderId, CancellationToken token ); Task< ShipStationOrder > GetOrderByIdAsync( string orderId, CancellationToken token ); Task< IEnumerable< ShipStationOrderShipment > > GetOrderShipmentsByIdAsync( string orderId, CancellationToken token ); + Task< IEnumerable< ShipStationOrderShipment > > GetOrderShipmentsByCreatedDateAsync( DateTime createdDate, CancellationToken token ); Task< IEnumerable< ShipStationOrderFulfillment > > GetOrderFulfillmentsByIdAsync( string orderId, CancellationToken token ); + Task< IEnumerable< ShipStationOrderFulfillment > > GetOrderFulfillmentsByCreatedDateAsync( DateTime createdDate, CancellationToken token ); void UpdateOrder( ShipStationOrder order, CancellationToken token ); Task UpdateOrderAsync( ShipStationOrder order, CancellationToken token ); diff --git a/src/ShipStationAccess/V2/Models/Order/ShipStationOrderFulfillment.cs b/src/ShipStationAccess/V2/Models/Order/ShipStationOrderFulfillment.cs index 6a3493f..8f6de54 100644 --- a/src/ShipStationAccess/V2/Models/Order/ShipStationOrderFulfillment.cs +++ b/src/ShipStationAccess/V2/Models/Order/ShipStationOrderFulfillment.cs @@ -11,13 +11,13 @@ [ DataMember( Name = "fulfillments" ) ] public IEnumerable< ShipStationOrderFulfillment > Fulfillments { get; set; } [ DataMember( Name = "total" ) ] - public int Total { get; set; } + public int TotalFulfillments { get; set; } [ DataMember( Name = "page" ) ] public int Page { get; set; } [ DataMember( Name = "pages" ) ] - public int Pages { get; set; } + public int TotalPages { get; set; } } [ DataContract ] diff --git a/src/ShipStationAccess/V2/Models/Order/ShipStationOrderShipment.cs b/src/ShipStationAccess/V2/Models/Order/ShipStationOrderShipment.cs index b5d9c6f..c342589 100644 --- a/src/ShipStationAccess/V2/Models/Order/ShipStationOrderShipment.cs +++ b/src/ShipStationAccess/V2/Models/Order/ShipStationOrderShipment.cs @@ -11,13 +11,13 @@ [ DataMember( Name = "shipments" ) ] public IEnumerable< ShipStationOrderShipment > Shipments { get; set; } [ DataMember( Name = "total" ) ] - public int Total { get; set; } + public int TotalShipments { get; set; } [ DataMember( Name = "page" ) ] public int Page { get; set; } [ DataMember( Name = "pages" ) ] - public int Pages { get; set; } + public int TotalPages { get; set; } } [ DataContract ] diff --git a/src/ShipStationAccess/V2/Services/ParamsBuilder.cs b/src/ShipStationAccess/V2/Services/ParamsBuilder.cs index 2dc494b..a9d603a 100644 --- a/src/ShipStationAccess/V2/Services/ParamsBuilder.cs +++ b/src/ShipStationAccess/V2/Services/ParamsBuilder.cs @@ -40,11 +40,24 @@ public static string CreateOrderShipmentsParams( string orderId, bool includeShi return endpoint; } + public static string CreateOrderShipmentsParams( DateTime createdDate, bool includeShipmentItems = true ) + { + var endpoint = string.Format( "?{0}={1}&{2}={3}", + ShipStationParam.OrdersCreatedDateFrom.Name, createdDate.ToString( "s" ), + ShipStationParam.IncludeShipmentItems.Name, includeShipmentItems ); + return endpoint; + } + public static string CreateOrderFulfillmentsParams( string orderId ) { return string.Format( "?{0}={1}", ShipStationParam.OrderId.Name, orderId ); } + public static string CreateOrderFulfillmentsParams( DateTime createdDate ) + { + return string.Format( "?{0}={1}", ShipStationParam.OrdersCreatedDateFrom.Name, createdDate.ToString( "s" ) ); + } + public static string CreateGetNextPageParams( ShipStationCommandConfig config ) { var endpoint = string.Format( "?{0}={1}&{2}={3}", diff --git a/src/ShipStationAccess/V2/ShipStationService.cs b/src/ShipStationAccess/V2/ShipStationService.cs index be2254d..534be06 100644 --- a/src/ShipStationAccess/V2/ShipStationService.cs +++ b/src/ShipStationAccess/V2/ShipStationService.cs @@ -25,9 +25,9 @@ public sealed class ShipStationService: IShipStationService private readonly IWebRequestServices _webRequestServices; private readonly SyncRunContext _syncRunContext; private readonly ShipStationTimeouts _timeouts; - - // lowered max limit for less order loss on Shipsation API's internal errors - private const int RequestMaxLimit = 20; + private const int OrdersPageSize = 500; + // Max orders that can be processed without needing complex shipping and fulfillment handling. + private const int OrderProcessingThreshold = 10; internal ShipStationService( IWebRequestServices webRequestServices, SyncRunContext syncRunContext, ShipStationTimeouts timeouts ) { @@ -122,7 +122,7 @@ public IEnumerable< ShipStationOrder > GetOrders( DateTime dateFrom, DateTime da do { - var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, RequestMaxLimit ) ); + var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, OrdersPageSize ) ); var ordersEndPoint = endPoint.ConcatParams( nextPageParams ); ActionPolicies.Get.Do( () => @@ -202,7 +202,7 @@ public async Task< SummaryResponse< ShipStationOrder > > GetCreatedOrdersAsync( { var createdOrdersEndpoint = ParamsBuilder.CreateNewOrdersParams( dateFrom, dateTo ); var createdOrdersResponse = new SummaryResponse< ShipStationOrder >(); - await this.DownloadOrdersAsync( createdOrdersResponse, createdOrdersEndpoint, 1, RequestMaxLimit, token ).ConfigureAwait( false ); + await this.DownloadOrdersAsync( createdOrdersResponse, createdOrdersEndpoint, 1, OrdersPageSize, token ).ConfigureAwait( false ); if ( createdOrdersResponse.Data.Any() ) { ShipStationLogger.Log.Info( Constants.LoggingCommonPrefix + "Created orders downloaded - {Orders}/{ExpectedOrders} orders from {Endpoint}. Response: '{Response}'", @@ -223,7 +223,7 @@ public async Task< SummaryResponse< ShipStationOrder > > GetModifiedOrdersAsync( { var modifiedOrdersEndpoint = ParamsBuilder.CreateModifiedOrdersParams( dateFrom, dateTo ); var modifiedOrdersResponse = new SummaryResponse< ShipStationOrder >(); - await this.DownloadOrdersAsync( modifiedOrdersResponse, modifiedOrdersEndpoint, 1, RequestMaxLimit, token ).ConfigureAwait( false ); + await this.DownloadOrdersAsync( modifiedOrdersResponse, modifiedOrdersEndpoint, 1, OrdersPageSize, token ).ConfigureAwait( false ); if ( modifiedOrdersResponse.Data.Any() ) { ShipStationLogger.Log.Info( Constants.LoggingCommonPrefix + "Modified orders downloaded - {Orders}/{ExpectedOrders} orders from {Endpoint}. Response: '{Response}'", @@ -307,9 +307,9 @@ public async Task DownloadOrdersAsync( SummaryResponse< ShipStationOrder > summa summary.Data.AddRange( ordersPage.Data ); currentPage++; - if ( currentPageSize != RequestMaxLimit ) + if ( currentPageSize != OrdersPageSize ) { - var newPageSize = PageSizeAdjuster.DoublePageSize( currentPageSize, RequestMaxLimit ); + var newPageSize = PageSizeAdjuster.DoublePageSize( currentPageSize, OrdersPageSize ); var newCurrentPage = PageSizeAdjuster.GetNextPageIndex( summary.TotalEntitiesHandled, newPageSize ); if ( !PageSizeAdjuster.AreEntitiesWillBeDownloadedAgainAfterChangingThePageSize( newCurrentPage, newPageSize, summary.TotalEntitiesHandled ) ) @@ -390,7 +390,7 @@ public IEnumerable< ShipStationOrder > GetOrders( string storeId, string orderNu do { - var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, RequestMaxLimit ) ); + var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, OrdersPageSize ) ); var ordersEndPoint = endPoint.ConcatParams( nextPageParams ); ActionPolicies.Get.Do( () => @@ -448,7 +448,39 @@ await ActionPolicies.GetAsync.Do( async () => private async Task FindShipmentsAndFulfillments( IEnumerable< ShipStationOrder > orders, CancellationToken token ) { - foreach( var order in orders ) + if( orders.ToList().Count > OrderProcessingThreshold ) + { + await GetShipmentsAndFullfilmentsByCreationDate( orders, token ).ConfigureAwait( false ); + } + else + { + await GetShipmentsAndFullfilmentByOrderId( orders, token ).ConfigureAwait( false ); + } + } + + /// + /// Uses the minimum order's 'createDate' value to fetch all shipments and fulfillments created on or after that date, and updates the corresponding collections in the orders list. + /// + private async Task GetShipmentsAndFullfilmentsByCreationDate( IEnumerable< ShipStationOrder > orders, CancellationToken token ) + { + var ordersList = orders.ToList(); + var minOrderDate = ordersList.Min( o => o.CreateDate ); + var shipments = ( await this.GetOrderShipmentsByCreatedDateAsync( minOrderDate, token ).ConfigureAwait( false ) ).ToList(); + var fulfillments = ( await this.GetOrderFulfillmentsByCreatedDateAsync( minOrderDate, token ).ConfigureAwait( false ) ).ToList(); + + foreach( var order in ordersList ) + { + order.Shipments = shipments.Where( s => s.OrderId == order.OrderId ).ToList(); + order.Fulfillments = fulfillments.Where( f => f.OrderId == order.OrderId ).ToList(); + } + } + + /// + /// Makes two separate calls to ShipStation's API to obtain shipments and fulfillment data per each order and updates the corresponding collections in the orders list. + /// + private async Task GetShipmentsAndFullfilmentByOrderId( IEnumerable< ShipStationOrder > orders, CancellationToken token ) + { + foreach( var order in orders.ToList() ) { order.Shipments = await this.GetOrderShipmentsByIdAsync( order.OrderId.ToString(), token ).ConfigureAwait( false ); order.Fulfillments = await this.GetOrderFulfillmentsByIdAsync( order.OrderId.ToString(), token ).ConfigureAwait( false ); @@ -465,7 +497,7 @@ public async Task< IEnumerable< ShipStationOrderShipment > > GetOrderShipmentsBy do { var getOrderShipmentsEndpoint = ParamsBuilder.CreateOrderShipmentsParams( orderId ); - var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, RequestMaxLimit ) ); + var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, OrdersPageSize ) ); var orderShipmentsByPageEndPoint = getOrderShipmentsEndpoint.ConcatParams( nextPageParams ); ShipStationOrderShipments ordersShipmentsPage = null; @@ -478,7 +510,39 @@ await ActionPolicies.GetAsync.Do( async () => break; ++currentPage; - totalShipStationShipmentsPages = ordersShipmentsPage.Pages + 1; + totalShipStationShipmentsPages = ordersShipmentsPage.TotalPages + 1; + + orderShipments.AddRange( ordersShipmentsPage.Shipments ); + } + while( currentPage <= totalShipStationShipmentsPages ); + + return orderShipments; + } + + public async Task< IEnumerable< ShipStationOrderShipment > > GetOrderShipmentsByCreatedDateAsync( DateTime createdDate, CancellationToken token ) + { + var orderShipments = new List< ShipStationOrderShipment >(); + + var currentPage = 1; + int? totalShipStationShipmentsPages; + + do + { + var getOrderShipmentsEndpoint = ParamsBuilder.CreateOrderShipmentsParams( createdDate ); + var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, OrdersPageSize ) ); + var orderShipmentsByPageEndPoint = getOrderShipmentsEndpoint.ConcatParams( nextPageParams ); + + ShipStationOrderShipments ordersShipmentsPage = null; + await ActionPolicies.GetAsync.Do( async () => + { + ordersShipmentsPage = await this._webRequestServices.GetResponseAsync< ShipStationOrderShipments >( ShipStationCommand.GetOrderShipments, orderShipmentsByPageEndPoint, token, _timeouts[ ShipStationOperationEnum.GetOrderShipments ] ).ConfigureAwait( false ); + } ); + + if ( ordersShipmentsPage?.Shipments == null || !ordersShipmentsPage.Shipments.Any() ) + break; + + ++currentPage; + totalShipStationShipmentsPages = ordersShipmentsPage.TotalPages + 1; orderShipments.AddRange( ordersShipmentsPage.Shipments ); } @@ -497,7 +561,39 @@ public async Task< IEnumerable< ShipStationOrderFulfillment > > GetOrderFulfillm do { var getOrderFulfillmentsEndpoint = ParamsBuilder.CreateOrderFulfillmentsParams( orderId ); - var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, RequestMaxLimit ) ); + var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, OrdersPageSize ) ); + var orderFulfillmentsByPageEndPoint = getOrderFulfillmentsEndpoint.ConcatParams( nextPageParams ); + + ShipStationOrderFulfillments orderFulfillmentsPage = null; + await ActionPolicies.GetAsync.Do( async () => + { + orderFulfillmentsPage = await this._webRequestServices.GetResponseAsync< ShipStationOrderFulfillments >( ShipStationCommand.GetOrderFulfillments, orderFulfillmentsByPageEndPoint, token, _timeouts[ ShipStationOperationEnum.GetOrderFulfillments ] ).ConfigureAwait( false ); + } ); + + if ( orderFulfillmentsPage?.Fulfillments == null || !orderFulfillmentsPage.Fulfillments.Any() ) + break; + + ++currentPage; + totalShipStationFulfillmentsPages = orderFulfillmentsPage.TotalPages + 1; + + orderFulfillments.AddRange( orderFulfillmentsPage.Fulfillments ); + } + while( currentPage <= totalShipStationFulfillmentsPages ); + + return orderFulfillments; + } + + public async Task< IEnumerable< ShipStationOrderFulfillment > > GetOrderFulfillmentsByCreatedDateAsync( DateTime createdDate, CancellationToken token ) + { + var orderFulfillments = new List< ShipStationOrderFulfillment >(); + + var currentPage = 1; + int? totalShipStationFulfillmentsPages; + + do + { + var getOrderFulfillmentsEndpoint = ParamsBuilder.CreateOrderFulfillmentsParams( createdDate ); + var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, OrdersPageSize ) ); var orderFulfillmentsByPageEndPoint = getOrderFulfillmentsEndpoint.ConcatParams( nextPageParams ); ShipStationOrderFulfillments orderFulfillmentsPage = null; @@ -510,7 +606,7 @@ await ActionPolicies.GetAsync.Do( async () => break; ++currentPage; - totalShipStationFulfillmentsPages = orderFulfillmentsPage.Pages + 1; + totalShipStationFulfillmentsPages = orderFulfillmentsPage.TotalPages + 1; orderFulfillments.AddRange( orderFulfillmentsPage.Fulfillments ); } diff --git a/src/ShipStationAccessTests/Orders/OrderTests.cs b/src/ShipStationAccessTests/Orders/OrderTests.cs index abcefba..6d1005d 100644 --- a/src/ShipStationAccessTests/Orders/OrderTests.cs +++ b/src/ShipStationAccessTests/Orders/OrderTests.cs @@ -23,6 +23,8 @@ public class OrderTests : BaseTest { private const string TestOrderWithShipments = "564221696"; private const string TestOrderWithFulfillments = "576752152"; + private readonly DateTime TestOrderWithShipmentsCreatedDate = new DateTime( 2020, 6, 10 ); + private readonly DateTime TestOrderWithFulfillmentsCreatedDate = new DateTime( 2020, 7, 20 ); [Test] public void DeserializeOrderWithNullablePaymentDateTest() @@ -217,6 +219,24 @@ public async Task GetOrderFulfillmentsAsync() orderFulfillments.Count().Should().BeGreaterThan( 0 ); } + [ Test ] + [ Explicit ] + public async Task GetOrderShipmentsByCreatedDateAsync_ShouldReturnOrdersWithShipments() + { + var orderShipments = await this._shipStationService.GetOrderShipmentsByCreatedDateAsync( TestOrderWithShipmentsCreatedDate, CancellationToken.None ); + + orderShipments.Count().Should().BeGreaterThan( 0 ); + } + + [ Test ] + [ Explicit ] + public async Task GetOrderFulfillmentsByCreatedDateAsync_ShouldReturnOrdersWithFullfilments() + { + var orderFulfillments = await this._shipStationService.GetOrderFulfillmentsByCreatedDateAsync( TestOrderWithFulfillmentsCreatedDate, CancellationToken.None ); + + orderFulfillments.Count().Should().BeGreaterThan( 0 ); + } + [ Test ] public async Task GivenNotExistingOrderId_WhenGetOrderAsyncIsCalled_ThenNullResponseIsExpected() {