GUARD-3049 Improve Orders Sync (#13)

* GUARD-3049 Add new endpoints to obtain shipments and fullfilment data by creation date

* GUARD-3049 Extracted logic into its own methods, renamed variables, added comments.

* GUARD-3049 Minor changes

* GUARD-3049 Rename orders properties

* GUARD-3049 Update package version

.build.ps1

@@ -76,7 +76,7 @@ task NuGet Package, Version, {
 <package>
 	<metadata>
 		<id>$project_name</id>
-		<version>$Version</version>
+		<version>$Version-alpha</version>
 		<authors>Agile Harbor</authors>
 		<owners>Agile Harbor</owners>
 		<projectUrl>https://github.com/agileharbor/$project_name</projectUrl>

src/Global/GlobalAssemblyInfo.cs

@@ -22,4 +22,4 @@
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
 
-[ assembly : AssemblyVersion( "1.6.1" ) ]
+[ assembly : AssemblyVersion( "1.7.1" ) ]

src/ShipStationAccess/V2/IShipStationService.cs

@@ -21,7 +21,9 @@ public interface IShipStationService
 		ShipStationOrder GetOrderById( string orderId, CancellationToken token );
 		Task< ShipStationOrder > GetOrderByIdAsync( string orderId, CancellationToken token );
 		Task< IEnumerable< ShipStationOrderShipment > > GetOrderShipmentsByIdAsync( string orderId, CancellationToken token );
+		Task< IEnumerable< ShipStationOrderShipment > > GetOrderShipmentsByCreatedDateAsync( DateTime createdDate, CancellationToken token );
 		Task< IEnumerable< ShipStationOrderFulfillment > > GetOrderFulfillmentsByIdAsync( string orderId, CancellationToken token );
+		Task< IEnumerable< ShipStationOrderFulfillment > > GetOrderFulfillmentsByCreatedDateAsync( DateTime createdDate, CancellationToken token );
 
 		void UpdateOrder( ShipStationOrder order, CancellationToken token );
 		Task UpdateOrderAsync( ShipStationOrder order, CancellationToken token );

src/ShipStationAccess/V2/Models/Order/ShipStationOrderFulfillment.cs

@@ -11,13 +11,13 @@ [ DataMember( Name = "fulfillments" ) ]
 		public IEnumerable< ShipStationOrderFulfillment > Fulfillments { get; set; }
 
 		[ DataMember( Name = "total" ) ]
-		public int Total { get; set; }
+		public int TotalFulfillments { get; set; }
 
 		[ DataMember( Name = "page" ) ]
 		public int Page { get; set; }
 
 		[ DataMember( Name = "pages" ) ]
-		public int Pages { get; set; }
+		public int TotalPages { get; set; }
 	}
 
 	[ DataContract ]

src/ShipStationAccess/V2/Models/Order/ShipStationOrderShipment.cs

@@ -11,13 +11,13 @@ [ DataMember( Name = "shipments" ) ]
 		public IEnumerable< ShipStationOrderShipment > Shipments { get; set; }
 
 		[ DataMember( Name = "total" ) ]
-		public int Total { get; set; }
+		public int TotalShipments { get; set; }
 
 		[ DataMember( Name = "page" ) ]
 		public int Page { get; set; }
 
 		[ DataMember( Name = "pages" ) ]
-		public int Pages { get; set; }
+		public int TotalPages { get; set; }
 	}
 
 	[ DataContract ]

src/ShipStationAccess/V2/Services/ParamsBuilder.cs

@@ -40,11 +40,24 @@ public static string CreateOrderShipmentsParams( string orderId, bool includeShi
 			return endpoint;
 		}
 
+		public static string CreateOrderShipmentsParams( DateTime createdDate, bool includeShipmentItems = true )
+		{
+			var endpoint = string.Format( "?{0}={1}&{2}={3}",
+				ShipStationParam.OrdersCreatedDateFrom.Name, createdDate.ToString( "s" ),
+				ShipStationParam.IncludeShipmentItems.Name, includeShipmentItems );
+			return endpoint;
+		}
+
 		public static string CreateOrderFulfillmentsParams( string orderId )
 		{
 			return string.Format( "?{0}={1}", ShipStationParam.OrderId.Name, orderId );
 		}
 
+		public static string CreateOrderFulfillmentsParams( DateTime createdDate )
+		{
+			return string.Format( "?{0}={1}", ShipStationParam.OrdersCreatedDateFrom.Name, createdDate.ToString( "s" ) );
+		}
+
 		public static string CreateGetNextPageParams( ShipStationCommandConfig config )
 		{
 			var endpoint = string.Format( "?{0}={1}&{2}={3}",

src/ShipStationAccess/V2/ShipStationService.cs

@@ -25,9 +25,9 @@ public sealed class ShipStationService: IShipStationService
 		private readonly IWebRequestServices _webRequestServices;
 		private readonly SyncRunContext _syncRunContext;
 		private readonly ShipStationTimeouts _timeouts;
-
-		// lowered max limit for less order loss on Shipsation API's internal errors
-		private const int RequestMaxLimit = 20;
+		private const int OrdersPageSize = 500;
+		// Max orders that can be processed without needing complex shipping and fulfillment handling.
+		private const int OrderProcessingThreshold = 10;
 
 		internal ShipStationService( IWebRequestServices webRequestServices, SyncRunContext syncRunContext, ShipStationTimeouts timeouts )
 		{
@@ -122,7 +122,7 @@ public IEnumerable< ShipStationOrder > GetOrders( DateTime dateFrom, DateTime da
 
 				do
 				{
-					var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, RequestMaxLimit ) );
+					var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, OrdersPageSize ) );
 					var ordersEndPoint = endPoint.ConcatParams( nextPageParams );
 
 					ActionPolicies.Get.Do( () =>
@@ -202,7 +202,7 @@ public async Task< SummaryResponse< ShipStationOrder > > GetCreatedOrdersAsync(
 		{
 			var createdOrdersEndpoint = ParamsBuilder.CreateNewOrdersParams( dateFrom, dateTo );
 			var createdOrdersResponse = new SummaryResponse< ShipStationOrder >();
-			await this.DownloadOrdersAsync( createdOrdersResponse, createdOrdersEndpoint, 1, RequestMaxLimit, token ).ConfigureAwait( false );
+			await this.DownloadOrdersAsync( createdOrdersResponse, createdOrdersEndpoint, 1, OrdersPageSize, token ).ConfigureAwait( false );
 			if ( createdOrdersResponse.Data.Any() )
 			{
 				ShipStationLogger.Log.Info( Constants.LoggingCommonPrefix + "Created orders downloaded - {Orders}/{ExpectedOrders} orders from {Endpoint}. Response: '{Response}'",
@@ -223,7 +223,7 @@ public async Task< SummaryResponse< ShipStationOrder > > GetModifiedOrdersAsync(
 		{
 			var modifiedOrdersEndpoint = ParamsBuilder.CreateModifiedOrdersParams( dateFrom, dateTo );
 			var modifiedOrdersResponse = new SummaryResponse< ShipStationOrder >();
-			await this.DownloadOrdersAsync( modifiedOrdersResponse, modifiedOrdersEndpoint, 1, RequestMaxLimit, token ).ConfigureAwait( false );
+			await this.DownloadOrdersAsync( modifiedOrdersResponse, modifiedOrdersEndpoint, 1, OrdersPageSize, token ).ConfigureAwait( false );
 			if ( modifiedOrdersResponse.Data.Any() )
 			{
 				ShipStationLogger.Log.Info( Constants.LoggingCommonPrefix + "Modified orders downloaded - {Orders}/{ExpectedOrders} orders from {Endpoint}. Response: '{Response}'",
@@ -307,9 +307,9 @@ public async Task DownloadOrdersAsync( SummaryResponse< ShipStationOrder > summa
 				summary.Data.AddRange( ordersPage.Data );
 				currentPage++;
 
-				if ( currentPageSize != RequestMaxLimit )
+				if ( currentPageSize != OrdersPageSize )
 				{
-					var newPageSize = PageSizeAdjuster.DoublePageSize( currentPageSize, RequestMaxLimit );
+					var newPageSize = PageSizeAdjuster.DoublePageSize( currentPageSize, OrdersPageSize );
 					var newCurrentPage = PageSizeAdjuster.GetNextPageIndex( summary.TotalEntitiesHandled, newPageSize );
 
 					if ( !PageSizeAdjuster.AreEntitiesWillBeDownloadedAgainAfterChangingThePageSize( newCurrentPage, newPageSize, summary.TotalEntitiesHandled ) )
@@ -390,7 +390,7 @@ public IEnumerable< ShipStationOrder > GetOrders( string storeId, string orderNu
 
 				do
 				{
-					var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, RequestMaxLimit ) );
+					var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, OrdersPageSize ) );
 					var ordersEndPoint = endPoint.ConcatParams( nextPageParams );
 
 					ActionPolicies.Get.Do( () =>
@@ -448,7 +448,39 @@ await ActionPolicies.GetAsync.Do( async () =>
 
 		private async Task FindShipmentsAndFulfillments( IEnumerable< ShipStationOrder > orders, CancellationToken token )
 		{
-			foreach( var order in orders )
+			if( orders.ToList().Count > OrderProcessingThreshold )
+			{
+				await GetShipmentsAndFullfilmentsByCreationDate( orders, token ).ConfigureAwait( false );
+			}
+			else
+			{
+				await GetShipmentsAndFullfilmentByOrderId( orders, token ).ConfigureAwait( false );
+			}
+		}
+
+		/// <summary>
+		/// Uses the minimum order's 'createDate' value to fetch all shipments and fulfillments created on or after that date, and updates the corresponding collections in the orders list.
+		/// </summary>
+		private async Task GetShipmentsAndFullfilmentsByCreationDate( IEnumerable< ShipStationOrder > orders, CancellationToken token )
+		{
+			var ordersList = orders.ToList();
+			var minOrderDate = ordersList.Min( o => o.CreateDate );
+			var shipments = ( await this.GetOrderShipmentsByCreatedDateAsync( minOrderDate, token ).ConfigureAwait( false ) ).ToList();
+			var fulfillments = ( await this.GetOrderFulfillmentsByCreatedDateAsync( minOrderDate, token ).ConfigureAwait( false ) ).ToList();
+
+			foreach( var order in ordersList )
+			{
+				order.Shipments = shipments.Where( s => s.OrderId == order.OrderId ).ToList();
+				order.Fulfillments = fulfillments.Where( f => f.OrderId == order.OrderId ).ToList();
+			}
+		}
+
+		/// <summary>
+		/// Makes two separate calls to ShipStation's API to obtain shipments and fulfillment data per each order and updates the corresponding collections in the orders list.
+		/// </summary>
+		private async Task GetShipmentsAndFullfilmentByOrderId( IEnumerable< ShipStationOrder > orders, CancellationToken token )
+		{
+			foreach( var order in orders.ToList() )
 			{
 				order.Shipments = await this.GetOrderShipmentsByIdAsync( order.OrderId.ToString(), token ).ConfigureAwait( false );
 				order.Fulfillments = await this.GetOrderFulfillmentsByIdAsync( order.OrderId.ToString(), token ).ConfigureAwait( false );
@@ -465,7 +497,7 @@ public async Task< IEnumerable< ShipStationOrderShipment > > GetOrderShipmentsBy
 			do
 			{
 				var getOrderShipmentsEndpoint = ParamsBuilder.CreateOrderShipmentsParams( orderId );
-				var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, RequestMaxLimit ) );
+				var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, OrdersPageSize ) );
 				var orderShipmentsByPageEndPoint = getOrderShipmentsEndpoint.ConcatParams( nextPageParams );
 
 				ShipStationOrderShipments ordersShipmentsPage = null;
@@ -478,7 +510,39 @@ await ActionPolicies.GetAsync.Do( async () =>
 					break;
 
 				++currentPage;
-				totalShipStationShipmentsPages = ordersShipmentsPage.Pages + 1;
+				totalShipStationShipmentsPages = ordersShipmentsPage.TotalPages + 1;
+
+				orderShipments.AddRange( ordersShipmentsPage.Shipments );
+			}
+			while( currentPage <= totalShipStationShipmentsPages );
+
+			return orderShipments;
+		}
+
+		public async Task< IEnumerable< ShipStationOrderShipment > > GetOrderShipmentsByCreatedDateAsync( DateTime createdDate, CancellationToken token )
+		{
+			var orderShipments = new List< ShipStationOrderShipment >();
+
+			var currentPage = 1;
+			int? totalShipStationShipmentsPages;
+
+			do
+			{
+				var getOrderShipmentsEndpoint = ParamsBuilder.CreateOrderShipmentsParams( createdDate );
+				var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, OrdersPageSize ) );
+				var orderShipmentsByPageEndPoint = getOrderShipmentsEndpoint.ConcatParams( nextPageParams );
+
+				ShipStationOrderShipments ordersShipmentsPage = null;
+				await ActionPolicies.GetAsync.Do( async () =>
+				{
+					ordersShipmentsPage = await this._webRequestServices.GetResponseAsync< ShipStationOrderShipments >( ShipStationCommand.GetOrderShipments, orderShipmentsByPageEndPoint, token, _timeouts[ ShipStationOperationEnum.GetOrderShipments ] ).ConfigureAwait( false );
+				} );
+
+				if ( ordersShipmentsPage?.Shipments == null || !ordersShipmentsPage.Shipments.Any() )
+					break;
+
+				++currentPage;
+				totalShipStationShipmentsPages = ordersShipmentsPage.TotalPages + 1;
 
 				orderShipments.AddRange( ordersShipmentsPage.Shipments );
 			}
@@ -497,7 +561,39 @@ public async Task< IEnumerable< ShipStationOrderFulfillment > > GetOrderFulfillm
 			do
 			{
 				var getOrderFulfillmentsEndpoint = ParamsBuilder.CreateOrderFulfillmentsParams( orderId );
-				var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, RequestMaxLimit ) );
+				var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, OrdersPageSize ) );
+				var orderFulfillmentsByPageEndPoint = getOrderFulfillmentsEndpoint.ConcatParams( nextPageParams );
+
+				ShipStationOrderFulfillments orderFulfillmentsPage = null;
+				await ActionPolicies.GetAsync.Do( async () =>
+				{
+					orderFulfillmentsPage = await this._webRequestServices.GetResponseAsync< ShipStationOrderFulfillments >( ShipStationCommand.GetOrderFulfillments, orderFulfillmentsByPageEndPoint, token, _timeouts[ ShipStationOperationEnum.GetOrderFulfillments ] ).ConfigureAwait( false );
+				} );
+
+				if ( orderFulfillmentsPage?.Fulfillments == null || !orderFulfillmentsPage.Fulfillments.Any() )
+					break;
+
+				++currentPage;
+				totalShipStationFulfillmentsPages = orderFulfillmentsPage.TotalPages + 1;
+
+				orderFulfillments.AddRange( orderFulfillmentsPage.Fulfillments );
+			}
+			while( currentPage <= totalShipStationFulfillmentsPages );
+
+			return orderFulfillments;
+		}
+
+		public async Task< IEnumerable< ShipStationOrderFulfillment > > GetOrderFulfillmentsByCreatedDateAsync( DateTime createdDate, CancellationToken token )
+		{
+			var orderFulfillments = new List< ShipStationOrderFulfillment >();
+
+			var currentPage = 1;
+			int? totalShipStationFulfillmentsPages;
+
+			do
+			{
+				var getOrderFulfillmentsEndpoint = ParamsBuilder.CreateOrderFulfillmentsParams( createdDate );
+				var nextPageParams = ParamsBuilder.CreateGetNextPageParams( new ShipStationCommandConfig( currentPage, OrdersPageSize ) );
 				var orderFulfillmentsByPageEndPoint = getOrderFulfillmentsEndpoint.ConcatParams( nextPageParams );
 
 				ShipStationOrderFulfillments orderFulfillmentsPage = null;
@@ -510,7 +606,7 @@ await ActionPolicies.GetAsync.Do( async () =>
 					break;
 
 				++currentPage;
-				totalShipStationFulfillmentsPages = orderFulfillmentsPage.Pages + 1;
+				totalShipStationFulfillmentsPages = orderFulfillmentsPage.TotalPages + 1;
 
 				orderFulfillments.AddRange( orderFulfillmentsPage.Fulfillments );
 			}

src/ShipStationAccessTests/Orders/OrderTests.cs

@@ -23,6 +23,8 @@ public class OrderTests : BaseTest
 	{
 		private const string TestOrderWithShipments = "564221696";
 		private const string TestOrderWithFulfillments = "576752152";
+		private readonly DateTime TestOrderWithShipmentsCreatedDate = new DateTime( 2020, 6, 10 );
+		private readonly DateTime TestOrderWithFulfillmentsCreatedDate = new DateTime( 2020, 7, 20 );
 
 		[Test]
 		public void DeserializeOrderWithNullablePaymentDateTest()
@@ -217,6 +219,24 @@ public async Task GetOrderFulfillmentsAsync()
 			orderFulfillments.Count().Should().BeGreaterThan( 0 );
 		}
 
+		[ Test ]
+		[ Explicit ]
+		public async Task GetOrderShipmentsByCreatedDateAsync_ShouldReturnOrdersWithShipments()
+		{
+			var orderShipments = await this._shipStationService.GetOrderShipmentsByCreatedDateAsync( TestOrderWithShipmentsCreatedDate, CancellationToken.None );
+
+			orderShipments.Count().Should().BeGreaterThan( 0 );
+		}
+
+		[ Test ]
+		[ Explicit ]
+		public async Task GetOrderFulfillmentsByCreatedDateAsync_ShouldReturnOrdersWithFullfilments()
+		{
+			var orderFulfillments = await this._shipStationService.GetOrderFulfillmentsByCreatedDateAsync( TestOrderWithFulfillmentsCreatedDate, CancellationToken.None );
+
+			orderFulfillments.Count().Should().BeGreaterThan( 0 );
+		}
+
 		[ Test ]
 		public async Task GivenNotExistingOrderId_WhenGetOrderAsyncIsCalled_ThenNullResponseIsExpected()
 		{