From a6adb0c37005275f3cbe5421861c607565295db3 Mon Sep 17 00:00:00 2001 From: Erik Parmann Date: Wed, 17 Apr 2019 12:50:35 +0200 Subject: [PATCH] Make client batched POST forward correct index Previosly it forwarded the last indexes of the prediction for all batches, resulting in loss of most predictions. --- gordo_components/client/client.py | 6 ++++- tests/gordo_components/client/test_client.py | 24 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/gordo_components/client/client.py b/gordo_components/client/client.py index d6319e088..63d3952dd 100644 --- a/gordo_components/client/client.py +++ b/gordo_components/client/client.py @@ -302,12 +302,16 @@ async def _process_post_prediction_task( # Get the output values values = np.array(resp["output"]) + # Chunks can have None as end-point + chunk_stop = chunk.stop if chunk.stop else len(X) + # Chunks can also be larger than the actual data + chunk_stop = min(chunk_stop, len(X)) predictions = pd.DataFrame( data=values, columns=[f"input_{sensor}" for sensor in X.columns] + [f"output_{sensor}" for sensor in X.columns], # match any offsetting from windowed models - index=X.index[-len(values) :], + index=X.index[chunk_stop - len(values) : chunk_stop], ) # Forward predictions to any other consumer if registered. diff --git a/tests/gordo_components/client/test_client.py b/tests/gordo_components/client/test_client.py index 178edb1f9..c48fdfb30 100644 --- a/tests/gordo_components/client/test_client.py +++ b/tests/gordo_components/client/test_client.py @@ -111,6 +111,29 @@ def test_client_predictions_with_or_without_data_provider( """ Run the prediction client with or without a data provider """ + use_client_predictions(trained_model_directory, use_data_provider, batch_size=1000) + + +@pytest.mark.dockertest +@pytest.mark.parametrize("batch_size", (10, 100)) +@pytest.mark.parametrize("trained_model_directory", (SENSORS,), indirect=True) +def test_client_predictions_different_batch_sizes( + trained_model_directory: pytest.fixture, batch_size: int +): + """ + Run the prediction client with different batch-sizes + """ + use_client_predictions( + trained_model_directory, use_data_provider=True, batch_size=batch_size + ) + + +def use_client_predictions( + trained_model_directory: pytest.fixture, use_data_provider: bool, batch_size: int +): + """ + Run the prediction client with or without a data provider + """ with watchman( host="localhost", @@ -165,6 +188,7 @@ def test_client_predictions_with_or_without_data_provider( prediction_forwarder=ForwardPredictionsIntoInflux( destination_influx_uri=uri ), + batch_size=batch_size, ) # Should have discovered machine-1 & machine-2