From 7b09072d1518c0f5f4d6bb308fd52e690cfe3507 Mon Sep 17 00:00:00 2001 From: Anis DAIMAR Date: Wed, 20 Jan 2021 15:52:59 +0100 Subject: [PATCH 1/3] Fix compiler.py:join_aliases; was causing KeyError when select_related was realized on empty field (null data on field) --- rest_models/backend/compiler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rest_models/backend/compiler.py b/rest_models/backend/compiler.py index 84b9d80..1cc5264 100644 --- a/rest_models/backend/compiler.py +++ b/rest_models/backend/compiler.py @@ -496,7 +496,10 @@ def join_aliases(aliases, responsereader, existing_aliases): # resolve the values of next jump val_for_model = responsereader[alias.model] current_data = existing_aliases[alias.parent] - val = current_data[alias.field.concrete and alias.field.db_column or alias.attrname] + val = current_data.get(alias.field.concrete and alias.field.db_column or alias.attrname, None) + if not val: + yield existing_aliases + return if not isinstance(val, list): val = [val] for pk in val: From 6fe098768f5c6a0d6432e0b02f026e6dd3dd289c Mon Sep 17 00:00:00 2001 From: Anis DAIMAR Date: Wed, 20 Jan 2021 17:28:01 +0100 Subject: [PATCH 2/3] Revert "Fix compiler.py:join_aliases; was causing KeyError when select_related was realized on empty field (null data on field)" This reverts commit 7b09072d1518c0f5f4d6bb308fd52e690cfe3507. --- rest_models/backend/compiler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/rest_models/backend/compiler.py b/rest_models/backend/compiler.py index 1cc5264..84b9d80 100644 --- a/rest_models/backend/compiler.py +++ b/rest_models/backend/compiler.py @@ -496,10 +496,7 @@ def join_aliases(aliases, responsereader, existing_aliases): # resolve the values of next jump val_for_model = responsereader[alias.model] current_data = existing_aliases[alias.parent] - val = current_data.get(alias.field.concrete and alias.field.db_column or alias.attrname, None) - if not val: - yield existing_aliases - return + val = current_data[alias.field.concrete and alias.field.db_column or alias.attrname] if not isinstance(val, list): val = [val] for pk in val: From 5e54a2d57e3d19634991f5e53e8a6e683e5f8e39 Mon Sep 17 00:00:00 2001 From: Anis DAIMAR Date: Thu, 2 Dec 2021 18:18:45 +0100 Subject: [PATCH 3/3] compiler: log and retry on json decode error --- rest_models/backend/compiler.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/rest_models/backend/compiler.py b/rest_models/backend/compiler.py index 84b9d80..5054b05 100644 --- a/rest_models/backend/compiler.py +++ b/rest_models/backend/compiler.py @@ -6,6 +6,7 @@ import logging import re from collections import defaultdict, namedtuple +from json import JSONDecodeError import six from django.core.exceptions import ImproperlyConfigured @@ -990,13 +991,15 @@ def execute_sql(self, result_type=MULTI, chunked_fetch=False, chunk_size=None): pk, params = self.build_params_and_pk() url = get_resource_path(self.query.model, pk) - response = self.connection.cursor().get( - url, - params=params - ) - self.raise_on_response(url, params, response) + response = self.make_request(params, url) + + try: + json = response.json() + except JSONDecodeError: + extra = {'params': params, 'response': response} + logger.error('json decode error while calling {}; retrying'.format(url), extra=extra) + json = self.make_request(params, url).json() - json = response.json() meta = self.get_meta(json, response) if meta: # pagination and others thing @@ -1038,6 +1041,14 @@ def next_from_query(): result = self.result_iter(response_reader) return result + def make_request(self, params, url): + response = self.connection.cursor().get( + url, + params=params + ) + self.raise_on_response(url, params, response) + return response + class SQLInsertCompiler(SQLCompiler): def resolve_data_n_files(self, obj):