From aaa8e9ecf84fd52b20e7c986266e6011be8547f2 Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Mon, 17 Aug 2020 22:27:11 +0300 Subject: [PATCH 1/8] expire schemas after 7 days --- redash/models/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/redash/models/__init__.py b/redash/models/__init__.py index 1d5528aaca..337853b5fb 100644 --- a/redash/models/__init__.py +++ b/redash/models/__init__.py @@ -204,7 +204,8 @@ def get_schema(self, refresh=False): ) out_schema = schema finally: - redis_connection.set(self._schema_key, json_dumps(out_schema)) + ttl = int(datetime.timedelta(days=7).total_seconds()) + redis_connection.set(self._schema_key, json_dumps(out_schema), ex=ttl) return out_schema From 3eb55b7d00d57c9df249c8d73420da70a17fc907 Mon Sep 17 00:00:00 2001 From: Omer Lachish Date: Mon, 17 Aug 2020 22:33:52 +0300 Subject: [PATCH 2/8] expire schemas 7 days after the last refresh_schemas scheduled time --- redash/models/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/redash/models/__init__.py b/redash/models/__init__.py index 337853b5fb..9203e42689 100644 --- a/redash/models/__init__.py +++ b/redash/models/__init__.py @@ -204,7 +204,11 @@ def get_schema(self, refresh=False): ) out_schema = schema finally: - ttl = int(datetime.timedelta(days=7).total_seconds()) + ttl = int( + datetime.timedelta( + minutes=settings.SCHEMAS_REFRESH_SCHEDULE, days=7 + ).total_seconds() + ) redis_connection.set(self._schema_key, json_dumps(out_schema), ex=ttl) return out_schema From 0e43e12eab8bd44d5f6d83096febdfe2b54330bb Mon Sep 17 00:00:00 2001 From: konnectr <1konnectrl@gmail.com> Date: Sun, 30 Jul 2023 21:05:55 +0500 Subject: [PATCH 3/8] format files --- redash/models/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/redash/models/__init__.py b/redash/models/__init__.py index ddcdcba3cd..8ddebf261e 100644 --- a/redash/models/__init__.py +++ b/redash/models/__init__.py @@ -214,11 +214,7 @@ def get_schema(self, refresh=False): logging.exception("Error sorting schema columns for data_source {}".format(self.id)) out_schema = schema finally: - ttl = int( - datetime.timedelta( - minutes=settings.SCHEMAS_REFRESH_SCHEDULE, days=7 - ).total_seconds() - ) + ttl = int(datetime.timedelta(minutes=settings.SCHEMAS_REFRESH_SCHEDULE, days=7).total_seconds()) redis_connection.set(self._schema_key, json_dumps(out_schema), ex=ttl) return out_schema From d2c41bc411d24236981c160cd4126437f9bcccbe Mon Sep 17 00:00:00 2001 From: Guido Petri <18634426+guidopetri@users.noreply.github.com> Date: Thu, 31 Aug 2023 07:53:56 -0500 Subject: [PATCH 4/8] add expire schema test --- tests/models/test_data_sources.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/models/test_data_sources.py b/tests/models/test_data_sources.py index dc53444cdd..7b5c848734 100644 --- a/tests/models/test_data_sources.py +++ b/tests/models/test_data_sources.py @@ -86,6 +86,14 @@ def test_model_uses_schema_sorter(self): self.assertEqual(out_schema, sorted_schema) + @patch("redash.redis_connection.set") + def test_expires_schema(self, mock_redis): + # default of 30min + 7 days + expected_ttl = 606600 + + data_source = self.factory.data_source.get_schema(refresh=True) + mock_redis.assert_called_with(data_source._schema_key, ex=expected_ttl) + class TestDataSourceCreate(BaseTestCase): def test_adds_data_source_to_default_group(self): From 126e7e524cff60d10b2092c2332642a9b6ad83a3 Mon Sep 17 00:00:00 2001 From: Guido Petri <18634426+guidopetri@users.noreply.github.com> Date: Thu, 31 Aug 2023 08:12:08 -0500 Subject: [PATCH 5/8] patch schema away --- tests/models/test_data_sources.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/models/test_data_sources.py b/tests/models/test_data_sources.py index 7b5c848734..ac3c9dd35f 100644 --- a/tests/models/test_data_sources.py +++ b/tests/models/test_data_sources.py @@ -91,7 +91,9 @@ def test_expires_schema(self, mock_redis): # default of 30min + 7 days expected_ttl = 606600 - data_source = self.factory.data_source.get_schema(refresh=True) + with mock.patch("redash.query_runner.pg.PostgreSQL.get_schema") as patched_get_schema: + data_source = self.factory.data_source.get_schema(refresh=True) + mock_redis.assert_called_with(data_source._schema_key, ex=expected_ttl) From d600a4bcc6cc04b9a071f58d3f0748460b3b539d Mon Sep 17 00:00:00 2001 From: Guido Petri <18634426+guidopetri@users.noreply.github.com> Date: Thu, 31 Aug 2023 08:14:10 -0500 Subject: [PATCH 6/8] return nothing on schema get --- tests/models/test_data_sources.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/models/test_data_sources.py b/tests/models/test_data_sources.py index ac3c9dd35f..45a8ae4d5a 100644 --- a/tests/models/test_data_sources.py +++ b/tests/models/test_data_sources.py @@ -92,6 +92,7 @@ def test_expires_schema(self, mock_redis): expected_ttl = 606600 with mock.patch("redash.query_runner.pg.PostgreSQL.get_schema") as patched_get_schema: + patched_get_schema.return_value = None data_source = self.factory.data_source.get_schema(refresh=True) mock_redis.assert_called_with(data_source._schema_key, ex=expected_ttl) From bfacbf46017329badf7892dba9d4c985c96168e2 Mon Sep 17 00:00:00 2001 From: Guido Petri <18634426+guidopetri@users.noreply.github.com> Date: Thu, 31 Aug 2023 19:09:59 -0500 Subject: [PATCH 7/8] fix redis key name --- tests/models/test_data_sources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models/test_data_sources.py b/tests/models/test_data_sources.py index 45a8ae4d5a..7e2932c0e8 100644 --- a/tests/models/test_data_sources.py +++ b/tests/models/test_data_sources.py @@ -95,7 +95,7 @@ def test_expires_schema(self, mock_redis): patched_get_schema.return_value = None data_source = self.factory.data_source.get_schema(refresh=True) - mock_redis.assert_called_with(data_source._schema_key, ex=expected_ttl) + mock_redis.assert_called_with("data_source:schema:1", ex=expected_ttl) class TestDataSourceCreate(BaseTestCase): From a51d6d49c3eb79d219aedb2d249a0063c39ba09c Mon Sep 17 00:00:00 2001 From: Guido Petri <18634426+guidopetri@users.noreply.github.com> Date: Thu, 31 Aug 2023 19:40:45 -0500 Subject: [PATCH 8/8] finally run tests locally and fix tests --- tests/models/test_data_sources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/models/test_data_sources.py b/tests/models/test_data_sources.py index 7e2932c0e8..7db4d463a3 100644 --- a/tests/models/test_data_sources.py +++ b/tests/models/test_data_sources.py @@ -93,9 +93,9 @@ def test_expires_schema(self, mock_redis): with mock.patch("redash.query_runner.pg.PostgreSQL.get_schema") as patched_get_schema: patched_get_schema.return_value = None - data_source = self.factory.data_source.get_schema(refresh=True) + self.factory.data_source.get_schema(refresh=True) - mock_redis.assert_called_with("data_source:schema:1", ex=expected_ttl) + mock_redis.assert_called_with("data_source:schema:1", "null", ex=expected_ttl) class TestDataSourceCreate(BaseTestCase):