From 7ecd62922399dbd7173b8def35e3283075dec3e0 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 7 Oct 2024 10:26:46 +0300 Subject: [PATCH 1/4] Fix: metrics wasn't handling non ORM join correctly --- redash/metrics/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/redash/metrics/database.py b/redash/metrics/database.py index 152427b2e5..6f16bdbc6d 100644 --- a/redash/metrics/database.py +++ b/redash/metrics/database.py @@ -5,7 +5,7 @@ from sqlalchemy.engine import Engine from sqlalchemy.event import listens_for from sqlalchemy.orm.util import _ORMJoin -from sqlalchemy.sql.selectable import Alias +from sqlalchemy.sql.selectable import Alias, Join from redash import statsd_client @@ -18,7 +18,7 @@ def _table_name_from_select_element(elt): if isinstance(t, Alias): t = t.original.froms[0] - while isinstance(t, _ORMJoin): + while isinstance(t, _ORMJoin) or isinstance(t, Join): t = t.left return t.name From 4633fe2010ad6579508597271bbf9219c25ecc69 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 7 Oct 2024 10:27:01 +0300 Subject: [PATCH 2/4] Rehash queries correctly --- migrations/versions/9e8c841d1a30_fix_hash.py | 64 ++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 migrations/versions/9e8c841d1a30_fix_hash.py diff --git a/migrations/versions/9e8c841d1a30_fix_hash.py b/migrations/versions/9e8c841d1a30_fix_hash.py new file mode 100644 index 0000000000..052d03009d --- /dev/null +++ b/migrations/versions/9e8c841d1a30_fix_hash.py @@ -0,0 +1,64 @@ +"""fix_hash + +Revision ID: 9e8c841d1a30 +Revises: 7205816877ec +Create Date: 2024-10-05 18:55:35.730573 + +""" +import logging +from alembic import op +import sqlalchemy as sa +from sqlalchemy.sql import table +from sqlalchemy import select + +from redash.query_runner import BaseQueryRunner, get_query_runner + + +# revision identifiers, used by Alembic. +revision = '9e8c841d1a30' +down_revision = '7205816877ec' +branch_labels = None +depends_on = None + + +def update_query_hash(record): + should_apply_auto_limit = record['options'].get("apply_auto_limit", False) if record['options'] else False + query_runner = get_query_runner(record['type'], {}) if record['type'] else BaseQueryRunner({}) + query_text = record['query'] + + parameters_dict = {p["name"]: p.get("value") for p in record['options'].get('parameters', [])} if record.options else {} + if any(parameters_dict): + print(f"Query {record.id} has parameters. Hash might be incorrect.") + + return query_runner.gen_query_hash(query_text, should_apply_auto_limit) + + +def upgrade(): + conn = op.get_bind() + + metadata = sa.MetaData(bind=conn) + queries = sa.Table("queries", metadata, autoload=True) + data_sources = sa.Table("data_sources", metadata, autoload=True) + + joined_table = queries.outerjoin(data_sources, queries.c.data_source_id == data_sources.c.id) + + query = select([ + queries.c.id.label("query_id"), + queries.c.query, + queries.c.query_hash, + queries.c.options, + data_sources.c.id.label("data_source_id"), + data_sources.c.type + ]).select_from(joined_table) + + for record in conn.execute(query): + new_hash = update_query_hash(record) + print(f"Updating hash for query {record['query_id']} from {record['query_hash']} to {new_hash}") + conn.execute( + queries.update() + .where(queries.c.id == record['query_id']) + .values(query_hash=new_hash)) + + +def downgrade(): + pass \ No newline at end of file From baaf3c5acafb8d2ac8eafea24a38e67adcce642d Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 7 Oct 2024 10:37:07 +0300 Subject: [PATCH 3/4] add manage.py queries rehash cli command to rehash queries --- redash/cli/queries.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/redash/cli/queries.py b/redash/cli/queries.py index 54943dee3c..dddb35724e 100644 --- a/redash/cli/queries.py +++ b/redash/cli/queries.py @@ -5,6 +5,22 @@ manager = AppGroup(help="Queries management commands.") +@manager.command(name="rehash") +def rehash(): + from redash import models + + for q in models.Query.query.all(): + old_hash = q.query_hash + q.update_query_hash() + new_hash = q.query_hash + + if old_hash != new_hash: + print(f"Query {q.id} has changed hash from {old_hash} to {new_hash}") + models.db.session.add(q) + + models.db.session.commit() + + @manager.command(name="add_tag") @argument("query_id") @argument("tag") From a5ae9504ab378d037fd1fd298a015225981fe7fe Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Sat, 19 Oct 2024 22:26:20 +0300 Subject: [PATCH 4/4] Update migrations/versions/9e8c841d1a30_fix_hash.py --- migrations/versions/9e8c841d1a30_fix_hash.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/versions/9e8c841d1a30_fix_hash.py b/migrations/versions/9e8c841d1a30_fix_hash.py index 052d03009d..966d393391 100644 --- a/migrations/versions/9e8c841d1a30_fix_hash.py +++ b/migrations/versions/9e8c841d1a30_fix_hash.py @@ -28,7 +28,7 @@ def update_query_hash(record): parameters_dict = {p["name"]: p.get("value") for p in record['options'].get('parameters', [])} if record.options else {} if any(parameters_dict): - print(f"Query {record.id} has parameters. Hash might be incorrect.") + print(f"Query {record['query_id']} has parameters. Hash might be incorrect.") return query_runner.gen_query_hash(query_text, should_apply_auto_limit)