diff --git a/.github/workflows/restyled.yml b/.github/workflows/restyled.yml new file mode 100644 index 0000000000..3482740947 --- /dev/null +++ b/.github/workflows/restyled.yml @@ -0,0 +1,36 @@ +name: Restyled + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + restyled: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - uses: restyled-io/actions/setup@v4 + - id: restyler + uses: restyled-io/actions/run@v4 + with: + fail-on-differences: true + + - if: | + !cancelled() && + steps.restyler.outputs.success == 'true' && + github.event.pull_request.head.repo.full_name == github.repository + uses: peter-evans/create-pull-request@v6 + with: + base: ${{ steps.restyler.outputs.restyled-base }} + branch: ${{ steps.restyler.outputs.restyled-head }} + title: ${{ steps.restyler.outputs.restyled-title }} + body: ${{ steps.restyler.outputs.restyled-body }} + labels: "restyled" + reviewers: ${{ github.event.pull_request.user.login }} + delete-branch: true diff --git a/migrations/versions/9e8c841d1a30_fix_hash.py b/migrations/versions/9e8c841d1a30_fix_hash.py new file mode 100644 index 0000000000..966d393391 --- /dev/null +++ b/migrations/versions/9e8c841d1a30_fix_hash.py @@ -0,0 +1,64 @@ +"""fix_hash + +Revision ID: 9e8c841d1a30 +Revises: 7205816877ec +Create Date: 2024-10-05 18:55:35.730573 + +""" +import logging +from alembic import op +import sqlalchemy as sa +from sqlalchemy.sql import table +from sqlalchemy import select + +from redash.query_runner import BaseQueryRunner, get_query_runner + + +# revision identifiers, used by Alembic. +revision = '9e8c841d1a30' +down_revision = '7205816877ec' +branch_labels = None +depends_on = None + + +def update_query_hash(record): + should_apply_auto_limit = record['options'].get("apply_auto_limit", False) if record['options'] else False + query_runner = get_query_runner(record['type'], {}) if record['type'] else BaseQueryRunner({}) + query_text = record['query'] + + parameters_dict = {p["name"]: p.get("value") for p in record['options'].get('parameters', [])} if record.options else {} + if any(parameters_dict): + print(f"Query {record['query_id']} has parameters. Hash might be incorrect.") + + return query_runner.gen_query_hash(query_text, should_apply_auto_limit) + + +def upgrade(): + conn = op.get_bind() + + metadata = sa.MetaData(bind=conn) + queries = sa.Table("queries", metadata, autoload=True) + data_sources = sa.Table("data_sources", metadata, autoload=True) + + joined_table = queries.outerjoin(data_sources, queries.c.data_source_id == data_sources.c.id) + + query = select([ + queries.c.id.label("query_id"), + queries.c.query, + queries.c.query_hash, + queries.c.options, + data_sources.c.id.label("data_source_id"), + data_sources.c.type + ]).select_from(joined_table) + + for record in conn.execute(query): + new_hash = update_query_hash(record) + print(f"Updating hash for query {record['query_id']} from {record['query_hash']} to {new_hash}") + conn.execute( + queries.update() + .where(queries.c.id == record['query_id']) + .values(query_hash=new_hash)) + + +def downgrade(): + pass \ No newline at end of file diff --git a/package.json b/package.json index d68aaeb5ab..fb058a0339 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "redash-client", - "version": "24.09.0-dev", + "version": "24.10.0-dev", "description": "The frontend part of Redash.", "main": "index.js", "scripts": { diff --git a/poetry.lock b/poetry.lock index 3de21181ad..b44c3257f8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -891,43 +891,38 @@ files = [ [[package]] name = "cryptography" -version = "42.0.8" +version = "43.0.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e"}, - {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949"}, - {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b"}, - {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7"}, - {file = "cryptography-42.0.8-cp37-abi3-win32.whl", hash = "sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2"}, - {file = "cryptography-42.0.8-cp37-abi3-win_amd64.whl", hash = "sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba"}, - {file = "cryptography-42.0.8-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c"}, - {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1"}, - {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14"}, - {file = "cryptography-42.0.8-cp39-abi3-win32.whl", hash = "sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c"}, - {file = "cryptography-42.0.8-cp39-abi3-win_amd64.whl", hash = "sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71"}, - {file = "cryptography-42.0.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648"}, - {file = "cryptography-42.0.8-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad"}, - {file = "cryptography-42.0.8.tar.gz", hash = "sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2"}, + {file = "cryptography-43.0.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a"}, + {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042"}, + {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494"}, + {file = "cryptography-43.0.1-cp37-abi3-win32.whl", hash = "sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2"}, + {file = "cryptography-43.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d"}, + {file = "cryptography-43.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1"}, + {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa"}, + {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4"}, + {file = "cryptography-43.0.1-cp39-abi3-win32.whl", hash = "sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47"}, + {file = "cryptography-43.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ea25acb556320250756e53f9e20a4177515f012c9eaea17eb7587a8c4d8ae034"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c1332724be35d23a854994ff0b66530119500b6053d0bd3363265f7e5e77288d"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:fba1007b3ef89946dbbb515aeeb41e30203b004f0b4b00e5e16078b518563289"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5b43d1ea6b378b54a1dc99dd8a2b5be47658fe9a7ce0a58ff0b55f4b43ef2b84"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:88cce104c36870d70c49c7c8fd22885875d950d9ee6ab54df2745f83ba0dc365"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:9d3cdb25fa98afdd3d0892d132b8d7139e2c087da1712041f6b762e4f807cc96"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e710bf40870f4db63c3d7d929aa9e09e4e7ee219e703f949ec4073b4294f6172"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7c05650fe8023c5ed0d46793d4b7d7e6cd9c04e68eabe5b0aeea836e37bdcec2"}, + {file = "cryptography-43.0.1.tar.gz", hash = "sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d"}, ] [package.dependencies] @@ -940,7 +935,7 @@ nox = ["nox"] pep8test = ["check-sdist", "click", "mypy", "ruff"] sdist = ["build"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test = ["certifi", "cryptography-vectors (==43.0.1)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] test-randomorder = ["pytest-randomly"] [[package]] @@ -4228,13 +4223,13 @@ requests = ">=2.0.1,<3.0.0" [[package]] name = "restrictedpython" -version = "6.2" +version = "7.3" description = "RestrictedPython is a defined subset of the Python language which allows to provide a program input into a trusted environment." optional = false -python-versions = ">=3.6, <3.12" +python-versions = "<3.13,>=3.7" files = [ - {file = "RestrictedPython-6.2-py3-none-any.whl", hash = "sha256:7c2ffa4904300d67732f841d8a975dcdc53eba4c1cdc9d84b97684ef12304a3d"}, - {file = "RestrictedPython-6.2.tar.gz", hash = "sha256:db73eb7e3b39650f0d21d10cc8dda9c0e2986e621c94b0c5de32fb0dee3a08af"}, + {file = "RestrictedPython-7.3-py3-none-any.whl", hash = "sha256:40a6170bbcfc48b32962831d9281a61608c8e56e7c02fd8e2397225f516a6ed4"}, + {file = "RestrictedPython-7.3.tar.gz", hash = "sha256:8888304c7858fdcfd86c50b58561797375ba40319d2b6ffb5d24b08b6a2dcd61"}, ] [package.extras] @@ -4607,37 +4602,37 @@ files = [ [[package]] name = "snowflake-connector-python" -version = "3.12.0" +version = "3.12.3" description = "Snowflake Connector for Python" optional = false python-versions = ">=3.8" files = [ - {file = "snowflake_connector_python-3.12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:edf28df8be24845cfcec653b160d2b8c048d5cb0c85b051f4957f0b0aae1e493"}, - {file = "snowflake_connector_python-3.12.0-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:c2bbdbbb028d7d542815ed68b28200728aa6707b9354e3a447fdc8c7a34bcdce"}, - {file = "snowflake_connector_python-3.12.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92c9a19a23033df709e63baa6ccdf6eff65210143a8c9c67a0a24bba862034b"}, - {file = "snowflake_connector_python-3.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d33d845e4c68d33e73a9f64100b53342c18607ac25c4f2a27dbed2078078d12"}, - {file = "snowflake_connector_python-3.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:c1d43bfaa885aab712f14f9ced232abe5023adfca7fbf7a7a0768a162523e9d6"}, - {file = "snowflake_connector_python-3.12.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6a0cc03fb44808f3ddc464ee272f141564c8daea14475e1df5c2a54c7acb2ddf"}, - {file = "snowflake_connector_python-3.12.0-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:564752d22accc43351b50f676b03aa9f2b441be2641e3cf9a7790faf54eff210"}, - {file = "snowflake_connector_python-3.12.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27d6a1a180832c7b551d38df1094a70fb79917f90c57893b9ce7e219362f6c1"}, - {file = "snowflake_connector_python-3.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60675fd83022daef40541d717d006695149c512b283e35741b61a4f48ba537e9"}, - {file = "snowflake_connector_python-3.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:a567b937b0179d1e95a8ad7200943d286f38d0e76df90af10f747ed9149dd681"}, - {file = "snowflake_connector_python-3.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:dc333fcfc383a8cab8bd7e890a7c76703e26598925a05954c75d2c50bff06071"}, - {file = "snowflake_connector_python-3.12.0-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:3c06bfba4a329fd4ec3feba0ada7b31f86ed4e156a9766bced52c2814d001fd2"}, - {file = "snowflake_connector_python-3.12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acf84b07dd2f22adfaa7d52ccd6be1722bd5a0e2b1a9b08681c3851bea05768f"}, - {file = "snowflake_connector_python-3.12.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:019b8a61e5af689451d502df2af8793fc6f20b5b0a3548fd8ad03aa8b62e7f2d"}, - {file = "snowflake_connector_python-3.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:45f9b9678694f10571c1f7ec7d0d741663ad0ff61a71ae53aa71be47faa19978"}, - {file = "snowflake_connector_python-3.12.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:21cbaef51fbed719de01155079df3d004cee963d3723c1ebdb8980923f893e04"}, - {file = "snowflake_connector_python-3.12.0-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:c86d4a7d49f42ea0bb34218cb49c401ba995892abcfb509ea749cd0a74a8b28a"}, - {file = "snowflake_connector_python-3.12.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1aa34aec0f96d7fc7271e38c68ee0d58529875d05e084afb4fc8f09b694643c4"}, - {file = "snowflake_connector_python-3.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c2f621030b26a220711c64518e00059736b79c1da53afa6a8ce68b31c1941014"}, - {file = "snowflake_connector_python-3.12.0-cp38-cp38-win_amd64.whl", hash = "sha256:368e46f1d079056e028bfe8f7171fabef62eb00bcf590df294220b7a5be5d56c"}, - {file = "snowflake_connector_python-3.12.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2735e16fffded0900f7484030613b79699afc1ed4e5cff086bd139a0ce965594"}, - {file = "snowflake_connector_python-3.12.0-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:c06a8e2e12284b4a4d462d0073fb4983e90ad2d6a2382926f9e3409f06c81d0b"}, - {file = "snowflake_connector_python-3.12.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:880e6e95171cd7374a86da14132fdfc4b622665f134561f4d43e3f35bdacf67d"}, - {file = "snowflake_connector_python-3.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e245b84c164433454ce49d78e6bcf5c2e62e25657358bf34ab533166e588f80"}, - {file = "snowflake_connector_python-3.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:85a5565b8813d164f33f32a825a70443008fe009aae050307f128a1ca892f9ed"}, - {file = "snowflake_connector_python-3.12.0.tar.gz", hash = "sha256:320e0b6f8cd8556e19c8b87249c931700238b2958313afc7a33108d67da87d82"}, + {file = "snowflake_connector_python-3.12.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:497a096fc379ef0846b2f1cf11a8d7620f0d090f08a77d9e93473845014d57d1"}, + {file = "snowflake_connector_python-3.12.3-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:055c5808d524497213e4cc9ae91ec3e46cb8342b314e78bc3e139d733dc16741"}, + {file = "snowflake_connector_python-3.12.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a5dc512d62ef693041ed2ad82931231caddc16e14ffc2842da3e3dd4240b83d"}, + {file = "snowflake_connector_python-3.12.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a46448f7279d444084eb84a9cddea67662e80ccfaddf41713b9e9aab2b1242e9"}, + {file = "snowflake_connector_python-3.12.3-cp310-cp310-win_amd64.whl", hash = "sha256:821b774b77129ce9f03729456ac1f21d69fedb50e5ce957178131c7bb3d8279f"}, + {file = "snowflake_connector_python-3.12.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:82290134978d11628026b447052219ce8d880e36937204f1f0332dfc3f2e92e9"}, + {file = "snowflake_connector_python-3.12.3-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:20b5c8000ee9cee11b0f9a6ae26640f0d498ce77f7e2ec649a2f0d306523792d"}, + {file = "snowflake_connector_python-3.12.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca6500d16bdbd37da88e589cc3e82b90272471d3aabfe4a79ec1cf4696675acf"}, + {file = "snowflake_connector_python-3.12.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b455ba117a68da436e253899674fae1a93669eaefdde8a903c03eb65b7e87c86"}, + {file = "snowflake_connector_python-3.12.3-cp311-cp311-win_amd64.whl", hash = "sha256:205219fcaeee2d33db5d0d023d60518e3bd8272ce1679be2199d7f362d255054"}, + {file = "snowflake_connector_python-3.12.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3d830ca32c864b730cba5d92900d850752199635c4fb0ae0a70ee677f62aee70"}, + {file = "snowflake_connector_python-3.12.3-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:597b0c74ec57ba693191ae2de8db9536e349ee32cab152df657473e498b6fd87"}, + {file = "snowflake_connector_python-3.12.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2215d8a4c5e25ea0d2183fe693c3fdf058cd6035e5c84710d532dc04ab4ffd31"}, + {file = "snowflake_connector_python-3.12.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8ba9c261904c1ba7cae6035c7881224cf979da39c8b7c7cb10236fdfc57e505"}, + {file = "snowflake_connector_python-3.12.3-cp312-cp312-win_amd64.whl", hash = "sha256:f0d0fcb948ef0812ab162ec9767622f345554043a07439c0c1a9474c86772320"}, + {file = "snowflake_connector_python-3.12.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fe742a0b2fb1c79a21e95b97c49a05783bc00314d1184d227c5fe5b57688af12"}, + {file = "snowflake_connector_python-3.12.3-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:a8584a44a6bb41d2056cf1b833e629c76e28c5303d2c875c1a23bda46a1cd43a"}, + {file = "snowflake_connector_python-3.12.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd990db8e4886c32ba5c63758e8dc4814e2e75f5fd3fe79d43f7e5ee0fc46793"}, + {file = "snowflake_connector_python-3.12.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4fe7f91f6e44bda877e77403a586d7487ca2c52dc1a32a705b2fea33f9c763a"}, + {file = "snowflake_connector_python-3.12.3-cp38-cp38-win_amd64.whl", hash = "sha256:4994e95eff593dc44c28243ef0ae8d27b8b1aeb96dd64cbcea5bcf0e4dfb77fb"}, + {file = "snowflake_connector_python-3.12.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ac33a7dd54b35f94c4b91369971dbd6467a914dff4b01c46e77e7e6901d7eca4"}, + {file = "snowflake_connector_python-3.12.3-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:a26876322811fe2b93f6d814dcfe016f1df680a12624026ecf57a6bcdf20f969"}, + {file = "snowflake_connector_python-3.12.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c0bb390be2e15b6b7cccab7fbe1ef94e1e9ab13790c974aa44761298cdc2641"}, + {file = "snowflake_connector_python-3.12.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7340f73af4ae72e6af8fe28a1b8e196a0c99943071afc96ce419efb4da80035"}, + {file = "snowflake_connector_python-3.12.3-cp39-cp39-win_amd64.whl", hash = "sha256:c314749bd0151218b654a7d4646a39067ab650bdc86dfebb1884b056b0bdb4b4"}, + {file = "snowflake_connector_python-3.12.3.tar.gz", hash = "sha256:02873c7f7a3b10322e28dddc2be6907f8ab8ecad93d6d6af14c77c2f53091b88"}, ] [package.dependencies] @@ -4645,7 +4640,7 @@ asn1crypto = ">0.24.0,<2.0.0" certifi = ">=2017.4.17" cffi = ">=1.9,<2.0.0" charset-normalizer = ">=2,<4" -cryptography = ">=3.1.0,<43.0.0" +cryptography = ">=3.1.0" filelock = ">=3.5,<4" idna = ">=2.5,<4" packaging = "*" @@ -5456,4 +5451,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "18011b0f5ea6b616c9d90f8a3a73a31dd285e30e6f642424a255fa54777eb62f" +content-hash = "00eb72e7f054606807de9f5dc727b446684a22ec7d450e18b5be9592ef017924" diff --git a/pyproject.toml b/pyproject.toml index daffbe938b..b83d0083bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ force-exclude = ''' [tool.poetry] name = "redash" -version = "24.09.0-dev" +version = "24.10.0-dev" description = "Make Your Company Data Driven. Connect to any data source, easily visualize, dashboard and share your data." authors = ["Arik Fraimovich "] # to be added to/removed from the mailing list, please reach out to Arik via the above email or Discord @@ -29,7 +29,7 @@ authlib = "0.15.5" backoff = "2.2.1" blinker = "1.6.2" click = "8.1.3" -cryptography = "42.0.8" +cryptography = "43.0.1" disposable-email-domains = ">=0.0.52" flask = "2.3.2" flask-limiter = "3.3.1" @@ -65,7 +65,7 @@ pyyaml = "6.0.1" redis = "4.6.0" regex = "2023.8.8" requests = "2.32.3" -restrictedpython = "6.2" +restrictedpython = "7.3" rq = "1.16.1" rq-scheduler = "0.13.1" semver = "2.8.1" @@ -130,7 +130,7 @@ python-rapidjson = "1.20" requests-aws-sign = "0.1.5" sasl = ">=0.1.3" simple-salesforce = "0.74.3" -snowflake-connector-python = "3.12.0" +snowflake-connector-python = "3.12.3" td-client = "1.0.0" thrift = ">=0.8.0" thrift-sasl = ">=0.1.0" diff --git a/redash/__init__.py b/redash/__init__.py index 0bd35de321..4459042d24 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -14,7 +14,7 @@ from redash.destinations import import_destinations from redash.query_runner import import_query_runners -__version__ = "24.09.0-dev" +__version__ = "24.10.0-dev" if os.environ.get("REMOTE_DEBUG"): diff --git a/redash/cli/queries.py b/redash/cli/queries.py index 54943dee3c..dddb35724e 100644 --- a/redash/cli/queries.py +++ b/redash/cli/queries.py @@ -5,6 +5,22 @@ manager = AppGroup(help="Queries management commands.") +@manager.command(name="rehash") +def rehash(): + from redash import models + + for q in models.Query.query.all(): + old_hash = q.query_hash + q.update_query_hash() + new_hash = q.query_hash + + if old_hash != new_hash: + print(f"Query {q.id} has changed hash from {old_hash} to {new_hash}") + models.db.session.add(q) + + models.db.session.commit() + + @manager.command(name="add_tag") @argument("query_id") @argument("tag") diff --git a/redash/metrics/database.py b/redash/metrics/database.py index 152427b2e5..6f16bdbc6d 100644 --- a/redash/metrics/database.py +++ b/redash/metrics/database.py @@ -5,7 +5,7 @@ from sqlalchemy.engine import Engine from sqlalchemy.event import listens_for from sqlalchemy.orm.util import _ORMJoin -from sqlalchemy.sql.selectable import Alias +from sqlalchemy.sql.selectable import Alias, Join from redash import statsd_client @@ -18,7 +18,7 @@ def _table_name_from_select_element(elt): if isinstance(t, Alias): t = t.original.froms[0] - while isinstance(t, _ORMJoin): + while isinstance(t, _ORMJoin) or isinstance(t, Join): t = t.left return t.name diff --git a/redash/monitor.py b/redash/monitor.py index 77521975c5..4ba3f00956 100644 --- a/redash/monitor.py +++ b/redash/monitor.py @@ -59,7 +59,7 @@ def get_status(): def rq_job_ids(): - queues = Queue.all(connection=redis_connection) + queues = Queue.all(connection=rq_redis_connection) started_jobs = [StartedJobRegistry(queue=q).get_job_ids() for q in queues] queued_jobs = [q.job_ids for q in queues] diff --git a/redash/tasks/worker.py b/redash/tasks/worker.py index 1983168d3d..004a30be8b 100644 --- a/redash/tasks/worker.py +++ b/redash/tasks/worker.py @@ -6,7 +6,7 @@ from rq import Queue as BaseQueue from rq.job import Job as BaseJob from rq.job import JobStatus -from rq.timeouts import HorseMonitorTimeoutException, UnixSignalDeathPenalty +from rq.timeouts import HorseMonitorTimeoutException from rq.utils import utcnow from rq.worker import ( HerokuWorker, # HerokuWorker implements graceful shutdown on SIGTERM @@ -113,30 +113,44 @@ def enforce_hard_limit(self, job): ) self.kill_horse() - def monitor_work_horse(self, job, queue): + def monitor_work_horse(self, job: "Job", queue: "Queue"): """The worker will monitor the work horse and make sure that it either executes successfully or the status of the job is set to failed + + Args: + job (Job): _description_ + queue (Queue): _description_ """ self.monitor_started = utcnow() + retpid = ret_val = rusage = None job.started_at = utcnow() while True: try: - with UnixSignalDeathPenalty(self.job_monitoring_interval, HorseMonitorTimeoutException): - retpid, ret_val = os.waitpid(self._horse_pid, 0) + with self.death_penalty_class(self.job_monitoring_interval, HorseMonitorTimeoutException): + retpid, ret_val, rusage = self.wait_for_horse() break except HorseMonitorTimeoutException: # Horse has not exited yet and is still running. # Send a heartbeat to keep the worker alive. - self.heartbeat(self.job_monitoring_interval + 5) + self.set_current_job_working_time((utcnow() - job.started_at).total_seconds()) job.refresh() + # Kill the job from this side if something is really wrong (interpreter lock/etc). + if job.timeout != -1 and self.current_job_working_time > (job.timeout + 60): # type: ignore + self.heartbeat(self.job_monitoring_interval + 60) + self.kill_horse() + self.wait_for_horse() + break + + self.maintain_heartbeats(job) if job.is_cancelled: self.stop_executing_job(job) if self.soft_limit_exceeded(job): self.enforce_hard_limit(job) + except OSError as e: # In case we encountered an OSError due to EINTR (which is # caused by a SIGINT or SIGTERM signal during @@ -149,29 +163,32 @@ def monitor_work_horse(self, job, queue): # Send a heartbeat to keep the worker alive. self.heartbeat() + self.set_current_job_working_time(0) + self._horse_pid = 0 # Set horse PID to 0, horse has finished working if ret_val == os.EX_OK: # The process exited normally. return + job_status = job.get_status() + if job_status is None: # Job completed and its ttl has expired return - if job_status not in [JobStatus.FINISHED, JobStatus.FAILED]: + elif self._stopped_job_id == job.id: + # Work-horse killed deliberately + self.log.warning("Job stopped by user, moving job to FailedJobRegistry") + if job.stopped_callback: + job.execute_stopped_callback(self.death_penalty_class) + self.handle_job_failure(job, queue=queue, exc_string="Job stopped by user, work-horse terminated.") + elif job_status not in [JobStatus.FINISHED, JobStatus.FAILED]: if not job.ended_at: job.ended_at = utcnow() # Unhandled failure: move the job to the failed queue - self.log.warning( - ( - "Moving job to FailedJobRegistry " - "(work-horse terminated unexpectedly; waitpid returned {})" # fmt: skip - ).format(ret_val) - ) - - self.handle_job_failure( - job, - queue=queue, - exc_string="Work-horse process was terminated unexpectedly " - "(waitpid returned %s)" % ret_val, # fmt: skip - ) + signal_msg = f" (signal {os.WTERMSIG(ret_val)})" if ret_val and os.WIFSIGNALED(ret_val) else "" + exc_string = f"Work-horse terminated unexpectedly; waitpid returned {ret_val}{signal_msg}; " + self.log.warning("Moving job to FailedJobRegistry (%s)", exc_string) + + self.handle_work_horse_killed(job, retpid, ret_val, rusage) + self.handle_job_failure(job, queue=queue, exc_string=exc_string) class RedashWorker(StatsdRecordingWorker, HardLimitingWorker): diff --git a/tests/test_monitor.py b/tests/test_monitor.py new file mode 100644 index 0000000000..a5bbc3d446 --- /dev/null +++ b/tests/test_monitor.py @@ -0,0 +1,23 @@ +from unittest.mock import MagicMock, patch + +from redash import rq_redis_connection +from redash.monitor import rq_job_ids + + +def test_rq_job_ids_uses_rq_redis_connection(): + mock_queue = MagicMock() + mock_queue.job_ids = [] + + mock_registry = MagicMock() + mock_registry.get_job_ids.return_value = [] + + with patch("redash.monitor.Queue") as mock_Queue, patch( + "redash.monitor.StartedJobRegistry" + ) as mock_StartedJobRegistry: + mock_Queue.all.return_value = [mock_queue] + mock_StartedJobRegistry.return_value = mock_registry + + rq_job_ids() + + mock_Queue.all.assert_called_once_with(connection=rq_redis_connection) + mock_StartedJobRegistry.assert_called_once_with(queue=mock_queue) diff --git a/yarn.lock b/yarn.lock index fac47619df..78a06e13b0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8137,9 +8137,9 @@ http-parser-js@>=0.5.1: integrity sha512-SGeBX54F94Wgu5RH3X5jsDtf4eHyRogWX1XGT3b4HuW3tQPM4AaBzoUji/4AAJNXCEOWZ5O0DgZmJw1947gD5Q== http-proxy-middleware@^2.0.3: - version "2.0.6" - resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.6.tgz#e1a4dd6979572c7ab5a4e4b55095d1f32a74963f" - integrity sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw== + version "2.0.7" + resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz#915f236d92ae98ef48278a95dedf17e991936ec6" + integrity sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA== dependencies: "@types/http-proxy" "^1.17.8" http-proxy "^1.18.1"