Merge branch 'master' into fix_typo_elasticsearch

.nvmrc

@@ -1 +1 @@
-v14.16.1
+v16.20.1

.pre-commit-config.yaml

@@ -13,4 +13,7 @@ repos:
     hooks:
       - id: flake8
         exclude: "migration/.*|.git|viz-lib|node_modules|migrations|bin/upgrade"
-
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: requirements-txt-fixer

client/app/services/query-result.js

@@ -113,6 +113,10 @@ export function fetchDataFromJob(jobId, interval = 1000) {
   });
 }
 
+export function isDateTime(v) {
+  return isString(v) && moment(v).isValid() && /^\d{4}-\d{2}-\d{2}T/.test(v);
+}
+
 class QueryResult {
   constructor(props) {
     this.deferred = defer();
@@ -147,7 +151,7 @@ class QueryResult {
           let newType = null;
           if (isNumber(v)) {
             newType = "float";
-          } else if (isString(v) && v.match(/^\d{4}-\d{2}-\d{2}T/)) {
+          } else if (isDateTime(v)) {
             row[k] = moment.utc(v);
             newType = "datetime";
           } else if (isString(v) && v.match(/^\d{4}-\d{2}-\d{2}$/)) {

client/app/services/query-result.test.js

@@ -0,0 +1,17 @@
+import { isDateTime } from "@/services/query-result";
+
+describe("isDateTime", () => {
+  it.each([
+    ["2022-01-01T00:00:00", true],
+    ["2022-01-01T00:00:00+09:00", true],
+    ["2021-01-27T00:00:01.733983944+03:00 stderr F {", false],
+    ["2021-01-27Z00:00:00+09:00", false],
+    ["2021-01-27", false],
+    ["foo bar", false],
+    [2022, false],
+    [null, false],
+    ["", false],
+  ])("isDateTime('%s'). expected '%s'.", (value, expected) => {
+    expect(isDateTime(value)).toBe(expected);
+  });
+});

client/cypress/integration/dashboard/dashboard_list.js

@@ -0,0 +1,24 @@
+describe("Dashboard list sort", () => {
+  beforeEach(() => {
+    cy.login();
+  });
+
+  it("creates one dashboard", () => {
+    cy.visit("/dashboards");
+    cy.getByTestId("CreateButton").click();
+    cy.getByTestId("CreateDashboardMenuItem").click();
+    cy.getByTestId("CreateDashboardDialog").within(() => {
+      cy.get("input").type("A Foo Bar");
+      cy.getByTestId("DashboardSaveButton").click();
+    });
+  });
+
+  describe("Sorting table does not crash page ", () => {
+    it("sorts", () => {
+      cy.visit("/dashboards");
+      cy.contains("Name").click();
+      cy.wait(1000); // eslint-disable-line cypress/no-unnecessary-waiting
+      cy.getByTestId("ErrorMessage").should("not.exist");
+    });
+  });
+});

pytest.ini

@@ -1,2 +1,5 @@
 [pytest]
 norecursedirs = *.egg .eggs dist build docs .tox
+filterwarnings =
+    once::DeprecationWarning
+    once::PendingDeprecationWarning

redash/destinations/__init__.py

@@ -31,7 +31,7 @@ def enabled(cls):
     def configuration_schema(cls):
         return {}
 
-    def notify(self, alert, query, user, new_state, app, host, options):
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
         raise NotImplementedError()
 
     @classmethod

redash/destinations/chatwork.py

@@ -29,7 +29,7 @@ def configuration_schema(cls):
     def icon(cls):
         return "fa-comment"
 
-    def notify(self, alert, query, user, new_state, app, host, options):
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
         try:
             # Documentation: http://developer.chatwork.com/ja/endpoint_rooms.html#POST-rooms-room_id-messages
             url = "https://api.chatwork.com/v2/rooms/{room_id}/messages".format(room_id=options.get("room_id"))

redash/destinations/discord.py

@@ -28,7 +28,7 @@ def configuration_schema(cls):
     def icon(cls):
         return "fa-discord"
 
-    def notify(self, alert, query, user, new_state, app, host, options):
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
         # Documentation: https://birdie0.github.io/discord-webhooks-guide/discord_webhook.html
         fields = [
             {

redash/destinations/email.py

@@ -27,7 +27,7 @@ def configuration_schema(cls):
     def icon(cls):
         return "fa-envelope"
 
-    def notify(self, alert, query, user, new_state, app, host, options):
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
         recipients = [email for email in options.get("addresses", "").split(",") if email]
 
         if not recipients:

redash/destinations/hangoutschat.py

@@ -37,7 +37,7 @@ def configuration_schema(cls):
     def icon(cls):
         return "fa-bolt"
 
-    def notify(self, alert, query, user, new_state, app, host, options):
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
         try:
             if new_state == "triggered":
                 message = '<b><font color="#c0392b">Triggered</font></b>'

redash/destinations/mattermost.py

@@ -24,7 +24,7 @@ def configuration_schema(cls):
     def icon(cls):
         return "fa-bolt"
 
-    def notify(self, alert, query, user, new_state, app, host, options):
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
         if alert.custom_subject:
             text = alert.custom_subject
         elif new_state == "triggered":

redash/destinations/microsoft_teams_webhook.py

@@ -74,7 +74,7 @@ def configuration_schema(cls):
     def icon(cls):
         return "fa-bolt"
 
-    def notify(self, alert, query, user, new_state, app, host, options):
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
         """
         :type app: redash.Redash
         """

redash/destinations/pagerduty.py

@@ -40,7 +40,7 @@ def configuration_schema(cls):
     def icon(cls):
         return "creative-commons-pd-alt"
 
-    def notify(self, alert, query, user, new_state, app, host, options):
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
         if alert.custom_subject:
             default_desc = alert.custom_subject
         elif options.get("description"):

redash/destinations/slack.py

@@ -21,7 +21,7 @@ def configuration_schema(cls):
     def icon(cls):
         return "fa-slack"
 
-    def notify(self, alert, query, user, new_state, app, host, options):
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
         # Documentation: https://api.slack.com/docs/attachments
         fields = [
             {

redash/destinations/webhook.py

@@ -26,12 +26,13 @@ def configuration_schema(cls):
     def icon(cls):
         return "fa-bolt"
 
-    def notify(self, alert, query, user, new_state, app, host, options):
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
         try:
             data = {
                 "event": "alert_state_change",
                 "alert": serialize_alert(alert, full=False),
                 "url_base": host,
+                "metadata": metadata,
             }
 
             data["alert"]["description"] = alert.custom_body

redash/handlers/query_results.py

@@ -1,6 +1,7 @@
 import unicodedata
 from urllib.parse import quote
 
+import regex
 from flask import make_response, request
 from flask_login import current_user
 from flask_restful import abort
@@ -115,7 +116,8 @@ def run_query(query, parameters, data_source, query_id, should_apply_auto_limit,
 def get_download_filename(query_result, query, filetype):
     retrieved_at = query_result.retrieved_at.strftime("%Y_%m_%d")
     if query:
-        filename = to_filename(query.name) if query.name != "" else str(query.id)
+        query_name = regex.sub(r"\p{C}", "", query.name)
+        filename = to_filename(query_name) if query_name != "" else str(query.id)
     else:
         filename = str(query_result.id)
     return "{}_{}.{}".format(filename, retrieved_at, filetype)

redash/models/__init__.py

@@ -1318,10 +1318,10 @@ def all(cls, org):
 
         return notification_destinations
 
-    def notify(self, alert, query, user, new_state, app, host):
+    def notify(self, alert, query, user, new_state, app, host, metadata):
         schema = get_configuration_schema_for_destination_type(self.type)
         self.options.set_schema(schema)
-        return self.destination.notify(alert, query, user, new_state, app, host, self.options)
+        return self.destination.notify(alert, query, user, new_state, app, host, metadata, self.options)
 
 
 @generic_repr("id", "user_id", "destination_id", "alert_id")
@@ -1358,16 +1358,16 @@ def to_dict(self):
     def all(cls, alert_id):
         return AlertSubscription.query.join(User).filter(AlertSubscription.alert_id == alert_id)
 
-    def notify(self, alert, query, user, new_state, app, host):
+    def notify(self, alert, query, user, new_state, app, host, metadata):
         if self.destination:
-            return self.destination.notify(alert, query, user, new_state, app, host)
+            return self.destination.notify(alert, query, user, new_state, app, host, metadata)
         else:
             # User email subscription, so create an email destination object
             config = {"addresses": self.user.email}
             schema = get_configuration_schema_for_destination_type("email")
             options = ConfigurationContainer(config, schema)
             destination = get_destination("email", options)
-            return destination.notify(alert, query, user, new_state, app, host, options)
+            return destination.notify(alert, query, user, new_state, app, host, metadata, options)
 
 
 @generic_repr("id", "trigger", "user_id", "org_id")

redash/models/users.py

@@ -163,7 +163,7 @@ def is_api_user(self):
 
     @property
     def profile_image_url(self):
-        if self._profile_image_url is not None:
+        if self._profile_image_url:
             return self._profile_image_url
 
         email_md5 = hashlib.md5(self.email.lower().encode()).hexdigest()

redash/query_runner/__init__.py

@@ -1,4 +1,5 @@
 import logging
+from collections import defaultdict
 from contextlib import ExitStack
 from functools import wraps
 
@@ -213,14 +214,14 @@ def run_query(self, query, user):
 
     def fetch_columns(self, columns):
         column_names = set()
-        duplicates_counter = 1
+        duplicates_counters = defaultdict(int)
         new_columns = []
 
         for col in columns:
             column_name = col[0]
-            if column_name in column_names:
-                column_name = "{}{}".format(column_name, duplicates_counter)
-                duplicates_counter += 1
+            while column_name in column_names:
+                duplicates_counters[col[0]] += 1
+                column_name = "{}{}".format(col[0], duplicates_counters[col[0]])
 
             column_names.add(column_name)
             new_columns.append({"name": column_name, "friendly_name": column_name, "type": col[1]})

redash/query_runner/big_query.py

@@ -96,9 +96,12 @@ def _get_total_bytes_processed_for_resp(bq_response):
 
 
 class BigQuery(BaseQueryRunner):
-    should_annotate_query = False
     noop_query = "SELECT 1"
 
+    def __init__(self, configuration):
+        super().__init__(configuration)
+        self.should_annotate_query = configuration["useQueryAnnotation"]
+
     @classmethod
     def enabled(cls):
         return enabled
@@ -129,6 +132,11 @@ def configuration_schema(cls):
                     "type": "number",
                     "title": "Maximum Billing Tier",
                 },
+                "useQueryAnnotation": {
+                    "type": "boolean",
+                    "title": "Use Query Annotation",
+                    "default": False,
+                },
             },
             "required": ["jsonKeyFile", "projectId"],
             "order": [
@@ -140,6 +148,7 @@ def configuration_schema(cls):
                 "totalMBytesProcessedLimit",
                 "maximumBillingTier",
                 "userDefinedFunctionResourceUri",
+                "useQueryAnnotation",
             ],
             "secret": ["jsonKeyFile"],
         }