Merge branch 'master' into yoshioka/multi-byte-search-for-user-and-fa…

…vorite

.nvmrc

@@ -1 +1 @@
-v14.16.1
+v16.20.1

.pre-commit-config.yaml

@@ -13,4 +13,7 @@ repos:
     hooks:
       - id: flake8
         exclude: "migration/.*|.git|viz-lib|node_modules|migrations|bin/upgrade"
-
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: requirements-txt-fixer

client/app/components/ApplicationArea/index.jsx

@@ -17,6 +17,13 @@ export default function ApplicationArea() {
   useEffect(() => {
     function globalErrorHandler(event) {
       event.preventDefault();
+      if (event.message === "Uncaught SyntaxError: Unexpected token '<'") {
+        // if we see a javascript error on unexpected token where the unexpected token is '<', this usually means that a fallback html file (like index.html)
+        // was served as content of script rather than the expected script, give a friendlier message in the console on what could be going on
+        console.error(
+          `[Uncaught SyntaxError: Unexpected token '<'] usually means that a fallback html file was returned from server rather than the expected script. Check that the server is properly serving the file ${event.filename}.`
+        );
+      }
       setUnhandledError(event.error);
     }
 

client/app/components/queries/QueryEditor/ace.js

@@ -27,6 +27,10 @@ defineDummySnippets("sql");
 defineDummySnippets("json");
 defineDummySnippets("yaml");
 
+// without this line, ace will try to load a non-existent mode-custom.js file
+// for data sources with syntax = "custom"
+ace.define("ace/mode/custom", [], () => {});
+
 function buildTableColumnKeywords(table) {
   const keywords = [];
   table.columns.forEach(column => {

client/app/services/query-result.js

@@ -113,6 +113,10 @@ export function fetchDataFromJob(jobId, interval = 1000) {
   });
 }
 
+export function isDateTime(v) {
+  return isString(v) && moment(v).isValid() && /^\d{4}-\d{2}-\d{2}T/.test(v);
+}
+
 class QueryResult {
   constructor(props) {
     this.deferred = defer();
@@ -147,7 +151,7 @@ class QueryResult {
           let newType = null;
           if (isNumber(v)) {
             newType = "float";
-          } else if (isString(v) && v.match(/^\d{4}-\d{2}-\d{2}T/)) {
+          } else if (isDateTime(v)) {
             row[k] = moment.utc(v);
             newType = "datetime";
           } else if (isString(v) && v.match(/^\d{4}-\d{2}-\d{2}$/)) {

client/app/services/query-result.test.js

@@ -0,0 +1,17 @@
+import { isDateTime } from "@/services/query-result";
+
+describe("isDateTime", () => {
+  it.each([
+    ["2022-01-01T00:00:00", true],
+    ["2022-01-01T00:00:00+09:00", true],
+    ["2021-01-27T00:00:01.733983944+03:00 stderr F {", false],
+    ["2021-01-27Z00:00:00+09:00", false],
+    ["2021-01-27", false],
+    ["foo bar", false],
+    [2022, false],
+    [null, false],
+    ["", false],
+  ])("isDateTime('%s'). expected '%s'.", (value, expected) => {
+    expect(isDateTime(value)).toBe(expected);
+  });
+});

pytest.ini

@@ -1,2 +1,5 @@
 [pytest]
 norecursedirs = *.egg .eggs dist build docs .tox
+filterwarnings =
+    once::DeprecationWarning
+    once::PendingDeprecationWarning

redash/destinations/__init__.py

@@ -31,7 +31,7 @@ def enabled(cls):
     def configuration_schema(cls):
         return {}
 
-    def notify(self, alert, query, user, new_state, app, host, options):
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
         raise NotImplementedError()
 
     @classmethod

redash/destinations/asana.py

@@ -0,0 +1,64 @@
+import logging
+import textwrap
+
+import requests
+
+from redash.destinations import BaseDestination, register
+from redash.models import Alert
+
+
+class Asana(BaseDestination):
+    @classmethod
+    def configuration_schema(cls):
+        return {
+            "type": "object",
+            "properties": {
+                "pat": {"type": "string", "title": "Asana Personal Access Token"},
+                "project_id": {"type": "string", "title": "Asana Project ID"},
+            },
+            "secret": ["pat"],
+            "required": ["pat", "project_id"],
+        }
+
+    @classmethod
+    def icon(cls):
+        return "fa-asana"
+
+    @property
+    def api_base_url(self):
+        return "https://app.asana.com/api/1.0/tasks"
+
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
+        # Documentation: https://developers.asana.com/docs/tasks
+        state = "TRIGGERED" if new_state == Alert.TRIGGERED_STATE else "RECOVERED"
+
+        notes = textwrap.dedent(
+            f"""
+        {alert.name} has {state}.
+
+        Query: {host}/queries/{query.id}
+        Alert: {host}/alerts/{alert.id}
+        """
+        ).strip()
+
+        data = {
+            "name": f"[Redash Alert] {state}: {alert.name}",
+            "notes": notes,
+            "projects": [options["project_id"]],
+        }
+
+        try:
+            resp = requests.post(
+                self.api_base_url,
+                data=data,
+                timeout=5.0,
+                headers={"Authorization": f"Bearer {options['pat']}"},
+            )
+            logging.warning(resp.text)
+            if resp.status_code != 201:
+                logging.error("Asana send ERROR. status_code => {status}".format(status=resp.status_code))
+        except Exception as e:
+            logging.exception("Asana send ERROR. {exception}".format(exception=e))
+
+
+register(Asana)

redash/destinations/chatwork.py

@@ -29,7 +29,7 @@ def configuration_schema(cls):
     def icon(cls):
         return "fa-comment"
 
-    def notify(self, alert, query, user, new_state, app, host, options):
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
         try:
             # Documentation: http://developer.chatwork.com/ja/endpoint_rooms.html#POST-rooms-room_id-messages
             url = "https://api.chatwork.com/v2/rooms/{room_id}/messages".format(room_id=options.get("room_id"))

redash/destinations/discord.py

@@ -28,7 +28,7 @@ def configuration_schema(cls):
     def icon(cls):
         return "fa-discord"
 
-    def notify(self, alert, query, user, new_state, app, host, options):
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
         # Documentation: https://birdie0.github.io/discord-webhooks-guide/discord_webhook.html
         fields = [
             {

redash/destinations/email.py

@@ -27,7 +27,7 @@ def configuration_schema(cls):
     def icon(cls):
         return "fa-envelope"
 
-    def notify(self, alert, query, user, new_state, app, host, options):
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
         recipients = [email for email in options.get("addresses", "").split(",") if email]
 
         if not recipients:

redash/destinations/hangoutschat.py

@@ -37,7 +37,7 @@ def configuration_schema(cls):
     def icon(cls):
         return "fa-bolt"
 
-    def notify(self, alert, query, user, new_state, app, host, options):
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
         try:
             if new_state == "triggered":
                 message = '<b><font color="#c0392b">Triggered</font></b>'

redash/destinations/mattermost.py

@@ -24,7 +24,7 @@ def configuration_schema(cls):
     def icon(cls):
         return "fa-bolt"
 
-    def notify(self, alert, query, user, new_state, app, host, options):
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
         if alert.custom_subject:
             text = alert.custom_subject
         elif new_state == "triggered":

redash/destinations/microsoft_teams_webhook.py

@@ -74,7 +74,7 @@ def configuration_schema(cls):
     def icon(cls):
         return "fa-bolt"
 
-    def notify(self, alert, query, user, new_state, app, host, options):
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
         """
         :type app: redash.Redash
         """

redash/destinations/pagerduty.py

@@ -40,7 +40,7 @@ def configuration_schema(cls):
     def icon(cls):
         return "creative-commons-pd-alt"
 
-    def notify(self, alert, query, user, new_state, app, host, options):
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
         if alert.custom_subject:
             default_desc = alert.custom_subject
         elif options.get("description"):

redash/destinations/slack.py

@@ -21,7 +21,7 @@ def configuration_schema(cls):
     def icon(cls):
         return "fa-slack"
 
-    def notify(self, alert, query, user, new_state, app, host, options):
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
         # Documentation: https://api.slack.com/docs/attachments
         fields = [
             {

redash/destinations/webhook.py

@@ -26,12 +26,13 @@ def configuration_schema(cls):
     def icon(cls):
         return "fa-bolt"
 
-    def notify(self, alert, query, user, new_state, app, host, options):
+    def notify(self, alert, query, user, new_state, app, host, metadata, options):
         try:
             data = {
                 "event": "alert_state_change",
                 "alert": serialize_alert(alert, full=False),
                 "url_base": host,
+                "metadata": metadata,
             }
 
             data["alert"]["description"] = alert.custom_body

redash/handlers/query_results.py

@@ -1,6 +1,7 @@
 import unicodedata
 from urllib.parse import quote
 
+import regex
 from flask import make_response, request
 from flask_login import current_user
 from flask_restful import abort
@@ -115,7 +116,8 @@ def run_query(query, parameters, data_source, query_id, should_apply_auto_limit,
 def get_download_filename(query_result, query, filetype):
     retrieved_at = query_result.retrieved_at.strftime("%Y_%m_%d")
     if query:
-        filename = to_filename(query.name) if query.name != "" else str(query.id)
+        query_name = regex.sub(r"\p{C}", "", query.name)
+        filename = to_filename(query_name) if query_name != "" else str(query.id)
     else:
         filename = str(query_result.id)
     return "{}_{}.{}".format(filename, retrieved_at, filetype)

redash/models/__init__.py

@@ -1328,10 +1328,10 @@ def all(cls, org):
 
         return notification_destinations
 
-    def notify(self, alert, query, user, new_state, app, host):
+    def notify(self, alert, query, user, new_state, app, host, metadata):
         schema = get_configuration_schema_for_destination_type(self.type)
         self.options.set_schema(schema)
-        return self.destination.notify(alert, query, user, new_state, app, host, self.options)
+        return self.destination.notify(alert, query, user, new_state, app, host, metadata, self.options)
 
 
 @generic_repr("id", "user_id", "destination_id", "alert_id")
@@ -1368,16 +1368,16 @@ def to_dict(self):
     def all(cls, alert_id):
         return AlertSubscription.query.join(User).filter(AlertSubscription.alert_id == alert_id)
 
-    def notify(self, alert, query, user, new_state, app, host):
+    def notify(self, alert, query, user, new_state, app, host, metadata):
         if self.destination:
-            return self.destination.notify(alert, query, user, new_state, app, host)
+            return self.destination.notify(alert, query, user, new_state, app, host, metadata)
         else:
             # User email subscription, so create an email destination object
             config = {"addresses": self.user.email}
             schema = get_configuration_schema_for_destination_type("email")
             options = ConfigurationContainer(config, schema)
             destination = get_destination("email", options)
-            return destination.notify(alert, query, user, new_state, app, host, options)
+            return destination.notify(alert, query, user, new_state, app, host, metadata, options)
 
 
 @generic_repr("id", "trigger", "user_id", "org_id")

redash/models/users.py

@@ -163,7 +163,7 @@ def is_api_user(self):
 
     @property
     def profile_image_url(self):
-        if self._profile_image_url is not None:
+        if self._profile_image_url:
             return self._profile_image_url
 
         email_md5 = hashlib.md5(self.email.lower().encode()).hexdigest()

redash/query_runner/__init__.py

@@ -1,4 +1,5 @@
 import logging
+from collections import defaultdict
 from contextlib import ExitStack
 from functools import wraps
 
@@ -213,14 +214,14 @@ def run_query(self, query, user):
 
     def fetch_columns(self, columns):
         column_names = set()
-        duplicates_counter = 1
+        duplicates_counters = defaultdict(int)
         new_columns = []
 
         for col in columns:
             column_name = col[0]
-            if column_name in column_names:
-                column_name = "{}{}".format(column_name, duplicates_counter)
-                duplicates_counter += 1
+            while column_name in column_names:
+                duplicates_counters[col[0]] += 1
+                column_name = "{}{}".format(col[0], duplicates_counters[col[0]])
 
             column_names.add(column_name)
             new_columns.append({"name": column_name, "friendly_name": column_name, "type": col[1]})

redash/query_runner/big_query.py

@@ -96,9 +96,12 @@ def _get_total_bytes_processed_for_resp(bq_response):
 
 
 class BigQuery(BaseQueryRunner):
-    should_annotate_query = False
     noop_query = "SELECT 1"
 
+    def __init__(self, configuration):
+        super().__init__(configuration)
+        self.should_annotate_query = configuration["useQueryAnnotation"]
+
     @classmethod
     def enabled(cls):
         return enabled
@@ -129,6 +132,11 @@ def configuration_schema(cls):
                     "type": "number",
                     "title": "Maximum Billing Tier",
                 },
+                "useQueryAnnotation": {
+                    "type": "boolean",
+                    "title": "Use Query Annotation",
+                    "default": False,
+                },
             },
             "required": ["jsonKeyFile", "projectId"],
             "order": [
@@ -140,6 +148,7 @@ def configuration_schema(cls):
                 "totalMBytesProcessedLimit",
                 "maximumBillingTier",
                 "userDefinedFunctionResourceUri",
+                "useQueryAnnotation",
             ],
             "secret": ["jsonKeyFile"],
         }