From 26d30a95a969654f9c6bb4c303b725dce562b673 Mon Sep 17 00:00:00 2001
From: Charlie <cgu@pinterest.com>
Date: Mon, 18 Nov 2024 23:50:41 -0800
Subject: [PATCH 1/6] tmp

---
 .../components/QueryEditor/QueryEditor.tsx    |  8 +-
 .../webapp/components/QueryEditor/monokai.ts  |  7 ++
 .../lib/codemirror/codemirror-language.ts     | 85 +++++++++++++++++++
 .../webapp/lib/codemirror/codemirror-mixed.ts | 53 +++++++-----
 .../webapp/lib/sql-helper/sql-setting.ts      |  6 +-
 5 files changed, 130 insertions(+), 29 deletions(-)
 create mode 100644 querybook/webapp/components/QueryEditor/monokai.ts
 create mode 100644 querybook/webapp/lib/codemirror/codemirror-language.ts

diff --git a/querybook/webapp/components/QueryEditor/QueryEditor.tsx b/querybook/webapp/components/QueryEditor/QueryEditor.tsx
index bcb249e17..48a0ac382 100644
--- a/querybook/webapp/components/QueryEditor/QueryEditor.tsx
+++ b/querybook/webapp/components/QueryEditor/QueryEditor.tsx
@@ -1,6 +1,5 @@
 import { acceptCompletion, startCompletion } from '@codemirror/autocomplete';
 import { EditorView } from '@codemirror/view';
-import { monokai } from '@uiw/codemirror-theme-monokai';
 import CodeMirror, { ReactCodeMirrorRef } from '@uiw/react-codemirror';
 import clsx from 'clsx';
 import React, {
@@ -35,6 +34,8 @@ import { TableToken } from 'lib/sql-helper/sql-lexer';
 import { navigateWithinEnv } from 'lib/utils/query-string';
 import { IconButton } from 'ui/Button/IconButton';
 
+import { CustomMonokaiDarkTheme } from './monokai';
+
 import './QueryEditor.scss';
 
 export interface IQueryEditorProps {
@@ -359,7 +360,7 @@ export const QueryEditor: React.FC<
 
         const extensions = useMemo(
             () => [
-                mixedSQL(),
+                mixedSQL(language),
                 keyMapExtention,
                 statusBarExtension,
                 eventsExtension,
@@ -372,6 +373,7 @@ export const QueryEditor: React.FC<
                 sqlCompleteExtension,
             ],
             [
+                language,
                 keyMapExtention,
                 statusBarExtension,
                 eventsExtension,
@@ -432,7 +434,7 @@ export const QueryEditor: React.FC<
                 {floatButtons}
                 <CodeMirror
                     ref={editorRef}
-                    theme={theme === 'dark' ? monokai : 'light'}
+                    theme={theme === 'dark' ? CustomMonokaiDarkTheme : 'light'}
                     className="ReactCodeMirror"
                     value={value}
                     height="100%"
diff --git a/querybook/webapp/components/QueryEditor/monokai.ts b/querybook/webapp/components/QueryEditor/monokai.ts
new file mode 100644
index 000000000..f503ae272
--- /dev/null
+++ b/querybook/webapp/components/QueryEditor/monokai.ts
@@ -0,0 +1,7 @@
+import { monokaiInit } from '@uiw/codemirror-theme-monokai';
+import { tags as t } from '@lezer/highlight';
+
+export const CustomMonokaiDarkTheme = monokaiInit({
+    settings: {},
+    styles: [{ tag: [t.name], color: 'var(--text-dark)' }],
+});
diff --git a/querybook/webapp/lib/codemirror/codemirror-language.ts b/querybook/webapp/lib/codemirror/codemirror-language.ts
new file mode 100644
index 000000000..09b9db9cc
--- /dev/null
+++ b/querybook/webapp/lib/codemirror/codemirror-language.ts
@@ -0,0 +1,85 @@
+import {
+    StandardSQL,
+    MSSQL,
+    MySQL,
+    PostgreSQL,
+    SQLite,
+    SQLDialect,
+} from '@codemirror/lang-sql';
+import { SettingsByLanguage } from 'lib/sql-helper/sql-setting';
+
+function builtInTypeSplit(language: string, builtIn: string): [string, string] {
+    /**
+     * This is needed because codemirror 5 mixes type and builtin keywords together
+     *
+     * returns [builtIn, Type] deduped
+     */
+
+    const settings = SettingsByLanguage[language];
+    const builtInSet = new Set(builtIn.split(' '));
+
+    const builtInWithoutType = Array.from(builtInSet.difference(settings.type));
+
+    return [builtInWithoutType.join(' '), Array.from(settings.type).join(' ')];
+}
+
+const trinoKeywords =
+    'abs absent acos add admin after all all_match alter analyze and any any_match approx_distinct approx_most_frequent approx_percentile approx_set arbitrary array_agg array_distinct array_except array_intersect array_join array_max array_min array_position array_remove array_sort array_union arrays_overlap as asc asin at at_timezone atan atan2 authorization avg bar bernoulli beta_cdf between bing_tile bing_tile_at bing_tile_coordinates bing_tile_polygon bing_tile_quadkey bing_tile_zoom_level bing_tiles_around bit_count bitwise_and bitwise_and_agg bitwise_left_shift bitwise_not bitwise_or bitwise_or_agg bitwise_right_shift bitwise_right_shift_arithmetic bitwise_xor bool_and bool_or both by call cardinality cascade case cast catalogs cbrt ceil ceiling char2hexint checksum chr classify coalesce codepoint column columns combinations comment commit committed concat concat_ws conditional constraint contains contains_sequence convex_hull_agg copartition corr cos cosh cosine_similarity count count_if covar_pop covar_samp crc32 create cross cube cume_dist current current_catalog current_date current_groups current_path current_role current_schema current_time current_timestamp current_timezone current_user data date_add date_diff date_format date_parse date_trunc day day_of_month day_of_week day_of_year deallocate default define definer degrees delete dense_rank deny desc describe descriptor distinct distributed dow doy drop e element_at else empty empty_approx_set encoding end error escape evaluate_classifier_predictions every except excluding execute exists exp explain extract false features fetch filter final first first_value flatten floor following for format format_datetime format_number from from_base from_base32 from_base64 from_base64url from_big_endian_32 from_big_endian_64 from_encoded_polyline from_geojson_geometry from_hex from_ieee754_32 from_ieee754_64 from_iso8601_date from_iso8601_timestamp from_iso8601_timestamp_nanos from_unixtime from_unixtime_nanos from_utf8 full functions geometric_mean geometry_from_hadoop_shape geometry_invalid_reason geometry_nearest_points geometry_to_bing_tiles geometry_union geometry_union_agg grant granted grants graphviz great_circle_distance greatest group grouping groups hamming_distance hash_counts having histogram hmac_md5 hmac_sha1 hmac_sha256 hmac_sha512 hour human_readable_seconds if ignore in including index infinity initial inner input insert intersect intersection_cardinality into inverse_beta_cdf inverse_normal_cdf invoker io is is_finite is_infinite is_json_scalar is_nan isolation jaccard_index join json_array json_array_contains json_array_get json_array_length json_exists json_extract json_extract_scalar json_format json_object json_parse json_query json_size json_value keep key keys kurtosis lag last last_day_of_month last_value lateral lead leading learn_classifier learn_libsvm_classifier learn_libsvm_regressor learn_regressor least left length level levenshtein_distance like limit line_interpolate_point line_interpolate_points line_locate_point listagg ln local localtime localtimestamp log log10 log2 logical lower lpad ltrim luhn_check make_set_digest map_agg map_concat map_entries map_filter map_from_entries map_keys map_union map_values map_zip_with match match_recognize matched matches materialized max max_by md5 measures merge merge_set_digest millisecond min min_by minute mod month multimap_agg multimap_from_entries murmur3 nan natural next nfc nfd nfkc nfkd ngrams no none none_match normal_cdf normalize not now nth_value ntile null nullif nulls numeric_histogram object objectid_timestamp of offset omit on one only option or order ordinality outer output over overflow parse_data_size parse_datetime parse_duration partition partitions passing past path pattern per percent_rank permute pi position pow power preceding prepare privileges properties prune qdigest_agg quarter quotes radians rand random range rank read recursive reduce reduce_agg refresh regexp_count regexp_extract regexp_extract_all regexp_like regexp_position regexp_replace regexp_split regr_intercept regr_slope regress rename render repeat repeatable replace reset respect restrict returning reverse revoke rgb right role roles rollback rollup round row_number rows rpad rtrim running scalar schema schemas second security seek select sequence serializable session set sets sha1 sha256 sha512 show shuffle sign simplify_geometry sin skewness skip slice some soundex spatial_partitioning spatial_partitions split split_part split_to_map split_to_multimap spooky_hash_v2_32 spooky_hash_v2_64 sqrt st_area st_asbinary st_astext st_boundary st_buffer st_centroid st_contains st_convexhull st_coorddim st_crosses st_difference st_dimension st_disjoint st_distance st_endpoint st_envelope st_envelopeaspts st_equals st_exteriorring st_geometries st_geometryfromtext st_geometryn st_geometrytype st_geomfrombinary st_interiorringn st_interiorrings st_intersection st_intersects st_isclosed st_isempty st_isring st_issimple st_isvalid st_length st_linefromtext st_linestring st_multipoint st_numgeometries st_numinteriorring st_numpoints st_overlaps st_point st_pointn st_points st_polygon st_relate st_startpoint st_symdifference st_touches st_union st_within st_x st_xmax st_xmin st_y st_ymax st_ymin start starts_with stats stddev stddev_pop stddev_samp string strpos subset substr substring sum system table tables tablesample tan tanh tdigest_agg text then ties timestamp_objectid timezone_hour timezone_minute to to_base to_base32 to_base64 to_base64url to_big_endian_32 to_big_endian_64 to_char to_date to_encoded_polyline to_geojson_geometry to_geometry to_hex to_ieee754_32 to_ieee754_64 to_iso8601 to_milliseconds to_spherical_geography to_timestamp to_unixtime to_utf8 trailing transaction transform transform_keys transform_values translate trim trim_array true truncate try try_cast type typeof uescape unbounded uncommitted unconditional union unique unknown unmatched unnest update upper url_decode url_encode url_extract_fragment url_extract_host url_extract_parameter url_extract_path url_extract_port url_extract_protocol url_extract_query use user using utf16 utf32 utf8 validate value value_at_quantile values values_at_quantiles var_pop var_samp variance verbose version view week week_of_year when where width_bucket wilson_interval_lower wilson_interval_upper window with with_timezone within without word_stem work wrapper write xxhash64 year year_of_week yow zip zip_with';
+const trinoBuiltin =
+    'array bigint bingtile boolean char codepoints color date decimal double function geometry hyperloglog int integer interval ipaddress joniregexp json json2016 jsonpath kdbtree likepattern map model objectid p4hyperloglog precision qdigest re2jregexp real regressor row setdigest smallint sphericalgeography tdigest time timestamp tinyint uuid varbinary varchar zone';
+
+const [processedTrinoBuiltin, processedTrinoType] = builtInTypeSplit(
+    'trino',
+    trinoBuiltin
+);
+
+// Source https://github.com/codemirror/codemirror5/blob/master/mode/sql/sql.js
+const TrinoSQL = SQLDialect.define({
+    operatorChars: '[]|<>=!-+*/%',
+    charSetCasts: false,
+    doubleQuotedStrings: false,
+    unquotedBitLiterals: true,
+    hashComments: false,
+    spaceAfterDashes: false,
+    keywords: trinoKeywords,
+    builtin: processedTrinoBuiltin,
+    types: processedTrinoType,
+});
+
+const sparkSQLKeywords =
+    'add after all alter analyze and anti archive array as asc at between bucket buckets by cache cascade case cast change clear cluster clustered codegen collection column columns comment commit compact compactions compute concatenate cost create cross cube current current_date current_timestamp database databases data dbproperties defined delete delimited deny desc describe dfs directories distinct distribute drop else end escaped except exchange exists explain export extended external false fields fileformat first following for format formatted from full function functions global grant group grouping having if ignore import in index indexes inner inpath inputformat insert intersect interval into is items join keys last lateral lazy left like limit lines list load local location lock locks logical macro map minus msck natural no not null nulls of on optimize option options or order out outer outputformat over overwrite partition partitioned partitions percent preceding principals purge range recordreader recordwriter recover reduce refresh regexp rename repair replace reset restrict revoke right rlike role roles rollback rollup row rows schema schemas select semi separated serde serdeproperties set sets show skewed sort sorted start statistics stored stratify struct table tables tablesample tblproperties temp temporary terminated then to touch transaction transactions transform true truncate unarchive unbounded uncache union unlock unset use using values view when where window with';
+const sparkSQLBuiltin =
+    'abs acos acosh add_months aggregate and any approx_count_distinct approx_percentile array array_contains array_distinct array_except array_intersect array_join array_max array_min array_position array_remove array_repeat array_sort array_union arrays_overlap arrays_zip ascii asin asinh assert_true atan atan2 atanh avg base64 between bigint bin binary bit_and bit_count bit_get bit_length bit_or bit_xor bool_and bool_or boolean bround btrim cardinality case cast cbrt ceil ceiling char char_length character_length chr coalesce collect_list collect_set concat concat_ws conv corr cos cosh cot count count_if count_min_sketch covar_pop covar_samp crc32 cume_dist current_catalog current_database current_date current_timestamp current_timezone current_user date date_add date_format date_from_unix_date date_part date_sub date_trunc datediff day dayofmonth dayofweek dayofyear decimal decode degrees delimited dense_rank div double e element_at elt encode every exists exp explode explode_outer expm1 extract factorial filter find_in_set first first_value flatten float floor forall format_number format_string from_csv from_json from_unixtime from_utc_timestamp get_json_object getbit greatest grouping grouping_id hash hex hour hypot if ifnull in initcap inline inline_outer input_file_block_length input_file_block_start input_file_name inputformat instr int isnan isnotnull isnull java_method json_array_length json_object_keys json_tuple kurtosis lag last last_day last_value lcase lead least left length levenshtein like ln locate log log10 log1p log2 lower lpad ltrim make_date make_dt_interval make_interval make_timestamp make_ym_interval map map_concat map_entries map_filter map_from_arrays map_from_entries map_keys map_values map_zip_with max max_by md5 mean min min_by minute mod monotonically_increasing_id month months_between named_struct nanvl negative next_day not now nth_value ntile nullif nvl nvl2 octet_length or outputformat overlay parse_url percent_rank percentile percentile_approx pi pmod posexplode posexplode_outer position positive pow power printf quarter radians raise_error rand randn random rank rcfile reflect regexp regexp_extract regexp_extract_all regexp_like regexp_replace repeat replace reverse right rint rlike round row_number rpad rtrim schema_of_csv schema_of_json second sentences sequence sequencefile serde session_window sha sha1 sha2 shiftleft shiftright shiftrightunsigned shuffle sign signum sin sinh size skewness slice smallint some sort_array soundex space spark_partition_id split sqrt stack std stddev stddev_pop stddev_samp str_to_map string struct substr substring substring_index sum tan tanh textfile timestamp timestamp_micros timestamp_millis timestamp_seconds tinyint to_csv to_date to_json to_timestamp to_unix_timestamp to_utc_timestamp transform transform_keys transform_values translate trim trunc try_add try_divide typeof ucase unbase64 unhex uniontype unix_date unix_micros unix_millis unix_seconds unix_timestamp upper uuid var_pop var_samp variance version weekday weekofyear when width_bucket window xpath xpath_boolean xpath_double xpath_float xpath_int xpath_long xpath_number xpath_short xpath_string xxhash64 year zip_with';
+
+const [processedSparkSQLBuiltin, processedSparkSQLType] = builtInTypeSplit(
+    'sparksql',
+    sparkSQLBuiltin
+);
+
+const SparkSQL = SQLDialect.define({
+    charSetCasts: false,
+    doubleQuotedStrings: true,
+    unquotedBitLiterals: true,
+    hashComments: false,
+    spaceAfterDashes: false,
+    keywords: sparkSQLKeywords,
+    types: processedSparkSQLType,
+    builtin: processedSparkSQLBuiltin,
+    operatorChars: '*/+-%<>!=~&|^',
+});
+
+const LanguageToCodeMirrorSQLDialect = {
+    sql: StandardSQL,
+    mysql: MySQL,
+    mssql: MSSQL,
+    postgresql: PostgreSQL,
+    sqlite: SQLite,
+    presto: TrinoSQL,
+    trino: TrinoSQL,
+    sparksql: SparkSQL,
+    spark: SparkSQL, // unused alias
+};
+
+export function getCodeMirrorSQLDialect(language: string): SQLDialect {
+    return LanguageToCodeMirrorSQLDialect[language] || StandardSQL;
+}
diff --git a/querybook/webapp/lib/codemirror/codemirror-mixed.ts b/querybook/webapp/lib/codemirror/codemirror-mixed.ts
index ee0260d42..86892315f 100644
--- a/querybook/webapp/lib/codemirror/codemirror-mixed.ts
+++ b/querybook/webapp/lib/codemirror/codemirror-mixed.ts
@@ -1,8 +1,10 @@
 import { parseMixed } from '@lezer/common';
 
-import { StandardSQL, sql } from '@codemirror/lang-sql';
+import { sql } from '@codemirror/lang-sql';
 import { LanguageSupport } from '@codemirror/language';
 import { liquid } from '@codemirror/lang-liquid';
+import { styleTags, tags as t } from '@lezer/highlight';
+import { getCodeMirrorSQLDialect } from './codemirror-language';
 
 const liquidBasedSQL = liquid({ base: sql() });
 
@@ -13,30 +15,35 @@ const matchingBraces = {
     '{%': '%}',
 };
 
-const mixedSQLLanguage = StandardSQL.language.configure({
-    wrap: parseMixed((node, input) => {
-        if (node.name === 'String' || node.name === 'QuotedIdentifier') {
-            return liquidString;
-        }
-        if (node.name === 'Braces') {
-            // must have at least length 4 to wrap
-            if (node.to - node.from < 4) {
-                return null;
-            }
-            const startText = input.read(node.from, node.from + 2);
-            const endText = input.read(node.to - 2, node.to);
-            const bracesMatch =
-                startText in matchingBraces &&
-                matchingBraces[startText] === endText;
-            if (bracesMatch) {
+export function mixedSQL(language: string) {
+    const baseSQLLanguage = getCodeMirrorSQLDialect(language);
+    const mixedSQLLanguage = baseSQLLanguage.language.configure({
+        props: [
+            styleTags({
+                'CompositeIdentifier/Identifier': t.special(t.propertyName),
+            }),
+        ],
+        wrap: parseMixed((node, input) => {
+            if (node.name === 'String' || node.name === 'QuotedIdentifier') {
                 return liquidString;
             }
-        }
-
-        return null;
-    }),
-});
+            if (node.name === 'Braces') {
+                // must have at least length 4 to wrap
+                if (node.to - node.from < 4) {
+                    return null;
+                }
+                const startText = input.read(node.from, node.from + 2);
+                const endText = input.read(node.to - 2, node.to);
+                const bracesMatch =
+                    startText in matchingBraces &&
+                    matchingBraces[startText] === endText;
+                if (bracesMatch) {
+                    return liquidString;
+                }
+            }
 
-export function mixedSQL() {
+            return null;
+        }),
+    });
     return new LanguageSupport(mixedSQLLanguage, []);
 }
diff --git a/querybook/webapp/lib/sql-helper/sql-setting.ts b/querybook/webapp/lib/sql-helper/sql-setting.ts
index 27ea41bb9..eb8512018 100644
--- a/querybook/webapp/lib/sql-helper/sql-setting.ts
+++ b/querybook/webapp/lib/sql-helper/sql-setting.ts
@@ -10,10 +10,10 @@ export interface ILanguageSetting {
     quoteChars?: [quoteStart: string, quoteEnd: string];
 }
 
-const SQL_KEYWORDS =
-    'alter and as asc between by count create delete desc distinct drop from group having in insert into is join like not on or order select set table union update values where limit ';
+export const SQL_KEYWORDS =
+    'alter and as asc between by count create delete desc distinct drop from group having in insert into is join like not on or order select set table union update values where limit';
 
-const SettingsByLanguage: Record<string, ILanguageSetting> = {
+export const SettingsByLanguage: Record<string, ILanguageSetting> = {
     hive: {
         keywords: new Set(
             // 2.2.0

From 70e1c6e68bafe666f9557fb8843b4468a93d1cca Mon Sep 17 00:00:00 2001
From: Charlie <cgu@pinterest.com>
Date: Tue, 19 Nov 2024 09:44:38 -0800
Subject: [PATCH 2/6] Update keywords set

---
 .../components/QueryEditor/QueryEditor.tsx    |  2 +
 .../lib/codemirror/codemirror-language.ts     | 44 +++++++++++++++----
 .../webapp/lib/sql-helper/sql-setting.ts      |  4 +-
 3 files changed, 39 insertions(+), 11 deletions(-)

diff --git a/querybook/webapp/components/QueryEditor/QueryEditor.tsx b/querybook/webapp/components/QueryEditor/QueryEditor.tsx
index 48a0ac382..116d32313 100644
--- a/querybook/webapp/components/QueryEditor/QueryEditor.tsx
+++ b/querybook/webapp/components/QueryEditor/QueryEditor.tsx
@@ -1,4 +1,5 @@
 import { acceptCompletion, startCompletion } from '@codemirror/autocomplete';
+import { indentService } from '@codemirror/language';
 import { EditorView } from '@codemirror/view';
 import CodeMirror, { ReactCodeMirrorRef } from '@uiw/react-codemirror';
 import clsx from 'clsx';
@@ -371,6 +372,7 @@ export const QueryEditor: React.FC<
                 searchExtension,
                 selectionExtension,
                 sqlCompleteExtension,
+                indentService.of((context, pos) => context.lineIndent(pos - 1)),
             ],
             [
                 language,
diff --git a/querybook/webapp/lib/codemirror/codemirror-language.ts b/querybook/webapp/lib/codemirror/codemirror-language.ts
index 09b9db9cc..06c68f3a9 100644
--- a/querybook/webapp/lib/codemirror/codemirror-language.ts
+++ b/querybook/webapp/lib/codemirror/codemirror-language.ts
@@ -6,21 +6,45 @@ import {
     SQLite,
     SQLDialect,
 } from '@codemirror/lang-sql';
-import { SettingsByLanguage } from 'lib/sql-helper/sql-setting';
+import { getLanguageSetting } from 'lib/sql-helper/sql-setting';
 
-function builtInTypeSplit(language: string, builtIn: string): [string, string] {
+// source: https://github.com/codemirror/lang-sql/blob/main/src/tokens.ts
+const SQLTypes =
+    'array binary bit boolean char character clob date decimal double float int integer interval large national nchar nclob numeric object precision real smallint time timestamp varchar varying ';
+const SQLKeywords =
+    'absolute action add after all allocate alter and any are as asc assertion at authorization before begin between both breadth by call cascade cascaded case cast catalog check close collate collation column commit condition connect connection constraint constraints constructor continue corresponding count create cross cube current current_date current_default_transform_group current_transform_group_for_type current_path current_role current_time current_timestamp current_user cursor cycle data day deallocate declare default deferrable deferred delete depth deref desc describe descriptor deterministic diagnostics disconnect distinct do domain drop dynamic each else elseif end end-exec equals escape except exception exec execute exists exit external fetch first for foreign found from free full function general get global go goto grant group grouping handle having hold hour identity if immediate in indicator initially inner inout input insert intersect into is isolation join key language last lateral leading leave left level like limit local localtime localtimestamp locator loop map match method minute modifies module month names natural nesting new next no none not of old on only open option or order ordinality out outer output overlaps pad parameter partial path prepare preserve primary prior privileges procedure public read reads recursive redo ref references referencing relative release repeat resignal restrict result return returns revoke right role rollback rollup routine row rows savepoint schema scroll search second section select session session_user set sets signal similar size some space specific specifictype sql sqlexception sqlstate sqlwarning start state static system_user table temporary then timezone_hour timezone_minute to trailing transaction translation treat trigger under undo union unique unnest until update usage user using value values view when whenever where while with without work write year zone';
+const SQLTypesSet = new Set(SQLTypes.split(' '));
+const SQLKeywordsSet = new Set(SQLKeywords.split(' '));
+
+function builtInTypeSplit(
+    language: string,
+    keywords: string,
+    builtIn: string
+): [string, string] {
     /**
-     * This is needed because codemirror 5 mixes type and builtin keywords together
+     * This is needed because codemirror 5 mixes type and builtin keywords together.
+     * In our case, we want:
+     *  - keywords: standard SQL keywords, from SQLKeywords
+     *  - builtin: functions, operators, etc
+     *  - type: data types
      *
-     * returns [builtIn, Type] deduped
+     * returns [builtIn, Type]
      */
 
-    const settings = SettingsByLanguage[language];
+    const settings = getLanguageSetting(language);
+    const keywordsSet = new Set(keywords.split(' '));
+    const typesSet = settings.type.union(SQLTypesSet);
     const builtInSet = new Set(builtIn.split(' '));
 
-    const builtInWithoutType = Array.from(builtInSet.difference(settings.type));
+    const nonStandardKeywordSet = keywordsSet.difference(SQLKeywordsSet);
+    const nonStandardKeywordAndBuiltInSet = nonStandardKeywordSet
+        .union(builtInSet)
+        .difference(typesSet);
 
-    return [builtInWithoutType.join(' '), Array.from(settings.type).join(' ')];
+    return [
+        Array.from(nonStandardKeywordAndBuiltInSet).join(' '),
+        Array.from(typesSet).join(' '),
+    ];
 }
 
 const trinoKeywords =
@@ -30,6 +54,7 @@ const trinoBuiltin =
 
 const [processedTrinoBuiltin, processedTrinoType] = builtInTypeSplit(
     'trino',
+    trinoKeywords,
     trinoBuiltin
 );
 
@@ -41,7 +66,7 @@ const TrinoSQL = SQLDialect.define({
     unquotedBitLiterals: true,
     hashComments: false,
     spaceAfterDashes: false,
-    keywords: trinoKeywords,
+    keywords: SQLKeywords,
     builtin: processedTrinoBuiltin,
     types: processedTrinoType,
 });
@@ -53,6 +78,7 @@ const sparkSQLBuiltin =
 
 const [processedSparkSQLBuiltin, processedSparkSQLType] = builtInTypeSplit(
     'sparksql',
+    sparkSQLKeywords,
     sparkSQLBuiltin
 );
 
@@ -62,7 +88,7 @@ const SparkSQL = SQLDialect.define({
     unquotedBitLiterals: true,
     hashComments: false,
     spaceAfterDashes: false,
-    keywords: sparkSQLKeywords,
+    keywords: SQLKeywords,
     types: processedSparkSQLType,
     builtin: processedSparkSQLBuiltin,
     operatorChars: '*/+-%<>!=~&|^',
diff --git a/querybook/webapp/lib/sql-helper/sql-setting.ts b/querybook/webapp/lib/sql-helper/sql-setting.ts
index eb8512018..202045843 100644
--- a/querybook/webapp/lib/sql-helper/sql-setting.ts
+++ b/querybook/webapp/lib/sql-helper/sql-setting.ts
@@ -10,10 +10,10 @@ export interface ILanguageSetting {
     quoteChars?: [quoteStart: string, quoteEnd: string];
 }
 
-export const SQL_KEYWORDS =
+const SQL_KEYWORDS =
     'alter and as asc between by count create delete desc distinct drop from group having in insert into is join like not on or order select set table union update values where limit';
 
-export const SettingsByLanguage: Record<string, ILanguageSetting> = {
+const SettingsByLanguage: Record<string, ILanguageSetting> = {
     hive: {
         keywords: new Set(
             // 2.2.0

From 46fa5c4b3da673b9a17969f6ae363df95a3dbb50 Mon Sep 17 00:00:00 2001
From: Charlie <cgu@pinterest.com>
Date: Tue, 19 Nov 2024 12:05:02 -0800
Subject: [PATCH 3/6] use xcode theme for light and add more coloring

---
 package.json                                  |  1 +
 .../components/QueryEditor/QueryEditor.tsx    |  8 +++++--
 .../webapp/components/QueryEditor/monokai.ts  |  7 ------
 .../webapp/components/QueryEditor/themes.ts   | 22 +++++++++++++++++++
 .../lib/codemirror/codemirror-language.ts     |  6 ++---
 yarn.lock                                     |  7 ++++++
 6 files changed, 39 insertions(+), 12 deletions(-)
 delete mode 100644 querybook/webapp/components/QueryEditor/monokai.ts
 create mode 100644 querybook/webapp/components/QueryEditor/themes.ts

diff --git a/package.json b/package.json
index 03ba9e16e..b12bc9117 100644
--- a/package.json
+++ b/package.json
@@ -41,6 +41,7 @@
         "@types/diff": "^5.0.2",
         "@uiw/codemirror-extensions-events": "^4.23.6",
         "@uiw/codemirror-theme-monokai": "^4.23.6",
+        "@uiw/codemirror-theme-xcode": "^4.23.6",
         "@uiw/codemirror-themes": "^4.23.5",
         "@uiw/react-codemirror": "^4.23.5",
         "@welldone-software/why-did-you-render": "^6.1.1",
diff --git a/querybook/webapp/components/QueryEditor/QueryEditor.tsx b/querybook/webapp/components/QueryEditor/QueryEditor.tsx
index 116d32313..362e43b1c 100644
--- a/querybook/webapp/components/QueryEditor/QueryEditor.tsx
+++ b/querybook/webapp/components/QueryEditor/QueryEditor.tsx
@@ -35,7 +35,7 @@ import { TableToken } from 'lib/sql-helper/sql-lexer';
 import { navigateWithinEnv } from 'lib/utils/query-string';
 import { IconButton } from 'ui/Button/IconButton';
 
-import { CustomMonokaiDarkTheme } from './monokai';
+import { CustomMonokaiDarkTheme, CustomXcodeTheme } from './themes';
 
 import './QueryEditor.scss';
 
@@ -436,7 +436,11 @@ export const QueryEditor: React.FC<
                 {floatButtons}
                 <CodeMirror
                     ref={editorRef}
-                    theme={theme === 'dark' ? CustomMonokaiDarkTheme : 'light'}
+                    theme={
+                        theme === 'dark'
+                            ? CustomMonokaiDarkTheme
+                            : CustomXcodeTheme
+                    }
                     className="ReactCodeMirror"
                     value={value}
                     height="100%"
diff --git a/querybook/webapp/components/QueryEditor/monokai.ts b/querybook/webapp/components/QueryEditor/monokai.ts
deleted file mode 100644
index f503ae272..000000000
--- a/querybook/webapp/components/QueryEditor/monokai.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import { monokaiInit } from '@uiw/codemirror-theme-monokai';
-import { tags as t } from '@lezer/highlight';
-
-export const CustomMonokaiDarkTheme = monokaiInit({
-    settings: {},
-    styles: [{ tag: [t.name], color: 'var(--text-dark)' }],
-});
diff --git a/querybook/webapp/components/QueryEditor/themes.ts b/querybook/webapp/components/QueryEditor/themes.ts
new file mode 100644
index 000000000..1986da1bf
--- /dev/null
+++ b/querybook/webapp/components/QueryEditor/themes.ts
@@ -0,0 +1,22 @@
+import { monokaiInit } from '@uiw/codemirror-theme-monokai';
+import { tags as t } from '@lezer/highlight';
+import { xcodeLightInit } from '@uiw/codemirror-theme-xcode';
+
+export const CustomMonokaiDarkTheme = monokaiInit({
+    settings: {},
+    styles: [
+        { tag: [t.name], color: 'var(--text-dark)' },
+        { tag: [t.constant(t.name), t.standard(t.name)], color: '#FD971F' },
+    ],
+});
+
+export const CustomXcodeTheme = xcodeLightInit({
+    settings: {
+        background: 'var(--bg-color)',
+        gutterBackground: 'var(--bg-color)',
+    },
+    styles: [
+        { tag: [t.special(t.propertyName)], color: '#327A9E' },
+        { tag: [t.constant(t.name), t.standard(t.name)], color: '#D23423' },
+    ],
+});
diff --git a/querybook/webapp/lib/codemirror/codemirror-language.ts b/querybook/webapp/lib/codemirror/codemirror-language.ts
index 06c68f3a9..2d9fec3e8 100644
--- a/querybook/webapp/lib/codemirror/codemirror-language.ts
+++ b/querybook/webapp/lib/codemirror/codemirror-language.ts
@@ -48,7 +48,7 @@ function builtInTypeSplit(
 }
 
 const trinoKeywords =
-    'abs absent acos add admin after all all_match alter analyze and any any_match approx_distinct approx_most_frequent approx_percentile approx_set arbitrary array_agg array_distinct array_except array_intersect array_join array_max array_min array_position array_remove array_sort array_union arrays_overlap as asc asin at at_timezone atan atan2 authorization avg bar bernoulli beta_cdf between bing_tile bing_tile_at bing_tile_coordinates bing_tile_polygon bing_tile_quadkey bing_tile_zoom_level bing_tiles_around bit_count bitwise_and bitwise_and_agg bitwise_left_shift bitwise_not bitwise_or bitwise_or_agg bitwise_right_shift bitwise_right_shift_arithmetic bitwise_xor bool_and bool_or both by call cardinality cascade case cast catalogs cbrt ceil ceiling char2hexint checksum chr classify coalesce codepoint column columns combinations comment commit committed concat concat_ws conditional constraint contains contains_sequence convex_hull_agg copartition corr cos cosh cosine_similarity count count_if covar_pop covar_samp crc32 create cross cube cume_dist current current_catalog current_date current_groups current_path current_role current_schema current_time current_timestamp current_timezone current_user data date_add date_diff date_format date_parse date_trunc day day_of_month day_of_week day_of_year deallocate default define definer degrees delete dense_rank deny desc describe descriptor distinct distributed dow doy drop e element_at else empty empty_approx_set encoding end error escape evaluate_classifier_predictions every except excluding execute exists exp explain extract false features fetch filter final first first_value flatten floor following for format format_datetime format_number from from_base from_base32 from_base64 from_base64url from_big_endian_32 from_big_endian_64 from_encoded_polyline from_geojson_geometry from_hex from_ieee754_32 from_ieee754_64 from_iso8601_date from_iso8601_timestamp from_iso8601_timestamp_nanos from_unixtime from_unixtime_nanos from_utf8 full functions geometric_mean geometry_from_hadoop_shape geometry_invalid_reason geometry_nearest_points geometry_to_bing_tiles geometry_union geometry_union_agg grant granted grants graphviz great_circle_distance greatest group grouping groups hamming_distance hash_counts having histogram hmac_md5 hmac_sha1 hmac_sha256 hmac_sha512 hour human_readable_seconds if ignore in including index infinity initial inner input insert intersect intersection_cardinality into inverse_beta_cdf inverse_normal_cdf invoker io is is_finite is_infinite is_json_scalar is_nan isolation jaccard_index join json_array json_array_contains json_array_get json_array_length json_exists json_extract json_extract_scalar json_format json_object json_parse json_query json_size json_value keep key keys kurtosis lag last last_day_of_month last_value lateral lead leading learn_classifier learn_libsvm_classifier learn_libsvm_regressor learn_regressor least left length level levenshtein_distance like limit line_interpolate_point line_interpolate_points line_locate_point listagg ln local localtime localtimestamp log log10 log2 logical lower lpad ltrim luhn_check make_set_digest map_agg map_concat map_entries map_filter map_from_entries map_keys map_union map_values map_zip_with match match_recognize matched matches materialized max max_by md5 measures merge merge_set_digest millisecond min min_by minute mod month multimap_agg multimap_from_entries murmur3 nan natural next nfc nfd nfkc nfkd ngrams no none none_match normal_cdf normalize not now nth_value ntile null nullif nulls numeric_histogram object objectid_timestamp of offset omit on one only option or order ordinality outer output over overflow parse_data_size parse_datetime parse_duration partition partitions passing past path pattern per percent_rank permute pi position pow power preceding prepare privileges properties prune qdigest_agg quarter quotes radians rand random range rank read recursive reduce reduce_agg refresh regexp_count regexp_extract regexp_extract_all regexp_like regexp_position regexp_replace regexp_split regr_intercept regr_slope regress rename render repeat repeatable replace reset respect restrict returning reverse revoke rgb right role roles rollback rollup round row_number rows rpad rtrim running scalar schema schemas second security seek select sequence serializable session set sets sha1 sha256 sha512 show shuffle sign simplify_geometry sin skewness skip slice some soundex spatial_partitioning spatial_partitions split split_part split_to_map split_to_multimap spooky_hash_v2_32 spooky_hash_v2_64 sqrt st_area st_asbinary st_astext st_boundary st_buffer st_centroid st_contains st_convexhull st_coorddim st_crosses st_difference st_dimension st_disjoint st_distance st_endpoint st_envelope st_envelopeaspts st_equals st_exteriorring st_geometries st_geometryfromtext st_geometryn st_geometrytype st_geomfrombinary st_interiorringn st_interiorrings st_intersection st_intersects st_isclosed st_isempty st_isring st_issimple st_isvalid st_length st_linefromtext st_linestring st_multipoint st_numgeometries st_numinteriorring st_numpoints st_overlaps st_point st_pointn st_points st_polygon st_relate st_startpoint st_symdifference st_touches st_union st_within st_x st_xmax st_xmin st_y st_ymax st_ymin start starts_with stats stddev stddev_pop stddev_samp string strpos subset substr substring sum system table tables tablesample tan tanh tdigest_agg text then ties timestamp_objectid timezone_hour timezone_minute to to_base to_base32 to_base64 to_base64url to_big_endian_32 to_big_endian_64 to_char to_date to_encoded_polyline to_geojson_geometry to_geometry to_hex to_ieee754_32 to_ieee754_64 to_iso8601 to_milliseconds to_spherical_geography to_timestamp to_unixtime to_utf8 trailing transaction transform transform_keys transform_values translate trim trim_array true truncate try try_cast type typeof uescape unbounded uncommitted unconditional union unique unknown unmatched unnest update upper url_decode url_encode url_extract_fragment url_extract_host url_extract_parameter url_extract_path url_extract_port url_extract_protocol url_extract_query use user using utf16 utf32 utf8 validate value value_at_quantile values values_at_quantiles var_pop var_samp variance verbose version view week week_of_year when where width_bucket wilson_interval_lower wilson_interval_upper window with with_timezone within without word_stem work wrapper write xxhash64 year year_of_week yow zip zip_with';
+    'abs absent acos add admin after all all_match alter analyze and any any_match approx_distinct approx_most_frequent approx_percentile approx_set arbitrary array_agg array_distinct array_except array_intersect array_join array_max array_min array_position array_remove array_sort array_union arrays_overlap as asc asin at at_timezone atan atan2 authorization avg bar bernoulli beta_cdf between bing_tile bing_tile_at bing_tile_coordinates bing_tile_polygon bing_tile_quadkey bing_tile_zoom_level bing_tiles_around bit_count bitwise_and bitwise_and_agg bitwise_left_shift bitwise_not bitwise_or bitwise_or_agg bitwise_right_shift bitwise_right_shift_arithmetic bitwise_xor bool_and bool_or both by call cardinality cascade case cast catalogs cbrt ceil ceiling char2hexint checksum chr classify coalesce codepoint column columns combinations comment commit committed concat concat_ws conditional constraint contains contains_sequence convex_hull_agg copartition corr cos cosh cosine_similarity count count_if covar_pop covar_samp crc32 create cross cube cume_dist current current_catalog current_date current_groups current_path current_role current_schema current_time current_timestamp current_timezone current_user data date_add date_diff date_format date_parse date_trunc day day_of_month day_of_week day_of_year deallocate default define definer degrees delete dense_rank deny desc describe descriptor distinct distributed dow doy drop element_at else empty empty_approx_set encoding end error escape evaluate_classifier_predictions every except excluding execute exists exp explain extract false features fetch filter final first first_value flatten floor following for format format_datetime format_number from from_base from_base32 from_base64 from_base64url from_big_endian_32 from_big_endian_64 from_encoded_polyline from_geojson_geometry from_hex from_ieee754_32 from_ieee754_64 from_iso8601_date from_iso8601_timestamp from_iso8601_timestamp_nanos from_unixtime from_unixtime_nanos from_utf8 full functions geometric_mean geometry_from_hadoop_shape geometry_invalid_reason geometry_nearest_points geometry_to_bing_tiles geometry_union geometry_union_agg grant granted grants graphviz great_circle_distance greatest group grouping groups hamming_distance hash_counts having histogram hmac_md5 hmac_sha1 hmac_sha256 hmac_sha512 hour human_readable_seconds if ignore in including index infinity initial inner input insert intersect intersection_cardinality into inverse_beta_cdf inverse_normal_cdf invoker io is is_finite is_infinite is_json_scalar is_nan isolation jaccard_index join json_array json_array_contains json_array_get json_array_length json_exists json_extract json_extract_scalar json_format json_object json_parse json_query json_size json_value keep key keys kurtosis lag last last_day_of_month last_value lateral lead leading learn_classifier learn_libsvm_classifier learn_libsvm_regressor learn_regressor least left length level levenshtein_distance like limit line_interpolate_point line_interpolate_points line_locate_point listagg ln local localtime localtimestamp log log10 log2 logical lower lpad ltrim luhn_check make_set_digest map_agg map_concat map_entries map_filter map_from_entries map_keys map_union map_values map_zip_with match match_recognize matched matches materialized max max_by md5 measures merge merge_set_digest millisecond min min_by minute mod month multimap_agg multimap_from_entries murmur3 nan natural next nfc nfd nfkc nfkd ngrams no none none_match normal_cdf normalize not now nth_value ntile null nullif nulls numeric_histogram object objectid_timestamp of offset omit on one only option or order ordinality outer output over overflow parse_data_size parse_datetime parse_duration partition partitions passing past path pattern per percent_rank permute pi position pow power preceding prepare privileges properties prune qdigest_agg quarter quotes radians rand random range rank read recursive reduce reduce_agg refresh regexp_count regexp_extract regexp_extract_all regexp_like regexp_position regexp_replace regexp_split regr_intercept regr_slope regress rename render repeat repeatable replace reset respect restrict returning reverse revoke rgb right role roles rollback rollup round row_number rows rpad rtrim running scalar schema schemas second security seek select sequence serializable session set sets sha1 sha256 sha512 show shuffle sign simplify_geometry sin skewness skip slice some soundex spatial_partitioning spatial_partitions split split_part split_to_map split_to_multimap spooky_hash_v2_32 spooky_hash_v2_64 sqrt st_area st_asbinary st_astext st_boundary st_buffer st_centroid st_contains st_convexhull st_coorddim st_crosses st_difference st_dimension st_disjoint st_distance st_endpoint st_envelope st_envelopeaspts st_equals st_exteriorring st_geometries st_geometryfromtext st_geometryn st_geometrytype st_geomfrombinary st_interiorringn st_interiorrings st_intersection st_intersects st_isclosed st_isempty st_isring st_issimple st_isvalid st_length st_linefromtext st_linestring st_multipoint st_numgeometries st_numinteriorring st_numpoints st_overlaps st_point st_pointn st_points st_polygon st_relate st_startpoint st_symdifference st_touches st_union st_within st_x st_xmax st_xmin st_y st_ymax st_ymin start starts_with stats stddev stddev_pop stddev_samp string strpos subset substr substring sum system table tables tablesample tan tanh tdigest_agg text then ties timestamp_objectid timezone_hour timezone_minute to to_base to_base32 to_base64 to_base64url to_big_endian_32 to_big_endian_64 to_char to_date to_encoded_polyline to_geojson_geometry to_geometry to_hex to_ieee754_32 to_ieee754_64 to_iso8601 to_milliseconds to_spherical_geography to_timestamp to_unixtime to_utf8 trailing transaction transform transform_keys transform_values translate trim trim_array true truncate try try_cast type typeof uescape unbounded uncommitted unconditional union unique unknown unmatched unnest update upper url_decode url_encode url_extract_fragment url_extract_host url_extract_parameter url_extract_path url_extract_port url_extract_protocol url_extract_query use user using utf16 utf32 utf8 validate value value_at_quantile values values_at_quantiles var_pop var_samp variance verbose version view week week_of_year when where width_bucket wilson_interval_lower wilson_interval_upper window with with_timezone within without word_stem work wrapper write xxhash64 year year_of_week yow zip zip_with';
 const trinoBuiltin =
     'array bigint bingtile boolean char codepoints color date decimal double function geometry hyperloglog int integer interval ipaddress joniregexp json json2016 jsonpath kdbtree likepattern map model objectid p4hyperloglog precision qdigest re2jregexp real regressor row setdigest smallint sphericalgeography tdigest time timestamp tinyint uuid varbinary varchar zone';
 
@@ -72,9 +72,9 @@ const TrinoSQL = SQLDialect.define({
 });
 
 const sparkSQLKeywords =
-    'add after all alter analyze and anti archive array as asc at between bucket buckets by cache cascade case cast change clear cluster clustered codegen collection column columns comment commit compact compactions compute concatenate cost create cross cube current current_date current_timestamp database databases data dbproperties defined delete delimited deny desc describe dfs directories distinct distribute drop else end escaped except exchange exists explain export extended external false fields fileformat first following for format formatted from full function functions global grant group grouping having if ignore import in index indexes inner inpath inputformat insert intersect interval into is items join keys last lateral lazy left like limit lines list load local location lock locks logical macro map minus msck natural no not null nulls of on optimize option options or order out outer outputformat over overwrite partition partitioned partitions percent preceding principals purge range recordreader recordwriter recover reduce refresh regexp rename repair replace reset restrict revoke right rlike role roles rollback rollup row rows schema schemas select semi separated serde serdeproperties set sets show skewed sort sorted start statistics stored stratify struct table tables tablesample tblproperties temp temporary terminated then to touch transaction transactions transform true truncate unarchive unbounded uncache union unlock unset use using values view when where window with';
+    'add after all alter analyze and anti archive array as asc at between bucket buckets by cache cascade case cast change clear cluster clustered codegen collection column columns comment commit compact compactions compute concatenate cost create cross cube current current_date current_timestamp database databases data dbproperties defined delete delimited deny desc describe dfs directories distinct distribute drop else end escaped except exchange exists explain export extended external false fields fileformat first following for format formatted from full function functions global grant group grouping having if ignore import in index indexes inner inpath inputformat insert intersect interval into is items jar join keys last lateral lazy left like limit lines list load local location lock locks logical macro map minus msck natural no not null nulls of on optimize option options or order out outer outputformat over overwrite partition partitioned partitions percent preceding principals purge range recordreader recordwriter recover reduce refresh regexp rename repair replace reset restrict revoke right rlike role roles rollback rollup row rows schema schemas select semi separated serde serdeproperties set sets show skewed sort sorted start statistics stored stratify struct table tables tablesample tblproperties temp temporary terminated then to touch transaction transactions transform true truncate unarchive unbounded uncache union unlock unset use using values view when where window with';
 const sparkSQLBuiltin =
-    'abs acos acosh add_months aggregate and any approx_count_distinct approx_percentile array array_contains array_distinct array_except array_intersect array_join array_max array_min array_position array_remove array_repeat array_sort array_union arrays_overlap arrays_zip ascii asin asinh assert_true atan atan2 atanh avg base64 between bigint bin binary bit_and bit_count bit_get bit_length bit_or bit_xor bool_and bool_or boolean bround btrim cardinality case cast cbrt ceil ceiling char char_length character_length chr coalesce collect_list collect_set concat concat_ws conv corr cos cosh cot count count_if count_min_sketch covar_pop covar_samp crc32 cume_dist current_catalog current_database current_date current_timestamp current_timezone current_user date date_add date_format date_from_unix_date date_part date_sub date_trunc datediff day dayofmonth dayofweek dayofyear decimal decode degrees delimited dense_rank div double e element_at elt encode every exists exp explode explode_outer expm1 extract factorial filter find_in_set first first_value flatten float floor forall format_number format_string from_csv from_json from_unixtime from_utc_timestamp get_json_object getbit greatest grouping grouping_id hash hex hour hypot if ifnull in initcap inline inline_outer input_file_block_length input_file_block_start input_file_name inputformat instr int isnan isnotnull isnull java_method json_array_length json_object_keys json_tuple kurtosis lag last last_day last_value lcase lead least left length levenshtein like ln locate log log10 log1p log2 lower lpad ltrim make_date make_dt_interval make_interval make_timestamp make_ym_interval map map_concat map_entries map_filter map_from_arrays map_from_entries map_keys map_values map_zip_with max max_by md5 mean min min_by minute mod monotonically_increasing_id month months_between named_struct nanvl negative next_day not now nth_value ntile nullif nvl nvl2 octet_length or outputformat overlay parse_url percent_rank percentile percentile_approx pi pmod posexplode posexplode_outer position positive pow power printf quarter radians raise_error rand randn random rank rcfile reflect regexp regexp_extract regexp_extract_all regexp_like regexp_replace repeat replace reverse right rint rlike round row_number rpad rtrim schema_of_csv schema_of_json second sentences sequence sequencefile serde session_window sha sha1 sha2 shiftleft shiftright shiftrightunsigned shuffle sign signum sin sinh size skewness slice smallint some sort_array soundex space spark_partition_id split sqrt stack std stddev stddev_pop stddev_samp str_to_map string struct substr substring substring_index sum tan tanh textfile timestamp timestamp_micros timestamp_millis timestamp_seconds tinyint to_csv to_date to_json to_timestamp to_unix_timestamp to_utc_timestamp transform transform_keys transform_values translate trim trunc try_add try_divide typeof ucase unbase64 unhex uniontype unix_date unix_micros unix_millis unix_seconds unix_timestamp upper uuid var_pop var_samp variance version weekday weekofyear when width_bucket window xpath xpath_boolean xpath_double xpath_float xpath_int xpath_long xpath_number xpath_short xpath_string xxhash64 year zip_with';
+    'abs acos acosh add_months aggregate and any approx_count_distinct approx_percentile array array_contains array_distinct array_except array_intersect array_join array_max array_min array_position array_remove array_repeat array_sort array_union arrays_overlap arrays_zip ascii asin asinh assert_true atan atan2 atanh avg base64 between bigint bin binary bit_and bit_count bit_get bit_length bit_or bit_xor bool_and bool_or boolean bround btrim cardinality case cast cbrt ceil ceiling char char_length character_length chr coalesce collect_list collect_set concat concat_ws conv corr cos cosh cot count count_if count_min_sketch covar_pop covar_samp crc32 cume_dist current_catalog current_database current_date current_timestamp current_timezone current_user date date_add date_format date_from_unix_date date_part date_sub date_trunc datediff day dayofmonth dayofweek dayofyear decimal decode degrees delimited dense_rank div double element_at elt encode every exists exp explode explode_outer expm1 extract factorial filter find_in_set first first_value flatten float floor forall format_number format_string from_csv from_json from_unixtime from_utc_timestamp get_json_object getbit greatest grouping grouping_id hash hex hour hypot if ifnull in initcap inline inline_outer input_file_block_length input_file_block_start input_file_name inputformat instr int isnan isnotnull isnull java_method json_array_length json_object_keys json_tuple kurtosis lag last last_day last_value lcase lead least left length levenshtein like ln locate log log10 log1p log2 lower lpad ltrim make_date make_dt_interval make_interval make_timestamp make_ym_interval map map_concat map_entries map_filter map_from_arrays map_from_entries map_keys map_values map_zip_with max max_by md5 mean min min_by minute mod monotonically_increasing_id month months_between named_struct nanvl negative next_day not now nth_value ntile nullif nvl nvl2 octet_length or outputformat overlay parse_url percent_rank percentile percentile_approx pi pmod posexplode posexplode_outer position positive pow power printf quarter radians raise_error rand randn random rank rcfile reflect regexp regexp_extract regexp_extract_all regexp_like regexp_replace repeat replace reverse right rint rlike round row_number rpad rtrim schema_of_csv schema_of_json second sentences sequence sequencefile serde session_window sha sha1 sha2 shiftleft shiftright shiftrightunsigned shuffle sign signum sin sinh size skewness slice smallint some sort_array soundex space spark_partition_id split sqrt stack std stddev stddev_pop stddev_samp str_to_map string struct substr substring substring_index sum tan tanh textfile timestamp timestamp_micros timestamp_millis timestamp_seconds tinyint to_csv to_date to_json to_timestamp to_unix_timestamp to_utc_timestamp transform transform_keys transform_values translate trim trunc try_add try_divide typeof ucase unbase64 unhex uniontype unix_date unix_micros unix_millis unix_seconds unix_timestamp upper uuid var_pop var_samp variance version weekday weekofyear when width_bucket window xpath xpath_boolean xpath_double xpath_float xpath_int xpath_long xpath_number xpath_short xpath_string xxhash64 year zip_with';
 
 const [processedSparkSQLBuiltin, processedSparkSQLType] = builtInTypeSplit(
     'sparksql',
diff --git a/yarn.lock b/yarn.lock
index 804259673..05455c581 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -6257,6 +6257,13 @@
   dependencies:
     "@uiw/codemirror-themes" "4.23.6"
 
+"@uiw/codemirror-theme-xcode@^4.23.6":
+  version "4.23.6"
+  resolved "https://registry.yarnpkg.com/@uiw/codemirror-theme-xcode/-/codemirror-theme-xcode-4.23.6.tgz#fdc070f743088d9792e1034194b3b4b639e9c410"
+  integrity sha512-t4shD5Cq6e1mqzG6w/+oMCw3oXn6QSbYZbCHDZ7GrbN51f6ZttKVOSZgUInlNeh9Spbvag8ZE5osMFFv6+twcQ==
+  dependencies:
+    "@uiw/codemirror-themes" "4.23.6"
+
 "@uiw/codemirror-themes@4.23.6":
   version "4.23.6"
   resolved "https://registry.yarnpkg.com/@uiw/codemirror-themes/-/codemirror-themes-4.23.6.tgz#47a101733a9c4aa382696178bc4b7bc0ecf0e5fa"

From 0f2dad376acbc0a7e5ea50a05e4c51f29f7add58 Mon Sep 17 00:00:00 2001
From: Charlie <cgu@pinterest.com>
Date: Tue, 19 Nov 2024 12:56:14 -0800
Subject: [PATCH 4/6] fix lint

---
 querybook/webapp/lib/codemirror/codemirror-language.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/querybook/webapp/lib/codemirror/codemirror-language.ts b/querybook/webapp/lib/codemirror/codemirror-language.ts
index 2d9fec3e8..509d1928d 100644
--- a/querybook/webapp/lib/codemirror/codemirror-language.ts
+++ b/querybook/webapp/lib/codemirror/codemirror-language.ts
@@ -24,9 +24,9 @@ function builtInTypeSplit(
     /**
      * This is needed because codemirror 5 mixes type and builtin keywords together.
      * In our case, we want:
-     *  - keywords: standard SQL keywords, from SQLKeywords
-     *  - builtin: functions, operators, etc
-     *  - type: data types
+     * - keywords: standard SQL keywords, from SQLKeywords
+     * - builtin: functions, operators, etc
+     * - type: data types
      *
      * returns [builtIn, Type]
      */

From d840712238feb32b346d9deab8134a45259e5df7 Mon Sep 17 00:00:00 2001
From: Charlie <cgu@pinterest.com>
Date: Tue, 19 Nov 2024 14:28:57 -0800
Subject: [PATCH 5/6] updated color for light theme

---
 .../CodeMirrorTooltip/FunctionDocumentationTooltip.tsx    | 4 ++--
 querybook/webapp/components/QueryEditor/QueryEditor.tsx   | 1 +
 querybook/webapp/components/QueryEditor/themes.ts         | 3 ++-
 .../queryEditor/extensions/useHoverTooltipExtension.tsx   | 8 +++++---
 4 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/querybook/webapp/components/CodeMirrorTooltip/FunctionDocumentationTooltip.tsx b/querybook/webapp/components/CodeMirrorTooltip/FunctionDocumentationTooltip.tsx
index 7a0803213..fffe9002c 100644
--- a/querybook/webapp/components/CodeMirrorTooltip/FunctionDocumentationTooltip.tsx
+++ b/querybook/webapp/components/CodeMirrorTooltip/FunctionDocumentationTooltip.tsx
@@ -1,4 +1,4 @@
-import React, { useCallback, useEffect } from 'react';
+import React, { useEffect } from 'react';
 import { useDispatch, useSelector } from 'react-redux';
 
 import { IFunctionDescription } from 'const/metastore';
@@ -59,7 +59,7 @@ export const FunctionDocumentationTooltipByName: React.FunctionComponent<{
         if (language) {
             dispatch(fetchFunctionDocumentationIfNeeded(language));
         }
-    }, [language]);
+    }, [dispatch, language]);
 
     const functionDefs = functionDocumentationByNameByLanguage?.[language];
     const functionNameLower = (functionName || '').toLowerCase();
diff --git a/querybook/webapp/components/QueryEditor/QueryEditor.tsx b/querybook/webapp/components/QueryEditor/QueryEditor.tsx
index 362e43b1c..857949635 100644
--- a/querybook/webapp/components/QueryEditor/QueryEditor.tsx
+++ b/querybook/webapp/components/QueryEditor/QueryEditor.tsx
@@ -295,6 +295,7 @@ export const QueryEditor: React.FC<
             useHoverTooltipExtension({
                 codeAnalysisRef,
                 metastoreId: 1,
+                language,
             });
 
         const openTableModalCommand = useCallback((editorView: EditorView) => {
diff --git a/querybook/webapp/components/QueryEditor/themes.ts b/querybook/webapp/components/QueryEditor/themes.ts
index 1986da1bf..b2d21a827 100644
--- a/querybook/webapp/components/QueryEditor/themes.ts
+++ b/querybook/webapp/components/QueryEditor/themes.ts
@@ -16,7 +16,8 @@ export const CustomXcodeTheme = xcodeLightInit({
         gutterBackground: 'var(--bg-color)',
     },
     styles: [
-        { tag: [t.special(t.propertyName)], color: '#327A9E' },
+        { tag: [t.special(t.propertyName)], color: '#005cc5' },
         { tag: [t.constant(t.name), t.standard(t.name)], color: '#D23423' },
+        { tag: [t.number], color: '#098658' },
     ],
 });
diff --git a/querybook/webapp/hooks/queryEditor/extensions/useHoverTooltipExtension.tsx b/querybook/webapp/hooks/queryEditor/extensions/useHoverTooltipExtension.tsx
index 5edfb72a3..c5677521c 100644
--- a/querybook/webapp/hooks/queryEditor/extensions/useHoverTooltipExtension.tsx
+++ b/querybook/webapp/hooks/queryEditor/extensions/useHoverTooltipExtension.tsx
@@ -16,9 +16,11 @@ import { reduxStore } from 'redux/store';
 export const useHoverTooltipExtension = ({
     codeAnalysisRef,
     metastoreId,
+    language,
 }: {
     codeAnalysisRef: MutableRefObject<ICodeAnalysis>;
     metastoreId: number;
+    language: string;
 }) => {
     const getTableAtV5Position = useCallback(
         (codeAnalysis, v5Pos: { line: number; ch: number }) => {
@@ -57,7 +59,7 @@ export const useHoverTooltipExtension = ({
             const v5Pos = offsetToPos(editorView, selection.from);
             return getTableAtV5Position(codeAnalysisRef.current, v5Pos);
         },
-        [getTableAtV5Position]
+        [codeAnalysisRef, getTableAtV5Position]
     );
 
     const getHoverTooltips: HoverTooltipSource = useCallback(
@@ -80,7 +82,7 @@ export const useHoverTooltipExtension = ({
             } else if (nextChar === '(') {
                 tooltipComponent = (
                     <FunctionDocumentationTooltipByName
-                        language="sqlite"
+                        language={language}
                         functionName={token.text}
                     />
                 );
@@ -107,7 +109,7 @@ export const useHoverTooltipExtension = ({
                 },
             };
         },
-        []
+        [codeAnalysisRef, getTableAtV5Position, language, metastoreId]
     );
 
     const extension = useMemo(

From cf8c7f1e04784c32777d4b7224a8a5637500c863 Mon Sep 17 00:00:00 2001
From: Charlie <cgu@pinterest.com>
Date: Tue, 19 Nov 2024 14:38:17 -0800
Subject: [PATCH 6/6] comments

---
 .../components/QueryEditor/QueryEditor.tsx    | 33 +++++++++++--------
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/querybook/webapp/components/QueryEditor/QueryEditor.tsx b/querybook/webapp/components/QueryEditor/QueryEditor.tsx
index 857949635..c3d9bbd29 100644
--- a/querybook/webapp/components/QueryEditor/QueryEditor.tsx
+++ b/querybook/webapp/components/QueryEditor/QueryEditor.tsx
@@ -294,23 +294,28 @@ export const QueryEditor: React.FC<
         const { extension: hoverTooltipExtension, getTableAtCursor } =
             useHoverTooltipExtension({
                 codeAnalysisRef,
-                metastoreId: 1,
+                metastoreId,
                 language,
             });
 
-        const openTableModalCommand = useCallback((editorView: EditorView) => {
-            const table = getTableAtCursor(editorView);
-            if (table) {
-                getTableByName(table.schema, table.name).then((tableInfo) => {
-                    if (tableInfo) {
-                        navigateWithinEnv(`/table/${tableInfo.id}/`, {
-                            isModal: true,
-                        });
-                    }
-                });
-            }
-            return true;
-        }, []);
+        const openTableModalCommand = useCallback(
+            (editorView: EditorView) => {
+                const table = getTableAtCursor(editorView);
+                if (table) {
+                    getTableByName(table.schema, table.name).then(
+                        (tableInfo) => {
+                            if (tableInfo) {
+                                navigateWithinEnv(`/table/${tableInfo.id}/`, {
+                                    isModal: true,
+                                });
+                            }
+                        }
+                    );
+                }
+                return true;
+            },
+            [getTableAtCursor, getTableByName]
+        );
 
         const keyBindings = useMemo(
             () => [