Merge pull request #41 from quickwit-oss/feat_aggregation_filter

Issue #20: add dynamic fields
quickwit-oss · Dec 28, 2023 · dac4ba6 · dac4ba6
2 parents 67f56ba + debd8ae
commit dac4ba6
Show file tree

Hide file tree

Showing 7 changed files with 81 additions and 45 deletions.
diff --git a/pkg/quickwit/quickwit.go b/pkg/quickwit/quickwit.go
@@ -143,7 +143,9 @@ func (ds *QuickwitDatasource) CallResource(ctx context.Context, req *backend.Cal
 	// - empty string for fetching db version
 	// - ?/_mapping for fetching index mapping
 	// - _msearch for executing getTerms queries
-	if req.Path != "" && !strings.Contains(req.Path, "indexes/") && req.Path != "_elastic/_msearch" {
+	// - _field_caps for getting all the aggregeables fields
+	var isFieldCaps = req.Path != "" && strings.Contains(req.Path, "_elastic") && strings.Contains(req.Path, "/_field_caps")
+	if req.Path != "" && !strings.Contains(req.Path, "indexes/") && req.Path != "_elastic/_msearch" && !isFieldCaps {
 		return fmt.Errorf("invalid resource URL: %s", req.Path)
 	}
 

diff --git a/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx b/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx
@@ -31,7 +31,7 @@ describe('Metric Editor', () => {
 
     const wrapper = ({ children }: PropsWithChildren<{}>) => (
       <ElasticsearchProvider
-        datasource={{ getFields } as ElasticDatasource}
+        datasource={{ getFields: getFields } as ElasticDatasource}
         query={query}
         app={undefined}
         range={getDefaultTimeRange()}

diff --git a/src/datasource.ts b/src/datasource.ts
@@ -34,7 +34,7 @@ import {
   SupplementaryQueryType,
   TimeRange,
 } from '@grafana/data';
-import { BucketAggregation, DataLinkConfig, ElasticsearchQuery, Field as QuickwitField, FieldMapping, IndexMetadata, Logs, TermsQuery } from './types';
+import { BucketAggregation, DataLinkConfig, ElasticsearchQuery, Field as QuickwitField, FieldMapping, IndexMetadata, Logs, TermsQuery, FieldCapabilitiesResponse } from './types';
 import { 
   DataSourceWithBackend, 
   getTemplateSrv, 
@@ -51,6 +51,7 @@ import { bucketAggregationConfig } from 'components/QueryEditor/BucketAggregatio
 import { isBucketAggregationWithField } from 'components/QueryEditor/BucketAggregationsEditor/aggregations';
 import ElasticsearchLanguageProvider from 'LanguageProvider';
 import { ReactNode } from 'react';
+import { fieldTypeMap } from 'utils';
 
 export const REF_ID_STARTER_LOG_VOLUME = 'log-volume-';
 
@@ -338,36 +339,35 @@ export class QuickwitDataSource
     );
   }
 
-  // TODO: instead of being a string, this could be a custom type representing all the elastic types
-  // FIXME: This doesn't seem to return actual MetricFindValues, we should either change the return type
-  // or fix the implementation.
-  getFields(type?: string[], _range?: TimeRange): Observable<MetricFindValue[]> {
-    const typeMap: Record<string, string> = {
-      u64: 'number',
-      i64: 'number',
-      datetime: 'date',
-      text: 'string',
-    };
-    return from(this.getResource('indexes/' + this.index)).pipe(
-      map((index_metadata) => {
-        const shouldAddField = (field: QuickwitField) => {
-          const translated_type = typeMap[field.field_mapping.type];
+  getFields(aggregatable?: boolean, type?: string[], _range?: TimeRange): Observable<MetricFindValue[]> {
+    // TODO: use the time range.
+    return from(this.getResource('_elastic/' + this.index + '/_field_caps')).pipe(
+      map((field_capabilities_response: FieldCapabilitiesResponse) => {
+        const shouldAddField = (field: any) => {
+          if (aggregatable !== undefined && field.aggregatable !== aggregatable) {
+            return false
+          }
           if (type?.length === 0) {
             return true;
           }
-          return type?.includes(translated_type);
+          return type?.includes(field.type) || type?.includes(fieldTypeMap[field.type]);
         };
-
-        const fields = getAllFields(index_metadata.index_config.doc_mapping.field_mappings);
-        const filteredFields = fields.filter(shouldAddField);
-
-        // transform to array
-        return _map(filteredFields, (field) => {
-          return {
-            text: field.json_path,
-            value: typeMap[field.field_mapping.type],
-          };
-        });
+        const fieldCapabilities = Object.entries(field_capabilities_response.fields)
+          .flatMap(([field_name, field_capabilities]) => {
+            return Object.values(field_capabilities)
+              .map(field_capability => {
+                field_capability.field_name = field_name;
+                return field_capability;
+              });
+          })
+          .filter(shouldAddField)
+          .map(field_capability => {
+            return {
+              text: field_capability.field_name,
+              value: fieldTypeMap[field_capability.type],  
+            }
+          });
+        return fieldCapabilities;
       })
     );
   }
@@ -376,7 +376,7 @@ export class QuickwitDataSource
    * Get tag keys for adhoc filters
    */
   getTagKeys() {
-    return lastValueFrom(this.getFields());
+    return lastValueFrom(this.getFields(true));
   }
 
   /**
@@ -523,12 +523,10 @@ export class QuickwitDataSource
     const range = options?.range;
     const parsedQuery = JSON.parse(query);
     if (query) {
-      // Interpolation of variables with a list of values for which we don't
-      // know the field name is not supported yet.
-      // if (parsedQuery.find === 'fields') {
-      //   parsedQuery.type = this.interpolateLuceneQuery(parsedQuery.type);
-      //   return lastValueFrom(this.getFields(parsedQuery.type, range));
-      // }
+      if (parsedQuery.find === 'fields') {
+        parsedQuery.type = this.interpolateLuceneQuery(parsedQuery.type);
+        return lastValueFrom(this.getFields(true, parsedQuery.type, range));
+      }
       if (parsedQuery.find === 'terms') {
         parsedQuery.field = this.interpolateLuceneQuery(parsedQuery.field);
         parsedQuery.query = this.interpolateLuceneQuery(parsedQuery.query);

diff --git a/src/hooks/useFields.test.tsx b/src/hooks/useFields.test.tsx
@@ -27,7 +27,7 @@ describe('useFields hook', () => {
 
     const wrapper = ({ children }: PropsWithChildren<{}>) => (
       <ElasticsearchProvider
-        datasource={{ getFields } as ElasticDatasource}
+        datasource={{ getFields: getFields } as ElasticDatasource}
         query={query}
         app={undefined}
         range={timeRange}
@@ -48,39 +48,39 @@ describe('useFields hook', () => {
       { wrapper, initialProps: 'cardinality' }
     );
     result.current();
-    expect(getFields).toHaveBeenLastCalledWith([], timeRange);
+    expect(getFields).toHaveBeenLastCalledWith(true, [], timeRange);
 
     // All other metric aggregations only work on numbers
     rerender('avg');
     result.current();
-    expect(getFields).toHaveBeenLastCalledWith(['number'], timeRange);
+    expect(getFields).toHaveBeenLastCalledWith(true, ['number'], timeRange);
 
     //
     // BUCKET AGGREGATIONS
     //
     // Date Histrogram only works on dates
     rerender('date_histogram');
     result.current();
-    expect(getFields).toHaveBeenLastCalledWith(['date'], timeRange);
+    expect(getFields).toHaveBeenLastCalledWith(true, ['date'], timeRange);
 
     // Histrogram only works on numbers
     rerender('histogram');
     result.current();
-    expect(getFields).toHaveBeenLastCalledWith(['number'], timeRange);
+    expect(getFields).toHaveBeenLastCalledWith(true, ['number'], timeRange);
 
     // Geohash Grid only works on geo_point data
     rerender('geohash_grid');
     result.current();
-    expect(getFields).toHaveBeenLastCalledWith(['geo_point'], timeRange);
+    expect(getFields).toHaveBeenLastCalledWith(true, ['geo_point'], timeRange);
 
     // All other bucket aggregation work on any kind of data
     rerender('terms');
     result.current();
-    expect(getFields).toHaveBeenLastCalledWith([], timeRange);
+    expect(getFields).toHaveBeenLastCalledWith(true, [], timeRange);
 
     // top_metrics work on only on numeric data in 7.7
     rerender('top_metrics');
     result.current();
-    expect(getFields).toHaveBeenLastCalledWith(['number'], timeRange);
+    expect(getFields).toHaveBeenLastCalledWith(true, ['number'], timeRange);
   });
 });
diff --git a/src/hooks/useFields.ts b/src/hooks/useFields.ts
@@ -62,7 +62,7 @@ export const useFields = (type: AggregationType | string[]) => {
   return async (q?: string) => {
     // _mapping doesn't support filtering, we avoid sending a request everytime q changes
     if (!rawFields) {
-      rawFields = await lastValueFrom(datasource.getFields(filter, range));
+      rawFields = await lastValueFrom(datasource.getFields(true, filter, range));
     }
 
     return rawFields.filter(({ text }) => q === undefined || text.includes(q)).map(toSelectableValue);

diff --git a/src/types.ts b/src/types.ts
@@ -150,3 +150,23 @@ export type Field = {
   path_segments: string[];
   field_mapping: FieldMapping;
 }
+
+export type FieldCapabilityType = "long" | "keyword" | "text" | "date" | "date_nanos" | "binary" | "double" | "boolean" | "ip" | "nested" | "object" ;
+
+export type FieldCapability = {
+  field_name: string; // Field not present in response but added on the front side.
+  type: FieldCapabilityType;
+  metadata_field: boolean;
+  searchable: boolean;
+  aggregatable: boolean;
+  indices: String[];
+}
+
+export type FieldCapabilitiesResponse = {
+  indices: String[];
+  fields: {
+    [key: string]: {
+      [key in FieldCapabilityType]: FieldCapability;
+    }
+  };
+}
diff --git a/src/utils.ts b/src/utils.ts
@@ -101,3 +101,19 @@ export const getScriptValue = (metric: MetricAggregationWithInlineScript) =>
 
 export const unsupportedVersionMessage =
   'Support for Elasticsearch versions after their end-of-life (currently versions < 7.16) was removed. Using unsupported version of Elasticsearch may lead to unexpected and incorrect results.';
+
+export const fieldTypeMap: Record<string, string> = {
+  date: 'date',
+  date_nanos: 'date',
+  keyword: 'string',
+  text: 'string',
+  binary: 'string',
+  byte: 'number',
+  long: 'number',
+  unsigned_long: 'number',
+  double: 'number',
+  integer: 'number',
+  short: 'number',
+  float: 'number',
+  scaled_float: 'number'
+};