Add option to specify default flexSearch case-sensitivity

.github/workflows/ci.yml

@@ -6,7 +6,7 @@ jobs:
         strategy:
             matrix:
                 arango-image: ['arangodb:3.6', 'arangodb:3.7', 'arangodb:3.8']
-                node-version: [12.x, 14.x, 16.x]
+                node-version: [14.x, 16.x]
         steps:
             - uses: actions/checkout@v2
             - name: Use Node.js ${{ matrix.node-version }}

core-exports.ts

@@ -1,4 +1,4 @@
-export { RequestProfile, ProjectOptions, RequestContext, ModelValidationOptions } from './src/config/interfaces';
+export { RequestProfile, ProjectOptions, RequestContext, ModelOptions } from './src/config/interfaces';
 export { FieldResolverParameters } from './src/graphql/operation-based-resolvers';
 export { Project, ProjectConfig } from './src/project/project';
 export { InvalidProjectError } from './src/project/invalid-project-error';

spec/regression/papers/model/paper.graphqls

@@ -6,8 +6,11 @@ enum Category {
     Programming
 }
 
+# need to specify flexSearchOrder with an id field so createdAt does not included (which we cannot test because it changes)
 "A scientific paper"
-type Paper @rootEntity(flexSearch: true, flexSearchLanguage: EN) @roles(readWrite: ["admin"]) {
+type Paper
+    @rootEntity(flexSearch: true, flexSearchLanguage: EN, flexSearchOrder: [{ field: "id", direction: ASC }])
+    @roles(readWrite: ["admin"]) {
     key: String @key @flexSearch # need a key field for the reference
     title: String @index @flexSearch # for pagination performance test
     "The date this paper has been published in a scientific journal or conference"

spec/regression/regression-suite.ts

@@ -71,7 +71,7 @@ export class RegressionSuite {
                 authRoles: context.authRoles,
                 flexSearchMaxFilterableAndSortableAmount: context.flexSearchMaxFilterableAndSortableAmount
             }),
-            modelValidationOptions: {
+            modelOptions: {
                 forbiddenRootEntityNames: []
             },
             ...options,

src/config/interfaces.ts

@@ -25,20 +25,25 @@ export interface SchemaOptions {
     readonly maxOrderByRootEntityDepth?: number;
 }
 
-export interface ModelValidationOptions {
-    /**
-     * A list of root entity names that are not allowed.
-     */
-    readonly forbiddenRootEntityNames?: ReadonlyArray<string>;
-}
-
 export interface ModelOptions {
     /**
      * Determines whether a slash in a source name indicates the target namespace for that source
      *
      * Defaults to true. Explicitly specify false to disable this.
      */
     readonly useSourceDirectoriesAsNamespaces?: boolean;
+
+    /**
+     * Specifies the default for case-sensitiveness of flexSearch fields (can be overridden with the decorator)
+     *
+     * Default is false
+     */
+    readonly isFlexSearchIndexCaseSensitiveByDefault?: boolean;
+
+    /**
+     * A list of root entity names that are not allowed.
+     */
+    readonly forbiddenRootEntityNames?: ReadonlyArray<string>;
 }
 
 export interface ProjectOptions {
@@ -47,7 +52,7 @@ export interface ProjectOptions {
     readonly getExecutionOptions?: (args: ExecutionOptionsCallbackArgs) => ExecutionOptions;
 
     readonly schemaOptions?: SchemaOptions;
-    readonly modelValidationOptions?: ModelValidationOptions;
+
     readonly modelOptions?: ModelOptions;
 
     /**

src/database/inmemory/js-generator.ts

@@ -964,11 +964,11 @@ register(OperatorWithAnalyzerQueryNode, (node, context) => {
     let rhs = processNode(node.rhs, context);
 
     if (isCaseInsensitive) {
-        lhs = js`${lhs}.toLowerCase()`;
+        lhs = js`(${lhs})?.toLowerCase()`;
         const rhsVar = js.variable('rhs');
         rhs = jsExt.evaluatingLambda(
             rhsVar,
-            js`(Array.isArray(${rhsVar}) ? ${rhsVar}.map(value => value.toLowerCase()) : ${rhsVar}.toLowerCase())`,
+            js`(Array.isArray(${rhsVar}) ? ${rhsVar}.map(value => value?.toLowerCase()) : (${rhsVar})?.toLowerCase())`,
             rhs
         );
     }

src/model/config/indices.ts

@@ -1,4 +1,5 @@
 import { DirectiveNode, ObjectValueNode, StringValueNode } from 'graphql';
+import { OrderDirection } from '../implementation/order';
 
 export interface IndexDefinitionConfig {
     readonly name?: string;
@@ -23,7 +24,7 @@ export interface IndexDefinitionConfig {
 
 export interface FlexSearchPrimarySortClauseConfig {
     readonly field: string;
-    readonly asc: boolean;
+    readonly direction: OrderDirection;
 }
 
 export interface FlexSearchIndexConfig {

src/model/config/model.ts

@@ -1,4 +1,4 @@
-import { ModelValidationOptions } from '../../config/interfaces';
+import { ModelOptions } from '../../config/interfaces';
 import { ValidationMessage } from '../validation';
 import { BillingConfig } from './billing';
 import { LocalizationConfig } from './i18n';
@@ -11,6 +11,6 @@ export interface ModelConfig {
     readonly validationMessages?: ReadonlyArray<ValidationMessage>;
     readonly i18n?: ReadonlyArray<LocalizationConfig>;
     readonly billing?: BillingConfig;
-    readonly modelValidationOptions?: ModelValidationOptions;
     readonly timeToLiveConfigs?: ReadonlyArray<TimeToLiveConfig>;
+    readonly options?: ModelOptions;
 }

src/model/create-model.ts

@@ -16,7 +16,7 @@ import {
     TypeDefinitionNode,
     valueFromAST
 } from 'graphql';
-import { ModelValidationOptions } from '../config/interfaces';
+import { ModelOptions } from '../config/interfaces';
 import {
     ParsedGraphQLProjectSource,
     ParsedObjectProjectSource,
@@ -110,21 +110,22 @@ import {
 } from './config';
 import { BillingConfig } from './config/billing';
 import { Model } from './implementation';
+import { OrderDirection } from './implementation/order';
 import { parseBillingConfigs } from './parse-billing';
 import { parseI18nConfigs } from './parse-i18n';
 import { parseTTLConfigs } from './parse-ttl';
 import { ValidationContext, ValidationMessage } from './validation';
 
-export function createModel(parsedProject: ParsedProject, modelValidationOptions?: ModelValidationOptions): Model {
+export function createModel(parsedProject: ParsedProject, options?: ModelOptions): Model {
     const validationContext = new ValidationContext();
     return new Model({
-        types: createTypeInputs(parsedProject, validationContext),
+        types: createTypeInputs(parsedProject, validationContext, options ?? {}),
         permissionProfiles: extractPermissionProfiles(parsedProject),
         i18n: extractI18n(parsedProject),
         validationMessages: validationContext.validationMessages,
         billing: extractBilling(parsedProject),
-        modelValidationOptions,
-        timeToLiveConfigs: extractTimeToLive(parsedProject)
+        timeToLiveConfigs: extractTimeToLive(parsedProject),
+        options
     });
 }
 
@@ -144,7 +145,11 @@ const VALIDATION_ERROR_MISSING_OBJECT_TYPE_DIRECTIVE = `Add one of @${ROOT_ENTIT
 const VALIDATION_ERROR_INVALID_DEFINITION_KIND =
     'This kind of definition is not allowed. Only object and enum type definitions are allowed.';
 
-function createTypeInputs(parsedProject: ParsedProject, context: ValidationContext): ReadonlyArray<TypeConfig> {
+function createTypeInputs(
+    parsedProject: ParsedProject,
+    context: ValidationContext,
+    options: ModelOptions
+): ReadonlyArray<TypeConfig> {
     const graphQLSchemaParts = parsedProject.sources.filter(
         parsedSource => parsedSource.kind === ParsedProjectSourceBaseKind.GRAPHQL
     ) as ReadonlyArray<ParsedGraphQLProjectSource>;
@@ -173,7 +178,7 @@ function createTypeInputs(parsedProject: ParsedProject, context: ValidationConte
                         };
                         return enumTypeInput;
                     case OBJECT_TYPE_DEFINITION:
-                        return createObjectTypeInput(definition, schemaPart, context);
+                        return createObjectTypeInput(definition, schemaPart, context, options);
                     default:
                         return undefined;
                 }
@@ -196,15 +201,16 @@ function createEnumValues(valueNodes: ReadonlyArray<EnumValueDefinitionNode>): R
 function createObjectTypeInput(
     definition: ObjectTypeDefinitionNode,
     schemaPart: ParsedGraphQLProjectSource,
-    context: ValidationContext
+    context: ValidationContext,
+    options: ModelOptions
 ): ObjectTypeConfig {
     const entityType = getKindOfObjectTypeNode(definition, context);
 
     const common = {
         name: definition.name.value,
         description: definition.description ? definition.description.value : undefined,
         astNode: definition,
-        fields: (definition.fields || []).map(field => createFieldInput(field, context)),
+        fields: (definition.fields || []).map(field => createFieldInput(field, context, options)),
         namespacePath: getNamespacePath(definition, schemaPart.namespacePath),
         flexSearchLanguage: getDefaultLanguage(definition, context)
     };
@@ -337,7 +343,7 @@ function getFlexSearchOrder(rootEntityDirective?: DirectiveNode): ReadonlyArray<
             .map((value: any) => {
                 return {
                     field: value.field,
-                    asc: value.direction === 'ASC' ? true : false
+                    direction: value.direction === 'DESC' ? OrderDirection.DESCENDING : OrderDirection.ASCENDING
                 };
             });
     }
@@ -457,7 +463,11 @@ function getLanguage(fieldNode: FieldDefinitionNode, context: ValidationContext)
     }
 }
 
-function createFieldInput(fieldNode: FieldDefinitionNode, context: ValidationContext): FieldConfig {
+function createFieldInput(
+    fieldNode: FieldDefinitionNode,
+    context: ValidationContext,
+    options: ModelOptions
+): FieldConfig {
     const inverseOfASTNode = getInverseOfASTNode(fieldNode, context);
     const relationDeleteActionASTNode = getRelationDeleteActionASTNode(fieldNode, context);
     const referenceDirectiveASTNode = findDirectiveWithName(fieldNode, REFERENCE_DIRECTIVE);
@@ -500,7 +510,7 @@ function createFieldInput(fieldNode: FieldDefinitionNode, context: ValidationCon
         isFlexSearchIndexCaseSensitive:
             flexSearchIndexCaseSensitiveNode?.value.kind === 'BooleanValue'
                 ? flexSearchIndexCaseSensitiveNode.value.value
-                : undefined,
+                : options.isFlexSearchIndexCaseSensitiveByDefault,
         isFlexSearchIndexedASTNode: findDirectiveWithName(fieldNode, FLEX_SEARCH_INDEXED_DIRECTIVE),
         isFlexSearchFulltextIndexed: hasDirectiveWithName(fieldNode, FLEX_SEARCH_FULLTEXT_INDEXED_DIRECTIVE),
         isFlexSearchFulltextIndexedASTNode: findDirectiveWithName(fieldNode, FLEX_SEARCH_FULLTEXT_INDEXED_DIRECTIVE),

src/model/implementation/field.ts

@@ -1522,7 +1522,7 @@ export class Field implements ModelComponent {
     }
 
     get isFlexSearchIndexCaseSensitive(): boolean {
-        return this.input.isFlexSearchIndexCaseSensitive ?? true;
+        return this.input.isFlexSearchIndexCaseSensitive ?? false;
     }
 
     get flexSearchAnalyzer(): string | undefined {

src/model/implementation/model.ts

@@ -1,6 +1,6 @@
 import { groupBy, uniqBy } from 'lodash';
 import memorize from 'memorize-decorator';
-import { ModelValidationOptions } from '../../config/interfaces';
+import { ModelOptions } from '../../config/interfaces';
 import { flatMap, objectEntries, objectValues } from '../../utils/utils';
 import { ModelConfig, TypeKind } from '../config';
 import { NamespacedPermissionProfileConfigMap } from '../index';
@@ -31,7 +31,11 @@ export class Model implements ModelComponent {
     readonly i18n: ModelI18n;
     readonly permissionProfiles: ReadonlyArray<PermissionProfile>;
     readonly billingEntityTypes: ReadonlyArray<BillingEntityType>;
-    readonly modelValidationOptions?: ModelValidationOptions;
+    /**
+     * @deprecated use options
+     */
+    readonly modelValidationOptions?: ModelOptions;
+    readonly options?: ModelOptions;
     readonly timeToLiveTypes: ReadonlyArray<TimeToLiveType>;
 
     constructor(private input: ModelConfig) {
@@ -50,7 +54,8 @@ export class Model implements ModelComponent {
         this.billingEntityTypes = input.billing
             ? input.billing.billingEntities.map(value => new BillingEntityType(value, this))
             : [];
-        this.modelValidationOptions = input.modelValidationOptions;
+        this.options = input.options;
+        this.modelValidationOptions = input.options;
         this.timeToLiveTypes = input.timeToLiveConfigs
             ? input.timeToLiveConfigs.map(ttlConfig => new TimeToLiveType(ttlConfig, this))
             : [];
@@ -243,10 +248,10 @@ export class Model implements ModelComponent {
     }
 
     get forbiddenRootEntityNames(): ReadonlyArray<string> {
-        if (!this.modelValidationOptions || !this.modelValidationOptions.forbiddenRootEntityNames) {
+        if (!this.options || !this.options.forbiddenRootEntityNames) {
             return ['BillingEntity'];
         }
-        return this.modelValidationOptions!.forbiddenRootEntityNames;
+        return this.options!.forbiddenRootEntityNames;
     }
 }
 

src/model/implementation/root-entity-type.ts

@@ -2,6 +2,7 @@ import { GraphQLID, GraphQLString } from 'graphql';
 import memorize from 'memorize-decorator';
 import {
     ACCESS_GROUP_FIELD,
+    ENTITY_CREATED_AT,
     FLEX_SEARCH_FULLTEXT_INDEXED_DIRECTIVE,
     FLEX_SEARCH_INDEXED_DIRECTIVE,
     FLEX_SEARCH_ORDER_ARGUMENT,
@@ -349,21 +350,29 @@ export class RootEntityType extends ObjectTypeBase {
         // primary sort is only used for sorting, so make sure it's unique
         // - this makes querying more consistent
         // - this enables us to use primary sort for cursor-based pagination (which requires an absolute sort order)
+        // the default primary sort should be createdAt_DESC, because this is useful most of the time. to avoid
+        // surprises when you do specify a primary sort, always add this default at the end (as long as it's not already
+        // included in the index)
+        if (!clauses.some(clause => clause.field === ENTITY_CREATED_AT)) {
+            clauses = [
+                ...clauses,
+                {
+                    field: ENTITY_CREATED_AT,
+                    direction: OrderDirection.DESCENDING
+                }
+            ];
+        }
         if (!clauses.some(clause => clause.field === this.discriminatorField.name)) {
             clauses = [
                 ...clauses,
                 {
                     field: this.discriminatorField.name,
-                    asc: true
+                    direction: OrderDirection.DESCENDING
                 }
             ];
         }
         return clauses.map(
-            c =>
-                new FlexSearchPrimarySortClause(
-                    new FieldPath({ path: c.field, baseType: this }),
-                    c.asc ? OrderDirection.ASCENDING : OrderDirection.DESCENDING
-                )
+            c => new FlexSearchPrimarySortClause(new FieldPath({ path: c.field, baseType: this }), c.direction)
         );
     }
 

src/project/project.ts

@@ -118,7 +118,6 @@ export class Project {
             getOperationIdentifier: config.getOperationIdentifier,
             processError: config.processError,
             schemaOptions: config.schemaOptions,
-            modelValidationOptions: config.modelValidationOptions,
             modelOptions: config.modelOptions
         };
     }

src/schema-generation/flex-search-filter-input-types/filter-fields.ts

@@ -389,6 +389,22 @@ export function resolveFilterField(
             not(new FlexSearchFieldExistsQueryNode(valueNode, analyzer))
         );
     }
+    // field < x and field <= x should also find NULL values, because that's how it behaves in non-flexsearch case
+    if (
+        (filterField.operatorName === INPUT_FIELD_LT || filterField.operatorName === INPUT_FIELD_LTE) &&
+        filterValue != null
+    ) {
+        const isNull = new BinaryOperationQueryNode(
+            new BinaryOperationQueryNode(valueNode, BinaryOperator.EQUAL, NullQueryNode.NULL),
+            BinaryOperator.OR,
+            not(new FlexSearchFieldExistsQueryNode(valueNode, analyzer))
+        );
+        return new BinaryOperationQueryNode(
+            isNull,
+            BinaryOperator.OR,
+            filterField.resolveOperator(valueNode, literalNode, analyzer)
+        );
+    }
     if (filterField.operatorName == INPUT_FIELD_IN && Array.isArray(filterValue) && filterValue.includes(null)) {
         return new BinaryOperationQueryNode(
             filterField.resolveOperator(valueNode, literalNode, analyzer),