From 643dacafbfcfb195783a92b387004c9ddb6da5d2 Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Mon, 15 Apr 2024 15:20:16 +0200 Subject: [PATCH 01/62] Create user last connection datetime --- app/gen-server/entity/User.ts | 3 +++ app/gen-server/lib/HomeDBManager.ts | 18 +++++++++++++++--- .../1713186031023-UserLastConnection.ts | 18 ++++++++++++++++++ app/server/lib/Client.ts | 4 ++++ app/server/lib/requestUtils.ts | 6 +++--- test/gen-server/migrations.ts | 4 +++- 6 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 app/gen-server/migration/1713186031023-UserLastConnection.ts diff --git a/app/gen-server/entity/User.ts b/app/gen-server/entity/User.ts index 2ed1016910..c93837cbf2 100644 --- a/app/gen-server/entity/User.ts +++ b/app/gen-server/entity/User.ts @@ -29,6 +29,9 @@ export class User extends BaseEntity { @Column({name: 'first_login_at', type: Date, nullable: true}) public firstLoginAt: Date | null; + @Column({name: 'last_connection_at', type: Date, nullable: true}) + public lastConnectionAt: Date | null; + @OneToOne(type => Organization, organization => organization.owner) public personalOrg: Organization; diff --git a/app/gen-server/lib/HomeDBManager.ts b/app/gen-server/lib/HomeDBManager.ts index 1a46573b3d..e5fc2eeb57 100644 --- a/app/gen-server/lib/HomeDBManager.ts +++ b/app/gen-server/lib/HomeDBManager.ts @@ -213,6 +213,7 @@ function isNonGuestGroup(group: Group): group is NonGuestGroup { export interface UserProfileChange { name?: string; isFirstTimeUser?: boolean; + newConnection?: boolean; } // Identifies a request to access a document. This combination of values is also used for caching @@ -614,6 +615,14 @@ export class HomeDBManager extends EventEmitter { // any automation for first logins if (!props.isFirstTimeUser) { isWelcomed = true; } } + if (props.newConnection === true) { + // set last connection time to now (remove milliseconds for compatibility with other + // timestamps in db set by typeorm, and since second level precision is fine) + const nowish = new Date(); + nowish.setMilliseconds(0); + user.lastConnectionAt = nowish; + needsSave = true; + } if (needsSave) { await user.save(); } @@ -701,12 +710,15 @@ export class HomeDBManager extends EventEmitter { user.name = (profile && (profile.name || email.split('@')[0])) || ''; needUpdate = true; } - if (profile && !user.firstLoginAt) { - // set first login time to now (remove milliseconds for compatibility with other + if (profile) { + // set first login time and last connection time to now (remove milliseconds for compatibility with other // timestamps in db set by typeorm, and since second level precision is fine) const nowish = new Date(); nowish.setMilliseconds(0); - user.firstLoginAt = nowish; + user.lastConnectionAt = nowish; + if (!user.firstLoginAt) { + user.firstLoginAt = nowish; + } needUpdate = true; } if (!user.picture && profile && profile.picture) { diff --git a/app/gen-server/migration/1713186031023-UserLastConnection.ts b/app/gen-server/migration/1713186031023-UserLastConnection.ts new file mode 100644 index 0000000000..e1688502f0 --- /dev/null +++ b/app/gen-server/migration/1713186031023-UserLastConnection.ts @@ -0,0 +1,18 @@ +import { MigrationInterface, QueryRunner, TableColumn} from 'typeorm'; + +export class UserLastConnection1713186031023 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + const sqlite = queryRunner.connection.driver.options.type === 'sqlite'; + const datetime = sqlite ? "datetime" : "timestamp with time zone"; + await queryRunner.addColumn('users', new TableColumn({ + name: 'last_connection_at', + type: datetime, + isNullable: true + })); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropColumn('users', 'last_connection_at'); + } +} diff --git a/app/server/lib/Client.ts b/app/server/lib/Client.ts index 0364ca365a..ca2ba8aafa 100644 --- a/app/server/lib/Client.ts +++ b/app/server/lib/Client.ts @@ -501,6 +501,10 @@ export class Client { const user = this._profile ? await this._fetchUser(dbManager) : dbManager.getAnonymousUser(); this._user = user ? dbManager.makeFullUser(user) : undefined; this._firstLoginAt = user?.firstLoginAt || null; + if (this._user) { + // Send the information to the dbManager that the user has a new activity + await dbManager.updateUser(this._user.id, {newConnection: true}); + } } private async _onMessage(message: string): Promise { diff --git a/app/server/lib/requestUtils.ts b/app/server/lib/requestUtils.ts index a20923f532..274ebc6c43 100644 --- a/app/server/lib/requestUtils.ts +++ b/app/server/lib/requestUtils.ts @@ -21,9 +21,9 @@ export const TEST_HTTPS_OFFSET = process.env.GRIST_TEST_HTTPS_OFFSET ? // Database fields that we permit in entities but don't want to cross the api. const INTERNAL_FIELDS = new Set([ - 'apiKey', 'billingAccountId', 'firstLoginAt', 'filteredOut', 'ownerId', 'gracePeriodStart', 'stripeCustomerId', - 'stripeSubscriptionId', 'stripePlanId', 'stripeProductId', 'userId', 'isFirstTimeUser', 'allowGoogleLogin', - 'authSubject', 'usage', 'createdBy' + 'apiKey', 'billingAccountId', 'firstLoginAt', 'lastConnectionAt', 'filteredOut', 'ownerId', 'gracePeriodStart', + 'stripeCustomerId', 'stripeSubscriptionId', 'stripePlanId', 'stripeProductId', 'userId', 'isFirstTimeUser', + 'allowGoogleLogin', 'authSubject', 'usage', 'createdBy' ]); /** diff --git a/test/gen-server/migrations.ts b/test/gen-server/migrations.ts index 0dd60c1664..a7faeb39f4 100644 --- a/test/gen-server/migrations.ts +++ b/test/gen-server/migrations.ts @@ -41,6 +41,8 @@ import {ForkIndexes1678737195050 as ForkIndexes} from 'app/gen-server/migration/ import {ActivationPrefs1682636695021 as ActivationPrefs} from 'app/gen-server/migration/1682636695021-ActivationPrefs'; import {AssistantLimit1685343047786 as AssistantLimit} from 'app/gen-server/migration/1685343047786-AssistantLimit'; import {Shares1701557445716 as Shares} from 'app/gen-server/migration/1701557445716-Shares'; +import {UserLastConnection1713186031023 + as UserLastConnection} from 'app/gen-server/migration/1713186031023-UserLastConnection'; const home: HomeDBManager = new HomeDBManager(); @@ -49,7 +51,7 @@ const migrations = [Initial, Login, PinDocs, UserPicture, DisplayEmail, DisplayE CustomerIndex, ExtraIndexes, OrgHost, DocRemovedAt, Prefs, ExternalBilling, DocOptions, Secret, UserOptions, GracePeriodStart, DocumentUsage, Activations, UserConnectId, UserUUID, UserUniqueRefUUID, - Forks, ForkIndexes, ActivationPrefs, AssistantLimit, Shares]; + Forks, ForkIndexes, ActivationPrefs, AssistantLimit, Shares, UserLastConnection]; // Assert that the "members" acl rule and group exist (or not). function assertMembersGroup(org: Organization, exists: boolean) { From cd8667a33376137c60e75e19f0c602cf7b4166a7 Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Tue, 16 Apr 2024 11:27:11 +0200 Subject: [PATCH 02/62] use date and not datetime for lastConnectionAt --- app/gen-server/lib/HomeDBManager.ts | 24 +++++++++---------- .../1713186031023-UserLastConnection.ts | 4 +--- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/app/gen-server/lib/HomeDBManager.ts b/app/gen-server/lib/HomeDBManager.ts index e5fc2eeb57..cbc899e8ce 100644 --- a/app/gen-server/lib/HomeDBManager.ts +++ b/app/gen-server/lib/HomeDBManager.ts @@ -616,12 +616,15 @@ export class HomeDBManager extends EventEmitter { if (!props.isFirstTimeUser) { isWelcomed = true; } } if (props.newConnection === true) { - // set last connection time to now (remove milliseconds for compatibility with other - // timestamps in db set by typeorm, and since second level precision is fine) - const nowish = new Date(); - nowish.setMilliseconds(0); - user.lastConnectionAt = nowish; - needsSave = true; + // set last connection to today (keep date, remove time) + const today = new Date(); + today.setHours(0, 0, 0, 0); + if (today.getFullYear() !== user.lastConnectionAt?.getFullYear() || + today.getMonth() !== user.lastConnectionAt?.getMonth() || + today.getDate() !== user.lastConnectionAt?.getDate()){ + user.lastConnectionAt = today; + needsSave = true; + } } if (needsSave) { await user.save(); @@ -710,15 +713,12 @@ export class HomeDBManager extends EventEmitter { user.name = (profile && (profile.name || email.split('@')[0])) || ''; needUpdate = true; } - if (profile) { - // set first login time and last connection time to now (remove milliseconds for compatibility with other + if (profile && !user.firstLoginAt) { + // set first login time to now (remove milliseconds for compatibility with other // timestamps in db set by typeorm, and since second level precision is fine) const nowish = new Date(); nowish.setMilliseconds(0); - user.lastConnectionAt = nowish; - if (!user.firstLoginAt) { - user.firstLoginAt = nowish; - } + user.firstLoginAt = nowish; needUpdate = true; } if (!user.picture && profile && profile.picture) { diff --git a/app/gen-server/migration/1713186031023-UserLastConnection.ts b/app/gen-server/migration/1713186031023-UserLastConnection.ts index e1688502f0..793c0638eb 100644 --- a/app/gen-server/migration/1713186031023-UserLastConnection.ts +++ b/app/gen-server/migration/1713186031023-UserLastConnection.ts @@ -3,11 +3,9 @@ import { MigrationInterface, QueryRunner, TableColumn} from 'typeorm'; export class UserLastConnection1713186031023 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { - const sqlite = queryRunner.connection.driver.options.type === 'sqlite'; - const datetime = sqlite ? "datetime" : "timestamp with time zone"; await queryRunner.addColumn('users', new TableColumn({ name: 'last_connection_at', - type: datetime, + type: "date", isNullable: true })); } From 62acdf38580a4e200b847b4f27a377f79aec8069 Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Tue, 30 Apr 2024 18:15:22 +0200 Subject: [PATCH 03/62] use moment to get today date --- app/gen-server/lib/HomeDBManager.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/app/gen-server/lib/HomeDBManager.ts b/app/gen-server/lib/HomeDBManager.ts index cbc899e8ce..835484ef75 100644 --- a/app/gen-server/lib/HomeDBManager.ts +++ b/app/gen-server/lib/HomeDBManager.ts @@ -72,6 +72,7 @@ import { import uuidv4 from "uuid/v4"; import flatten = require('lodash/flatten'); import pick = require('lodash/pick'); +import moment = require('moment-timezone'); // Support transactions in Sqlite in async code. This is a monkey patch, affecting // the prototypes of various TypeORM classes. @@ -616,13 +617,10 @@ export class HomeDBManager extends EventEmitter { if (!props.isFirstTimeUser) { isWelcomed = true; } } if (props.newConnection === true) { - // set last connection to today (keep date, remove time) - const today = new Date(); - today.setHours(0, 0, 0, 0); - if (today.getFullYear() !== user.lastConnectionAt?.getFullYear() || - today.getMonth() !== user.lastConnectionAt?.getMonth() || - today.getDate() !== user.lastConnectionAt?.getDate()){ - user.lastConnectionAt = today; + // set last connection to today (need date only, no time) + const today = moment().startOf('day'); + if (today !== moment(user.lastConnectionAt).startOf('day')) { + user.lastConnectionAt = today.toDate(); needsSave = true; } } From 4d57c60be3bcaad468879c09e2b8923abfbb0efa Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Fri, 17 May 2024 10:47:12 +0200 Subject: [PATCH 04/62] update user last connection on each authorization request --- app/gen-server/lib/HomeDBManager.ts | 13 ++++--------- app/server/lib/Authorizer.ts | 5 +++++ app/server/lib/Client.ts | 4 ---- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/app/gen-server/lib/HomeDBManager.ts b/app/gen-server/lib/HomeDBManager.ts index 835484ef75..84b692d8f8 100644 --- a/app/gen-server/lib/HomeDBManager.ts +++ b/app/gen-server/lib/HomeDBManager.ts @@ -72,7 +72,6 @@ import { import uuidv4 from "uuid/v4"; import flatten = require('lodash/flatten'); import pick = require('lodash/pick'); -import moment = require('moment-timezone'); // Support transactions in Sqlite in async code. This is a monkey patch, affecting // the prototypes of various TypeORM classes. @@ -214,7 +213,7 @@ function isNonGuestGroup(group: Group): group is NonGuestGroup { export interface UserProfileChange { name?: string; isFirstTimeUser?: boolean; - newConnection?: boolean; + lastConnectionAt?: Date; } // Identifies a request to access a document. This combination of values is also used for caching @@ -616,13 +615,9 @@ export class HomeDBManager extends EventEmitter { // any automation for first logins if (!props.isFirstTimeUser) { isWelcomed = true; } } - if (props.newConnection === true) { - // set last connection to today (need date only, no time) - const today = moment().startOf('day'); - if (today !== moment(user.lastConnectionAt).startOf('day')) { - user.lastConnectionAt = today.toDate(); - needsSave = true; - } + if (props.lastConnectionAt && user.lastConnectionAt !== props.lastConnectionAt) { + user.lastConnectionAt = props.lastConnectionAt; + needsSave = true; } if (needsSave) { await user.save(); diff --git a/app/server/lib/Authorizer.ts b/app/server/lib/Authorizer.ts index b8c859815b..d167ef4f67 100644 --- a/app/server/lib/Authorizer.ts +++ b/app/server/lib/Authorizer.ts @@ -23,6 +23,7 @@ import * as cookie from 'cookie'; import {NextFunction, Request, RequestHandler, Response} from 'express'; import {IncomingMessage} from 'http'; import onHeaders from 'on-headers'; +import moment from 'moment-timezone'; export interface RequestWithLogin extends Request { sessionID: string; @@ -358,6 +359,10 @@ export async function addRequestUser( mreq.user = user; mreq.userId = user.id; mreq.userIsAuthorized = true; + const today = moment().startOf('day'); + if (today !== moment(user.lastConnectionAt).startOf('day')) { + await dbManager.updateUser(mreq.userId, {lastConnectionAt: today.toDate()}); + } } } } diff --git a/app/server/lib/Client.ts b/app/server/lib/Client.ts index ca2ba8aafa..0364ca365a 100644 --- a/app/server/lib/Client.ts +++ b/app/server/lib/Client.ts @@ -501,10 +501,6 @@ export class Client { const user = this._profile ? await this._fetchUser(dbManager) : dbManager.getAnonymousUser(); this._user = user ? dbManager.makeFullUser(user) : undefined; this._firstLoginAt = user?.firstLoginAt || null; - if (this._user) { - // Send the information to the dbManager that the user has a new activity - await dbManager.updateUser(this._user.id, {newConnection: true}); - } } private async _onMessage(message: string): Promise { From 6a07bcbbd512fe4c966e1e0ced0bbb23a9dee4a5 Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Mon, 20 May 2024 17:33:17 +0200 Subject: [PATCH 05/62] fix (migration-test): use SQL queries to avoid using User entity in migrations --- .../migration/1663851423064-UserUUID.ts | 16 +++++++++------- .../migration/1664528376930-UserRefUnique.ts | 17 +++++++++-------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/app/gen-server/migration/1663851423064-UserUUID.ts b/app/gen-server/migration/1663851423064-UserUUID.ts index ba0e71b1f7..ce4a9827ef 100644 --- a/app/gen-server/migration/1663851423064-UserUUID.ts +++ b/app/gen-server/migration/1663851423064-UserUUID.ts @@ -1,4 +1,3 @@ -import {User} from 'app/gen-server/entity/User'; import {makeId} from 'app/server/lib/idUtils'; import {MigrationInterface, QueryRunner, TableColumn} from "typeorm"; @@ -15,12 +14,15 @@ export class UserUUID1663851423064 implements MigrationInterface { // Updating so many rows in a multiple queries is not ideal. We will send updates in chunks. // 300 seems to be a good number, for 24k rows we have 80 queries. - const userList = await queryRunner.manager.createQueryBuilder() - .select("users") - .from(User, "users") - .getMany(); - userList.forEach(u => u.ref = makeId()); - await queryRunner.manager.save(userList, { chunk: 300 }); + const userList = await queryRunner.query("SELECT * FROM users;"); + let transaction = ""; + for (let i = 0; i < userList.length; i += 1) { + transaction += `UPDATE users SET ref = '${makeId()}' WHERE id = ${userList[i].id};`; + if (i % 300 === 0 || i === userList.length - 1) { + await queryRunner.query(transaction); + transaction = ""; + } + } // We are not making this column unique yet, because it can fail // if there are some old workers still running, and any new user diff --git a/app/gen-server/migration/1664528376930-UserRefUnique.ts b/app/gen-server/migration/1664528376930-UserRefUnique.ts index 2753604250..0fe54cb021 100644 --- a/app/gen-server/migration/1664528376930-UserRefUnique.ts +++ b/app/gen-server/migration/1664528376930-UserRefUnique.ts @@ -1,4 +1,3 @@ -import {User} from 'app/gen-server/entity/User'; import {makeId} from 'app/server/lib/idUtils'; import {MigrationInterface, QueryRunner} from "typeorm"; @@ -8,13 +7,15 @@ export class UserRefUnique1664528376930 implements MigrationInterface { // the ref column unique. // Update users that don't have unique ref set. - const userList = await queryRunner.manager.createQueryBuilder() - .select("users") - .from(User, "users") - .where("ref is null") - .getMany(); - userList.forEach(u => u.ref = makeId()); - await queryRunner.manager.save(userList, {chunk: 300}); + const userList = await queryRunner.query("SELECT * FROM users WHERE ref is null"); + let transaction = ""; + for (let i = 0; i < userList.length; i += 1) { + transaction += `UPDATE users SET ref = '${makeId()}' WHERE id = ${userList[i].id};`; + if (i % 300 === 0 || i === userList.length - 1) { + await queryRunner.query(transaction); + transaction = ""; + } + } // Mark column as unique and non-nullable. const users = (await queryRunner.getTable('users'))!; From 73420a7d0d86b2ef71d6fdb848ffd9d1466f010a Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Wed, 22 May 2024 11:03:53 +0200 Subject: [PATCH 06/62] update lastConnectionAt directly in getUserByLogin --- app/gen-server/lib/HomeDBManager.ts | 11 ++++++----- app/server/lib/Authorizer.ts | 5 ----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/app/gen-server/lib/HomeDBManager.ts b/app/gen-server/lib/HomeDBManager.ts index 84b692d8f8..fede8a1551 100644 --- a/app/gen-server/lib/HomeDBManager.ts +++ b/app/gen-server/lib/HomeDBManager.ts @@ -72,6 +72,7 @@ import { import uuidv4 from "uuid/v4"; import flatten = require('lodash/flatten'); import pick = require('lodash/pick'); +import moment from 'moment-timezone'; // Support transactions in Sqlite in async code. This is a monkey patch, affecting // the prototypes of various TypeORM classes. @@ -213,7 +214,6 @@ function isNonGuestGroup(group: Group): group is NonGuestGroup { export interface UserProfileChange { name?: string; isFirstTimeUser?: boolean; - lastConnectionAt?: Date; } // Identifies a request to access a document. This combination of values is also used for caching @@ -615,10 +615,6 @@ export class HomeDBManager extends EventEmitter { // any automation for first logins if (!props.isFirstTimeUser) { isWelcomed = true; } } - if (props.lastConnectionAt && user.lastConnectionAt !== props.lastConnectionAt) { - user.lastConnectionAt = props.lastConnectionAt; - needsSave = true; - } if (needsSave) { await user.save(); } @@ -743,6 +739,11 @@ export class HomeDBManager extends EventEmitter { user.options = {...(user.options ?? {}), authSubject: userOptions.authSubject}; needUpdate = true; } + const today = moment().startOf('day'); + if (today !== moment(user.lastConnectionAt).startOf('day')) { + user.lastConnectionAt = today.toDate(); + needUpdate = true; + } if (needUpdate) { login.user = user; await manager.save([user, login]); diff --git a/app/server/lib/Authorizer.ts b/app/server/lib/Authorizer.ts index d167ef4f67..b8c859815b 100644 --- a/app/server/lib/Authorizer.ts +++ b/app/server/lib/Authorizer.ts @@ -23,7 +23,6 @@ import * as cookie from 'cookie'; import {NextFunction, Request, RequestHandler, Response} from 'express'; import {IncomingMessage} from 'http'; import onHeaders from 'on-headers'; -import moment from 'moment-timezone'; export interface RequestWithLogin extends Request { sessionID: string; @@ -359,10 +358,6 @@ export async function addRequestUser( mreq.user = user; mreq.userId = user.id; mreq.userIsAuthorized = true; - const today = moment().startOf('day'); - if (today !== moment(user.lastConnectionAt).startOf('day')) { - await dbManager.updateUser(mreq.userId, {lastConnectionAt: today.toDate()}); - } } } } From 6610ab27099164a44b3bb1af3f945d8218beb150 Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Tue, 28 May 2024 11:51:18 +0200 Subject: [PATCH 07/62] feat [last-connection]: use chunk in migration and parameterizing the queries --- .../migration/1663851423064-UserUUID.ts | 14 ++------ .../migration/1664528376930-UserRefUnique.ts | 11 ++----- app/gen-server/sqlUtils.ts | 33 +++++++++++++++++++ 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/app/gen-server/migration/1663851423064-UserUUID.ts b/app/gen-server/migration/1663851423064-UserUUID.ts index ce4a9827ef..66764b9415 100644 --- a/app/gen-server/migration/1663851423064-UserUUID.ts +++ b/app/gen-server/migration/1663851423064-UserUUID.ts @@ -1,5 +1,6 @@ -import {makeId} from 'app/server/lib/idUtils'; + import {MigrationInterface, QueryRunner, TableColumn} from "typeorm"; +import {addRefToUserList} from "../sqlUtils"; export class UserUUID1663851423064 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { @@ -12,17 +13,8 @@ export class UserUUID1663851423064 implements MigrationInterface { isUnique: false, })); - // Updating so many rows in a multiple queries is not ideal. We will send updates in chunks. - // 300 seems to be a good number, for 24k rows we have 80 queries. const userList = await queryRunner.query("SELECT * FROM users;"); - let transaction = ""; - for (let i = 0; i < userList.length; i += 1) { - transaction += `UPDATE users SET ref = '${makeId()}' WHERE id = ${userList[i].id};`; - if (i % 300 === 0 || i === userList.length - 1) { - await queryRunner.query(transaction); - transaction = ""; - } - } + await addRefToUserList(queryRunner, userList); // We are not making this column unique yet, because it can fail // if there are some old workers still running, and any new user diff --git a/app/gen-server/migration/1664528376930-UserRefUnique.ts b/app/gen-server/migration/1664528376930-UserRefUnique.ts index 0fe54cb021..dd88a565bc 100644 --- a/app/gen-server/migration/1664528376930-UserRefUnique.ts +++ b/app/gen-server/migration/1664528376930-UserRefUnique.ts @@ -1,5 +1,5 @@ -import {makeId} from 'app/server/lib/idUtils'; import {MigrationInterface, QueryRunner} from "typeorm"; +import {addRefToUserList} from "../sqlUtils"; export class UserRefUnique1664528376930 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { @@ -8,14 +8,7 @@ export class UserRefUnique1664528376930 implements MigrationInterface { // Update users that don't have unique ref set. const userList = await queryRunner.query("SELECT * FROM users WHERE ref is null"); - let transaction = ""; - for (let i = 0; i < userList.length; i += 1) { - transaction += `UPDATE users SET ref = '${makeId()}' WHERE id = ${userList[i].id};`; - if (i % 300 === 0 || i === userList.length - 1) { - await queryRunner.query(transaction); - transaction = ""; - } - } + await addRefToUserList(queryRunner, userList); // Mark column as unique and non-nullable. const users = (await queryRunner.getTable('users'))!; diff --git a/app/gen-server/sqlUtils.ts b/app/gen-server/sqlUtils.ts index 6108642fb7..c619a28117 100644 --- a/app/gen-server/sqlUtils.ts +++ b/app/gen-server/sqlUtils.ts @@ -2,6 +2,8 @@ import {DatabaseType, QueryRunner, SelectQueryBuilder} from 'typeorm'; import {RelationCountLoader} from 'typeorm/query-builder/relation-count/RelationCountLoader'; import {RelationIdLoader} from 'typeorm/query-builder/relation-id/RelationIdLoader'; import {RawSqlResultsToEntityTransformer} from "typeorm/query-builder/transformer/RawSqlResultsToEntityTransformer"; +import {makeId} from 'app/server/lib/idUtils'; +import {chunk} from 'lodash'; /** * @@ -125,6 +127,37 @@ export function datetime(dbType: DatabaseType) { } } +export function addParamToQuery(dbType: DatabaseType, index: number) { + switch (dbType) { + case 'postgres': + return `$${index}`; + case 'sqlite': + return `?`; + default: + throw new Error(`addParamToQuery not implemented for ${dbType}`); + } +} + +export async function addRefToUserList(queryRunner: QueryRunner, userList: any[]){ + const dbType = queryRunner.connection.driver.options.type; + + // Updating so many rows in a multiple queries is not ideal. We will send updates in chunks. + // 300 seems to be a good number, for 24k rows we have 80 queries. + const userChunks = chunk(userList, 300); + for (const users of userChunks) { + await queryRunner.connection.transaction(async manager => { + const queries = users.map((user: any, _index: number, _array: any[]) => { + return manager.query( + `UPDATE users + SET ref = ${addParamToQuery(dbType, 1)} + WHERE id = ${addParamToQuery(dbType, 2)}`, + [makeId(), user.id]); + }); + await Promise.all(queries); + }); + } +} + /** * * Generate SQL code from one QueryBuilder, get the "raw" results, and then decode From 22114c7c08407de1f972696a48802c7f4ea05a18 Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Tue, 28 May 2024 13:56:18 +0200 Subject: [PATCH 08/62] remove Trailing spaces --- app/server/lib/requestUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/server/lib/requestUtils.ts b/app/server/lib/requestUtils.ts index d5f1a78a04..de0326d478 100644 --- a/app/server/lib/requestUtils.ts +++ b/app/server/lib/requestUtils.ts @@ -21,7 +21,7 @@ export const TEST_HTTPS_OFFSET = process.env.GRIST_TEST_HTTPS_OFFSET ? // Database fields that we permit in entities but don't want to cross the api. const INTERNAL_FIELDS = new Set([ - 'apiKey', 'billingAccountId', 'firstLoginAt', 'lastConnectionAt', 'filteredOut', 'ownerId', 'gracePeriodStart', + 'apiKey', 'billingAccountId', 'firstLoginAt', 'lastConnectionAt', 'filteredOut', 'ownerId', 'gracePeriodStart', 'stripeCustomerId', 'stripeSubscriptionId', 'stripeProductId', 'userId', 'isFirstTimeUser', 'allowGoogleLogin', 'authSubject', 'usage', 'createdBy' ]); From c418a0ba274268399c6effd6783d60d77ef7d22f Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Wed, 29 May 2024 09:54:08 +0200 Subject: [PATCH 09/62] reorganize imports --- app/gen-server/migration/1663851423064-UserUUID.ts | 2 +- app/gen-server/migration/1664528376930-UserRefUnique.ts | 2 +- app/gen-server/migration/1713186031023-UserLastConnection.ts | 2 +- app/gen-server/sqlUtils.ts | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/gen-server/migration/1663851423064-UserUUID.ts b/app/gen-server/migration/1663851423064-UserUUID.ts index 66764b9415..9d26cd6727 100644 --- a/app/gen-server/migration/1663851423064-UserUUID.ts +++ b/app/gen-server/migration/1663851423064-UserUUID.ts @@ -1,6 +1,6 @@ import {MigrationInterface, QueryRunner, TableColumn} from "typeorm"; -import {addRefToUserList} from "../sqlUtils"; +import {addRefToUserList} from "app/gen-server/sqlUtils"; export class UserUUID1663851423064 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { diff --git a/app/gen-server/migration/1664528376930-UserRefUnique.ts b/app/gen-server/migration/1664528376930-UserRefUnique.ts index dd88a565bc..6474789fec 100644 --- a/app/gen-server/migration/1664528376930-UserRefUnique.ts +++ b/app/gen-server/migration/1664528376930-UserRefUnique.ts @@ -1,5 +1,5 @@ import {MigrationInterface, QueryRunner} from "typeorm"; -import {addRefToUserList} from "../sqlUtils"; +import {addRefToUserList} from "app/gen-server/sqlUtils"; export class UserRefUnique1664528376930 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { diff --git a/app/gen-server/migration/1713186031023-UserLastConnection.ts b/app/gen-server/migration/1713186031023-UserLastConnection.ts index 793c0638eb..ee522ff4c2 100644 --- a/app/gen-server/migration/1713186031023-UserLastConnection.ts +++ b/app/gen-server/migration/1713186031023-UserLastConnection.ts @@ -1,4 +1,4 @@ -import { MigrationInterface, QueryRunner, TableColumn} from 'typeorm'; +import {MigrationInterface, QueryRunner, TableColumn} from 'typeorm'; export class UserLastConnection1713186031023 implements MigrationInterface { diff --git a/app/gen-server/sqlUtils.ts b/app/gen-server/sqlUtils.ts index c619a28117..46346e9b13 100644 --- a/app/gen-server/sqlUtils.ts +++ b/app/gen-server/sqlUtils.ts @@ -1,9 +1,9 @@ +import {makeId} from 'app/server/lib/idUtils'; +import {chunk} from 'lodash'; import {DatabaseType, QueryRunner, SelectQueryBuilder} from 'typeorm'; import {RelationCountLoader} from 'typeorm/query-builder/relation-count/RelationCountLoader'; import {RelationIdLoader} from 'typeorm/query-builder/relation-id/RelationIdLoader'; import {RawSqlResultsToEntityTransformer} from "typeorm/query-builder/transformer/RawSqlResultsToEntityTransformer"; -import {makeId} from 'app/server/lib/idUtils'; -import {chunk} from 'lodash'; /** * From 4428d8c7ae4782bae02eb3becc5c65ffe1ac4cfc Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Wed, 29 May 2024 14:30:55 +0200 Subject: [PATCH 10/62] reorder import --- app/gen-server/lib/HomeDBManager.ts | 2 +- app/gen-server/migration/1663851423064-UserUUID.ts | 2 +- app/gen-server/migration/1664528376930-UserRefUnique.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/gen-server/lib/HomeDBManager.ts b/app/gen-server/lib/HomeDBManager.ts index 5e793d77de..44a454720f 100644 --- a/app/gen-server/lib/HomeDBManager.ts +++ b/app/gen-server/lib/HomeDBManager.ts @@ -61,6 +61,7 @@ import {getScope} from 'app/server/lib/requestUtils'; import {WebHookSecret} from "app/server/lib/Triggers"; import {EventEmitter} from 'events'; import {Request} from "express"; +import moment from 'moment-timezone'; import { Brackets, Connection, @@ -72,7 +73,6 @@ import { import uuidv4 from "uuid/v4"; import flatten = require('lodash/flatten'); import pick = require('lodash/pick'); -import moment from 'moment-timezone'; import defaultsDeep = require('lodash/defaultsDeep'); // Support transactions in Sqlite in async code. This is a monkey patch, affecting diff --git a/app/gen-server/migration/1663851423064-UserUUID.ts b/app/gen-server/migration/1663851423064-UserUUID.ts index 9d26cd6727..eb6df46361 100644 --- a/app/gen-server/migration/1663851423064-UserUUID.ts +++ b/app/gen-server/migration/1663851423064-UserUUID.ts @@ -1,6 +1,6 @@ -import {MigrationInterface, QueryRunner, TableColumn} from "typeorm"; import {addRefToUserList} from "app/gen-server/sqlUtils"; +import {MigrationInterface, QueryRunner, TableColumn} from "typeorm"; export class UserUUID1663851423064 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { diff --git a/app/gen-server/migration/1664528376930-UserRefUnique.ts b/app/gen-server/migration/1664528376930-UserRefUnique.ts index 6474789fec..0aa74c716a 100644 --- a/app/gen-server/migration/1664528376930-UserRefUnique.ts +++ b/app/gen-server/migration/1664528376930-UserRefUnique.ts @@ -1,5 +1,5 @@ -import {MigrationInterface, QueryRunner} from "typeorm"; import {addRefToUserList} from "app/gen-server/sqlUtils"; +import {MigrationInterface, QueryRunner} from "typeorm"; export class UserRefUnique1664528376930 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { From 2c696c98a1b24b1bf146a85730eddaebc0a42dd2 Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Mon, 3 Jun 2024 12:07:38 +0200 Subject: [PATCH 11/62] simplification of addRefToUserList + add testing --- app/gen-server/sqlUtils.ts | 24 ++++++++---------------- test/gen-server/migrations.ts | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/app/gen-server/sqlUtils.ts b/app/gen-server/sqlUtils.ts index 46346e9b13..323b7525e5 100644 --- a/app/gen-server/sqlUtils.ts +++ b/app/gen-server/sqlUtils.ts @@ -127,17 +127,6 @@ export function datetime(dbType: DatabaseType) { } } -export function addParamToQuery(dbType: DatabaseType, index: number) { - switch (dbType) { - case 'postgres': - return `$${index}`; - case 'sqlite': - return `?`; - default: - throw new Error(`addParamToQuery not implemented for ${dbType}`); - } -} - export async function addRefToUserList(queryRunner: QueryRunner, userList: any[]){ const dbType = queryRunner.connection.driver.options.type; @@ -147,11 +136,14 @@ export async function addRefToUserList(queryRunner: QueryRunner, userList: any[] for (const users of userChunks) { await queryRunner.connection.transaction(async manager => { const queries = users.map((user: any, _index: number, _array: any[]) => { - return manager.query( - `UPDATE users - SET ref = ${addParamToQuery(dbType, 1)} - WHERE id = ${addParamToQuery(dbType, 2)}`, - [makeId(), user.id]); + switch (dbType) { + case 'postgres': + return manager.query(`UPDATE users SET ref = $1 WHERE id = $2`, [makeId(), user.id]); + case 'sqlite': + return manager.query(`UPDATE users SET ref = ? WHERE id = ?`, [makeId(), user.id]); + default: + throw new Error(`addParamToQuery not implemented for ${dbType}`); + } }); await Promise.all(queries); }); diff --git a/test/gen-server/migrations.ts b/test/gen-server/migrations.ts index aa793f83f3..26ec44fe74 100644 --- a/test/gen-server/migrations.ts +++ b/test/gen-server/migrations.ts @@ -116,6 +116,22 @@ describe('migrations', function() { // be doing something. }); + it('can migrate UserUUID and UserUniqueRefUUID with user in table', async function() { + this.timeout(60000); + const runner = home.connection.createQueryRunner(); + for (const migration of migrations) { + if (migration === UserUUID) { + // Create 400 users to test the chunk (each chunk is 300 users) + for (let i = 0; i < 400; i++) { + await runner.query(`INSERT INTO users (id, name, is_first_time_user) VALUES (${i}, 'name${i}', true)`); + } + } + + await (new migration()).up(runner); + } + await addSeedData(home.connection); + }); + it('can correctly switch display_email column to non-null with data', async function() { this.timeout(60000); const sqlite = home.connection.driver.options.type === 'sqlite'; From 6d64b87a1003a28066206e3337101832ec166e97 Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Tue, 18 Jun 2024 11:23:01 +0200 Subject: [PATCH 12/62] test: add check all users have unique ref --- test/gen-server/migrations.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/gen-server/migrations.ts b/test/gen-server/migrations.ts index 26ec44fe74..afaf149a1b 100644 --- a/test/gen-server/migrations.ts +++ b/test/gen-server/migrations.ts @@ -44,6 +44,7 @@ import {Shares1701557445716 as Shares} from 'app/gen-server/migration/1701557445 import {Billing1711557445716 as BillingFeatures} from 'app/gen-server/migration/1711557445716-Billing'; import {UserLastConnection1713186031023 as UserLastConnection} from 'app/gen-server/migration/1713186031023-UserLastConnection'; +import { User } from "app/gen-server/entity/User"; const home: HomeDBManager = new HomeDBManager(); @@ -119,16 +120,27 @@ describe('migrations', function() { it('can migrate UserUUID and UserUniqueRefUUID with user in table', async function() { this.timeout(60000); const runner = home.connection.createQueryRunner(); + + // Create 400 users to test the chunk (each chunk is 300 users) + const nbUsersToCreate = 400; for (const migration of migrations) { if (migration === UserUUID) { - // Create 400 users to test the chunk (each chunk is 300 users) - for (let i = 0; i < 400; i++) { + for (let i = 0; i < nbUsersToCreate; i++) { await runner.query(`INSERT INTO users (id, name, is_first_time_user) VALUES (${i}, 'name${i}', true)`); } } await (new migration()).up(runner); } + + // Check that all refs are unique + const userList = await runner.manager.createQueryBuilder() + .select("users") + .from(User, "users") + .getMany(); + const setOfUserRefs = new Set(userList.map(u => u.ref)); + assert.equal(nbUsersToCreate, userList.length); + assert.equal(setOfUserRefs.size, userList.length); await addSeedData(home.connection); }); From 2f1ec75f78516297bc3bb21721ef02106d640162 Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Tue, 18 Jun 2024 11:23:40 +0200 Subject: [PATCH 13/62] Update error message app/gen-server/sqlUtils.ts Co-authored-by: George Gevoian <85144792+georgegevoian@users.noreply.github.com> --- app/gen-server/sqlUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/gen-server/sqlUtils.ts b/app/gen-server/sqlUtils.ts index 323b7525e5..7513dee13b 100644 --- a/app/gen-server/sqlUtils.ts +++ b/app/gen-server/sqlUtils.ts @@ -142,7 +142,7 @@ export async function addRefToUserList(queryRunner: QueryRunner, userList: any[] case 'sqlite': return manager.query(`UPDATE users SET ref = ? WHERE id = ?`, [makeId(), user.id]); default: - throw new Error(`addParamToQuery not implemented for ${dbType}`); + throw new Error(`addRefToUserList not implemented for ${dbType}`); } }); await Promise.all(queries); From 5ffbf62593a85394eb940332223f2fb15badd2d8 Mon Sep 17 00:00:00 2001 From: George Gevoian Date: Fri, 24 May 2024 00:23:00 -0700 Subject: [PATCH 14/62] (core) Fix and move filter tests to grist-core Summary: A few tests that hadn't been ported to grist-core yet began failing after a change in behavior with the column filter menu. Test Plan: Existing tests. Reviewers: jarek Reviewed By: jarek Differential Revision: https://phab.getgrist.com/D4260 --- test/nbrowser/ColumnFilterMenu.ts | 621 +++++++++++++++++++++++++++++ test/nbrowser/ColumnFilterMenu2.ts | 120 ++++++ test/nbrowser/ColumnFilterMenu3.ts | 211 ++++++++++ test/nbrowser/SectionFilter.ts | 531 ++++++++++++++++++++++++ 4 files changed, 1483 insertions(+) create mode 100644 test/nbrowser/ColumnFilterMenu.ts create mode 100644 test/nbrowser/ColumnFilterMenu2.ts create mode 100644 test/nbrowser/ColumnFilterMenu3.ts create mode 100644 test/nbrowser/SectionFilter.ts diff --git a/test/nbrowser/ColumnFilterMenu.ts b/test/nbrowser/ColumnFilterMenu.ts new file mode 100644 index 0000000000..863aef5a26 --- /dev/null +++ b/test/nbrowser/ColumnFilterMenu.ts @@ -0,0 +1,621 @@ +import { UserAPI } from 'app/common/UserAPI'; +import { addToRepl, assert, driver, Key } from 'mocha-webdriver'; +import * as gu from 'test/nbrowser/gristUtils'; +import { setupTestSuite } from 'test/nbrowser/testUtils'; + +const limitShown = 500; + +// Sum all of the counts directly on the browser using `driver.executeScript(...)`. There could me +// over 500 of them and using the classic driver.findAll(...) approach makes it too slow and causes +// the test to crash (timeout). +function getCount() { + return driver.executeScript(` + return Array.from(document.querySelectorAll('.test-filter-menu-count'), e => e.innerText) + .map(s => s.split(',').join('')) + .map(Number) + .reduce((acc, v) => acc + v, 0); +`); +} + +// find a filter value by name +function findByName(regex: RegExp | string) { + return driver.findContent('.test-filter-menu-list label', regex); +} + +describe('ColumnFilterMenu', function() { + this.timeout(20000); + const cleanup = setupTestSuite(); + addToRepl('findByName', findByName); + let doc: any; + let api: UserAPI; + + it('should handle empty lists consistently', async function() { + // A formula returning an empty RecordSet in a RefList columns results in storing [] instead of null. + // This previously caused a bug where the empty list was 'flattened' and the cell not appearing in filters at all. + const session = await gu.session().teamSite.login(); + const api = session.createHomeApi(); + const docId = await session.tempNewDoc(cleanup, 'FilterEmptyLists', {load: false}); + + await api.applyUserActions(docId, [ + ['AddTable', 'Table2', [ + { + id: 'A', type: 'RefList:Table2', isFormula: true, + // This means that the first cell will contain [] while the second will contain null. + // The test asserts that both end up being treated the same. + formula: 'if $id == 1: return table.lookupRecords(B="foobar")' + }, + {id: 'B'}, + ]], + ['BulkAddRecord', 'Table2', [null, null], {B: [1, 2]}], + ]); + + await session.loadDoc(`/doc/${docId}/p/2`); + + await gu.rightClick(gu.getCell({rowNum: 1, col: 'A'})); + await driver.findContent('.grist-floating-menu li', 'Filter by this value').click(); + + assert.deepEqual( + await gu.getVisibleGridCells({cols: ['A', 'B'], rowNums: [1, 2, 3]}), + [ + '', '1', + '', '2', + '', '' + ] + ); + + await gu.openColumnMenu('A', 'Filter'); + + assert.deepEqual( + await driver.findAll('.test-filter-menu-list .test-filter-menu-count', (e) => e.getText()), + ['2'], + ); + }); + + it('should show only first 500', async function() { + const session = await gu.session().teamSite.login(); + await session.tempDoc(cleanup, 'World.grist'); + + // check row count is > 4000 + const total = await gu.getGridRowCount() - 1; + assert.equal(total, 4079); + + // scroll back to top + await gu.sendKeys(Key.chord(await gu.modKey(), Key.UP)); + + // open filter menu for first column + await gu.openColumnMenu('Name', 'Filter'); + + // check ther are 500 entry shown + assert.lengthOf(await driver.findAll('.test-filter-menu-list label'), limitShown); + + // check `Other` summary is present + assert.deepEqual( + await driver.findAll('.test-filter-menu-summary', (e) => e.find('label').getText()), + ['Other Values (3,501)', 'Future Values'] + ); + + // check counts add up + assert.equal(await getCount(), total); + + // type 'A' to search + await gu.sendKeys('A'); + + // check summary has `Other matching` and Other Non-matching` + assert.deepEqual( + await driver.findAll('.test-filter-menu-summary', (e) => e.find('label').getText()), + ['Other Matching (2,493)', 'Other Non-Matching (1,008)'] + ); + + // check count adds up + assert.equal(await getCount(), total); + + // clear search input + await gu.sendKeys(Key.BACK_SPACE); + + // Click All Except / Other Matching / Other NOn-Matching + await driver.findContent('.test-filter-menu-bulk-action', /None/).click(); + + // click Aba and Abadan + await driver.findContent('.test-filter-menu-list label', /Aba/).click(); + await driver.findContent('.test-filter-menu-list label', /Abadan/).click(); + + // Apply filter + await driver.find('.test-filter-menu-apply-btn').click(); + + // check grid contains aba and abadan + assert.deepEqual( + await gu.getVisibleGridCells({cols: ['Name'], rowNums: [1, 2, 3]}), + [ + 'Aba', + 'Abadan', + '' + ] + ); + + }); + + it('should uncheck \'Other Values\' checkbox when user clicks \'None\'', async () => { + // open the Name filter + await gu.openColumnMenu('Name', 'Filter'); + + // click None + await driver.findContent('.test-filter-menu-bulk-action', /None/).click(); + + // check Other values was propertly unchecked + assert.equal( + await driver.findContent('.test-filter-menu-summary', /Other Values/).find('input').matches(':checked'), + false + ); + + assert.equal( + await driver.findContent('.test-filter-menu-summary', /Future Values/).find('input').matches(':checked'), + false + ); + }); + + it('should take other filters into account', async () => { + + const session = await gu.session().teamSite.login(); + doc = await session.tempDoc(cleanup, 'SortFilterIconTest.grist'); + api = session.createHomeApi(); + + // check table content + assert.deepEqual( + await gu.getVisibleGridCells({cols: ['Name', 'Count'], rowNums: [1, 2, 3, 4, 5, 6]}), + [ 'Apples', '1', + 'Oranges', '3', + 'Bananas', '2', + 'Grapes', '-1', + 'Grapefruit', 'n/a', + 'Clementines', '5' + ]); + + // add Name Filter + await gu.openColumnMenu('Name', 'Filter'); + + // Click Oranges + await findByName('Oranges').click(); + + // Click Apply + await driver.find('.test-filter-menu-apply-btn').click(); + + // add Count filters + await driver.find('.test-add-filter-btn').click(); + await driver.findContent('.grist-floating-menu li', /Count/).click(); + + // Check that there's only 5 values left ('3' is missing) + assert.deepEqual(await driver.findAll('.test-filter-menu-list label', (e) => e.getText()), + ['n/a', '-1', '1', '2', '5']); + + // Check `Others` shows unique count + assert.equal(await driver.find('.test-filter-menu-summary').getText(), + 'Others (1)'); + + // Check `Others` is checked + assert.equal(await driver.find('.test-filter-menu-summary').find('input').matches(':checked'), true); + + // Click `Other` + await driver.find('.test-filter-menu-summary').find('input').click(); + + // Click '1' + await findByName(/^1/).click(); + + // Click Apply + await driver.find('.test-filter-menu-apply-btn').click(); + + // Open the Name menu filter + await driver.findContent('.test-filter-field', /Name/).click(); + + // Check there's only 4 values left + assert.deepEqual(await driver.findAll('.test-filter-menu-list label', (e) => e.getText()), + ['Bananas', 'Clementines', 'Grapefruit', 'Grapes']); + + // check `Others` shows 2 unique values + assert.equal(await driver.find('.test-filter-menu-summary').getText(), + 'Others (2)'); + + // check `Others` is in indeterminate state + assert.equal(await driver.find('.test-filter-menu-summary').find('input').matches(':checked'), false); + assert.equal(await driver.find('.test-filter-menu-summary').find('input').matches(':indeterminate'), true); + + // Click `Others` + await driver.find('.test-filter-menu-summary').find('input').click(); + + // check `Others` is checked + assert.equal(await driver.find('.test-filter-menu-summary').find('input').matches(':checked'), true); + assert.equal(await driver.find('.test-filter-menu-summary').find('input').matches(':indeterminate'), false); + + // Click `Others` + await driver.find('.test-filter-menu-summary').find('input').click(); + + // check `Others` is checked + assert.equal(await driver.find('.test-filter-menu-summary').find('input').matches(':checked'), false); + assert.equal(await driver.find('.test-filter-menu-summary').find('input').matches(':indeterminate'), false); + + // Click Apply + await driver.find('.test-filter-menu-apply-btn').click(); + + // open Count filter menu + await driver.findContent('.test-filter-field', /Count/).click(); + + // Click all and click Apply + await driver.findContent('.test-filter-menu-bulk-action', /All/).click(); + await driver.find('.test-filter-menu-apply-btn').click(); + + // open Name filter menu + await driver.findContent('.test-filter-field', /Name/).click(); + + // Check Apples and Oranges are unchecked + assert.deepEqual(await driver.findAll('.test-filter-menu-list label', (e) => e.getText()), + ['Apples', 'Bananas', 'Clementines', 'Grapefruit', 'Grapes', 'Oranges']); + assert.equal(await findByName('Apples').find('input').matches(':checked'), false); + assert.equal(await findByName('Oranges').find('input').matches(':checked'), false); + + // click Apply + await driver.find('.test-filter-menu-apply-btn').click(); + + // Open count Filter menu + await driver.findContent('.test-filter-field', /Count/).click(); + + // Check there's only 4 values left + assert.deepEqual(await driver.findAll('.test-filter-menu-list label', (e) => e.getText()), + ['n/a', '-1', '2', '5']); + + // Click Others + await driver.find('.test-filter-menu-summary').click(); + + // click Apply + await driver.find('.test-filter-menu-apply-btn').click(); + + // Open Name filter menu + await driver.findContent('.test-filter-field', /Name/).click(); + + // Check Others is unchecked + assert.equal(await driver.find('.test-filter-menu-summary').find('input').matches(':checked'), false); + assert.equal(await driver.find('.test-filter-menu-summary').find('input').matches(':indeterminate'), false); + + // Click Others + await driver.find('.test-filter-menu-summary').find('input').click(); + await driver.find('.test-filter-menu-apply-btn').click(); + + // Open count filter + await driver.findContent('.test-filter-field', /Count/).click(); + + // Click All and click apply + await driver.findContent('.test-filter-menu-bulk-action', /All/).click(); + await driver.find('.test-filter-menu-apply-btn').click(); + + // open Name filter menu + await driver.findContent('.test-filter-field', /Name/).click(); + + // Check both apples and orages are not checked + assert.equal(await findByName('Apples').find('input').matches(':checked'), false); + assert.equal(await findByName('Oranges').find('input').matches(':checked'), false); + + // Revert to all + await driver.findContent('.test-filter-menu-bulk-action', /All/).click(); + await driver.find('.test-filter-menu-apply-btn').click(); + + // Open Count filter menu and click All + await driver.findContent('.test-filter-field', /Count/).click(); + await driver.findContent('.test-filter-menu-bulk-action', /All/).click(); + await driver.find('.test-filter-menu-apply-btn').click(); + }); + + it('should show count of unique values next to summaries', async () => { + + // add another Apples + await driver.find('.record-add .field').click(); + await driver.sendKeys('Apples', Key.ENTER); + await gu.waitForServer(); + assert.deepEqual( + await gu.getVisibleGridCells({cols: ['Name', 'Count'], rowNums: [1, 2, 3, 4, 5, 6, 7]}), + [ 'Apples', '1', + 'Oranges', '3', + 'Bananas', '2', + 'Grapes', '-1', + 'Grapefruit', 'n/a', + 'Clementines', '5', + 'Apples', '0' + ]); + + // open the Count filter + await driver.findContent('.test-filter-field', /Count/).click(); + + // uncheck 0 and 1 + await findByName(/^0/).click(); + await findByName(/^1/).click(); + + // Click Apply + await driver.find('.test-filter-menu-apply-btn').click(); + + // open the Name filter + await driver.findContent('.test-filter-field', /Name/).click(); + + // check Apples is missing + assert.deepEqual(await driver.findAll('.test-filter-menu-list label', (e) => e.getText()), + ['Bananas', 'Clementines', 'Grapefruit', 'Grapes', 'Oranges']); + + // check count is (1) + assert.deepEqual( + await driver.findAll('.test-filter-menu-summary', (e) => e.find('label').getText()), + ['Others (1)'] + ); + + // close filter + await driver.sendKeys(Key.ESCAPE); + }); + + it('should show a working range filter for numeric columns', async function() { + + // open the Count filter + await driver.findContent('.test-filter-field', /Count/).click(); + + // set min to '2' + await gu.setRangeFilterBound('min', '2'); + await driver.find('.test-filter-menu-apply-btn').click(); + + // check values + assert.deepEqual( + await gu.getVisibleGridCells({cols: ['Name', 'Count'], rowNums: [1, 2, 3, 4]}), + [ 'Oranges', '3', + 'Bananas', '2', + 'Clementines', '5', + '', '' + ] + ); + + // reopen the filter + await driver.findContent('.test-filter-field', /Count/).click(); + + // set max to '4' + await gu.setRangeFilterBound('max', '4'); + await driver.find('.test-filter-menu-apply-btn').click(); + + assert.deepEqual( + await gu.getVisibleGridCells({cols: ['Name', 'Count'], rowNums: [1, 2, 3, 4]}), + [ 'Oranges', '3', + 'Bananas', '2', + '', '', + undefined, undefined + ] + ); + + // remove both min and max + await driver.findContent('.test-filter-field', /Count/).click(); + await gu.setRangeFilterBound('min', null); + await gu.setRangeFilterBound('max', null); + await driver.find('.test-filter-menu-apply-btn').click(); + + // check all values are there + assert.deepEqual( + await gu.getVisibleGridCells({cols: ['Name', 'Count'], rowNums: [1, 2, 3, 4, 5, 6, 7]}), + [ 'Apples', '1', + 'Oranges', '3', + 'Bananas', '2', + 'Grapes', '-1', + 'Grapefruit', 'n/a', + 'Clementines', '5', + 'Apples', '0' + ]); + + }); + + it('should remove new filters when Cancel is clicked in a new filter', async function() { + // Create a new Date filter. + await gu.openColumnMenu('Date', 'Filter'); + assert.deepEqual( + [ + {checked: true, value: 'n/a', count: 1}, + {checked: true, value: '', count: 2}, + {checked: true, value: '2019-07-15', count: 1}, + {checked: true, value: '2019-07-16', count: 1}, + {checked: true, value: '2019-07-17', count: 1}, + {checked: true, value: '2019-07-18', count: 1} + ], + await gu.getFilterMenuState() + ); + + // Check that the Date filter is pinned. + assert.deepEqual( + [ + {name: 'Name', hasUnsavedChanges: true}, + {name: 'Count', hasUnsavedChanges: true}, + {name: 'Date', hasUnsavedChanges: true}, + ], + await gu.getPinnedFilters() + ); + + // Set a min filter of '2019-07-16'. + await gu.setRangeFilterBound('min', '2019-07-16'); + + // Click Cancel, and check that the filter is no longer applied to the table data. + await gu.waitToPass(async () => { + await driver.find('.test-filter-menu-cancel-btn').click(); + assert.isFalse(await driver.find('.test-filter-menu-wrapper').isPresent()); + }); + assert.deepEqual( + await gu.getVisibleGridCells({cols: ['Name', 'Count'], rowNums: [1, 2, 3, 4, 5, 6, 7]}), + [ 'Apples', '1', + 'Oranges', '3', + 'Bananas', '2', + 'Grapes', '-1', + 'Grapefruit', 'n/a', + 'Clementines', '5', + 'Apples', '0' + ] + ); + + // Check that the Date filter was removed. + await gu.openSectionMenu('sortAndFilter'); + assert.isFalse(await driver.findContent('.test-filter-config-filter', /Date/).isPresent()); + await gu.sendKeys(Key.ESCAPE); + assert.deepEqual( + [ + {name: 'Name', hasUnsavedChanges: true}, + {name: 'Count', hasUnsavedChanges: true}, + ], + await gu.getPinnedFilters() + ); + }); + + it('should revert to open state when Cancel is clicked in an existing filter', async function() { + // Open the Count filter. + await driver.findContent('.test-filter-field', /Count/).click(); + + // Filter out 1 and 2. + await driver.findContent('.test-filter-menu-list label', /1/).click(); + await driver.findContent('.test-filter-menu-list label', /2/).click(); + + // Unpin the filter. + await driver.find('.test-filter-menu-pin-btn').click(); + + // Click Cancel, and check that the filter is no longer applied to the table data. + await driver.find('.test-filter-menu-cancel-btn').click(); + assert.deepEqual( + await gu.getVisibleGridCells({cols: ['Name', 'Count'], rowNums: [1, 2, 3, 4, 5, 6, 7]}), + [ 'Apples', '1', + 'Oranges', '3', + 'Bananas', '2', + 'Grapes', '-1', + 'Grapefruit', 'n/a', + 'Clementines', '5', + 'Apples', '0' + ] + ); + + // Check that Count is still pinned to the filter bar. + assert.deepEqual( + [ + {name: 'Name', hasUnsavedChanges: true}, + {name: 'Count', hasUnsavedChanges: true}, + ], + await gu.getPinnedFilters() + ); + + // Check the filter menu state of Count. + await driver.findContent('.test-filter-field', /Count/).click(); + assert.deepEqual( + [ + {checked: true, value: 'n/a', count: 1}, + {checked: true, value: '-1', count: 1}, + {checked: true, value: '0', count: 1}, + {checked: true, value: '1', count: 1}, + {checked: true, value: '2', count: 1}, + {checked: true, value: '3', count: 1}, + {checked: true, value: '5', count: 1}, + ], + await gu.getFilterMenuState() + ); + + await gu.sendKeys(Key.ESCAPE); + }); + + async function testDateLikeColumn(colId: 'Date'|'DateTime') { + + const timeChunk = colId === 'DateTime' ? ' 12:00am' : ''; + const colRegex = new RegExp(colId + '\\b'); + + // add Date Filter + await driver.find('.test-add-filter-btn').click(); + await driver.findContent('.grist-floating-menu li', colRegex).click(); + + // set min to '2019-07-16' + await gu.setRangeFilterBound('min', '2019-07-16'); + await driver.find('.test-filter-menu-apply-btn').click(); + await gu.waitAppFocus(true); + + // check values + assert.deepEqual( + await gu.getVisibleGridCells({cols: ['Name', colId], rowNums: [1, 2, 3, 4]}), + [ 'Apples', '2019-07-17' + timeChunk, + 'Oranges', '2019-07-16' + timeChunk, + 'Bananas', '2019-07-18' + timeChunk, + '', '' + ] + ); + + // reopen the filter + await driver.findContent('.test-filter-field', colRegex).click(); + + // set max to '2019-07-17' + await gu.setRangeFilterBound('max', '2019-07-17'); + await driver.find('.test-filter-menu-apply-btn').click(); + await gu.waitAppFocus(true); + + assert.deepEqual( + await gu.getVisibleGridCells({cols: ['Name', colId], rowNums: [1, 2, 3, 4]}), + [ 'Apples', '2019-07-17' + timeChunk, + 'Oranges', '2019-07-16' + timeChunk, + '', '', + undefined, undefined + ] + ); + + // remove both min and max + await driver.findContent('.test-filter-field', colRegex).click(); + await gu.setRangeFilterBound('min', null); + await gu.setRangeFilterBound('max', null); + await driver.find('.test-filter-menu-apply-btn').click(); + await gu.waitAppFocus(true); + + // check all values are there + assert.deepEqual( + await gu.getVisibleGridCells({cols: ['Name', colId], rowNums: [1, 2, 3, 4, 5, 6, 7]}), + [ 'Apples', '2019-07-17' + timeChunk, + 'Oranges', '2019-07-16' + timeChunk, + 'Bananas', '2019-07-18' + timeChunk, + 'Grapes', '', + 'Grapefruit', '2019-07-15' + timeChunk, + 'Clementines', 'n/a', + 'Apples', '', + ]); + } + + + it('should show a working range filter for Date column', async function() { + await testDateLikeColumn('Date'); + }); + + it('should show a working range filter for DateTime column', async function() { + + // adds a DateTime column + await api.applyUserActions(doc.id, [ + ['AddVisibleColumn', 'Table1', 'DateTime', { + type: "DateTime:UTC", widgetOptions: '{"dateFormat": "YYYY-MM-DD", "timeFormat": "h:mma"}' + }], + ['BulkUpdateRecord', 'Table1', [1, 2, 3, 4, 5, 6], { + DateTime: [ + // TODO: fix timezone + "2019-07-17T00:00Z", + "2019-07-16T00:00Z", + "2019-07-18T00:00Z", + "", + "2019-07-15T00:00Z", + "n/a", + ] + }], + ]); + + await testDateLikeColumn('DateTime'); + }); + + it('should have working date range filter also when column is hidden', async function() { + + // hide Date column + await gu.toggleSidePanel('right', 'open'); + await driver.find('.test-right-tab-pagewidget').click(); + await gu.moveToHidden('Date'); + + // add Date filter + await driver.findContent('.test-filter-field', 'Date').click(); + + // start typing date in min bounds and send TAB + await driver.find('.test-filter-menu-min').click(); + await gu.sendKeys('2019-07-14', Key.TAB); + + // check min is set to a valid date + assert.equal(await driver.find('.test-filter-menu-min input').value(), '2019-07-14'); + }); + +}); diff --git a/test/nbrowser/ColumnFilterMenu2.ts b/test/nbrowser/ColumnFilterMenu2.ts new file mode 100644 index 0000000000..40a1c06915 --- /dev/null +++ b/test/nbrowser/ColumnFilterMenu2.ts @@ -0,0 +1,120 @@ +import * as gu from 'test/nbrowser/gristUtils'; +import { setupTestSuite } from "test/nbrowser/testUtils"; +import { assert, driver } from 'mocha-webdriver'; + + +function getItems() { + return driver.findAll('.test-filter-menu-list label', async (e) => ({ + checked: await e.find('input').isSelected(), + label: await e.getText(), + count: await e.findClosest('div').find('.test-filter-menu-count').getText() + })); +} + +describe('ColumnFilterMenu2', function() { + + this.timeout(20000); + const cleanup = setupTestSuite(); + let mainSession: gu.Session; + let docId: string; + let api: any; + + before(async function() { + mainSession = await gu.session().teamSite.user('user1').login(); + docId = await mainSession.tempNewDoc(cleanup, 'ColumnFilterMenu2.grist', {load: false}); + api = mainSession.createHomeApi(); + // Prepare a table with some interestingly-formatted columns, and some data. + await api.applyUserActions(docId, [ + ['AddTable', 'Test', []], + ['AddVisibleColumn', 'Test', 'Bool', { + type: 'Bool', widgetOptions: JSON.stringify({widget:"TextBox"}) + }], + ['AddVisibleColumn', 'Test', 'Choice', { + type: 'Choice', widgetOptions: JSON.stringify({choices: ['foo', 'bar']}) + }], + ['AddVisibleColumn', 'Test', 'ChoiceList', { + type: 'ChoiceList', widgetOptions: JSON.stringify({choices: ['foo', 'bar']}) + }], + ['AddRecord', 'Test', null, {Bool: true, Choice: 'foo', ChoiceList: ['L', 'foo']}], + ]); + return docId; + }); + + afterEach(() => gu.checkForErrors()); + + it('should show all options for Bool columns', async () => { + await mainSession.loadDoc(`/doc/${docId}/p/2`); + + await gu.openColumnMenu('Bool', 'Filter'); + assert.deepEqual(await getItems(), [ + {checked: true, label: 'false', count: '0'}, + {checked: true, label: 'true', count: '1'}, + ]); + + // click false + await driver.findContent('.test-filter-menu-list label', 'false').click(); + assert.deepEqual(await getItems(), [ + {checked: false, label: 'false', count: '0'}, + {checked: true, label: 'true', count: '1'}, + ]); + + // add new record with Bool=false + const {retValues} = await api.applyUserActions(docId, [ + ['AddRecord', 'Test', null, {Bool: false}], + ]); + + // check record is not shown on screen + assert.deepEqual( + await gu.getVisibleGridCells({cols: ['Bool', 'Choice', 'ChoiceList'], rowNums: [1, 2]}), + ['true', 'foo', 'foo', + '', '', '' + ] as any + ); + + // remove added record + await api.applyUserActions(docId, [ + ['RemoveRecord', 'Test', retValues[0]] + ]); + }); + + it('should show all options for Choice/ChoiceList columns', async () => { + await gu.openColumnMenu('Choice', 'Filter'); + assert.deepEqual(await getItems(), [ + {checked: true, label: 'bar', count: '0'}, + {checked: true, label: 'foo', count: '1'}, + ]); + + // click bar + await driver.findContent('.test-filter-menu-list label', 'bar').click(); + assert.deepEqual(await getItems(), [ + {checked: false, label: 'bar', count: '0'}, + {checked: true, label: 'foo', count: '1'}, + ]); + + // add new record with Choice=bar + const {retValues} = await api.applyUserActions(docId, [ + ['AddRecord', 'Test', null, {Choice: 'bar'}], + ]); + + // check record is not shown on screen + assert.deepEqual( + await gu.getVisibleGridCells({cols: ['Bool', 'Choice', 'ChoiceList'], rowNums: [1, 2]}), + ['true', 'foo', 'foo', + '', '', '' + ] as any + ); + + // remove added record + await api.applyUserActions(docId, [ + ['RemoveRecord', 'Test', retValues[0]] + ]); + + // check ChoiceList filter offeres all options + await gu.openColumnMenu('ChoiceList', 'Filter'); + assert.deepEqual(await getItems(), [ + {checked: true, label: 'bar', count: '0'}, + {checked: true, label: 'foo', count: '1'}, + ]); + }); + +}); diff --git a/test/nbrowser/ColumnFilterMenu3.ts b/test/nbrowser/ColumnFilterMenu3.ts new file mode 100644 index 0000000000..3c23168918 --- /dev/null +++ b/test/nbrowser/ColumnFilterMenu3.ts @@ -0,0 +1,211 @@ +import { assert, driver, Key } from "mocha-webdriver"; +import * as gu from 'test/nbrowser/gristUtils'; +import { setupTestSuite } from "test/nbrowser/testUtils"; + +void(driver); +void(Key); + +async function getValues() { + return driver.findAll('.test-filter-menu-list label', e => e.getText()); +} + +describe('ColumnFilterMenu3', function() { + this.timeout(30000); + const cleanup = setupTestSuite(); + let mainSession: gu.Session; + let docId: string; + before(async () => { + mainSession = await gu.session().teamSite.user('user1').login(); + docId = await mainSession.tempNewDoc(cleanup, 'Search3.grist', {load: false}); + const api = mainSession.createHomeApi(); + // Prepare a table with some interestingly-formatted columns, and some data. + const {retValues} = await api.applyUserActions(docId, [ + ['AddTable', 'Test', []], + ['AddVisibleColumn', 'Test', 'Date', {type: 'Date', widgetOptions: '{"dateFormat":"DD-MM-YYYY"}'}], + ['AddVisibleColumn', 'Test', 'Numeric', {type: 'Numeric'}], + ['AddVisibleColumn', 'Test', 'Int', {type: 'Int'}], + ['AddVisibleColumn', 'Test', 'Ref', {type: 'Ref:Test'}], + ['AddVisibleColumn', 'Test', 'RefList', {type: 'RefList:Test'}], + ]); + await api.applyUserActions(docId, [ + ['UpdateRecord', '_grist_Tables_column', retValues[4].colRef, {visibleCol: retValues[1].colRef}], + ['UpdateRecord', '_grist_Tables_column', retValues[5].colRef, {visibleCol: retValues[1].colRef}], + ['SetDisplayFormula', 'Test', null, retValues[4].colRef, '$Ref.Date'], + ['SetDisplayFormula', 'Test', null, retValues[5].colRef, '$RefList.Date'], + ['AddRecord', 'Test', null, {Date: '22-12-2011', Numeric: 2, Int: 2, Ref: 1, + RefList: ['L', 1, 2]}], + ['AddRecord', 'Test', null, {Date: '20-12-2021', Numeric: 22, Int: 22, Ref: 2, + RefList: ['L', 1]}], + ['AddRecord', 'Test', null, {Date: '20-12-2011', Numeric: 3, Int: 3, Ref: 3, + RefList: ['L', 1, 2, 3]}], + ]); + await mainSession.loadDoc(`/doc/${docId}/p/2`); + }); + + afterEach(async () => { + // close menu if one was opened + if (await driver.find('.grist-floating-menu').isPresent()) { + await driver.sendKeys(Key.ESCAPE); + } + if (await driver.find('.test-filter-menu-wrapper').isPresent()) { + await driver.sendKeys(Key.ESCAPE); + } + }); + + it('should correctly focus between inputs in Numeric columns', async () => { + // A bug was introduced where the search input could no longer be focused if either range + // input had focus. + await gu.openColumnMenu('Numeric', 'Filter'); + + const assertSearchCanBeFocused = async () => { + await driver.find('.test-filter-menu-search-input').click(); + assert.equal( + await driver.switchTo().activeElement().getId(), + await driver.find('.test-filter-menu-search-input').getId() + ); + }; + + await driver.find('.test-filter-menu-min').click(); + await assertSearchCanBeFocused(); + await driver.find('.test-filter-menu-max').click(); + await assertSearchCanBeFocused(); + }); + + it('should have correct order for Numeric column', async () => { + await gu.openColumnMenu('Numeric', 'Filter'); + assert.deepEqual(await getValues(), ['2', '3', '22']); + }); + + it('should have correct order for Integer column', async () => { + await gu.openColumnMenu('Int', 'Filter'); + assert.deepEqual(await getValues(), ['2', '3', '22']); + await driver.find('.test-filter-menu-apply-btn'); + }); + + it('should have correct order for Date column', async () => { + await gu.openColumnMenu('Date', 'Filter'); + assert.deepEqual(await getValues(), ['20-12-2011', '22-12-2011', '20-12-2021']); + }); + + describe('Ref', function() { + + it('should have correct order for Numeric column', async () => { + await gu.toggleSidePanel('right', 'open'); + await gu.openColumnMenu('Ref', 'Options'); + await gu.setRefShowColumn('Numeric'); + await gu.openColumnMenu('Ref', 'Filter'); + assert.deepEqual(await getValues(), ['2', '3', '22']); + }); + it('should have correct order for Integer column', async () => { + await gu.setRefShowColumn('Int'); + await gu.openColumnMenu('Ref', 'Filter'); + assert.deepEqual(await getValues(), ['2', '3', '22']); + }); + it('should have correct order for Date column', async () => { + await gu.setRefShowColumn('Date'); + await gu.openColumnMenu('Ref', 'Filter'); + assert.deepEqual(await getValues(), ['20-12-2011', '22-12-2011', '20-12-2021']); + }); + }); + + describe('RefList', function() { + it('should have correct order for Numeric column', async () => { + await gu.openColumnMenu('RefList', 'Options'); + await gu.setRefShowColumn('Numeric'); + await gu.openColumnMenu('RefList', 'Filter'); + assert.deepEqual(await getValues(), ['2', '3', '22']); + }); + it('should have correct order for Integer column', async () => { + await gu.setRefShowColumn('Int'); + await gu.openColumnMenu('RefList', 'Filter'); + assert.deepEqual(await getValues(), ['2', '3', '22']); + }); + it('should have correct order for Date column', async () => { + await gu.setRefShowColumn('Date'); + await gu.openColumnMenu('RefList', 'Filter'); + assert.deepEqual(await getValues(), ['20-12-2011', '22-12-2011', '20-12-2021']); + }); + }); + + describe('id mismatch', function() { + // This test intent to replicate a bug that happened with filters. For the bug to happen we need + // to have a view field row id (here view field of col B) that matches the row id of another + // column (here col A). When this happen, and when col A is hidden, and when users open the + // column menu for B, the filter apply mistakingly to column A values as well, which could + // intail unexpected result depending on the values of A. + + let docId2: string; + before(async () => { + docId2 = await mainSession.tempNewDoc(cleanup, 'ColumnFilterMenu3IdMismatch.grist', {load: false}); + const api = mainSession.createHomeApi(); + await api.applyUserActions(docId2, [ + ['BulkAddRecord', 'Table1', [null, null, null], {A: [1, 3, 3], B: [1, 1, 3]}], + ['RemoveRecord', "_grist_Views_section_field", 1], // Hide 'A' column + ]); + }); + it('filters should work correctly', async function() { + await mainSession.loadDoc(`/doc/${docId2}/p/1`); + + // filter B by {max: 2} + await gu.openColumnMenu('B', 'Filter'); + await gu.setRangeFilterBound('max', '2'); + await driver.find('.test-filter-menu-apply-btn').click(); + + // check filter does not behaves in-correctly (here mostly to show what the problem looked + // like) + assert.notDeepEqual( + await gu.getVisibleGridCells({cols: ['B'], rowNums: [1, 2, 3]}), + [ '1', '', undefined] + ); + + // check filter does behave correctly + assert.deepEqual( + await gu.getVisibleGridCells({cols: ['B'], rowNums: [1, 2, 3]}), + [ '1', '1', ''] + ); + }); + }); + + describe('empty choice columns', function() { + // Previously, a bug would cause an error to be thrown when filtering an empty + // choice or choice list column. This suite replicates that scenario. + + async function assertEmptyRowCount(count: number) { + assert.deepEqual( + await driver.findAll('.test-filter-menu-list label', (e) => e.getText()), + [''] + ); + assert.deepEqual( + await driver.findAll('.test-filter-menu-list .test-filter-menu-count', (e) => e.getText()), + [count.toString()], + ); + } + + async function assertEmptyColumnIsFilterable( + columnType: 'Choice' | 'Choice List' | 'Reference List' + ) { + const columnLabel = `Empty ${columnType}`; + await gu.addColumn(columnLabel); + await gu.setType(new RegExp(`${columnType}$`)); + await gu.openColumnMenu(columnLabel, 'Filter'); + await assertEmptyRowCount(2); + await gu.sendKeys(Key.ESCAPE); + } + + afterEach(() => gu.checkForErrors()); + + it('should not throw an error when filtering empty choice columns', async function() { + await assertEmptyColumnIsFilterable('Choice'); + }); + + it('should not throw an error when filtering empty choice list columns', async function() { + await assertEmptyColumnIsFilterable('Choice List'); + }); + + it('should not throw an error when filtering empty reference list columns', async function() { + // Note: this wasn't impacted by the aforementioned bug; this test is only included for + // completeness. + await assertEmptyColumnIsFilterable('Reference List'); + }); + }); +}); diff --git a/test/nbrowser/SectionFilter.ts b/test/nbrowser/SectionFilter.ts new file mode 100644 index 0000000000..ab4caf17a0 --- /dev/null +++ b/test/nbrowser/SectionFilter.ts @@ -0,0 +1,531 @@ +import {assert, driver, Key, until} from 'mocha-webdriver'; +import * as gu from 'test/nbrowser/gristUtils'; +import {setupTestSuite} from 'test/nbrowser/testUtils'; + +describe('SectionFilter', function() { + this.timeout(60000); + const cleanup = setupTestSuite(); + + describe('Core tests', function() { + + before(async function() { + this.timeout(10000); + const session = await gu.session().teamSite.login(); + await session.tempNewDoc(cleanup); + }); + + it('should be able to open / close filter menu', async () => { + const menu = await gu.openColumnMenu('A', 'Filter'); + assert.equal(await menu.find('.test-filter-menu-list').getText(), 'No matching values'); + await driver.sendKeys(Key.ESCAPE); + await driver.wait(until.stalenessOf(menu)); + }); + + it('should filter out records in response to filter menu selections', async () => { + this.timeout(10000); + + await gu.enterGridRows({col: 'A', rowNum: 1}, [ + ['Apples', '1'], + ['Oranges', '2'], + ['Bananas', '1'], + ['Apples', '2'], + ['Bananas', '1'], + ['Apples', '2'], + ]); + + const menu = await gu.openColumnMenu('A', 'Filter'); + assert.deepEqual(await gu.getFilterMenuState(), [ + { checked: true, value: 'Apples', count: 3}, + { checked: true, value: 'Bananas', count: 2}, + { checked: true, value: 'Oranges', count: 1} + ]); + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3, 4, 5, 6]), + ['Apples', 'Oranges', 'Bananas', 'Apples', 'Bananas', 'Apples']); + + await menu.findContent('label', /Apples/).click(); + assert.deepEqual(await gu.getFilterMenuState(), [ + { checked: false, value: 'Apples', count: 3}, + { checked: true, value: 'Bananas', count: 2}, + { checked: true, value: 'Oranges', count: 1} + ]); + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3]), + ['Oranges', 'Bananas', 'Bananas']); + + await menu.findContent('label', /Apples/).click(); + assert.deepEqual(await gu.getFilterMenuState(), [ + { checked: true, value: 'Apples', count: 3}, + { checked: true, value: 'Bananas', count: 2}, + { checked: true, value: 'Oranges', count: 1} + ]); + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3, 4, 5, 6]), + ['Apples', 'Oranges', 'Bananas', 'Apples', 'Bananas', 'Apples']); + + await driver.sendKeys(Key.ESCAPE); + }); + + it('should undo filter changes on cancel', async () => { + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3, 4, 5, 6]), + ['Apples', 'Oranges', 'Bananas', 'Apples', 'Bananas', 'Apples']); + + const menu = await gu.openColumnMenu('A', 'Filter'); + + await menu.findContent('label', /Apples/).click(); + assert.deepEqual(await gu.getFilterMenuState(), [ + { checked: false, value: 'Apples', count: 3}, + { checked: true, value: 'Bananas', count: 2}, + { checked: true, value: 'Oranges', count: 1} + ]); + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3]), + ['Oranges', 'Bananas', 'Bananas']); + + await menu.find('.test-filter-menu-cancel-btn').click(); + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3, 4, 5, 6]), + ['Apples', 'Oranges', 'Bananas', 'Apples', 'Bananas', 'Apples']); + }); + + it('should display new/updated rows even when only certain values are filtered in', async () => { + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3, 4, 5, 6]), + ['Apples', 'Oranges', 'Bananas', 'Apples', 'Bananas', 'Apples']); + + let menu = await gu.openColumnMenu('A', 'Filter'); + + // Put the filter into the "inclusion" state, with nothing selected initially. + assert.deepEqual( + await driver.findAll('.test-filter-menu-bulk-action:not(:disabled)', (e) => e.getText()), + ['None']); + await driver.findContent('.test-filter-menu-bulk-action', /None/).click(); + assert.deepEqual( + await driver.findAll('.test-filter-menu-bulk-action:not(:disabled)', (e) => e.getText()), + ['All']); + + // Include only "Apples". + await menu.findContent('label', /Apples/).click(); + assert.deepEqual(await gu.getFilterMenuState(), [ + { checked: true, value: 'Apples', count: 3}, + { checked: false, value: 'Bananas', count: 2}, + { checked: false, value: 'Oranges', count: 1} + ]); + + await driver.find('.test-filter-menu-apply-btn').click(); + + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3, 4]), + ['Apples', 'Apples', 'Apples', '']); + + // Update first row to Oranges; it should remain shown. + await gu.getCell(0, 1).click(); + await gu.enterCell('Oranges'); + + // Enter a new row using a keyboard shortcut. + await driver.find('body').sendKeys(Key.chord(await gu.modKey(), Key.ENTER)); + + // Enter a new row by typing in a value into the "add-row". + await driver.find('.gridview_row .record-add .field').click(); + await gu.enterCell('Bananas'); + + // Ensure all 3 changes are visible. + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3, 4, 5, 6]), + ['Oranges', 'Apples', '', 'Apples', 'Bananas', '']); + + // Check that the filter menu looks as expected. + menu = await gu.openColumnMenu('A', 'Filter'); + assert.deepEqual(await gu.getFilterMenuState(), [ + { checked: false, value: '', count: 1}, + { checked: true, value: 'Apples', count: 2}, + { checked: false, value: 'Bananas', count: 3}, + { checked: false, value: 'Oranges', count: 2} + ]); + + // Apply the filter to make it only-Apples again. + await menu.find('.test-filter-menu-apply-btn').click(); + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3]), + ['Apples', 'Apples', '']); + + // Reset the filter + menu = await gu.openColumnMenu('A', 'Filter'); + assert.deepEqual( + await driver.findAll('.test-filter-menu-bulk-action:not([class*=-disabled])', (e) => e.getText()), + ['All', 'None']); + await driver.findContent('.test-filter-menu-bulk-action', /All/).click(); + await menu.find('.test-filter-menu-apply-btn').click(); + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3, 4, 5, 6, 7, 8]), + ['Oranges', 'Oranges', 'Bananas', 'Apples', 'Bananas', '', 'Apples', 'Bananas']); + + // Restore changes of this test case. + await gu.undo(3); + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3, 4, 5, 6]), + ['Apples', 'Oranges', 'Bananas', 'Apples', 'Bananas', 'Apples']); + }); + + it('should display new/updated rows even when filtered, but refilter on menu changes', async () => { + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3, 4, 5, 6]), + ['Apples', 'Oranges', 'Bananas', 'Apples', 'Bananas', 'Apples']); + + let menu = await gu.openColumnMenu('A', 'Filter'); + + await menu.findContent('label', /Apples/).click(); + await driver.find('.test-filter-menu-apply-btn').click(); + + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3]), + ['Oranges', 'Bananas', 'Bananas']); + + // Update Oranges to Apples and make sure it's not filtered out + await (await gu.getCell(0, 1)).click(); + await gu.enterCell('Apples'); + + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3]), + ['Apples', 'Bananas', 'Bananas']); + + // Set back to Oranges and make sure it stays + await driver.sendKeys(Key.UP); + await gu.enterCell('Oranges'); + + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3]), + ['Oranges', 'Bananas', 'Bananas']); + + // Enter two new rows and make sure they're also not filtered out + await driver.find('.gridview_row .record-add .field').click(); + await gu.enterCell('Apples'); + await gu.enterCell('Bananas'); + + // Enter a new row using a keyboard shortcut. + await driver.find('body').sendKeys(Key.chord(await gu.modKey(), Key.ENTER)); + await gu.waitForServer(); + + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3, 4, 5, 6]), + ['Oranges', 'Bananas', 'Bananas', 'Apples', 'Bananas', '']); + + menu = await gu.openColumnMenu('A', 'Filter'); + assert.deepEqual(await gu.getFilterMenuState(), [ + { checked: true, value: '', count: 1}, + { checked: false, value: 'Apples', count: 4}, + { checked: true, value: 'Bananas', count: 3}, + { checked: true, value: 'Oranges', count: 1} + ]); + + await menu.findContent('label', /Apples/).click(); + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3, 4, 5, 6, 7, 8]), + ['Apples', 'Oranges', 'Bananas', 'Apples', 'Bananas', 'Apples', 'Apples', 'Bananas']); + await menu.findContent('label', /Apples/).click(); + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3, 4]), + ['Oranges', 'Bananas', 'Bananas', 'Bananas']); + await driver.sendKeys(Key.ESCAPE); + }); + }); + + describe('Type tests', function() { + + before(async function() { + const session = await gu.session().teamSite.login(); + await session.tempDoc(cleanup, 'FilterTest.grist'); + }); + + it('should properly filter strings', async () => { + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3, 4, 5, 6, 7, 8]), + ['Foo', 'Bar', '1', '2.0', '2016-01-01', '5+6', '', '']); + + const menu = await gu.openColumnMenu('Text', 'Filter'); + assert.deepEqual(await gu.getFilterMenuState(), [ + { checked: true, value: '', count: 1}, + { checked: true, value: '1', count: 1}, + { checked: true, value: '2.0', count: 1}, + { checked: true, value: '5+6', count: 1}, + { checked: true, value: '2016-01-01', count: 1}, + { checked: true, value: 'Bar', count: 1}, + { checked: true, value: 'Foo', count: 1} + ]); + await menu.findContent('label', /^$/).click(); + await menu.findContent('label', /Bar/).click(); + assert.deepEqual(await gu.getVisibleGridCells(0, [1, 2, 3, 4, 5, 6, 7]), + ['Foo', '1', '2.0', '2016-01-01', '5+6', '', undefined]); + await menu.find('.test-filter-menu-cancel-btn').click(); + }); + + + it('should properly filter numbers', async () => { + assert.deepEqual(await gu.getVisibleGridCells(1, [1, 2, 3, 4, 5, 6, 7, 8]), + ['5.00', '6.00', '7.00', '-1.00', 'foo', '0.00', '', '']); + + const menu = await gu.openColumnMenu('Number', 'Filter'); + assert.deepEqual(await gu.getFilterMenuState(), [ + { checked: true, value: '', count: 1}, + { checked: true, value: 'foo', count: 1}, + { checked: true, value: '-1.00', count: 1}, + { checked: true, value: '0.00', count: 1}, + { checked: true, value: '5.00', count: 1}, + { checked: true, value: '6.00', count: 1}, + { checked: true, value: '7.00', count: 1}, + ]); + await menu.findContent('label', /^$/).click(); + await menu.findContent('label', /7/).click(); + await menu.findContent('label', /foo/).click(); + assert.deepEqual(await gu.getVisibleGridCells(1, [1, 2, 3, 4, 5, 6]), + ['5.00', '6.00', '-1.00', '0.00', '', undefined]); + await menu.find('.test-filter-menu-cancel-btn').click(); + }); + + it('should properly filter dates', async () => { + assert.deepEqual(await gu.getVisibleGridCells(2, [1, 2, 3, 4, 5, 6, 7, 8]), + ['2019-06-03', '2019-06-07', '2019-06-05', 'bar', '2019-06-123', '0', '', '']); + + const menu = await gu.openColumnMenu('Date', 'Filter'); + assert.deepEqual(await gu.getFilterMenuState(), [ + { checked: true, value: '', count: 1}, + { checked: true, value: '2019-06-123', count: 1}, + { checked: true, value: 'bar', count: 1}, + { checked: true, value: '0', count: 1}, + { checked: true, value: '2019-06-03', count: 1}, + { checked: true, value: '2019-06-05', count: 1}, + { checked: true, value: '2019-06-07', count: 1}, + ]); + await menu.findContent('label', /^$/).click(); + await menu.findContent('label', /2019-06-05/).click(); + await menu.findContent('label', /bar/).click(); + assert.deepEqual(await gu.getVisibleGridCells(2, [1, 2, 3, 4, 5, 6]), + ['2019-06-03', '2019-06-07', '2019-06-123', '0', '', undefined]); + await menu.find('.test-filter-menu-cancel-btn').click(); + }); + + it('should properly search through list of date to filter', async () => { + const menu = await gu.openColumnMenu('Date', 'Filter'); + assert.lengthOf(await gu.getFilterMenuState(), 7); + await driver.sendKeys('07'); + assert.deepEqual(await gu.getFilterMenuState(), [ + { checked: true, value: '2019-06-07', count: 1} + ]); + assert.deepEqual( + await menu.findAll('.test-filter-menu-list label', (e) => e.getText()), + ['2019-06-07'] + ); + await menu.findContent('.test-filter-menu-bulk-action', /All Shown/).click(); + assert.deepEqual( + await gu.getVisibleGridCells(2, [1, 2]), + ['2019-06-07', ''] + ); + await menu.find('.test-filter-menu-cancel-btn').click(); + }); + + it('should properly filter formulas', async () => { + assert.deepEqual(await gu.getVisibleGridCells(3, [1, 2, 3, 4, 5, 6, 7, 8]), + ['25', '36', '49', '1', '#TypeError', '0', '#TypeError', '']); + + const menu = await gu.openColumnMenu('Formula', 'Filter'); + assert.deepEqual(await gu.getFilterMenuState(), [ + { checked: true, value: '#TypeError', count: 2}, + { checked: true, value: '0', count: 1}, + { checked: true, value: '1', count: 1}, + { checked: true, value: '25', count: 1}, + { checked: true, value: '36', count: 1}, + { checked: true, value: '49', count: 1}, + ]); + + await menu.findContent('label', /0/).click(); + await menu.findContent('label', /#TypeError/).click(); + await menu.findContent('label', /25/).click(); + + assert.deepEqual(await gu.getVisibleGridCells(3, [1, 2, 3, 4, 5]), + ['36', '49', '1', '', undefined]); + await menu.find('.test-filter-menu-cancel-btn').click(); + }); + + it('should properly filter references', async () => { + assert.deepEqual(await gu.getVisibleGridCells(4, [1, 2, 3, 4, 5, 6, 7, 8]), + ['alice', 'carol', 'bob', 'denis', '0', 'denis', '', '']); + + const menu = await gu.openColumnMenu('Reference', 'Filter'); + assert.deepEqual(await gu.getFilterMenuState(), [ + { checked: true, value: '', count: 1}, + { checked: true, value: '#Invalid Ref: 0', count: 1}, + { checked: true, value: '#Invalid Ref: denis', count: 2}, + { checked: true, value: 'alice', count: 1}, + { checked: true, value: 'bob', count: 1}, + { checked: true, value: 'carol', count: 1}, + ]); + + await menu.findContent('label', /^$/).click(); + await menu.findContent('label', /#Invalid Ref: denis/).click(); + await menu.findContent('label', /bob/).click(); + + assert.deepEqual(await gu.getVisibleGridCells(4, [1, 2, 3, 4, 5]), + ['alice', 'carol', '0', '', undefined]); + await menu.find('.test-filter-menu-cancel-btn').click(); + }); + + it('should properly filter choice lists', async () => { + assert.deepEqual(await gu.getVisibleGridCells(5, [1, 2, 3, 4, 5, 6, 7, 8]), + ['Foo\nBar\nBaz', 'Foo\nBar', 'Foo', 'InvalidChoice', 'Baz\nBaz\nBaz', 'Bar\nBaz', '', '']); + + const menu = await gu.openColumnMenu('ChoiceList', 'Filter'); + assert.deepEqual(await gu.getFilterMenuState(), [ + { checked: true, value: '', count: 1}, + { checked: true, value: 'Bar', count: 3}, + { checked: true, value: 'Baz', count: 5}, + { checked: true, value: 'Foo', count: 3}, + { checked: true, value: 'InvalidChoice', count: 1}, + ]); + + // Check that all the choices are rendered in the right colors. + const choiceColors = await menu.findAll( + 'label .test-filter-menu-choice-token', + async (c) => [await c.getCssValue('background-color'), await c.getCssValue('color')] + ); + + assert.deepEqual( + choiceColors, + [ + [ 'rgba(254, 204, 129, 1)', 'rgba(0, 0, 0, 1)' ], + [ 'rgba(53, 253, 49, 1)', 'rgba(0, 0, 0, 1)' ], + [ 'rgba(204, 254, 254, 1)', 'rgba(0, 0, 0, 1)' ], + [ 'rgba(255, 255, 255, 1)', 'rgba(0, 0, 0, 1)' ] + ] + ); + + // Check that Foo is rendered with font options. + const boldFonts = await menu.findAll( + 'label .test-filter-menu-choice-token.font-italic.font-bold', + (c) => c.getText() + ); + + assert.deepEqual(boldFonts, ['Foo']); + + await menu.findContent('label', /^$/).click(); + await menu.findContent('label', /Bar/).click(); + await menu.findContent('label', /Baz/).click(); + + assert.deepEqual(await gu.getVisibleGridCells(5, [1, 2, 3, 4, 5]), + ['Foo\nBar\nBaz', 'Foo\nBar', 'Foo', 'InvalidChoice', '']); + await menu.find('.test-filter-menu-cancel-btn').click(); + }); + + it('should properly filter errors in choice lists', async () => { + assert.deepEqual(await gu.getVisibleGridCells(6, [1, 2, 3, 4, 5, 6, 7, 8]), + ['25.0', '36.0', '49.0', '1.0', '#TypeError', '', '#TypeError', '']); + + await gu.scrollIntoView(gu.getColumnHeader('ChoiceListErrors')); + const menu = await gu.openColumnMenu('ChoiceListErrors', 'Filter'); + assert.deepEqual(await gu.getFilterMenuState(), [ + { checked: true, value: '', count: 1}, + { checked: true, value: '#TypeError', count: 2}, + { checked: true, value: '1.0', count: 1}, + { checked: true, value: '25.0', count: 1}, + { checked: true, value: '36.0', count: 1}, + { checked: true, value: '49.0', count: 1}, + { checked: true, value: 'A', count: 0}, + { checked: true, value: 'B', count: 0}, + { checked: true, value: 'C', count: 0}, + { checked: true, value: 'D', count: 0}, + ]); + + await menu.findContent('label', /^$/).click(); + await menu.findContent('label', /#TypeError/).click(); + await menu.findContent('label', /25\.0/).click(); + await menu.findContent('label', /36\.0/).click(); + await menu.findContent('label', /49\.0/).click(); + + assert.deepEqual(await gu.getVisibleGridCells(6, [1, 2]), + ['1.0', '']); + await menu.find('.test-filter-menu-cancel-btn').click(); + }); + + it('should properly filter choices', async () => { + assert.deepEqual(await gu.getVisibleGridCells(7, [1, 2, 3, 4, 5, 6, 7, 8]), + ['Red', 'Orange', 'Yellow', 'InvalidChoice', '', 'Red', '', '']); + + const menu = await gu.openColumnMenu('Choice', 'Filter'); + assert.deepEqual(await gu.getFilterMenuState(), [ + { checked: true, value: '', count: 2}, + { checked: true, value: 'InvalidChoice', count: 1}, + { checked: true, value: 'Orange', count: 1}, + { checked: true, value: 'Red', count: 2}, + { checked: true, value: 'Yellow', count: 1}, + ]); + + // Check that all the choices are rendered in the right colors. + const choiceColors = await menu.findAll( + 'label .test-filter-menu-choice-token', + async (c) => [await c.getCssValue('background-color'), await c.getCssValue('color')] + ); + + assert.deepEqual( + choiceColors, + [ + [ 'rgba(255, 255, 255, 1)', 'rgba(0, 0, 0, 1)' ], + [ 'rgba(254, 204, 129, 1)', 'rgba(0, 0, 0, 1)' ], + [ 'rgba(252, 54, 59, 1)', 'rgba(255, 255, 255, 1)' ], + [ 'rgba(255, 250, 205, 1)', 'rgba(0, 0, 0, 1)' ] + ] + ); + + // Check that Red is rendered with font options. + const withFonts = await menu.findAll( + 'label .test-filter-menu-choice-token.font-underline.font-strikethrough', + (c) => c.getText() + ); + + assert.deepEqual(withFonts, ['Red']); + + await menu.findContent('label', /InvalidChoice/).click(); + await menu.findContent('label', /Orange/).click(); + await menu.findContent('label', /Yellow/).click(); + + assert.deepEqual(await gu.getVisibleGridCells(7, [1, 2, 3, 4, 5]), + ['Red', '', 'Red', '', '']); + await menu.find('.test-filter-menu-cancel-btn').click(); + }); + + it('should properly filter reference lists', async () => { + assert.deepEqual(await gu.getVisibleGridCells(8, [1, 2, 3, 4, 5, 6, 7, 8]), + ['alice\ncarol', 'bob', 'carol\nbob\nalice', '[u\'denis\']', '[u\'0\']', '[u\'denis\', u\'edward\']', '', '']); + + const menu = await gu.openColumnMenu('ReferenceList', 'Filter'); + assert.deepEqual(await gu.getFilterMenuState(), [ + { checked: true, value: '', count: 1 }, + { checked: true, value: '#Invalid RefList: [u\'0\']', count: 1 }, + { + checked: true, + value: '#Invalid RefList: [u\'denis\', u\'edward\']', + count: 1 + }, + { + checked: true, + value: '#Invalid RefList: [u\'denis\']', + count: 1 + }, + { checked: true, value: 'alice', count: 2 }, + { checked: true, value: 'bob', count: 2 }, + { checked: true, value: 'carol', count: 2 } + ]); + + await menu.findContent('label', /^$/).click(); + await menu.findContent('label', /bob/).click(); + await menu.findContent('label', /#Invalid RefList: \[u'0'\]/).click(); + + assert.deepEqual(await gu.getVisibleGridCells(8, [1, 2, 3, 4, 5]), + ['alice\ncarol', 'carol\nbob\nalice', '[u\'denis\']', '[u\'denis\', u\'edward\']', '']); + await menu.find('.test-filter-menu-cancel-btn').click(); + }); + + it('should reflect the section show column setting in the filter menu', async () => { + // Scroll col 3 into view to make sure col 4 is clickable + await gu.scrollIntoView(gu.getCell(3, 1)); + + // Change the show column setting of the Reference column to 'color'. + await gu.getCell(4, 1).click(); + await gu.toggleSidePanel('right', 'open'); + await driver.find('.test-right-tab-field').click(); + await gu.setRefShowColumn('color'); + + // Open the filter menu for Reference, and check that the values are now from 'color'. + const menu = await gu.openColumnMenu('Reference', 'Filter'); + assert.deepEqual(await gu.getFilterMenuState(), [ + { checked: true, value: '', count: 1 }, + { checked: true, value: '#Invalid Ref: 0', count: 1 }, + { checked: true, value: '#Invalid Ref: denis', count: 2 }, + { checked: true, value: 'blue', count: 1 }, + { checked: true, value: 'green', count: 1 }, + { checked: true, value: 'red', count: 1 } + ]); + + await menu.find('.test-filter-menu-cancel-btn').click(); + }); + }); +}); From 71d8d47323563be5b6eef0f02bb4693c8f02236f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaros=C5=82aw=20Sadzi=C5=84ski?= Date: Fri, 24 May 2024 08:55:15 +0200 Subject: [PATCH 15/62] (core) Fix for ACL page for temporary documents. Summary: When the ACL page was visited by an anonymous user (for a temporary doc), it tried to load the "View as" list, which failed. Apart from example users, it was also trying to load all users the document is shared with by checking the home db. However, since the document is not persisted, it failed. Test Plan: Added new test Reviewers: Spoffy Reviewed By: Spoffy Subscribers: Spoffy Differential Revision: https://phab.getgrist.com/D4257 --- app/server/lib/ActiveDoc.ts | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index d86280bb26..a322077b5b 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -79,7 +79,8 @@ import {schema, SCHEMA_VERSION} from 'app/common/schema'; import {MetaRowRecord, SingleCell} from 'app/common/TableData'; import {TelemetryEvent, TelemetryMetadataByLevel} from 'app/common/Telemetry'; import {FetchUrlOptions, UploadResult} from 'app/common/uploads'; -import {Document as APIDocument, DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI'; +import {Document as APIDocument, DocReplacementOptions, + DocState, DocStateComparison, NEW_DOCUMENT_CODE} from 'app/common/UserAPI'; import {convertFromColumn} from 'app/common/ValueConverter'; import {guessColInfo} from 'app/common/ValueGuesser'; import {parseUserAction} from 'app/common/ValueParser'; @@ -1574,17 +1575,22 @@ export class ActiveDoc extends EventEmitter { }; const isShared = new Set(); - // Collect users the document is shared with. const userId = getDocSessionUserId(docSession); if (!userId) { throw new Error('Cannot determine user'); } - const db = this.getHomeDbManager(); - if (db) { - const access = db.unwrapQueryResult( - await db.getDocAccess({userId, urlId: this.docName}, { - flatten: true, excludeUsersWithoutAccess: true, - })); - result.users = access.users; - result.users.forEach(user => isShared.add(normalizeEmail(user.email))); + + const parsed = parseUrlId(this.docName); + // If this is not a temporary document (i.e. created by anonymous user). + if (parsed.trunkId !== NEW_DOCUMENT_CODE) { + // Collect users the document is shared with. + const db = this.getHomeDbManager(); + if (db) { + const access = db.unwrapQueryResult( + await db.getDocAccess({userId, urlId: this.docName}, { + flatten: true, excludeUsersWithoutAccess: true, + })); + result.users = access.users; + result.users.forEach(user => isShared.add(normalizeEmail(user.email))); + } } // Collect users from user attribute tables. Omit duplicates with users the document is From 7d628db157fc5c41099ef65445fdf98bb18220ef Mon Sep 17 00:00:00 2001 From: George Gevoian Date: Tue, 28 May 2024 15:13:10 -0700 Subject: [PATCH 16/62] (core) Removing virtual tables when they are not needed Summary: Clearing virtual tables after user navigates away from the pages that show them. Leaving them behind will reveal them on the Raw Data page, with a buggy experience as user can't view the data there. Test Plan: Extended tests. Reviewers: paulfitz Reviewed By: paulfitz Subscribers: jarek, georgegevoian Differential Revision: https://phab.getgrist.com/D4258 --- app/client/models/DocData.ts | 6 +- app/client/models/VirtualTable.ts | 112 +++++++++++++++++------------- app/client/ui/AdminPanel.ts | 1 + app/client/ui/AdminPanelCss.ts | 4 +- app/client/ui/TimingPage.ts | 4 +- app/client/ui/WebhookPage.ts | 7 +- test/nbrowser/Timing.ts | 19 +++++ 7 files changed, 96 insertions(+), 57 deletions(-) diff --git a/app/client/models/DocData.ts b/app/client/models/DocData.ts index 747a6f1905..2c2234fd59 100644 --- a/app/client/models/DocData.ts +++ b/app/client/models/DocData.ts @@ -186,10 +186,14 @@ export class DocData extends BaseDocData { return this.sendActions([action], optDesc).then((retValues) => retValues[0]); } - public registerVirtualTable(tableId: string, Cons: typeof TableData) { + public registerVirtualTableFactory(tableId: string, Cons: typeof TableData) { this._virtualTablesFunc.set(tableId, Cons); } + public unregisterVirtualTableFactory(tableId: string) { + this._virtualTablesFunc.delete(tableId); + } + // See documentation of sendActions(). private async _sendActionsImpl(actions: UserAction[], optDesc?: string): Promise { const tableName = String(actions[0]?.[1]); diff --git a/app/client/models/VirtualTable.ts b/app/client/models/VirtualTable.ts index 2bcab26ef8..c955beb4c0 100644 --- a/app/client/models/VirtualTable.ts +++ b/app/client/models/VirtualTable.ts @@ -1,6 +1,5 @@ import { reportError } from 'app/client/models/errors'; import { GristDoc } from 'app/client/components/GristDoc'; -import { DocData } from 'app/client/models/DocData'; import { TableData } from 'app/client/models/TableData'; import { concatenateSummaries, summarizeStoredAndUndo } from 'app/common/ActionSummarizer'; import { TableDelta } from 'app/common/ActionSummary'; @@ -8,7 +7,6 @@ import { ProcessedAction } from 'app/common/AlternateActions'; import { DisposableWithEvents } from 'app/common/DisposableWithEvents'; import { DocAction, TableDataAction, UserAction } from 'app/common/DocActions'; import { DocDataCache } from 'app/common/DocDataCache'; -import { ColTypeMap } from 'app/common/TableData'; import { RowRecord } from 'app/plugin/GristData'; import debounce = require('lodash/debounce'); @@ -43,6 +41,7 @@ export interface IEdit { export interface IExternalTable { name: string; // the tableId of the virtual table (e.g. GristHidden_WebhookTable) initialActions: DocAction[]; // actions to create the table. + destroyActions?: DocAction[]; // actions to destroy the table (auto generated if not defined), pass [] to disable. fetchAll(): Promise; // get initial state of the table. sync(editor: IEdit): Promise; // incorporate external changes. beforeEdit(editor: IEdit): Promise; // called prior to committing a change. @@ -63,43 +62,39 @@ export class VirtualTableData extends TableData { public ext: IExternalTable; public cache: DocDataCache; - constructor(docData: DocData, tableId: string, tableData: TableDataAction|null, columnTypes: ColTypeMap) { - super(docData, tableId, tableData, columnTypes); - } - - public setExt(_ext: IExternalTable) { - this.ext = _ext; - this.cache = new DocDataCache(this.ext.initialActions); - } - - public get name() { - return this.ext.name; - } - - public fetchData() { + public override fetchData() { return super.fetchData(async () => { const data = await this.ext.fetchAll(); - this.cache.docData.getTable(this.name)?.loadData(data); + this.cache.docData.getTable(this.getName())?.loadData(data); return data; }); } - public async sendTableActions(userActions: UserAction[]): Promise { + public override async sendTableActions(userActions: UserAction[]): Promise { const actions = await this._sendTableActionsCore(userActions, {isUser: true}); await this.ext.afterEdit(this._editor(actions)); return actions.map(action => action.retValues); } - public sync() { - return this.ext.sync(this._editor()); - } - - public async sendTableAction(action: UserAction): Promise { + public override async sendTableAction(action: UserAction): Promise { const retValues = await this.sendTableActions([action]); return retValues[0]; } + public setExt(_ext: IExternalTable) { + this.ext = _ext; + this.cache = new DocDataCache(this.ext.initialActions); + } + + public getName() { + return this.ext.name; + } + + public sync() { + return this.ext.sync(this._editor()); + } + public async schemaChange() { await this.ext.afterAnySchemaChange(this._editor()); } @@ -108,7 +103,7 @@ export class VirtualTableData extends TableData { const summary = concatenateSummaries( actions .map(action => summarizeStoredAndUndo(action.stored, action.undo))); - const delta = summary.tableDeltas[this.name]; + const delta = summary.tableDeltas[this.getName()]; return { actions, delta, @@ -135,7 +130,7 @@ export class VirtualTableData extends TableData { } const actions = await this.cache.sendTableActions(userActions); if (isUser) { - const newTable = await this.cache.docData.requireTable(this.name); + const newTable = await this.cache.docData.requireTable(this.getName()); try { await this.ext.beforeEdit({ ...this._editor(actions), @@ -155,7 +150,7 @@ export class VirtualTableData extends TableData { this.docData.receiveAction(docAction); this.cache.docData.receiveAction(docAction); if (isUser) { - const code = `ext-${this.name}-${_counterForUndoActions}`; + const code = `ext-${this.getName()}-${_counterForUndoActions}`; _counterForUndoActions++; this.gristDoc.getUndoStack().pushAction({ actionNum: code, @@ -197,46 +192,67 @@ export class VirtualTableData extends TableData { * one second after last call (or at most 2 seconds after the first * call). */ -export class VirtualTable { - public lazySync = debounce(this.sync, 1000, { +export class VirtualTableRegistration extends DisposableWithEvents { + public lazySync = debounce(this._sync, 1000, { maxWait: 2000, trailing: true, }); - public tableData: VirtualTableData; + private _tableData: VirtualTableData; - public constructor(private _owner: DisposableWithEvents, - _gristDoc: GristDoc, - _ext: IExternalTable) { - if (!_gristDoc.docModel.docData.getTable(_ext.name)) { + constructor(gristDoc: GristDoc, ext: IExternalTable) { + super(); + if (!gristDoc.docModel.docData.getTable(ext.name)) { - // register the virtual table - _gristDoc.docModel.docData.registerVirtualTable(_ext.name, VirtualTableData); + // Register the virtual table + gristDoc.docModel.docData.registerVirtualTableFactory(ext.name, VirtualTableData); // then process initial actions - for (const action of _ext.initialActions) { - _gristDoc.docData.receiveAction(action); + for (const action of ext.initialActions) { + gristDoc.docData.receiveAction(action); } - // pass in gristDoc and external interface - this.tableData = _gristDoc.docModel.docData.getTable(_ext.name)! as VirtualTableData; + this._tableData = gristDoc.docModel.docData.getTable(ext.name)! as VirtualTableData; //this.tableData.docApi = this.docApi; - this.tableData.gristDoc = _gristDoc; - this.tableData.setExt(_ext); + this._tableData.gristDoc = gristDoc; + this._tableData.setExt(ext); // subscribe to schema changes - this.tableData.schemaChange().catch(e => reportError(e)); - _owner.listenTo(_gristDoc, 'schemaUpdateAction', () => this.tableData.schemaChange()); + this._tableData.schemaChange().catch(e => reportError(e)); + this.listenTo(gristDoc, 'schemaUpdateAction', () => this._tableData.schemaChange()); } else { - this.tableData = _gristDoc.docModel.docData.getTable(_ext.name)! as VirtualTableData; + throw new Error(`Virtual table ${ext.name} already exists`); } - // debounce is typed as returning a promise, but doesn't appear to actually do so? + // debounce is typed as returning a promise, but doesn't appear to actually //do so? Promise.resolve(this.lazySync()).catch(e => reportError(e)); + + this.onDispose(() => { + const reverse = ext.destroyActions ?? generateDestroyActions(ext.initialActions); + reverse.forEach(action => gristDoc.docModel.docData.receiveAction(action)); + gristDoc.docModel.docData.unregisterVirtualTableFactory(ext.name); + }); } - public async sync() { - if (this._owner.isDisposed()) { + private async _sync() { + if (this.isDisposed()) { return; } - await this.tableData.sync(); + await this._tableData.sync(); } } + +/** + * This is a helper method that generates undo actions for actions that create a virtual + * table. It just removes everything using the ids in the initial actions. It tries to fail + * if actions are more complex than simple create table/columns actions. + */ +function generateDestroyActions(initialActions: DocAction[]): DocAction[] { + return initialActions.map(action => { + switch (action[0]) { + case 'AddTable': return ['RemoveTable', action[1]]; + case 'AddColumn': return ['RemoveColumn', action[1]]; + case 'AddRecord': return ['RemoveRecord', action[1], action[2]]; + case 'BulkAddRecord': return ['BulkRemoveRecord', action[1], action[2]]; + default: throw new Error(`Cannot generate destroy action for ${action[0]}`); + } + }).reverse() as unknown as DocAction[]; +} diff --git a/app/client/ui/AdminPanel.ts b/app/client/ui/AdminPanel.ts index dfecb1ff8a..b726b66779 100644 --- a/app/client/ui/AdminPanel.ts +++ b/app/client/ui/AdminPanel.ts @@ -109,6 +109,7 @@ Please log in as an administrator.`)), url: dom('pre', `/admin?boot-key=${exampleKey}`) }), ), + testId('admin-panel-error'), ]); } diff --git a/app/client/ui/AdminPanelCss.ts b/app/client/ui/AdminPanelCss.ts index b4bd9709b8..9fb8b15fe8 100644 --- a/app/client/ui/AdminPanelCss.ts +++ b/app/client/ui/AdminPanelCss.ts @@ -2,13 +2,13 @@ import {transition} from 'app/client/ui/transitions'; import {toggle} from 'app/client/ui2018/checkbox'; import {mediaSmall, testId, theme, vars} from 'app/client/ui2018/cssVars'; import {icon} from 'app/client/ui2018/icons'; -import {dom, DomContents, IDisposableOwner, Observable, styled} from 'grainjs'; +import {dom, DomContents, DomElementArg, IDisposableOwner, Observable, styled} from 'grainjs'; export function HidableToggle(owner: IDisposableOwner, value: Observable) { return toggle(value, dom.hide((use) => use(value) === null)); } -export function AdminSection(owner: IDisposableOwner, title: DomContents, items: DomContents[]) { +export function AdminSection(owner: IDisposableOwner, title: DomContents, items: DomElementArg[]) { return cssSection( cssSectionTitle(title), ...items, diff --git a/app/client/ui/TimingPage.ts b/app/client/ui/TimingPage.ts index c64cac283d..14d0ab9474 100644 --- a/app/client/ui/TimingPage.ts +++ b/app/client/ui/TimingPage.ts @@ -2,7 +2,7 @@ import BaseView = require('app/client/components/BaseView'); import {GristDoc} from 'app/client/components/GristDoc'; import {ViewSectionHelper} from 'app/client/components/ViewLayout'; import {makeT} from 'app/client/lib/localization'; -import {IEdit, IExternalTable, VirtualTable} from 'app/client/models/VirtualTable'; +import {IEdit, IExternalTable, VirtualTableRegistration} from 'app/client/models/VirtualTable'; import {urlState} from 'app/client/models/gristUrlState'; import {docListHeader} from 'app/client/ui/DocMenuCss'; import {isNarrowScreenObs, mediaSmall} from 'app/client/ui2018/cssVars'; @@ -163,7 +163,7 @@ export class TimingPage extends DisposableWithEvents { // And wire up the UI. const ext = this.autoDispose(new TimingExternalTable(data)); - new VirtualTable(this, this._gristDoc, ext); + this.autoDispose(new VirtualTableRegistration(this._gristDoc, ext)); this._data.set(data); } } diff --git a/app/client/ui/WebhookPage.ts b/app/client/ui/WebhookPage.ts index f04bc10416..9e263fc0c7 100644 --- a/app/client/ui/WebhookPage.ts +++ b/app/client/ui/WebhookPage.ts @@ -2,7 +2,7 @@ import {GristDoc} from 'app/client/components/GristDoc'; import {ViewSectionHelper} from 'app/client/components/ViewLayout'; import {makeT} from 'app/client/lib/localization'; import {reportMessage, reportSuccess} from 'app/client/models/errors'; -import {IEdit, IExternalTable, VirtualTable} from 'app/client/models/VirtualTable'; +import {IEdit, IExternalTable, VirtualTableRegistration} from 'app/client/models/VirtualTable'; import {docListHeader} from 'app/client/ui/DocMenuCss'; import {bigPrimaryButton} from 'app/client/ui2018/buttons'; import {mediaSmall, testId} from 'app/client/ui2018/cssVars'; @@ -337,7 +337,7 @@ class WebhookExternalTable implements IExternalTable { export class WebhookPage extends DisposableWithEvents { public docApi = this.gristDoc.docPageModel.appModel.api.getDocAPI(this.gristDoc.docId()); - public sharedTable: VirtualTable; + public sharedTable: VirtualTableRegistration; private _webhookExternalTable: WebhookExternalTable; @@ -345,10 +345,9 @@ export class WebhookPage extends DisposableWithEvents { super(); //this._webhooks = observableArray(); this._webhookExternalTable = new WebhookExternalTable(this.docApi); - const table = new VirtualTable(this, gristDoc, this._webhookExternalTable); + const table = this.autoDispose(new VirtualTableRegistration(gristDoc, this._webhookExternalTable)); this.listenTo(gristDoc, 'webhooks', async () => { await table.lazySync(); - }); } diff --git a/test/nbrowser/Timing.ts b/test/nbrowser/Timing.ts index 7c15115102..b21e130438 100644 --- a/test/nbrowser/Timing.ts +++ b/test/nbrowser/Timing.ts @@ -147,6 +147,25 @@ describe("Timing", function () { await driver.navigate().refresh(); await gu.waitForUrl('/settings'); }); + + it('clears virtual table when navigated away', async function() { + // Start timing and go to results. + await startTiming.click(); + await modal.wait(); + await optionReload.click(); + await modalConfirm.click(); + + // Wait for the results page. + await gu.waitToPass(async () => { + assert.isTrue(await driver.findContentWait('div', 'Formula timer', 1000).isDisplayed()); + assert.equal(await gu.getCell(0, 1).getText(), 'Table1'); + }); + + // Now go to the raw data page, and make sure we see only Table1. + await driver.find('.test-tools-raw').click(); + await driver.findWait('.test-raw-data-list', 2000); + assert.deepEqual(await driver.findAll('.test-raw-data-table-id', e => e.getText()), ['Table1']); + }); }); const element = (testId: string) => ({ From eb6a6bd3be3ec71775a9ef93bcd0cd038b74bd8e Mon Sep 17 00:00:00 2001 From: Camille L Date: Tue, 28 May 2024 15:48:41 +0000 Subject: [PATCH 17/62] Translated using Weblate (French) Currently translated at 98.8% (1282 of 1297 strings) Translation: Grist/client Translate-URL: https://hosted.weblate.org/projects/grist/client/fr/ --- static/locales/fr.client.json | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/static/locales/fr.client.json b/static/locales/fr.client.json index b70c488b1d..e2a8c84968 100644 --- a/static/locales/fr.client.json +++ b/static/locales/fr.client.json @@ -311,7 +311,18 @@ "For currency columns": "Pour les colonnes de devises", "Hard reset of data engine": "Réinitialisation du moteur de données", "Locale": "Langue", - "For number and date formats": "Pour les colonnes de nombre et date" + "For number and date formats": "Pour les colonnes de nombre et date", + "Base doc URL: {{docApiUrl}}": "URL de base pour ce document : {{docApiUrl}}", + "Document ID to use whenever the REST API calls for {{docId}}. See {{apiURL}}": "ID du document à utiliser lorsque l'API REST fait appel à {{docId}}. Voir {{apiURL}}", + "Python version used": "Version de Python utilisée", + "Notify other services on doc changes": "Informer d'autres services des changements du documents", + "Manage webhooks": "Gérer les points d'ancrage Web", + "ID for API use": "ID pour l'utilisation de l'API", + "Find slow formulas": "Trouver les formules lentes", + "python2 (legacy)": "python2 (encien)", + "Try API calls from the browser": "Essayer les appels API à partir du navigateur", + "Time Zone": "Fuseau horaire", + "python3 (recommended)": "python3 (recommandé)" }, "DocumentUsage": { "Usage statistics are only available to users with full access to the document data.": "Les statistiques d'utilisation ne sont disponibles qu'aux utilisateurs ayant un accès complet aux données du document.", @@ -1509,7 +1520,8 @@ "Updates": "Mises à jour", "unconfigured": "non configuré", "unknown": "inconnu", - "Auto-check when this page loads": "Vérification automatique au chargement de cette page" + "Auto-check when this page loads": "Vérification automatique au chargement de cette page", + "Sandbox settings for data engine": "Paramètres de la Sandbox pour le moteur de données" }, "Field": { "No choices configured": "Aucun choix configuré", From 519fdfa99ea67bfc8bef01dcef2e71f755da5eb0 Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Wed, 29 May 2024 20:02:00 +0200 Subject: [PATCH 18/62] feat: add new translations (#1004) --- app/client/ui/AppHeader.ts | 6 ++--- static/locales/en.client.json | 48 +++++++++++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/app/client/ui/AppHeader.ts b/app/client/ui/AppHeader.ts index 109eeb2aba..8449d1c98a 100644 --- a/app/client/ui/AppHeader.ts +++ b/app/client/ui/AppHeader.ts @@ -118,7 +118,7 @@ export class AppHeader extends Disposable { // Show 'Organization Settings' when on a home page of a valid org. (!this._docPageModel && this._currentOrg && !this._currentOrg.owner ? menuItem(() => manageTeamUsersApp({app: this._appModel}), - 'Manage Team', testId('orgmenu-manage-team'), + t('Manage Team'), testId('orgmenu-manage-team'), dom.cls('disabled', !roles.canEditAccess(this._currentOrg.access))) : // Don't show on doc pages, or for personal orgs. null), @@ -153,12 +153,12 @@ export class AppHeader extends Disposable { (isBillingManager ? menuItemLink( urlState().setLinkUrl({billing: 'billing'}), - 'Billing Account', + t('Billing Account'), testId('orgmenu-billing'), ) : menuItem( () => null, - 'Billing Account', + t('Billing Account'), dom.cls('disabled', true), testId('orgmenu-billing'), ) diff --git a/static/locales/en.client.json b/static/locales/en.client.json index 98f05d4832..4513915ed7 100644 --- a/static/locales/en.client.json +++ b/static/locales/en.client.json @@ -118,7 +118,9 @@ "Legacy": "Legacy", "Personal Site": "Personal Site", "Team Site": "Team Site", - "Grist Templates": "Grist Templates" + "Grist Templates": "Grist Templates", + "Billing Account": "Billing Account", + "Manage Team": "Manage Team" }, "AppModel": { "This team site is suspended. Documents can be read, but not modified.": "This team site is suspended. Documents can be read, but not modified." @@ -327,7 +329,17 @@ "Time Zone": "Time Zone", "Try API calls from the browser": "Try API calls from the browser", "python2 (legacy)": "python2 (legacy)", - "python3 (recommended)": "python3 (recommended)" + "python3 (recommended)": "python3 (recommended)", + "Cancel": "Cancel", + "Force reload the document while timing formulas, and show the result.": "Force reload the document while timing formulas, and show the result.", + "Formula timer": "Formula timer", + "Reload data engine": "Reload data engine", + "Reload data engine?": "Reload data engine?", + "Start timing": "Start timing", + "Stop timing...": "Stop timing...", + "Time reload": "Time reload", + "Timing is on": "Timing is on", + "You can make changes to the document, then stop timing to see the results.": "You can make changes to the document, then stop timing to see the results." }, "DocumentUsage": { "Attachments Size": "Size of Attachments", @@ -525,7 +537,8 @@ "Trash": "Trash", "Workspace will be moved to Trash.": "Workspace will be moved to Trash.", "Workspaces": "Workspaces", - "Tutorial": "Tutorial" + "Tutorial": "Tutorial", + "Terms of service": "Terms of service" }, "Importer": { "Merge rows that match these fields:": "Merge rows that match these fields:", @@ -1017,7 +1030,9 @@ "Add conditional style": "Add conditional style", "Error in style rule": "Error in style rule", "Row Style": "Row Style", - "Rule must return True or False": "Rule must return True or False" + "Rule must return True or False": "Rule must return True or False", + "Conditional Style": "Conditional Style", + "IF...": "IF..." }, "CurrencyPicker": { "Invalid currency": "Invalid currency" @@ -1536,7 +1551,20 @@ "Security Settings": "Security Settings", "Updates": "Updates", "unconfigured": "unconfigured", - "unknown": "unknown" + "unknown": "unknown", + "Administrator Panel Unavailable": "Administrator Panel Unavailable", + "Authentication": "Authentication", + "Check failed.": "Check failed.", + "Check succeeded.": "Check succeeded.", + "Current authentication method": "Current authentication method", + "Details": "Details", + "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.": "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.", + "No fault detected.": "No fault detected.", + "Notes": "Notes", + "Or, as a fallback, you can set: {{bootKey}} in the environment and visit: {{url}}": "Or, as a fallback, you can set: {{bootKey}} in the environment and visit: {{url}}", + "Results": "Results", + "Self Checks": "Self Checks", + "You do not have access to the administrator panel.\nPlease log in as an administrator.": "You do not have access to the administrator panel.\nPlease log in as an administrator." }, "Columns": { "Remove Column": "Remove Column" @@ -1587,5 +1615,15 @@ "Custom": "Custom", "Form": "Form", "Table": "Table" + }, + "TimingPage": { + "Average Time (s)": "Average Time (s)", + "Column ID": "Column ID", + "Formula timer": "Formula timer", + "Loading timing data. Don't close this tab.": "Loading timing data. Don't close this tab.", + "Max Time (s)": "Max Time (s)", + "Number of Calls": "Number of Calls", + "Table ID": "Table ID", + "Total Time (s)": "Total Time (s)" } } From ae48fb6b0654dc9edbe0355e76b8b4dece3ddc4a Mon Sep 17 00:00:00 2001 From: Spoffy <4805393+Spoffy@users.noreply.github.com> Date: Thu, 30 May 2024 15:32:32 +0100 Subject: [PATCH 19/62] Removes spacing from admin page auth translation key (#1001) Updates the authentication message on the admin page, removing newlines and tabs. This cleans up the formatting of the resulting translation key (in `en.client.json`). Context: https://github.com/gristlabs/grist-core/pull/987#discussion_r1603799796 --- app/client/ui/AdminPanel.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/client/ui/AdminPanel.ts b/app/client/ui/AdminPanel.ts index b726b66779..37d4f4e205 100644 --- a/app/client/ui/AdminPanel.ts +++ b/app/client/ui/AdminPanel.ts @@ -237,8 +237,8 @@ Please log in as an administrator.`)), private _buildAuthenticationNotice(owner: IDisposableOwner) { return t('Grist allows different types of authentication to be configured, including SAML and OIDC. \ - We recommend enabling one of these if Grist is accessible over the network or being made available \ - to multiple people.'); +We recommend enabling one of these if Grist is accessible over the network or being made available \ +to multiple people.'); } private _buildUpdates(owner: MultiHolder) { From 0ce7cda8d252d34540caa6b3910b7aa6c366b068 Mon Sep 17 00:00:00 2001 From: Roman Holinec <3ko@pixeon.sk> Date: Thu, 30 May 2024 02:41:38 +0000 Subject: [PATCH 20/62] Translated using Weblate (Slovak) Currently translated at 7.2% (94 of 1297 strings) Translation: Grist/client Translate-URL: https://hosted.weblate.org/projects/grist/client/sk/ --- static/locales/sk.client.json | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/static/locales/sk.client.json b/static/locales/sk.client.json index 910f031b51..901fad8d67 100644 --- a/static/locales/sk.client.json +++ b/static/locales/sk.client.json @@ -78,9 +78,37 @@ "Support Grist": "Podpora Grist", "Upgrade Plan": "Plán Inovácie", "Sign In": "Prihlásiť sa", - "Use This Template": "Použiť túto Šablónu" + "Use This Template": "Použiť túto Šablónu", + "Sign Up": "Prihlásiť sa" }, "ViewAsDropdown": { - "View As": "Zobraziť Ako" + "View As": "Zobraziť Ako", + "Users from table": "Používatelia z tabuľky", + "Example Users": "Príklady používateľov" + }, + "ActionLog": { + "Table {{tableId}} was subsequently removed in action #{{actionNum}}": "Tabuľka {{tableId}} bola následne odstránená v akcii #{{actionNum}}", + "Action Log failed to load": "Nepodarilo sa načítať denník akcií", + "Column {{colId}} was subsequently removed in action #{{action.actionNum}}": "Stĺpec {{colId}} bol následne odstránený v akcii #{{action.actionNum}}", + "This row was subsequently removed in action {{action.actionNum}}": "Tento riadok bol následne odstránený v akcii {{action.actionNum}}", + "All tables": "Všetky tabuľky" + }, + "ApiKey": { + "By generating an API key, you will be able to make API calls for your own account.": "Vygenerovaním kľúča API budete môcť uskutočňovať volania API pre svoj vlastný účet.", + "This API key can be used to access your account via the API. Don’t share your API key with anyone.": "Tento kľúč API je možné použiť na prístup k vášmu účtu prostredníctvom rozhrania API. Nezdieľajte svoj API kľúč s nikým.", + "You're about to delete an API key. This will cause all future requests using this API key to be rejected. Do you still want to delete?": "Chystáte sa odstrániť kľúč API. To spôsobí, že všetky budúce požiadavky používajúce tento kľúč API budú odmietnuté. Stále chcete odstrániť?", + "Click to show": "Kliknutím zobraziť", + "Create": "Vytvoriť", + "Remove": "Odobrať", + "Remove API Key": "Odobrať kľúč API", + "This API key can be used to access this account anonymously via the API.": "Tento kľúč API možno použiť na anonymný prístup k tomuto účtu prostredníctvom rozhrania API." + }, + "AddNewButton": { + "Add New": "Pridať Nový" + }, + "App": { + "Description": "Popis", + "Key": "Kľúč", + "Memory Error": "Chyba pamäte" } } From fa5aa6af181eac5de17e71ef9f5738ecfa15f0db Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 31 May 2024 09:32:46 -0400 Subject: [PATCH 21/62] automated update to translation keys (#987) Co-authored-by: Paul's Grist Bot --- static/locales/en.client.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/static/locales/en.client.json b/static/locales/en.client.json index 4513915ed7..3ba12ffcc9 100644 --- a/static/locales/en.client.json +++ b/static/locales/en.client.json @@ -1564,7 +1564,8 @@ "Or, as a fallback, you can set: {{bootKey}} in the environment and visit: {{url}}": "Or, as a fallback, you can set: {{bootKey}} in the environment and visit: {{url}}", "Results": "Results", "Self Checks": "Self Checks", - "You do not have access to the administrator panel.\nPlease log in as an administrator.": "You do not have access to the administrator panel.\nPlease log in as an administrator." + "You do not have access to the administrator panel.\nPlease log in as an administrator.": "You do not have access to the administrator panel.\nPlease log in as an administrator.", + "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.": "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people." }, "Columns": { "Remove Column": "Remove Column" From 6b061a6fd1bb2dca051c14b632ff5d02bdb19892 Mon Sep 17 00:00:00 2001 From: Camille L Date: Fri, 31 May 2024 08:07:34 +0000 Subject: [PATCH 22/62] Translated using Weblate (French) Currently translated at 98.7% (1316 of 1333 strings) Translation: Grist/client Translate-URL: https://hosted.weblate.org/projects/grist/client/fr/ --- static/locales/fr.client.json | 44 +++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/static/locales/fr.client.json b/static/locales/fr.client.json index e2a8c84968..1c64e75c48 100644 --- a/static/locales/fr.client.json +++ b/static/locales/fr.client.json @@ -114,7 +114,9 @@ "Legacy": "Ancienne version", "Personal Site": "Espace personnel", "Team Site": "Espace d'équipe", - "Grist Templates": "Modèles Grist" + "Grist Templates": "Modèles Grist", + "Billing Account": "Compte de facturation", + "Manage Team": "Gestion de l'équipe" }, "AppModel": { "This team site is suspended. Documents can be read, but not modified.": "Le site de cette équipe est suspendu. Les documents peuvent être lus, mais pas modifiés." @@ -322,7 +324,18 @@ "python2 (legacy)": "python2 (encien)", "Try API calls from the browser": "Essayer les appels API à partir du navigateur", "Time Zone": "Fuseau horaire", - "python3 (recommended)": "python3 (recommandé)" + "python3 (recommended)": "python3 (recommandé)", + "Start timing": "Début du chrono", + "Time reload": "Durée du rechargement", + "Stop timing...": "Arrêter le chrono...", + "Cancel": "Annuler", + "Reload data engine": "Recharger le moteur de données", + "Reload data engine?": "Recharger le moteur de données ?", + "Force reload the document while timing formulas, and show the result.": "Forcer le rechargement du document pendant le chronométrage des formules et afficher le résultat.", + "Formula timer": "Chronomètre de formule", + "Timing is on": "Le chronomètre tourne", + "You can make changes to the document, then stop timing to see the results.": "Vous pouvez apporter des modifications au document, puis arrêter le chronométrage pour voir les résultats.", + "Formula times": "Minuteur de formule" }, "DocumentUsage": { "Usage statistics are only available to users with full access to the document data.": "Les statistiques d'utilisation ne sont disponibles qu'aux utilisateurs ayant un accès complet aux données du document.", @@ -1064,7 +1077,9 @@ "Add another rule": "Ajouter une autre règle", "Error in style rule": "Erreur dans la règle de style", "Row Style": "Style de ligne", - "Rule must return True or False": "La règle doit retourner Vrai ou Faux" + "Rule must return True or False": "La règle doit retourner Vrai ou Faux", + "Conditional Style": "Style conditionnel", + "IF...": "SI..." }, "CurrencyPicker": { "Invalid currency": "Devise invalide" @@ -1521,7 +1536,18 @@ "unconfigured": "non configuré", "unknown": "inconnu", "Auto-check when this page loads": "Vérification automatique au chargement de cette page", - "Sandbox settings for data engine": "Paramètres de la Sandbox pour le moteur de données" + "Sandbox settings for data engine": "Paramètres de la Sandbox pour le moteur de données", + "Check failed.": "La vérification a échoué.", + "Check succeeded.": "Vérification réussie.", + "No fault detected.": "Aucun défaut n'a été détecté.", + "Notes": "Notes", + "Authentication": "Authentification", + "Administrator Panel Unavailable": "Panneau d'administration indisponible", + "Current authentication method": "Méthode actuelle d'authentification", + "Details": "Détails", + "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.": "Grist permet de configurer différents types d'authentification, notamment SAML et OIDC. Nous recommandons d'activer l'un de ces types d'authentification si Grist est accessible via le réseau ou s'il est mis à la disposition de plusieurs personnes.", + "You do not have access to the administrator panel.\nPlease log in as an administrator.": "Vous n'avez pas accès au panneau d'administrateur.\nVeuillez vous connecter en tant qu'administrateur.", + "Results": "Résultats" }, "Field": { "No choices configured": "Aucun choix configuré", @@ -1586,5 +1612,15 @@ "Card List": "Liste de fiches", "Card": "Fiche", "Calendar": "Calendrier" + }, + "TimingPage": { + "Max Time (s)": "Temps maximum (s)", + "Average Time (s)": "Temps moyen (s)", + "Column ID": "ID de la colonne", + "Loading timing data. Don't close this tab.": "Chargement des données de chronométrage. Ne pas fermer cet onglet.", + "Formula timer": "Chronomètre de formule", + "Number of Calls": "Nombre d'appels", + "Table ID": "ID de la table", + "Total Time (s)": "Temps total" } } From 403a4806c50791a70821cd7a1a80a0bf2f42aa83 Mon Sep 17 00:00:00 2001 From: gallegonovato Date: Fri, 31 May 2024 09:05:58 +0000 Subject: [PATCH 23/62] Translated using Weblate (Spanish) Currently translated at 97.5% (1301 of 1333 strings) Translation: Grist/client Translate-URL: https://hosted.weblate.org/projects/grist/client/es/ --- static/locales/es.client.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/static/locales/es.client.json b/static/locales/es.client.json index f3807e53fa..891afbd2eb 100644 --- a/static/locales/es.client.json +++ b/static/locales/es.client.json @@ -102,7 +102,9 @@ "Legacy": "Legado", "Personal Site": "Sitio Personal", "Team Site": "Sitio de Equipo", - "Grist Templates": "Plantillas Grist" + "Grist Templates": "Plantillas Grist", + "Manage Team": "Administrar equipo", + "Billing Account": "Cuenta de facturación" }, "CellContextMenu": { "Clear cell": "Borrar celda", @@ -277,7 +279,9 @@ "python2 (legacy)": "python2 (legado)", "Notify other services on doc changes": "Notificar a otros servicios los cambios de documentos", "Python": "Python", - "python3 (recommended)": "python3 (recomendado)" + "python3 (recommended)": "python3 (recomendado)", + "Cancel": "Cancelar", + "Force reload the document while timing formulas, and show the result.": "Fuerza la recarga del documento mientras sincronizas las fórmulas y muestra el resultado." }, "DuplicateTable": { "Copy all data in addition to the table structure.": "Copiar todos los datos además de la estructura de la tabla.", From d480eca71c631df23aa6f931ebc18d5972a55e40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C4=8Dek=20Prijatelj?= Date: Sat, 1 Jun 2024 06:13:11 +0000 Subject: [PATCH 24/62] Translated using Weblate (Slovenian) Currently translated at 100.0% (1333 of 1333 strings) Translation: Grist/client Translate-URL: https://hosted.weblate.org/projects/grist/client/sl/ --- static/locales/sl.client.json | 48 +++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/static/locales/sl.client.json b/static/locales/sl.client.json index 966bf905d6..180d32132f 100644 --- a/static/locales/sl.client.json +++ b/static/locales/sl.client.json @@ -262,7 +262,8 @@ "Access Details": "Podrobnosti o dostopu", "Workspaces": "Delovni prostori", "Workspace will be moved to Trash.": "Delovni prostor se bo premaknil v koš.", - "Examples & Templates": "Predloge" + "Examples & Templates": "Predloge", + "Terms of service": "Pogoji storitve" }, "OnBoardingPopups": { "Finish": "Zaključek", @@ -357,7 +358,9 @@ "Personal Site": "Osebna stran", "Team Site": "Spletna stran ekipe", "Grist Templates": "Grist predloge", - "Legacy": "Zapuščina" + "Legacy": "Zapuščina", + "Billing Account": "Račun za obračunavanje", + "Manage Team": "Upravljanje ekipe" }, "ChartView": { "Pick a column": "Izberite stolpec", @@ -525,7 +528,17 @@ "Time Zone": "Časovni pas", "Try API calls from the browser": "Poskusi klice API-ja iz brskalnika", "python3 (recommended)": "python3 (priporočeno)", - "python2 (legacy)": "python2 (odsvetovano)" + "python2 (legacy)": "python2 (odsvetovano)", + "Formula timer": "Časovnik formule", + "Reload data engine": "Ponovno naloži podatkovni mehanizem", + "Reload data engine?": "Ponovno naložiti podatkovni mehanizem?", + "Start timing": "Začni meriti čas", + "Stop timing...": "Ustavi merjenje časa ...", + "Time reload": "Ponovno nalaganje časa", + "Cancel": "Prekliči", + "Force reload the document while timing formulas, and show the result.": "Prisilno znova naloži dokument med časovnimi formulami in prikaži rezultat.", + "Timing is on": "Merjenje časa je vklopljeno", + "You can make changes to the document, then stop timing to see the results.": "Dokument lahko spremeniš in nato ustaviš merjenje časa, da vidiš rezultat." }, "GridOptions": { "Horizontal Gridlines": "Vodoravne linije", @@ -1289,7 +1302,9 @@ "Row Style": "Slog vrstice", "Add another rule": "Dodaj dodatno pravilo", "Add conditional style": "Dodaj pogojni slog", - "Error in style rule": "Napaka v slogovnem pravilu" + "Error in style rule": "Napaka v slogovnem pravilu", + "IF...": "ČE ...", + "Conditional Style": "Pogojni slog" }, "EditorTooltip": { "Convert column to formula": "Pretvori stolpec v formulo" @@ -1522,7 +1537,20 @@ "unconfigured": "nekonfigurirano", "unknown": "neznano", "Grist releases are at ": "Grist verzije so pri ", - "Grist allows for very powerful formulas, using Python. We recommend setting the environment variable GRIST_SANDBOX_FLAVOR to gvisor if your hardware supports it (most will), to run formulas in each document within a sandbox isolated from other documents and isolated from the network.": "Grist omogoča zelo zmogljive formule z uporabo Pythona. Priporočamo, da spremenljivko okolja GRIST_SANDBOX_FLAVOR nastavite na gvisor, če vaša strojna oprema to podpira (večina bo), da zaženete formule v vsakem dokumentu znotraj peskovnika, izoliranega od drugih dokumentov in izoliranega od omrežja." + "Grist allows for very powerful formulas, using Python. We recommend setting the environment variable GRIST_SANDBOX_FLAVOR to gvisor if your hardware supports it (most will), to run formulas in each document within a sandbox isolated from other documents and isolated from the network.": "Grist omogoča zelo zmogljive formule z uporabo Pythona. Priporočamo, da spremenljivko okolja GRIST_SANDBOX_FLAVOR nastavite na gvisor, če vaša strojna oprema to podpira (večina bo), da zaženete formule v vsakem dokumentu znotraj peskovnika, izoliranega od drugih dokumentov in izoliranega od omrežja.", + "Authentication": "Preverjanje pristnosti", + "Check succeeded.": "Preverjanje uspelo.", + "Current authentication method": "Trenutna metoda preverjanja pristnosti", + "Details": "Podrobnosti", + "No fault detected.": "Ni zaznane napake.", + "Notes": "Opombe", + "Check failed.": "Preverjanje ni uspelo.", + "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.": "Grist omogoča konfiguracijo različnih vrst avtentikacije, vključno s SAML in OIDC. Priporočamo, da omogočiš enega od teh, če je Grist dostopen prek omrežja ali je na voljo več osebam.", + "Or, as a fallback, you can set: {{bootKey}} in the environment and visit: {{url}}": "Lahko pa kot nadomestno možnost nastavite: {{bootKey}} v okolju in obiščete: {{url}}", + "Results": "Rezultati", + "Self Checks": "Samopregledi", + "You do not have access to the administrator panel.\nPlease log in as an administrator.": "Nimaš dostopa do skrbniške plošče.\nPrijavi se kot skrbnik.", + "Administrator Panel Unavailable": "Skrbniška plošča ni na voljo" }, "ChoiceEditor": { "Error in dropdown condition": "Napaka v spustnem meniju", @@ -1587,5 +1615,15 @@ "Custom": "Po meri", "Form": "Forma", "Table": "Tabela" + }, + "TimingPage": { + "Loading timing data. Don't close this tab.": "Nalaganje časovnih podatkov. Ne zapri tega zavihka.", + "Max Time (s)": "Največji čas (i)", + "Number of Calls": "Število klicev", + "Table ID": "ID tabele", + "Total Time (s)": "Skupni čas", + "Average Time (s)": "Povprečni čas (i)", + "Column ID": "ID stolpca", + "Formula timer": "Časovnik formule" } } From 5a1d41a2ddc9195b9433762b21fbae8b6ec21f33 Mon Sep 17 00:00:00 2001 From: Florentina Petcu Date: Fri, 31 May 2024 08:13:26 +0000 Subject: [PATCH 25/62] Translated using Weblate (Romanian) Currently translated at 79.8% (1065 of 1333 strings) Translation: Grist/client Translate-URL: https://hosted.weblate.org/projects/grist/client/ro/ --- static/locales/ro.client.json | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/static/locales/ro.client.json b/static/locales/ro.client.json index f47fcd7c7e..19e75da608 100644 --- a/static/locales/ro.client.json +++ b/static/locales/ro.client.json @@ -455,7 +455,9 @@ "SELECTOR FOR": "SELECTOR PENTRU", "Sort & Filter": "Sortare și filtrare", "Widget": "Widget", - "Reset form": "Resetare formular" + "Reset form": "Resetare formular", + "Enter text": "Introdu text", + "Configuration": "Configurare" }, "FloatingPopup": { "Maximize": "Maximizați", @@ -884,7 +886,8 @@ "Help Center": "Centru de ajutor", "Opted In": "A optat pentru a participa", "Contribute": "Contribuie", - "Support Grist page": "Pagina Susține Grist" + "Support Grist page": "Pagina Susține Grist", + "Admin Panel": "Panou administrator" }, "ValidationPanel": { "Update formula (Shift+Enter)": "Actualizați formula (Shift+Enter)", @@ -1292,5 +1295,13 @@ }, "HiddenQuestionConfig": { "Hidden fields": "Câmpuri ascunse" + }, + "AdminPanel": { + "Support Grist Labs on GitHub": "Sponsor Grist Labs pe GitHub", + "Telemetry": "Telemetry", + "Admin Panel": "Panou administrator", + "Home": "Acasă", + "Sponsor": "Sponsor", + "Support Grist": "Sponsor Grist" } } From 51aa4e57c644ae3651b09a11947024f62522937c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C4=8Dek=20Prijatelj?= Date: Sun, 2 Jun 2024 07:56:21 +0000 Subject: [PATCH 26/62] Translated using Weblate (Slovenian) Currently translated at 100.0% (1334 of 1334 strings) Translation: Grist/client Translate-URL: https://hosted.weblate.org/projects/grist/client/sl/ --- static/locales/sl.client.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/static/locales/sl.client.json b/static/locales/sl.client.json index 180d32132f..f359c24ba6 100644 --- a/static/locales/sl.client.json +++ b/static/locales/sl.client.json @@ -1550,7 +1550,8 @@ "Results": "Rezultati", "Self Checks": "Samopregledi", "You do not have access to the administrator panel.\nPlease log in as an administrator.": "Nimaš dostopa do skrbniške plošče.\nPrijavi se kot skrbnik.", - "Administrator Panel Unavailable": "Skrbniška plošča ni na voljo" + "Administrator Panel Unavailable": "Skrbniška plošča ni na voljo", + "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.": "Grist omogoča konfiguracijo različnih vrst avtentikacije, vključno s SAML in OIDC. Priporočamo, da omogočiš enega od teh, če je Grist dostopen prek omrežja ali je na voljo več osebam." }, "ChoiceEditor": { "Error in dropdown condition": "Napaka v spustnem meniju", From b701349866e7ce0ab5003ab8e19c8692094afc8b Mon Sep 17 00:00:00 2001 From: Roman Holinec <3ko@pixeon.sk> Date: Sat, 1 Jun 2024 19:01:30 +0000 Subject: [PATCH 27/62] Translated using Weblate (Slovak) Currently translated at 12.4% (166 of 1334 strings) Translation: Grist/client Translate-URL: https://hosted.weblate.org/projects/grist/client/sk/ --- static/locales/sk.client.json | 90 ++++++++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/static/locales/sk.client.json b/static/locales/sk.client.json index 901fad8d67..898c96cfe1 100644 --- a/static/locales/sk.client.json +++ b/static/locales/sk.client.json @@ -109,6 +109,94 @@ "App": { "Description": "Popis", "Key": "Kľúč", - "Memory Error": "Chyba pamäte" + "Memory Error": "Chyba pamäte", + "Translators: please translate this only when your language is ready to be offered to users": "Prekladatelia: preložte to, prosím, len vtedy, keď je váš jazyk pripravený na poskytovanie používateľom" + }, + "AppHeader": { + "Manage Team": "Riadiť Tím", + "Billing Account": "Fakturačný účet", + "Home Page": "Domovská stránka", + "Legacy": "Dedičstvo", + "Personal Site": "Osobná stránka", + "Team Site": "Tímová stránka", + "Grist Templates": "Grist Šablóny" + }, + "CellContextMenu": { + "Delete {{count}} columns_one": "Odstrániť stĺpec", + "Delete {{count}} columns_other": "Odstrániť {{count}} stĺpcov", + "Delete {{count}} rows_other": "Odstrániť {{count}} riadkov", + "Duplicate rows_one": "Duplikovať riadok", + "Insert column to the right": "Vložiť stĺpec doprava", + "Reset {{count}} columns_one": "Resetovať stĺpec", + "Reset {{count}} columns_other": "Resetovanie {{count}} stĺpcov", + "Reset {{count}} entire columns_one": "Resetovať celý stĺpec", + "Comment": "Komentár", + "Reset {{count}} entire columns_other": "Resetovať {{count}} celé stĺpce", + "Cut": "Vystrihnúť", + "Duplicate rows_other": "Duplikovať riadky", + "Filter by this value": "Filtrovať podľa tejto hodnoty", + "Insert row": "Vložiť riadok", + "Insert row above": "Vložiť riadok vyššie", + "Insert row below": "Vložiť riadok nižšie", + "Insert column to the left": "Vložiť stĺpec doľava", + "Copy": "Kopírovať", + "Paste": "Vložiť", + "Clear cell": "Vymazať bunku", + "Clear values": "Vyčistiť hodnoty", + "Copy anchor link": "Kopírovať odkaz na kotvu", + "Delete {{count}} rows_one": "Odstrániť riadok" + }, + "ChartView": { + "Create separate series for each value of the selected column.": "Vytvoriť samostatné série pre každú hodnotu vybratého stĺpca.", + "Pick a column": "Vybrať stĺpec", + "Toggle chart aggregation": "Prepnúť združovanie grafu", + "selected new group data columns": "vybrať nové stĺpce skupiny dát" + }, + "ColumnFilterMenu": { + "All Shown": "Všetko zobrazené", + "Filter by Range": "Filtrovať podľa rozsahu", + "No matching values": "Žiadne zodpovedajúce hodnoty", + "Max": "Max", + "Start": "Štart", + "End": "Koniec", + "Other Matching": "Iné zhodné", + "Other Non-Matching": "Iné nezhodné", + "Other Values": "Iné hodnoty", + "Future Values": "Budúce hodnoty", + "Others": "Iné", + "None": "Žiadne", + "Min": "Min", + "Search": "Vyhľadať", + "Search values": "Hľadať hodnoty", + "All": "Všetko", + "All Except": "Všetko Okrem" + }, + "CustomSectionConfig": { + " (optional)": " (voliteľné)", + "Add": "Pridať", + "Enter Custom URL": "Zadať vlastnú adresu URL", + "Full document access": "Úplný prístup k dokumentu", + "Learn more about custom widgets": "Prečítať si viac o vlastných widgetoch", + "Open configuration": "Otvoriť konfiguráciu", + "Pick a {{columnType}} column": "Vybrať stĺpec {{columnType}}", + "Select Custom Widget": "Vybrať Vlastný Widget", + "Pick a column": "Vybrať stĺpec", + "Read selected table": "Prečítať vybranú tabuľku", + "Widget does not require any permissions.": "Widget nevyžaduje žiadne povolenia.", + "Widget needs to {{read}} the current table.": "Widget vyžaduje {{read}} aktuálnu tabuľku.", + "Widget needs {{fullAccess}} to this document.": "Widget vyžaduje {{fullAccess}} k tomuto dokumentu.", + "No document access": "Bez prístupu k dokumentu" + }, + "AppModel": { + "This team site is suspended. Documents can be read, but not modified.": "Táto tímová stránka je pozastavená. Dokumenty je možné čítať, ale nie upravovať." + }, + "CodeEditorPanel": { + "Access denied": "Prístup zamietnutý", + "Code View is available only when you have full document access.": "Zobrazenie kódu je dostupné iba vtedy, keď máte úplný prístup k dokumentu." + }, + "ColorSelect": { + "Apply": "Použiť", + "Cancel": "Zrušiť", + "Default cell style": "Predvolený štýl bunky" } } From cc5089376cf4a9e7b6449a17dd1c1c932fd1b3a4 Mon Sep 17 00:00:00 2001 From: Leslie H <142967379+SleepyLeslie@users.noreply.github.com> Date: Mon, 3 Jun 2024 15:20:10 +0000 Subject: [PATCH 28/62] Bump minio to v8.0.0 (#991) --- app/server/lib/MinIOExternalStorage.ts | 55 ++++++++++++++++++-------- package.json | 3 +- yarn.lock | 53 ++++++++++++++----------- 3 files changed, 68 insertions(+), 43 deletions(-) diff --git a/app/server/lib/MinIOExternalStorage.ts b/app/server/lib/MinIOExternalStorage.ts index e341d15888..f5b5788616 100644 --- a/app/server/lib/MinIOExternalStorage.ts +++ b/app/server/lib/MinIOExternalStorage.ts @@ -5,20 +5,39 @@ import {IncomingMessage} from 'http'; import * as fse from 'fs-extra'; import * as minio from 'minio'; -// The minio typings appear to be quite stale. Extend them here to avoid -// lots of casts to any. -type MinIOClient = minio.Client & { - statObject(bucket: string, key: string, options: {versionId?: string}): Promise; - getObject(bucket: string, key: string, options: {versionId?: string}): Promise; - listObjects(bucket: string, key: string, recursive: boolean, - options: {IncludeVersion?: boolean}): minio.BucketStream; - removeObjects(bucket: string, versions: Array<{name: string, versionId: string}>): Promise; -}; - -type MinIOBucketItemStat = minio.BucketItemStat & { - versionId?: string; - metaData?: Record; -}; +// The minio-js v8.0.0 typings are sometimes incorrect. Here are some workarounds. +interface MinIOClient extends + // Some of them are not directly extendable, must be omitted first and then redefined. + Omit + { + // The official typing returns `Promise`, dropping some useful metadata. + getObject(bucket: string, key: string, options: {versionId?: string}): Promise; + // The official typing dropped "options" in their .d.ts file, but it is present in the underlying impl. + listObjects(bucket: string, key: string, recursive: boolean, + options: {IncludeVersion?: boolean}): minio.BucketStream; + // The released v8.0.0 wrongly returns `Promise`; borrowed from PR #1297 + getBucketVersioning(bucketName: string): Promise; + // The released v8.0.0 typing is outdated; copied over from commit 8633968. + removeObjects(bucketName: string, objectsList: RemoveObjectsParam): Promise + } + +type MinIOVersioningStatus = "" | { + Status: "Enabled" | "Suspended", + MFADelete?: string, + ExcludeFolders?: boolean, + ExcludedPrefixes?: {Prefix: string}[] +} + +type RemoveObjectsParam = string[] | { name: string, versionId?: string }[] + +type RemoveObjectsResponse = null | undefined | { + Error?: { + Code?: string + Message?: string + Key?: string + VersionId?: string + } +} /** * An external store implemented using the MinIO client, which @@ -38,7 +57,7 @@ export class MinIOExternalStorage implements ExternalStorage { region: string }, private _batchSize?: number, - private _s3 = new minio.Client(options) as MinIOClient + private _s3 = new minio.Client(options) as unknown as MinIOClient ) { } @@ -70,7 +89,7 @@ export class MinIOExternalStorage implements ExternalStorage { public async upload(key: string, fname: string, metadata?: ObjMetadata) { const stream = fse.createReadStream(fname); const result = await this._s3.putObject( - this.bucket, key, stream, + this.bucket, key, stream, undefined, metadata ? {Metadata: toExternalMetadata(metadata)} : undefined ); // Empirically VersionId is available in result for buckets with versioning enabled. @@ -111,7 +130,9 @@ export class MinIOExternalStorage implements ExternalStorage { public async hasVersioning(): Promise { const versioning = await this._s3.getBucketVersioning(this.bucket); - return versioning && versioning.Status === 'Enabled'; + // getBucketVersioning() may return an empty string when versioning has never been enabled. + // This situation is not addressed in minio-js v8.0.0, but included in our workaround. + return versioning !== '' && versioning?.Status === 'Enabled'; } public async versions(key: string, options?: { includeDeleteMarkers?: boolean }) { diff --git a/package.json b/package.json index bbe1c5e588..efe831db5f 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,6 @@ "@types/lru-cache": "5.1.1", "@types/marked": "4.0.8", "@types/mime-types": "2.1.0", - "@types/minio": "7.0.15", "@types/mocha": "10.0.1", "@types/moment-timezone": "0.5.9", "@types/mousetrap": "1.6.2", @@ -167,7 +166,7 @@ "locale-currency": "0.0.2", "lodash": "4.17.21", "marked": "4.2.12", - "minio": "7.1.3", + "minio": "8.0.0", "moment": "2.29.4", "moment-timezone": "0.5.35", "morgan": "1.9.1", diff --git a/yarn.lock b/yarn.lock index a55cc5a219..798534bf60 100644 --- a/yarn.lock +++ b/yarn.lock @@ -964,13 +964,6 @@ resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a" integrity sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw== -"@types/minio@7.0.15": - version "7.0.15" - resolved "https://registry.yarnpkg.com/@types/minio/-/minio-7.0.15.tgz#6fbf2e17aeae172cbf181ea52b1faa05a601ce42" - integrity sha512-1VR05lWJDuxkn/C7d87MPAJs0p+onKnkUN3nyQ0xrrtaziZQmONy/nxXRaAVWheEyIb6sl0TTi77I/GAQDN5Lw== - dependencies: - "@types/node" "*" - "@types/mocha@10.0.1": version "10.0.1" resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-10.0.1.tgz#2f4f65bb08bc368ac39c96da7b2f09140b26851b" @@ -2166,6 +2159,11 @@ buffer-crc32@^0.2.1, buffer-crc32@^0.2.13: resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ== +buffer-crc32@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-1.0.0.tgz#a10993b9055081d55304bd9feb4a072de179f405" + integrity sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w== + buffer-equal-constant-time@1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz" @@ -3673,6 +3671,11 @@ eventemitter3@^4.0.0: resolved "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz" integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw== +eventemitter3@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-5.0.1.tgz#53f5ffd0a492ac800721bb42c66b841de96423c4" + integrity sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA== + events@^1.1.1, events@~1.1.0: version "1.1.1" resolved "https://registry.npmjs.org/events/-/events-1.1.1.tgz" @@ -5136,11 +5139,6 @@ json-stable-stringify@~0.0.0: dependencies: jsonify "~0.0.0" -json-stream@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/json-stream/-/json-stream-1.0.0.tgz#1a3854e28d2bbeeab31cc7ddf683d2ddc5652708" - integrity sha512-H/ZGY0nIAg3QcOwE1QN/rK/Fa7gJn7Ii5obwp6zyPO4xiPNwpIMjqy2gwjBEGqzkF/vSWEIBQCBuN19hYiL6Qg== - json5@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.2.tgz#63d98d60f21b313b77c4d6da18bfa69d80e1d593" @@ -5674,24 +5672,24 @@ minimist@^1.1.0, minimist@^1.2.0, minimist@^1.2.5: resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== -minio@7.1.3: - version "7.1.3" - resolved "https://registry.yarnpkg.com/minio/-/minio-7.1.3.tgz#86dc95f3671045d6956920db757bb63f25bf20ee" - integrity sha512-xPrLjWkTT5E7H7VnzOjF//xBp9I40jYB4aWhb2xTFopXXfw+Wo82DDWngdUju7Doy3Wk7R8C4LAgwhLHHnf0wA== +minio@8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/minio/-/minio-8.0.0.tgz#c712bace51232e20c30568ca160a41a5ac1d13ce" + integrity sha512-GkM/lk+Gzwd4fAQvLlB+cy3NV3PRADe0tNXnH9JD5BmdAHKIp+5vypptbjdkU85xWBIQsa2xK35GpXjmYXBBYA== dependencies: async "^3.2.4" block-stream2 "^2.1.0" browser-or-node "^2.1.1" - buffer-crc32 "^0.2.13" + buffer-crc32 "^1.0.0" + eventemitter3 "^5.0.1" fast-xml-parser "^4.2.2" ipaddr.js "^2.0.1" - json-stream "^1.0.0" lodash "^4.17.21" mime-types "^2.1.35" query-string "^7.1.3" + stream-json "^1.8.0" through2 "^4.0.2" web-encoding "^1.1.5" - xml "^1.0.1" xml2js "^0.5.0" minipass-collect@^1.0.2: @@ -7491,6 +7489,11 @@ stream-browserify@^2.0.0: inherits "~2.0.1" readable-stream "^2.0.2" +stream-chain@^2.2.5: + version "2.2.5" + resolved "https://registry.yarnpkg.com/stream-chain/-/stream-chain-2.2.5.tgz#b30967e8f14ee033c5b9a19bbe8a2cba90ba0d09" + integrity sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA== + stream-combiner2@^1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/stream-combiner2/-/stream-combiner2-1.1.1.tgz" @@ -7517,6 +7520,13 @@ stream-http@^2.0.0: to-arraybuffer "^1.0.0" xtend "^4.0.0" +stream-json@^1.8.0: + version "1.8.0" + resolved "https://registry.yarnpkg.com/stream-json/-/stream-json-1.8.0.tgz#53f486b2e3b4496c506131f8d7260ba42def151c" + integrity sha512-HZfXngYHUAr1exT4fxlbc1IOce1RYxp2ldeaf97LYCOPSoOqY/1Psp7iGvpb+6JIOgkra9zDYnPX01hGAHzEPw== + dependencies: + stream-chain "^2.2.5" + stream-splicer@^2.0.0: version "2.0.1" resolved "https://registry.npmjs.org/stream-splicer/-/stream-splicer-2.0.1.tgz" @@ -8632,11 +8642,6 @@ xml2js@^0.5.0: sax ">=0.6.0" xmlbuilder "~11.0.0" -xml@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5" - integrity sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw== - xmlbuilder2@^2.4.0: version "2.4.1" resolved "https://registry.yarnpkg.com/xmlbuilder2/-/xmlbuilder2-2.4.1.tgz#899c783a833188c5a5aa6f3c5428a3963f3e479d" From 0e78637cd4ae77d7a0d5d44ce9407e723af2ae2d Mon Sep 17 00:00:00 2001 From: George Gevoian Date: Wed, 29 May 2024 14:55:21 -0700 Subject: [PATCH 29/62] (core) Support `user` variable in dropdown conditions Summary: Dropdown conditions can now reference a `user` variable, similar to the one available in Access Rules. Test Plan: Browser test. Reviewers: jarek, paulfitz Reviewed By: jarek, paulfitz Differential Revision: https://phab.getgrist.com/D4255 --- .../components/DropdownConditionConfig.ts | 24 +- app/client/components/TypeTransform.ts | 2 +- app/client/lib/ReferenceUtils.ts | 12 +- app/client/models/DocPageModel.ts | 18 +- app/client/widgets/AbstractWidget.js | 4 +- app/client/widgets/ChoiceEditor.js | 2 +- app/client/widgets/ChoiceListEditor.ts | 16 +- app/client/widgets/ChoiceTextBox.ts | 11 +- app/client/widgets/DateTimeTextBox.js | 6 +- app/client/widgets/FieldBuilder.ts | 2 +- app/client/widgets/NTextBox.ts | 3 +- app/client/widgets/NewAbstractWidget.ts | 4 +- app/client/widgets/NumericTextBox.ts | 5 +- app/client/widgets/Reference.ts | 7 +- app/client/widgets/ReferenceEditor.ts | 6 +- app/client/widgets/ReferenceListEditor.ts | 6 +- app/common/DocListAPI.ts | 2 + app/common/GranularAccessClause.ts | 23 -- app/common/PredicateFormula.ts | 10 +- app/common/RecordView.ts | 43 +++ app/common/User.ts | 89 ++++++ app/server/lib/ActiveDoc.ts | 4 + app/server/lib/DocManager.ts | 4 +- app/server/lib/GranularAccess.ts | 289 +++++++----------- app/server/lib/PermissionInfo.ts | 7 +- test/nbrowser/DropdownConditionEditor.ts | 91 +++++- test/server/lib/ACLFormula.ts | 4 +- 27 files changed, 426 insertions(+), 268 deletions(-) create mode 100644 app/common/RecordView.ts create mode 100644 app/common/User.ts diff --git a/app/client/components/DropdownConditionConfig.ts b/app/client/components/DropdownConditionConfig.ts index a07da7857d..1094f825ca 100644 --- a/app/client/components/DropdownConditionConfig.ts +++ b/app/client/components/DropdownConditionConfig.ts @@ -1,4 +1,5 @@ import {buildDropdownConditionEditor} from 'app/client/components/DropdownConditionEditor'; +import {GristDoc} from 'app/client/components/GristDoc'; import {makeT} from 'app/client/lib/localization'; import {ViewFieldRec} from 'app/client/models/DocModel'; import {cssLabel, cssRow} from 'app/client/ui/RightPanelStyles'; @@ -7,7 +8,9 @@ import {textButton } from 'app/client/ui2018/buttons'; import {testId, theme} from 'app/client/ui2018/cssVars'; import {ISuggestionWithValue} from 'app/common/ActiveDocAPI'; import {getPredicateFormulaProperties} from 'app/common/PredicateFormula'; +import {UserInfo} from 'app/common/User'; import {Computed, Disposable, dom, Observable, styled} from 'grainjs'; +import isPlainObject from 'lodash/isPlainObject'; const t = makeT('DropdownConditionConfig'); @@ -99,7 +102,7 @@ export class DropdownConditionConfig extends Disposable { private _editorElement: HTMLElement; - constructor(private _field: ViewFieldRec) { + constructor(private _field: ViewFieldRec, private _gristDoc: GristDoc) { super(); this.autoDispose(this._text.addListener(() => { @@ -167,6 +170,10 @@ export class DropdownConditionConfig extends Disposable { private _getAutocompleteSuggestions(): ISuggestionWithValue[] { const variables = ['choice']; + const user = this._gristDoc.docPageModel.user.get(); + if (user) { + variables.push(...getUserCompletions(user)); + } const refColumns = this._refColumns.get(); if (refColumns) { variables.push('choice.id', ...refColumns.map(({colId}) => `choice.${colId.peek()}`)); @@ -176,7 +183,6 @@ export class DropdownConditionConfig extends Disposable { ...columns.map(({colId}) => `$${colId.peek()}`), ...columns.map(({colId}) => `rec.${colId.peek()}`), ); - const suggestions = [ 'and', 'or', 'not', 'in', 'is', 'True', 'False', 'None', 'OWNER', 'EDITOR', 'VIEWER', @@ -186,6 +192,20 @@ export class DropdownConditionConfig extends Disposable { } } +function getUserCompletions(user: UserInfo) { + return Object.entries(user).flatMap(([key, value]) => { + if (key === 'LinkKey') { + return 'user.LinkKey.'; + } else if (isPlainObject(value)) { + return Object.keys(value as {[key: string]: any}) + .filter(valueKey => valueKey !== 'manualSort') + .map(valueKey => `user.${key}.${valueKey}`); + } else { + return `user.${key}`; + } + }); +} + const cssSetDropdownConditionRow = styled(cssRow, ` margin-top: 16px; `); diff --git a/app/client/components/TypeTransform.ts b/app/client/components/TypeTransform.ts index c47fcd86b1..0cc4cbd974 100644 --- a/app/client/components/TypeTransform.ts +++ b/app/client/components/TypeTransform.ts @@ -63,7 +63,7 @@ export class TypeTransform extends ColumnTransform { if (use(this._isFormWidget)) { return transformWidget.buildFormTransformConfigDom(); } else { - return transformWidget.buildTransformConfigDom(); + return transformWidget.buildTransformConfigDom(this.gristDoc); } }), dom.maybe(this._reviseTypeChange, () => diff --git a/app/client/lib/ReferenceUtils.ts b/app/client/lib/ReferenceUtils.ts index 04bf671655..a5fa381cf0 100644 --- a/app/client/lib/ReferenceUtils.ts +++ b/app/client/lib/ReferenceUtils.ts @@ -1,13 +1,13 @@ +import {GristDoc} from 'app/client/components/GristDoc'; import {ACIndex, ACResults} from 'app/client/lib/ACIndex'; import {makeT} from 'app/client/lib/localization'; import {ICellItem} from 'app/client/models/ColumnACIndexes'; import {ColumnCache} from 'app/client/models/ColumnCache'; -import {DocData} from 'app/client/models/DocData'; import {ColumnRec} from 'app/client/models/entities/ColumnRec'; import {ViewFieldRec} from 'app/client/models/entities/ViewFieldRec'; import {TableData} from 'app/client/models/TableData'; import {getReferencedTableId, isRefListType} from 'app/common/gristTypes'; -import {EmptyRecordView} from 'app/common/PredicateFormula'; +import {EmptyRecordView} from 'app/common/RecordView'; import {BaseFormatter} from 'app/common/ValueFormatter'; import {Disposable, dom, Observable} from 'grainjs'; @@ -26,9 +26,10 @@ export class ReferenceUtils extends Disposable { public readonly hasDropdownCondition = Boolean(this.field.dropdownCondition.peek()?.text); private readonly _columnCache: ColumnCache>; + private readonly _docData = this._gristDoc.docData; private _dropdownConditionError = Observable.create(this, null); - constructor(public readonly field: ViewFieldRec, private readonly _docData: DocData) { + constructor(public readonly field: ViewFieldRec, private readonly _gristDoc: GristDoc) { super(); const colType = field.column().type(); @@ -38,7 +39,7 @@ export class ReferenceUtils extends Disposable { } this.refTableId = refTableId; - const tableData = _docData.getTable(refTableId); + const tableData = this._docData.getTable(refTableId); if (!tableData) { throw new Error("Invalid referenced table " + refTableId); } @@ -131,12 +132,13 @@ export class ReferenceUtils extends Disposable { if (!table) { throw new Error(`Table ${tableId} not found`); } const {result: predicate} = dropdownConditionCompiled; + const user = this._gristDoc.docPageModel.user.get() ?? undefined; const rec = table.getRecord(rowId) || new EmptyRecordView(); return (item: ICellItem) => { const choice = item.rowId === 'new' ? new EmptyRecordView() : this.tableData.getRecord(item.rowId); if (!choice) { throw new Error(`Reference ${item.rowId} not found`); } - return predicate({rec, choice}); + return predicate({user, rec, choice}); }; } } diff --git a/app/client/models/DocPageModel.ts b/app/client/models/DocPageModel.ts index 4805218e4f..f0a2683b63 100644 --- a/app/client/models/DocPageModel.ts +++ b/app/client/models/DocPageModel.ts @@ -23,6 +23,7 @@ import {Features, mergedFeatures, Product} from 'app/common/Features'; import {buildUrlId, IGristUrlState, parseUrlId, UrlIdParts} from 'app/common/gristUrls'; import {getReconnectTimeout} from 'app/common/gutil'; import {canEdit, isOwner} from 'app/common/roles'; +import {UserInfo} from 'app/common/User'; import {Document, NEW_DOCUMENT_CODE, Organization, UserAPI, Workspace} from 'app/common/UserAPI'; import {Holder, Observable, subscribe} from 'grainjs'; import {Computed, Disposable, dom, DomArg, DomElementArg} from 'grainjs'; @@ -38,6 +39,7 @@ export interface DocInfo extends Document { isPreFork: boolean; isFork: boolean; isRecoveryMode: boolean; + user: UserInfo|null; userOverride: UserOverride|null; isBareFork: boolean; // a document created without logging in, which is treated as a // fork without an original. @@ -78,6 +80,7 @@ export interface DocPageModel { isPrefork: Observable; isFork: Observable; isRecoveryMode: Observable; + user: Observable; userOverride: Observable; isBareFork: Observable; isSnapshot: Observable; @@ -134,6 +137,7 @@ export class DocPageModelImpl extends Disposable implements DocPageModel { public readonly isFork = Computed.create(this, this.currentDoc, (use, doc) => doc ? doc.isFork : false); public readonly isRecoveryMode = Computed.create(this, this.currentDoc, (use, doc) => doc ? doc.isRecoveryMode : false); + public readonly user = Computed.create(this, this.currentDoc, (use, doc) => doc ? doc.user : null); public readonly userOverride = Computed.create(this, this.currentDoc, (use, doc) => doc ? doc.userOverride : null); public readonly isBareFork = Computed.create(this, this.currentDoc, (use, doc) => doc ? doc.isBareFork : false); public readonly isSnapshot = Computed.create(this, this.currentDoc, (use, doc) => doc ? doc.isSnapshot : false); @@ -265,8 +269,9 @@ export class DocPageModelImpl extends Disposable implements DocPageModel { // TODO It would be bad if a new doc gets opened while this getDoc() is pending... const newDoc = await getDoc(this._api, urlId); const isRecoveryMode = Boolean(this.currentDoc.get()?.isRecoveryMode); + const user = this.currentDoc.get()?.user || null; const userOverride = this.currentDoc.get()?.userOverride || null; - this.currentDoc.set({...buildDocInfo(newDoc, openMode), isRecoveryMode, userOverride}); + this.currentDoc.set({...buildDocInfo(newDoc, openMode), isRecoveryMode, user, userOverride}); return newDoc; } @@ -407,11 +412,13 @@ It also disables formulas. [{{error}}]", {error: err.message}) linkParameters, originalUrlId: options.originalUrlId, }); - if (openDocResponse.recoveryMode || openDocResponse.userOverride) { - doc.isRecoveryMode = Boolean(openDocResponse.recoveryMode); - doc.userOverride = openDocResponse.userOverride || null; - this.currentDoc.set({...doc}); + const {user, recoveryMode, userOverride} = openDocResponse; + doc.user = user; + if (recoveryMode || userOverride) { + doc.isRecoveryMode = Boolean(recoveryMode); + doc.userOverride = userOverride || null; } + this.currentDoc.set({...doc}); if (openDocResponse.docUsage) { this.updateCurrentDocUsage(openDocResponse.docUsage); } @@ -520,6 +527,7 @@ function buildDocInfo(doc: Document, mode: OpenDocMode | undefined): DocInfo { ...doc, isFork, isRecoveryMode: false, // we don't know yet, will learn when doc is opened. + user: null, // ditto. userOverride: null, // ditto. isPreFork, isBareFork, diff --git a/app/client/widgets/AbstractWidget.js b/app/client/widgets/AbstractWidget.js index b7a1c23b3d..bee224369a 100644 --- a/app/client/widgets/AbstractWidget.js +++ b/app/client/widgets/AbstractWidget.js @@ -21,7 +21,7 @@ dispose.makeDisposable(AbstractWidget); /** * Builds the DOM showing configuration buttons and fields in the sidebar. */ -AbstractWidget.prototype.buildConfigDom = function() { +AbstractWidget.prototype.buildConfigDom = function(_gristDoc) { throw new Error("Not Implemented"); }; @@ -29,7 +29,7 @@ AbstractWidget.prototype.buildConfigDom = function() { * Builds the transform prompt config DOM in the few cases where it is necessary. * Child classes need not override this function if they do not require transform config options. */ -AbstractWidget.prototype.buildTransformConfigDom = function() { +AbstractWidget.prototype.buildTransformConfigDom = function(_gristDoc) { return null; }; diff --git a/app/client/widgets/ChoiceEditor.js b/app/client/widgets/ChoiceEditor.js index f01fdbfa94..7f32aaf990 100644 --- a/app/client/widgets/ChoiceEditor.js +++ b/app/client/widgets/ChoiceEditor.js @@ -129,7 +129,7 @@ ChoiceEditor.prototype.buildDropdownConditionFilter = function() { return buildDropdownConditionFilter({ dropdownConditionCompiled: dropdownConditionCompiled.result, - docData: this.options.gristDoc.docData, + gristDoc: this.options.gristDoc, tableId: this.options.field.tableId(), rowId: this.options.rowId, }); diff --git a/app/client/widgets/ChoiceListEditor.ts b/app/client/widgets/ChoiceListEditor.ts index 238f47f089..1837ff2c43 100644 --- a/app/client/widgets/ChoiceListEditor.ts +++ b/app/client/widgets/ChoiceListEditor.ts @@ -1,10 +1,10 @@ import {createGroup} from 'app/client/components/commands'; +import {GristDoc} from 'app/client/components/GristDoc'; import {ACIndexImpl, ACItem, ACResults, buildHighlightedDom, HighlightFunc, normalizeText} from 'app/client/lib/ACIndex'; import {IAutocompleteOptions} from 'app/client/lib/autocomplete'; import {makeT} from 'app/client/lib/localization'; import {IToken, TokenField, tokenFieldStyles} from 'app/client/lib/TokenField'; -import {DocData} from 'app/client/models/DocData'; import {colors, testId, theme} from 'app/client/ui2018/cssVars'; import {menuCssClass} from 'app/client/ui2018/menus'; import {createMobileButtons, getButtonMargins} from 'app/client/widgets/EditorButtons'; @@ -12,7 +12,8 @@ import {EditorPlacement} from 'app/client/widgets/EditorPlacement'; import {FieldOptions, NewBaseEditor} from 'app/client/widgets/NewBaseEditor'; import {csvEncodeRow} from 'app/common/csvFormat'; import {CellValue} from "app/common/DocActions"; -import {CompiledPredicateFormula, EmptyRecordView} from 'app/common/PredicateFormula'; +import {CompiledPredicateFormula} from 'app/common/PredicateFormula'; +import {EmptyRecordView} from 'app/common/RecordView'; import {decodeObject, encodeObject} from 'app/plugin/objtypes'; import {ChoiceOptions, getRenderFillColor, getRenderTextColor} from 'app/client/widgets/ChoiceTextBox'; import {choiceToken, cssChoiceACItem, cssChoiceToken} from 'app/client/widgets/ChoiceToken'; @@ -246,7 +247,7 @@ export class ChoiceListEditor extends NewBaseEditor { return buildDropdownConditionFilter({ dropdownConditionCompiled: dropdownConditionCompiled.result, - docData: this.options.gristDoc.docData, + gristDoc: this.options.gristDoc, tableId: this.options.field.tableId(), rowId: this.options.rowId, }); @@ -311,7 +312,7 @@ export class ChoiceListEditor extends NewBaseEditor { export interface GetACFilterFuncParams { dropdownConditionCompiled: CompiledPredicateFormula; - docData: DocData; + gristDoc: GristDoc; tableId: string; rowId: number; } @@ -319,12 +320,13 @@ export interface GetACFilterFuncParams { export function buildDropdownConditionFilter( params: GetACFilterFuncParams ): (item: ChoiceItem) => boolean { - const {dropdownConditionCompiled, docData, tableId, rowId} = params; - const table = docData.getTable(tableId); + const {dropdownConditionCompiled, gristDoc, tableId, rowId} = params; + const table = gristDoc.docData.getTable(tableId); if (!table) { throw new Error(`Table ${tableId} not found`); } + const user = gristDoc.docPageModel.user.get() ?? undefined; const rec = table.getRecord(rowId) || new EmptyRecordView(); - return (item: ChoiceItem) => dropdownConditionCompiled({rec, choice: item.label}); + return (item: ChoiceItem) => dropdownConditionCompiled({user, rec, choice: item.label}); } const cssCellEditor = styled('div', ` diff --git a/app/client/widgets/ChoiceTextBox.ts b/app/client/widgets/ChoiceTextBox.ts index d27409477c..4f7dfc5c8a 100644 --- a/app/client/widgets/ChoiceTextBox.ts +++ b/app/client/widgets/ChoiceTextBox.ts @@ -4,6 +4,7 @@ import { FormSelectConfig, } from 'app/client/components/Forms/FormConfig'; import {DropdownConditionConfig} from 'app/client/components/DropdownConditionConfig'; +import {GristDoc} from 'app/client/components/GristDoc'; import {makeT} from 'app/client/lib/localization'; import {DataRowModel} from 'app/client/models/DataRowModel'; import {ViewFieldRec} from 'app/client/models/entities/ViewFieldRec'; @@ -79,17 +80,17 @@ export class ChoiceTextBox extends NTextBox { ); } - public buildConfigDom() { + public buildConfigDom(gristDoc: GristDoc) { return [ - super.buildConfigDom(), + super.buildConfigDom(gristDoc), this.buildChoicesConfigDom(), - dom.create(DropdownConditionConfig, this.field), + dom.create(DropdownConditionConfig, this.field, gristDoc), ]; } - public buildTransformConfigDom() { + public buildTransformConfigDom(gristDoc: GristDoc) { return [ - super.buildConfigDom(), + super.buildConfigDom(gristDoc), this.buildChoicesConfigDom(), ]; } diff --git a/app/client/widgets/DateTimeTextBox.js b/app/client/widgets/DateTimeTextBox.js index 4c6424cea8..da8db6fbe5 100644 --- a/app/client/widgets/DateTimeTextBox.js +++ b/app/client/widgets/DateTimeTextBox.js @@ -54,7 +54,7 @@ _.extend(DateTimeTextBox.prototype, DateTextBox.prototype); * Builds the config dom for the DateTime TextBox. If isTransformConfig is true, * builds only the necessary dom for the transform config menu. */ -DateTimeTextBox.prototype.buildConfigDom = function(isTransformConfig) { +DateTimeTextBox.prototype.buildConfigDom = function(_gristDoc, isTransformConfig) { const disabled = ko.pureComputed(() => { return this.field.config.options.disabled('timeFormat')() || this.field.column().disableEditData(); }); @@ -92,8 +92,8 @@ DateTimeTextBox.prototype.buildConfigDom = function(isTransformConfig) { ); }; -DateTimeTextBox.prototype.buildTransformConfigDom = function() { - return this.buildConfigDom(true); +DateTimeTextBox.prototype.buildTransformConfigDom = function(gristDoc) { + return this.buildConfigDom(gristDoc, true); }; // clean up old koform styles diff --git a/app/client/widgets/FieldBuilder.ts b/app/client/widgets/FieldBuilder.ts index 1e4511fafd..35d932211b 100644 --- a/app/client/widgets/FieldBuilder.ts +++ b/app/client/widgets/FieldBuilder.ts @@ -499,7 +499,7 @@ export class FieldBuilder extends Disposable { // the dom created by the widgetImpl to get out of sync. return dom('div', kd.maybe(() => !this._isTransformingType() && this.widgetImpl(), (widget: NewAbstractWidget) => - dom('div', widget.buildConfigDom()) + dom('div', widget.buildConfigDom(this.gristDoc)) ) ); } diff --git a/app/client/widgets/NTextBox.ts b/app/client/widgets/NTextBox.ts index e939c31509..b346c4d651 100644 --- a/app/client/widgets/NTextBox.ts +++ b/app/client/widgets/NTextBox.ts @@ -1,4 +1,5 @@ import { FormFieldRulesConfig } from 'app/client/components/Forms/FormConfig'; +import { GristDoc } from 'app/client/components/GristDoc'; import { fromKoSave } from 'app/client/lib/fromKoSave'; import { makeT } from 'app/client/lib/localization'; import { DataRowModel } from 'app/client/models/DataRowModel'; @@ -32,7 +33,7 @@ export class NTextBox extends NewAbstractWidget { })); } - public buildConfigDom(): DomContents { + public buildConfigDom(_gristDoc: GristDoc): DomContents { const toggle = () => { const newValue = !this.field.config.wrap.peek(); this.field.config.wrap.setAndSave(newValue).catch(reportError); diff --git a/app/client/widgets/NewAbstractWidget.ts b/app/client/widgets/NewAbstractWidget.ts index 86812e20e8..947d869477 100644 --- a/app/client/widgets/NewAbstractWidget.ts +++ b/app/client/widgets/NewAbstractWidget.ts @@ -60,7 +60,7 @@ export abstract class NewAbstractWidget extends Disposable { /** * Builds the DOM showing configuration buttons and fields in the sidebar. */ - public buildConfigDom(): DomContents { + public buildConfigDom(_gristDoc: GristDoc): DomContents { return null; } @@ -68,7 +68,7 @@ export abstract class NewAbstractWidget extends Disposable { * Builds the transform prompt config DOM in the few cases where it is necessary. * Child classes need not override this function if they do not require transform config options. */ - public buildTransformConfigDom(): DomContents { + public buildTransformConfigDom(_gristDoc: GristDoc): DomContents { return null; } diff --git a/app/client/widgets/NumericTextBox.ts b/app/client/widgets/NumericTextBox.ts index 04470433dd..8e52029a90 100644 --- a/app/client/widgets/NumericTextBox.ts +++ b/app/client/widgets/NumericTextBox.ts @@ -2,6 +2,7 @@ * See app/common/NumberFormat for description of options we support. */ import {FormFieldRulesConfig} from 'app/client/components/Forms/FormConfig'; +import {GristDoc} from 'app/client/components/GristDoc'; import {fromKoSave} from 'app/client/lib/fromKoSave'; import {makeT} from 'app/client/lib/localization'; import {ViewFieldRec} from 'app/client/models/entities/ViewFieldRec'; @@ -39,7 +40,7 @@ export class NumericTextBox extends NTextBox { super(field); } - public buildConfigDom(): DomContents { + public buildConfigDom(gristDoc: GristDoc): DomContents { // Holder for all computeds created here. It gets disposed with the returned DOM element. const holder = new MultiHolder(); @@ -89,7 +90,7 @@ export class NumericTextBox extends NTextBox { const disabledStyle = cssButtonSelect.cls('-disabled', disabled); return [ - super.buildConfigDom(), + super.buildConfigDom(gristDoc), cssLabel(t('Number Format')), cssRow( dom.autoDispose(holder), diff --git a/app/client/widgets/Reference.ts b/app/client/widgets/Reference.ts index 81da964b63..76e9d2dd69 100644 --- a/app/client/widgets/Reference.ts +++ b/app/client/widgets/Reference.ts @@ -4,6 +4,7 @@ import { FormSelectConfig } from 'app/client/components/Forms/FormConfig'; import {DropdownConditionConfig} from 'app/client/components/DropdownConditionConfig'; +import {GristDoc} from 'app/client/components/GristDoc'; import {makeT} from 'app/client/lib/localization'; import {DataRowModel} from 'app/client/models/DataRowModel'; import {TableRec} from 'app/client/models/DocModel'; @@ -53,12 +54,12 @@ export class Reference extends NTextBox { }); } - public buildConfigDom() { + public buildConfigDom(gristDoc: GristDoc) { return [ this.buildTransformConfigDom(), - dom.create(DropdownConditionConfig, this.field), + dom.create(DropdownConditionConfig, this.field, gristDoc), cssLabel(t('CELL FORMAT')), - super.buildConfigDom(), + super.buildConfigDom(gristDoc), ]; } diff --git a/app/client/widgets/ReferenceEditor.ts b/app/client/widgets/ReferenceEditor.ts index 793589172c..5cbe3af7f2 100644 --- a/app/client/widgets/ReferenceEditor.ts +++ b/app/client/widgets/ReferenceEditor.ts @@ -23,8 +23,8 @@ export class ReferenceEditor extends NTextEditor { constructor(options: FieldOptions) { super(options); - const docData = options.gristDoc.docData; - this._utils = new ReferenceUtils(options.field, docData); + const gristDoc = options.gristDoc; + this._utils = new ReferenceUtils(options.field, gristDoc); const vcol = this._utils.visibleColModel; this._enableAddNew = ( @@ -47,7 +47,7 @@ export class ReferenceEditor extends NTextEditor { // The referenced table has probably already been fetched (because there must already be a // Reference widget instantiated), but it's better to avoid this assumption. - docData.fetchTable(this._utils.refTableId).then(() => { + gristDoc.docData.fetchTable(this._utils.refTableId).then(() => { if (this.isDisposed()) { return; } if (needReload && this.textInput.value === '') { this.textInput.value = undef(options.state, options.editValue, this._idToText()); diff --git a/app/client/widgets/ReferenceListEditor.ts b/app/client/widgets/ReferenceListEditor.ts index c2060199fc..bf2beddab5 100644 --- a/app/client/widgets/ReferenceListEditor.ts +++ b/app/client/widgets/ReferenceListEditor.ts @@ -55,8 +55,8 @@ export class ReferenceListEditor extends NewBaseEditor { constructor(protected options: FieldOptions) { super(options); - const docData = options.gristDoc.docData; - this._utils = new ReferenceUtils(options.field, docData); + const gristDoc = options.gristDoc; + this._utils = new ReferenceUtils(options.field, gristDoc); const vcol = this._utils.visibleColModel; this._enableAddNew = ( @@ -130,7 +130,7 @@ export class ReferenceListEditor extends NewBaseEditor { // The referenced table has probably already been fetched (because there must already be a // Reference widget instantiated), but it's better to avoid this assumption. - docData.fetchTable(this._utils.refTableId).then(() => { + gristDoc.docData.fetchTable(this._utils.refTableId).then(() => { if (this.isDisposed()) { return; } if (needReload) { this._tokenField.setTokens( diff --git a/app/common/DocListAPI.ts b/app/common/DocListAPI.ts index 0663963295..c780310445 100644 --- a/app/common/DocListAPI.ts +++ b/app/common/DocListAPI.ts @@ -3,6 +3,7 @@ import {TableDataAction} from 'app/common/DocActions'; import {FilteredDocUsageSummary} from 'app/common/DocUsage'; import {Role} from 'app/common/roles'; import {StringUnion} from 'app/common/StringUnion'; +import {UserInfo} from 'app/common/User'; import {FullUser} from 'app/common/UserAPI'; // Possible flavors of items in a list of documents. @@ -75,6 +76,7 @@ export interface OpenLocalDocResult { doc: {[tableId: string]: TableDataAction}; log: MinimalActionGroup[]; isTimingOn: boolean; + user: UserInfo; recoveryMode?: boolean; userOverride?: UserOverride; docUsage?: FilteredDocUsageSummary; diff --git a/app/common/GranularAccessClause.ts b/app/common/GranularAccessClause.ts index 3f3b77690a..e207bae4af 100644 --- a/app/common/GranularAccessClause.ts +++ b/app/common/GranularAccessClause.ts @@ -1,7 +1,6 @@ import {PartialPermissionSet} from 'app/common/ACLPermissions'; import {CellValue, RowRecord} from 'app/common/DocActions'; import {CompiledPredicateFormula} from 'app/common/PredicateFormula'; -import {Role} from 'app/common/roles'; import {MetaRowRecord} from 'app/common/TableData'; export interface RuleSet { @@ -25,12 +24,6 @@ export interface RulePart { memo?: string; } -// Light wrapper for reading records or user attributes. -export interface InfoView { - get(key: string): CellValue; - toJSON(): {[key: string]: any}; -} - // As InfoView, but also supporting writing. export interface InfoEditor { get(key: string): CellValue; @@ -38,22 +31,6 @@ export interface InfoEditor { toJSON(): {[key: string]: any}; } -// Represents user info, which may include properties which are themselves RowRecords. -export interface UserInfo { - Name: string | null; - Email: string | null; - Access: Role | null; - Origin: string | null; - LinkKey: Record; - UserID: number | null; - UserRef: string | null; - SessionID: string | null; - ShareRef: number | null; // This is a rowId in the _grist_Shares table, if the user - // is accessing a document via a share. Otherwise null. - [attributes: string]: unknown; - toJSON(): {[key: string]: any}; -} - export interface UserAttributeRule { origRecord?: RowRecord; // Original record used to create this UserAttributeRule. name: string; // Should be unique among UserAttributeRules. diff --git a/app/common/PredicateFormula.ts b/app/common/PredicateFormula.ts index 2d9d5f30fc..6a3b125067 100644 --- a/app/common/PredicateFormula.ts +++ b/app/common/PredicateFormula.ts @@ -10,7 +10,8 @@ */ import {CellValue, RowRecord} from 'app/common/DocActions'; import {ErrorWithCode} from 'app/common/ErrorWithCode'; -import {InfoView, UserInfo} from 'app/common/GranularAccessClause'; +import {InfoView} from 'app/common/RecordView'; +import {UserInfo} from 'app/common/User'; import {decodeObject} from 'app/plugin/objtypes'; import constant = require('lodash/constant'); @@ -31,11 +32,6 @@ export interface PredicateFormulaInput { choice?: string|RowRecord|InfoView; } -export class EmptyRecordView implements InfoView { - public get(_colId: string): CellValue { return null; } - public toJSON() { return {}; } -} - /** * The result of compiling ParsedPredicateFormula. */ @@ -102,7 +98,7 @@ export function compilePredicateFormula( break; } case 'dropdown-condition': { - validNames = ['rec', 'choice']; + validNames = ['rec', 'choice', 'user']; break; } } diff --git a/app/common/RecordView.ts b/app/common/RecordView.ts new file mode 100644 index 0000000000..d1359b0c5c --- /dev/null +++ b/app/common/RecordView.ts @@ -0,0 +1,43 @@ +import {CellValue, TableDataAction} from 'app/common/DocActions'; + +/** Light wrapper for reading records or user attributes. */ +export interface InfoView { + get(key: string): CellValue; + toJSON(): {[key: string]: any}; +} + +/** + * A row-like view of TableDataAction, which is columnar in nature. + * + * If index value is undefined, acts as an EmptyRecordRow. + */ +export class RecordView implements InfoView { + public constructor(public data: TableDataAction, public index: number|undefined) { + } + + public get(colId: string): CellValue { + if (this.index === undefined) { return null; } + if (colId === 'id') { + return this.data[2][this.index]; + } + return this.data[3][colId]?.[this.index]; + } + + public has(colId: string) { + return colId === 'id' || colId in this.data[3]; + } + + public toJSON() { + if (this.index === undefined) { return {}; } + const results: {[key: string]: any} = {id: this.index}; + for (const key of Object.keys(this.data[3])) { + results[key] = this.data[3][key]?.[this.index]; + } + return results; + } +} + +export class EmptyRecordView implements InfoView { + public get(_colId: string): CellValue { return null; } + public toJSON() { return {}; } +} diff --git a/app/common/User.ts b/app/common/User.ts new file mode 100644 index 0000000000..4e3419a148 --- /dev/null +++ b/app/common/User.ts @@ -0,0 +1,89 @@ +import {getTableId} from 'app/common/DocActions'; +import {EmptyRecordView, RecordView} from 'app/common/RecordView'; +import {Role} from 'app/common/roles'; + +/** + * Information about a user, including any user attributes. + */ +export interface UserInfo { + Name: string | null; + Email: string | null; + Access: Role | null; + Origin: string | null; + LinkKey: Record; + UserID: number | null; + UserRef: string | null; + SessionID: string | null; + /** + * This is a rowId in the _grist_Shares table, if the user is accessing a document + * via a share. Otherwise null. + */ + ShareRef: number | null; + [attributes: string]: unknown; +} + +/** + * Wrapper class for `UserInfo`. + * + * Contains methods for converting itself to different representations. + */ +export class User implements UserInfo { + public Name: string | null = null; + public UserID: number | null = null; + public Access: Role | null = null; + public Origin: string | null = null; + public LinkKey: Record = {}; + public Email: string | null = null; + public SessionID: string | null = null; + public UserRef: string | null = null; + public ShareRef: number | null = null; + [attribute: string]: any; + + constructor(info: Record = {}) { + Object.assign(this, info); + } + + /** + * Returns a JSON representation of this class that excludes full row data, + * only keeping user info and table/row ids for any user attributes. + * + * Used by the sandbox to support `user` variables in formulas (see `user.py`). + */ + public toJSON() { + return this._toObject((value) => { + if (value instanceof RecordView) { + return [getTableId(value.data), value.get('id')]; + } else if (value instanceof EmptyRecordView) { + return null; + } else { + return value; + } + }); + } + + /** + * Returns a record representation of this class, with all user attributes + * converted from `RecordView` instances to their JSON representations. + * + * Used by the client to support `user` variables in dropdown conditions. + */ + public toUserInfo(): UserInfo { + return this._toObject((value) => { + if (value instanceof RecordView) { + return value.toJSON(); + } else if (value instanceof EmptyRecordView) { + return null; + } else { + return value; + } + }) as UserInfo; + } + + private _toObject(mapValue: (value: unknown) => unknown) { + const results: {[key: string]: any} = {}; + for (const [key, value] of Object.entries(this)) { + results[key] = mapValue(value); + } + return results; + } +} diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index a322077b5b..addf127820 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -443,6 +443,10 @@ export class ActiveDoc extends EventEmitter { return this._granularAccess.filterDocUsageSummary(docSession, this.getDocUsageSummary()); } + public getUser(docSession: OptDocSession) { + return this._granularAccess.getUser(docSession); + } + public async getUserOverride(docSession: OptDocSession) { return this._granularAccess.getUserOverride(docSession); } diff --git a/app/server/lib/DocManager.ts b/app/server/lib/DocManager.ts index 3222ae087b..437559f857 100644 --- a/app/server/lib/DocManager.ts +++ b/app/server/lib/DocManager.ts @@ -395,9 +395,10 @@ export class DocManager extends EventEmitter { } } - const [metaTables, recentActions, userOverride] = await Promise.all([ + const [metaTables, recentActions, user, userOverride] = await Promise.all([ activeDoc.fetchMetaTables(docSession), activeDoc.getRecentMinimalActions(docSession), + activeDoc.getUser(docSession), activeDoc.getUserOverride(docSession), ]); @@ -414,6 +415,7 @@ export class DocManager extends EventEmitter { doc: metaTables, log: recentActions, recoveryMode: activeDoc.recoveryMode, + user: user.toUserInfo(), userOverride, docUsage, isTimingOn: activeDoc.isTimingOn, diff --git a/app/server/lib/GranularAccess.ts b/app/server/lib/GranularAccess.ts index e6116d36e0..69fcf2ee70 100644 --- a/app/server/lib/GranularAccess.ts +++ b/app/server/lib/GranularAccess.ts @@ -26,12 +26,14 @@ import { UserOverride } from 'app/common/DocListAPI'; import { DocUsageSummary, FilteredDocUsageSummary } from 'app/common/DocUsage'; import { normalizeEmail } from 'app/common/emails'; import { ErrorWithCode } from 'app/common/ErrorWithCode'; -import { InfoEditor, InfoView, UserInfo } from 'app/common/GranularAccessClause'; +import { InfoEditor } from 'app/common/GranularAccessClause'; import * as gristTypes from 'app/common/gristTypes'; import { getSetMapValue, isNonNullish, pruneArray } from 'app/common/gutil'; -import { compilePredicateFormula, EmptyRecordView, PredicateFormulaInput } from 'app/common/PredicateFormula'; +import { compilePredicateFormula, PredicateFormulaInput } from 'app/common/PredicateFormula'; import { MetaRowRecord, SingleCell } from 'app/common/TableData'; +import { EmptyRecordView, InfoView, RecordView } from 'app/common/RecordView'; import { canEdit, canView, isValidRole, Role } from 'app/common/roles'; +import { User } from 'app/common/User'; import { FullUser, UserAccessData } from 'app/common/UserAPI'; import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager'; import { GristObjCode } from 'app/plugin/GristData'; @@ -330,11 +332,106 @@ export class GranularAccess implements GranularAccessForBundle { this._userAttributesMap = new WeakMap(); } - public getUser(docSession: OptDocSession): Promise { - return this._getUser(docSession); + /** + * Construct the UserInfo needed for evaluating rules. This also enriches the user with values + * created by user-attribute rules. + */ + public async getUser(docSession: OptDocSession): Promise { + const linkParameters = docSession.authorizer?.getLinkParameters() || {}; + let access: Role | null; + let fullUser: FullUser | null; + const attrs = this._getUserAttributes(docSession); + access = getDocSessionAccess(docSession); + + const linkId = getDocSessionShare(docSession); + let shareRef: number = 0; + if (linkId) { + const rowIds = this._docData.getMetaTable('_grist_Shares').filterRowIds({ + linkId, + }); + if (rowIds.length > 1) { + throw new Error('Share identifier is not unique'); + } + if (rowIds.length === 1) { + shareRef = rowIds[0]; + } + } + + if (docSession.forkingAsOwner) { + // For granular access purposes, we become an owner. + // It is a bit of a bluff, done on the understanding that this session will + // never be used to edit the document, and that any edits will be done on a + // fork. + access = 'owners'; + } + + // If aclAsUserId/aclAsUser is set, then override user for acl purposes. + if (linkParameters.aclAsUserId || linkParameters.aclAsUser) { + if (access !== 'owners') { throw new ErrorWithCode('ACL_DENY', 'only an owner can override user'); } + if (attrs.override) { + // Used cached properties. + access = attrs.override.access; + fullUser = attrs.override.user; + } else { + attrs.override = await this._getViewAsUser(linkParameters); + fullUser = attrs.override.user; + } + } else { + fullUser = getDocSessionUser(docSession); + } + const user = new User(); + user.Access = access; + user.ShareRef = shareRef || null; + const isAnonymous = fullUser?.id === this._homeDbManager?.getAnonymousUserId() || + fullUser?.id === null; + user.UserID = (!isAnonymous && fullUser?.id) || null; + user.Email = fullUser?.email || null; + user.Name = fullUser?.name || null; + // If viewed from a websocket, collect any link parameters included. + // TODO: could also get this from rest api access, just via a different route. + user.LinkKey = linkParameters; + // Include origin info if accessed via the rest api. + // TODO: could also get this for websocket access, just via a different route. + user.Origin = docSession.req?.get('origin') || null; + user.SessionID = isAnonymous ? `a${getDocSessionAltSessionId(docSession)}` : `u${user.UserID}`; + user.IsLoggedIn = !isAnonymous; + user.UserRef = fullUser?.ref || null; // Empty string should be treated as null. + + if (this._ruler.ruleCollection.ruleError && !this._recoveryMode) { + // It is important to signal that the doc is in an unexpected state, + // and prevent it opening. + throw this._ruler.ruleCollection.ruleError; + } + + for (const clause of this._ruler.ruleCollection.getUserAttributeRules().values()) { + if (clause.name in user) { + log.warn(`User attribute ${clause.name} ignored; conflicts with an existing one`); + continue; + } + if (attrs.rows[clause.name]) { + user[clause.name] = attrs.rows[clause.name]; + continue; + } + let rec = new EmptyRecordView(); + let rows: TableDataAction|undefined; + try { + // Use lodash's get() that supports paths, e.g. charId of 'a.b' would look up `user.a.b`. + // TODO: add indexes to db. + rows = await this._fetchQueryFromDB({ + tableId: clause.tableId, + filters: { [clause.lookupColId]: [get(user, clause.charId)] } + }); + } catch (e) { + log.warn(`User attribute ${clause.name} failed`, e); + } + if (rows && rows[2].length > 0) { rec = new RecordView(rows, 0); } + user[clause.name] = rec; + attrs.rows[clause.name] = rec; + } + return user; } - public async getCachedUser(docSession: OptDocSession): Promise { + public async getCachedUser(docSession: OptDocSession): Promise { const access = await this._getAccess(docSession); return access.getUser(); } @@ -345,7 +442,7 @@ export class GranularAccess implements GranularAccessForBundle { */ public async inputs(docSession: OptDocSession): Promise { return { - user: await this._getUser(docSession), + user: await this.getUser(docSession), docId: this._docId }; } @@ -479,7 +576,7 @@ export class GranularAccess implements GranularAccessForBundle { public async canApplyBundle() { if (!this._activeBundle) { throw new Error('no active bundle'); } const {docActions, docSession, isDirect} = this._activeBundle; - const currentUser = await this._getUser(docSession); + const currentUser = await this.getUser(docSession); const userIsOwner = await this.isOwner(docSession); if (this._activeBundle.hasDeliberateRuleChange && !userIsOwner) { throw new ErrorWithCode('ACL_DENY', 'Only owners can modify access rules'); @@ -1004,7 +1101,7 @@ export class GranularAccess implements GranularAccessForBundle { } public async getUserOverride(docSession: OptDocSession): Promise { - await this._getUser(docSession); + await this.getUser(docSession); return this._getUserAttributes(docSession).override; } @@ -1120,7 +1217,7 @@ export class GranularAccess implements GranularAccessForBundle { const linkParameters = docSession.authorizer?.getLinkParameters() || {}; const baseAccess = getDocSessionAccess(docSession); if ((linkParameters.aclAsUserId || linkParameters.aclAsUser) && baseAccess === 'owners') { - const info = await this._getUser(docSession); + const info = await this.getUser(docSession); return info.Access; } return baseAccess; @@ -1838,105 +1935,6 @@ export class GranularAccess implements GranularAccessForBundle { } } - /** - * Construct the UserInfo needed for evaluating rules. This also enriches the user with values - * created by user-attribute rules. - */ - private async _getUser(docSession: OptDocSession): Promise { - const linkParameters = docSession.authorizer?.getLinkParameters() || {}; - let access: Role | null; - let fullUser: FullUser | null; - const attrs = this._getUserAttributes(docSession); - access = getDocSessionAccess(docSession); - - const linkId = getDocSessionShare(docSession); - let shareRef: number = 0; - if (linkId) { - const rowIds = this._docData.getMetaTable('_grist_Shares').filterRowIds({ - linkId, - }); - if (rowIds.length > 1) { - throw new Error('Share identifier is not unique'); - } - if (rowIds.length === 1) { - shareRef = rowIds[0]; - } - } - - if (docSession.forkingAsOwner) { - // For granular access purposes, we become an owner. - // It is a bit of a bluff, done on the understanding that this session will - // never be used to edit the document, and that any edits will be done on a - // fork. - access = 'owners'; - } - - // If aclAsUserId/aclAsUser is set, then override user for acl purposes. - if (linkParameters.aclAsUserId || linkParameters.aclAsUser) { - if (access !== 'owners') { throw new ErrorWithCode('ACL_DENY', 'only an owner can override user'); } - if (attrs.override) { - // Used cached properties. - access = attrs.override.access; - fullUser = attrs.override.user; - } else { - attrs.override = await this._getViewAsUser(linkParameters); - fullUser = attrs.override.user; - } - } else { - fullUser = getDocSessionUser(docSession); - } - const user = new User(); - user.Access = access; - user.ShareRef = shareRef || null; - const isAnonymous = fullUser?.id === this._homeDbManager?.getAnonymousUserId() || - fullUser?.id === null; - user.UserID = (!isAnonymous && fullUser?.id) || null; - user.Email = fullUser?.email || null; - user.Name = fullUser?.name || null; - // If viewed from a websocket, collect any link parameters included. - // TODO: could also get this from rest api access, just via a different route. - user.LinkKey = linkParameters; - // Include origin info if accessed via the rest api. - // TODO: could also get this for websocket access, just via a different route. - user.Origin = docSession.req?.get('origin') || null; - user.SessionID = isAnonymous ? `a${getDocSessionAltSessionId(docSession)}` : `u${user.UserID}`; - user.IsLoggedIn = !isAnonymous; - user.UserRef = fullUser?.ref || null; // Empty string should be treated as null. - - if (this._ruler.ruleCollection.ruleError && !this._recoveryMode) { - // It is important to signal that the doc is in an unexpected state, - // and prevent it opening. - throw this._ruler.ruleCollection.ruleError; - } - - for (const clause of this._ruler.ruleCollection.getUserAttributeRules().values()) { - if (clause.name in user) { - log.warn(`User attribute ${clause.name} ignored; conflicts with an existing one`); - continue; - } - if (attrs.rows[clause.name]) { - user[clause.name] = attrs.rows[clause.name]; - continue; - } - let rec = new EmptyRecordView(); - let rows: TableDataAction|undefined; - try { - // Use lodash's get() that supports paths, e.g. charId of 'a.b' would look up `user.a.b`. - // TODO: add indexes to db. - rows = await this._fetchQueryFromDB({ - tableId: clause.tableId, - filters: { [clause.lookupColId]: [get(user, clause.charId)] } - }); - } catch (e) { - log.warn(`User attribute ${clause.name} failed`, e); - } - if (rows && rows[2].length > 0) { rec = new RecordView(rows, 0); } - user[clause.name] = rec; - attrs.rows[clause.name] = rec; - } - return user; - } - /** * Get the "View As" user specified in link parameters. * If aclAsUserId is set, we get the user with the specified id. @@ -2583,7 +2581,7 @@ export class GranularAccess implements GranularAccessForBundle { /** * Tests if the user can modify cell's data. */ - private async _canApplyCellActions(currentUser: UserInfo, userIsOwner: boolean) { + private async _canApplyCellActions(currentUser: User, userIsOwner: boolean) { // Owner can modify all comments, without exceptions. if (userIsOwner) { return; @@ -2654,7 +2652,7 @@ export class Ruler { } export interface RulerOwner { - getUser(docSession: OptDocSession): Promise; + getUser(docSession: OptDocSession): Promise; inputs(docSession: OptDocSession): Promise; } @@ -2688,36 +2686,6 @@ interface ActionCursor { // access control state. } -/** - * A row-like view of TableDataAction, which is columnar in nature. If index value - * is undefined, acts as an EmptyRecordRow. - */ -export class RecordView implements InfoView { - public constructor(public data: TableDataAction, public index: number|undefined) { - } - - public get(colId: string): CellValue { - if (this.index === undefined) { return null; } - if (colId === 'id') { - return this.data[2][this.index]; - } - return this.data[3][colId]?.[this.index]; - } - - public has(colId: string) { - return colId === 'id' || colId in this.data[3]; - } - - public toJSON() { - if (this.index === undefined) { return {}; } - const results: {[key: string]: any} = {}; - for (const key of Object.keys(this.data[3])) { - results[key] = this.data[3][key]?.[this.index]; - } - return results; - } -} - /** * A read-write view of a DataAction, for use in censorship. */ @@ -3222,47 +3190,6 @@ export function filterColValues(action: DataAction, return [action, ...[...parts.keys()].sort().map(key => parts.get(key)!)]; } -/** - * Information about a user, including any user attributes. - * - * Serializes into a more compact JSON form that excludes full - * row data, only keeping user info and table/row ids for any - * user attributes. - * - * See `user.py` for the sandbox equivalent that deserializes objects of this class. - */ -export class User implements UserInfo { - public Name: string | null = null; - public UserID: number | null = null; - public Access: Role | null = null; - public Origin: string | null = null; - public LinkKey: Record = {}; - public Email: string | null = null; - public SessionID: string | null = null; - public UserRef: string | null = null; - public ShareRef: number | null = null; - [attribute: string]: any; - - constructor(_info: Record = {}) { - Object.assign(this, _info); - } - - public toJSON() { - const results: {[key: string]: any} = {}; - for (const [key, value] of Object.entries(this)) { - if (value instanceof RecordView) { - // Only include the table id and first matching row id. - results[key] = [getTableId(value.data), value.get('id')]; - } else if (value instanceof EmptyRecordView) { - results[key] = null; - } else { - results[key] = value; - } - } - return results; - } -} - export function validTableIdString(tableId: any): string { if (typeof tableId !== 'string') { throw new Error(`Expected tableId to be a string`); } return tableId; diff --git a/app/server/lib/PermissionInfo.ts b/app/server/lib/PermissionInfo.ts index 7ef7860f63..d779bf9603 100644 --- a/app/server/lib/PermissionInfo.ts +++ b/app/server/lib/PermissionInfo.ts @@ -3,8 +3,9 @@ import { ALL_PERMISSION_PROPS, emptyPermissionSet, MixedPermissionSet, PartialPermissionSet, PermissionSet, TablePermissionSet, toMixed } from 'app/common/ACLPermissions'; import { ACLRuleCollection } from 'app/common/ACLRuleCollection'; -import { RuleSet, UserInfo } from 'app/common/GranularAccessClause'; +import { RuleSet } from 'app/common/GranularAccessClause'; import { PredicateFormulaInput } from 'app/common/PredicateFormula'; +import { User } from 'app/common/User'; import { getSetMapValue } from 'app/common/gutil'; import log from 'app/server/lib/log'; import { mapValues } from 'lodash'; @@ -80,8 +81,8 @@ abstract class RuleInfo { return this._mergeFullAccess(tableAccess); } - public getUser(): UserInfo { - return this._input.user!; + public getUser(): User { + return this._input.user! as User; } protected abstract _processRule(ruleSet: RuleSet, defaultAccess?: () => MixedT): MixedT; diff --git a/test/nbrowser/DropdownConditionEditor.ts b/test/nbrowser/DropdownConditionEditor.ts index cdab44ab2b..9dff24b85f 100644 --- a/test/nbrowser/DropdownConditionEditor.ts +++ b/test/nbrowser/DropdownConditionEditor.ts @@ -1,3 +1,4 @@ +import {UserAPI} from 'app/common/UserAPI'; import {assert, driver, Key} from 'mocha-webdriver'; import * as gu from 'test/nbrowser/gristUtils'; import {setupTestSuite} from 'test/nbrowser/testUtils'; @@ -5,22 +6,56 @@ import {setupTestSuite} from 'test/nbrowser/testUtils'; describe('DropdownConditionEditor', function () { this.timeout(20000); const cleanup = setupTestSuite(); + let api: UserAPI; + let docId: string; before(async () => { - const session = await gu.session().login(); - await session.tempDoc(cleanup, 'DropdownCondition.grist'); + const session = await gu.session().user('user1').login(); + api = session.createHomeApi(); + docId = (await session.tempDoc(cleanup, 'DropdownCondition.grist')).id; + await api.updateDocPermissions(docId, {users: { + [gu.translateUser('user2').email]: 'editors', + }}); + await addUserAttributes(); + await gu.openPage('Employees'); await gu.openColumnPanel(); }); afterEach(() => gu.checkForErrors()); + async function addUserAttributes() { + await api.applyUserActions(docId, [ + ['AddTable', 'Roles', [{id: 'Email'}, {id: 'Admin', type: 'Bool'}]], + ['AddRecord', 'Roles', null, {Email: gu.translateUser('user1').email, Admin: true}], + ['AddRecord', 'Roles', null, {Email: gu.translateUser('user2').email, Admin: false}], + ]); + await driver.find('.test-tools-access-rules').click(); + await gu.waitForServer(); + await driver.findContentWait('button', /Add User Attributes/, 2000).click(); + const userAttrRule = await driver.find('.test-rule-userattr'); + await userAttrRule.find('.test-rule-userattr-name').click(); + await driver.sendKeys('Roles', Key.ENTER); + await userAttrRule.find('.test-rule-userattr-attr').click(); + await driver.sendKeys('Email', Key.ENTER); + await userAttrRule.find('.test-rule-userattr-table').click(); + await driver.findContent('.test-select-menu li', 'Roles').click(); + await userAttrRule.find('.test-rule-userattr-col').click(); + await driver.sendKeys('Email', Key.ENTER); + await driver.find('.test-rules-save').click(); + await gu.waitForServer(); + } + describe(`in choice columns`, function() { + before(async () => { + const session = await gu.session().user('user1').login(); + await session.loadDoc(`/doc/${docId}`); + }); + it('creates dropdown conditions', async function() { await gu.getCell(1, 1).click(); - assert.isFalse(await driver.find('.test-field-dropdown-condition').isPresent()); - await driver.find('.test-field-set-dropdown-condition').click(); + await driver.find('.test-field-dropdown-condition').click(); await gu.waitAppFocus(false); - await gu.sendKeys('c'); + await gu.sendKeys(await gu.selectAllKey(), Key.DELETE, 'c'); await gu.waitToPass(async () => { const completions = await driver.findAll('.ace_autocomplete .ace_line', el => el.getText()); assert.deepEqual(completions, [ @@ -28,6 +63,7 @@ describe('DropdownConditionEditor', function () { 're\nc\n.Name\n ', 're\nc\n.Role\n ', 're\nc\n.Supervisor\n ', + 'user.A\nc\ncess\n ', ]); }); await gu.sendKeysSlowly(['hoice not in $']); @@ -141,6 +177,11 @@ describe('DropdownConditionEditor', function () { }); describe(`in reference columns`, function() { + before(async () => { + const session = await gu.session().user('user1').login(); + await session.loadDoc(`/doc/${docId}`); + }); + it('creates dropdown conditions', async function() { await gu.getCell(2, 1).click(); assert.isFalse(await driver.find('.test-field-dropdown-condition').isPresent()); @@ -288,4 +329,44 @@ describe('DropdownConditionEditor', function () { await gu.sendKeys(Key.ESCAPE); }); }); + + it('supports user variable', async function() { + // Filter dropdown values based on a user attribute. + await gu.getCell(1, 1).click(); + await driver.find('.test-field-set-dropdown-condition').click(); + await gu.waitAppFocus(false); + await gu.sendKeysSlowly(['user.']); + await gu.waitToPass(async () => { + const completions = await driver.findAll('.ace_autocomplete .ace_line', el => el.getText()); + assert.deepEqual(completions, [ + 'user.\nAccess\n ', + 'user.\nEmail\n ', + 'user.\nIsLoggedIn\n ', + 'user.\nLinkKey.\n ', + 'user.\nName\n ', + 'user.\nOrigin\n ', + 'user.\nRoles.Admin\n ', + 'user.\nRoles.Email\n ', + '' + ]); + }); + await gu.sendKeys('Roles.Admin == True', Key.ENTER); + await gu.waitForServer(); + + // Check that user1 (who is an admin) can see dropdown values. + await gu.sendKeys(Key.ENTER); + assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [ + 'Trainee', + 'Supervisor', + ]); + await gu.sendKeys(Key.ESCAPE); + + // Switch to user2 (who is not an admin), and check that they can't see any dropdown values. + const session = await gu.session().user('user2').login(); + await session.loadDoc(`/doc/${docId}`); + await gu.getCell(1, 1).click(); + await gu.sendKeys(Key.ENTER); + assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), []); + await gu.sendKeys(Key.ESCAPE); + }); }); diff --git a/test/server/lib/ACLFormula.ts b/test/server/lib/ACLFormula.ts index 8951c0f942..1fcd89dc65 100644 --- a/test/server/lib/ACLFormula.ts +++ b/test/server/lib/ACLFormula.ts @@ -1,9 +1,9 @@ import {CellValue} from 'app/common/DocActions'; -import {InfoView} from 'app/common/GranularAccessClause'; import {GristObjCode} from 'app/plugin/GristData'; import {CompiledPredicateFormula, compilePredicateFormula} from 'app/common/PredicateFormula'; +import {InfoView} from 'app/common/RecordView'; +import {User} from 'app/common/User'; import {makeExceptionalDocSession} from 'app/server/lib/DocSession'; -import {User} from 'app/server/lib/GranularAccess'; import {assert} from 'chai'; import {createDocTools} from 'test/server/docTools'; import * as testUtils from 'test/server/testUtils'; From 7ed70b9e8893be447d8ef5f06f8121385309a84b Mon Sep 17 00:00:00 2001 From: Dmitry S Date: Mon, 10 Jun 2024 12:55:26 -0400 Subject: [PATCH 30/62] (core) Fix for flaky GridViewNewColumnMenu test which may have been flaky because of window resizing. Test Plan: Only a test change Reviewers: jarek Reviewed By: jarek Differential Revision: https://phab.getgrist.com/D4267 --- test/nbrowser/GridViewNewColumnMenu.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/nbrowser/GridViewNewColumnMenu.ts b/test/nbrowser/GridViewNewColumnMenu.ts index 0054662e67..d8faa9c482 100644 --- a/test/nbrowser/GridViewNewColumnMenu.ts +++ b/test/nbrowser/GridViewNewColumnMenu.ts @@ -341,6 +341,13 @@ describe('GridViewNewColumnMenu', function () { describe('create formula column', function(){ revertThis(); + + before(async function() { + // Previous test runs in a smaller screen. It restores the window, but it's hard to know + // when all the resizing has taken effect, and easier to just reload the doc. + await gu.reloadDoc(); + }); + it('should show "create formula column" option with tooltip', async function () { // open add new colum menu await clickAddColumn(); From b4344c19d88bcd20884c08b0652a13dda3ea032d Mon Sep 17 00:00:00 2001 From: Paul Janzen Date: Mon, 3 Jun 2024 17:05:34 +0000 Subject: [PATCH 31/62] Translated using Weblate (Portuguese (Brazil)) Currently translated at 100.0% (1334 of 1334 strings) Translation: Grist/client Translate-URL: https://hosted.weblate.org/projects/grist/client/pt_BR/ --- static/locales/pt_BR.client.json | 49 ++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/static/locales/pt_BR.client.json b/static/locales/pt_BR.client.json index 6c7f2223ca..950fd01c80 100644 --- a/static/locales/pt_BR.client.json +++ b/static/locales/pt_BR.client.json @@ -114,7 +114,9 @@ "Legacy": "Legado", "Personal Site": "Site pessoal", "Team Site": "Site da Equipe", - "Grist Templates": "Modelos de Grist" + "Grist Templates": "Modelos de Grist", + "Billing Account": "Conta de faturamento", + "Manage Team": "Gerenciar Equipe" }, "AppModel": { "This team site is suspended. Documents can be read, but not modified.": "Este site da equipe está suspenso. Os documentos podem ser lidos, mas não modificados." @@ -335,7 +337,17 @@ "Notify other services on doc changes": "Notifique outros serviços em alterações doc", "Reload": "Recarregar", "Time Zone": "Fuso horário", - "python3 (recommended)": "python3 (recomendado)" + "python3 (recommended)": "python3 (recomendado)", + "Time reload": "Recarga de tempo", + "Timing is on": "O tempo está ligado", + "You can make changes to the document, then stop timing to see the results.": "Você pode fazer alterações no documento e, em seguida, interromper a cronometragem para ver os resultados.", + "Cancel": "Cancelar", + "Force reload the document while timing formulas, and show the result.": "Forçar o recarregamento do documento durante a cronometragem das fórmulas e mostrar o resultado.", + "Formula timer": "Temporizador de Fórmula", + "Reload data engine": "Recarregar o motor de dados", + "Reload data engine?": "Recarregar o motor de dados?", + "Start timing": "Iniciar cronometragem", + "Stop timing...": "Pare de cronometrar..." }, "DocumentUsage": { "Attachments Size": "Tamanho dos Anexos", @@ -557,7 +569,8 @@ "Trash": "Lixeira", "Workspace will be moved to Trash.": "A Área de Trabalho será movida pra Lixeira.", "Workspaces": "Áreas de Trabalho", - "Tutorial": "Tutorial" + "Tutorial": "Tutorial", + "Terms of service": "Termos de serviço" }, "Importer": { "Merge rows that match these fields:": "Mesclar linhas que correspondem a estes campos:", @@ -1083,7 +1096,9 @@ "Rule must return True or False": "A regra deve retornar Verdadeiro ou Falso", "Error in style rule": "Erro na regra de estilo", "Add another rule": "Adicionar outra regra", - "Add conditional style": "Adicionar estilo condicional" + "Add conditional style": "Adicionar estilo condicional", + "Conditional Style": "Estilo condicional", + "IF...": "SE..." }, "CurrencyPicker": { "Invalid currency": "Moeda inválida" @@ -1586,7 +1601,21 @@ "Auto-check when this page loads": "Verificar automaticamente quando esta página carregar", "Checking for updates...": "Verificando atualizações...", "Error checking for updates": "Erro ao verificar atualizações", - "Grist allows for very powerful formulas, using Python. We recommend setting the environment variable GRIST_SANDBOX_FLAVOR to gvisor if your hardware supports it (most will), to run formulas in each document within a sandbox isolated from other documents and isolated from the network.": "Grist permite fórmulas muito poderosas, usando Python. Recomendamos definir a variável de ambiente GRIST_SANDBOX_FLAVOR para gvisor se o seu hardware o suporta (a maioria suportará), para executar fórmulas em cada documento dentro de uma caixa de areia isolada de outros documentos e isolada da rede." + "Grist allows for very powerful formulas, using Python. We recommend setting the environment variable GRIST_SANDBOX_FLAVOR to gvisor if your hardware supports it (most will), to run formulas in each document within a sandbox isolated from other documents and isolated from the network.": "Grist permite fórmulas muito poderosas, usando Python. Recomendamos definir a variável de ambiente GRIST_SANDBOX_FLAVOR para gvisor se o seu hardware o suporta (a maioria suportará), para executar fórmulas em cada documento dentro de uma caixa de areia isolada de outros documentos e isolada da rede.", + "Details": "Detalhes", + "No fault detected.": "Nenhuma falha detectada.", + "Notes": "Notas", + "Or, as a fallback, you can set: {{bootKey}} in the environment and visit: {{url}}": "Ou, como alternativa, você pode definir: {{bootKey}} no ambiente e visitar: {{url}}", + "Results": "Resultados", + "Self Checks": "Autoverificações", + "You do not have access to the administrator panel.\nPlease log in as an administrator.": "Você não tem acesso ao painel do administrador.\nFaça login como administrador.", + "Administrator Panel Unavailable": "Painel do administrador indisponível", + "Check succeeded.": "A verificação foi bem-sucedida.", + "Authentication": "Autenticação", + "Check failed.": "A verificação falhou.", + "Current authentication method": "Método de autenticação atual", + "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.": "O Grist permite a configuração de diferentes tipos de autenticação, incluindo SAML e OIDC. Recomendamos ativar um desses tipos se o Grist for acessível pela rede ou estiver sendo disponibilizado para várias pessoas.", + "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.": "O Grist permite a configuração de diferentes tipos de autenticação, incluindo SAML e OIDC. Recomendamos ativar um desses tipos se o Grist for acessível pela rede ou estiver sendo disponibilizado para várias pessoas." }, "Field": { "No choices configured": "Nenhuma opção configurada", @@ -1651,5 +1680,15 @@ "Custom": "Personalizado", "Table": "Tabela", "Form": "Formulário" + }, + "TimingPage": { + "Average Time (s)": "Tempo(s) médio(s)", + "Max Time (s)": "Tempo(s) máximo(s)", + "Total Time (s)": "Tempo(s) total(s)", + "Formula timer": "Temporizador de Fórmula", + "Column ID": "ID da Coluna", + "Loading timing data. Don't close this tab.": "Carregando dados de tempo. Não feche essa guia.", + "Number of Calls": "Número de chamadas", + "Table ID": "ID da tabela" } } From f4f0c2bd20ba8288cfaa29c5f2070b9c0e5641be Mon Sep 17 00:00:00 2001 From: Paul Janzen Date: Mon, 3 Jun 2024 17:13:39 +0000 Subject: [PATCH 32/62] Translated using Weblate (German) Currently translated at 100.0% (1334 of 1334 strings) Translation: Grist/client Translate-URL: https://hosted.weblate.org/projects/grist/client/de/ --- static/locales/de.client.json | 49 +++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/static/locales/de.client.json b/static/locales/de.client.json index 67fb9665a6..af9e532d41 100644 --- a/static/locales/de.client.json +++ b/static/locales/de.client.json @@ -114,7 +114,9 @@ "Legacy": "Hinterlassenschaft", "Personal Site": "Persönliche Seite", "Team Site": "Teamseite", - "Grist Templates": "Grist Vorlagen" + "Grist Templates": "Grist Vorlagen", + "Billing Account": "Abrechnungskonto", + "Manage Team": "Team verwalten" }, "AppModel": { "This team site is suspended. Documents can be read, but not modified.": "Diese Teamseite ist gesperrt. Die Dokumente können gelesen, aber nicht geändert werden." @@ -335,7 +337,17 @@ "Data Engine": "Datenmaschine", "Default for DateTime columns": "Standard für DateTime-Spalten", "Document ID": "Dokument-ID", - "Document ID to use whenever the REST API calls for {{docId}}. See {{apiURL}}": "Dokument-ID, die bei Aufrufen der REST-API für {{docId}} zu verwenden ist. Siehe {{apiURL}}" + "Document ID to use whenever the REST API calls for {{docId}}. See {{apiURL}}": "Dokument-ID, die bei Aufrufen der REST-API für {{docId}} zu verwenden ist. Siehe {{apiURL}}", + "Reload data engine": "Datenmaschine neu laden", + "Reload data engine?": "Datenmaschine neu laden?", + "Start timing": "Startzeitpunkt", + "Stop timing...": "Stoppt die Zeitmessung...", + "Time reload": "Zeit nachladen", + "Force reload the document while timing formulas, and show the result.": "Erzwingen Sie das Neuladen des Dokuments während der Zeitmessung von Formeln, und zeigen Sie das Ergebnis an.", + "Formula timer": "Formel Timer", + "Cancel": "Abbrechen", + "Timing is on": "Das Timing läuft", + "You can make changes to the document, then stop timing to see the results.": "Sie können Änderungen an dem Dokument vornehmen und dann die Zeitmessung stoppen, um die Ergebnisse zu sehen." }, "DocumentUsage": { "Attachments Size": "Größe der Anhänge", @@ -557,7 +569,8 @@ "Trash": "Papierkorb", "Workspace will be moved to Trash.": "Der Arbeitsbereich wird in den Papierkorb verschoben.", "Workspaces": "Arbeitsbereiche", - "Tutorial": "Tutorial" + "Tutorial": "Tutorial", + "Terms of service": "Nutzungsbedingungen" }, "Importer": { "Merge rows that match these fields:": "Zeilen zusammenführen, die mit diesen Feldern übereinstimmen:", @@ -1099,7 +1112,9 @@ "Error in style rule": "Fehler in der Stilregel", "Rule must return True or False": "Regel muss wahr oder falsch zurückgeben", "Add another rule": "Eine weitere Regel hinzufügen", - "Row Style": "Zeilenstil" + "Row Style": "Zeilenstil", + "IF...": "WENN...", + "Conditional Style": "Bedingter Stil" }, "CurrencyPicker": { "Invalid currency": "Ungültige Währung" @@ -1582,7 +1597,21 @@ "unconfigured": "unkonfiguriert", "unknown": "unbekannt", "Check now": "Jetzt prüfen", - "Grist allows for very powerful formulas, using Python. We recommend setting the environment variable GRIST_SANDBOX_FLAVOR to gvisor if your hardware supports it (most will), to run formulas in each document within a sandbox isolated from other documents and isolated from the network.": "Grist ermöglicht sehr leistungsfähige Formeln, die Python verwenden. Wir empfehlen, die Umgebungsvariable GRIST_SANDBOX_FLAVOR auf gvisor zu setzen, wenn Ihre Hardware dies unterstützt (was bei den meisten der Fall ist), um Formeln in jedem Dokument innerhalb einer Sandbox auszuführen, die von anderen Dokumenten und vom Netzwerk isoliert ist." + "Grist allows for very powerful formulas, using Python. We recommend setting the environment variable GRIST_SANDBOX_FLAVOR to gvisor if your hardware supports it (most will), to run formulas in each document within a sandbox isolated from other documents and isolated from the network.": "Grist ermöglicht sehr leistungsfähige Formeln, die Python verwenden. Wir empfehlen, die Umgebungsvariable GRIST_SANDBOX_FLAVOR auf gvisor zu setzen, wenn Ihre Hardware dies unterstützt (was bei den meisten der Fall ist), um Formeln in jedem Dokument innerhalb einer Sandbox auszuführen, die von anderen Dokumenten und vom Netzwerk isoliert ist.", + "Results": "Ergebnisse", + "Self Checks": "Selbstkontrolle", + "Check failed.": "Prüfung fehlgeschlagen.", + "Administrator Panel Unavailable": "Administrator-Panel Nicht verfügbar", + "Authentication": "Authentifizierung", + "Check succeeded.": "Prüfung gelungen.", + "Notes": "Anmerkungen", + "You do not have access to the administrator panel.\nPlease log in as an administrator.": "Sie haben keinen Zugriff auf das Administrator-Panel.\nBitte melden Sie sich als Administrator an.", + "Current authentication method": "Aktuelle Authentifizierungsmethode", + "Details": "Details", + "No fault detected.": "Kein Fehler erkannt.", + "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.": "In Grist können verschiedene Arten der Authentifizierung konfiguriert werden, darunter SAML und OIDC. Wir empfehlen, eine davon zu aktivieren, wenn Grist über das Netzwerk zugänglich ist oder mehreren Personen zur Verfügung gestellt wird.", + "Or, as a fallback, you can set: {{bootKey}} in the environment and visit: {{url}}": "Als Ausweichmöglichkeit können Sie auch {{bootKey}} in der Umgebung einstellen und {{url}} besuchen", + "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.": "In Grist können verschiedene Arten der Authentifizierung konfiguriert werden, darunter SAML und OIDC. Wir empfehlen, eine davon zu aktivieren, wenn Grist über das Netzwerk zugänglich ist oder mehreren Personen zur Verfügung gestellt wird." }, "Section": { "Insert section above": "Abschnitt oben einfügen", @@ -1651,5 +1680,15 @@ "Table": "Tabelle", "Chart": "Diagramm", "Custom": "Benutzerdefiniert" + }, + "TimingPage": { + "Max Time (s)": "Max Zeit(en)", + "Number of Calls": "Anzahl der Anrufe", + "Table ID": "Tabelle ID", + "Total Time (s)": "Gesamtzeit(en)", + "Formula timer": "Formel Timer", + "Average Time (s)": "Durchschnittliche Zeit(en)", + "Loading timing data. Don't close this tab.": "Zeitpunktsdaten laden. Schließen Sie diese Registerkarte nicht.", + "Column ID": "Spalte ID" } } From b9600c50bcf62ed7dd84eaf97d1561ebf63c3471 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=BB=D0=B0=D0=B4=D0=B8=D0=BC=D0=B8=D1=80=20=D0=92?= Date: Mon, 3 Jun 2024 14:11:06 +0000 Subject: [PATCH 33/62] Translated using Weblate (Russian) Currently translated at 99.6% (1329 of 1334 strings) Translation: Grist/client Translate-URL: https://hosted.weblate.org/projects/grist/client/ru/ --- static/locales/ru.client.json | 98 ++++++++++++++++++++++++++++++++--- 1 file changed, 90 insertions(+), 8 deletions(-) diff --git a/static/locales/ru.client.json b/static/locales/ru.client.json index 56ba559370..5fbdc3b4f1 100644 --- a/static/locales/ru.client.json +++ b/static/locales/ru.client.json @@ -97,7 +97,9 @@ "Home Page": "Домашняя страница", "Legacy": "Устаревший", "Team Site": "Сайт группы", - "Grist Templates": "Шаблоны Grist" + "Grist Templates": "Шаблоны Grist", + "Billing Account": "Платежный аккаунт", + "Manage Team": "Управление командой" }, "ApiKey": { "Remove API Key": "Удалить ключ API", @@ -421,7 +423,43 @@ "Document ID copied to clipboard": "Идентификатор документа скопирован в буфер обмена", "Manage Webhooks": "Управление вебхуками", "Webhooks": "Вебхуки", - "API Console": "API-консоль" + "API Console": "API-консоль", + "Default for DateTime columns": "По умолчанию для столбцов ДатаВремя", + "Document ID": "ID Документа", + "Notify other services on doc changes": "Уведомлять другие службы об изменениях в документе", + "python2 (legacy)": "python2 (устаревший)", + "Formula timer": "Таймер формулы", + "Reload data engine": "Перезагрузка механизма обработки данных", + "Start timing": "Старт тайминга", + "Stop timing...": "Остановка тайминга...", + "Currency": "Валюта", + "Data Engine": "Механизм обработки данных", + "Hard reset of data engine": "Жесткий сброс системы обработки данных", + "python3 (recommended)": "python3 (рекомендуется)", + "Cancel": "Отмена", + "Force reload the document while timing formulas, and show the result.": "Принудительно перезагрузите документ, синхронизируя формулы, и покажите результат.", + "Time reload": "Время перезагрузки", + "Reload data engine?": "Перезагрузить механизм обработки данных?", + "Timing is on": "Тайминг включен", + "You can make changes to the document, then stop timing to see the results.": "Вы можете внести изменения в документ, а затем остановить отсчет времени, чтобы увидеть результаты.", + "Coming soon": "Скоро будет", + "Copy to clipboard": "Скопировать в буфер обмена", + "API URL copied to clipboard": "API URL скопирован в буфер обмена", + "API console": "Консоль API", + "API documentation.": "Документация по API.", + "Base doc URL: {{docApiUrl}}": "Базовый URL документа: {{docApiUrl}}", + "Find slow formulas": "Найти медленные формулы", + "For currency columns": "Для столбцов валют", + "For number and date formats": "Для форматов чисел и дат", + "Document ID to use whenever the REST API calls for {{docId}}. See {{apiURL}}": "ID Документа, используется всегда при запросе REST API {{docId}}. Смотреть {{apiURL}}", + "ID for API use": "ID для использования API", + "Manage webhooks": "Управление webhook-ами", + "Locale": "Локализация", + "Reload": "Перезагрузить", + "Python": "Python", + "Python version used": "Python используемая версия", + "Time Zone": "Часовой пояс", + "Try API calls from the browser": "Попробуйте вызовы API из браузера" }, "DocPageModel": { "Add Widget to Page": "Добавить виджет на страницу", @@ -524,7 +562,8 @@ "Trash": "Корзина", "Workspaces": "Рабочие пространства", "Workspace will be moved to Trash.": "Рабочее пространство будет перемещено в корзину.", - "Tutorial": "Обучение" + "Tutorial": "Обучение", + "Terms of service": "Условия использования" }, "GridViewMenus": { "Add to sort": "Добавить в сортировку", @@ -667,7 +706,8 @@ }, "OnBoardingPopups": { "Finish": "Закончить", - "Next": "Дальше" + "Next": "Дальше", + "Previous": "Предыдущий" }, "PageWidgetPicker": { "Group by": "Группировать по", @@ -1009,7 +1049,8 @@ "Don't show tips": "Не показывать советы", "Undo to restore": "Отменить для восстановления", "Got it": "Принято", - "Don't show again": "Больше не показывать" + "Don't show again": "Больше не показывать", + "TIP": "Совет" }, "search": { "Find Next ": "Найти далее ", @@ -1059,7 +1100,9 @@ "Add conditional style": "Добавить условное форматирование", "Error in style rule": "Ошибка в правиле форматирования", "Row Style": "Форматирование строки", - "Rule must return True or False": "Правило должно возвращать значение Истина или Ложь" + "Rule must return True or False": "Правило должно возвращать значение Истина или Ложь", + "IF...": "ЕСЛИ...", + "Conditional Style": "Условный стиль" }, "CurrencyPicker": { "Invalid currency": "Неверная валюта" @@ -1493,7 +1536,21 @@ "Newer version available": "Доступна более новая версия", "OK": "OK", "unconfigured": "несконфигурированно", - "Security Settings": "Настройки безопасности" + "Security Settings": "Настройки безопасности", + "Authentication": "Аутентификация", + "Check failed.": "Проверка не удалась.", + "Check succeeded.": "Проверка прошла успешно.", + "Details": "Подробности", + "No fault detected.": "Неисправностей не обнаружено.", + "Notes": "Примечания", + "Results": "Результаты", + "Self Checks": "Самопроверки", + "Administrator Panel Unavailable": "Панель администратора недоступна", + "Current authentication method": "Текущий метод аутентификации", + "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.": "Grist позволяет настраивать различные типы аутентификации, включая SAML и OIDC. Мы рекомендуем включить один из них, если Grist доступен по сети или доступен нескольким пользователям.", + "Or, as a fallback, you can set: {{bootKey}} in the environment and visit: {{url}}": "Или, в качестве запасного варианта, вы можете установить: {{bootKey}} в окружающей среде и посетить: {{url}}", + "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.": "Grist позволяет настраивать различные типы аутентификации, включая SAML и OIDC. Мы рекомендуем включить один из них, если Grist доступен по сети или доступен нескольким пользователям.", + "You do not have access to the administrator panel.\nPlease log in as an administrator.": "У вас нет доступа к панели администратора.\nПожалуйста, войдите в систему как администратор." }, "CreateTeamModal": { "Billing is not supported in grist-core": "Выставление счетов в grist-core не поддерживается", @@ -1511,7 +1568,7 @@ }, "Field": { "No choices configured": "Опции отсутствуют", - "No values in show column of referenced table": "No values in show column of referenced table" + "No values in show column of referenced table": "Нет значений в столбце отображения ссылочной таблицы." }, "Columns": { "Remove Column": "Удалить столбец" @@ -1543,5 +1600,30 @@ "No choices matching condition": "Нет вариантов, соответствующих условию", "No choices to select": "Нет вариантов для выбора", "Error in dropdown condition": "Ошибка в условиях выпадающего списка" + }, + "widgetTypesMap": { + "Custom": "Кастомный", + "Form": "Форма", + "Table": "Таблица", + "Calendar": "Календарь", + "Card": "Карточка", + "Card List": "Список карточек", + "Chart": "Диаграмма" + }, + "TimingPage": { + "Table ID": "ID таблицы", + "Formula timer": "Таймер формулы", + "Average Time (s)": "Среднее время (с)", + "Loading timing data. Don't close this tab.": "Загрузка данных о времени. Не закрывайте эту вкладку.", + "Column ID": "ID столбца", + "Max Time (s)": "Макс. время (с)", + "Number of Calls": "Количество Вызовов", + "Total Time (s)": "Общее время (с)" + }, + "FormRenderer": { + "Reset": "Сброс", + "Search": "Поиск", + "Select...": "Выбрать...", + "Submit": "Отправить" } } From 42742b8574b6f3d6fec35a7ec52a8242187756ff Mon Sep 17 00:00:00 2001 From: Florentina Petcu Date: Tue, 4 Jun 2024 06:06:57 +0000 Subject: [PATCH 34/62] Translated using Weblate (Romanian) Currently translated at 80.2% (1071 of 1334 strings) Translation: Grist/client Translate-URL: https://hosted.weblate.org/projects/grist/client/ro/ --- static/locales/ro.client.json | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/static/locales/ro.client.json b/static/locales/ro.client.json index 19e75da608..6b53dd553d 100644 --- a/static/locales/ro.client.json +++ b/static/locales/ro.client.json @@ -245,7 +245,8 @@ "Home Page": "Pagina principală", "Team Site": "Spaţiul echipei", "Legacy": "Versiune veche", - "Grist Templates": "Șabloane Grist" + "Grist Templates": "Șabloane Grist", + "Manage Team": "Gestionează echipa" }, "ViewAsDropdown": { "View As": "Vizualizare ca", @@ -842,7 +843,12 @@ "Python": "Python", "Python version used": "Versiunea Python folosită", "Reload": "Reîncarcă", - "For number and date formats": "Pentru formatele de număr și dată" + "For number and date formats": "Pentru formatele de număr și dată", + "Cancel": "Anulează", + "Start timing": "Începe cronometrarea", + "Manage webhooks": "Gestionează ancore Web", + "Stop timing...": "Oprește cronometrarea...", + "python3 (recommended)": "python3 (recomandat)" }, "ColumnTitle": { "Column ID copied to clipboard": "ID-ul coloanei a fost copiat în clipboard", From 2c740627bdf8ef2a4a7d4bdb427caecd0573dff9 Mon Sep 17 00:00:00 2001 From: Paul Janzen Date: Tue, 4 Jun 2024 13:25:40 +0000 Subject: [PATCH 35/62] Translated using Weblate (Spanish) Currently translated at 100.0% (1334 of 1334 strings) Translation: Grist/client Translate-URL: https://hosted.weblate.org/projects/grist/client/es/ --- static/locales/es.client.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/static/locales/es.client.json b/static/locales/es.client.json index 891afbd2eb..8d6b2f3258 100644 --- a/static/locales/es.client.json +++ b/static/locales/es.client.json @@ -281,7 +281,8 @@ "Python": "Python", "python3 (recommended)": "python3 (recomendado)", "Cancel": "Cancelar", - "Force reload the document while timing formulas, and show the result.": "Fuerza la recarga del documento mientras sincronizas las fórmulas y muestra el resultado." + "Force reload the document while timing formulas, and show the result.": "Fuerza la recarga del documento mientras sincronizas las fórmulas y muestra el resultado.", + "Formula timer": "Temporizador de formulas" }, "DuplicateTable": { "Copy all data in addition to the table structure.": "Copiar todos los datos además de la estructura de la tabla.", From 792dc900cfd9e8d015a76cc3b8728e83bf68f861 Mon Sep 17 00:00:00 2001 From: gallegonovato Date: Tue, 4 Jun 2024 10:20:55 +0000 Subject: [PATCH 36/62] Translated using Weblate (Spanish) Currently translated at 100.0% (1334 of 1334 strings) Translation: Grist/client Translate-URL: https://hosted.weblate.org/projects/grist/client/es/ --- static/locales/es.client.json | 42 +++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/static/locales/es.client.json b/static/locales/es.client.json index 8d6b2f3258..c816385468 100644 --- a/static/locales/es.client.json +++ b/static/locales/es.client.json @@ -282,7 +282,14 @@ "python3 (recommended)": "python3 (recomendado)", "Cancel": "Cancelar", "Force reload the document while timing formulas, and show the result.": "Fuerza la recarga del documento mientras sincronizas las fórmulas y muestra el resultado.", - "Formula timer": "Temporizador de formulas" + "Formula timer": "Temporizador de formulas", + "Time reload": "Duración de la recarga", + "Timing is on": "El tiempo está activado", + "Start timing": "Iniciar cronometraje", + "Reload data engine": "Recargar el motor de datos", + "Reload data engine?": "¿Recargar motor de datos?", + "You can make changes to the document, then stop timing to see the results.": "Puede realizar cambios en el documento y luego detener el cronometraje para ver los resultados.", + "Stop timing...": "Dejando de cronometrar..." }, "DuplicateTable": { "Copy all data in addition to the table structure.": "Copiar todos los datos además de la estructura de la tabla.", @@ -478,7 +485,8 @@ "Trash": "Papelera", "Workspace will be moved to Trash.": "El espacio de trabajo se moverá a la papelera.", "Workspaces": "Espacios de trabajo", - "Tutorial": "Tutorial" + "Tutorial": "Tutorial", + "Terms of service": "Términos del servicio" }, "LeftPanelCommon": { "Help Center": "Centro de ayuda" @@ -1167,7 +1175,9 @@ "Error in style rule": "Error en la regla de estilo", "Rule must return True or False": "La regla debe regresar Verdadera o Falsa", "Add conditional style": "Añadir estilo condicional", - "Row Style": "Estilo de fila" + "Row Style": "Estilo de fila", + "IF...": "SI...", + "Conditional Style": "Estilo condicional" }, "Reference": { "SHOW COLUMN": "MOSTRAR COLUMNA", @@ -1581,7 +1591,21 @@ "OK": "De acuerdo", "Sandbox settings for data engine": "Configuración del entorno de pruebas para el motor de datos", "unconfigured": "desconfigurado", - "Learn more.": "Más información." + "Learn more.": "Más información.", + "Authentication": "Autentificación", + "Check succeeded.": "Verificación exitosa.", + "Notes": "Notas", + "Administrator Panel Unavailable": "Panel de administrador no disponible", + "Check failed.": "La verificación falló.", + "Current authentication method": "Método de autenticación actual", + "Details": "Detalles", + "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.": "Grist permite configurar diferentes tipos de autenticación, incluidos SAML y OIDC. Recomendamos habilitar uno de estos si se puede acceder a Grist a través de la red o si está disponible a varias personas.", + "No fault detected.": "No se detectó ningún error.", + "Results": "Resultados", + "Or, as a fallback, you can set: {{bootKey}} in the environment and visit: {{url}}": "O, como alternativa, puedes configurar: {{bootKey}} en el entorno y visita: {{url}}", + "You do not have access to the administrator panel.\nPlease log in as an administrator.": "No tienes acceso al panel de administrador.\nInicia sesión como administrador.", + "Self Checks": "Controles automáticos", + "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.": "Grist permite configurar diferentes tipos de autenticación, incluidos SAML y OIDC. Recomendamos habilitar uno de estos si se puede acceder a Grist a través de la red o si está disponible para varias personas." }, "CreateTeamModal": { "Cancel": "Cancelar", @@ -1646,5 +1670,15 @@ "Custom": "Personalizado", "Form": "Formulario", "Table": "Tabla" + }, + "TimingPage": { + "Average Time (s)": "Tiempo promedio (s)", + "Formula timer": "Temporizador de formulas", + "Loading timing data. Don't close this tab.": "Cargando datos del cronometraje. No cierres esta pestaña.", + "Number of Calls": "Número de llamadas", + "Table ID": "ID de la tabla", + "Total Time (s)": "Tiempo total (s)", + "Column ID": "ID de la columna", + "Max Time (s)": "Tiempo máximo (s)" } } From 5d80db5ff68e9ecfb140cccffe092f93a5c96aa1 Mon Sep 17 00:00:00 2001 From: George Gevoian <85144792+georgegevoian@users.noreply.github.com> Date: Wed, 5 Jun 2024 09:18:02 -0700 Subject: [PATCH 37/62] Add user id middleware to form pages (#1020) --- app/server/lib/FlexServer.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/app/server/lib/FlexServer.ts b/app/server/lib/FlexServer.ts index 9d3b28cc1e..46e4a50847 100644 --- a/app/server/lib/FlexServer.ts +++ b/app/server/lib/FlexServer.ts @@ -1123,6 +1123,7 @@ export class FlexServer implements GristServer { welcomeNewUser ], formMiddleware: [ + this._userIdMiddleware, forcedLoginMiddleware, ], forceLogin: this._redirectToLoginUnconditionally, From a7fbc2e40172bc2419fb6aa1727a3559f8d6cc8b Mon Sep 17 00:00:00 2001 From: Paul Fitzpatrick Date: Wed, 5 Jun 2024 14:29:44 -0400 Subject: [PATCH 38/62] include pyodide in the docker image (#1019) Grist has for some time supported a sandbox based on pyodide. It is a bit slower to start than the gvisor-based sandbox, but can run in situations where it can't. Until now it hasn't been easy to use when running Grist as a container, since the support files weren't included. This change rectifies that omission. Nothing changes by default. But now if you start Grist as a container and set `GRIST_SANDBOX_FLAVOR=pyodide`, it should work rather than fail. --- Dockerfile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Dockerfile b/Dockerfile index 20a645e76f..50e5cc6c4d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,6 +35,12 @@ COPY stubs /grist/stubs COPY buildtools /grist/buildtools RUN yarn run build:prod +# Prepare material for optional pyodide sandbox +COPY sandbox/pyodide /grist/sandbox/pyodide +COPY sandbox/requirements3.txt /grist/sandbox/requirements3.txt +RUN \ + cd /grist/sandbox/pyodide && make setup + ################################################################################ ## Python collection stage ################################################################################ @@ -108,6 +114,9 @@ ADD sandbox /grist/sandbox ADD plugins /grist/plugins ADD static /grist/static +# Make optional pyodide sandbox available +COPY --from=builder /grist/sandbox/pyodide /grist/sandbox/pyodide + # Finalize static directory RUN \ mv /grist/static-built/* /grist/static && \ From fe10c80cec88c86890368dabdc4908ba14faff46 Mon Sep 17 00:00:00 2001 From: fflorent Date: Wed, 29 May 2024 08:40:55 +0200 Subject: [PATCH 39/62] Dockerfile: use tini to reap zombie processes --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 50e5cc6c4d..f6cafa437b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,7 +79,7 @@ FROM node:18-buster-slim # Install pgrep for managing gvisor processes. RUN \ apt-get update && \ - apt-get install -y --no-install-recommends libexpat1 libsqlite3-0 procps && \ + apt-get install -y --no-install-recommends libexpat1 libsqlite3-0 procps tini && \ rm -rf /var/lib/apt/lists/* # Keep all storage user may want to persist in a distinct directory @@ -151,4 +151,5 @@ ENV \ EXPOSE 8484 +ENTRYPOINT ["/usr/bin/tini", "-s", "--"] CMD ["./sandbox/run.sh"] From 7c62a3816c7504f325e35eeb84c9658edeef05b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20Guti=C3=A9rrez=20Hermoso?= Date: Thu, 6 Jun 2024 16:14:35 -0400 Subject: [PATCH 40/62] admin: fix warning in websocket probe This is a small thing, but when visiting the admin, the websocket test doesn't send valid JSON, which the receiving endpoint expects. This results in a harmless exception being thrown. While this test should eventually be modified to be run from the frontend, for now let's just make a small fix and send valid JSON in order to avoid that JSON parsing exception. --- app/server/lib/BootProbes.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/server/lib/BootProbes.ts b/app/server/lib/BootProbes.ts index df9427f231..61ac66ebcc 100644 --- a/app/server/lib/BootProbes.ts +++ b/app/server/lib/BootProbes.ts @@ -118,7 +118,7 @@ const _webSocketsProbe: Probe = { url, }; ws.on('open', () => { - ws.send('Just nod if you can hear me.'); + ws.send('{"msg": "Just nod if you can hear me."}'); resolve({ status: 'success', details, From 117717d0967c58d8687959ed548fe8cfbf493416 Mon Sep 17 00:00:00 2001 From: Spoffy <4805393+Spoffy@users.noreply.github.com> Date: Fri, 7 Jun 2024 20:07:25 +0100 Subject: [PATCH 41/62] Attempts to make DropdownConditionEditor tests less flaky (#1026) Attempts to fix "DropdownConditionEditor in choice columns creates dropdown conditions", and adds comments to inform future investigators. --- test/nbrowser/DropdownConditionEditor.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/nbrowser/DropdownConditionEditor.ts b/test/nbrowser/DropdownConditionEditor.ts index 9dff24b85f..1ad75b0f9a 100644 --- a/test/nbrowser/DropdownConditionEditor.ts +++ b/test/nbrowser/DropdownConditionEditor.ts @@ -66,8 +66,13 @@ describe('DropdownConditionEditor', function () { 'user.A\nc\ncess\n ', ]); }); - await gu.sendKeysSlowly(['hoice not in $']); + await gu.sendKeysSlowly(['hoice not in ']); + // Attempts to reduce test flakiness by delaying input of $. Not guaranteed to do anything. + await driver.sleep(100); + await gu.sendKeys('$'); await gu.waitToPass(async () => { + // This test is sometimes flaky here. It will consistently return the wrong value, usually an array of + // empty strings. The running theory is it's an issue in Ace editor. const completions = await driver.findAll('.ace_autocomplete .ace_line', el => el.getText()); assert.deepEqual(completions, [ '$\nName\n ', From 07bf8ef002fb42f77fec7c114f380f639191e19a Mon Sep 17 00:00:00 2001 From: Spoffy <4805393+Spoffy@users.noreply.github.com> Date: Mon, 10 Jun 2024 21:39:57 +0100 Subject: [PATCH 42/62] Fixes flaky ViewLayoutCollapse test (#1027) This fixes the flaky test in "ViewLayoutCollapse.ts": "fix: should not dispose the instance when drag is cancelled". The 'mouseenter' event wasn't consistently triggering properly on the drop target (LayoutEditor.ts - line 342) when the mouse was moved onto it. The change simulates a "drag" over the drop target, moving the mouse into multiple positions over it, seemingly fixing the problem. --- test/nbrowser/ViewLayoutCollapse.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/nbrowser/ViewLayoutCollapse.ts b/test/nbrowser/ViewLayoutCollapse.ts index 88f10ad4d6..9086e92135 100644 --- a/test/nbrowser/ViewLayoutCollapse.ts +++ b/test/nbrowser/ViewLayoutCollapse.ts @@ -311,7 +311,9 @@ describe("ViewLayoutCollapse", function() { // Move back and drop. await gu.getSection(COMPANIES_CHART).getRect(); - await move(getDragElement(COMPANIES_CHART)); + await move(getDragElement(COMPANIES_CHART), {x : 50}); + await driver.sleep(100); + await move(getDragElement(COMPANIES_CHART), {x : 100}); await driver.sleep(100); await move(getDragElement(COMPANIES_CHART), {x : 200}); await gu.waitToPass(async () => { From 0a3978c6d456be582efb8bb27ad2f3867405dbed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaros=C5=82aw=20Sadzi=C5=84ski?= Date: Wed, 12 Jun 2024 11:33:13 +0200 Subject: [PATCH 43/62] (core) Renaming installationId metadata for checkUpdateAPI telemetry endpoint. Summary: CheckUpdateAPI is now storing client's installation id in a new field called 'deploymentId'. Previously it was using installationId which is reserved (and overriden) by the home server. Test Plan: Existing and manual Reviewers: paulfitz Reviewed By: paulfitz Subscribers: paulfitz Differential Revision: https://phab.getgrist.com/D4268 --- app/common/Telemetry.ts | 2 +- app/server/lib/Telemetry.ts | 11 +++++++++++ app/server/lib/UpdateManager.ts | 6 +++--- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/app/common/Telemetry.ts b/app/common/Telemetry.ts index 328a3c7ebc..aab28a27c3 100644 --- a/app/common/Telemetry.ts +++ b/app/common/Telemetry.ts @@ -1746,7 +1746,7 @@ export const TelemetryContracts: TelemetryContracts = { minimumTelemetryLevel: Level.limited, retentionPeriod: 'indefinitely', metadataContracts: { - installationId: { + deploymentId: { description: 'The installation id of the client.', dataType: 'string', }, diff --git a/app/server/lib/Telemetry.ts b/app/server/lib/Telemetry.ts index edef19126a..6d34160046 100644 --- a/app/server/lib/Telemetry.ts +++ b/app/server/lib/Telemetry.ts @@ -334,6 +334,17 @@ export class Telemetry implements ITelemetry { try { this._numPendingForwardEventRequests += 1; const {category: eventCategory} = TelemetryContracts[event]; + + if (metadata) { + if ('installationId' in metadata || + 'eventSource' in metadata || + 'eventName' in metadata || + 'eventCategory' in metadata) + { + throw new Error('metadata contains reserved keys'); + } + } + await this._doForwardEvent(JSON.stringify({ event, metadata: { diff --git a/app/server/lib/UpdateManager.ts b/app/server/lib/UpdateManager.ts index 47556b74ce..c7ac9f6765 100644 --- a/app/server/lib/UpdateManager.ts +++ b/app/server/lib/UpdateManager.ts @@ -86,7 +86,7 @@ export class UpdateManager { // This is the most interesting part for us, to track installation ids and match them // with the version of the client. Won't be send without telemetry opt in. - const installationId = optStringParam( + const deploymentId = optStringParam( payload("installationId"), "installationId" ); @@ -104,8 +104,8 @@ export class UpdateManager { .getTelemetry() .logEvent(req as RequestWithLogin, "checkedUpdateAPI", { full: { - installationId, - deploymentType, + deploymentId, + deploymentType }, }); From 2b49dee8bda8290c1fed0cc5e0d30d5046905c91 Mon Sep 17 00:00:00 2001 From: Roman Holinec <3ko@pixeon.sk> Date: Mon, 10 Jun 2024 14:00:41 +0000 Subject: [PATCH 44/62] Translated using Weblate (Slovak) Currently translated at 13.7% (184 of 1334 strings) Translation: Grist/client Translate-URL: https://hosted.weblate.org/projects/grist/client/sk/ --- static/locales/sk.client.json | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/static/locales/sk.client.json b/static/locales/sk.client.json index 898c96cfe1..0ee86e76da 100644 --- a/static/locales/sk.client.json +++ b/static/locales/sk.client.json @@ -185,7 +185,9 @@ "Widget does not require any permissions.": "Widget nevyžaduje žiadne povolenia.", "Widget needs to {{read}} the current table.": "Widget vyžaduje {{read}} aktuálnu tabuľku.", "Widget needs {{fullAccess}} to this document.": "Widget vyžaduje {{fullAccess}} k tomuto dokumentu.", - "No document access": "Bez prístupu k dokumentu" + "No document access": "Bez prístupu k dokumentu", + "Clear selection": "Vyčistiť výber", + "No {{columnType}} columns in table.": "V tabuľke nie sú žiadne stĺpce {{columnType}}." }, "AppModel": { "This team site is suspended. Documents can be read, but not modified.": "Táto tímová stránka je pozastavená. Dokumenty je možné čítať, ale nie upravovať." @@ -198,5 +200,25 @@ "Apply": "Použiť", "Cancel": "Zrušiť", "Default cell style": "Predvolený štýl bunky" + }, + "DataTables": { + "Delete {{formattedTableName}} data, and remove it from all pages?": "Odstrániť údaje {{formattedTableName}} a odobrať ich zo všetkých stránok?", + "Duplicate Table": "Duplikovať Tabuľku", + "Raw Data Tables": "Tbuľky s nespracovanými údajmi", + "Table ID copied to clipboard": "ID tabuľky bolo skopírované do schránky", + "Record Card": "Karta Záznamu", + "Edit Record Card": "Úprava Karty Záznamu", + "Click to copy": "Kliknutím skopírovať", + "You do not have edit access to this document": "Nemáte prístup k úprave tohto dokumentu", + "Record Card Disabled": "Zakázaná Karta Záznamu", + "Rename Table": "Premenovať tabuľku", + "{{action}} Record Card": "{{action}} Kartu Záznamu", + "Remove Table": "Odstrániť tabuľku" + }, + "DocHistory": { + "Activity": "Aktivita", + "Beta": "Beta", + "Compare to Previous": "Porovnať s Predchádzajúcim", + "Compare to Current": "Porovnať s Aktuálnym" } } From ce9b1a8a8f0ad208f561f92799b9d9671c6296be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20Guti=C3=A9rrez=20Hermoso?= Date: Tue, 11 Jun 2024 22:09:40 -0400 Subject: [PATCH 45/62] v1.1.15 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index efe831db5f..9cd1e9c3bb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "grist-core", - "version": "1.1.14", + "version": "1.1.15", "license": "Apache-2.0", "description": "Grist is the evolution of spreadsheets", "homepage": "https://github.com/gristlabs/grist-core", From 224dbdf644c47355dfb51547ba37e8556be111f4 Mon Sep 17 00:00:00 2001 From: Paul Fitzpatrick Date: Wed, 12 Jun 2024 09:34:31 -0400 Subject: [PATCH 46/62] make the example key on admin panel without auth work when insecure (#1024) The example key shown on the admin panel to users who are not known to be administrators is generated using a method that is only available in secure environments. This adds a fallback for insecure environments. The key is less solid but again, it is just an example, and for an insecure environment. Tested manually running locally and using a hostname set in /etc/hosts. --- app/client/ui/AdminPanel.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/app/client/ui/AdminPanel.ts b/app/client/ui/AdminPanel.ts index 37d4f4e205..bfd2bd6c95 100644 --- a/app/client/ui/AdminPanel.ts +++ b/app/client/ui/AdminPanel.ts @@ -98,7 +98,7 @@ export class AdminPanel extends Disposable { * which could include a legit adminstrator if auth is misconfigured. */ private _buildMainContentForOthers(owner: MultiHolder) { - const exampleKey = 'example-' + window.crypto.randomUUID(); + const exampleKey = _longCodeForExample(); return dom.create(AdminSection, t('Administrator Panel Unavailable'), [ dom('p', t(`You do not have access to the administrator panel. Please log in as an administrator.`)), @@ -649,3 +649,19 @@ export const cssLabel = styled('div', ` text-align: right; padding-right: 5px; `); + + +/** + * Make a long code to use in the example, so that if people copy + * and paste it lazily, they end up decently secure, or at least a + * lot more secure than a key like "REPLACE_WITH_YOUR_SECRET" + */ +function _longCodeForExample() { + // Crypto in insecure contexts doesn't have randomUUID + if (window.isSecureContext) { + return 'example-a' + window.crypto.randomUUID(); + } + return 'example-b' + 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'.replace(/x/g, () => { + return Math.floor(Math.random() * 16).toString(16); + }); +} From f7bd26583763321e5a705859de86dd27d747e43c Mon Sep 17 00:00:00 2001 From: Spoffy Date: Thu, 13 Jun 2024 19:36:05 +0100 Subject: [PATCH 47/62] (core) Makes EE frontend behave as core if EE isn't activated Summary: - Makes EE decide which ActivationPage to use - Makes ProductUpgrades use core implementation if not activated - Changes banners to proxy to core implementation if EE not activated - [Fix] Enables new site creation in EE as in Core: - Core enables people to freely create new team sites. - Enterprise currently redirects to the pricing page. - This enables enterprise to also create team sites, instead of redirecting. Test Plan: Manually test in EE, unit tests in Jenkins Reviewers: paulfitz, jordigh Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D4264 --- app/client/components/CoreBanners.ts | 10 ++++++++++ app/client/ui/AppUI.ts | 2 +- app/client/ui/CreateTeamModal.ts | 8 ++++---- app/client/ui/DefaultActivationPage.ts | 18 ++++++++++++++++++ stubs/app/client/components/Banners.ts | 10 +--------- stubs/app/client/ui/ActivationPage.ts | 15 +++++---------- 6 files changed, 39 insertions(+), 24 deletions(-) create mode 100644 app/client/components/CoreBanners.ts create mode 100644 app/client/ui/DefaultActivationPage.ts diff --git a/app/client/components/CoreBanners.ts b/app/client/components/CoreBanners.ts new file mode 100644 index 0000000000..7df6398a7f --- /dev/null +++ b/app/client/components/CoreBanners.ts @@ -0,0 +1,10 @@ +import {AppModel} from 'app/client/models/AppModel'; +import {DocPageModel} from 'app/client/models/DocPageModel'; + +export function buildHomeBanners(_app: AppModel) { + return null; +} + +export function buildDocumentBanners(_docPageModel: DocPageModel) { + return null; +} diff --git a/app/client/ui/AppUI.ts b/app/client/ui/AppUI.ts index 10e7b8406b..23ff48d3c2 100644 --- a/app/client/ui/AppUI.ts +++ b/app/client/ui/AppUI.ts @@ -82,7 +82,7 @@ function createMainPage(appModel: AppModel, appObj: App) { } else if (pageType === 'admin') { return domAsync(loadAdminPanel().then(m => dom.create(m.AdminPanel, appModel))); } else if (pageType === 'activation') { - return domAsync(loadActivationPage().then(ap => dom.create(ap.ActivationPage, appModel))); + return domAsync(loadActivationPage().then(ap => dom.create(ap.getActivationPage(), appModel))); } else { return dom.create(pagePanelsDoc, appModel, appObj); } diff --git a/app/client/ui/CreateTeamModal.ts b/app/client/ui/CreateTeamModal.ts index 438fc31c26..4c633e9896 100644 --- a/app/client/ui/CreateTeamModal.ts +++ b/app/client/ui/CreateTeamModal.ts @@ -24,10 +24,10 @@ export async function buildNewSiteModal(context: Disposable, options: { appModel: AppModel, plan?: PlanSelection, onCreate?: () => void -}) { +}): Promise { const { onCreate } = options; - return showModal( + showModal( context, (_owner: Disposable, ctrl: IModalControl) => dom.create(NewSiteModalContent, ctrl, onCreate), dom.cls(cssModalIndex.className), @@ -87,12 +87,12 @@ export function buildUpgradeModal(owner: Disposable, options: { throw new UserError(t(`Billing is not supported in grist-core`)); } -export interface UpgradeButton { +export interface IUpgradeButton { showUpgradeCard(...args: DomArg[]): DomContents; showUpgradeButton(...args: DomArg[]): DomContents; } -export function buildUpgradeButton(owner: IDisposableOwner, app: AppModel): UpgradeButton { +export function buildUpgradeButton(owner: IDisposableOwner, app: AppModel): IUpgradeButton { return { showUpgradeCard: () => null, showUpgradeButton: () => null, diff --git a/app/client/ui/DefaultActivationPage.ts b/app/client/ui/DefaultActivationPage.ts new file mode 100644 index 0000000000..859db9e85b --- /dev/null +++ b/app/client/ui/DefaultActivationPage.ts @@ -0,0 +1,18 @@ +import {AppModel} from 'app/client/models/AppModel'; +import { Disposable, IDomCreator } from 'grainjs'; + +export type IActivationPageCreator = IDomCreator<[AppModel]> + +/** + * A blank ActivationPage stand-in, as it's possible for the frontend to try and load an "activation page", + * even though there's no activation in core. + */ +export class DefaultActivationPage extends Disposable { + constructor(_appModel: AppModel) { + super(); + } + + public buildDom() { + return null; + } +} diff --git a/stubs/app/client/components/Banners.ts b/stubs/app/client/components/Banners.ts index 7df6398a7f..b0aa190bcb 100644 --- a/stubs/app/client/components/Banners.ts +++ b/stubs/app/client/components/Banners.ts @@ -1,10 +1,2 @@ -import {AppModel} from 'app/client/models/AppModel'; -import {DocPageModel} from 'app/client/models/DocPageModel'; +export { buildHomeBanners, buildDocumentBanners } from 'app/client/components/CoreBanners'; -export function buildHomeBanners(_app: AppModel) { - return null; -} - -export function buildDocumentBanners(_docPageModel: DocPageModel) { - return null; -} diff --git a/stubs/app/client/ui/ActivationPage.ts b/stubs/app/client/ui/ActivationPage.ts index aa2ce08a57..99b90e5717 100644 --- a/stubs/app/client/ui/ActivationPage.ts +++ b/stubs/app/client/ui/ActivationPage.ts @@ -1,12 +1,7 @@ -import {AppModel} from 'app/client/models/AppModel'; -import {Disposable} from 'grainjs'; +import { + DefaultActivationPage, IActivationPageCreator +} from "app/client/ui/DefaultActivationPage"; -export class ActivationPage extends Disposable { - constructor(_appModel: AppModel) { - super(); - } - - public buildDom() { - return null; - } +export function getActivationPage(): IActivationPageCreator { + return DefaultActivationPage; } From 76c2218045199ddcbefa75681081cac3e0511d2f Mon Sep 17 00:00:00 2001 From: Dmitry S Date: Wed, 12 Jun 2024 17:47:25 -0400 Subject: [PATCH 48/62] (core) Update documentation of certain functions Summary: - lookupOne/lookupRecords explain `sort_by` param better, and link to more detailed article. - Incorporate a typo fix from Help Center - Fix the omission of TASTEME never having been documented. Test Plan: Corresponding update to Help Center can be reviewed at https://github.com/gristlabs/grist-help/pull/351 Reviewers: jarek Reviewed By: jarek Subscribers: jarek Differential Revision: https://phab.getgrist.com/D4269 --- sandbox/grist/functions/text.py | 18 +++++++++++++++--- sandbox/grist/table.py | 19 ++++++++++++++++--- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/sandbox/grist/functions/text.py b/sandbox/grist/functions/text.py index 3d9a7c0d5a..0fe8c4ca56 100644 --- a/sandbox/grist/functions/text.py +++ b/sandbox/grist/functions/text.py @@ -647,6 +647,18 @@ def T(value): six.text_type(value) if isinstance(value, AltText) else u"") def TASTEME(food): + """ + For any given piece of text, decides if it is tasty or not. + + This is not serious. It appeared as an Easter egg, and is kept as such. It is in fact a puzzle + to figure out the underlying simple rule. It has been surprisingly rarely cracked, even after + reading the source code, which is freely available and may entertain Python fans. + + >>> TASTEME('Banana') + True + >>> TASTEME('Garlic') + False + """ chews = re.findall(r'\b[A-Z]+\b', food.upper()) claw = slice(2, None) spit = lambda chow: chow[claw] @@ -657,9 +669,9 @@ def TASTEME(food): @unimplemented def TEXT(number, format_type): # pylint: disable=unused-argument """ - Converts a number into text according to a specified format. It is not yet implemented in + Converts a number into text according to a specified format. It is not yet implemented in Grist. You can use the similar Python functions str() to convert numbers into strings, and - optionally format() to specify the number format. + optionally format() to specify the number format. """ raise NotImplementedError() @@ -681,7 +693,7 @@ def TRIM(text): def UPPER(text): """ - Converts a specified string to uppercase. Same as `text.lower()`. + Converts a specified string to uppercase. Same as `text.upper()`. >>> UPPER("e. e. cummings") 'E. E. CUMMINGS' diff --git a/sandbox/grist/table.py b/sandbox/grist/table.py index 9b976403f1..b582ab9d47 100644 --- a/sandbox/grist/table.py +++ b/sandbox/grist/table.py @@ -68,13 +68,17 @@ def lookupRecords(self, **field_value_pairs): any expression, most commonly a field in the current row (e.g. `$SomeField`) or a constant (e.g. a quoted string like `"Some Value"`) (examples below). - If `sort_by=field` is given, sort the results by that field. + + You may set the optional `sort_by` parameter to the column ID by which to sort multiple matching + results, to determine which of them is returned. You can prefix the column ID with "-" to + reverse the order. For example: ``` People.lookupRecords(Email=$Work_Email) People.lookupRecords(First_Name="George", Last_Name="Washington") People.lookupRecords(Last_Name="Johnson", sort_by="First_Name") + Orders.lookupRecords(Customer=$id, sort_by="-OrderDate") ``` See [RecordSet](#recordset) for useful properties offered by the returned object. @@ -82,6 +86,8 @@ def lookupRecords(self, **field_value_pairs): See [CONTAINS](#contains) for an example utilizing `UserTable.lookupRecords` to find records where a field of a list type (such as `Choice List` or `Reference List`) contains the given value. + + Learn more about [lookupRecords](references-lookups.md#lookuprecords). """ return self.table.lookup_records(**field_value_pairs) @@ -92,14 +98,21 @@ def lookupOne(self, **field_value_pairs): Returns a [Record](#record) matching the given field=value arguments. The value may be any expression, most commonly a field in the current row (e.g. `$SomeField`) or a constant (e.g. a quoted string - like `"Some Value"`). If multiple records match, returns one of them. If none match, returns the - special empty record. + like `"Some Value"`). If multiple records are found, the first match is returned. + + You may set the optional `sort_by` parameter to the column ID by which to sort multiple matching + results, to determine which of them is returned. You can prefix the column ID with "-" to + reverse the order. For example: ``` People.lookupOne(First_Name="Lewis", Last_Name="Carroll") People.lookupOne(Email=$Work_Email) + Tickets.lookupOne(Person=$id, sort_by="Date") # Find the first ticket for the person + Tickets.lookupOne(Person=$id, sort_by="-Date") # Find the last ticket for the person ``` + + Learn more about [lookupOne](references-lookups.md#lookupone). """ return self.table.lookup_one_record(**field_value_pairs) From 4ebdae62f456a96415afe4f9d9b8b93545662f0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaros=C5=82aw=20Sadzi=C5=84ski?= Date: Fri, 14 Jun 2024 12:12:24 +0200 Subject: [PATCH 49/62] (core) Restoring GRIST_DEFAULT_PRODUCT functionality Summary: The GRIST_DEFAULT_PRODUCT wasn't used for grist-ee, now it is respected. Test Plan: I've build grist-ee docker image from github and run it using our instruction (both for recreating the issue and confirming it is fixed) ``` docker run -p 8484:8484 \ -v $PWD:/persist \ -e GRIST_SESSION_SECRET=invent-a-secret-here \ -e GRIST_SINGLE_ORG=cool-beans -it gristlabs/grist-ee ``` For grist-core I recreated/confirmed it is fixed it just by `GRIST_SINGLE_ORG=team npm start` in the core folder. I also created some team sites using stubbed UI and confirmed that they were using the GRIST_DEFAULT_PRODUCT product. Reviewers: paulfitz Reviewed By: paulfitz Subscribers: paulfitz Differential Revision: https://phab.getgrist.com/D4271 --- app/client/ui/CreateTeamModal.ts | 4 +++- app/gen-server/entity/Product.ts | 5 ++--- stubs/app/server/server.ts | 2 -- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/app/client/ui/CreateTeamModal.ts b/app/client/ui/CreateTeamModal.ts index 4c633e9896..4dccfc2eb3 100644 --- a/app/client/ui/CreateTeamModal.ts +++ b/app/client/ui/CreateTeamModal.ts @@ -136,7 +136,9 @@ function buildTeamPage({ } await create(); } finally { - disabled.set(false); + if (!disabled.isDisposed()) { + disabled.set(false); + } } } const clickOnEnter = dom.onKeyPress({ diff --git a/app/gen-server/entity/Product.ts b/app/gen-server/entity/Product.ts index 0ecde6daa8..42ce65261c 100644 --- a/app/gen-server/entity/Product.ts +++ b/app/gen-server/entity/Product.ts @@ -147,12 +147,11 @@ export const PRODUCTS: IProduct[] = [ */ export function getDefaultProductNames() { const defaultProduct = process.env.GRIST_DEFAULT_PRODUCT; - const personalFreePlan = PERSONAL_FREE_PLAN; return { // Personal site start off on a functional plan. - personal: defaultProduct || personalFreePlan, + personal: defaultProduct || PERSONAL_FREE_PLAN, // Team site starts off on a limited plan, requiring subscription. - teamInitial: defaultProduct || 'stub', + teamInitial: defaultProduct || STUB_PLAN, // Team site that has been 'turned off'. teamCancel: 'suspended', // Functional team site. diff --git a/stubs/app/server/server.ts b/stubs/app/server/server.ts index cfb13172aa..20ab2d269f 100644 --- a/stubs/app/server/server.ts +++ b/stubs/app/server/server.ts @@ -7,7 +7,6 @@ import {commonUrls} from 'app/common/gristUrls'; import {isAffirmative} from 'app/common/gutil'; import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager'; -import {TEAM_FREE_PLAN} from 'app/common/Features'; const debugging = isAffirmative(process.env.DEBUG) || isAffirmative(process.env.VERBOSE); @@ -90,7 +89,6 @@ async function setupDb() { }, { setUserAsOwner: false, useNewPlan: true, - product: TEAM_FREE_PLAN })); } } From 8864643b276195f3e31ca5a6a78707abb83a927c Mon Sep 17 00:00:00 2001 From: Leslie H <142967379+SleepyLeslie@users.noreply.github.com> Date: Thu, 13 Jun 2024 22:26:23 +0000 Subject: [PATCH 50/62] Update README to rebrand grist-electron (#1039) --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1c9c46d8a7..fad5aa62b5 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,12 @@ Grist is a modern relational spreadsheet. It combines the flexibility of a spreadsheet with the robustness of a database. * `grist-core` (this repo) has what you need to run a powerful spreadsheet hosting server. -* [`grist-electron`](https://github.com/gristlabs/grist-electron) is a Linux/macOS/Windows desktop app for viewing and editing spreadsheets stored locally. +* [`grist-desktop`](https://github.com/gristlabs/grist-desktop) is a Linux/macOS/Windows desktop app for viewing and editing spreadsheets stored locally. * [`grist-static`](https://github.com/gristlabs/grist-static) is a fully in-browser build of Grist for displaying spreadsheets on a website without back-end support. The `grist-core` repo is the heart of Grist, including the hosted services offered by [Grist Labs](https://getgrist.com), an NYC-based company 🇺🇸 and Grist's main developer. The French government agency [ANCT Données et Territoires](https://donnees.incubateur.anct.gouv.fr/toolbox/grist) 🇫🇷 has also made significant contributions to the codebase. -The `grist-core`, `grist-electron`, and `grist-static` repositories are all open source (Apache License, Version 2.0). +The `grist-core`, `grist-desktop`, and `grist-static` repositories are all open source (Apache License, Version 2.0). > Questions? Feedback? Want to share what you're building with Grist? Join our [official Discord server](https://discord.gg/MYKpYQ3fbP) or visit our [Community forum](https://community.getgrist.com/). @@ -35,7 +35,7 @@ Here are some specific feature highlights of Grist: - Enables [backups](https://support.getgrist.com/exports/#backing-up-an-entire-document) that you can confidently restore in full. - Great for moving between different hosts. * Can be displayed on a static website with [`grist-static`](https://github.com/gristlabs/grist-static) – no special server needed. - * A self-contained desktop app for viewing and editing locally: [`grist-electron`](https://github.com/gristlabs/grist-electron). + * A self-contained desktop app for viewing and editing locally: [`grist-desktop`](https://github.com/gristlabs/grist-desktop). * Convenient editing and formatting features. - Choices and [choice lists](https://support.getgrist.com/col-types/#choice-list-columns), for adding colorful tags to records. - [References](https://support.getgrist.com/col-refs/#creating-a-new-reference-list-column) and reference lists, for cross-referencing records in other tables. @@ -81,7 +81,7 @@ If you just want a quick demo of Grist: * You can try Grist out at the hosted service run by Grist Labs at [docs.getgrist.com](https://docs.getgrist.com) (no registration needed). * Or you can see a fully in-browser build of Grist at [gristlabs.github.io/grist-static](https://gristlabs.github.io/grist-static/). - * Or you can download Grist as a desktop app from [github.com/gristlabs/grist-electron](https://github.com/gristlabs/grist-electron). + * Or you can download Grist as a desktop app from [github.com/gristlabs/grist-desktop](https://github.com/gristlabs/grist-desktop). To get `grist-core` running on your computer with [Docker](https://www.docker.com/get-started), do: From 8468fa566d9d8b37d7e4a2c54cfd381b85cea535 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaros=C5=82aw=20Sadzi=C5=84ski?= Date: Tue, 18 Jun 2024 12:12:20 +0200 Subject: [PATCH 51/62] (core) Adding fixSiteProducts that changes orgs from teamFree to Free product if it was set be default Summary: After release on 2024-06-12 (1.1.15) the GRIST_DEFAULT_PRODUCT env variable wasn't respected by the method that started the server in single org mode. In all deployments (apart from saas), the default product used for new sites is set to `Free`, but the code that starts the server enforced `teamFree` product. This change adds a fix routine that fixes this issue by rewriting team sites from `teamFree` product to `Free` product only if: - The default product is set to `Free` - The deployment type is something other then 'saas'. Additionally there is a test that will fail after 2024.10.01, as this fix should be removed before this date. Test Plan: Added test Reviewers: paulfitz Reviewed By: paulfitz Subscribers: paulfitz Differential Revision: https://phab.getgrist.com/D4272 --- app/gen-server/lib/Housekeeper.ts | 78 ++++++++++++++++ stubs/app/server/server.ts | 8 ++ test/server/fixSiteProducts.ts | 146 ++++++++++++++++++++++++++++++ 3 files changed, 232 insertions(+) create mode 100644 test/server/fixSiteProducts.ts diff --git a/app/gen-server/lib/Housekeeper.ts b/app/gen-server/lib/Housekeeper.ts index f3412d0316..14c68ecf9a 100644 --- a/app/gen-server/lib/Housekeeper.ts +++ b/app/gen-server/lib/Housekeeper.ts @@ -1,8 +1,10 @@ import { ApiError } from 'app/common/ApiError'; import { delay } from 'app/common/delay'; import { buildUrlId } from 'app/common/gristUrls'; +import { BillingAccount } from 'app/gen-server/entity/BillingAccount'; import { Document } from 'app/gen-server/entity/Document'; import { Organization } from 'app/gen-server/entity/Organization'; +import { Product } from 'app/gen-server/entity/Product'; import { Workspace } from 'app/gen-server/entity/Workspace'; import { HomeDBManager, Scope } from 'app/gen-server/lib/HomeDBManager'; import { fromNow } from 'app/gen-server/sqlUtils'; @@ -462,3 +464,79 @@ async function forEachWithBreaks(logText: string, items: T[], callback: (item } log.rawInfo(logText, {itemsProcesssed, itemsTotal, timeMs: Date.now() - start}); } + +/** + * For a brief moment file `stubs/app/server/server.ts` was ignoring the GRIST_DEFAULT_PRODUCT + * variable, which is currently set for all deployment types to 'Free' product. As a result orgs + * created after 2024-06-12 (1.1.15) were created with 'teamFree' product instead of 'Free'. + * It only affected deployments that were using: + * - GRIST_DEFAULT_PRODUCT variable set to 'Free' + * - GRIST_SINGLE_ORG set to enforce single org mode. + * + * This method fixes the product for all orgs created with 'teamFree' product, if the default + * product that should be used is 'Free' and the deployment type is not 'saas' ('saas' deployment + * isn't using GRIST_DEFAULT_PRODUCT variable). This method should be removed after 2024.10.01. + * + * There is a corresponding test that will fail if this method (and that test) are not removed. + * + * @returns true if the method was run, false otherwise. + */ +export async function fixSiteProducts(options: { + deploymentType: string, + db: HomeDBManager, + dry?: boolean, +}) { + const {deploymentType, dry, db} = options; + + const hasDefaultProduct = () => Boolean(process.env.GRIST_DEFAULT_PRODUCT); + const defaultProductIsFree = () => process.env.GRIST_DEFAULT_PRODUCT === 'Free'; + const notSaasDeployment = () => deploymentType !== 'saas'; + const mustRun = hasDefaultProduct() && defaultProductIsFree() && notSaasDeployment(); + if (!mustRun) { + return false; + } + const removeMeDate = new Date('2024-10-01'); + const warningMessage = `WARNING: This method should be removed after ${removeMeDate.toDateString()}.`; + if (new Date() > removeMeDate) { + console.warn(warningMessage); + } + + // Find all billing accounts on teamFree product and change them to the Free. + + return await db.connection.transaction(async (t) => { + const freeProduct = await t.findOne(Product, {where: {name: 'Free'}}); + + const freeTeamProduct = await t.findOne(Product, {where: {name: 'teamFree'}}); + + if (!freeTeamProduct) { + console.warn('teamFree product not found.'); + return false; + } + + if (!freeProduct) { + console.warn('Free product not found.'); + return false; + } + + if (dry) { + await t.createQueryBuilder() + .select('ba') + .from(BillingAccount, 'ba') + .where('ba.product = :productId', {productId: freeTeamProduct.id}) + .getMany() + .then((accounts) => { + accounts.forEach(a => { + console.log(`Would change account ${a.id} from ${a.product.id} to ${freeProduct.id}`); + }); + }); + } else { + await t.createQueryBuilder() + .update(BillingAccount) + .set({product: freeProduct.id}) + .where({product: freeTeamProduct.id}) + .execute(); + } + + return true; + }); +} diff --git a/stubs/app/server/server.ts b/stubs/app/server/server.ts index 20ab2d269f..b6f272a92c 100644 --- a/stubs/app/server/server.ts +++ b/stubs/app/server/server.ts @@ -7,6 +7,7 @@ import {commonUrls} from 'app/common/gristUrls'; import {isAffirmative} from 'app/common/gutil'; import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager'; +import {fixSiteProducts} from 'app/gen-server/lib/Housekeeper'; const debugging = isAffirmative(process.env.DEBUG) || isAffirmative(process.env.VERBOSE); @@ -135,6 +136,13 @@ export async function main() { if (process.env.GRIST_SERVE_PLUGINS_PORT) { await server.startCopy('pluginServer', parseInt(process.env.GRIST_SERVE_PLUGINS_PORT, 10)); } + + await fixSiteProducts({ + deploymentType: server.getDeploymentType(), + db: server.getHomeDBManager(), + dry: true + }); + return server; } diff --git a/test/server/fixSiteProducts.ts b/test/server/fixSiteProducts.ts new file mode 100644 index 0000000000..b42d5a016b --- /dev/null +++ b/test/server/fixSiteProducts.ts @@ -0,0 +1,146 @@ +import {Organization} from 'app/gen-server/entity/Organization'; +import {fixSiteProducts} from 'app/gen-server/lib/Housekeeper'; +import {TestServer} from 'test/gen-server/apiUtils'; +import * as testUtils from 'test/server/testUtils'; +import {assert} from 'chai'; +import sinon from "sinon"; +import {getDefaultProductNames} from 'app/gen-server/entity/Product'; + +const email = 'chimpy@getgrist.com'; +const profile = {email, name: email}; +const org = 'single-org'; + +describe('fixSiteProducts', function() { + this.timeout(6000); + + let oldEnv: testUtils.EnvironmentSnapshot; + let server: TestServer; + + before(async function() { + oldEnv = new testUtils.EnvironmentSnapshot(); + // By default we will simulate 'core' deployment that has 'Free' team site as default product. + process.env.GRIST_TEST_SERVER_DEPLOYMENT_TYPE = 'core'; + process.env.GRIST_DEFAULT_PRODUCT = 'Free'; + server = new TestServer(this); + await server.start(); + }); + + after(async function() { + oldEnv.restore(); + await server.stop(); + }); + + it('fix should be deleted after 2024-10-01', async function() { + const now = new Date(); + const remove_date = new Date('2024-10-01'); + assert.isTrue(now < remove_date, 'This test and a fix method should be deleted after 2024-10-01'); + }); + + it('fixes sites that where created with a wrong product', async function() { + const db = server.dbManager; + const user = await db.getUserByLogin(email, {profile}) as any; + const getOrg = (id: number) => db.connection.manager.findOne( + Organization, + {where: {id}, relations: ['billingAccount', 'billingAccount.product']}); + + const productOrg = (id: number) => getOrg(id)?.then(org => org?.billingAccount?.product?.name); + + const freeOrgId = db.unwrapQueryResult(await db.addOrg(user, { + name: org, + domain: org, + }, { + setUserAsOwner: false, + useNewPlan: true, + product: 'teamFree', + })); + + const teamOrgId = db.unwrapQueryResult(await db.addOrg(user, { + name: 'fix-team-org', + domain: 'fix-team-org', + }, { + setUserAsOwner: false, + useNewPlan: true, + product: 'team', + })); + + // Make sure it is created with teamFree product. + assert.equal(await productOrg(freeOrgId), 'teamFree'); + + // Run the fixer. + assert.isTrue(await fixSiteProducts({ + db, + deploymentType: server.server.getDeploymentType(), + })); + + // Make sure we fixed the product is on Free product. + assert.equal(await productOrg(freeOrgId), 'Free'); + + // Make sure the other org is still on team product. + assert.equal(await productOrg(teamOrgId), 'team'); + }); + + it("doesn't run when on saas deployment", async function() { + process.env.GRIST_TEST_SERVER_DEPLOYMENT_TYPE = 'saas'; + + // Stub it in the server. Notice that we assume some knowledge about how the server is implemented - that it won't + // cache this value (nor any other component) and always read it when needed. Otherwise we would need to recreate + // the server each time. + const sandbox = sinon.createSandbox(); + sandbox.stub(server.server, 'getDeploymentType').returns('saas'); + assert.equal(server.server.getDeploymentType(), 'saas'); + + assert.isFalse(await fixSiteProducts({ + db: server.dbManager, + deploymentType: server.server.getDeploymentType(), + })); + + sandbox.restore(); + }); + + it("doesn't run when default product is not set", async function() { + // Make sure we are in 'core'. + assert.equal(server.server.getDeploymentType(), 'core'); + + // But only when Free product is the default one. + process.env.GRIST_DEFAULT_PRODUCT = 'teamFree'; + assert.equal(getDefaultProductNames().teamInitial, 'teamFree'); // sanity check that Grist sees it. + + assert.isFalse(await fixSiteProducts({ + db: server.dbManager, + deploymentType: server.server.getDeploymentType(), + })); + + process.env.GRIST_DEFAULT_PRODUCT = 'team'; + assert.equal(getDefaultProductNames().teamInitial, 'team'); + + assert.isFalse(await fixSiteProducts({ + db: server.dbManager, + deploymentType: server.server.getDeploymentType(), + })); + + delete process.env.GRIST_DEFAULT_PRODUCT; + assert.equal(getDefaultProductNames().teamInitial, 'stub'); + + const db = server.dbManager; + const user = await db.getUserByLogin(email, {profile}) as any; + const orgId = db.unwrapQueryResult(await db.addOrg(user, { + name: 'sanity-check-org', + domain: 'sanity-check-org', + }, { + setUserAsOwner: false, + useNewPlan: true, + product: 'teamFree', + })); + + const getOrg = (id: number) => db.connection.manager.findOne(Organization, + {where: {id}, relations: ['billingAccount', 'billingAccount.product']}); + const productOrg = (id: number) => getOrg(id)?.then(org => org?.billingAccount?.product?.name); + assert.equal(await productOrg(orgId), 'teamFree'); + + assert.isFalse(await fixSiteProducts({ + db: server.dbManager, + deploymentType: server.server.getDeploymentType(), + })); + assert.equal(await productOrg(orgId), 'teamFree'); + }); +}); From 822aefc81e83be800c9ba139c9978b8a8af8f0b4 Mon Sep 17 00:00:00 2001 From: Dmitry S Date: Tue, 18 Jun 2024 09:56:11 -0400 Subject: [PATCH 52/62] (core) Disable formula timing UI for non-owners Summary: For non-owners, the timing section of Document Settings is now disabled. For non-editors, the "Reload" section is disabled. Test Plan: Added a test case for timing being disabled. Reviewers: jarek Reviewed By: jarek Differential Revision: https://phab.getgrist.com/D4275 --- app/client/ui/AdminPanelCss.ts | 19 +++++++++++++++++-- app/client/ui/DocumentSettings.ts | 5 +++++ test/nbrowser/Timing.ts | 24 ++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/app/client/ui/AdminPanelCss.ts b/app/client/ui/AdminPanelCss.ts index 9fb8b15fe8..2853a3fa36 100644 --- a/app/client/ui/AdminPanelCss.ts +++ b/app/client/ui/AdminPanelCss.ts @@ -1,3 +1,4 @@ +import {hoverTooltip} from 'app/client/ui/tooltips'; import {transition} from 'app/client/ui/transitions'; import {toggle} from 'app/client/ui2018/checkbox'; import {mediaSmall, testId, theme, vars} from 'app/client/ui2018/cssVars'; @@ -21,6 +22,7 @@ export function AdminSectionItem(owner: IDisposableOwner, options: { description?: DomContents, value?: DomContents, expandedContent?: DomContents, + disabled?: false|string, }) { const itemContent = (...prefix: DomContents[]) => [ cssItemName( @@ -34,7 +36,7 @@ export function AdminSectionItem(owner: IDisposableOwner, options: { testId(`admin-panel-item-value-${options.id}`), dom.on('click', ev => ev.stopPropagation())), ]; - if (options.expandedContent) { + if (options.expandedContent && !options.disabled) { const isCollapsed = Observable.create(owner, true); return cssItem( cssItemShort( @@ -56,7 +58,13 @@ export function AdminSectionItem(owner: IDisposableOwner, options: { ); } else { return cssItem( - cssItemShort(itemContent()), + cssItemShort(itemContent(), + cssItemShort.cls('-disabled', Boolean(options.disabled)), + options.disabled ? hoverTooltip(options.disabled, { + placement: 'bottom-end', + modifiers: {offset: {offset: '0, -10'}}, + }) : null, + ), testId(`admin-panel-item-${options.id}`), ); } @@ -109,6 +117,9 @@ const cssItemShort = styled('div', ` &-expandable:hover { background-color: ${theme.lightHover}; } + &-disabled { + opacity: .5; + } @container line (max-width: 500px) { & { @@ -157,6 +168,10 @@ const cssItemValue = styled('div', ` margin: -16px; padding: 16px; cursor: auto; + + .${cssItemShort.className}-disabled & { + pointer-events: none; + } `); const cssCollapseIcon = styled(icon, ` diff --git a/app/client/ui/DocumentSettings.ts b/app/client/ui/DocumentSettings.ts index 615292e71d..b2c9ba287b 100644 --- a/app/client/ui/DocumentSettings.ts +++ b/app/client/ui/DocumentSettings.ts @@ -28,6 +28,7 @@ import {EngineCode} from 'app/common/DocumentSettings'; import {commonUrls, GristLoadConfig} from 'app/common/gristUrls'; import {not, propertyCompare} from 'app/common/gutil'; import {getCurrency, locales} from 'app/common/Locales'; +import {isOwner, isOwnerOrEditor} from 'app/common/roles'; import {Computed, Disposable, dom, fromKo, IDisposableOwner, makeTestId, Observable, styled} from 'grainjs'; import * as moment from 'moment-timezone'; @@ -58,6 +59,8 @@ export class DocSettingsPage extends Disposable { const canChangeEngine = getSupportedEngineChoices().length > 0; const docPageModel = this._gristDoc.docPageModel; const isTimingOn = this._gristDoc.isTimingOn; + const isDocOwner = isOwner(docPageModel.currentDoc.get()); + const isDocEditor = isOwnerOrEditor(docPageModel.currentDoc.get()); return cssContainer( dom.create(AdminSection, t('Document Settings'), [ @@ -115,6 +118,7 @@ export class DocSettingsPage extends Disposable { 'This allows diagnosing which formulas are responsible for slow performance when a ' + 'document is first opened, or when a document responds to changes.' )), + disabled: isDocOwner ? false : t('Only available to document owners'), }), dom.create(AdminSectionItem, { @@ -122,6 +126,7 @@ export class DocSettingsPage extends Disposable { name: t('Reload'), description: t('Hard reset of data engine'), value: cssSmallButton(t('Reload data engine'), dom.on('click', this._reloadEngine.bind(this, true))), + disabled: isDocEditor ? false : t('Only available to document editors'), }), canChangeEngine ? dom.create(AdminSectionItem, { diff --git a/test/nbrowser/Timing.ts b/test/nbrowser/Timing.ts index b21e130438..9659cdf4d2 100644 --- a/test/nbrowser/Timing.ts +++ b/test/nbrowser/Timing.ts @@ -166,6 +166,30 @@ describe("Timing", function () { await driver.findWait('.test-raw-data-list', 2000); assert.deepEqual(await driver.findAll('.test-raw-data-table-id', e => e.getText()), ['Table1']); }); + + it('should be disabled for non-owners', async function() { + await userApi.updateDocPermissions(docId, {users: { + [gu.translateUser('user2').email]: 'editors', + }}); + + const session = await gu.session().teamSite.user('user2').login(); + await session.loadDoc(`/doc/${docId}`); + await gu.openDocumentSettings(); + + const start = driver.find('.test-settings-timing-start'); + assert.equal(await start.isPresent(), true); + + // Check that we have an informative tooltip. + await start.mouseMove(); + assert.match(await driver.findWait('.test-tooltip', 2000).getText(), /Only available to document owners/); + + // Nothing should happen on click. We click the location rather than the element, since the + // element isn't actually clickable. + await start.mouseMove(); + await driver.withActions(a => a.press().release()); + await driver.sleep(100); + assert.equal(await driver.find(".test-settings-timing-modal").isPresent(), false); + }); }); const element = (testId: string) => ({ From 54e0d7e64d247bd9882ed9459c2e97fc61d0490d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaros=C5=82aw=20Sadzi=C5=84ski?= Date: Tue, 18 Jun 2024 18:29:06 +0200 Subject: [PATCH 53/62] (core) Removing dry option from fixSiteProducts Summary: fixSiteProducts was always called with a dry option. This option was just added for debuging test failure, it should have been removed. Test Plan: Manual. - on grist core, prepare site with `teamFree` product - then to recreate run the previous version as `GRIST_SINGLE_ORG=cool-beans GRIST_DEFAULT_PRODUCT=Free npm start` - then to confirm it is fixed, run the same command as above Site should be changed from `teamFree` to `Free`. Reviewers: paulfitz Reviewed By: paulfitz Subscribers: paulfitz Differential Revision: https://phab.getgrist.com/D4276 --- app/gen-server/lib/Housekeeper.ts | 30 +++++++----------------------- stubs/app/server/server.ts | 3 +-- 2 files changed, 8 insertions(+), 25 deletions(-) diff --git a/app/gen-server/lib/Housekeeper.ts b/app/gen-server/lib/Housekeeper.ts index 14c68ecf9a..116a3c508c 100644 --- a/app/gen-server/lib/Housekeeper.ts +++ b/app/gen-server/lib/Housekeeper.ts @@ -483,10 +483,9 @@ async function forEachWithBreaks(logText: string, items: T[], callback: (item */ export async function fixSiteProducts(options: { deploymentType: string, - db: HomeDBManager, - dry?: boolean, + db: HomeDBManager }) { - const {deploymentType, dry, db} = options; + const {deploymentType, db} = options; const hasDefaultProduct = () => Boolean(process.env.GRIST_DEFAULT_PRODUCT); const defaultProductIsFree = () => process.env.GRIST_DEFAULT_PRODUCT === 'Free'; @@ -502,10 +501,8 @@ export async function fixSiteProducts(options: { } // Find all billing accounts on teamFree product and change them to the Free. - return await db.connection.transaction(async (t) => { const freeProduct = await t.findOne(Product, {where: {name: 'Free'}}); - const freeTeamProduct = await t.findOne(Product, {where: {name: 'teamFree'}}); if (!freeTeamProduct) { @@ -518,24 +515,11 @@ export async function fixSiteProducts(options: { return false; } - if (dry) { - await t.createQueryBuilder() - .select('ba') - .from(BillingAccount, 'ba') - .where('ba.product = :productId', {productId: freeTeamProduct.id}) - .getMany() - .then((accounts) => { - accounts.forEach(a => { - console.log(`Would change account ${a.id} from ${a.product.id} to ${freeProduct.id}`); - }); - }); - } else { - await t.createQueryBuilder() - .update(BillingAccount) - .set({product: freeProduct.id}) - .where({product: freeTeamProduct.id}) - .execute(); - } + await t.createQueryBuilder() + .update(BillingAccount) + .set({product: freeProduct.id}) + .where({product: freeTeamProduct.id}) + .execute(); return true; }); diff --git a/stubs/app/server/server.ts b/stubs/app/server/server.ts index b6f272a92c..e8761cef15 100644 --- a/stubs/app/server/server.ts +++ b/stubs/app/server/server.ts @@ -139,8 +139,7 @@ export async function main() { await fixSiteProducts({ deploymentType: server.getDeploymentType(), - db: server.getHomeDBManager(), - dry: true + db: server.getHomeDBManager() }); return server; From bd27669042e5506ace49abfd1ae66715ad436490 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 18 Jun 2024 10:54:52 -0400 Subject: [PATCH 54/62] automated update to translation keys (#1053) Co-authored-by: Paul's Grist Bot --- static/locales/en.client.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/static/locales/en.client.json b/static/locales/en.client.json index 3ba12ffcc9..7a8ee8df46 100644 --- a/static/locales/en.client.json +++ b/static/locales/en.client.json @@ -339,7 +339,9 @@ "Stop timing...": "Stop timing...", "Time reload": "Time reload", "Timing is on": "Timing is on", - "You can make changes to the document, then stop timing to see the results.": "You can make changes to the document, then stop timing to see the results." + "You can make changes to the document, then stop timing to see the results.": "You can make changes to the document, then stop timing to see the results.", + "Only available to document editors": "Only available to document editors", + "Only available to document owners": "Only available to document owners" }, "DocumentUsage": { "Attachments Size": "Size of Attachments", From 883b8e951a55a11c1dc7ee01b99a25748ecbe9de Mon Sep 17 00:00:00 2001 From: Florent Date: Tue, 18 Jun 2024 16:57:06 +0200 Subject: [PATCH 55/62] HomeDBManager refactoration: extract method related to Users management in its own module (#1049) The HomeDBManager remains the exposed class to the other parts of the code: any module under gen-server/lib/homedb like UsersManager is intended to be used solely by HomeDBManager, and in order to use their methods, an indirection has to be created to pass through HomeDBManager. --- app/common/UserAPI.ts | 3 + app/gen-server/lib/HomeDBManager.ts | 893 ++++------------------ app/gen-server/lib/homedb/Interfaces.ts | 40 + app/gen-server/lib/homedb/UsersManager.ts | 761 ++++++++++++++++++ 4 files changed, 945 insertions(+), 752 deletions(-) create mode 100644 app/gen-server/lib/homedb/Interfaces.ts create mode 100644 app/gen-server/lib/homedb/UsersManager.ts diff --git a/app/common/UserAPI.ts b/app/common/UserAPI.ts index c10486adc7..9c6e824ac5 100644 --- a/app/common/UserAPI.ts +++ b/app/common/UserAPI.ts @@ -35,6 +35,9 @@ export const ANONYMOUS_USER_EMAIL = 'anon@getgrist.com'; // Nominal email address of a user who, if you share with them, everyone gets access. export const EVERYONE_EMAIL = 'everyone@getgrist.com'; +// Nominal email address of a user who can view anything (for thumbnails). +export const PREVIEWER_EMAIL = 'thumbnail@getgrist.com'; + // A special 'docId' that means to create a new document. export const NEW_DOCUMENT_CODE = 'new'; diff --git a/app/gen-server/lib/HomeDBManager.ts b/app/gen-server/lib/HomeDBManager.ts index 44a454720f..06409d1c8b 100644 --- a/app/gen-server/lib/HomeDBManager.ts +++ b/app/gen-server/lib/HomeDBManager.ts @@ -4,11 +4,10 @@ import {mapGetOrSet, mapSetOrClear, MapWithTTL} from 'app/common/AsyncCreate'; import {getDataLimitStatus} from 'app/common/DocLimits'; import {createEmptyOrgUsageSummary, DocumentUsage, OrgUsageSummary} from 'app/common/DocUsage'; import {normalizeEmail} from 'app/common/emails'; -import {ANONYMOUS_PLAN, canAddOrgMembers, Features, PERSONAL_FREE_PLAN} from 'app/common/Features'; +import {ANONYMOUS_PLAN, canAddOrgMembers, Features} from 'app/common/Features'; import {buildUrlId, MIN_URLID_PREFIX_LENGTH, parseUrlId} from 'app/common/gristUrls'; -import {FullUser, UserProfile} from 'app/common/LoginSessionAPI'; +import {UserProfile} from 'app/common/LoginSessionAPI'; import {checkSubdomainValidity} from 'app/common/orgNameUtils'; -import {UserOrgPrefs} from 'app/common/Prefs'; import * as roles from 'app/common/roles'; import {StringUnion} from 'app/common/StringUnion'; import { @@ -22,9 +21,10 @@ import { Organization as OrgInfo, PermissionData, PermissionDelta, + PREVIEWER_EMAIL, UserAccessData, UserOptions, - WorkspaceProperties + WorkspaceProperties, } from "app/common/UserAPI"; import {AclRule, AclRuleDoc, AclRuleOrg, AclRuleWs} from "app/gen-server/entity/AclRule"; import {Alias} from "app/gen-server/entity/Alias"; @@ -32,7 +32,6 @@ import {BillingAccount} from "app/gen-server/entity/BillingAccount"; import {BillingAccountManager} from "app/gen-server/entity/BillingAccountManager"; import {Document} from "app/gen-server/entity/Document"; import {Group} from "app/gen-server/entity/Group"; -import {Login} from "app/gen-server/entity/Login"; import {AccessOption, AccessOptionWithRole, Organization} from "app/gen-server/entity/Organization"; import {Pref} from "app/gen-server/entity/Pref"; import {getDefaultProductNames, personalFreeFeatures, Product} from "app/gen-server/entity/Product"; @@ -41,6 +40,10 @@ import {Share} from "app/gen-server/entity/Share"; import {User} from "app/gen-server/entity/User"; import {Workspace} from "app/gen-server/entity/Workspace"; import {Limit} from 'app/gen-server/entity/Limit'; +import { + AvailableUsers, GetUserOptions, NonGuestGroup, Resource, UserProfileChange +} from 'app/gen-server/lib/homedb/Interfaces'; +import {SUPPORT_EMAIL, UsersManager} from 'app/gen-server/lib/homedb/UsersManager'; import {Permissions} from 'app/gen-server/lib/Permissions'; import {scrubUserFromOrg} from "app/gen-server/lib/scrubUserFromOrg"; import {applyPatch} from 'app/gen-server/lib/TypeORMPatches'; @@ -59,21 +62,20 @@ import log from 'app/server/lib/log'; import {Permit} from 'app/server/lib/Permit'; import {getScope} from 'app/server/lib/requestUtils'; import {WebHookSecret} from "app/server/lib/Triggers"; + import {EventEmitter} from 'events'; import {Request} from "express"; -import moment from 'moment-timezone'; +import {defaultsDeep, flatten, pick} from 'lodash'; import { Brackets, Connection, DatabaseType, EntityManager, + ObjectLiteral, SelectQueryBuilder, WhereExpression } from "typeorm"; import uuidv4 from "uuid/v4"; -import flatten = require('lodash/flatten'); -import pick = require('lodash/pick'); -import defaultsDeep = require('lodash/defaultsDeep'); // Support transactions in Sqlite in async code. This is a monkey patch, affecting // the prototypes of various TypeORM classes. @@ -81,6 +83,7 @@ import defaultsDeep = require('lodash/defaultsDeep'); // fixed. See https://github.com/typeorm/typeorm/issues/1884#issuecomment-380767213 applyPatch(); +export { SUPPORT_EMAIL }; export const NotifierEvents = StringUnion( 'addUser', 'userChange', @@ -94,18 +97,6 @@ export const NotifierEvents = StringUnion( export type NotifierEvent = typeof NotifierEvents.type; -// Nominal email address of a user who can view anything (for thumbnails). -export const PREVIEWER_EMAIL = 'thumbnail@getgrist.com'; - -// A special user allowed to add/remove the EVERYONE_EMAIL to/from a resource. -export const SUPPORT_EMAIL = appSettings.section('access').flag('supportEmail').requireString({ - envVar: 'GRIST_SUPPORT_EMAIL', - defaultValue: 'support@getgrist.com', -}); - -// A list of emails we don't expect to see logins for. -const NON_LOGIN_EMAILS = [PREVIEWER_EMAIL, EVERYONE_EMAIL, ANONYMOUS_USER_EMAIL]; - // Name of a special workspace with examples in it. export const EXAMPLE_WORKSPACE_NAME = 'Examples & Templates'; @@ -120,8 +111,6 @@ const listPublicSites = appSettings.section('access').flag('listPublicSites').re // which is a burden under heavy traffic. const DOC_AUTH_CACHE_TTL = 5000; -type Resource = Organization|Workspace|Document; - export interface QueryResult { status: number; data?: T; @@ -172,16 +161,6 @@ export interface UserChange { membersAfter: Map; } -// A specification of the users available during a request. This can be a single -// user, identified by a user id, or a collection of profiles (typically drawn from -// the session). -type AvailableUsers = number | UserProfile[]; - -// A type guard to check for single-user case. -function isSingleUser(users: AvailableUsers): users is number { - return typeof users === 'number'; -} - // The context in which a query is being made. Includes what we know // about the user, and for requests made from pages, the active organization. export interface Scope { @@ -205,18 +184,6 @@ export interface DocScope extends Scope { urlId: string; } -type NonGuestGroup = Group & { name: roles.NonGuestRole }; - -// Returns whether the given group is a valid non-guest group. -function isNonGuestGroup(group: Group): group is NonGuestGroup { - return roles.isNonGuestRole(group.name); -} - -export interface UserProfileChange { - name?: string; - isFirstTimeUser?: boolean; -} - // Identifies a request to access a document. This combination of values is also used for caching // DocAuthResult for DOC_AUTH_CACHE_TTL. Other request scope information is passed along. export interface DocAuthKey { @@ -236,12 +203,6 @@ export interface DocAuthResult { cachedDoc?: Document; // For cases where stale info is ok. } -interface GetUserOptions { - manager?: EntityManager; - profile?: UserProfile; - userOptions?: UserOptions; -} - // Represent a DocAuthKey as a string. The format is ": ". // flushSingleDocAuthCache() depends on this format. function stringifyDocAuthKey(key: DocAuthKey): string { @@ -285,9 +246,9 @@ export type BillingOptions = Partial { - await this._getSpecialUserId({ - email: ANONYMOUS_USER_EMAIL, - name: "Anonymous" - }); - await this._getSpecialUserId({ - email: PREVIEWER_EMAIL, - name: "Preview" - }); - await this._getSpecialUserId({ - email: EVERYONE_EMAIL, - name: "Everyone" - }); - await this._getSpecialUserId({ - email: SUPPORT_EMAIL, - name: "Support" - }); + }) { + await this._usersManager.initializeSpecialIds(); if (!options?.skipWorkspaces) { // Find the example workspace. If there isn't one named just right, take the first workspace @@ -421,7 +366,7 @@ export class HomeDBManager extends EventEmitter { // anonymous users. const supportWorkspaces = await this._workspaces() .leftJoinAndSelect('workspaces.org', 'orgs') - .where('orgs.owner_id = :userId', { userId: this.getSupportUserId() }) + .where('orgs.owner_id = :userId', { userId: this._usersManager.getSupportUserId() }) .orderBy('workspaces.created_at') .getMany(); const exampleWorkspace = supportWorkspaces.find(ws => ws.name === EXAMPLE_WORKSPACE_NAME) || supportWorkspaces[0]; @@ -472,330 +417,78 @@ export class HomeDBManager extends EventEmitter { } /** - * Clear all user preferences associated with the given email addresses. * For use in tests. + * @see UsersManager.prototype.testClearUserPrefs */ public async testClearUserPrefs(emails: string[]) { - return await this._connection.transaction(async manager => { - for (const email of emails) { - const user = await this.getUserByLogin(email, {manager}); - if (user) { - await manager.delete(Pref, {userId: user.id}); - } - } - }); + return this._usersManager.testClearUserPrefs(emails); } public async getUserByKey(apiKey: string): Promise { - // Include logins relation for Authorization convenience. - return await User.findOne({where: {apiKey}, relations: ["logins"]}) || undefined; + return this._usersManager.getUserByKey(apiKey); } public async getUserByRef(ref: string): Promise { - return await User.findOne({where: {ref}, relations: ["logins"]}) || undefined; + return this._usersManager.getUserByRef(ref); } - public async getUser( - userId: number, - options: {includePrefs?: boolean} = {} - ): Promise { - const {includePrefs} = options; - const relations = ["logins"]; - if (includePrefs) { relations.push("prefs"); } - return await User.findOne({where: {id: userId}, relations}) || undefined; + public async getUser(userId: number, options: {includePrefs?: boolean} = {}) { + return this._usersManager.getUser(userId, options); } - public async getFullUser(userId: number): Promise { - const user = await User.findOne({where: {id: userId}, relations: ["logins"]}); - if (!user) { throw new ApiError("unable to find user", 400); } - return this.makeFullUser(user); + public async getFullUser(userId: number) { + return this._usersManager.getFullUser(userId); } /** - * Convert a user record into the format specified in api. + * @see UsersManager.prototype.makeFullUser */ - public makeFullUser(user: User): FullUser { - if (!user.logins?.[0]?.displayEmail) { - throw new ApiError("unable to find mandatory user email", 400); - } - const displayEmail = user.logins[0].displayEmail; - const loginEmail = user.loginEmail; - const result: FullUser = { - id: user.id, - email: displayEmail, - // Only include loginEmail when it's different, to avoid overhead when FullUser is sent - // around, and also to avoid updating too many tests. - loginEmail: loginEmail !== displayEmail ? loginEmail : undefined, - name: user.name, - picture: user.picture, - ref: user.ref, - locale: user.options?.locale, - prefs: user.prefs?.find((p)=> p.orgId === null)?.prefs, - }; - if (this.getAnonymousUserId() === user.id) { - result.anonymous = true; - } - if (this.getSupportUserId() === user.id) { - result.isSupport = true; - } - return result; + public makeFullUser(user: User) { + return this._usersManager.makeFullUser(user); } /** - * Ensures that user with external id exists and updates its profile and email if necessary. - * - * @param profile External profile + * @see UsersManager.prototype.ensureExternalUser */ public async ensureExternalUser(profile: UserProfile) { - await this._connection.transaction(async manager => { - // First find user by the connectId from the profile - const existing = await manager.findOne(User, { - where: {connectId: profile.connectId || undefined}, - relations: ["logins"], - }); - - // If a user does not exist, create it with data from the external profile. - if (!existing) { - const newUser = await this.getUserByLoginWithRetry(profile.email, { - profile, - manager - }); - if (!newUser) { - throw new ApiError("Unable to create user", 500); - } - // No need to survey this user. - newUser.isFirstTimeUser = false; - await newUser.save(); - } else { - // Else update profile and login information from external profile. - let updated = false; - let login: Login = existing.logins[0]!; - const properEmail = normalizeEmail(profile.email); - - if (properEmail !== existing.loginEmail) { - login = login ?? new Login(); - login.email = properEmail; - login.displayEmail = profile.email; - existing.logins.splice(0, 1, login); - login.user = existing; - updated = true; - } - - if (profile?.name && profile?.name !== existing.name) { - existing.name = profile.name; - updated = true; - } - - if (profile?.picture && profile?.picture !== existing.picture) { - existing.picture = profile.picture; - updated = true; - } - - if (updated) { - await manager.save([existing, login]); - } - } - }); + return this._usersManager.ensureExternalUser(profile); } - public async updateUser(userId: number, props: UserProfileChange): Promise { - let isWelcomed: boolean = false; - let user: User|null = null; - await this._connection.transaction(async manager => { - user = await manager.findOne(User, {relations: ['logins'], - where: {id: userId}}); - let needsSave = false; - if (!user) { throw new ApiError("unable to find user", 400); } - if (props.name && props.name !== user.name) { - user.name = props.name; - needsSave = true; - } - if (props.isFirstTimeUser !== undefined && props.isFirstTimeUser !== user.isFirstTimeUser) { - user.isFirstTimeUser = props.isFirstTimeUser; - needsSave = true; - // If we are turning off the isFirstTimeUser flag, then right - // after this transaction commits is a great time to trigger - // any automation for first logins - if (!props.isFirstTimeUser) { isWelcomed = true; } - } - if (needsSave) { - await user.save(); - } - }); + public async updateUser(userId: number, props: UserProfileChange) { + const { user, isWelcomed } = await this._usersManager.updateUser(userId, props); if (user && isWelcomed) { this.emit('firstLogin', this.makeFullUser(user)); } } public async updateUserName(userId: number, name: string) { - const user = await User.findOne({where: {id: userId}}); - if (!user) { throw new ApiError("unable to find user", 400); } - user.name = name; - await user.save(); + return this._usersManager.updateUserName(userId, name); } public async updateUserOptions(userId: number, props: Partial) { - const user = await User.findOne({where: {id: userId}}); - if (!user) { throw new ApiError("unable to find user", 400); } - - const newOptions = {...(user.options ?? {}), ...props}; - user.options = newOptions; - await user.save(); + return this._usersManager.updateUserOptions(userId, props); } - // Fetch user from login, creating the user if previously unseen, allowing one retry - // for an email key conflict failure. This is in case our transaction conflicts with a peer - // doing the same thing. This is quite likely if the first page visited by a previously - // unseen user fires off multiple api calls. + /** + * @see UsersManager.prototype.getUserByLoginWithRetry + */ public async getUserByLoginWithRetry(email: string, options: GetUserOptions = {}): Promise { - try { - return await this.getUserByLogin(email, options); - } catch (e) { - if (e.name === 'QueryFailedError' && e.detail && - e.detail.match(/Key \(email\)=[^ ]+ already exists/)) { - // This is a postgres-specific error message. This problem cannot arise in sqlite, - // because we have to serialize sqlite transactions in any case to get around a typeorm - // limitation. - return await this.getUserByLogin(email, options); - } - throw e; - } + return this._usersManager.getUserByLoginWithRetry(email, options); } /** - * - * Fetches a user record based on an email address. If a user record already - * exists linked to the email address supplied, that is the record returned. - * Otherwise a fresh record is created, linked to the supplied email address. - * The supplied `options` are used when creating a fresh record, or updating - * unset/outdated fields of an existing record. - * + * @see UsersManager.prototype.getUserByLogin */ public async getUserByLogin(email: string, options: GetUserOptions = {}): Promise { - const {manager: transaction, profile, userOptions} = options; - const normalizedEmail = normalizeEmail(email); - const userByLogin = await this._runInTransaction(transaction, async manager => { - let needUpdate = false; - const userQuery = manager.createQueryBuilder() - .select('user') - .from(User, 'user') - .leftJoinAndSelect('user.logins', 'logins') - .leftJoinAndSelect('user.personalOrg', 'personalOrg') - .where('email = :email', {email: normalizedEmail}); - let user = await userQuery.getOne(); - let login: Login; - if (!user) { - user = new User(); - // Special users do not have first time user set so that they don't get redirected to the - // welcome page. - user.isFirstTimeUser = !NON_LOGIN_EMAILS.includes(normalizedEmail); - login = new Login(); - login.email = normalizedEmail; - login.user = user; - needUpdate = true; - } else { - login = user.logins[0]; - } - - // Check that user and login records are up to date. - if (!user.name) { - // Set the user's name if our provider knows it. Otherwise use their username - // from email, for lack of something better. If we don't have a profile at this - // time, then leave the name blank in the hopes of learning it when the user logs in. - user.name = (profile && (profile.name || email.split('@')[0])) || ''; - needUpdate = true; - } - if (profile && !user.firstLoginAt) { - // set first login time to now (remove milliseconds for compatibility with other - // timestamps in db set by typeorm, and since second level precision is fine) - const nowish = new Date(); - nowish.setMilliseconds(0); - user.firstLoginAt = nowish; - needUpdate = true; - } - if (!user.picture && profile && profile.picture) { - // Set the user's profile picture if our provider knows it. - user.picture = profile.picture; - needUpdate = true; - } - if (profile && profile.email && profile.email !== login.displayEmail) { - // Use provider's version of email address for display. - login.displayEmail = profile.email; - needUpdate = true; - } - - if (profile?.connectId && profile?.connectId !== user.connectId) { - user.connectId = profile.connectId; - needUpdate = true; - } - - if (!login.displayEmail) { - // Save some kind of display email if we don't have anything at all for it yet. - // This could be coming from how someone wrote it in a UserManager dialog, for - // instance. It will get overwritten when the user logs in if the provider's - // version is different. - login.displayEmail = email; - needUpdate = true; - } - if (!user.options?.authSubject && userOptions?.authSubject) { - // Link subject from password-based authentication provider if not previously linked. - user.options = {...(user.options ?? {}), authSubject: userOptions.authSubject}; - needUpdate = true; - } - const today = moment().startOf('day'); - if (today !== moment(user.lastConnectionAt).startOf('day')) { - user.lastConnectionAt = today.toDate(); - needUpdate = true; - } - if (needUpdate) { - login.user = user; - await manager.save([user, login]); - } - if (!user.personalOrg && !NON_LOGIN_EMAILS.includes(login.email)) { - // Add a personal organization for this user. - // We don't add a personal org for anonymous/everyone/previewer "users" as it could - // get a bit confusing. - const result = await this.addOrg(user, {name: "Personal"}, { - setUserAsOwner: true, - useNewPlan: true, - product: PERSONAL_FREE_PLAN, - }, manager); - if (result.status !== 200) { - throw new Error(result.errMessage); - } - needUpdate = true; - - // We just created a personal org; set userOrgPrefs that should apply for new users only. - const userOrgPrefs: UserOrgPrefs = {showGristTour: true}; - const orgId = result.data; - if (orgId) { - await this.updateOrg({userId: user.id}, orgId, {userOrgPrefs}, manager); - } - } - if (needUpdate) { - // We changed the db - reload user in order to give consistent results. - // In principle this could be optimized, but this is simpler to maintain. - user = await userQuery.getOne(); - } - return user; - }); - return userByLogin; + return this._usersManager.getUserByLogin(email, options); } /** + * @see UsersManager.prototype.getExistingUserByLogin * Find a user by email. Don't create the user if it doesn't already exist. */ - public async getExistingUserByLogin( - email: string, - manager?: EntityManager - ): Promise { - const normalizedEmail = normalizeEmail(email); - return await (manager || this._connection).createQueryBuilder() - .select('user') - .from(User, 'user') - .leftJoinAndSelect('user.logins', 'logins') - .where('email = :email', {email: normalizedEmail}) - .getOne() || undefined; + public async getExistingUserByLogin(email: string, manager?: EntityManager): Promise { + return this._usersManager.getExistingUserByLogin(email, manager); } /** @@ -827,50 +520,16 @@ export class HomeDBManager extends EventEmitter { public async getOrgBillableMemberCount(org: string|number|Organization): Promise { return (await this._getOrgMembers(org)) .filter(u => !u.options?.isConsultant) // remove consultants. - .filter(u => !this.getExcludedUserIds().includes(u.id)) // remove support user and other + .filter(u => !this._usersManager.getExcludedUserIds().includes(u.id)) // remove support user and other .length; } /** - * Deletes a user from the database. For the moment, the only person with the right - * to delete a user is the user themselves. - * Users have logins, a personal org, and entries in the group_users table. All are - * removed together in a transaction. All material in the personal org will be lost. - * - * @param scope: request scope, including the id of the user initiating this action - * @param userIdToDelete: the id of the user to delete from the database - * @param name: optional cross-check, delete only if user name matches this + * @see UsersManager.prototype.deleteUser */ public async deleteUser(scope: Scope, userIdToDelete: number, name?: string): Promise> { - const userIdDeleting = scope.userId; - if (userIdDeleting !== userIdToDelete) { - throw new ApiError('not permitted to delete this user', 403); - } - await this._connection.transaction(async manager => { - const user = await manager.findOne(User, {where: {id: userIdToDelete}, - relations: ["logins", "personalOrg", "prefs"]}); - if (!user) { throw new ApiError('user not found', 404); } - if (name) { - if (user.name !== name) { - throw new ApiError(`user name did not match ('${name}' vs '${user.name}')`, 400); - } - } - if (user.personalOrg) { await this.deleteOrg(scope, user.personalOrg.id, manager); } - await manager.remove([...user.logins]); - // We don't have a GroupUser entity, and adding one tickles lots of TypeOrm quirkiness, - // so use a plain query to delete entries in the group_users table. - await manager.createQueryBuilder() - .delete() - .from('group_users') - .where('user_id = :userId', {userId: userIdToDelete}) - .execute(); - - await manager.delete(User, userIdToDelete); - }); - return { - status: 200 - }; + return this._usersManager.deleteUser(scope, userIdToDelete, name); } /** @@ -884,14 +543,14 @@ export class HomeDBManager extends EventEmitter { // Anonymous access to the merged org is a special case. We return an // empty organization, not backed by the database, and which can contain // nothing but the example documents always added to the merged org. - if (this.isMergedOrg(orgKey) && userId === this.getAnonymousUserId()) { + if (this.isMergedOrg(orgKey) && userId === this._usersManager.getAnonymousUserId()) { const anonOrg: OrgInfo = { id: 0, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), domain: this.mergedOrgDomain(), name: 'Anonymous', - owner: this.makeFullUser(this.getAnonymousUser()), + owner: this.makeFullUser(this._usersManager.getAnonymousUser()), access: 'viewers', billingAccount: { id: 0, @@ -916,7 +575,7 @@ export class HomeDBManager extends EventEmitter { qb = this._addBillingAccount(qb, scope.userId); let effectiveUserId = scope.userId; if (scope.specialPermit && scope.specialPermit.org === orgKey) { - effectiveUserId = this.getPreviewerUserId(); + effectiveUserId = this._usersManager.getPreviewerUserId(); } qb = this._withAccess(qb, effectiveUserId, 'orgs'); qb = qb.leftJoinAndSelect('orgs.owner', 'owner'); @@ -960,7 +619,7 @@ export class HomeDBManager extends EventEmitter { includeOrgsAndManagers: boolean, transaction?: EntityManager): Promise { const org = this.unwrapQueryResult(await this.getOrg(scope, orgKey, transaction)); - if (!org.billingAccount.isManager && scope.userId !== this.getPreviewerUserId() && + if (!org.billingAccount.isManager && scope.userId !== this._usersManager.getPreviewerUserId() && // The special permit (used for the support user) allows access to the billing account. scope.specialPermit?.org !== orgKey) { throw new ApiError('User does not have access to billing account', 401); @@ -1023,7 +682,7 @@ export class HomeDBManager extends EventEmitter { const query = this._orgWorkspaces(scope, orgKey, options); // Allow an empty result for the merged org for the anonymous user. The anonymous user // has no home org or workspace. For all other sitations, expect at least one workspace. - const emptyAllowed = this.isMergedOrg(orgKey) && scope.userId === this.getAnonymousUserId(); + const emptyAllowed = this.isMergedOrg(orgKey) && scope.userId === this._usersManager.getAnonymousUserId(); const result = await this._verifyAclPermissions(query, { scope, emptyAllowed }); // Return the workspaces, not the org(s). if (result.status === 200) { @@ -1146,7 +805,6 @@ export class HomeDBManager extends EventEmitter { return options.find(option => option.access === role) || null; } - /** * Returns a SelectQueryBuilder which gives an array of orgs already filtered by * the given user' (or users') access. @@ -1159,7 +817,7 @@ export class HomeDBManager extends EventEmitter { options?: {ignoreEveryoneShares?: boolean}): Promise> { let queryBuilder = this._orgs() .leftJoinAndSelect('orgs.owner', 'users', 'orgs.owner_id = users.id'); - if (isSingleUser(users)) { + if (UsersManager.isSingleUser(users)) { // When querying with a single user in mind, we keep our api promise // of returning their personal org first in the list. queryBuilder = queryBuilder @@ -1172,7 +830,7 @@ export class HomeDBManager extends EventEmitter { queryBuilder = this._withAccess(queryBuilder, users, 'orgs'); // Add a direct, efficient filter to remove irrelevant personal orgs from consideration. queryBuilder = this._filterByOrgGroups(queryBuilder, users, domain, options); - if (this._isAnonymousUser(users) && !listPublicSites) { + if (this._usersManager.isAnonymousUser(users) && !listPublicSites) { // The anonymous user is a special case. It may have access to potentially // many orgs, but listing them all would be kind of a misfeature. but reporting // nothing would complicate the client. We compromise, and report at most @@ -1233,7 +891,7 @@ export class HomeDBManager extends EventEmitter { // TODO: look up the document properly, perhaps delegating // to the regular path through this method. workspace: this.unwrapQueryResult( - await this.getWorkspace({userId: this.getSupportUserId()}, + await this.getWorkspace({userId: this._usersManager.getSupportUserId()}, this._exampleWorkspaceId)), aliases: [], access: 'editors', // a share may have view/edit access, @@ -1248,7 +906,7 @@ export class HomeDBManager extends EventEmitter { // We imagine current user owning trunk if there is no embedded userId, or // the embedded userId matches the current user. const access = (forkUserId === undefined || forkUserId === userId) ? 'owners' : - (userId === this.getPreviewerUserId() ? 'viewers' : null); + (userId === this._usersManager.getPreviewerUserId() ? 'viewers' : null); if (!access) { throw new ApiError("access denied", 403); } doc = { name: 'Untitled', @@ -1258,7 +916,7 @@ export class HomeDBManager extends EventEmitter { isPinned: false, urlId: null, workspace: this.unwrapQueryResult( - await this.getWorkspace({userId: this.getSupportUserId()}, + await this.getWorkspace({userId: this._usersManager.getSupportUserId()}, this._exampleWorkspaceId)), aliases: [], access @@ -1271,7 +929,7 @@ export class HomeDBManager extends EventEmitter { // work. let qb = this._doc({...key, showAll: true}, {manager: transaction}) .leftJoinAndSelect('orgs.owner', 'org_users'); - if (userId !== this.getAnonymousUserId()) { + if (userId !== this._usersManager.getAnonymousUserId()) { qb = this._addForks(userId, qb); } qb = this._addIsSupportWorkspace(userId, qb, 'orgs', 'workspaces'); @@ -1330,7 +988,7 @@ export class HomeDBManager extends EventEmitter { public async getRawDocById(docId: string, transaction?: EntityManager) { return await this.getDoc({ urlId: docId, - userId: this.getPreviewerUserId(), + userId: this._usersManager.getPreviewerUserId(), showAll: true }, transaction); } @@ -1449,7 +1107,7 @@ export class HomeDBManager extends EventEmitter { // If we are support user, use team product // A bit fragile: this is called during creation of support@ user, before // getSupportUserId() is available, but with setUserAsOwner of true. - user.id === this.getSupportUserId() ? productNames.team : + user.id === this._usersManager.getSupportUserId() ? productNames.team : // Otherwise use teamInitial product (a stub). productNames.teamInitial; @@ -2177,7 +1835,7 @@ export class HomeDBManager extends EventEmitter { } // Get the ids of users to update. const billingAccountId = billingAccount.id; - const analysis = await this._verifyAndLookupDeltaEmails(userId, permissionDelta, true, transaction); + const analysis = await this._usersManager.verifyAndLookupDeltaEmails(userId, permissionDelta, true, transaction); this._failIfPowerfulAndChangingSelf(analysis); const {userIdDelta} = analysis; if (!userIdDelta) { throw new ApiError('No userIdDelta', 500); } @@ -2223,7 +1881,7 @@ export class HomeDBManager extends EventEmitter { const {userId} = scope; const notifications: Array<() => void> = []; const result = await this._connection.transaction(async manager => { - const analysis = await this._verifyAndLookupDeltaEmails(userId, delta, true, manager); + const analysis = await this._usersManager.verifyAndLookupDeltaEmails(userId, delta, true, manager); const {userIdDelta} = analysis; let orgQuery = this.org(scope, orgKey, { manager, @@ -2245,7 +1903,7 @@ export class HomeDBManager extends EventEmitter { const org: Organization = queryResult.data; const groups = getNonGuestGroups(org); if (userIdDelta) { - const membersBefore = getUsersWithRole(groups, this.getExcludedUserIds()); + const membersBefore = UsersManager.getUsersWithRole(groups, this._usersManager.getExcludedUserIds()); const countBefore = removeRole(membersBefore).length; await this._updateUserPermissions(groups, userIdDelta, manager); this._checkUserChangeAllowed(userId, groups); @@ -2258,7 +1916,7 @@ export class HomeDBManager extends EventEmitter { } } // Emit an event if the number of org users is changing. - const membersAfter = getUsersWithRole(groups, this.getExcludedUserIds()); + const membersAfter = UsersManager.getUsersWithRole(groups, this._usersManager.getExcludedUserIds()); const countAfter = removeRole(membersAfter).length; notifications.push(this._userChangeNotification(userId, org, countBefore, countAfter, membersBefore, membersAfter)); @@ -2280,7 +1938,7 @@ export class HomeDBManager extends EventEmitter { const {userId} = scope; const notifications: Array<() => void> = []; const result = await this._connection.transaction(async manager => { - const analysis = await this._verifyAndLookupDeltaEmails(userId, delta, false, manager); + const analysis = await this._usersManager.verifyAndLookupDeltaEmails(userId, delta, false, manager); let {userIdDelta} = analysis; let wsQuery = this._workspace(scope, wsId, { manager, @@ -2318,14 +1976,16 @@ export class HomeDBManager extends EventEmitter { userIdDelta[userId] = roles.OWNER; } } - const membersBefore = this._withoutExcludedUsers(new Map(groups.map(grp => [grp.name, grp.memberUsers]))); + const membersBefore = this._usersManager.withoutExcludedUsers( + new Map(groups.map(grp => [grp.name, grp.memberUsers])) + ); if (userIdDelta) { // To check limits on shares, we track group members before and after call // to _updateUserPermissions. Careful, that method mutates groups. - const nonOrgMembersBefore = this._getUserDifference(groups, orgGroups); + const nonOrgMembersBefore = this._usersManager.getUserDifference(groups, orgGroups); await this._updateUserPermissions(groups, userIdDelta, manager); this._checkUserChangeAllowed(userId, groups); - const nonOrgMembersAfter = this._getUserDifference(groups, orgGroups); + const nonOrgMembersAfter = this._usersManager.getUserDifference(groups, orgGroups); const features = ws.org.billingAccount.getFeatures(); const limit = features.maxSharesPerWorkspace; if (limit !== undefined) { @@ -2353,7 +2013,7 @@ export class HomeDBManager extends EventEmitter { const notifications: Array<() => void> = []; const result = await this._connection.transaction(async manager => { const {userId} = scope; - const analysis = await this._verifyAndLookupDeltaEmails(userId, delta, false, manager); + const analysis = await this._usersManager.verifyAndLookupDeltaEmails(userId, delta, false, manager); let {userIdDelta} = analysis; const doc = await this._loadDocAccess(scope, analysis.permissionThreshold, manager); this._failIfPowerfulAndChangingSelf(analysis, {data: doc, status: 200}); @@ -2376,10 +2036,10 @@ export class HomeDBManager extends EventEmitter { // to _updateUserPermissions. Careful, that method mutates groups. const org = doc.workspace.org; const orgGroups = getNonGuestGroups(org); - const nonOrgMembersBefore = this._getUserDifference(groups, orgGroups); + const nonOrgMembersBefore = this._usersManager.getUserDifference(groups, orgGroups); await this._updateUserPermissions(groups, userIdDelta, manager); this._checkUserChangeAllowed(userId, groups); - const nonOrgMembersAfter = this._getUserDifference(groups, orgGroups); + const nonOrgMembersAfter = this._usersManager.getUserDifference(groups, orgGroups); const features = org.billingAccount.getFeatures(); this._restrictAllDocShares(features, nonOrgMembersBefore, nonOrgMembersAfter); } @@ -2405,7 +2065,7 @@ export class HomeDBManager extends EventEmitter { } const org: Organization = queryResult.data; const userRoleMap = getMemberUserRoles(org, this.defaultGroupNames); - const users = getResourceUsers(org).filter(u => userRoleMap[u.id]).map(u => { + const users = UsersManager.getResourceUsers(org).filter(u => userRoleMap[u.id]).map(u => { const access = userRoleMap[u.id]; return { ...this.makeFullUser(u), @@ -2462,7 +2122,7 @@ export class HomeDBManager extends EventEmitter { const orgMapWithMembership = getMemberUserRoles(org, this.defaultGroupNames); // Iterate through the org since all users will be in the org. - const users: UserAccessData[] = getResourceUsers([workspace, org]).map(u => { + const users: UserAccessData[] = UsersManager.getResourceUsers([workspace, org]).map(u => { const orgAccess = orgMapWithMembership[u.id] || null; return { ...this.makeFullUser(u), @@ -2522,7 +2182,7 @@ export class HomeDBManager extends EventEmitter { const orgMapWithMembership = getMemberUserRoles(doc.workspace.org, this.defaultGroupNames); const wsMaxInheritedRole = this._getMaxInheritedRole(doc.workspace); // Iterate through the org since all users will be in the org. - let users: UserAccessData[] = getResourceUsers([doc, doc.workspace, doc.workspace.org]).map(u => { + let users: UserAccessData[] = UsersManager.getResourceUsers([doc, doc.workspace, doc.workspace.org]).map(u => { // Merge the strongest roles from the resource and parent resources. Note that the parent // resource access levels must be tempered by the maxInheritedRole values of their children. const inheritFromOrg = roles.getWeakestRole(orgMap[u.id] || null, wsMaxInheritedRole); @@ -2535,7 +2195,7 @@ export class HomeDBManager extends EventEmitter { roles.getStrongestRole(wsMap[u.id] || null, inheritFromOrg) ), isMember: orgAccess && orgAccess !== 'guests', - isSupport: u.id === this.getSupportUserId() ? true : undefined, + isSupport: u.id === this._usersManager.getSupportUserId() ? true : undefined, }; }); let maxInheritedRole = this._getMaxInheritedRole(doc); @@ -2629,7 +2289,7 @@ export class HomeDBManager extends EventEmitter { } const workspace: Workspace = wsQueryResult.data; // Collect all first-level users of the doc being moved. - const firstLevelUsers = getResourceUsers(doc); + const firstLevelUsers = UsersManager.getResourceUsers(doc); const docGroups = doc.aclRules.map(rule => rule.group); if (doc.workspace.org.id !== workspace.org.id) { // Doc is going to a new org. Check that there is room for it there. @@ -2640,8 +2300,8 @@ export class HomeDBManager extends EventEmitter { const sourceOrgGroups = getNonGuestGroups(sourceOrg); const destOrg = workspace.org; const destOrgGroups = getNonGuestGroups(destOrg); - const nonOrgMembersBefore = this._getUserDifference(docGroups, sourceOrgGroups); - const nonOrgMembersAfter = this._getUserDifference(docGroups, destOrgGroups); + const nonOrgMembersBefore = this._usersManager.getUserDifference(docGroups, sourceOrgGroups); + const nonOrgMembersAfter = this._usersManager.getUserDifference(docGroups, destOrgGroups); const features = destOrg.billingAccount.getFeatures(); this._restrictAllDocShares(features, nonOrgMembersBefore, nonOrgMembersAfter, false); } @@ -2819,93 +2479,31 @@ export class HomeDBManager extends EventEmitter { .getOne() || undefined; } - /** - * Get the anonymous user, as a constructed object rather than a database lookup. - */ - public getAnonymousUser(): User { - const user = new User(); - user.id = this.getAnonymousUserId(); - user.name = "Anonymous"; - user.isFirstTimeUser = false; - const login = new Login(); - login.displayEmail = login.email = ANONYMOUS_USER_EMAIL; - user.logins = [login]; - user.ref = ''; - return user; - } - - /** - * - * Get the id of the anonymous user. - * - */ - public getAnonymousUserId(): number { - const id = this._specialUserIds[ANONYMOUS_USER_EMAIL]; - if (!id) { throw new Error("Anonymous user not available"); } - return id; + public getAnonymousUser() { + return this._usersManager.getAnonymousUser(); } - /** - * Get the id of the thumbnail user. - */ - public getPreviewerUserId(): number { - const id = this._specialUserIds[PREVIEWER_EMAIL]; - if (!id) { throw new Error("Previewer user not available"); } - return id; + public getAnonymousUserId() { + return this._usersManager.getAnonymousUserId(); } - /** - * Get the id of the 'everyone' user. - */ - public getEveryoneUserId(): number { - const id = this._specialUserIds[EVERYONE_EMAIL]; - if (!id) { throw new Error("'everyone' user not available"); } - return id; + public getPreviewerUserId() { + return this._usersManager.getPreviewerUserId(); } - /** - * Get the id of the 'support' user. - */ - public getSupportUserId(): number { - const id = this._specialUserIds[SUPPORT_EMAIL]; - if (!id) { throw new Error("'support' user not available"); } - return id; + public getEveryoneUserId() { + return this._usersManager.getEveryoneUserId(); } - /** - * Get ids of users to be excluded from member counts and emails. - */ - public getExcludedUserIds(): number[] { - return [this.getSupportUserId(), this.getAnonymousUserId(), this.getEveryoneUserId()]; + public getSupportUserId() { + return this._usersManager.getSupportUserId(); } /** - * - * Take a list of user profiles coming from the client's session, correlate - * them with Users and Logins in the database, and construct full profiles - * with user ids, standardized display emails, pictures, and anonymous flags. - * + * @see UsersManager.prototype.completeProfiles */ - public async completeProfiles(profiles: UserProfile[]): Promise { - if (profiles.length === 0) { return []; } - const qb = this._connection.createQueryBuilder() - .select('logins') - .from(Login, 'logins') - .leftJoinAndSelect('logins.user', 'user') - .where('logins.email in (:...emails)', {emails: profiles.map(profile => normalizeEmail(profile.email))}); - const completedProfiles: {[email: string]: FullUser} = {}; - for (const login of await qb.getMany()) { - completedProfiles[login.email] = { - id: login.user.id, - email: login.displayEmail, - name: login.user.name, - picture: login.user.picture, - anonymous: login.user.id === this.getAnonymousUserId(), - locale: login.user.options?.locale - }; - } - return profiles.map(profile => completedProfiles[normalizeEmail(profile.email)]) - .filter(profile => profile); + public async completeProfiles(profiles: UserProfile[]) { + return this._usersManager.completeProfiles(profiles); } /** @@ -3167,7 +2765,7 @@ export class HomeDBManager extends EventEmitter { } org = result.entities[0]; } - return getResourceUsers(org, this.defaultNonGuestGroupNames); + return UsersManager.getResourceUsers(org, this.defaultNonGuestGroupNames); } private async _getOrCreateLimit(accountId: number, limitType: LimitType, force: boolean): Promise { @@ -3204,7 +2802,6 @@ export class HomeDBManager extends EventEmitter { return result; } - private _org(scope: Scope|null, includeSupport: boolean, org: string|number|null, options: QueryOptions = {}): SelectQueryBuilder { let query = this._orgs(options.manager); @@ -3224,7 +2821,7 @@ export class HomeDBManager extends EventEmitter { // TODO If the specialPermit is used across the network, requests could refer to orgs in // different ways (number vs string), causing this comparison to fail. if (options.allowSpecialPermit && scope.specialPermit && scope.specialPermit.org === org) { - effectiveUserId = this.getPreviewerUserId(); + effectiveUserId = this._usersManager.getPreviewerUserId(); threshold = Permissions.VIEW; } // Compute whether we have access to the doc @@ -3244,7 +2841,7 @@ export class HomeDBManager extends EventEmitter { private _orgWorkspaces(scope: Scope, org: string|number|null, options: QueryOptions = {}): SelectQueryBuilder { const {userId} = scope; - const supportId = this._specialUserIds[SUPPORT_EMAIL]; + const supportId = this._usersManager.getSpecialUserId(SUPPORT_EMAIL); let query = this.org(scope, org, options) .leftJoinAndSelect('orgs.workspaces', 'workspaces') .leftJoinAndSelect('workspaces.docs', 'docs', this._onDoc(scope)) @@ -3263,7 +2860,7 @@ export class HomeDBManager extends EventEmitter { .addOrderBy('docs.created_at') .leftJoinAndSelect('orgs.owner', 'org_users'); - if (userId !== this.getAnonymousUserId()) { + if (userId !== this._usersManager.getAnonymousUserId()) { query = this._addForks(userId, query); } @@ -3274,7 +2871,7 @@ export class HomeDBManager extends EventEmitter { // Add a direct, efficient filter to remove irrelevant personal orgs from consideration. query = this._filterByOrgGroups(query, userId, null); // The anonymous user is a special case; include only examples from support user. - if (userId === this.getAnonymousUserId()) { + if (userId === this._usersManager.getAnonymousUserId()) { query = query.andWhere('orgs.owner_id = :supportId', { supportId }); } } @@ -3298,7 +2895,7 @@ export class HomeDBManager extends EventEmitter { .where('docs.urlId = :urlId', {urlId}); // Place restriction on active urlIds only. // Older urlIds are best-effort, and subject to // reuse (currently). - if (org.ownerId === this.getSupportUserId()) { + if (org.ownerId === this._usersManager.getSupportUserId()) { // This is the support user. Some of their documents end up as examples on team sites. // so urlIds need to be checked globally, which corresponds to placing no extra where // clause here. @@ -3355,7 +2952,10 @@ export class HomeDBManager extends EventEmitter { .andWhere('doc_users.id is not null'); const wsWithDocs = await wsWithDocsQuery.getOne(); await this._setGroupUsers(manager, wsGuestGroup.id, wsGuestGroup.memberUsers, - this._filterEveryone(getResourceUsers(wsWithDocs?.docs || []))); + this._usersManager.filterEveryone( + UsersManager.getResourceUsers(wsWithDocs?.docs || []) + ) + ); }); } @@ -3386,7 +2986,7 @@ export class HomeDBManager extends EventEmitter { } const orgGuestGroup = orgGroups[0]!; await this._setGroupUsers(manager, orgGuestGroup.id, orgGuestGroup.memberUsers, - this._filterEveryone(getResourceUsers(org.workspaces))); + this._usersManager.filterEveryone(UsersManager.getResourceUsers(org.workspaces))); }); } @@ -3420,25 +3020,6 @@ export class HomeDBManager extends EventEmitter { } } - /** - * Don't add everyone@ as a guest, unless also sharing with anon@. - * This means that material shared with everyone@ doesn't become - * listable/discoverable by default. - * - * This is a HACK to allow existing example doc setup to continue to - * work. It could be removed if we are willing to share the entire - * support org with users. E.g. move any material we don't want to - * share into a workspace that doesn't inherit ACLs. TODO: remove - * this hack, or enhance it up as a way to support discoverability / - * listing. It has the advantage of cloning well. - */ - private _filterEveryone(users: User[]): User[] { - const everyone = this.getEveryoneUserId(); - const anon = this.getAnonymousUserId(); - if (users.find(u => u.id === anon)) { return users; } - return users.filter(u => u.id !== everyone); - } - /** * Creates, initializes and saves a workspace in the given org with the given properties. * Product limits on number of workspaces allowed in org are not checked. @@ -3502,7 +3083,7 @@ export class HomeDBManager extends EventEmitter { * Adds any calculated fields related to billing accounts - currently just * products.paid. */ - private _addBillingAccountCalculatedFields(qb: SelectQueryBuilder) { + private _addBillingAccountCalculatedFields(qb: SelectQueryBuilder) { // We need to sum up whether the account is paid or not, so that UI can provide // a "billing" vs "upgrade" link. For the moment, we just check if there is // a subscription id. TODO: make sure this is correct in case of free plans. @@ -3513,16 +3094,16 @@ export class HomeDBManager extends EventEmitter { /** * Makes sure that product features for orgs are available in query result. */ - private _addFeatures(qb: SelectQueryBuilder, orgAlias: string = 'orgs') { + private _addFeatures(qb: SelectQueryBuilder, orgAlias: string = 'orgs') { qb = qb.leftJoinAndSelect(`${orgAlias}.billingAccount`, 'billing_accounts'); qb = qb.leftJoinAndSelect('billing_accounts.product', 'products'); // orgAlias.billingAccount.product.features should now be available return qb; } - private _addIsSupportWorkspace(users: AvailableUsers, qb: SelectQueryBuilder, + private _addIsSupportWorkspace(users: AvailableUsers, qb: SelectQueryBuilder, orgAlias: string, workspaceAlias: string) { - const supportId = this._specialUserIds[SUPPORT_EMAIL]; + const supportId = this._usersManager.getSpecialUserId(SUPPORT_EMAIL); // We'll be selecting a boolean and naming it as *_support. This matches the // SQL name `support` of a column in the Workspace entity whose javascript @@ -3531,7 +3112,7 @@ export class HomeDBManager extends EventEmitter { // If we happen to be the support user, don't treat our workspaces as anything // special, so we can work with them in the ordinary way. - if (isSingleUser(users) && users === supportId) { return qb.addSelect('false', alias); } + if (UsersManager.isSingleUser(users) && users === supportId) { return qb.addSelect('false', alias); } // Otherwise, treat workspaces owned by support as special. return qb.addSelect(`coalesce(${orgAlias}.owner_id = ${supportId}, false)`, alias); @@ -3540,7 +3121,7 @@ export class HomeDBManager extends EventEmitter { /** * Makes sure that doc forks are available in query result. */ - private _addForks(userId: number, qb: SelectQueryBuilder) { + private _addForks(userId: number, qb: SelectQueryBuilder) { return qb.leftJoin('docs.forks', 'forks', 'forks.created_by = :forkUserId') .setParameter('forkUserId', userId) .addSelect([ @@ -3552,24 +3133,6 @@ export class HomeDBManager extends EventEmitter { ]); } - /** - * - * Get the id of a special user, creating that user if it is not already present. - * - */ - private async _getSpecialUserId(profile: UserProfile) { - let id = this._specialUserIds[profile.email]; - if (!id) { - // get or create user - with retry, since there'll be a race to create the - // user if a bunch of servers start simultaneously and the user doesn't exist - // yet. - const user = await this.getUserByLoginWithRetry(profile.email, {profile}); - if (user) { id = this._specialUserIds[profile.email] = user.id; } - } - if (!id) { throw new Error(`Could not find or create user ${profile.email}`); } - return id; - } - /** * Modify an access level when the document is a fork. Here are the rules, as they * have evolved (the main constraint is that currently forks have no access info of @@ -3584,7 +3147,7 @@ export class HomeDBManager extends EventEmitter { ids: {userId: number, forkUserId?: number}, res: {access: roles.Role|null}) { if (doc.type === 'tutorial') { - if (ids.userId === this.getPreviewerUserId()) { + if (ids.userId === this._usersManager.getPreviewerUserId()) { res.access = 'viewers'; } else if (ids.forkUserId && ids.forkUserId === ids.userId) { res.access = 'owners'; @@ -3605,99 +3168,6 @@ export class HomeDBManager extends EventEmitter { } } } - - // This deals with the problem posed by receiving a PermissionDelta specifying a - // role for both alice@x and Alice@x. We do not distinguish between such emails. - // If there are multiple indistinguishabe emails, we preserve just one of them, - // assigning it the most powerful permission specified. The email variant perserved - // is the earliest alphabetically. - private _mergeIndistinguishableEmails(delta: PermissionDelta) { - if (!delta.users) { return; } - // We normalize emails for comparison, but track how they were capitalized - // in order to preserve it. This is worth doing since for the common case - // of a user being added to a resource prior to ever logging in, their - // displayEmail will be seeded from this value. - const displayEmails: {[email: string]: string} = {}; - // This will be our output. - const users: {[email: string]: roles.NonGuestRole|null} = {}; - for (const displayEmail of Object.keys(delta.users).sort()) { - const email = normalizeEmail(displayEmail); - const role = delta.users[displayEmail]; - const key = displayEmails[email] = displayEmails[email] || displayEmail; - users[key] = users[key] ? roles.getStrongestRole(users[key], role) : role; - } - delta.users = users; - } - - // Looks up the emails in the permission delta and adds them to the users map in - // the delta object. - // Returns a QueryResult based on the validity of the passed in PermissionDelta object. - private async _verifyAndLookupDeltaEmails( - userId: number, - delta: PermissionDelta, - isOrg: boolean = false, - transaction?: EntityManager - ): Promise { - if (!delta) { - throw new ApiError('Bad request: missing permission delta', 400); - } - this._mergeIndistinguishableEmails(delta); - const hasInherit = 'maxInheritedRole' in delta; - const hasUsers = delta.users; // allow zero actual changes; useful to reduce special - // cases in scripts - if ((isOrg && (hasInherit || !hasUsers)) || (!isOrg && !hasInherit && !hasUsers)) { - throw new ApiError('Bad request: invalid permission delta', 400); - } - // Lookup the email access changes and move them to the users object. - const userIdMap: {[userId: string]: roles.NonGuestRole|null} = {}; - if (hasInherit) { - // Verify maxInheritedRole - const role = delta.maxInheritedRole; - const validRoles = new Set(this.defaultBasicGroupNames); - if (role && !validRoles.has(role)) { - throw new ApiError(`Invalid maxInheritedRole ${role}`, 400); - } - } - if (delta.users) { - // Verify roles - const deltaRoles = Object.keys(delta.users).map(_userId => delta.users![_userId]); - // Cannot set role "members" on workspace/doc. - const validRoles = new Set(isOrg ? this.defaultNonGuestGroupNames : this.defaultBasicGroupNames); - for (const role of deltaRoles) { - if (role && !validRoles.has(role)) { - throw new ApiError(`Invalid user role ${role}`, 400); - } - } - // Lookup emails - const emailMap = delta.users; - const emails = Object.keys(emailMap); - const emailUsers = await Promise.all( - emails.map(async email => await this.getUserByLogin(email, {manager: transaction})) - ); - emails.forEach((email, i) => { - const userIdAffected = emailUsers[i]!.id; - // Org-level sharing with everyone would allow serious spamming - forbid it. - if (emailMap[email] !== null && // allow removing anything - userId !== this.getSupportUserId() && // allow support user latitude - userIdAffected === this.getEveryoneUserId() && - isOrg) { - throw new ApiError('This user cannot share with everyone at top level', 403); - } - userIdMap[userIdAffected] = emailMap[email]; - }); - } - const userIdDelta = delta.users ? userIdMap : null; - const userIds = Object.keys(userIdDelta || {}); - const removingSelf = userIds.length === 1 && userIds[0] === String(userId) && - delta.maxInheritedRole === undefined && userIdDelta?.[userId] === null; - const permissionThreshold = removingSelf ? Permissions.VIEW : Permissions.ACL_EDIT; - return { - userIdDelta, - permissionThreshold, - affectsSelf: userId in userIdMap, - }; - } - /** * A helper to throw an error if a user with ACL_EDIT permission attempts * to change their own access rights. The user permissions are expected to @@ -3732,7 +3202,7 @@ export class HomeDBManager extends EventEmitter { // Get the user objects which map to non-null values in the userDelta. const userIds = Object.keys(userDelta).filter(userId => userDelta[userId]) .map(userIdStr => parseInt(userIdStr, 10)); - const users = await this._getUsers(userIds, manager); + const users = await this._usersManager.getUsers(userIds, manager); // Add unaffected users to the delta so that we have a record of where they are. groups.forEach(grp => { @@ -3777,21 +3247,6 @@ export class HomeDBManager extends EventEmitter { return this._connection.transaction(op); } - /** - * Returns a Promise for an array of User entites for the given userIds. - */ - private async _getUsers(userIds: number[], optManager?: EntityManager): Promise { - if (userIds.length === 0) { - return []; - } - const manager = optManager || new EntityManager(this._connection); - const queryBuilder = manager.createQueryBuilder() - .select('users') - .from(User, 'users') - .where('users.id IN (:...userIds)', {userIds}); - return await queryBuilder.getMany(); - } - /** * Aggregate the given columns as a json object. The keys should be simple * alphanumeric strings, and the values should be the names of sql columns - @@ -3854,7 +3309,7 @@ export class HomeDBManager extends EventEmitter { let threshold = options.markPermissions; if (options.allowSpecialPermit && scope.specialPermit && scope.specialPermit.docId) { query = query.andWhere('docs.id = :docId', {docId: scope.specialPermit.docId}); - effectiveUserId = this.getPreviewerUserId(); + effectiveUserId = this._usersManager.getPreviewerUserId(); threshold = Permissions.VIEW; } // Compute whether we have access to the doc @@ -3888,7 +3343,7 @@ export class HomeDBManager extends EventEmitter { } else { query = query .setParameter('forkUserId', scope.userId) - .setParameter('forkAnonId', this.getAnonymousUserId()) + .setParameter('forkAnonId', this._usersManager.getAnonymousUserId()) .addSelect( // Access to forks is currently limited to the users that created them, with // the exception of anonymous users, who have no access to their forks. @@ -3941,7 +3396,7 @@ export class HomeDBManager extends EventEmitter { let threshold = options.markPermissions; if (options.allowSpecialPermit && scope.specialPermit && scope.specialPermit.workspaceId === wsId) { - effectiveUserId = this.getPreviewerUserId(); + effectiveUserId = this._usersManager.getPreviewerUserId(); threshold = Permissions.VIEW; } // Compute whether we have access to the ws @@ -3974,7 +3429,7 @@ export class HomeDBManager extends EventEmitter { // Always include the org of the support@ user, which contains the Samples workspace, // which we always show. (For isMergedOrg case, it's already included.) if (includeSupport) { - const supportId = this._specialUserIds[SUPPORT_EMAIL]; + const supportId = this._usersManager.getSpecialUserId(SUPPORT_EMAIL); return qb.andWhere(new Brackets((q) => this._wherePlainOrg(q, org).orWhere('orgs.owner_id = :supportId', {supportId}))); } else { @@ -4028,11 +3483,11 @@ export class HomeDBManager extends EventEmitter { .leftJoin('orgs.aclRules', 'acl_rules') .leftJoin('acl_rules.group', 'groups') .leftJoin('groups.memberUsers', 'members'); - if (isSingleUser(users)) { + if (UsersManager.isSingleUser(users)) { // Add an exception for the previewer user, if present. - const previewerId = this._specialUserIds[PREVIEWER_EMAIL]; + const previewerId = this._usersManager.getSpecialUserId(PREVIEWER_EMAIL); if (users === previewerId) { return qb; } - const everyoneId = this._specialUserIds[EVERYONE_EMAIL]; + const everyoneId = this._usersManager.getSpecialUserId(EVERYONE_EMAIL); if (options?.ignoreEveryoneShares) { return qb.where('members.id = :userId', {userId: users}); } @@ -4287,7 +3742,7 @@ export class HomeDBManager extends EventEmitter { throw new ApiError('Cannot find unique login for user', 500); } value.email = logins[0].displayEmail; - value.anonymous = (logins[0].userId === this.getAnonymousUserId()); + value.anonymous = (logins[0].userId === this._usersManager.getAnonymousUserId()); continue; } if (key === 'managers') { @@ -4401,8 +3856,8 @@ export class HomeDBManager extends EventEmitter { if (permissions !== null) { q = q.select('acl_rules.permissions'); } else { - const everyoneId = this._specialUserIds[EVERYONE_EMAIL]; - const anonId = this._specialUserIds[ANONYMOUS_USER_EMAIL]; + const everyoneId = this._usersManager.getSpecialUserId(EVERYONE_EMAIL); + const anonId = this._usersManager.getSpecialUserId(ANONYMOUS_USER_EMAIL); // Overall permissions are the bitwise-or of all individual // permissions from ACL rules. We also include // Permissions.PUBLIC if any of the ACL rules are for the @@ -4454,7 +3909,7 @@ export class HomeDBManager extends EventEmitter { q = q.andWhere(`acl_rules.${idColumn} = ${resType}.id`); if (permissions !== null) { q = q.andWhere(`(acl_rules.permissions & ${permissions}) = ${permissions}`).limit(1); - } else if (!isSingleUser(users)) { + } else if (!UsersManager.isSingleUser(users)) { q = q.addSelect('profiles.id'); q = q.addSelect('profiles.display_email'); q = q.addSelect('profiles.name'); @@ -4466,7 +3921,7 @@ export class HomeDBManager extends EventEmitter { } return q; }; - if (isSingleUser(users)) { + if (UsersManager.isSingleUser(users)) { return getBasicPermissions(qb.subQuery()); } else { return qb.subQuery() @@ -4496,7 +3951,7 @@ export class HomeDBManager extends EventEmitter { qb = qb // filter for the specified user being a direct or indirect member of the acl_rule's group .where(new Brackets(cond => { - if (isSingleUser(users)) { + if (UsersManager.isSingleUser(users)) { // Users is an integer, so ok to insert into sql. It we // didn't, we'd need to use distinct parameter names, since // we may include this code with different user ids in the @@ -4506,7 +3961,7 @@ export class HomeDBManager extends EventEmitter { cond = cond.orWhere(`gu2.user_id = ${users}`); cond = cond.orWhere(`gu3.user_id = ${users}`); // Support the special "everyone" user. - const everyoneId = this._specialUserIds[EVERYONE_EMAIL]; + const everyoneId = this._usersManager.getSpecialUserId(EVERYONE_EMAIL); if (everyoneId === undefined) { throw new Error("Special user id for EVERYONE_EMAIL not found"); } @@ -4517,7 +3972,7 @@ export class HomeDBManager extends EventEmitter { if (accessStyle === 'list') { // Support also the special anonymous user. Currently, by convention, sharing a // resource with anonymous should make it listable. - const anonId = this._specialUserIds[ANONYMOUS_USER_EMAIL]; + const anonId = this._usersManager.getSpecialUserId(ANONYMOUS_USER_EMAIL); if (anonId === undefined) { throw new Error("Special user id for ANONYMOUS_USER_EMAIL not found"); } @@ -4527,7 +3982,7 @@ export class HomeDBManager extends EventEmitter { cond = cond.orWhere(`gu3.user_id = ${anonId}`); } // Add an exception for the previewer user, if present. - const previewerId = this._specialUserIds[PREVIEWER_EMAIL]; + const previewerId = this._usersManager.getSpecialUserId(PREVIEWER_EMAIL); if (users === previewerId) { // All acl_rules granting view access are available to previewer user. cond = cond.orWhere('acl_rules.permissions = :permission', @@ -4541,7 +3996,7 @@ export class HomeDBManager extends EventEmitter { } return cond; })); - if (!isSingleUser(users)) { + if (!UsersManager.isSingleUser(users)) { // We need to join against a list of users. const emails = new Set(users.map(profile => normalizeEmail(profile.email))); if (emails.size > 0) { @@ -4575,7 +4030,7 @@ export class HomeDBManager extends EventEmitter { // Apply limits to the query. Results should be limited to a specific org // if request is from a branded webpage; results should be limited to a // specific user or set of users. - private _applyLimit(qb: SelectQueryBuilder, limit: Scope, + private _applyLimit(qb: SelectQueryBuilder, limit: Scope, resources: Array<'docs'|'workspaces'|'orgs'>, accessStyle: AccessStyle): SelectQueryBuilder { if (limit.org) { @@ -4687,7 +4142,7 @@ export class HomeDBManager extends EventEmitter { if (features.maxDocsPerOrg !== undefined) { // we need to count how many docs are in the current org, and if we // are already at or above the limit, then fail. - const wss = this.unwrapQueryResult(await this.getOrgWorkspaces({userId: this.getPreviewerUserId()}, + const wss = this.unwrapQueryResult(await this.getOrgWorkspaces({userId: this._usersManager.getPreviewerUserId()}, workspace.org.id, {manager})); const count = wss.map(ws => ws.docs.length).reduce((a, b) => a + b, 0); @@ -4707,11 +4162,7 @@ export class HomeDBManager extends EventEmitter { // For the moment only the support user can add both everyone@ and anon@ to a // resource, since that allows spam. TODO: enhance or remove. private _checkUserChangeAllowed(userId: number, groups: Group[]) { - if (userId === this.getSupportUserId()) { return; } - const ids = new Set(flatten(groups.map(g => g.memberUsers)).map(u => u.id)); - if (ids.has(this.getEveryoneUserId()) && ids.has(this.getAnonymousUserId())) { - throw new Error('this user cannot share with everyone and anonymous'); - } + return this._usersManager.checkUserChangeAllowed(userId, groups); } // Fetch a Document with all access information loaded. Make sure the user has the @@ -4787,30 +4238,6 @@ export class HomeDBManager extends EventEmitter { return () => this.emit('addUser', userId, resource, userIdDelta, membersBefore); } - // Given two arrays of groups, returns a map of users present in the first array but - // not the second, where the map is broken down by user role. - // This method is used for checking limits on shares. - // Excluded users are removed from the results. - private _getUserDifference(groupsA: Group[], groupsB: Group[]): Map { - const subtractSet: Set = - new Set(flatten(groupsB.map(grp => grp.memberUsers)).map(usr => usr.id)); - const result = new Map(); - for (const group of groupsA) { - const name = group.name; - if (!roles.isNonGuestRole(name)) { continue; } - result.set(name, group.memberUsers.filter(user => !subtractSet.has(user.id))); - } - return this._withoutExcludedUsers(result); - } - - private _withoutExcludedUsers(members: Map): Map { - const excludedUsers = this.getExcludedUserIds(); - for (const [role, users] of members.entries()) { - members.set(role, users.filter((user) => !excludedUsers.includes(user.id))); - } - return members; - } - private _billingManagerNotification(userId: number, addUserId: number, orgs: Organization[]) { return () => { this.emit('addBillingManager', userId, addUserId, orgs); @@ -4823,17 +4250,6 @@ export class HomeDBManager extends EventEmitter { }; } - /** - * Check for anonymous user, either encoded directly as an id, or as a singular - * profile (this case arises during processing of the session/access/all endpoint - * whether we are checking for available orgs without committing yet to a particular - * choice of user). - */ - private _isAnonymousUser(users: AvailableUsers): boolean { - return isSingleUser(users) ? users === this.getAnonymousUserId() : - users.length === 1 && normalizeEmail(users[0].email) === ANONYMOUS_USER_EMAIL; - } - // Set Workspace.removedAt to null (undeletion) or to a datetime (soft deletion) private _setWorkspaceRemovedAt(scope: Scope, wsId: number, removedAt: Date|null) { return this._connection.transaction(async manager => { @@ -4875,12 +4291,12 @@ export class HomeDBManager extends EventEmitter { maxInheritedRole: roles.BasicRole|null, docId?: string ): {personal: true, public: boolean}|undefined { - if (scope.userId === this.getPreviewerUserId()) { return; } + if (scope.userId === this._usersManager.getPreviewerUserId()) { return; } // If we have special access to the resource, don't filter user information. if (scope.specialPermit?.docId === docId && docId) { return; } - const thisUser = this.getAnonymousUserId() === scope.userId + const thisUser = this._usersManager.getAnonymousUserId() === scope.userId ? null : users.find(user => user.id === scope.userId); const realAccess = thisUser ? getRealAccess(thisUser, { maxInheritedRole, users }) : null; @@ -4970,24 +4386,6 @@ async function verifyEntity( }; } -// Returns all first-level memberUsers in the resources. Requires all resources' aclRules, groups -// and memberUsers to be populated. -// If optRoles is provided, only checks membership in resource groups with the given roles. -function getResourceUsers(res: Resource|Resource[], optRoles?: string[]): User[] { - res = Array.isArray(res) ? res : [res]; - const users: {[uid: string]: User} = {}; - let resAcls: AclRule[] = flatten(res.map(_res => _res.aclRules as AclRule[])); - if (optRoles) { - resAcls = resAcls.filter(_acl => optRoles.includes(_acl.group.name)); - } - resAcls.forEach((aclRule: AclRule) => { - aclRule.group.memberUsers.forEach((u: User) => users[u.id] = u); - }); - const userList = Object.keys(users).map(uid => users[uid]); - userList.sort((a, b) => a.id - b.id); - return userList; -} - // Returns a map of userIds to the user's strongest default role on the given resource. // The resource's aclRules, groups, and memberUsers must be populated. function getMemberUserRoles(res: Resource, allowRoles: T[]): {[userId: string]: T} { @@ -5020,24 +4418,6 @@ export function removeRole(usersWithRoles: Map) { return flatten([...usersWithRoles.values()]); } -function getNonGuestGroups(entity: Organization|Workspace|Document): NonGuestGroup[] { - return (entity.aclRules as AclRule[]).map(aclRule => aclRule.group).filter(isNonGuestGroup); -} - -// Returns a map of users indexed by their roles. Optionally excludes users whose ids are in -// excludeUsers. -function getUsersWithRole(groups: NonGuestGroup[], excludeUsers?: number[]): Map { - const members = new Map(); - for (const group of groups) { - let users = group.memberUsers; - if (excludeUsers) { - users = users.filter((user) => !excludeUsers.includes(user.id)); - } - members.set(group.name, users); - } - return members; -} - export async function makeDocAuthResult(docPromise: Promise): Promise { try { const doc = await docPromise; @@ -5057,3 +4437,12 @@ export function getDocAuthKeyFromScope(scope: Scope): DocAuthKey { if (!urlId) { throw new Error('document required'); } return {urlId, userId, org}; } + +// Returns whether the given group is a valid non-guest group. +function isNonGuestGroup(group: Group): group is NonGuestGroup { + return roles.isNonGuestRole(group.name); +} + +function getNonGuestGroups(entity: Organization|Workspace|Document): NonGuestGroup[] { + return (entity.aclRules as AclRule[]).map(aclRule => aclRule.group).filter(isNonGuestGroup); +} diff --git a/app/gen-server/lib/homedb/Interfaces.ts b/app/gen-server/lib/homedb/Interfaces.ts new file mode 100644 index 0000000000..0802d96ae0 --- /dev/null +++ b/app/gen-server/lib/homedb/Interfaces.ts @@ -0,0 +1,40 @@ +import { UserProfile } from "app/common/LoginSessionAPI"; +import { UserOptions } from "app/common/UserAPI"; +import * as roles from 'app/common/roles'; +import { Document } from "app/gen-server/entity/Document"; +import { Group } from "app/gen-server/entity/Group"; +import { Organization } from "app/gen-server/entity/Organization"; +import { Workspace } from "app/gen-server/entity/Workspace"; + +import { EntityManager } from "typeorm"; + +export interface QueryResult { + status: number; + data?: T; + errMessage?: string; +} + +export interface GetUserOptions { + manager?: EntityManager; + profile?: UserProfile; + userOptions?: UserOptions; +} + +export interface UserProfileChange { + name?: string; + isFirstTimeUser?: boolean; +} + +// A specification of the users available during a request. This can be a single +// user, identified by a user id, or a collection of profiles (typically drawn from +// the session). +export type AvailableUsers = number | UserProfile[]; + +export type NonGuestGroup = Group & { name: roles.NonGuestRole }; + +export type Resource = Organization|Workspace|Document; + +export type RunInTransaction = ( + transaction: EntityManager|undefined, + op: ((manager: EntityManager) => Promise) +) => Promise; diff --git a/app/gen-server/lib/homedb/UsersManager.ts b/app/gen-server/lib/homedb/UsersManager.ts new file mode 100644 index 0000000000..8c7618e9bf --- /dev/null +++ b/app/gen-server/lib/homedb/UsersManager.ts @@ -0,0 +1,761 @@ +import { ApiError } from 'app/common/ApiError'; +import { normalizeEmail } from 'app/common/emails'; +import { PERSONAL_FREE_PLAN } from 'app/common/Features'; +import { UserOrgPrefs } from 'app/common/Prefs'; +import * as roles from 'app/common/roles'; +import { + ANONYMOUS_USER_EMAIL, + EVERYONE_EMAIL, + FullUser, + PermissionDelta, + PREVIEWER_EMAIL, + UserOptions, + UserProfile +} from 'app/common/UserAPI'; +import { AclRule } from 'app/gen-server/entity/AclRule'; +import { Group } from 'app/gen-server/entity/Group'; +import { Login } from 'app/gen-server/entity/Login'; +import { User } from 'app/gen-server/entity/User'; +import { appSettings } from 'app/server/lib/AppSettings'; +import { HomeDBManager, PermissionDeltaAnalysis, Scope } from 'app/gen-server/lib/HomeDBManager'; +import { + AvailableUsers, GetUserOptions, NonGuestGroup, QueryResult, Resource, RunInTransaction, UserProfileChange +} from 'app/gen-server/lib/homedb/Interfaces'; +import { Permissions } from 'app/gen-server/lib/Permissions'; +import { Pref } from 'app/gen-server/entity/Pref'; + +import flatten from 'lodash/flatten'; +import { EntityManager } from 'typeorm'; +import moment from 'moment-timezone'; + +// A special user allowed to add/remove the EVERYONE_EMAIL to/from a resource. +export const SUPPORT_EMAIL = appSettings.section('access').flag('supportEmail').requireString({ + envVar: 'GRIST_SUPPORT_EMAIL', + defaultValue: 'support@getgrist.com', +}); + +// A list of emails we don't expect to see logins for. +const NON_LOGIN_EMAILS = [PREVIEWER_EMAIL, EVERYONE_EMAIL, ANONYMOUS_USER_EMAIL]; + +/** + * Class responsible for Users Management. + * + * It's only meant to be used by HomeDBManager. If you want to use one of its (instance or static) methods, + * please make an indirection which passes through HomeDBManager. + */ +export class UsersManager { + public static isSingleUser(users: AvailableUsers): users is number { + return typeof users === 'number'; + } + + // Returns all first-level memberUsers in the resources. Requires all resources' aclRules, groups + // and memberUsers to be populated. + // If optRoles is provided, only checks membership in resource groups with the given roles. + public static getResourceUsers(res: Resource|Resource[], optRoles?: string[]): User[] { + res = Array.isArray(res) ? res : [res]; + const users: {[uid: string]: User} = {}; + let resAcls: AclRule[] = flatten(res.map(_res => _res.aclRules as AclRule[])); + if (optRoles) { + resAcls = resAcls.filter(_acl => optRoles.includes(_acl.group.name)); + } + resAcls.forEach((aclRule: AclRule) => { + aclRule.group.memberUsers.forEach((u: User) => users[u.id] = u); + }); + const userList = Object.keys(users).map(uid => users[uid]); + userList.sort((a, b) => a.id - b.id); + return userList; + } + + // Returns a map of users indexed by their roles. Optionally excludes users whose ids are in + // excludeUsers. + public static getUsersWithRole(groups: NonGuestGroup[], excludeUsers?: number[]): Map { + const members = new Map(); + for (const group of groups) { + let users = group.memberUsers; + if (excludeUsers) { + users = users.filter((user) => !excludeUsers.includes(user.id)); + } + members.set(group.name, users); + } + return members; + } + + private _specialUserIds: {[name: string]: number} = {}; // id for anonymous user, previewer, etc + + private get _connection () { + return this._homeDb.connection; + } + + public constructor( + private readonly _homeDb: HomeDBManager, + private _runInTransaction: RunInTransaction + ) {} + + /** + * Clear all user preferences associated with the given email addresses. + * For use in tests. + */ + public async testClearUserPrefs(emails: string[]) { + return await this._connection.transaction(async manager => { + for (const email of emails) { + const user = await this.getUserByLogin(email, {manager}); + if (user) { + await manager.delete(Pref, {userId: user.id}); + } + } + }); + } + + public getSpecialUserId(key: string) { + return this._specialUserIds[key]; + } + + /** + * + * Get the id of the anonymous user. + * + */ + public getAnonymousUserId(): number { + const id = this._specialUserIds[ANONYMOUS_USER_EMAIL]; + if (!id) { throw new Error("Anonymous user not available"); } + return id; + } + + /** + * Get the id of the thumbnail user. + */ + public getPreviewerUserId(): number { + const id = this._specialUserIds[PREVIEWER_EMAIL]; + if (!id) { throw new Error("Previewer user not available"); } + return id; + } + + /** + * Get the id of the 'everyone' user. + */ + public getEveryoneUserId(): number { + const id = this._specialUserIds[EVERYONE_EMAIL]; + if (!id) { throw new Error("'everyone' user not available"); } + return id; + } + + /** + * Get the id of the 'support' user. + */ + public getSupportUserId(): number { + const id = this._specialUserIds[SUPPORT_EMAIL]; + if (!id) { throw new Error("'support' user not available"); } + return id; + } + + public async getUserByKey(apiKey: string): Promise { + // Include logins relation for Authorization convenience. + return await User.findOne({where: {apiKey}, relations: ["logins"]}) || undefined; + } + + public async getUserByRef(ref: string): Promise { + return await User.findOne({where: {ref}, relations: ["logins"]}) || undefined; + } + + public async getUser( + userId: number, + options: {includePrefs?: boolean} = {} + ): Promise { + const {includePrefs} = options; + const relations = ["logins"]; + if (includePrefs) { relations.push("prefs"); } + return await User.findOne({where: {id: userId}, relations}) || undefined; + } + + public async getFullUser(userId: number): Promise { + const user = await User.findOne({where: {id: userId}, relations: ["logins"]}); + if (!user) { throw new ApiError("unable to find user", 400); } + return this.makeFullUser(user); + } + + /** + * Convert a user record into the format specified in api. + */ + public makeFullUser(user: User): FullUser { + if (!user.logins?.[0]?.displayEmail) { + throw new ApiError("unable to find mandatory user email", 400); + } + const displayEmail = user.logins[0].displayEmail; + const loginEmail = user.loginEmail; + const result: FullUser = { + id: user.id, + email: displayEmail, + // Only include loginEmail when it's different, to avoid overhead when FullUser is sent + // around, and also to avoid updating too many tests. + loginEmail: loginEmail !== displayEmail ? loginEmail : undefined, + name: user.name, + picture: user.picture, + ref: user.ref, + locale: user.options?.locale, + prefs: user.prefs?.find((p)=> p.orgId === null)?.prefs, + }; + if (this.getAnonymousUserId() === user.id) { + result.anonymous = true; + } + if (this.getSupportUserId() === user.id) { + result.isSupport = true; + } + return result; + } + + /** + * Ensures that user with external id exists and updates its profile and email if necessary. + * + * @param profile External profile + */ + public async ensureExternalUser(profile: UserProfile) { + await this._connection.transaction(async manager => { + // First find user by the connectId from the profile + const existing = await manager.findOne(User, { + where: {connectId: profile.connectId || undefined}, + relations: ["logins"], + }); + + // If a user does not exist, create it with data from the external profile. + if (!existing) { + const newUser = await this.getUserByLoginWithRetry(profile.email, { + profile, + manager + }); + if (!newUser) { + throw new ApiError("Unable to create user", 500); + } + // No need to survey this user. + newUser.isFirstTimeUser = false; + await newUser.save(); + } else { + // Else update profile and login information from external profile. + let updated = false; + let login: Login = existing.logins[0]!; + const properEmail = normalizeEmail(profile.email); + + if (properEmail !== existing.loginEmail) { + login = login ?? new Login(); + login.email = properEmail; + login.displayEmail = profile.email; + existing.logins.splice(0, 1, login); + login.user = existing; + updated = true; + } + + if (profile?.name && profile?.name !== existing.name) { + existing.name = profile.name; + updated = true; + } + + if (profile?.picture && profile?.picture !== existing.picture) { + existing.picture = profile.picture; + updated = true; + } + + if (updated) { + await manager.save([existing, login]); + } + } + }); + } + + public async updateUser(userId: number, props: UserProfileChange) { + let isWelcomed: boolean = false; + let user: User|null = null; + await this._connection.transaction(async manager => { + user = await manager.findOne(User, {relations: ['logins'], + where: {id: userId}}); + let needsSave = false; + if (!user) { throw new ApiError("unable to find user", 400); } + if (props.name && props.name !== user.name) { + user.name = props.name; + needsSave = true; + } + if (props.isFirstTimeUser !== undefined && props.isFirstTimeUser !== user.isFirstTimeUser) { + user.isFirstTimeUser = props.isFirstTimeUser; + needsSave = true; + // If we are turning off the isFirstTimeUser flag, then right + // after this transaction commits is a great time to trigger + // any automation for first logins + if (!props.isFirstTimeUser) { isWelcomed = true; } + } + if (needsSave) { + await user.save(); + } + }); + return { user, isWelcomed }; + } + + public async updateUserName(userId: number, name: string) { + const user = await User.findOne({where: {id: userId}}); + if (!user) { throw new ApiError("unable to find user", 400); } + user.name = name; + await user.save(); + } + + public async updateUserOptions(userId: number, props: Partial) { + const user = await User.findOne({where: {id: userId}}); + if (!user) { throw new ApiError("unable to find user", 400); } + + const newOptions = {...(user.options ?? {}), ...props}; + user.options = newOptions; + await user.save(); + } + + /** + * Get the anonymous user, as a constructed object rather than a database lookup. + */ + public getAnonymousUser(): User { + const user = new User(); + user.id = this.getAnonymousUserId(); + user.name = "Anonymous"; + user.isFirstTimeUser = false; + const login = new Login(); + login.displayEmail = login.email = ANONYMOUS_USER_EMAIL; + user.logins = [login]; + user.ref = ''; + return user; + } + + // Fetch user from login, creating the user if previously unseen, allowing one retry + // for an email key conflict failure. This is in case our transaction conflicts with a peer + // doing the same thing. This is quite likely if the first page visited by a previously + // unseen user fires off multiple api calls. + public async getUserByLoginWithRetry(email: string, options: GetUserOptions = {}): Promise { + try { + return await this.getUserByLogin(email, options); + } catch (e) { + if (e.name === 'QueryFailedError' && e.detail && + e.detail.match(/Key \(email\)=[^ ]+ already exists/)) { + // This is a postgres-specific error message. This problem cannot arise in sqlite, + // because we have to serialize sqlite transactions in any case to get around a typeorm + // limitation. + return await this.getUserByLogin(email, options); + } + throw e; + } + } + + /** + * Find a user by email. Don't create the user if it doesn't already exist. + */ + public async getExistingUserByLogin( + email: string, + manager?: EntityManager + ): Promise { + const normalizedEmail = normalizeEmail(email); + return await (manager || this._connection).createQueryBuilder() + .select('user') + .from(User, 'user') + .leftJoinAndSelect('user.logins', 'logins') + .where('email = :email', {email: normalizedEmail}) + .getOne() || undefined; + } + + /** + * + * Fetches a user record based on an email address. If a user record already + * exists linked to the email address supplied, that is the record returned. + * Otherwise a fresh record is created, linked to the supplied email address. + * The supplied `options` are used when creating a fresh record, or updating + * unset/outdated fields of an existing record. + * + */ + public async getUserByLogin(email: string, options: GetUserOptions = {}): Promise { + const {manager: transaction, profile, userOptions} = options; + const normalizedEmail = normalizeEmail(email); + const userByLogin = await this._runInTransaction(transaction, async manager => { + let needUpdate = false; + const userQuery = manager.createQueryBuilder() + .select('user') + .from(User, 'user') + .leftJoinAndSelect('user.logins', 'logins') + .leftJoinAndSelect('user.personalOrg', 'personalOrg') + .where('email = :email', {email: normalizedEmail}); + let user = await userQuery.getOne(); + let login: Login; + if (!user) { + user = new User(); + // Special users do not have first time user set so that they don't get redirected to the + // welcome page. + user.isFirstTimeUser = !NON_LOGIN_EMAILS.includes(normalizedEmail); + login = new Login(); + login.email = normalizedEmail; + login.user = user; + needUpdate = true; + } else { + login = user.logins[0]; + } + + // Check that user and login records are up to date. + if (!user.name) { + // Set the user's name if our provider knows it. Otherwise use their username + // from email, for lack of something better. If we don't have a profile at this + // time, then leave the name blank in the hopes of learning it when the user logs in. + user.name = (profile && (profile.name || email.split('@')[0])) || ''; + needUpdate = true; + } + if (profile && !user.firstLoginAt) { + // set first login time to now (remove milliseconds for compatibility with other + // timestamps in db set by typeorm, and since second level precision is fine) + const nowish = new Date(); + nowish.setMilliseconds(0); + user.firstLoginAt = nowish; + needUpdate = true; + } + if (!user.picture && profile && profile.picture) { + // Set the user's profile picture if our provider knows it. + user.picture = profile.picture; + needUpdate = true; + } + if (profile && profile.email && profile.email !== login.displayEmail) { + // Use provider's version of email address for display. + login.displayEmail = profile.email; + needUpdate = true; + } + + if (profile?.connectId && profile?.connectId !== user.connectId) { + user.connectId = profile.connectId; + needUpdate = true; + } + + if (!login.displayEmail) { + // Save some kind of display email if we don't have anything at all for it yet. + // This could be coming from how someone wrote it in a UserManager dialog, for + // instance. It will get overwritten when the user logs in if the provider's + // version is different. + login.displayEmail = email; + needUpdate = true; + } + if (!user.options?.authSubject && userOptions?.authSubject) { + // Link subject from password-based authentication provider if not previously linked. + user.options = {...(user.options ?? {}), authSubject: userOptions.authSubject}; + needUpdate = true; + } + const today = moment().startOf('day'); + if (today !== moment(user.lastConnectionAt).startOf('day')) { + user.lastConnectionAt = today.toDate(); + needUpdate = true; + } + if (needUpdate) { + login.user = user; + await manager.save([user, login]); + } + if (!user.personalOrg && !NON_LOGIN_EMAILS.includes(login.email)) { + // Add a personal organization for this user. + // We don't add a personal org for anonymous/everyone/previewer "users" as it could + // get a bit confusing. + const result = await this._homeDb.addOrg(user, {name: "Personal"}, { + setUserAsOwner: true, + useNewPlan: true, + product: PERSONAL_FREE_PLAN, + }, manager); + if (result.status !== 200) { + throw new Error(result.errMessage); + } + needUpdate = true; + + // We just created a personal org; set userOrgPrefs that should apply for new users only. + const userOrgPrefs: UserOrgPrefs = {showGristTour: true}; + const orgId = result.data; + if (orgId) { + await this._homeDb.updateOrg({userId: user.id}, orgId, {userOrgPrefs}, manager); + } + } + if (needUpdate) { + // We changed the db - reload user in order to give consistent results. + // In principle this could be optimized, but this is simpler to maintain. + user = await userQuery.getOne(); + } + return user; + }); + return userByLogin; + } + + /** + * Deletes a user from the database. For the moment, the only person with the right + * to delete a user is the user themselves. + * Users have logins, a personal org, and entries in the group_users table. All are + * removed together in a transaction. All material in the personal org will be lost. + * + * @param scope: request scope, including the id of the user initiating this action + * @param userIdToDelete: the id of the user to delete from the database + * @param name: optional cross-check, delete only if user name matches this + */ + public async deleteUser(scope: Scope, userIdToDelete: number, + name?: string): Promise> { + const userIdDeleting = scope.userId; + if (userIdDeleting !== userIdToDelete) { + throw new ApiError('not permitted to delete this user', 403); + } + await this._connection.transaction(async manager => { + const user = await manager.findOne(User, {where: {id: userIdToDelete}, + relations: ["logins", "personalOrg", "prefs"]}); + if (!user) { throw new ApiError('user not found', 404); } + if (name) { + if (user.name !== name) { + throw new ApiError(`user name did not match ('${name}' vs '${user.name}')`, 400); + } + } + if (user.personalOrg) { await this._homeDb.deleteOrg(scope, user.personalOrg.id, manager); } + await manager.remove([...user.logins]); + // We don't have a GroupUser entity, and adding one tickles lots of TypeOrm quirkiness, + // so use a plain query to delete entries in the group_users table. + await manager.createQueryBuilder() + .delete() + .from('group_users') + .where('user_id = :userId', {userId: userIdToDelete}) + .execute(); + + await manager.delete(User, userIdToDelete); + }); + return { + status: 200 + }; + } + + // Looks up the emails in the permission delta and adds them to the users map in + // the delta object. + // Returns a QueryResult based on the validity of the passed in PermissionDelta object. + public async verifyAndLookupDeltaEmails( + userId: number, + delta: PermissionDelta, + isOrg: boolean = false, + transaction?: EntityManager + ): Promise { + if (!delta) { + throw new ApiError('Bad request: missing permission delta', 400); + } + this._mergeIndistinguishableEmails(delta); + const hasInherit = 'maxInheritedRole' in delta; + const hasUsers = delta.users; // allow zero actual changes; useful to reduce special + // cases in scripts + if ((isOrg && (hasInherit || !hasUsers)) || (!isOrg && !hasInherit && !hasUsers)) { + throw new ApiError('Bad request: invalid permission delta', 400); + } + // Lookup the email access changes and move them to the users object. + const userIdMap: {[userId: string]: roles.NonGuestRole|null} = {}; + if (hasInherit) { + // Verify maxInheritedRole + const role = delta.maxInheritedRole; + const validRoles = new Set(this._homeDb.defaultBasicGroupNames); + if (role && !validRoles.has(role)) { + throw new ApiError(`Invalid maxInheritedRole ${role}`, 400); + } + } + if (delta.users) { + // Verify roles + const deltaRoles = Object.keys(delta.users).map(_userId => delta.users![_userId]); + // Cannot set role "members" on workspace/doc. + const validRoles = new Set(isOrg ? this._homeDb.defaultNonGuestGroupNames : this._homeDb.defaultBasicGroupNames); + for (const role of deltaRoles) { + if (role && !validRoles.has(role)) { + throw new ApiError(`Invalid user role ${role}`, 400); + } + } + // Lookup emails + const emailMap = delta.users; + const emails = Object.keys(emailMap); + const emailUsers = await Promise.all( + emails.map(async email => await this.getUserByLogin(email, {manager: transaction})) + ); + emails.forEach((email, i) => { + const userIdAffected = emailUsers[i]!.id; + // Org-level sharing with everyone would allow serious spamming - forbid it. + if (emailMap[email] !== null && // allow removing anything + userId !== this.getSupportUserId() && // allow support user latitude + userIdAffected === this.getEveryoneUserId() && + isOrg) { + throw new ApiError('This user cannot share with everyone at top level', 403); + } + userIdMap[userIdAffected] = emailMap[email]; + }); + } + const userIdDelta = delta.users ? userIdMap : null; + const userIds = Object.keys(userIdDelta || {}); + const removingSelf = userIds.length === 1 && userIds[0] === String(userId) && + delta.maxInheritedRole === undefined && userIdDelta?.[userId] === null; + const permissionThreshold = removingSelf ? Permissions.VIEW : Permissions.ACL_EDIT; + return { + userIdDelta, + permissionThreshold, + affectsSelf: userId in userIdMap, + }; + } + + public async initializeSpecialIds(): Promise { + await this._maybeCreateSpecialUserId({ + email: ANONYMOUS_USER_EMAIL, + name: "Anonymous" + }); + await this._maybeCreateSpecialUserId({ + email: PREVIEWER_EMAIL, + name: "Preview" + }); + await this._maybeCreateSpecialUserId({ + email: EVERYONE_EMAIL, + name: "Everyone" + }); + await this._maybeCreateSpecialUserId({ + email: SUPPORT_EMAIL, + name: "Support" + }); + } + + /** + * Check for anonymous user, either encoded directly as an id, or as a singular + * profile (this case arises during processing of the session/access/all endpoint + * whether we are checking for available orgs without committing yet to a particular + * choice of user). + */ + public isAnonymousUser(users: AvailableUsers): boolean { + return UsersManager.isSingleUser(users) ? users === this.getAnonymousUserId() : + users.length === 1 && normalizeEmail(users[0].email) === ANONYMOUS_USER_EMAIL; + } + + /** + * Get ids of users to be excluded from member counts and emails. + */ + public getExcludedUserIds(): number[] { + return [this.getSupportUserId(), this.getAnonymousUserId(), this.getEveryoneUserId()]; + } + + /** + * Returns a Promise for an array of User entites for the given userIds. + */ + public async getUsers(userIds: number[], optManager?: EntityManager): Promise { + if (userIds.length === 0) { + return []; + } + const manager = optManager || new EntityManager(this._connection); + const queryBuilder = manager.createQueryBuilder() + .select('users') + .from(User, 'users') + .where('users.id IN (:...userIds)', {userIds}); + return await queryBuilder.getMany(); + } + + /** + * Don't add everyone@ as a guest, unless also sharing with anon@. + * This means that material shared with everyone@ doesn't become + * listable/discoverable by default. + * + * This is a HACK to allow existing example doc setup to continue to + * work. It could be removed if we are willing to share the entire + * support org with users. E.g. move any material we don't want to + * share into a workspace that doesn't inherit ACLs. TODO: remove + * this hack, or enhance it up as a way to support discoverability / + * listing. It has the advantage of cloning well. + */ + public filterEveryone(users: User[]): User[] { + const everyone = this.getEveryoneUserId(); + const anon = this.getAnonymousUserId(); + if (users.find(u => u.id === anon)) { return users; } + return users.filter(u => u.id !== everyone); + } + + // Given two arrays of groups, returns a map of users present in the first array but + // not the second, where the map is broken down by user role. + // This method is used for checking limits on shares. + // Excluded users are removed from the results. + public getUserDifference(groupsA: Group[], groupsB: Group[]): Map { + const subtractSet: Set = + new Set(flatten(groupsB.map(grp => grp.memberUsers)).map(usr => usr.id)); + const result = new Map(); + for (const group of groupsA) { + const name = group.name; + if (!roles.isNonGuestRole(name)) { continue; } + result.set(name, group.memberUsers.filter(user => !subtractSet.has(user.id))); + } + return this.withoutExcludedUsers(result); + } + + public withoutExcludedUsers(members: Map): Map { + const excludedUsers = this.getExcludedUserIds(); + for (const [role, users] of members.entries()) { + members.set(role, users.filter((user) => !excludedUsers.includes(user.id))); + } + return members; + } + + /** + * + * Take a list of user profiles coming from the client's session, correlate + * them with Users and Logins in the database, and construct full profiles + * with user ids, standardized display emails, pictures, and anonymous flags. + * + */ + public async completeProfiles(profiles: UserProfile[]): Promise { + if (profiles.length === 0) { return []; } + const qb = this._connection.createQueryBuilder() + .select('logins') + .from(Login, 'logins') + .leftJoinAndSelect('logins.user', 'user') + .where('logins.email in (:...emails)', {emails: profiles.map(profile => normalizeEmail(profile.email))}); + const completedProfiles: {[email: string]: FullUser} = {}; + for (const login of await qb.getMany()) { + completedProfiles[login.email] = { + id: login.user.id, + email: login.displayEmail, + name: login.user.name, + picture: login.user.picture, + anonymous: login.user.id === this.getAnonymousUserId(), + locale: login.user.options?.locale + }; + } + return profiles.map(profile => completedProfiles[normalizeEmail(profile.email)]) + .filter(profile => profile); + } + + // For the moment only the support user can add both everyone@ and anon@ to a + // resource, since that allows spam. TODO: enhance or remove. + public checkUserChangeAllowed(userId: number, groups: Group[]) { + if (userId === this.getSupportUserId()) { return; } + const ids = new Set(flatten(groups.map(g => g.memberUsers)).map(u => u.id)); + if (ids.has(this.getEveryoneUserId()) && ids.has(this.getAnonymousUserId())) { + throw new Error('this user cannot share with everyone and anonymous'); + } + } + + /** + * + * Get the id of a special user, creating that user if it is not already present. + * + */ + private async _maybeCreateSpecialUserId(profile: UserProfile) { + let id = this._specialUserIds[profile.email]; + if (!id) { + // get or create user - with retry, since there'll be a race to create the + // user if a bunch of servers start simultaneously and the user doesn't exist + // yet. + const user = await this.getUserByLoginWithRetry(profile.email, {profile}); + if (user) { id = this._specialUserIds[profile.email] = user.id; } + } + if (!id) { throw new Error(`Could not find or create user ${profile.email}`); } + return id; + } + + // This deals with the problem posed by receiving a PermissionDelta specifying a + // role for both alice@x and Alice@x. We do not distinguish between such emails. + // If there are multiple indistinguishabe emails, we preserve just one of them, + // assigning it the most powerful permission specified. The email variant perserved + // is the earliest alphabetically. + private _mergeIndistinguishableEmails(delta: PermissionDelta) { + if (!delta.users) { return; } + // We normalize emails for comparison, but track how they were capitalized + // in order to preserve it. This is worth doing since for the common case + // of a user being added to a resource prior to ever logging in, their + // displayEmail will be seeded from this value. + const displayEmails: {[email: string]: string} = {}; + // This will be our output. + const users: {[email: string]: roles.NonGuestRole|null} = {}; + for (const displayEmail of Object.keys(delta.users).sort()) { + const email = normalizeEmail(displayEmail); + const role = delta.users[displayEmail]; + const key = displayEmails[email] = displayEmails[email] || displayEmail; + users[key] = users[key] ? roles.getStrongestRole(users[key], role) : role; + } + delta.users = users; + } +} From 1081d63fa70a795dd5ef655fb7f8533074cc2710 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20Guti=C3=A9rrez=20Hermoso?= Date: Thu, 13 Jun 2024 17:45:41 -0400 Subject: [PATCH 56/62] supervisor: new file This is a new entrypoint, mostly intended for Docker, so we have one simple process controlling the main Grist process. The purpose of this is to be able to make Grist easily restartable with a new environment. --- sandbox/supervisor.mjs | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 sandbox/supervisor.mjs diff --git a/sandbox/supervisor.mjs b/sandbox/supervisor.mjs new file mode 100644 index 0000000000..2508cb1705 --- /dev/null +++ b/sandbox/supervisor.mjs @@ -0,0 +1,35 @@ +import {spawn} from 'child_process'; + +let grist; + +function startGrist(newConfig={}) { + saveNewConfig(newConfig); + // H/T https://stackoverflow.com/a/36995148/11352427 + grist = spawn('./sandbox/run.sh', { + stdio: ['inherit', 'inherit', 'inherit', 'ipc'] + }); + grist.on('message', function(data) { + if (data.action === 'restart') { + console.log('Restarting Grist with new environment'); + + // Note that we only set this event handler here, after we have + // a new environment to reload with. Small chance of a race here + // in case something else sends a SIGINT before we do it + // ourselves further below. + grist.on('exit', () => { + grist = startGrist(data.newConfig); + }); + + grist.kill('SIGINT'); + } + }); + return grist; +} + +// Stub function +function saveNewConfig(newConfig) { + // TODO: something here to actually persist the new config before + // restarting Grist. +} + +startGrist(); From ec55ec04afb1f628f6c68abc94b27d3e1e22ad50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20Guti=C3=A9rrez=20Hermoso?= Date: Fri, 7 Jun 2024 20:07:19 -0400 Subject: [PATCH 57/62] FlexServer: add new admin restart endpoint This adds an endpoint for the admin user to be able to signal to a controlling process to restart the server. This is intended for `docker-runner.mjs`. --- app/server/lib/FlexServer.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/server/lib/FlexServer.ts b/app/server/lib/FlexServer.ts index 46e4a50847..5aed483d0a 100644 --- a/app/server/lib/FlexServer.ts +++ b/app/server/lib/FlexServer.ts @@ -1883,6 +1883,22 @@ export class FlexServer implements GristServer { const probes = new BootProbes(this.app, this, '/api', adminMiddleware); probes.addEndpoints(); + this.app.post('/api/admin/restart', requireInstallAdmin, expressWrap(async (req, resp) => { + const newConfig = req.body.newConfig; + resp.on('finish', () => { + // If we have IPC with parent process (e.g. when running under + // Docker) tell the parent that we have a new environment so it + // can restart us. + if (process.send) { + process.send({ action: 'restart', newConfig }); + } + }); + // On the topic of http response codes, thus spake MDN: + // "409: This response is sent when a request conflicts with the current state of the server." + const status = process.send ? 200 : 409; + return resp.status(status).send(); + })); + // Restrict this endpoint to install admins this.app.get('/api/install/prefs', requireInstallAdmin, expressWrap(async (_req, resp) => { const activation = await this._activations.current(); From 8fab3aa1f431915d80d6428e6f9bfa7a4dfe250c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20Guti=C3=A9rrez=20Hermoso?= Date: Thu, 13 Jun 2024 17:45:47 -0400 Subject: [PATCH 58/62] Dockerfile: use docker-runner.mjs as new entrypoint --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f6cafa437b..4af861cc33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -113,6 +113,7 @@ ADD bower_components /grist/bower_components ADD sandbox /grist/sandbox ADD plugins /grist/plugins ADD static /grist/static +ADD docker-runner.mjs /grist/docker-runner.mjs # Make optional pyodide sandbox available COPY --from=builder /grist/sandbox/pyodide /grist/sandbox/pyodide @@ -152,4 +153,4 @@ ENV \ EXPOSE 8484 ENTRYPOINT ["/usr/bin/tini", "-s", "--"] -CMD ["./sandbox/run.sh"] +CMD ["node", "./sandbox/supervisor.mjs"] From 015dc5ed881d2d04891664827f7fefcf8557a142 Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Thu, 20 Jun 2024 15:14:51 +0200 Subject: [PATCH 59/62] fix: use only needed columns on db query --- .../migration/1663851423064-UserUUID.ts | 12 ++++++--- .../migration/1664528376930-UserRefUnique.ts | 10 +++++--- app/gen-server/sqlUtils.ts | 25 ------------------- 3 files changed, 16 insertions(+), 31 deletions(-) diff --git a/app/gen-server/migration/1663851423064-UserUUID.ts b/app/gen-server/migration/1663851423064-UserUUID.ts index eb6df46361..35c505239f 100644 --- a/app/gen-server/migration/1663851423064-UserUUID.ts +++ b/app/gen-server/migration/1663851423064-UserUUID.ts @@ -1,5 +1,5 @@ -import {addRefToUserList} from "app/gen-server/sqlUtils"; +import {makeId} from 'app/server/lib/idUtils'; import {MigrationInterface, QueryRunner, TableColumn} from "typeorm"; export class UserUUID1663851423064 implements MigrationInterface { @@ -13,8 +13,14 @@ export class UserUUID1663851423064 implements MigrationInterface { isUnique: false, })); - const userList = await queryRunner.query("SELECT * FROM users;"); - await addRefToUserList(queryRunner, userList); + // Updating so many rows in a multiple queries is not ideal. We will send updates in chunks. + // 300 seems to be a good number, for 24k rows we have 80 queries. + const userList = await queryRunner.manager.createQueryBuilder() + .select(["users.id", "users.ref"]) + .from("users", "users") + .getMany(); + userList.forEach(u => u.ref = makeId()); + await queryRunner.manager.save(userList, { chunk: 300 }); // We are not making this column unique yet, because it can fail // if there are some old workers still running, and any new user diff --git a/app/gen-server/migration/1664528376930-UserRefUnique.ts b/app/gen-server/migration/1664528376930-UserRefUnique.ts index 0aa74c716a..338b60cbc1 100644 --- a/app/gen-server/migration/1664528376930-UserRefUnique.ts +++ b/app/gen-server/migration/1664528376930-UserRefUnique.ts @@ -1,4 +1,4 @@ -import {addRefToUserList} from "app/gen-server/sqlUtils"; +import {makeId} from 'app/server/lib/idUtils'; import {MigrationInterface, QueryRunner} from "typeorm"; export class UserRefUnique1664528376930 implements MigrationInterface { @@ -7,8 +7,12 @@ export class UserRefUnique1664528376930 implements MigrationInterface { // the ref column unique. // Update users that don't have unique ref set. - const userList = await queryRunner.query("SELECT * FROM users WHERE ref is null"); - await addRefToUserList(queryRunner, userList); + const userList = await queryRunner.manager.createQueryBuilder() + .select(["users.id", "users.ref"]) + .from("users", "users") + .getMany(); + userList.forEach(u => u.ref = makeId()); + await queryRunner.manager.save(userList, { chunk: 300 }); // Mark column as unique and non-nullable. const users = (await queryRunner.getTable('users'))!; diff --git a/app/gen-server/sqlUtils.ts b/app/gen-server/sqlUtils.ts index 7513dee13b..6108642fb7 100644 --- a/app/gen-server/sqlUtils.ts +++ b/app/gen-server/sqlUtils.ts @@ -1,5 +1,3 @@ -import {makeId} from 'app/server/lib/idUtils'; -import {chunk} from 'lodash'; import {DatabaseType, QueryRunner, SelectQueryBuilder} from 'typeorm'; import {RelationCountLoader} from 'typeorm/query-builder/relation-count/RelationCountLoader'; import {RelationIdLoader} from 'typeorm/query-builder/relation-id/RelationIdLoader'; @@ -127,29 +125,6 @@ export function datetime(dbType: DatabaseType) { } } -export async function addRefToUserList(queryRunner: QueryRunner, userList: any[]){ - const dbType = queryRunner.connection.driver.options.type; - - // Updating so many rows in a multiple queries is not ideal. We will send updates in chunks. - // 300 seems to be a good number, for 24k rows we have 80 queries. - const userChunks = chunk(userList, 300); - for (const users of userChunks) { - await queryRunner.connection.transaction(async manager => { - const queries = users.map((user: any, _index: number, _array: any[]) => { - switch (dbType) { - case 'postgres': - return manager.query(`UPDATE users SET ref = $1 WHERE id = $2`, [makeId(), user.id]); - case 'sqlite': - return manager.query(`UPDATE users SET ref = ? WHERE id = ?`, [makeId(), user.id]); - default: - throw new Error(`addRefToUserList not implemented for ${dbType}`); - } - }); - await Promise.all(queries); - }); - } -} - /** * * Generate SQL code from one QueryBuilder, get the "raw" results, and then decode From 1fa3fb4cd6bb9e18db0fdb6057f808f784dbe4d2 Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Thu, 20 Jun 2024 15:55:12 +0200 Subject: [PATCH 60/62] fix: check if lastConnectionAt exist and use moment isSame function for comparasion --- app/gen-server/lib/homedb/UsersManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/gen-server/lib/homedb/UsersManager.ts b/app/gen-server/lib/homedb/UsersManager.ts index 8c7618e9bf..6c60c1ed4e 100644 --- a/app/gen-server/lib/homedb/UsersManager.ts +++ b/app/gen-server/lib/homedb/UsersManager.ts @@ -434,7 +434,7 @@ export class UsersManager { needUpdate = true; } const today = moment().startOf('day'); - if (today !== moment(user.lastConnectionAt).startOf('day')) { + if (!user.lastConnectionAt || !today.isSame(moment(user.lastConnectionAt).startOf('day'))) { user.lastConnectionAt = today.toDate(); needUpdate = true; } From 4ffb74983fb63d2ba2bcfc9f57c8965b4654f45a Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Thu, 20 Jun 2024 17:52:14 +0200 Subject: [PATCH 61/62] fix: use update instead of save function to avoid using user entity --- app/gen-server/migration/1663851423064-UserUUID.ts | 12 +++++++++++- .../migration/1664528376930-UserRefUnique.ts | 13 ++++++++++++- test/gen-server/migrations.ts | 5 ++--- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/app/gen-server/migration/1663851423064-UserUUID.ts b/app/gen-server/migration/1663851423064-UserUUID.ts index 35c505239f..4001d6bb8f 100644 --- a/app/gen-server/migration/1663851423064-UserUUID.ts +++ b/app/gen-server/migration/1663851423064-UserUUID.ts @@ -1,5 +1,6 @@ import {makeId} from 'app/server/lib/idUtils'; +import {chunk} from 'lodash'; import {MigrationInterface, QueryRunner, TableColumn} from "typeorm"; export class UserUUID1663851423064 implements MigrationInterface { @@ -20,7 +21,16 @@ export class UserUUID1663851423064 implements MigrationInterface { .from("users", "users") .getMany(); userList.forEach(u => u.ref = makeId()); - await queryRunner.manager.save(userList, { chunk: 300 }); + + const userChunks = chunk(userList, 300); + for (const users of userChunks) { + await queryRunner.connection.transaction(async manager => { + const queries = users.map((user: any, _index: number, _array: any[]) => { + return queryRunner.manager.update("users", user.id, user); + }); + await Promise.all(queries); + }); + } // We are not making this column unique yet, because it can fail // if there are some old workers still running, and any new user diff --git a/app/gen-server/migration/1664528376930-UserRefUnique.ts b/app/gen-server/migration/1664528376930-UserRefUnique.ts index 338b60cbc1..149be01ee5 100644 --- a/app/gen-server/migration/1664528376930-UserRefUnique.ts +++ b/app/gen-server/migration/1664528376930-UserRefUnique.ts @@ -1,4 +1,5 @@ import {makeId} from 'app/server/lib/idUtils'; +import {chunk} from 'lodash'; import {MigrationInterface, QueryRunner} from "typeorm"; export class UserRefUnique1664528376930 implements MigrationInterface { @@ -10,9 +11,19 @@ export class UserRefUnique1664528376930 implements MigrationInterface { const userList = await queryRunner.manager.createQueryBuilder() .select(["users.id", "users.ref"]) .from("users", "users") + .where("users.ref is null") .getMany(); userList.forEach(u => u.ref = makeId()); - await queryRunner.manager.save(userList, { chunk: 300 }); + + const userChunks = chunk(userList, 300); + for (const users of userChunks) { + await queryRunner.connection.transaction(async manager => { + const queries = users.map((user: any, _index: number, _array: any[]) => { + return queryRunner.manager.update("users", user.id, user); + }); + await Promise.all(queries); + }); + } // Mark column as unique and non-nullable. const users = (await queryRunner.getTable('users'))!; diff --git a/test/gen-server/migrations.ts b/test/gen-server/migrations.ts index afaf149a1b..e6a45b9862 100644 --- a/test/gen-server/migrations.ts +++ b/test/gen-server/migrations.ts @@ -44,7 +44,6 @@ import {Shares1701557445716 as Shares} from 'app/gen-server/migration/1701557445 import {Billing1711557445716 as BillingFeatures} from 'app/gen-server/migration/1711557445716-Billing'; import {UserLastConnection1713186031023 as UserLastConnection} from 'app/gen-server/migration/1713186031023-UserLastConnection'; -import { User } from "app/gen-server/entity/User"; const home: HomeDBManager = new HomeDBManager(); @@ -135,8 +134,8 @@ describe('migrations', function() { // Check that all refs are unique const userList = await runner.manager.createQueryBuilder() - .select("users") - .from(User, "users") + .select(["users.id", "users.ref"]) + .from("users", "users") .getMany(); const setOfUserRefs = new Set(userList.map(u => u.ref)); assert.equal(nbUsersToCreate, userList.length); From fc6de6bdc4c4ef3f6e5eeb51d14a5f34c86ff6a6 Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Fri, 28 Jun 2024 17:33:41 +0200 Subject: [PATCH 62/62] fix: use datetime and timestamp for lastConnectionAt --- app/gen-server/lib/homedb/UsersManager.ts | 29 +++++++++++-------- .../migration/1663851423064-UserUUID.ts | 1 - .../1713186031023-UserLastConnection.ts | 4 ++- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/app/gen-server/lib/homedb/UsersManager.ts b/app/gen-server/lib/homedb/UsersManager.ts index 6c60c1ed4e..168665f354 100644 --- a/app/gen-server/lib/homedb/UsersManager.ts +++ b/app/gen-server/lib/homedb/UsersManager.ts @@ -26,7 +26,6 @@ import { Pref } from 'app/gen-server/entity/Pref'; import flatten from 'lodash/flatten'; import { EntityManager } from 'typeorm'; -import moment from 'moment-timezone'; // A special user allowed to add/remove the EVERYONE_EMAIL to/from a resource. export const SUPPORT_EMAIL = appSettings.section('access').flag('supportEmail').requireString({ @@ -396,14 +395,6 @@ export class UsersManager { user.name = (profile && (profile.name || email.split('@')[0])) || ''; needUpdate = true; } - if (profile && !user.firstLoginAt) { - // set first login time to now (remove milliseconds for compatibility with other - // timestamps in db set by typeorm, and since second level precision is fine) - const nowish = new Date(); - nowish.setMilliseconds(0); - user.firstLoginAt = nowish; - needUpdate = true; - } if (!user.picture && profile && profile.picture) { // Set the user's profile picture if our provider knows it. user.picture = profile.picture; @@ -433,9 +424,23 @@ export class UsersManager { user.options = {...(user.options ?? {}), authSubject: userOptions.authSubject}; needUpdate = true; } - const today = moment().startOf('day'); - if (!user.lastConnectionAt || !today.isSame(moment(user.lastConnectionAt).startOf('day'))) { - user.lastConnectionAt = today.toDate(); + + // get date of now (remove milliseconds for compatibility with other + // timestamps in db set by typeorm, and since second level precision is fine) + const nowish = new Date(); + nowish.setMilliseconds(0); + if (profile && !user.firstLoginAt) { + // set first login time to now + user.firstLoginAt = nowish; + needUpdate = true; + } + const getTimestampStartOfDay = (date: Date) => { + const timestamp = Math.floor(date.getTime() / 1000); // unix timestamp seconds from epoc + const startOfDay = timestamp - (timestamp % 86400 /*24h*/); // start of a day in seconds since epoc + return startOfDay; + }; + if (!user.lastConnectionAt || getTimestampStartOfDay(user.lastConnectionAt) !== getTimestampStartOfDay(nowish)) { + user.lastConnectionAt = nowish; needUpdate = true; } if (needUpdate) { diff --git a/app/gen-server/migration/1663851423064-UserUUID.ts b/app/gen-server/migration/1663851423064-UserUUID.ts index 4001d6bb8f..60c8666829 100644 --- a/app/gen-server/migration/1663851423064-UserUUID.ts +++ b/app/gen-server/migration/1663851423064-UserUUID.ts @@ -1,4 +1,3 @@ - import {makeId} from 'app/server/lib/idUtils'; import {chunk} from 'lodash'; import {MigrationInterface, QueryRunner, TableColumn} from "typeorm"; diff --git a/app/gen-server/migration/1713186031023-UserLastConnection.ts b/app/gen-server/migration/1713186031023-UserLastConnection.ts index ee522ff4c2..52310a3898 100644 --- a/app/gen-server/migration/1713186031023-UserLastConnection.ts +++ b/app/gen-server/migration/1713186031023-UserLastConnection.ts @@ -3,9 +3,11 @@ import {MigrationInterface, QueryRunner, TableColumn} from 'typeorm'; export class UserLastConnection1713186031023 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { + const sqlite = queryRunner.connection.driver.options.type === 'sqlite'; + const datetime = sqlite ? "datetime" : "timestamp with time zone"; await queryRunner.addColumn('users', new TableColumn({ name: 'last_connection_at', - type: "date", + type: datetime, isNullable: true })); }