From f164c65983ce2bdcc636f5cb06e0a6883fbfdf1e Mon Sep 17 00:00:00 2001 From: Florent FAYOLLE Date: Tue, 10 Oct 2023 15:57:01 +0200 Subject: [PATCH 1/4] Option to print col id as headers instead of their name in CSV #688 --- app/server/lib/Export.ts | 3 +++ app/server/lib/ExportCSV.ts | 19 +++++++------ test/server/lib/DocApi.ts | 53 ++++++++++++++++++++++++++++--------- 3 files changed, 54 insertions(+), 21 deletions(-) diff --git a/app/server/lib/Export.ts b/app/server/lib/Export.ts index 0d590f2076..cdcd0b29b5 100644 --- a/app/server/lib/Export.ts +++ b/app/server/lib/Export.ts @@ -99,6 +99,7 @@ export interface ExportParameters { sortOrder?: number[]; filters?: Filter[]; linkingFilter?: FilterColValues; + colIdAsHeader?: boolean; } /** @@ -117,6 +118,7 @@ export function parseExportParameters(req: express.Request): ExportParameters { const sortOrder = optJsonParam(req.query.activeSortSpec, []) as number[]; const filters: Filter[] = optJsonParam(req.query.filters, []); const linkingFilter: FilterColValues = optJsonParam(req.query.linkingFilter, null); + const colIdAsHeader = gutil.isAffirmative(req.query.colIdAsHeader); return { tableId, @@ -124,6 +126,7 @@ export function parseExportParameters(req: express.Request): ExportParameters { sortOrder, filters, linkingFilter, + colIdAsHeader, }; } diff --git a/app/server/lib/ExportCSV.ts b/app/server/lib/ExportCSV.ts index 8f3949d51e..c48f2e53e2 100644 --- a/app/server/lib/ExportCSV.ts +++ b/app/server/lib/ExportCSV.ts @@ -17,11 +17,12 @@ bluebird.promisifyAll(csv); export async function downloadCSV(activeDoc: ActiveDoc, req: express.Request, res: express.Response, options: DownloadOptions) { log.info('Generating .csv file...'); - const {filename, tableId, viewSectionId, filters, sortOrder, linkingFilter} = options; + const {filename, tableId, viewSectionId, filters, sortOrder, linkingFilter, colIdAsHeader} = options; + const colPropertyAsHeader = colIdAsHeader ? 'colId' : 'label'; const data = viewSectionId ? await makeCSVFromViewSection( - activeDoc, viewSectionId, sortOrder || null, filters || null, linkingFilter || null, req) : - await makeCSVFromTable(activeDoc, tableId, req); + activeDoc, viewSectionId, sortOrder || null, filters || null, linkingFilter || null, colPropertyAsHeader, req) : + await makeCSVFromTable(activeDoc, tableId, colPropertyAsHeader, req); res.set('Content-Type', 'text/csv'); res.setHeader('Content-Disposition', contentDisposition(filename + '.csv')); res.send(data); @@ -44,10 +45,11 @@ export async function makeCSVFromViewSection( sortOrder: number[] | null, filters: Filter[] | null, linkingFilter: FilterColValues | null, + colPropertyAsHeader: 'label' | 'colId', req: express.Request) { const data = await exportSection(activeDoc, viewSectionId, sortOrder, filters, linkingFilter, req); - const file = convertToCsv(data); + const file = convertToCsv(data, colPropertyAsHeader); return file; } @@ -61,6 +63,7 @@ export async function makeCSVFromViewSection( export async function makeCSVFromTable( activeDoc: ActiveDoc, tableId: string, + colPropertyAsHeader: 'label' | 'colId', req: express.Request) { if (!activeDoc.docData) { @@ -76,7 +79,7 @@ export async function makeCSVFromTable( } const data = await exportTable(activeDoc, tableRef, req); - const file = convertToCsv(data); + const file = convertToCsv(data, colPropertyAsHeader); return file; } @@ -84,13 +87,13 @@ function convertToCsv({ rowIds, access, columns: viewColumns, - docSettings -}: ExportData) { + docSettings, +}: ExportData, colPropertyAsHeader: 'label' | 'colId') { // create formatters for columns const formatters = viewColumns.map(col => col.formatter); // Arrange the data into a row-indexed matrix, starting with column headers. - const csvMatrix = [viewColumns.map(col => col.label)]; + const csvMatrix = [viewColumns.map(col => col[colPropertyAsHeader])]; // populate all the rows with values as strings rowIds.forEach(row => { csvMatrix.push(access.map((getter, c) => formatters[c].formatAny(getter(row)))); diff --git a/test/server/lib/DocApi.ts b/test/server/lib/DocApi.ts index 8c7750d286..ca3eb660a8 100644 --- a/test/server/lib/DocApi.ts +++ b/test/server/lib/DocApi.ts @@ -230,6 +230,14 @@ describe('DocApi', function () { // Contains the tests. This is where you want to add more test. function testDocApi() { + async function generateDocAndUrl(docName: string = "Dummy") { + const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id; + const docId = await userApi.newDoc({name: docName}, wid); + const docUrl = `${serverUrl}/api/docs/${docId}`; + const tableUrl = `${serverUrl}/api/docs/${docId}/tables/Table1`; + return { docUrl, tableUrl, docId }; + } + it("creator should be owner of a created ws", async () => { const kiwiEmail = 'kiwi@getgrist.com'; const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id; @@ -1055,13 +1063,13 @@ function testDocApi() { }); describe("/docs/{did}/tables/{tid}/columns", function () { - async function generateDocAndUrl(docName: string = "Dummy") { - const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id; - const docId = await userApi.newDoc({name: docName}, wid); - const url = `${serverUrl}/api/docs/${docId}/tables/Table1/columns`; - return { url, docId }; + async function generateDocAndUrlForColumns(name: string) { + const { tableUrl, docId } = await generateDocAndUrl(name); + return { + docId, + url: `${tableUrl}/columns`, + }; } - describe("PUT /docs/{did}/tables/{tid}/columns", function () { async function getColumnFieldsMapById(url: string, params: any) { const result = await axios.get(url, {...chimpy, params}); @@ -1079,7 +1087,7 @@ function testDocApi() { expectedFieldsByColId: Record, opts?: { getParams?: any } ) { - const {url} = await generateDocAndUrl('ColumnsPut'); + const {url} = await generateDocAndUrlForColumns('ColumnsPut'); const body: ColumnsPut = { columns }; const resp = await axios.put(url, body, {...chimpy, params}); assert.equal(resp.status, 200); @@ -1150,7 +1158,7 @@ function testDocApi() { it('should forbid update by viewers', async function () { // given - const { url, docId } = await generateDocAndUrl('ColumnsPut'); + const { url, docId } = await generateDocAndUrlForColumns('ColumnsPut'); await userApi.updateDocPermissions(docId, {users: {'kiwi@getgrist.com': 'viewers'}}); // when @@ -1162,7 +1170,7 @@ function testDocApi() { it("should return 404 when table is not found", async function() { // given - const { url } = await generateDocAndUrl('ColumnsPut'); + const { url } = await generateDocAndUrlForColumns('ColumnsPut'); const notFoundUrl = url.replace("Table1", "NonExistingTable"); // when @@ -1176,7 +1184,7 @@ function testDocApi() { describe("DELETE /docs/{did}/tables/{tid}/columns/{colId}", function () { it('should delete some column', async function() { - const {url} = await generateDocAndUrl('ColumnDelete'); + const {url} = await generateDocAndUrlForColumns('ColumnDelete'); const deleteUrl = url + '/A'; const resp = await axios.delete(deleteUrl, chimpy); @@ -1190,7 +1198,7 @@ function testDocApi() { }); it('should return 404 if table not found', async function() { - const {url} = await generateDocAndUrl('ColumnDelete'); + const {url} = await generateDocAndUrlForColumns('ColumnDelete'); const deleteUrl = url.replace("Table1", "NonExistingTable") + '/A'; const resp = await axios.delete(deleteUrl, chimpy); @@ -1199,7 +1207,7 @@ function testDocApi() { }); it('should return 404 if column not found', async function() { - const {url} = await generateDocAndUrl('ColumnDelete'); + const {url} = await generateDocAndUrlForColumns('ColumnDelete'); const deleteUrl = url + '/NonExistingColId'; const resp = await axios.delete(deleteUrl, chimpy); @@ -1208,7 +1216,7 @@ function testDocApi() { }); it('should forbid column deletion by viewers', async function() { - const {url, docId} = await generateDocAndUrl('ColumnDelete'); + const {url, docId} = await generateDocAndUrlForColumns('ColumnDelete'); await userApi.updateDocPermissions(docId, {users: {'kiwi@getgrist.com': 'viewers'}}); const deleteUrl = url + '/A'; const resp = await axios.delete(deleteUrl, kiwi); @@ -2584,6 +2592,25 @@ function testDocApi() { assert.equal(resp2.data, 'A,B\nSanta,1\nBob,11\nAlice,2\nFelix,22\n'); }); + it('GET /docs/{did}/download/csv with colIdAsHeader shows columns id in the header instead of their name', + async function () { + const { docUrl } = await generateDocAndUrl('csvWithColIdAsHeader'); + const AColRef = 2; + const userActions = [ + ['AddRecord', 'Table1', null, {A: 'a1', B: 'b1'}], + ['UpdateRecord', '_grist_Tables_column', AColRef, { untieColIdFromLabel: true }], + ['UpdateRecord', '_grist_Tables_column', AColRef, { + label: 'Column label for A', + colId: 'AColId' + }] + ]; + const resp = await axios.post(`${docUrl}/apply`, userActions, chimpy); + assert.equal(resp.status, 200); + const csvResp = await axios.get(`${docUrl}/download/csv?tableId=Table1&colIdAsHeader=true`, chimpy); + assert.equal(csvResp.status, 200); + assert.equal(csvResp.data, 'AColId,B,C\na1,b1,\n'); + }); + it("GET /docs/{did}/download/csv respects permissions", async function () { // kiwi has no access to TestDoc const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=Table1`, kiwi); From dc5a8d9d73e61835b9196a294bbadb0dfaf874fc Mon Sep 17 00:00:00 2001 From: Florent FAYOLLE Date: Thu, 12 Oct 2023 15:36:56 +0200 Subject: [PATCH 2/4] Option to export xslx with colId as header #688 --- app/server/lib/DocApi.ts | 1 + app/server/lib/workerExporter.ts | 38 ++++++++++++++++++-------------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/app/server/lib/DocApi.ts b/app/server/lib/DocApi.ts index bcf93c7a36..190239469c 100644 --- a/app/server/lib/DocApi.ts +++ b/app/server/lib/DocApi.ts @@ -1211,6 +1211,7 @@ export class DocWorkerApi { viewSectionId: undefined, filters: [], sortOrder: [], + colIdAsHeader: false }; await downloadXLSX(activeDoc, req, res, options); })); diff --git a/app/server/lib/workerExporter.ts b/app/server/lib/workerExporter.ts index c6992c2006..c5280ac5c6 100644 --- a/app/server/lib/workerExporter.ts +++ b/app/server/lib/workerExporter.ts @@ -79,14 +79,14 @@ export async function doMakeXLSXFromOptions( stream: Stream, options: ExportParameters ) { - const {tableId, viewSectionId, filters, sortOrder, linkingFilter} = options; + const {tableId, viewSectionId, filters, sortOrder, linkingFilter, colIdAsHeader} = options; if (viewSectionId) { - return doMakeXLSXFromViewSection(activeDocSource, testDates, stream, viewSectionId, - sortOrder || null, filters || null, linkingFilter || null); + return doMakeXLSXFromViewSection({activeDocSource, testDates, stream, viewSectionId, colIdAsHeader, + sortOrder: sortOrder || null, filters: filters || null, linkingFilter: linkingFilter || null}); } else if (tableId) { - return doMakeXLSXFromTable(activeDocSource, testDates, stream, tableId); + return doMakeXLSXFromTable({activeDocSource, testDates, stream, tableId, colIdAsHeader}); } else { - return doMakeXLSX(activeDocSource, testDates, stream); + return doMakeXLSX({activeDocSource, testDates, stream, colIdAsHeader}); } } @@ -98,7 +98,9 @@ export async function doMakeXLSXFromOptions( * @param {Integer[]} activeSortOrder (optional) - overriding sort order. * @param {Filter[]} filters (optional) - filters defined from ui. */ -async function doMakeXLSXFromViewSection( +async function doMakeXLSXFromViewSection({ + activeDocSource, testDates, stream, viewSectionId, sortOrder, filters, linkingFilter, colIdAsHeader +}: { activeDocSource: ActiveDocSource, testDates: boolean, stream: Stream, @@ -106,9 +108,10 @@ async function doMakeXLSXFromViewSection( sortOrder: number[] | null, filters: Filter[] | null, linkingFilter: FilterColValues | null, -) { + colIdAsHeader?: boolean, +}) { const data = await doExportSection(activeDocSource, viewSectionId, sortOrder, filters, linkingFilter); - const {exportTable, end} = convertToExcel(stream, testDates); + const {exportTable, end} = convertToExcel(stream, testDates, {colIdAsHeader}); exportTable(data); return end(); } @@ -119,14 +122,15 @@ async function doMakeXLSXFromViewSection( * @param {Object} activeDoc - the activeDoc that the table being converted belongs to. * @param {Integer} tableId - id of the table to export. */ -async function doMakeXLSXFromTable( +async function doMakeXLSXFromTable({activeDocSource, testDates, stream, tableId, colIdAsHeader}: { activeDocSource: ActiveDocSource, testDates: boolean, stream: Stream, tableId: string, -) { + colIdAsHeader?: boolean, +}) { const data = await doExportTable(activeDocSource, {tableId}); - const {exportTable, end} = convertToExcel(stream, testDates); + const {exportTable, end} = convertToExcel(stream, testDates, {colIdAsHeader}); exportTable(data); return end(); } @@ -134,12 +138,13 @@ async function doMakeXLSXFromTable( /** * Creates excel document with all tables from an active Grist document. */ -async function doMakeXLSX( +async function doMakeXLSX({activeDocSource, testDates, stream, colIdAsHeader}: { activeDocSource: ActiveDocSource, testDates: boolean, stream: Stream, -): Promise { - const {exportTable, end} = convertToExcel(stream, testDates); + colIdAsHeader?: boolean, +}): Promise { + const {exportTable, end} = convertToExcel(stream, testDates, {colIdAsHeader}); await doExportDoc(activeDocSource, async (table: ExportData) => exportTable(table)); return end(); } @@ -152,7 +157,7 @@ async function doMakeXLSX( * (The second option is for grist-static; at the time of writing * WorkbookWriter doesn't appear to be available in a browser context). */ -function convertToExcel(stream: Stream|undefined, testDates: boolean): { +function convertToExcel(stream: Stream|undefined, testDates: boolean, options: { colIdAsHeader?: boolean }): { exportTable: (table: ExportData) => void, end: () => Promise, } { @@ -206,7 +211,8 @@ function convertToExcel(stream: Stream|undefined, testDates: boolean): { const formatters = columns.map(col => createExcelFormatter(col.formatter.type, col.formatter.widgetOpts)); // Generate headers for all columns with correct styles for whole column. // Actual header style for a first row will be overwritten later. - ws.columns = columns.map((col, c) => ({ header: col.label, style: formatters[c].style() })); + const colHeader = options.colIdAsHeader ? 'colId' : 'label'; + ws.columns = columns.map((col, c) => ({ header: col[colHeader], style: formatters[c].style() })); // style up the header row for (let i = 1; i <= columns.length; i++) { // apply to all rows (including header) From 2fce0e2376b55ee018e440bea1cc6438f648da49 Mon Sep 17 00:00:00 2001 From: Florent FAYOLLE Date: Thu, 12 Oct 2023 16:22:01 +0200 Subject: [PATCH 3/4] Harmonize between CSV and Excel export #688 --- app/server/lib/ExportCSV.ts | 54 ++++++++++++++++++++------------ app/server/lib/workerExporter.ts | 24 ++++++++++---- 2 files changed, 52 insertions(+), 26 deletions(-) diff --git a/app/server/lib/ExportCSV.ts b/app/server/lib/ExportCSV.ts index c48f2e53e2..8fa9b95c5b 100644 --- a/app/server/lib/ExportCSV.ts +++ b/app/server/lib/ExportCSV.ts @@ -18,11 +18,12 @@ export async function downloadCSV(activeDoc: ActiveDoc, req: express.Request, res: express.Response, options: DownloadOptions) { log.info('Generating .csv file...'); const {filename, tableId, viewSectionId, filters, sortOrder, linkingFilter, colIdAsHeader} = options; - const colPropertyAsHeader = colIdAsHeader ? 'colId' : 'label'; const data = viewSectionId ? - await makeCSVFromViewSection( - activeDoc, viewSectionId, sortOrder || null, filters || null, linkingFilter || null, colPropertyAsHeader, req) : - await makeCSVFromTable(activeDoc, tableId, colPropertyAsHeader, req); + await makeCSVFromViewSection({ + activeDoc, viewSectionId, sortOrder: sortOrder || null, filters: filters || null, + linkingFilter: linkingFilter || null, colIdAsHeader, req + }) : + await makeCSVFromTable({activeDoc, tableId, colIdAsHeader, req}); res.set('Content-Type', 'text/csv'); res.setHeader('Content-Disposition', contentDisposition(filename + '.csv')); res.send(data); @@ -33,38 +34,51 @@ export async function downloadCSV(activeDoc: ActiveDoc, req: express.Request, * * See https://github.com/wdavidw/node-csv for API details. * - * @param {Object} activeDoc - the activeDoc that the table being converted belongs to. - * @param {Integer} viewSectionId - id of the viewsection to export. - * @param {Integer[]} activeSortOrder (optional) - overriding sort order. - * @param {Filter[]} filters (optional) - filters defined from ui. + * @param {Object} options - options for the export. + * @param {Object} options.activeDoc - the activeDoc that the table being converted belongs to. + * @param {Integer} options.viewSectionId - id of the viewsection to export. + * @param {Integer[]} options.activeSortOrder (optional) - overriding sort order. + * @param {Filter[]} options.filters (optional) - filters defined from ui. + * @param {FilterColValues} options.linkingFilter (optional) - linking filter defined from ui. + * @param {boolean} [options.colIdAsHeader] - whether to use column id as header. + * @param {express.Request} options.req - the request object. + * * @return {Promise} Promise for the resulting CSV. */ -export async function makeCSVFromViewSection( +export async function makeCSVFromViewSection({ + activeDoc, viewSectionId, sortOrder = null, filters = null, linkingFilter = null, colIdAsHeader, req +}: { activeDoc: ActiveDoc, viewSectionId: number, sortOrder: number[] | null, filters: Filter[] | null, linkingFilter: FilterColValues | null, - colPropertyAsHeader: 'label' | 'colId', - req: express.Request) { + colIdAsHeader?: boolean, + req: express.Request +}) { const data = await exportSection(activeDoc, viewSectionId, sortOrder, filters, linkingFilter, req); - const file = convertToCsv(data, colPropertyAsHeader); + const file = convertToCsv(data, { colIdAsHeader }); return file; } /** * Returns a csv stream of a table that can be transformed or parsed. * - * @param {Object} activeDoc - the activeDoc that the table being converted belongs to. - * @param {Integer} tableId - id of the table to export. + * @param {Object} options - options for the export. + * @param {Object} options.activeDoc - the activeDoc that the table being converted belongs to. + * @param {Integer} options.tableId - id of the table to export. + * @param {boolean} [options.colIdAsHeader] - whether to use column id as header. + * @param {express.Request} options.req - the request object. + * * @return {Promise} Promise for the resulting CSV. */ -export async function makeCSVFromTable( +export async function makeCSVFromTable({ activeDoc, tableId, colIdAsHeader, req }: { activeDoc: ActiveDoc, tableId: string, - colPropertyAsHeader: 'label' | 'colId', - req: express.Request) { + colIdAsHeader?: boolean, + req: express.Request +}) { if (!activeDoc.docData) { throw new Error('No docData in active document'); @@ -79,7 +93,7 @@ export async function makeCSVFromTable( } const data = await exportTable(activeDoc, tableRef, req); - const file = convertToCsv(data, colPropertyAsHeader); + const file = convertToCsv(data, { colIdAsHeader }); return file; } @@ -87,12 +101,12 @@ function convertToCsv({ rowIds, access, columns: viewColumns, - docSettings, -}: ExportData, colPropertyAsHeader: 'label' | 'colId') { +}: ExportData, options: { colIdAsHeader?: boolean }) { // create formatters for columns const formatters = viewColumns.map(col => col.formatter); // Arrange the data into a row-indexed matrix, starting with column headers. + const colPropertyAsHeader = options.colIdAsHeader ? 'colId' : 'label'; const csvMatrix = [viewColumns.map(col => col[colPropertyAsHeader])]; // populate all the rows with values as strings rowIds.forEach(row => { diff --git a/app/server/lib/workerExporter.ts b/app/server/lib/workerExporter.ts index c5280ac5c6..36a99468fa 100644 --- a/app/server/lib/workerExporter.ts +++ b/app/server/lib/workerExporter.ts @@ -91,12 +91,18 @@ export async function doMakeXLSXFromOptions( } /** + * @async * Returns a XLSX stream of a view section that can be transformed or parsed. * - * @param {Object} activeDoc - the activeDoc that the table being converted belongs to. - * @param {Integer} viewSectionId - id of the viewsection to export. - * @param {Integer[]} activeSortOrder (optional) - overriding sort order. - * @param {Filter[]} filters (optional) - filters defined from ui. + * @param {Object} options - options for the export. + * @param {Object} options.activeDocSource - the activeDoc that the table being converted belongs to. + * @param {Integer} options.viewSectionId - id of the viewsection to export. + * @param {Integer[]} options.activeSortOrder (optional) - overriding sort order. + * @param {Filter[]} options.filters (optional) - filters defined from ui. + * @param {FilterColValues} options.linkingFilter (optional) + * @param {Stream} options.stream - the stream to write to. + * @param {boolean} options.testDates - whether to use static dates for testing. + * @param {boolean} [options.colIdAsHeader] - whether to use column id as header. */ async function doMakeXLSXFromViewSection({ activeDocSource, testDates, stream, viewSectionId, sortOrder, filters, linkingFilter, colIdAsHeader @@ -117,10 +123,16 @@ async function doMakeXLSXFromViewSection({ } /** + * @async * Returns a XLSX stream of a table that can be transformed or parsed. * - * @param {Object} activeDoc - the activeDoc that the table being converted belongs to. - * @param {Integer} tableId - id of the table to export. + * @param {Object} options - options for the export. + * @param {Object} options.activeDocSource - the activeDoc that the table being converted belongs to. + * @param {Integer} options.tableId - id of the table to export. + * @param {Stream} options.stream - the stream to write to. + * @param {boolean} options.testDates - whether to use static dates for testing. + * @param {boolean} [options.colIdAsHeader] - whether to use column id as header. + * */ async function doMakeXLSXFromTable({activeDocSource, testDates, stream, tableId, colIdAsHeader}: { activeDocSource: ActiveDocSource, From 4410bc0cd1e8a300a9b021958b3b19de01628565 Mon Sep 17 00:00:00 2001 From: Florent Date: Fri, 13 Oct 2023 17:49:37 +0200 Subject: [PATCH 4/4] Change colIdAsHeader to header=colId|label #688 --- app/server/lib/DocApi.ts | 4 ++-- app/server/lib/Export.ts | 10 +++++---- app/server/lib/ExportCSV.ts | 28 ++++++++++++------------- app/server/lib/workerExporter.ts | 36 ++++++++++++++++---------------- test/server/lib/DocApi.ts | 4 ++-- 5 files changed, 42 insertions(+), 40 deletions(-) diff --git a/app/server/lib/DocApi.ts b/app/server/lib/DocApi.ts index 190239469c..c3423ae9e3 100644 --- a/app/server/lib/DocApi.ts +++ b/app/server/lib/DocApi.ts @@ -1205,13 +1205,13 @@ export class DocWorkerApi { this._app.get('/api/docs/:docId/download/xlsx', canView, withDoc(async (activeDoc, req, res) => { // Query DB for doc metadata to get the doc title (to use as the filename). const {name: docTitle} = await this._dbManager.getDoc(req); - const options = !_.isEmpty(req.query) ? this._getDownloadOptions(req, docTitle) : { + const options: DownloadOptions = !_.isEmpty(req.query) ? this._getDownloadOptions(req, docTitle) : { filename: docTitle, tableId: '', viewSectionId: undefined, filters: [], sortOrder: [], - colIdAsHeader: false + header: 'label' }; await downloadXLSX(activeDoc, req, res, options); })); diff --git a/app/server/lib/Export.ts b/app/server/lib/Export.ts index cdcd0b29b5..ab4469893f 100644 --- a/app/server/lib/Export.ts +++ b/app/server/lib/Export.ts @@ -17,7 +17,7 @@ import {BaseFormatter, createFullFormatterFromDocData} from 'app/common/ValueFor import {ActiveDoc} from 'app/server/lib/ActiveDoc'; import {RequestWithLogin} from 'app/server/lib/Authorizer'; import {docSessionFromRequest} from 'app/server/lib/DocSession'; -import {optIntegerParam, optJsonParam, stringParam} from 'app/server/lib/requestUtils'; +import {optIntegerParam, optJsonParam, optStringParam, stringParam} from 'app/server/lib/requestUtils'; import {ServerColumnGetters} from 'app/server/lib/ServerColumnGetters'; import * as express from 'express'; import * as _ from 'underscore'; @@ -90,6 +90,8 @@ export interface ExportData { docSettings: DocumentSettings; } +export type ExportHeader = 'colId' | 'label'; + /** * Export parameters that identifies a section, filters, sort order. */ @@ -99,7 +101,7 @@ export interface ExportParameters { sortOrder?: number[]; filters?: Filter[]; linkingFilter?: FilterColValues; - colIdAsHeader?: boolean; + header?: ExportHeader; } /** @@ -118,7 +120,7 @@ export function parseExportParameters(req: express.Request): ExportParameters { const sortOrder = optJsonParam(req.query.activeSortSpec, []) as number[]; const filters: Filter[] = optJsonParam(req.query.filters, []); const linkingFilter: FilterColValues = optJsonParam(req.query.linkingFilter, null); - const colIdAsHeader = gutil.isAffirmative(req.query.colIdAsHeader); + const header = optStringParam(req.query.header, 'header', {allowed: ['label', 'colId']}) as ExportHeader | undefined; return { tableId, @@ -126,7 +128,7 @@ export function parseExportParameters(req: express.Request): ExportParameters { sortOrder, filters, linkingFilter, - colIdAsHeader, + header, }; } diff --git a/app/server/lib/ExportCSV.ts b/app/server/lib/ExportCSV.ts index 8fa9b95c5b..a0ff027d0d 100644 --- a/app/server/lib/ExportCSV.ts +++ b/app/server/lib/ExportCSV.ts @@ -1,7 +1,7 @@ import {ApiError} from 'app/common/ApiError'; import {ActiveDoc} from 'app/server/lib/ActiveDoc'; import {FilterColValues} from "app/common/ActiveDocAPI"; -import {DownloadOptions, ExportData, exportSection, exportTable, Filter} from 'app/server/lib/Export'; +import {DownloadOptions, ExportData, ExportHeader, exportSection, exportTable, Filter} from 'app/server/lib/Export'; import log from 'app/server/lib/log'; import * as bluebird from 'bluebird'; import contentDisposition from 'content-disposition'; @@ -17,13 +17,13 @@ bluebird.promisifyAll(csv); export async function downloadCSV(activeDoc: ActiveDoc, req: express.Request, res: express.Response, options: DownloadOptions) { log.info('Generating .csv file...'); - const {filename, tableId, viewSectionId, filters, sortOrder, linkingFilter, colIdAsHeader} = options; + const {filename, tableId, viewSectionId, filters, sortOrder, linkingFilter, header} = options; const data = viewSectionId ? await makeCSVFromViewSection({ activeDoc, viewSectionId, sortOrder: sortOrder || null, filters: filters || null, - linkingFilter: linkingFilter || null, colIdAsHeader, req + linkingFilter: linkingFilter || null, header, req }) : - await makeCSVFromTable({activeDoc, tableId, colIdAsHeader, req}); + await makeCSVFromTable({activeDoc, tableId, header, req}); res.set('Content-Type', 'text/csv'); res.setHeader('Content-Disposition', contentDisposition(filename + '.csv')); res.send(data); @@ -40,25 +40,25 @@ export async function downloadCSV(activeDoc: ActiveDoc, req: express.Request, * @param {Integer[]} options.activeSortOrder (optional) - overriding sort order. * @param {Filter[]} options.filters (optional) - filters defined from ui. * @param {FilterColValues} options.linkingFilter (optional) - linking filter defined from ui. - * @param {boolean} [options.colIdAsHeader] - whether to use column id as header. + * @param {string} options.header (optional) - which field of the column to use as header * @param {express.Request} options.req - the request object. * * @return {Promise} Promise for the resulting CSV. */ export async function makeCSVFromViewSection({ - activeDoc, viewSectionId, sortOrder = null, filters = null, linkingFilter = null, colIdAsHeader, req + activeDoc, viewSectionId, sortOrder = null, filters = null, linkingFilter = null, header, req }: { activeDoc: ActiveDoc, viewSectionId: number, sortOrder: number[] | null, filters: Filter[] | null, linkingFilter: FilterColValues | null, - colIdAsHeader?: boolean, + header?: ExportHeader, req: express.Request }) { const data = await exportSection(activeDoc, viewSectionId, sortOrder, filters, linkingFilter, req); - const file = convertToCsv(data, { colIdAsHeader }); + const file = convertToCsv(data, { header }); return file; } @@ -68,15 +68,15 @@ export async function makeCSVFromViewSection({ * @param {Object} options - options for the export. * @param {Object} options.activeDoc - the activeDoc that the table being converted belongs to. * @param {Integer} options.tableId - id of the table to export. - * @param {boolean} [options.colIdAsHeader] - whether to use column id as header. + * @param {string} options.header (optional) - which field of the column to use as header * @param {express.Request} options.req - the request object. * * @return {Promise} Promise for the resulting CSV. */ -export async function makeCSVFromTable({ activeDoc, tableId, colIdAsHeader, req }: { +export async function makeCSVFromTable({ activeDoc, tableId, header, req }: { activeDoc: ActiveDoc, tableId: string, - colIdAsHeader?: boolean, + header?: ExportHeader, req: express.Request }) { @@ -93,7 +93,7 @@ export async function makeCSVFromTable({ activeDoc, tableId, colIdAsHeader, req } const data = await exportTable(activeDoc, tableRef, req); - const file = convertToCsv(data, { colIdAsHeader }); + const file = convertToCsv(data, { header }); return file; } @@ -101,12 +101,12 @@ function convertToCsv({ rowIds, access, columns: viewColumns, -}: ExportData, options: { colIdAsHeader?: boolean }) { +}: ExportData, options: { header?: ExportHeader }) { // create formatters for columns const formatters = viewColumns.map(col => col.formatter); // Arrange the data into a row-indexed matrix, starting with column headers. - const colPropertyAsHeader = options.colIdAsHeader ? 'colId' : 'label'; + const colPropertyAsHeader = options.header ?? 'label'; const csvMatrix = [viewColumns.map(col => col[colPropertyAsHeader])]; // populate all the rows with values as strings rowIds.forEach(row => { diff --git a/app/server/lib/workerExporter.ts b/app/server/lib/workerExporter.ts index 36a99468fa..212ed5994e 100644 --- a/app/server/lib/workerExporter.ts +++ b/app/server/lib/workerExporter.ts @@ -1,7 +1,7 @@ import {PassThrough} from 'stream'; import {FilterColValues} from "app/common/ActiveDocAPI"; import {ActiveDocSource, doExportDoc, doExportSection, doExportTable, - ExportData, ExportParameters, Filter} from 'app/server/lib/Export'; + ExportData, ExportHeader, ExportParameters, Filter} from 'app/server/lib/Export'; import {createExcelFormatter} from 'app/server/lib/ExcelFormatter'; import * as log from 'app/server/lib/log'; import {Alignment, Border, Buffer as ExcelBuffer, stream as ExcelWriteStream, @@ -79,14 +79,14 @@ export async function doMakeXLSXFromOptions( stream: Stream, options: ExportParameters ) { - const {tableId, viewSectionId, filters, sortOrder, linkingFilter, colIdAsHeader} = options; + const {tableId, viewSectionId, filters, sortOrder, linkingFilter, header} = options; if (viewSectionId) { - return doMakeXLSXFromViewSection({activeDocSource, testDates, stream, viewSectionId, colIdAsHeader, + return doMakeXLSXFromViewSection({activeDocSource, testDates, stream, viewSectionId, header, sortOrder: sortOrder || null, filters: filters || null, linkingFilter: linkingFilter || null}); } else if (tableId) { - return doMakeXLSXFromTable({activeDocSource, testDates, stream, tableId, colIdAsHeader}); + return doMakeXLSXFromTable({activeDocSource, testDates, stream, tableId, header}); } else { - return doMakeXLSX({activeDocSource, testDates, stream, colIdAsHeader}); + return doMakeXLSX({activeDocSource, testDates, stream, header}); } } @@ -102,10 +102,10 @@ export async function doMakeXLSXFromOptions( * @param {FilterColValues} options.linkingFilter (optional) * @param {Stream} options.stream - the stream to write to. * @param {boolean} options.testDates - whether to use static dates for testing. - * @param {boolean} [options.colIdAsHeader] - whether to use column id as header. + * @param {string} options.header (optional) - which field of the column to use as header */ async function doMakeXLSXFromViewSection({ - activeDocSource, testDates, stream, viewSectionId, sortOrder, filters, linkingFilter, colIdAsHeader + activeDocSource, testDates, stream, viewSectionId, sortOrder, filters, linkingFilter, header }: { activeDocSource: ActiveDocSource, testDates: boolean, @@ -114,10 +114,10 @@ async function doMakeXLSXFromViewSection({ sortOrder: number[] | null, filters: Filter[] | null, linkingFilter: FilterColValues | null, - colIdAsHeader?: boolean, + header?: ExportHeader, }) { const data = await doExportSection(activeDocSource, viewSectionId, sortOrder, filters, linkingFilter); - const {exportTable, end} = convertToExcel(stream, testDates, {colIdAsHeader}); + const {exportTable, end} = convertToExcel(stream, testDates, {header}); exportTable(data); return end(); } @@ -131,18 +131,18 @@ async function doMakeXLSXFromViewSection({ * @param {Integer} options.tableId - id of the table to export. * @param {Stream} options.stream - the stream to write to. * @param {boolean} options.testDates - whether to use static dates for testing. - * @param {boolean} [options.colIdAsHeader] - whether to use column id as header. + * @param {string} options.header (optional) - which field of the column to use as header * */ -async function doMakeXLSXFromTable({activeDocSource, testDates, stream, tableId, colIdAsHeader}: { +async function doMakeXLSXFromTable({activeDocSource, testDates, stream, tableId, header}: { activeDocSource: ActiveDocSource, testDates: boolean, stream: Stream, tableId: string, - colIdAsHeader?: boolean, + header?: ExportHeader, }) { const data = await doExportTable(activeDocSource, {tableId}); - const {exportTable, end} = convertToExcel(stream, testDates, {colIdAsHeader}); + const {exportTable, end} = convertToExcel(stream, testDates, {header}); exportTable(data); return end(); } @@ -150,13 +150,13 @@ async function doMakeXLSXFromTable({activeDocSource, testDates, stream, tableId, /** * Creates excel document with all tables from an active Grist document. */ -async function doMakeXLSX({activeDocSource, testDates, stream, colIdAsHeader}: { +async function doMakeXLSX({activeDocSource, testDates, stream, header}: { activeDocSource: ActiveDocSource, testDates: boolean, stream: Stream, - colIdAsHeader?: boolean, + header?: ExportHeader, }): Promise { - const {exportTable, end} = convertToExcel(stream, testDates, {colIdAsHeader}); + const {exportTable, end} = convertToExcel(stream, testDates, {header}); await doExportDoc(activeDocSource, async (table: ExportData) => exportTable(table)); return end(); } @@ -169,7 +169,7 @@ async function doMakeXLSX({activeDocSource, testDates, stream, colIdAsHeader}: { * (The second option is for grist-static; at the time of writing * WorkbookWriter doesn't appear to be available in a browser context). */ -function convertToExcel(stream: Stream|undefined, testDates: boolean, options: { colIdAsHeader?: boolean }): { +function convertToExcel(stream: Stream|undefined, testDates: boolean, options: { header?: ExportHeader }): { exportTable: (table: ExportData) => void, end: () => Promise, } { @@ -223,7 +223,7 @@ function convertToExcel(stream: Stream|undefined, testDates: boolean, options: { const formatters = columns.map(col => createExcelFormatter(col.formatter.type, col.formatter.widgetOpts)); // Generate headers for all columns with correct styles for whole column. // Actual header style for a first row will be overwritten later. - const colHeader = options.colIdAsHeader ? 'colId' : 'label'; + const colHeader = options.header ?? 'label'; ws.columns = columns.map((col, c) => ({ header: col[colHeader], style: formatters[c].style() })); // style up the header row for (let i = 1; i <= columns.length; i++) { diff --git a/test/server/lib/DocApi.ts b/test/server/lib/DocApi.ts index ca3eb660a8..a796af56c9 100644 --- a/test/server/lib/DocApi.ts +++ b/test/server/lib/DocApi.ts @@ -2592,7 +2592,7 @@ function testDocApi() { assert.equal(resp2.data, 'A,B\nSanta,1\nBob,11\nAlice,2\nFelix,22\n'); }); - it('GET /docs/{did}/download/csv with colIdAsHeader shows columns id in the header instead of their name', + it('GET /docs/{did}/download/csv with header=colId shows columns id in the header instead of their name', async function () { const { docUrl } = await generateDocAndUrl('csvWithColIdAsHeader'); const AColRef = 2; @@ -2606,7 +2606,7 @@ function testDocApi() { ]; const resp = await axios.post(`${docUrl}/apply`, userActions, chimpy); assert.equal(resp.status, 200); - const csvResp = await axios.get(`${docUrl}/download/csv?tableId=Table1&colIdAsHeader=true`, chimpy); + const csvResp = await axios.get(`${docUrl}/download/csv?tableId=Table1&header=colId`, chimpy); assert.equal(csvResp.status, 200); assert.equal(csvResp.data, 'AColId,B,C\na1,b1,\n'); });