From 0b38f66283d6c12cba56dca2aa7266900a76b293 Mon Sep 17 00:00:00 2001 From: sambokar Date: Tue, 22 Oct 2024 11:43:15 -0400 Subject: [PATCH] fixing issues with file upload and processing system. bugs were found causing load issues and SQL overload issues, but have been temporarily resolved. refit is needed to update how the file set is processed to ensure that system will not collapse under weight of file set. --- .../measurementshub/validations/page.tsx | 1 - .../[dataType]/[[...slugs]]/route.ts | 1 - .../procedures/[validationType]/route.ts | 1 - .../datagrids/isolateddatagridcommons.tsx | 80 ++++++++++++++++- .../isolatedmultilinedatagridcommons.tsx | 55 ++++++------ .../datagrids/measurementscommons.tsx | 20 ++++- .../components/processors/processcensus.tsx | 86 +++++++++---------- .../processors/processorhelperfunctions.tsx | 6 +- .../components/processors/processormacros.tsx | 4 +- frontend/config/poolmonitor.ts | 38 ++++++-- frontend/config/sqlrdsdefinitions/views.ts | 3 + frontend/config/sqlrdsdefinitions/zones.ts | 1 + frontend/config/utils.ts | 6 -- 13 files changed, 207 insertions(+), 95 deletions(-) diff --git a/frontend/app/(hub)/measurementshub/validations/page.tsx b/frontend/app/(hub)/measurementshub/validations/page.tsx index 99afe371..8fd2c617 100644 --- a/frontend/app/(hub)/measurementshub/validations/page.tsx +++ b/frontend/app/(hub)/measurementshub/validations/page.tsx @@ -62,7 +62,6 @@ export default function ValidationsPage() { try { const response = await fetch('/api/validations/crud', { method: 'GET' }); const data = await response.json(); - console.log('data: ', data); setGlobalValidations(data); } catch (err) { console.error('Error fetching validations:', err); diff --git a/frontend/app/api/fixeddata/[dataType]/[[...slugs]]/route.ts b/frontend/app/api/fixeddata/[dataType]/[[...slugs]]/route.ts index e456eb55..0e37856f 100644 --- a/frontend/app/api/fixeddata/[dataType]/[[...slugs]]/route.ts +++ b/frontend/app/api/fixeddata/[dataType]/[[...slugs]]/route.ts @@ -365,7 +365,6 @@ export async function DELETE(request: NextRequest, { params }: { params: { dataT let conn: PoolConnection | null = null; const demappedGridID = gridID.charAt(0).toUpperCase() + gridID.substring(1); const { newRow } = await request.json(); - console.log('newrow: ', newRow); try { conn = await getConn(); await conn.beginTransaction(); diff --git a/frontend/app/api/validations/procedures/[validationType]/route.ts b/frontend/app/api/validations/procedures/[validationType]/route.ts index 7606a619..43f23c52 100644 --- a/frontend/app/api/validations/procedures/[validationType]/route.ts +++ b/frontend/app/api/validations/procedures/[validationType]/route.ts @@ -5,7 +5,6 @@ import { HTTPResponses } from '@/config/macros'; export async function POST(request: NextRequest, { params }: { params: { validationProcedureName: string } }) { try { const { schema, validationProcedureID, cursorQuery, p_CensusID, p_PlotID, minDBH, maxDBH, minHOM, maxHOM } = await request.json(); - console.log('data: ', schema, validationProcedureID, cursorQuery, p_CensusID, p_PlotID, minDBH, maxDBH, minHOM, maxHOM); // Execute the validation procedure using the provided inputs const validationResponse = await runValidation(validationProcedureID, params.validationProcedureName, schema, cursorQuery, { diff --git a/frontend/components/datagrids/isolateddatagridcommons.tsx b/frontend/components/datagrids/isolateddatagridcommons.tsx index 0b2970d2..b4ec5f18 100644 --- a/frontend/components/datagrids/isolateddatagridcommons.tsx +++ b/frontend/components/datagrids/isolateddatagridcommons.tsx @@ -241,7 +241,25 @@ export default function IsolatedDataGridCommons(props: Readonly { const values = getTableHeaders(FormType.attributes) .map(rowHeader => rowHeader.label) - .map(header => row[header]); + .map(header => row[header]) + .map(value => { + if (value === undefined || value === null || value === '') { + return null; + } + if (typeof value === 'number') { + return value; + } + const parsedValue = parseFloat(value); + if (!isNaN(parsedValue)) { + return parsedValue; + } + if (typeof value === 'string') { + value = value.replace(/"/g, '""'); + value = `"${value}"`; + } + + return value; + }); aCSVRows += values.join(',') + '\n'; }); const aBlob = new Blob([aCSVRows], { @@ -268,7 +286,25 @@ export default function IsolatedDataGridCommons(props: Readonly { const values = getTableHeaders(FormType.quadrats) .map(rowHeader => rowHeader.label) - .map(header => row[header]); + .map(header => row[header]) + .map(value => { + if (value === undefined || value === null || value === '') { + return null; + } + if (typeof value === 'number') { + return value; + } + const parsedValue = parseFloat(value); + if (!isNaN(parsedValue)) { + return parsedValue; + } + if (typeof value === 'string') { + value = value.replace(/"/g, '""'); + value = `"${value}"`; + } + + return value; + }); qCSVRows += values.join(',') + '\n'; }); const qBlob = new Blob([qCSVRows], { @@ -295,7 +331,25 @@ export default function IsolatedDataGridCommons(props: Readonly { const values = getTableHeaders(FormType.personnel) .map(rowHeader => rowHeader.label) - .map(header => row[header]); + .map(header => row[header]) + .map(value => { + if (value === undefined || value === null || value === '') { + return null; + } + if (typeof value === 'number') { + return value; + } + const parsedValue = parseFloat(value); + if (!isNaN(parsedValue)) { + return parsedValue; + } + if (typeof value === 'string') { + value = value.replace(/"/g, '""'); + value = `"${value}"`; + } + + return value; + }); pCSVRows += values.join(',') + '\n'; }); const pBlob = new Blob([pCSVRows], { @@ -323,7 +377,25 @@ export default function IsolatedDataGridCommons(props: Readonly { const values = getTableHeaders(FormType.species) .map(rowHeader => rowHeader.label) - .map(header => row[header]); + .map(header => row[header]) + .map(value => { + if (value === undefined || value === null || value === '') { + return null; + } + if (typeof value === 'number') { + return value; + } + const parsedValue = parseFloat(value); + if (!isNaN(parsedValue)) { + return parsedValue; + } + if (typeof value === 'string') { + value = value.replace(/"/g, '""'); + value = `"${value}"`; + } + + return value; + }); sCSVRows += values.join(',') + '\n'; }); const sBlob = new Blob([sCSVRows], { diff --git a/frontend/components/datagrids/isolatedmultilinedatagridcommons.tsx b/frontend/components/datagrids/isolatedmultilinedatagridcommons.tsx index b9c267fb..fc1b6ece 100644 --- a/frontend/components/datagrids/isolatedmultilinedatagridcommons.tsx +++ b/frontend/components/datagrids/isolatedmultilinedatagridcommons.tsx @@ -47,8 +47,8 @@ export default function IsolatedMultilineDataGridCommons(props: Readonly( - () => [ + const columns = useMemo(() => { + let baseColumns: GridColDef[] = [ { field: 'actions', headerName: 'Actions', @@ -80,33 +80,40 @@ export default function IsolatedMultilineDataGridCommons(props: Readonly ]; } }, - ...gridColumns, - { - field: 'date', - headerName: 'Date', - headerClassName: 'header', - flex: 1, - editable: true, - renderCell: renderDatePicker, - renderEditCell: renderEditDatePicker - }, - { - field: 'codes', - headerName: 'Codes', - headerClassName: 'header', - flex: 1, - align: 'center', - editable: true - } - ], - [gridColumns, unsavedChangesRef, apiRef, setRows] - ); + ...gridColumns + ]; + + if (gridType === 'measurements') { + baseColumns = [ + ...baseColumns, + { + field: 'date', + headerName: 'Date', + headerClassName: 'header', + flex: 1, + editable: true, + renderCell: renderDatePicker, + renderEditCell: renderEditDatePicker + }, + { + field: 'codes', + headerName: 'Codes', + headerClassName: 'header', + flex: 1, + align: 'center', + editable: true + } + ]; + } + + return baseColumns; + }, [gridColumns, gridType, unsavedChangesRef, apiRef, setHasUnsavedRows]); const processRowUpdate = useCallback>((newRow, oldRow) => { const rowId = newRow.id; diff --git a/frontend/components/datagrids/measurementscommons.tsx b/frontend/components/datagrids/measurementscommons.tsx index f8df58f4..f4761347 100644 --- a/frontend/components/datagrids/measurementscommons.tsx +++ b/frontend/components/datagrids/measurementscommons.tsx @@ -209,7 +209,25 @@ export default function MeasurementsCommons(props: Readonly { const values = getTableHeaders(FormType.measurements) .map(rowHeader => rowHeader.label) - .map(header => row[header]); + .map(header => row[header]) + .map(value => { + if (value === undefined || value === null || value === '') { + return null; + } + if (typeof value === 'number') { + return value; + } + const parsedValue = parseFloat(value); + if (!isNaN(parsedValue)) { + return parsedValue; + } + if (typeof value === 'string') { + value = value.replace(/"/g, '""'); + value = `"${value}"`; + } + + return value; + }); csvRows += values.join(',') + '\n'; }); const blob = new Blob([csvRows], { diff --git a/frontend/components/processors/processcensus.tsx b/frontend/components/processors/processcensus.tsx index b56015be..ce6a8bb4 100644 --- a/frontend/components/processors/processcensus.tsx +++ b/frontend/components/processors/processcensus.tsx @@ -11,15 +11,12 @@ export async function processCensus(props: Readonly): Pr console.error('Missing required parameters: plotID or censusID'); throw new Error('Process Census: Missing plotID or censusID'); } - const { tag, stemtag, spcode, quadrat, lx, ly, coordinateunit, dbh, dbhunit, hom, homunit, date, codes } = rowData; try { await connection.beginTransaction(); - // Fetch species const speciesID = await fetchPrimaryKey(schema, 'species', { SpeciesCode: spcode }, connection, 'SpeciesID'); - // Fetch quadrat const quadratID = await fetchPrimaryKey(schema, 'quadrats', { QuadratName: quadrat, PlotID: plotID }, connection, 'QuadratID'); @@ -27,8 +24,7 @@ export async function processCensus(props: Readonly): Pr // Handle Tree Upsert const treeID = await handleUpsert(connection, schema, 'trees', { TreeTag: tag, SpeciesID: speciesID }, 'TreeID'); - if (stemtag && lx && ly) { - console.log('Processing stem with StemTag:', stemtag); + if (stemtag || lx || ly) { // Handle Stem Upsert const stemID = await handleUpsert( connection, @@ -38,46 +34,47 @@ export async function processCensus(props: Readonly): Pr 'StemID' ); - if (dbh && hom && date) { - // Handle Core Measurement Upsert - const coreMeasurementID = await handleUpsert( - connection, - schema, - 'coremeasurements', - { - CensusID: censusID, - StemID: stemID, - IsValidated: null, - MeasurementDate: moment(date).format('YYYY-MM-DD'), - MeasuredDBH: dbh, - DBHUnit: dbhunit, - MeasuredHOM: hom, - HOMUnit: homunit - }, - 'CoreMeasurementID' - ); + // Handle Core Measurement Upsert + const coreMeasurementID = await handleUpsert( + connection, + schema, + 'coremeasurements', + { + CensusID: censusID, + StemID: stemID, + IsValidated: null, + MeasurementDate: date && moment(date).isValid() ? moment.utc(date).format('YYYY-MM-DD') : null, + MeasuredDBH: dbh ? parseFloat(dbh) : null, + DBHUnit: dbhunit, + MeasuredHOM: hom ? parseFloat(hom) : null, + HOMUnit: homunit, + Description: null, + UserDefinedFields: null + }, + 'CoreMeasurementID' + ); - // Handle CM Attributes Upsert - if (codes) { - const parsedCodes = codes - .split(';') - .map(code => code.trim()) - .filter(Boolean); - if (parsedCodes.length === 0) { - console.error('No valid attribute codes found:', codes); - } else { - for (const code of parsedCodes) { - const attributeRows = await runQuery(connection, `SELECT COUNT(*) as count FROM ${schema}.attributes WHERE Code = ?`, [code]); - if (!attributeRows || attributeRows.length === 0 || !attributeRows[0].count) { - throw createError(`Attribute code ${code} not found or query failed.`, { code }); - } - await handleUpsert(connection, schema, 'cmattributes', { CoreMeasurementID: coreMeasurementID, Code: code }, 'CMAID'); + // Handle CM Attributes Upsert + if (codes) { + const parsedCodes = codes + .split(';') + .map(code => code.trim()) + .filter(Boolean); + if (parsedCodes.length === 0) { + console.error('No valid attribute codes found:', codes); + } else { + for (const code of parsedCodes) { + const attributeRows = await runQuery(connection, `SELECT COUNT(*) as count FROM ${schema}.attributes WHERE Code = ?`, [code]); + if (!attributeRows || attributeRows.length === 0 || !attributeRows[0].count) { + throw createError(`Attribute code ${code} not found or query failed.`, { code }); } + await handleUpsert(connection, schema, 'cmattributes', { CoreMeasurementID: coreMeasurementID, Code: code }, 'CMAID'); } } + } - // Update Census Start/End Dates - const combinedQuery = ` + // Update Census Start/End Dates + const combinedQuery = ` UPDATE ${schema}.census c JOIN ( SELECT CensusID, MIN(MeasurementDate) AS FirstMeasurementDate, MAX(MeasurementDate) AS LastMeasurementDate @@ -88,11 +85,10 @@ export async function processCensus(props: Readonly): Pr SET c.StartDate = m.FirstMeasurementDate, c.EndDate = m.LastMeasurementDate WHERE c.CensusID = ${censusID};`; - await runQuery(connection, combinedQuery); - await connection.commit(); - console.log('Upsert successful. CoreMeasurement ID generated:', coreMeasurementID); - return coreMeasurementID; - } + await runQuery(connection, combinedQuery); + await connection.commit(); + console.log('Upsert successful. CoreMeasurement ID generated:', coreMeasurementID); + return coreMeasurementID; } } } catch (error: any) { diff --git a/frontend/components/processors/processorhelperfunctions.tsx b/frontend/components/processors/processorhelperfunctions.tsx index ae728615..665432b8 100644 --- a/frontend/components/processors/processorhelperfunctions.tsx +++ b/frontend/components/processors/processorhelperfunctions.tsx @@ -26,7 +26,11 @@ export async function insertOrUpdate(props: InsertUpdateProcessingProps): Promis if (columns.includes('censusID')) rowData['censusID'] = subProps.censusID?.toString() ?? null; const tableColumns = columns.map(fileColumn => mapping.columnMappings[fileColumn]).join(', '); const placeholders = columns.map(() => '?').join(', '); // Use '?' for placeholders in MySQL - const values = columns.map(fileColumn => rowData[fileColumn]); + const values = columns.map(fileColumn => { + const value = rowData[fileColumn]; + if (typeof value === 'string' && value === '') return null; + return value; + }); const query = ` INSERT INTO ${schema}.${mapping.tableName} (${tableColumns}) VALUES (${placeholders}) ON DUPLICATE KEY diff --git a/frontend/components/processors/processormacros.tsx b/frontend/components/processors/processormacros.tsx index 09a48104..d1d3b5d7 100644 --- a/frontend/components/processors/processormacros.tsx +++ b/frontend/components/processors/processormacros.tsx @@ -104,8 +104,8 @@ const sqlConfig: PoolOptions = { port: parseInt(process.env.AZURE_SQL_PORT!), database: process.env.AZURE_SQL_CATALOG_SCHEMA, waitForConnections: true, - connectionLimit: 100, // increased from 10 to prevent bottlenecks - queueLimit: 0, + connectionLimit: 150, // increased from 10 to prevent bottlenecks + queueLimit: 20, keepAliveInitialDelay: 10000, // 0 by default. enableKeepAlive: true, // false by default. connectTimeout: 20000 // 10 seconds by default. diff --git a/frontend/config/poolmonitor.ts b/frontend/config/poolmonitor.ts index fd494385..104b5611 100644 --- a/frontend/config/poolmonitor.ts +++ b/frontend/config/poolmonitor.ts @@ -10,6 +10,7 @@ export class PoolMonitor { private readonly config: PoolOptions; private poolClosed = false; private acquiredConnectionIds: Set = new Set(); + private reinitializing = false; constructor(config: PoolOptions) { this.config = config; @@ -20,7 +21,6 @@ export class PoolMonitor { if (!this.acquiredConnectionIds.has(connection.threadId)) { this.acquiredConnectionIds.add(connection.threadId); ++this.activeConnections; - ++this.totalConnectionsCreated; console.log(chalk.green(`Acquired: ${connection.threadId}`)); this.logPoolStatus(); this.resetInactivityTimer(); @@ -50,7 +50,6 @@ export class PoolMonitor { ++this.waitingForConnection; console.log(chalk.magenta('Enqueued.')); this.logPoolStatus(); - this.resetInactivityTimer(); }); // Initialize inactivity timer @@ -62,15 +61,22 @@ export class PoolMonitor { throw new Error('Connection pool is closed'); } + let connection: PoolConnection | null = null; try { console.log(chalk.cyan('Requesting new connection...')); - const connection = await this.pool.getConnection(); + connection = await this.pool.getConnection(); console.log(chalk.green('Connection acquired')); + --this.waitingForConnection; this.resetInactivityTimer(); return connection; } catch (error) { console.error(chalk.red('Error getting connection from pool:', error)); throw error; + } finally { + if (connection) { + connection.release(); + console.log(chalk.blue('Connection released in finally block')); + } } } @@ -84,6 +90,10 @@ export class PoolMonitor { async closeAllConnections(): Promise { try { + if (this.poolClosed) { + console.log(chalk.yellow('Pool already closed.')); + return; + } console.log(chalk.yellow('Ending pool connections...')); await this.pool.end(); this.poolClosed = true; @@ -94,12 +104,22 @@ export class PoolMonitor { } } - public reinitializePool() { - console.log(chalk.cyan('Reinitializing connection pool...')); - this.pool = createPool(this.config); - this.poolClosed = false; - this.acquiredConnectionIds.clear(); - console.log(chalk.cyan('Connection pool reinitialized.')); + async reinitializePool(): Promise { + if (this.reinitializing) return; // Prevent concurrent reinitialization + this.reinitializing = true; + + try { + console.log(chalk.cyan('Reinitializing connection pool...')); + await this.closeAllConnections(); // Ensure old pool is closed + this.pool = createPool(this.config); + this.poolClosed = false; + this.acquiredConnectionIds.clear(); + console.log(chalk.cyan('Connection pool reinitialized.')); + } catch (error) { + console.error(chalk.red('Error during reinitialization:', error)); + } finally { + this.reinitializing = false; + } } public isPoolClosed(): boolean { diff --git a/frontend/config/sqlrdsdefinitions/views.ts b/frontend/config/sqlrdsdefinitions/views.ts index 0e0fff41..c532cc6d 100644 --- a/frontend/config/sqlrdsdefinitions/views.ts +++ b/frontend/config/sqlrdsdefinitions/views.ts @@ -24,6 +24,7 @@ export type AllTaxonomiesViewResult = ResultType; export function getAllTaxonomiesViewHCs(): ColumnStates { return { + speciesID: false, familyID: false, genusID: false, speciesLimits: false @@ -73,6 +74,7 @@ export type MeasurementsSummaryResult = ResultType; export function getMeasurementsSummaryViewHCs(): ColumnStates { return { + coreMeasurementID: false, plotID: false, censusID: false, quadratID: false, @@ -131,6 +133,7 @@ export type StemTaxonomiesViewResult = ResultType; export function getStemTaxonomiesViewHCs(): ColumnStates { return { + stemID: false, treeID: false, speciesID: false, familyID: false, diff --git a/frontend/config/sqlrdsdefinitions/zones.ts b/frontend/config/sqlrdsdefinitions/zones.ts index b6c00c8b..2a6b9c1c 100644 --- a/frontend/config/sqlrdsdefinitions/zones.ts +++ b/frontend/config/sqlrdsdefinitions/zones.ts @@ -107,6 +107,7 @@ export const validateQuadratsRow: ValidationFunction = row => { export function getQuadratHCs(): ColumnStates { return { + quadratID: false, plotID: false, censusID: false }; diff --git a/frontend/config/utils.ts b/frontend/config/utils.ts index 92c2f5dc..e9332db5 100644 --- a/frontend/config/utils.ts +++ b/frontend/config/utils.ts @@ -169,10 +169,7 @@ export async function handleUpsert( throw new Error(`No data provided for upsert operation on table ${tableName}`); } - console.log('handleUpsert data:', data); - const query = createInsertOrUpdateQuery(schema, tableName, data); - console.log('handleUpsert query:', query); const result = await runQuery(connection, query, Object.values(data)); @@ -191,9 +188,6 @@ export async function handleUpsert( const findExistingQuery = `SELECT * FROM \`${schema}\`.\`${tableName}\` WHERE ${whereConditions}`; const values = Object.values(data).filter(value => value !== null); - console.log('handleUpsert findExisting query:', findExistingQuery); - console.log('handleUpsert findExisting values:', values); - const searchResult = await runQuery(connection, findExistingQuery, values); if (searchResult.length > 0) {