Skip to content

Commit

Permalink
feat(keystone): DOMA-10613 normal excel date format parsing (#5516)
Browse files Browse the repository at this point in the history
  • Loading branch information
@YEgorLu
YEgorLu authored Nov 21, 2024
1 parent 4e682de commit 174e9c3
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 225 deletions.
44 changes: 7 additions & 37 deletions apps/condo/domains/meter/utils/taskSchema/DomaMetersImporter.js
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,8 @@
const dayjs = require('dayjs')
const utc = require('dayjs/plugin/utc')
const { get, isEqual, isNil, has } = require('lodash')

const { ExcelParser } = require('@open-condo/keystone/file/file-types/excel')

const { AbstractMetersImporter } = require('./AbstractMetersImporter')
const { TransformRowError } = require('./MetersDataImporterTypes')

dayjs.extend(utc)

const DATE_COLUMN_INDEXES = [
11,
12,
13,
14,
15,
16,
17,
]

class DomaMetersImporter extends AbstractMetersImporter {
hasColumnsHeaders () {
return true
Expand Down Expand Up @@ -57,24 +41,10 @@ class DomaMetersImporter extends AbstractMetersImporter {
errors.push(this.errors.unknownIsAutomatic.message)
}

// do not put utc date in row, this will be printed in pdf if errors
const excelParsedDates = {}
// date can be os type text or excel date type, which parses number
DATE_COLUMN_INDEXES
.filter(index => ExcelParser.isExcelDate(row[index]))
.forEach(index => {
excelParsedDates[index] = ExcelParser.parseExcelDate(+row[index])
row[index] = dayjs.utc(excelParsedDates[index]).format('YYYY-MM-DD HH:mm:ss')
})

if (errors.length > 0) {
throw new TransformRowError(errors)
}

const getDateString = (row, index) => {
return excelParsedDates[index] || row[index]
}

return {
address: row[0],
addressInfo: {
Expand All @@ -84,20 +54,20 @@ class DomaMetersImporter extends AbstractMetersImporter {
accountNumber: row[3],
meterNumber: row[5],
meterResource: { id: this.mappers.resourceId[row[4]] },
date: getDateString(row, 11),
date: row[11],
value1: isNil(row[7]) ? undefined : row[7],
value2: isNil(row[8]) ? undefined : row[8],
value3: isNil(row[9]) ? undefined : row[9],
value4: isNil(row[10]) ? undefined : row[10],
meterMeta: {
numberOfTariffs: Number(row[6]),
place: row[18],
verificationDate: getDateString(row, 12),
nextVerificationDate: getDateString(row, 13),
installationDate: getDateString(row, 14),
commissioningDate: getDateString(row, 15),
sealingDate: getDateString(row, 16),
controlReadingsDate: getDateString(row, 17),
verificationDate: row[12],
nextVerificationDate: row[13],
installationDate: row[14],
commissioningDate: row[15],
sealingDate: row[16],
controlReadingsDate: row[17],
isAutomatic: get(this, ['mappers', 'isAutomatic', cell19Value]),
},
}
Expand Down
129 changes: 0 additions & 129 deletions apps/condo/domains/meter/utils/taskSchema/DomaMetersImporter.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -291,133 +291,4 @@ describe('DomaMetersImporter', () => {
expect(errors).toEqual([['unknownUnitType'], ['unknownResource'], ['unknownIsAutomatic']])
})

test('Transforms excel dates to format YYYY-MM-DD', () => {
// see packages/keystone/file/file-types/excel for more info about strange dates
const excelDate = '45532'
const expectedDate = '2024-08-28T00:00:00.000Z'
const excelDateWithTime = '45557.16903935185'
const expectedDateWithTime = '2024-09-22T04:03:25.000Z'

const fakeAddress = faker.address.streetAddress()
const rows = [
// address, unitName, unitType, accountNumber, meterType, meterNumber, tariffs, v1, v2, v3, v4, date, verificationDate, nextVerificationDate, installationDate, commissioningDate, sealingDate, controlReadingsDate, place, isAutomatic
[
fakeAddress,
'1',
'Квартира',
'001',
'ГВС',
'0001',
'1',
'101.1',
null,
'',
'',
excelDate,
excelDate,
excelDate,
excelDate,
excelDate,
excelDate,
excelDate,
'',
'yes',
],
[
fakeAddress,
'1',
'Квартира',
'001',
'ГВС',
'0001',
'1',
'101.1',
null,
'',
'',
excelDateWithTime,
excelDateWithTime,
excelDateWithTime,
excelDateWithTime,
excelDateWithTime,
excelDateWithTime,
excelDateWithTime,
'',
'yes',
],
]

const importer = new ImporterWrapper()
const result = []
const errors = []
for (const row of rows) {
try {
const transformedRow = importer.transformRowWrapper(row)
result.push(transformedRow)
} catch (err) {
errors.push(err.getMessages())
}
}

expect(result).toEqual([
{
address: fakeAddress,
addressInfo: {
unitType: 'flat',
unitName: '1',
},
accountNumber: '001',
meterNumber: '0001',
meterResource: {
id: '0f54223c-0631-11ec-9a03-0242ac130003',
},
date: expectedDate,
value1: '101.1',
value2: undefined,
value3: '',
value4: '',
meterMeta: {
numberOfTariffs: 1,
place: '',
verificationDate: expectedDate,
nextVerificationDate: expectedDate,
installationDate: expectedDate,
commissioningDate: expectedDate,
sealingDate: expectedDate,
controlReadingsDate: expectedDate,
isAutomatic: true,
},
},
{
address: fakeAddress,
addressInfo: {
unitType: 'flat',
unitName: '1',
},
accountNumber: '001',
meterNumber: '0001',
meterResource: {
id: '0f54223c-0631-11ec-9a03-0242ac130003',
},
date: expectedDateWithTime,
value1: '101.1',
value2: undefined,
value3: '',
value4: '',
meterMeta: {
numberOfTariffs: 1,
place: '',
verificationDate: expectedDateWithTime,
nextVerificationDate: expectedDateWithTime,
installationDate: expectedDateWithTime,
commissioningDate: expectedDateWithTime,
sealingDate: expectedDateWithTime,
controlReadingsDate: expectedDateWithTime,
isAutomatic: true,
},
},
])

expect(errors).toHaveLength(0)
})
})
69 changes: 10 additions & 59 deletions packages/keystone/file/file-types/excel.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
const excel = require('xlsx')

const FIVE_OR_MORE_DIGITS_IN_STRING = /^\d{5,}$/

class ExcelParser {

Expand Down Expand Up @@ -28,67 +27,20 @@ class ExcelParser {
return true
}

/**
* Excel stores date types as numbers <days from epoch>.<time data>
* @param serial {number} date from excel parser
* @returns {string} YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS
/** @param {excel.CellObject} cell
* @returns {string | undefined}
*/
static parseExcelDate (serial) {
const withTime = String(serial).includes('.')

// milliseconds since 1899-12-31T00:00:00Z, corresponds to Excel serial 0.
const xlSerialOffset = -2209075200000

let elapsedDays
// each serial up to 60 corresponds to a valid calendar date.
// serial 60 is 1900-02-29. This date does not exist on the calendar.
// we choose to interpret serial 60 (as well as 61) both as 1900-03-01
// so, if the serial is 61 or over, we have to subtract 1.
if (serial < 61) {
elapsedDays = serial
}
else {
elapsedDays = serial - 1
parseCell (cell) {
const isDefinedDate = cell && cell.t === 'd' && cell.v instanceof Date
if (isDefinedDate) {
return cell.v.toISOString()
} else if (cell) {
return cell.v
}

// javascript dates ignore leap seconds
// each day corresponds to a fixed number of milliseconds:
// 24 hrs * 60 mins * 60 s * 1000 ms
const millisPerDay = 86400000

const jsTimestamp = xlSerialOffset + elapsedDays * millisPerDay
const date = new Date(jsTimestamp)
const year = date.getUTCFullYear()
const month = (date.getUTCMonth() + 1).toString().padStart(2, '0')
const day = date.getUTCDate().toString().padStart(2, '0')

let dateString = `${year}-${month}-${day}`
let hours = '00'
let minutes = '00'
let seconds = '00'
let milliseconds = '000'
if (withTime) {
hours = date.getUTCHours().toString().padStart(2, '0')
minutes = date.getUTCMinutes().toString().padStart(2, '0')
seconds = date.getUTCSeconds().toString().padStart(2, '0')
milliseconds = date.getUTCMilliseconds().toString().padStart(3, '0')
}
dateString += `T${hours}:${minutes}:${seconds}.${milliseconds}Z`
return dateString
}

static isExcelDate (serial) {
if (typeof serial !== 'number' && !serial) {
return false
}
const [daysFromEpoch] = serial.toString().split('.')

// 2020-10-31 stands for 44927 days from epoch, so must be enough to not mistake with YYYY.MM
return daysFromEpoch && FIVE_OR_MORE_DIGITS_IN_STRING.test(daysFromEpoch)
}

async parse () {
const workbook = excel.read(this.buffer, { type: 'buffer' })
const workbook = excel.read(this.buffer, { type: 'buffer', cellDates: true })
const rows = []
workbook.SheetNames.forEach((sheetName) => {
const worksheet = workbook.Sheets[sheetName]
Expand All @@ -97,8 +49,7 @@ class ExcelParser {
const row = []
for (let colNum = range.s.c; colNum <= range.e.c; colNum++) {
const cellAddress = excel.utils.encode_cell({ r: rowNum, c: colNum })
const cellValue = worksheet[cellAddress] ? worksheet[cellAddress].v : undefined
row.push(cellValue)
row.push(this.parseCell(worksheet[cellAddress]))
}
rows.push(row)
}
Expand Down

0 comments on commit 174e9c3

Please sign in to comment.