Skip to content

Commit 2d50d6a

Browse files
waleedlatif1waleedlatif1
committed
fix(people): honor the 1904 date system and cover the sheet reader with workbook tests
1 parent 3ff0517 commit 2d50d6a

3 files changed

Lines changed: 152 additions & 65 deletions

File tree

‎apps/sim/app/o/[organizationId]/people/people-import-modal.tsx‎

Lines changed: 5 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -14,78 +14,18 @@ import {
1414
useScrollEdges,
1515
} from '@sim/emcn'
1616
import { getErrorMessage } from '@sim/utils/errors'
17-
import { PEOPLE_IMPORT_MAX_ROWS, type PeopleImportResult } from '@/lib/api/contracts/people'
18-
import { assertOoxmlPreviewWithinLimits } from '@/lib/file-parsers/ooxml-preview-guard'
17+
import type { PeopleImportResult } from '@/lib/api/contracts/people'
1918
import { saveBlob } from '@/lib/uploads/client/download'
19+
import {
20+
type PeopleSheet,
21+
readPeopleSheet,
22+
} from '@/app/o/[organizationId]/people/read-people-sheet'
2023
import { useImportPeople } from '@/hooks/queries/people'
2124

22-
const MAX_FILE_BYTES = 5 * 1024 * 1024
2325
const ACCEPTED_FILES = '.csv,.tsv,.xlsx,.xls'
2426
const TEMPLATE_CSV =
2527
'email,title,department,location,start_date,manager_email\nada@example.com,Engineer,Platform,Berlin,2024-01-31,grace@example.com\n'
2628

27-
interface PeopleSheet {
28-
fileName: string
29-
headers: string[]
30-
/** Each row's cells, in `headers` order. */
31-
rows: string[][]
32-
}
33-
34-
/**
35-
* The first sheet's named columns and its rows of cells, one per sheet row (blank ones included) so the
36-
* row numbers in the preview match the spreadsheet. Delimited text is read verbatim so the server
37-
* interprets dates the same way for every file. A spreadsheet date cell is read from its stored
38-
* value, not the text its locale displays, so `31/01/2024` and `1/31/24` both arrive as
39-
* `2024-01-31`.
40-
*/
41-
async function readPeopleSheet(file: File): Promise<PeopleSheet> {
42-
if (file.size > MAX_FILE_BYTES) throw new Error('Choose a file under 5 MB')
43-
const data = new Uint8Array(await file.arrayBuffer())
44-
if (/\.xlsx$/i.test(file.name)) await assertOoxmlPreviewWithinLimits(data)
45-
const XLSX = await import('xlsx')
46-
const workbook = XLSX.read(data, {
47-
type: 'array',
48-
raw: /\.(csv|tsv)$/i.test(file.name),
49-
dense: true,
50-
// Keeps each cell's number format, which is how a date cell is told from a number.
51-
cellNF: true,
52-
})
53-
const sheet = workbook.Sheets[workbook.SheetNames[0] ?? '']
54-
if (!sheet?.['!ref']) throw new Error('The file has no rows')
55-
const range = XLSX.utils.decode_range(sheet['!ref'])
56-
const grid = sheet['!data'] ?? []
57-
const text = (row: number, column: number): string => {
58-
const cell = grid[row]?.[column]
59-
if (!cell || cell.v === undefined || cell.v === null) return ''
60-
if (cell.t === 'n' && XLSX.SSF.is_date(cell.z ?? '')) {
61-
const date = XLSX.SSF.parse_date_code(Number(cell.v))
62-
return `${date.y}-${String(date.m).padStart(2, '0')}-${String(date.d).padStart(2, '0')}`
63-
}
64-
return String(cell.w ?? cell.v).trim()
65-
}
66-
const columns: { header: string; index: number }[] = []
67-
for (let index = range.s.c; index <= range.e.c; index++) {
68-
const header = text(range.s.r, index)
69-
if (!header) continue
70-
if (columns.some((column) => column.header === header)) {
71-
throw new Error(`The ${header} column appears more than once`)
72-
}
73-
columns.push({ header, index })
74-
}
75-
if (!columns.length) throw new Error('The first row should name the columns')
76-
const rowCount = range.e.r - range.s.r
77-
if (rowCount > PEOPLE_IMPORT_MAX_ROWS) {
78-
throw new Error(`Import at most ${PEOPLE_IMPORT_MAX_ROWS} rows at a time`)
79-
}
80-
return {
81-
fileName: file.name,
82-
headers: columns.map(({ header }) => header),
83-
rows: Array.from({ length: rowCount }, (_, offset) =>
84-
columns.map(({ index }) => text(range.s.r + 1 + offset, index))
85-
),
86-
}
87-
}
88-
8929
function downloadTemplate() {
9030
saveBlob(new Blob([TEMPLATE_CSV], { type: 'text/csv' }), 'people-template.csv')
9131
}

‎apps/sim/app/o/[organizationId]/people/read-people-sheet.test.ts‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
import { describe, expect, it } from 'vitest'
2+
import * as XLSX from 'xlsx'
3+
import { readPeopleSheet } from '@/app/o/[organizationId]/people/read-people-sheet'
4+
5+
/** January 31, 2024 as a date serial in each of Excel's two date systems. */
6+
const SERIAL_1900 = 45322
7+
const SERIAL_1904 = SERIAL_1900 - 1462
8+
9+
function workbookFile(rows: unknown[][], { format = 'm/d/yy', date1904 = false } = {}): File {
10+
const sheet = XLSX.utils.aoa_to_sheet(rows, { dense: false })
11+
for (const [address, cell] of Object.entries(sheet)) {
12+
if (!address.startsWith('!') && (cell as XLSX.CellObject).t === 'n') {
13+
;(cell as XLSX.CellObject).z = format
14+
}
15+
}
16+
const workbook = XLSX.utils.book_new()
17+
XLSX.utils.book_append_sheet(workbook, sheet, 'People')
18+
workbook.Workbook = { WBProps: { date1904 } }
19+
const bytes = XLSX.write(workbook, { type: 'array', bookType: 'xlsx' })
20+
return new File([bytes], 'people.xlsx')
21+
}
22+
23+
describe('readPeopleSheet', () => {
24+
it.each(['m/d/yy', 'dd/mm/yyyy', 'yyyy-mm-dd'])(
25+
'reads a date cell formatted %s as its calendar date, not its displayed text',
26+
async (format) => {
27+
const sheet = await readPeopleSheet(
28+
workbookFile(
29+
[
30+
['email', 'start date'],
31+
['ada@example.com', SERIAL_1900],
32+
],
33+
{ format }
34+
)
35+
)
36+
expect(sheet.rows).toEqual([['ada@example.com', '2024-01-31']])
37+
}
38+
)
39+
40+
it('reads dates from a workbook saved in the 1904 date system', async () => {
41+
const sheet = await readPeopleSheet(
42+
workbookFile(
43+
[
44+
['email', 'start date'],
45+
['ada@example.com', SERIAL_1904],
46+
],
47+
{ date1904: true }
48+
)
49+
)
50+
expect(sheet.rows).toEqual([['ada@example.com', '2024-01-31']])
51+
})
52+
53+
it('keeps blank rows in place so issue row numbers match the sheet', async () => {
54+
const sheet = await readPeopleSheet(
55+
workbookFile([
56+
['email', 'title'],
57+
['ada@example.com', 'Engineer'],
58+
[],
59+
['bo@example.com', 'Lead'],
60+
])
61+
)
62+
expect(sheet.rows).toEqual([
63+
['ada@example.com', 'Engineer'],
64+
['', ''],
65+
['bo@example.com', 'Lead'],
66+
])
67+
})
68+
69+
it('refuses a header row that repeats a column name', async () => {
70+
await expect(
71+
readPeopleSheet(
72+
workbookFile([
73+
['email', 'title', 'title'],
74+
['ada@example.com', 'a', 'b'],
75+
])
76+
)
77+
).rejects.toThrow('The title column appears more than once')
78+
})
79+
})

‎apps/sim/app/o/[organizationId]/people/read-people-sheet.ts‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import { PEOPLE_IMPORT_MAX_ROWS } from '@/lib/api/contracts/people'
2+
import { assertOoxmlPreviewWithinLimits } from '@/lib/file-parsers/ooxml-preview-guard'
3+
4+
const MAX_FILE_BYTES = 5 * 1024 * 1024
5+
6+
export interface PeopleSheet {
7+
fileName: string
8+
headers: string[]
9+
/** Each row's cells, in `headers` order. */
10+
rows: string[][]
11+
}
12+
13+
/**
14+
* The first sheet's named columns and its rows of cells, one per sheet row (blank ones included) so the
15+
* row numbers in the preview match the spreadsheet. Delimited text is read verbatim so the server
16+
* interprets dates the same way for every file. A spreadsheet date cell is read from its stored
17+
* value, not the text its locale displays, so `31/01/2024` and `1/31/24` both arrive as
18+
* `2024-01-31`.
19+
*/
20+
export async function readPeopleSheet(file: File): Promise<PeopleSheet> {
21+
if (file.size > MAX_FILE_BYTES) throw new Error('Choose a file under 5 MB')
22+
const data = new Uint8Array(await file.arrayBuffer())
23+
if (/\.xlsx$/i.test(file.name)) await assertOoxmlPreviewWithinLimits(data)
24+
const XLSX = await import('xlsx')
25+
const workbook = XLSX.read(data, {
26+
type: 'array',
27+
raw: /\.(csv|tsv)$/i.test(file.name),
28+
dense: true,
29+
// Keeps each cell's number format, which is how a date cell is told from a number.
30+
cellNF: true,
31+
})
32+
const sheet = workbook.Sheets[workbook.SheetNames[0] ?? '']
33+
// Workbooks saved in the 1904 date system count their date serials from a different epoch.
34+
const dateSystem = { date1904: Boolean(workbook.Workbook?.WBProps?.date1904) }
35+
if (!sheet?.['!ref']) throw new Error('The file has no rows')
36+
const range = XLSX.utils.decode_range(sheet['!ref'])
37+
const grid = sheet['!data'] ?? []
38+
const text = (row: number, column: number): string => {
39+
const cell = grid[row]?.[column]
40+
if (!cell || cell.v === undefined || cell.v === null) return ''
41+
if (cell.t === 'n' && XLSX.SSF.is_date(cell.z ?? '')) {
42+
const date = XLSX.SSF.parse_date_code(Number(cell.v), dateSystem)
43+
return `${date.y}-${String(date.m).padStart(2, '0')}-${String(date.d).padStart(2, '0')}`
44+
}
45+
return String(cell.w ?? cell.v).trim()
46+
}
47+
const columns: { header: string; index: number }[] = []
48+
for (let index = range.s.c; index <= range.e.c; index++) {
49+
const header = text(range.s.r, index)
50+
if (!header) continue
51+
if (columns.some((column) => column.header === header)) {
52+
throw new Error(`The ${header} column appears more than once`)
53+
}
54+
columns.push({ header, index })
55+
}
56+
if (!columns.length) throw new Error('The first row should name the columns')
57+
const rowCount = range.e.r - range.s.r
58+
if (rowCount > PEOPLE_IMPORT_MAX_ROWS) {
59+
throw new Error(`Import at most ${PEOPLE_IMPORT_MAX_ROWS} rows at a time`)
60+
}
61+
return {
62+
fileName: file.name,
63+
headers: columns.map(({ header }) => header),
64+
rows: Array.from({ length: rowCount }, (_, offset) =>
65+
columns.map(({ index }) => text(range.s.r + 1 + offset, index))
66+
),
67+
}
68+
}

0 commit comments

Comments
 (0)