From 042d69f43e5eeeef6510e5631301a6ac06537552 Mon Sep 17 00:00:00 2001 From: Dinh Le Date: Tue, 29 Sep 2026 20:05:06 +0700 Subject: [PATCH] fix(node): encode string chunks back to bytes instead of corrupting them A request stream yields string chunks when something upstream calls `setEncoding` on it, or when it is an object-mode stream. `toWebReadableStream` passed them to `new Uint8Array(value)`, which reads a string as a length: '123' became 123 zero bytes and most bodies vanished. The JSON and URL-encoded readers threw a TypeError on the same input, and file bodies ignored the stream's encoding. String chunks are now encoded back with the stream's own encoding (utf8 by default), which recovers the original bytes for base64, hex and latin1 as well. --- packages/node/src/body.test.ts | 23 +++++++++++++++++++++++ packages/node/src/body.ts | 9 ++++----- packages/node/src/utils.test.ts | 24 ++++++++++++++++++++++++ packages/node/src/utils.ts | 9 ++++++++- 4 files changed, 59 insertions(+), 6 deletions(-) diff --git a/packages/node/src/body.test.ts b/packages/node/src/body.test.ts index 394b8d1..891faf5 100644 --- a/packages/node/src/body.test.ts +++ b/packages/node/src/body.test.ts @@ -235,6 +235,29 @@ describe('toStandardBody', () => { }) }) + describe.each(['utf8', 'base64'] as const)('request with %s encoding set', (encoding) => { + let standardBody: any + + async function handler(req: IncomingMessage, res: ServerResponse): Promise { + req.setEncoding(encoding) + standardBody = await toStandardBody(req) + res.end() + } + + it('json', async () => { + await request(handler).post('/').set('standard-server', 'json').send('{"emoji":"😀"}') + + expect(standardBody).toEqual({ emoji: '😀' }) + }) + + it('file', async () => { + await request(handler).post('/').set('standard-server', 'file').send('emoji=😀') + + expect(standardBody).toBeInstanceOf(File) + expect(await standardBody.text()).toBe('emoji=😀') + }) + }) + describe('http2', () => { /** * Runs a request through a real http2 server, so `toStandardBody` receives an diff --git a/packages/node/src/body.ts b/packages/node/src/body.ts index 4ad7eac..d20b0a7 100644 --- a/packages/node/src/body.ts +++ b/packages/node/src/body.ts @@ -1,12 +1,11 @@ import type { StandardBody, StandardBodyHint, StandardHeaders } from '@standard-server/core' -import type { Buffer } from 'node:buffer' import type { ToEventStreamOptions } from './event-stream' import type { NodeHttpRequest } from './types' import { Readable } from 'node:stream' import { generateContentDisposition, getFilenameFromContentDisposition, resolveStandardBodyHint } from '@standard-server/core' import { isAsyncIteratorObject, parseEmptyableJSON, stringifyJSON } from '@standard-server/shared' import { toAsyncIteratorObject, toEventStream } from './event-stream' -import { toWebReadableStream } from './utils' +import { readableChunkToBytes, toWebReadableStream } from './utils' export interface ToStandardBodyOptions { /** @@ -169,7 +168,7 @@ async function _streamToString(stream: Readable): Promise { let string = '' for await (const chunk of stream) { - string += decoder.decode(chunk, { stream: true }) + string += decoder.decode(readableChunkToBytes(stream, chunk), { stream: true }) } // Flush any remaining bytes (e.g. incomplete multi-byte sequences) @@ -179,10 +178,10 @@ async function _streamToString(stream: Readable): Promise { } async function _streamToFile(stream: Readable, fileName: string, contentType: string): Promise { - const chunks: Buffer[] = [] + const chunks: Uint8Array[] = [] for await (const chunk of stream) { - chunks.push(chunk) + chunks.push(readableChunkToBytes(stream, chunk)) } return new File(chunks, fileName, { type: contentType }) diff --git a/packages/node/src/utils.test.ts b/packages/node/src/utils.test.ts index 5c25569..9fd7a98 100644 --- a/packages/node/src/utils.test.ts +++ b/packages/node/src/utils.test.ts @@ -597,6 +597,30 @@ describe('toWebReadableStream', () => { expect(Buffer.concat(received).equals(Buffer.concat(chunks))).toBe(true) }) + it('encodes string chunks of an object mode stream as utf-8', async () => { + const received: Uint8Array[] = [] + for await (const chunk of toWebReadableStream(Readable.from(['123', 'héllo']))) { + expect(chunk).toBeInstanceOf(Uint8Array) + received.push(chunk) + } + + expect(Buffer.concat(received).equals(Buffer.from('123héllo'))).toBe(true) + }) + + it.each(['utf8', 'base64', 'hex', 'latin1'] as const)('recovers the bytes of a stream with %s encoding set as copies', async (encoding) => { + const bytes = Buffer.from('héllo 😀') + const source = Readable.from([bytes.subarray(0, 8), bytes.subarray(8)], { objectMode: false }).setEncoding(encoding) + + const received: Uint8Array[] = [] + for await (const chunk of toWebReadableStream(source)) { + expect(chunk).toBeInstanceOf(Uint8Array) + expect(Buffer.isBuffer(chunk)).toBe(false) + received.push(chunk) + } + + expect(Buffer.concat(received).equals(bytes)).toBe(true) + }) + it('does not throw when a raw buffer stream is cancelled mid-read', async () => { const source = Readable.from((async function* () { for (let i = 0; i < 10_000; i++) { diff --git a/packages/node/src/utils.ts b/packages/node/src/utils.ts index d1805d1..38a758b 100644 --- a/packages/node/src/utils.ts +++ b/packages/node/src/utils.ts @@ -1,6 +1,7 @@ import type { Readable } from 'node:stream' import type Stream from 'node:stream' import type { NodeHttpResponse } from './types' +import { Buffer } from 'node:buffer' import { IncomingMessage } from 'node:http' import { Http2ServerRequest } from 'node:http2' @@ -40,7 +41,7 @@ export function toWebReadableStream(stream: Readable): ReadableStream | string): Uint8Array { + return typeof chunk === 'string' + ? Buffer.from(chunk, stream.readableEncoding ?? 'utf8') + : chunk +} + async function _drainIterator(iterator: AsyncIterator): Promise { while (!(await iterator.next()).done) { // discard