fix(node): encode string chunks back to bytes instead of corrupting them

packages/node/src/body.test.ts

@@ -235,6 +235,29 @@ describe('toStandardBody', () => {
     })
   })
 
+  describe.each(['utf8', 'base64'] as const)('request with %s encoding set', (encoding) => {
+    let standardBody: any
+
+    async function handler(req: IncomingMessage, res: ServerResponse): Promise<void> {
+      req.setEncoding(encoding)
+      standardBody = await toStandardBody(req)
+      res.end()
+    }
+
+    it('json', async () => {
+      await request(handler).post('/').set('standard-server', 'json').send('{"emoji":"😀"}')
+
+      expect(standardBody).toEqual({ emoji: '😀' })
+    })
+
+    it('file', async () => {
+      await request(handler).post('/').set('standard-server', 'file').send('emoji=😀')
+
+      expect(standardBody).toBeInstanceOf(File)
+      expect(await standardBody.text()).toBe('emoji=😀')
+    })
+  })
+
   describe('http2', () => {
     /**
      * Runs a request through a real http2 server, so `toStandardBody` receives an

packages/node/src/body.ts

@@ -1,12 +1,11 @@
 import type { StandardBody, StandardBodyHint, StandardHeaders } from '@standard-server/core'
-import type { Buffer } from 'node:buffer'
 import type { ToEventStreamOptions } from './event-stream'
 import type { NodeHttpRequest } from './types'
 import { Readable } from 'node:stream'
 import { generateContentDisposition, getFilenameFromContentDisposition, resolveStandardBodyHint } from '@standard-server/core'
 import { isAsyncIteratorObject, parseEmptyableJSON, stringifyJSON } from '@standard-server/shared'
 import { toAsyncIteratorObject, toEventStream } from './event-stream'
-import { toWebReadableStream } from './utils'
+import { readableChunkToBytes, toWebReadableStream } from './utils'
 
 export interface ToStandardBodyOptions {
   /**
@@ -169,7 +168,7 @@ async function _streamToString(stream: Readable): Promise<string> {
   let string = ''
 
   for await (const chunk of stream) {
-    string += decoder.decode(chunk, { stream: true })
+    string += decoder.decode(readableChunkToBytes(stream, chunk), { stream: true })
   }
 
   // Flush any remaining bytes (e.g. incomplete multi-byte sequences)
@@ -179,10 +178,10 @@ async function _streamToString(stream: Readable): Promise<string> {
 }
 
 async function _streamToFile(stream: Readable, fileName: string, contentType: string): Promise<File> {
-  const chunks: Buffer<ArrayBuffer>[] = []
+  const chunks: Uint8Array<ArrayBuffer>[] = []
 
   for await (const chunk of stream) {
-    chunks.push(chunk)
+    chunks.push(readableChunkToBytes(stream, chunk))
   }
 
   return new File(chunks, fileName, { type: contentType })

packages/node/src/utils.test.ts

@@ -597,6 +597,30 @@ describe('toWebReadableStream', () => {
     expect(Buffer.concat(received).equals(Buffer.concat(chunks))).toBe(true)
   })
 
+  it('encodes string chunks of an object mode stream as utf-8', async () => {
+    const received: Uint8Array[] = []
+    for await (const chunk of toWebReadableStream(Readable.from(['123', 'héllo']))) {
+      expect(chunk).toBeInstanceOf(Uint8Array)
+      received.push(chunk)
+    }
+
+    expect(Buffer.concat(received).equals(Buffer.from('123héllo'))).toBe(true)
+  })
+
+  it.each(['utf8', 'base64', 'hex', 'latin1'] as const)('recovers the bytes of a stream with %s encoding set as copies', async (encoding) => {
+    const bytes = Buffer.from('héllo 😀')
+    const source = Readable.from([bytes.subarray(0, 8), bytes.subarray(8)], { objectMode: false }).setEncoding(encoding)
+
+    const received: Uint8Array[] = []
+    for await (const chunk of toWebReadableStream(source)) {
+      expect(chunk).toBeInstanceOf(Uint8Array)
+      expect(Buffer.isBuffer(chunk)).toBe(false)
+      received.push(chunk)
+    }
+
+    expect(Buffer.concat(received).equals(bytes)).toBe(true)
+  })
+
   it('does not throw when a raw buffer stream is cancelled mid-read', async () => {
     const source = Readable.from((async function* () {
       for (let i = 0; i < 10_000; i++) {

packages/node/src/utils.ts

@@ -1,6 +1,7 @@
 import type { Readable } from 'node:stream'
 import type Stream from 'node:stream'
 import type { NodeHttpResponse } from './types'
+import { Buffer } from 'node:buffer'
 import { IncomingMessage } from 'node:http'
 import { Http2ServerRequest } from 'node:http2'
 
@@ -40,7 +41,7 @@ export function toWebReadableStream(stream: Readable): ReadableStream<Uint8Array
         controller.close()
       }
       else {
-        controller.enqueue(new Uint8Array(value))
+        controller.enqueue(new Uint8Array(readableChunkToBytes(stream, value)))
       }
     },
     cancel(reason) {
@@ -60,6 +61,12 @@ export function toWebReadableStream(stream: Readable): ReadableStream<Uint8Array
   })
 }
 
+export function readableChunkToBytes(stream: Readable, chunk: Uint8Array<ArrayBuffer> | string): Uint8Array<ArrayBuffer> {
+  return typeof chunk === 'string'
+    ? Buffer.from(chunk, stream.readableEncoding ?? 'utf8')
+    : chunk
+}
+
 async function _drainIterator(iterator: AsyncIterator<unknown>): Promise<void> {
   while (!(await iterator.next()).done) {
     // discard