fs: enable chunked reading for large files in readFileHandle

lib/fs.js

@@ -74,7 +74,6 @@
   aggregateTwoErrors,
   codes: {
     ERR_ACCESS_DENIED,
-    ERR_FS_FILE_TOO_LARGE,
     ERR_INVALID_ARG_VALUE,
   },
 } = require('internal/errors');
@@ -98,7 +97,6 @@
 } = require('internal/util');
 const {
   constants: {
-    kIoMaxLength,
     kMaxUserId,
   },
   copyObject,
@@ -361,13 +359,8 @@
   // TODO(BridgeAR): Check if allocating a smaller chunk is better performance
   // wise, similar to the promise based version (less peak memory and chunked
   // stringify operations vs multiple C++/JS boundary crossings).
   const size = context.size = isFileType(stats, S_IFREG) ? stats[8] : 0;
 
-  if (size > kIoMaxLength) {
-    err = new ERR_FS_FILE_TOO_LARGE(size);
-    return context.close(err);
-  }
-
   try {
     context.prepare();
   } catch (err) {
@@ -447,9 +440,6 @@
   let threw = true;
   let buffer;
   try {
-    if (size > kIoMaxLength) {
-      throw new ERR_FS_FILE_TOO_LARGE(size);
-    }
     buffer = Buffer.allocUnsafe(size);
     threw = false;
   } finally {

lib/internal/fs/promises.js

@@ -1161,7 +1161,7 @@
   } while (remaining > 0);
 }
 
 async function readFileHandleWithUserBuffer(filehandle, options, size) {
   const signal = options?.signal;
   const encoding = options?.encoding;
   const buffer = getReadFileBuffer(options, size);
@@ -1228,21 +1228,25 @@
 
   let size = 0;
   let length = 0;
-  if ((statFields[1/* mode */] & S_IFMT) === S_IFREG) {
-    size = statFields[8/* size */];
+
+  if ((statFields[1 /* mode */] & S_IFMT) === S_IFREG) {
+    size = statFields[8 /* size */];
+
+    if (size > kIoMaxLength) {
+      const limit = kIoMaxLength;
+      process.emitWarning(
+        `Detected \`fs.readFile()\` to read a file larger than the recommended limit (${size} > ${limit} bytes). Please consider using \`fs.createReadStream()\` instead to minimize memory overhead and increase performance.`,
+        'LargeFileWarning',
+        ERR_FS_FILE_TOO_LARGE,
+      );
+    }
+
     length = encoding ? MathMin(size, kReadFileBufferLength) : size;
   }
   if (length === 0) {
     length = kReadFileUnknownBufferLength;
   }
 
-  if (size > kIoMaxLength)
-    throw new ERR_FS_FILE_TOO_LARGE(size);
-
-  if (options.buffer !== undefined) {
-    return readFileHandleWithUserBuffer(filehandle, options, size);
-  }
-
   let totalRead = 0;
   const noSize = size === 0;
   let buffer = Buffer.allocUnsafeSlow(length);
@@ -1266,8 +1270,8 @@
     totalRead += bytesRead;
 
     if (bytesRead === 0 ||
-        totalRead === size ||
-        (bytesRead !== buffer.length && !chunkedRead && !noSize)) {
+      totalRead === size ||
+      (bytesRead !== buffer.length && !chunkedRead && !noSize)) {
       const singleRead = bytesRead === totalRead;
 
       const bytesToCheck = chunkedRead ? totalRead : bytesRead;
@@ -1290,9 +1294,10 @@
       result += decoder.end(buffer);
       return result;
     }
-    const readBuffer = bytesRead !== buffer.length ?
-      buffer.subarray(0, bytesRead) :
-      buffer;
+    const readBuffer =
+      bytesRead !== buffer.length ?
+        buffer.subarray(0, bytesRead) :
+        buffer;
     if (encoding) {
       result += decoder.write(readBuffer);
     } else if (size !== 0) {

test/parallel/test-fs-promises-file-handle-readFile.js

@@ -1,3 +1,3 @@
 'use strict';
 
 const common = require('../common');
@@ -5,19 +5,24 @@
 // The following tests validate base functionality for the fs.promises
 // FileHandle.readFile method.
 
-const fs = require('fs');
+const fs = require('node:fs');
 const {
   open,
   readFile,
-  writeFile,
   truncate,
+  writeFile,
 } = fs.promises;
-const path = require('path');
+const path = require('node:path');
+const os = require('node:os');
 const tmpdir = require('../common/tmpdir');
 const tick = require('../common/tick');
-const assert = require('assert');
+const assert = require('node:assert');
 const tmpDir = tmpdir.path;
 
+const skipMessage = 'intensive toString tests due to memory confinements';
+if (!common.enoughTestMem)
+  common.skip(skipMessage);
+
 tmpdir.refresh();
 
 async function validateReadFile() {
@@ -33,6 +38,34 @@
   assert.deepStrictEqual(buffer, readFileData);
 }
 
+async function validateLargeFileSupport() {
+  const LARGE_FILE_SIZE = 2147483647 + 10 * 1024 * 1024; // INT32_MAX + 10 MB
+  const FILE_PATH = path.join(os.tmpdir(), 'large-virtual-file.bin');
+
+  if (!tmpdir.hasEnoughSpace(LARGE_FILE_SIZE)) {
+    common.printSkipMessage(`Not enough space in ${os.tmpdir()}`);
+    return;
+  }
+
+  function createVirtualLargeFile() {
+    return Buffer.alloc(LARGE_FILE_SIZE, 'A');
+  }
+
+  const virtualFile = createVirtualLargeFile();
+
+  await writeFile(FILE_PATH, virtualFile);
+
+  const buffer = await readFile(FILE_PATH);
+
+  assert.strictEqual(
+    buffer.length,
+    LARGE_FILE_SIZE,
+    `Expected file size to be ${LARGE_FILE_SIZE}, but got ${buffer.length}`
+  );
+
+  await fs.promises.unlink(FILE_PATH);
+}
+
 async function validateReadFileProc() {
   // Test to make sure reading a file under the /proc directory works. Adapted
   // from test-fs-read-file-sync-hostname.js.
@@ -92,30 +125,43 @@
     }, 'tick-1');
   }
 
-  // Validate file size is within range for reading
+  // For validates the ability of the filesystem module to handle large files
   {
-    // Variable taken from https://github.com/nodejs/node/blob/1377163f3351/lib/internal/fs/promises.js#L5
-    const kIoMaxLength = 2 ** 31 - 1;
+    const largeFileSize = 5 * 1024 * 1024 * 1024; // 5 GiB
+    const newFile = path.resolve(tmpDir, 'dogs-running3.txt');
 
-    if (!tmpdir.hasEnoughSpace(kIoMaxLength)) {
-      // truncate() will fail with ENOSPC if there is not enough space.
+    if (!tmpdir.hasEnoughSpace(largeFileSize)) {
       common.printSkipMessage(`Not enough space in ${tmpDir}`);
     } else {
-      const newFile = path.resolve(tmpDir, 'dogs-running3.txt');
       await writeFile(newFile, Buffer.from('0'));
-      await truncate(newFile, kIoMaxLength + 1);
+      await truncate(newFile, largeFileSize);
 
       await using fileHandle = await open(newFile, 'r');
 
-      await assert.rejects(fileHandle.readFile(), {
-        name: 'RangeError',
-        code: 'ERR_FS_FILE_TOO_LARGE'
+      let warningEmitted = false;
+      let warningMessage = '';
+      process.once('warning', (warning) => {
+        if (warning.name === 'LargeFileWarning') {
+          warningEmitted = true;
+          warningMessage = warning.message;
+        }
       });
+
+      const data = await fileHandle.readFile();
+
+      const expectedWarningMsg = 'Expected LargeFileWarning to be emitted for 5GB file';
+      assert.strictEqual(warningEmitted, true, expectedWarningMsg);
+
+      assert.match(warningMessage,
+                   /larger than the recommended limit \(\d+ > \d+ bytes\)/);
+
+      console.log(`File read successfully, size: ${data.length} bytes`);
     }
   }
 }
 
 validateReadFile()
+  .then(validateLargeFileSupport)
   .then(validateReadFileProc)
   .then(doReadAndCancel)
   .then(common.mustCall());

test/parallel/test-fs-readfile.js

@@ -6,6 +6,7 @@ const common = require('../common');
 
 const tmpdir = require('../../test/common/tmpdir');
 const assert = require('assert');
+const path = require('node:path');
 const fs = require('fs');
 
 const prefix = `.removeme-fs-readfile-${process.pid}`;
@@ -52,28 +53,19 @@ for (const e of fileInfo) {
   }));
 }
 
-// readFile() and readFileSync() should fail if the file is too big.
+// Test to verify that readFile() and readFileSync() can handle large files
 {
-  const kIoMaxLength = 2 ** 31 - 1;
+  const kLargeFileSize = 3 * 1024 * 1024 * 1024; // 3 GiB
 
-  if (!tmpdir.hasEnoughSpace(kIoMaxLength)) {
+  if (!tmpdir.hasEnoughSpace(kLargeFileSize)) {
     // truncateSync() will fail with ENOSPC if there is not enough space.
     common.printSkipMessage(`Not enough space in ${tmpdir.path}`);
   } else {
-    const file = tmpdir.resolve(`${prefix}-too-large.txt`);
-    fs.writeFileSync(file, Buffer.from('0'));
-    fs.truncateSync(file, kIoMaxLength + 1);
-
-    fs.readFile(file, common.expectsError({
-      code: 'ERR_FS_FILE_TOO_LARGE',
-      name: 'RangeError',
-    }));
-    assert.throws(() => {
-      fs.readFileSync(file);
-    }, { code: 'ERR_FS_FILE_TOO_LARGE', name: 'RangeError' });
+    const file = path.join(tmpdir.path, 'temp-large-file.txt');
+    fs.writeFileSync(file, Buffer.alloc(1024));
+    fs.truncateSync(file, kLargeFileSize);
   }
 }
-
 {
   // Test cancellation, before
   const signal = AbortSignal.abort();