feat(metro-file-map): Lazily stat files and populate symlinks for Node crawled file trees

packages/metro-file-map/API.md

@@ -139,6 +139,7 @@ interface FileSystem_2 {
       subpath?: string | undefined;
     }>,
   ): {changedFiles: FileData; removedFiles: Set<string>};
+  getMtimeByNormalPath(file: Path): null | undefined | number;
   getOrComputeSha1(file: Path): Promise<null | undefined | {sha1: string; content?: Buffer | undefined}>;
   getSerializableSnapshot(): CacheData['fileSystemData'];
   getSha1(file: Path): null | undefined | string;

packages/metro-file-map/src/__tests__/index-test.js

@@ -176,6 +176,19 @@ jest.mock('fs', () => ({
     error.code = 'ENOENT';
     throw error;
   }),
+  readlinkSync: jest.fn(path => {
+    const entry = mockFs[path];
+    if (!entry) {
+      const error = new Error(`Cannot read path '${path}'.`);
+      // $FlowFixMe[prop-missing] code
+      error.code = 'ENOENT';
+      throw error;
+    }
+    if (typeof entry.link !== 'string') {
+      throw new Error(`Not a symlink: '${path}'.`);
+    }
+    return entry.link;
+  }),
   writeFileSync: jest.fn((path, data, options) => {
     expect(options).toBe(require('node:v8').serialize ? undefined : 'utf8');
     mockFs[path] = data;
@@ -751,6 +764,50 @@ describe('FileMap', () => {
     );
   });
 
+  test('defers symlink resolution for entries with null mtime', async () => {
+    const node = require('../crawlers/node').default;
+    const fsModule = require('node:fs');
+
+    // $FlowFixMe[prop-missing]
+    // $FlowFixMe[missing-local-annot]
+    node.mockImplementation(options => {
+      const changedFiles = createMap<FileMetadata>({
+        [path.join('fruits', 'Strawberry.js')]: [32, 42, 0, null, 0, null],
+        [path.join('fruits', 'LinkToStrawberry.js')]: [
+          null,
+          0,
+          0,
+          null,
+          1,
+          null,
+        ],
+      });
+      return Promise.resolve({changedFiles, removedFiles: new Set()});
+    });
+
+    const {fileSystem} = await buildNewFileMap({
+      enableSymlinks: true,
+      useWatchman: false,
+    });
+
+    expect(fsModule.promises.readlink).not.toHaveBeenCalledWith(
+      expect.stringContaining('LinkToStrawberry'),
+    );
+
+    expect(
+      fileSystem.lookup(
+        path.join('/', 'project', 'fruits', 'LinkToStrawberry.js'),
+      ),
+    ).toMatchObject({
+      exists: true,
+      realPath: path.join('/', 'project', 'fruits', 'Strawberry.js'),
+    });
+
+    expect(fsModule.readlinkSync).toHaveBeenCalledWith(
+      path.join('/', 'project', 'fruits', 'LinkToStrawberry.js'),
+    );
+  });
+
   test('handles a Haste module moving between builds', async () => {
     mockFs = object({
       [path.join('/', 'project', 'vegetables', 'Melon.js')]: `

packages/metro-file-map/src/crawlers/__tests__/integration-test.js

@@ -72,44 +72,55 @@ function oneOf(this: $FlowFixMe, actual: unknown, ...expectOneOf: unknown[]) {
  * https://fburl.com/gdoc/y8dn025u */
 expect.extend({oneOf});
 
-const CASES = [
-  [
-    true,
-    new Map([
-      ['foo.js', [expect.any(Number), 245, 0, null, 0, null]],
-      [
-        join('directory', 'bar.js'),
-        [expect.any(Number), 245, 0, null, 0, null],
-      ],
-      [
-        'link-to-directory',
-        [expect.any(Number), 9, 0, null, expect.oneOf(1, 'directory'), null],
-      ],
-      [
-        'link-to-foo.js',
-        [expect.any(Number), 6, 0, null, expect.oneOf(1, 'foo.js'), null],
-      ],
-    ]),
-  ],
-  [
-    false,
-    new Map([
-      [
-        join('directory', 'bar.js'),
-        [expect.any(Number), 245, 0, null, 0, null],
-      ],
-      ['foo.js', [expect.any(Number), 245, 0, null, 0, null]],
-    ]),
-  ],
-];
+function getCases(skipsStat: boolean) {
+  const fileEntry = skipsStat
+    ? [null, 0, 0, null, 0, null]
+    : [expect.any(Number), 245, 0, null, 0, null];
+  return [
+    [
+      true,
+      new Map([
+        ['foo.js', fileEntry],
+        [join('directory', 'bar.js'), fileEntry],
+        [
+          'link-to-directory',
+          skipsStat
+            ? [null, 0, 0, null, 1, null]
+            : [
+                expect.any(Number),
+                9,
+                0,
+                null,
+                expect.oneOf(1, 'directory'),
+                null,
+              ],
+        ],
+        [
+          'link-to-foo.js',
+          skipsStat
+            ? [null, 0, 0, null, 1, null]
+            : [expect.any(Number), 6, 0, null, expect.oneOf(1, 'foo.js'), null],
+        ],
+      ]),
+    ],
+    [
+      false,
+      new Map([
+        [join('directory', 'bar.js'), fileEntry],
+        ['foo.js', fileEntry],
+      ]),
+    ],
+  ];
+}
 
 describe.each(Object.keys(CRAWLERS))(
   'Crawler integration tests (%s)',
   crawlerName => {
     const crawl = CRAWLERS[crawlerName];
     const maybeTest = crawl ? test : test.skip;
+    const skipsStat = crawlerName === 'node-recursive';
 
-    maybeTest.each(CASES)(
+    maybeTest.each(getCases(skipsStat))(
       'Finds the expected files (includeSymlinks: %s)',
       async (includeSymlinks, expectedChangedFiles) => {
         invariant(crawl, 'crawl should not be null within maybeTest');

packages/metro-file-map/src/crawlers/__tests__/node-test.js

@@ -259,17 +259,113 @@ describe('node crawler', () => {
 
     expect(changedFiles).toEqual(
       createMap({
-        'fruits/directory/strawberry.js': [33, 42, 0, null, 0, null],
-        'fruits/tomato.js': [32, 42, 0, null, 0, null],
+        'fruits/directory/strawberry.js': [null, 0, 0, null, 0, null],
+        'fruits/tomato.js': [null, 0, 0, null, 0, null],
       }),
     );
     expect(removedFiles).toEqual(new Set());
     // once for /project/fruits, once for /project/fruits/directory
     expect(fs.readdir).toHaveBeenCalledTimes(2);
-    // once for strawberry.js, once for tomato.js
+    // Neither file is in the previous file system, so neither is statted.
+    expect(fs.lstat).toHaveBeenCalledTimes(0);
+  });
+
+  test('skips lstat for files with no prior mtime', async () => {
+    nodeCrawl = require('../node').default;
+    const fs = require('graceful-fs');
+
+    const files = createMap({
+      'fruits/tomato.js': [null, 0, 0, null, 0, null],
+      'fruits/directory/strawberry.js': [null, 0, 0, null, 0, null],
+    });
+
+    const {changedFiles, removedFiles} = await nodeCrawl({
+      console: global.console,
+      previousState: {fileSystem: getFS(files)},
+      extensions: ['js'],
+      ignore: pearMatcher,
+      rootDir,
+      roots: ['/project/fruits'],
+    });
+
+    expect(changedFiles).toEqual(new Map());
+    expect(removedFiles).toEqual(new Set());
+    expect(fs.lstat).toHaveBeenCalledTimes(0);
+  });
+
+  test('calls lstat only for files with existing mtime', async () => {
+    nodeCrawl = require('../node').default;
+    const fs = require('graceful-fs');
+
+    const files = createMap({
+      'fruits/tomato.js': [31, 42, 1, null, 0, null],
+      'fruits/directory/strawberry.js': [null, 0, 0, null, 0, null],
+    });
+
+    const {changedFiles, removedFiles} = await nodeCrawl({
+      console: global.console,
+      previousState: {fileSystem: getFS(files)},
+      extensions: ['js'],
+      ignore: pearMatcher,
+      rootDir,
+      roots: ['/project/fruits'],
+    });
+
+    expect(changedFiles).toEqual(
+      createMap({
+        'fruits/tomato.js': [32, 42, 0, null, 0, null],
+      }),
+    );
+    expect(removedFiles).toEqual(new Set());
+    expect(fs.lstat).toHaveBeenCalledTimes(1);
+  });
+
+  test('excludes unchanged files when lstat mtime matches cache', async () => {
+    nodeCrawl = require('../node').default;
+    const fs = require('graceful-fs');
+
+    const files = createMap({
+      'fruits/tomato.js': [32, 42, 1, null, 0, null],
+      'fruits/directory/strawberry.js': [33, 42, 1, null, 0, null],
+    });
+
+    const {changedFiles, removedFiles} = await nodeCrawl({
+      console: global.console,
+      previousState: {fileSystem: getFS(files)},
+      extensions: ['js'],
+      ignore: pearMatcher,
+      rootDir,
+      roots: ['/project/fruits'],
+    });
+
+    expect(changedFiles).toEqual(new Map());
+    expect(removedFiles).toEqual(new Set());
     expect(fs.lstat).toHaveBeenCalledTimes(2);
   });
 
+  test('marks symlinks correctly when stat is skipped', async () => {
+    nodeCrawl = require('../node').default;
+
+    const {changedFiles} = await nodeCrawl({
+      console: global.console,
+      previousState: {fileSystem: emptyFS},
+      extensions: ['js'],
+      includeSymlinks: true,
+      ignore: pearMatcher,
+      rootDir,
+      roots: ['/project/fruits'],
+    });
+
+    expect(changedFiles.get(normalize('fruits/symlink'))).toEqual([
+      null,
+      0,
+      0,
+      null,
+      1,
+      null,
+    ]);
+  });
+
   test('aborts the crawl on pre-aborted signal', async () => {
     nodeCrawl = require('../node').default;
     const err = new Error('aborted for test');
@@ -356,7 +452,6 @@ describe('node crawler', () => {
   (process.platform === 'win32' ? test.skip : test)(
     'crawls from a filesystem root without doubling separators',
     async () => {
-      const fs = require('graceful-fs');
       nodeCrawl = require('../node').default;
       const ignore = jest.fn(pearMatcher);
 
@@ -374,10 +469,7 @@ describe('node crawler', () => {
         'fruits/tomato.js',
       ]);
       expect(ignore).toHaveBeenCalledWith('/project');
-      expect(fs.lstat).toHaveBeenCalledWith(
-        '/project/fruits/tomato.js',
-        expect.any(Function),
-      );
+      expect(ignore).toHaveBeenCalledWith('/project/fruits/tomato.js');
     },
   );
 });

packages/metro-file-map/src/crawlers/node/index.js

@@ -16,8 +16,15 @@ import * as fs from 'graceful-fs';
 import * as path from 'node:path';
 
 function find(options: CrawlerOptions): Promise<FileData> {
-  const {console, extensions, ignore, includeSymlinks, rootDir, roots} =
-    options;
+  const {
+    console,
+    extensions,
+    ignore,
+    includeSymlinks,
+    previousState,
+    rootDir,
+    roots,
+  } = options;
   const result: FileData = new Map();
   const pathUtils = new RootPathUtils(rootDir);
   const exts = new Set(extensions);
@@ -80,6 +87,23 @@ function find(options: CrawlerOptions): Promise<FileData> {
               continue;
             }
 
+            const mtime =
+              previousState.fileSystem.getMtimeByNormalPath(childNormal);
+            if (mtime == null || mtime === 0) {
+              // A file that's new, or never accessed since it was crawled, is
+              // recorded without an lstat - its mtime and size are populated
+              // when it's first accessed.
+              result.set(childNormal, [
+                null,
+                0,
+                0,
+                null,
+                isSymbolicLink ? 1 : 0,
+                null,
+              ]);
+              continue;
+            }
+
             activeCalls++;
             fs.lstat(file, (err, stat) => {
               activeCalls--;

packages/metro-file-map/src/flow-types.js

@@ -329,6 +329,7 @@ export interface FileSystem {
     removedFiles: Set<string>,
   };
   getSerializableSnapshot(): CacheData['fileSystemData'];
+  getMtimeByNormalPath(file: Path): ?number;
   getSha1(file: Path): ?string;
   getOrComputeSha1(file: Path): Promise<?{sha1: string, content?: Buffer}>;
 

packages/metro-file-map/src/index.js

@@ -638,7 +638,9 @@ export default class FileMap extends EventEmitter {
 
       if (fileData[H.SYMLINK] === 0) {
         filesToProcess.push([normalFilePath, fileData]);
-      } else {
+      } else if (fileData[H.MTIME] != null && fileData[H.MTIME] !== 0) {
+        // The symlink will only be updated, if it's been accessed before
+        // If this is a newly crawled entry, it's skipped
         const maybeReadLink = this.#maybeReadLink(normalFilePath, fileData);
         if (maybeReadLink) {
           readLinkPromises.push(