metro-file-map: Give plugins getOrComputeFileData, record observed content

packages/metro-file-map/API.md

@@ -124,12 +124,12 @@ export class FileDataPlugin<PerFileData extends void | V8Serializable = void | V
   assertValid(): void;
   getCacheKey(): string;
   getFileSystem(): FileMapPluginInitOptions<null, PerFileData>['files'];
+  getOrComputeFileData(mixedPath: string, observations?: null | undefined | Observations): ReturnType<FileMapPluginInitOptions<null, PerFileData>['getOrComputeFileData']>;
   getSerializableSnapshot(): null;
   getWorker(): FileMapPluginWorker;
   initialize(initOptions: FileMapPluginInitOptions<null, PerFileData>): Promise<void>;
   readonly name: string;
   onChanged(_changes: ReadonlyFileSystemChanges<null | undefined | PerFileData>): void;
-  processFile(mixedPath: string): ReturnType<FileMapPluginInitOptions<null, PerFileData>['processFile']>;
 }
 
 export type FileDataPluginOptions = Readonly<
@@ -139,6 +139,16 @@ export type FileDataPluginOptions = Readonly<
   }
 >;
 
+export type FileDataResult<PerFileData> =
+  | Readonly<{exists: false}>
+  | Readonly<{exists: true; type: 'd'}>
+  | Readonly<{
+      exists: true;
+      type: 'f';
+      realPath: string;
+      data: null | undefined | PerFileData;
+    }>;
+
 class FileMap extends EventEmitter {
   constructor(options: InputOptions);
   build(): Promise<BuildResult>;

packages/metro-file-map/src/__tests__/index-test.js

@@ -445,6 +445,55 @@ describe('FileMap', () => {
     expect(FileMap.H).toBe(require('../constants').default);
   });
 
+  test('getOrComputeFileData records a file found as content, as well as what lookup records', async () => {
+    const {dependencyPlugin} = await buildNewFileMap();
+    if (dependencyPlugin == null) {
+      throw new Error('Expected a DependencyPlugin');
+    }
+    const observations = {
+      existence: new Set<string>(),
+      content: new Set<string>(),
+    };
+
+    expect(
+      dependencyPlugin.getOrComputeFileData(
+        p('/project/fruits/Banana.js'),
+        observations,
+      ),
+    ).toMatchObject({exists: true, type: 'f'});
+    expect(
+      dependencyPlugin.getOrComputeFileData(p('/project/fruits'), observations),
+    ).toEqual({exists: true, type: 'd'});
+    expect(
+      dependencyPlugin.getOrComputeFileData(
+        p('/project/fruits/Missing.js'),
+        observations,
+      ),
+    ).toEqual({exists: false});
+
+    expect(observations).toEqual({
+      content: new Set([path.join('fruits', 'Banana.js')]),
+      existence: new Set([
+        path.join('fruits', 'Banana.js'),
+        'fruits',
+        path.join('fruits', 'Missing.js'),
+      ]),
+    });
+  });
+
+  test("getOrComputeFileData gives null for a file a non-lazy plugin's worker skipped", async () => {
+    mockFs[p('/project/fruits/Pear.json')] = '{}';
+    const {dependencyPlugin} = await buildNewFileMap({
+      extensions: ['js', 'json'],
+    });
+    if (dependencyPlugin == null) {
+      throw new Error('Expected a DependencyPlugin');
+    }
+    expect(
+      dependencyPlugin.getOrComputeFileData(p('/project/fruits/Pear.json')),
+    ).toMatchObject({exists: true, type: 'f', data: null});
+  });
+
   test('ignores files given a pattern', async () => {
     mockFs[p('/project/fruits/Kiwi.js')] = `
       // Kiwi!
@@ -1643,26 +1692,27 @@ describe('FileMap', () => {
         watch: true,
         plugins: [lazyPlugin],
       });
+      const onMetadata = jest.fn();
       await fileMap.build();
       try {
         const bananaPath = p('/project/fruits/Banana.js');
-        const getPluginData = () => {
-          const result = lazyPlugin.getFileSystem().lookup(bananaPath);
-          if (!result.exists || result.type !== 'f') {
-            throw new Error('Expected a file');
-          }
-          return result.pluginData;
-        };
-        const onMetadata = jest.fn();
         fileMap.on('metadata', onMetadata);
 
-        // The crawl does not run a lazy plugin's worker
-        expect(getPluginData()).toBeUndefined();
-
         const expected = {length: String(mockFs[bananaPath]).length};
-        expect(lazyPlugin.processFile(bananaPath)).toEqual(expected);
-        // The data is stored, and the cache told there is something to save
-        expect(getPluginData()).toEqual(expected);
+        expect(lazyPlugin.getOrComputeFileData(bananaPath)).toEqual({
+          exists: true,
+          type: 'f',
+          realPath: bananaPath,
+          data: expected,
+        });
+        // The crawl did not run the worker, so it ran now, and the cache is
+        // told there is something to save
+        expect(onMetadata).toHaveBeenCalledTimes(1);
+
+        // The data is stored
+        expect(lazyPlugin.getOrComputeFileData(bananaPath)).toMatchObject({
+          data: expected,
+        });
         expect(onMetadata).toHaveBeenCalledTimes(1);
 
         mockFs[bananaPath] = '// A changed banana';
@@ -1673,11 +1723,11 @@ describe('FileMap', () => {
         });
         await waitForItToChange(fileMap);
 
-        // Nor does a change, which leaves the file unprocessed again
-        expect(getPluginData()).toBeUndefined();
-        expect(lazyPlugin.processFile(bananaPath)).toEqual({
-          length: '// A changed banana'.length,
+        // A change leaves the file unprocessed again
+        expect(lazyPlugin.getOrComputeFileData(bananaPath)).toMatchObject({
+          data: {length: '// A changed banana'.length},
         });
+        expect(onMetadata).toHaveBeenCalledTimes(2);
       } finally {
         await fileMap.end();
       }

packages/metro-file-map/src/flow-types.js

@@ -230,26 +230,43 @@ export type FileMapPluginInitOptions<
     }>,
     lookup(
       mixedPath: string,
-    ):
-      | {exists: false}
-      | {exists: true, type: 'f', readonly pluginData: PerFileData}
-      | {exists: true, type: 'd'},
+    ): {exists: false} | {exists: true, type: 'f' | 'd'},
   }>,
   pluginState: ?SerializableState,
   /**
-   * Synchronously run this plugin's worker, in-band, on the regular file at
-   * `mixedPath`, store the result as its plugin data and return it. Throws if
-   * the path is not a regular file, or the plugin has no worker.
+   * This plugin's data for the regular file at `mixedPath`. Throws if the
+   * plugin has no worker.
    *
-   * Plugin data is `undefined` for a file until a worker has run on it, which
-   * is when a lazy plugin should call this, and is reset to `undefined` when
-   * the file changes. `null` is a result like any other, so what a worker
-   * returns is stored with `undefined` replaced by `null`, and a file is
-   * processed at most once while it is unchanged.
+   * A lazy plugin's worker is run on a file, synchronously and in-band, the
+   * first time its data is asked for, and again after the file changes. The
+   * result is stored, with `undefined` replaced by `null`.
+   *
+   * A non-lazy plugin's worker has already run on every file it can give
+   * non-null data for, so a file it has not run on (eg, one its filter
+   * excludes) has `null` data, without running it.
+   *
+   * If `observations` is given, records what the result depends upon: a
+   * regular file found in `content`, since its data is derived from what it
+   * holds, and otherwise as `FileSystem.lookup` does. This is the same whether
+   * the data was stored or computed.
    */
-  processFile: (mixedPath: string) => PerFileData,
+  getOrComputeFileData: (
+    mixedPath: string,
+    observations?: ?Observations,
+  ) => FileDataResult<PerFileData>,
 }>;
 
+export type FileDataResult<out PerFileData> =
+  | Readonly<{exists: false}>
+  | Readonly<{exists: true, type: 'd'}>
+  | Readonly<{
+      exists: true,
+      type: 'f',
+      // The real, absolute path of the file.
+      realPath: string,
+      data: ?PerFileData,
+    }>;
+
 export type FileMapPluginWorker = Readonly<{
   worker: Readonly<{
     modulePath: string,
@@ -264,7 +281,7 @@ export type FileMapPluginWorker = Readonly<{
   /**
    * If true, the worker is not run on files as they are crawled or changed,
    * and `filter` is not consulted. It runs only on the files the plugin asks
-   * for, through the `processFile` it is given on initialization. Use this
+   * for, through the `getOrComputeFileData` it is given on initialization. Use this
    * where a small, unpredictable subset of matching files is ever needed.
    */
   lazy?: boolean,
@@ -454,6 +471,8 @@ export type LookupResult =
     }
   | {
       exists: true,
+      // The directory's path relative to the root, with symlinks resolved.
+      canonicalPath: CanonicalPath,
       // The real, normal, absolute path of the directory.
       realPath: string,
       // Currently lookup always follows symlinks, so can only return
@@ -462,6 +481,8 @@ export type LookupResult =
     }
   | {
       exists: true,
+      // The file's path relative to the root, with symlinks resolved.
+      canonicalPath: CanonicalPath,
       // The real, normal, absolute path of the file.
       realPath: string,
       // Currently lookup always follows symlinks, so can only return
@@ -493,7 +514,9 @@ export interface Observations {
    * Files whose content was observed (including traversed symlinks, whose
    * content is their target) and directories whose immediate entries were
    * enumerated. Invalidated by modification, as well as by addition or
-   * removal.
+   * removal, so a path here may also be in `existence` (eg, after a `lookup`
+   * and a content read of the same file) without changing what invalidates
+   * the result.
    */
   readonly content: Set<CanonicalPath>;
 }

packages/metro-file-map/src/index.js

@@ -164,6 +164,7 @@ export type {
   CrawlerOptions,
   CrawlResult,
   DependencyExtractor,
+  FileDataResult,
   FileMapRoot,
   WatcherStatus,
 } from './flow-types';
@@ -454,23 +455,15 @@ export default class FileMap extends EventEmitter {
             fileSystem,
           }),
           Promise.all(
-            plugins.map(({plugin, dataIdx}) =>
-              plugin.initialize({
+            plugins.map(({plugin, dataIdx}) => {
+              const isLazy = plugin.getWorker()?.lazy === true;
+              return plugin.initialize({
                 files: {
                   lookup: mixedPath => {
                     const result = fileSystem.lookup(mixedPath);
-                    if (!result.exists) {
-                      return {exists: false};
-                    }
-                    if (result.type === 'd') {
-                      return {exists: true, type: 'd'};
-                    }
-                    return {
-                      exists: true,
-                      type: 'f',
-                      pluginData:
-                        dataIdx != null ? result.metadata[dataIdx] : null,
-                    };
+                    return result.exists
+                      ? {exists: true, type: result.type}
+                      : {exists: false};
                   },
                   fileIterator: opts =>
                     mapIterable(
@@ -483,34 +476,52 @@ export default class FileMap extends EventEmitter {
                     ),
                 },
                 pluginState: initialData?.plugins.get(plugin.name),
-                processFile: mixedPath => {
+                getOrComputeFileData: (mixedPath, observations) => {
                   invariant(
                     dataIdx != null,
                     'metro-file-map: Plugin "%s" has no worker to process files with',
                     plugin.name,
                   );
-                  const result = fileSystem.lookup(mixedPath);
-                  if (!result.exists || result.type !== 'f') {
-                    throw new Error(
-                      `metro-file-map: Cannot process ${mixedPath}, which is not a regular file`,
-                    );
+                  const result = fileSystem.lookup(mixedPath, observations);
+                  if (!result.exists) {
+                    return {exists: false};
                   }
-                  const pluginData = this.#fileProcessor.processFileForPlugin(
-                    result.realPath,
-                    result.metadata,
-                    dataIdx - H.PLUGINDATA,
-                  );
-                  debug(
-                    'Lazily processed file for %s: %s',
-                    plugin.name,
-                    mixedPath,
-                  );
-                  // Inform caches that there is new data to save.
-                  this.emit('metadata');
-                  return pluginData;
+                  if (result.type === 'd') {
+                    return {exists: true, type: 'd'};
+                  }
+                  // Plugin data is derived from what the file holds, so it
+                  // changes when the file is modified.
+                  observations?.content.add(result.canonicalPath);
+                  let data = result.metadata[dataIdx];
+                  // `undefined` means no worker has run on this file.
+                  // eslint-disable-next-line lint/strictly-null
+                  if (data === undefined) {
+                    if (isLazy) {
+                      data = this.#fileProcessor.processFileForPlugin(
+                        result.realPath,
+                        result.metadata,
+                        dataIdx - H.PLUGINDATA,
+                      );
+                      debug(
+                        'Lazily processed file for %s: %s',
+                        plugin.name,
+                        mixedPath,
+                      );
+                      // Inform caches that there is new data to save.
+                      this.emit('metadata');
+                    } else {
+                      data = null;
+                    }
+                  }
+                  return {
+                    exists: true,
+                    type: 'f',
+                    realPath: result.realPath,
+                    data,
+                  };
                 },
-              }),
-            ),
+              });
+            }),
           ),
         ]);
 

packages/metro-file-map/src/lib/TreeFS.js

@@ -304,15 +304,15 @@ export default class TreeFS implements MutableFileSystem {
     }
     const realPath = this.#pathUtils.normalToAbsolute(canonicalPath);
     if (isDirectory(node)) {
-      return {exists: true, realPath, type: 'd'};
+      return {exists: true, canonicalPath, realPath, type: 'd'};
     }
     invariant(
       isRegularFile(node),
       'lookup follows symlinks, so should never return one (%s -> %s)',
       mixedPath,
       canonicalPath,
     );
-    return {exists: true, realPath, type: 'f', metadata: node};
+    return {exists: true, canonicalPath, realPath, type: 'f', metadata: node};
   }
 
   getAllFiles(): Array<Path> {

packages/metro-file-map/src/lib/__tests__/TreeFS-test.js

@@ -167,6 +167,7 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => {
         const observations = emptyObservations();
         expect(tfs.lookup(givenPath, observations)).toEqual({
           exists: true,
+          canonicalPath: canonical(expectedRealPath),
           realPath: expectedRealPath,
           type: 'f',
           metadata: expect.any(Array),
@@ -214,6 +215,7 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => {
         expect(tfs.lookup(givenPath, observations)).toEqual({
           exists: true,
           type: 'd',
+          canonicalPath: canonical(expectedRealPath),
           realPath: expectedRealPath,
         });
         expect(observations).toEqual({
@@ -289,6 +291,7 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => {
         const observations = emptyObservations();
         expect(tfs.lookup(mixedPath, observations)).toEqual({
           exists: true,
+          canonicalPath: canonical(expectedRealPath),
           realPath: expectedRealPath,
           type: 'f',
           metadata: expect.any(Array),