fix(core): create multi-glob function (#29880)

<!-- Please make sure you have read the submission guidelines before
posting an PR -->
<!--
https://github.com/nrwl/nx/blob/master/CONTRIBUTING.md#-submitting-a-pr
-->

<!-- Please make sure that your commit message follows our format -->
<!-- Example: `fix(nx): must begin with lowercase` -->

<!-- If this is a particularly complex change or feature addition, you
can request a dedicated Nx release for this pull request branch. Mention
someone from the Nx team or the `@nrwl/nx-pipelines-reviewers` and they
will confirm if the PR warrants its own release for testing purposes,
and generate it for you if appropriate. -->

## Current Behavior
<!-- This is the behavior we have today -->
Whenever there are multiple plugins using in a workspace, all the
configuration paths are collected and used as 1 giant glob to test
against the workspace context files.

## Expected Behavior
<!-- This is the behavior we should expect with the changes in this PR
-->
Each plugin configuration glob is now handled separately and are not
joined together into one giant one. This allows each glob pattern to
have separate files for each plugin.

## Related Issue(s)
<!-- Please link the issue being fixed so it gets closed when this is
merged. -->

Fixes #29473

---------

Co-authored-by: FrozenPandaz <[email protected]>

Author: 2 people

packages/nx/src/daemon/client/client.ts

@@ -34,7 +34,10 @@ import {
   ProjectGraphError,
 } from '../../project-graph/error-types';
 import { IS_WASM, NxWorkspaceFiles, TaskRun, TaskTarget } from '../../native';
-import { HandleGlobMessage } from '../message-types/glob';
+import {
+  HandleGlobMessage,
+  HandleMultiGlobMessage,
+} from '../message-types/glob';
 import {
   GET_NX_WORKSPACE_FILES,
   HandleNxWorkspaceFilesMessage,
@@ -339,6 +342,15 @@ export class DaemonClient {
     return this.sendToDaemonViaQueue(message);
   }
 
+  multiGlob(globs: string[], exclude?: string[]): Promise<string[][]> {
+    const message: HandleMultiGlobMessage = {
+      type: 'MULTI_GLOB',
+      globs,
+      exclude,
+    };
+    return this.sendToDaemonViaQueue(message);
+  }
+
   getWorkspaceContextFileData(): Promise<FileData[]> {
     const message: HandleContextFileDataMessage = {
       type: GET_CONTEXT_FILE_DATA,

packages/nx/src/daemon/message-types/glob.ts

@@ -16,3 +16,21 @@ export function isHandleGlobMessage(
     message['type'] === GLOB
   );
 }
+
+export const MULTI_GLOB = 'MULTI_GLOB' as const;
+export type HandleMultiGlobMessage = {
+  type: typeof MULTI_GLOB;
+  globs: string[];
+  exclude?: string[];
+};
+
+export function isHandleMultiGlobMessage(
+  message: unknown
+): message is HandleMultiGlobMessage {
+  return (
+    typeof message === 'object' &&
+    message !== null &&
+    'type' in message &&
+    message['type'] === MULTI_GLOB
+  );
+}

packages/nx/src/daemon/server/handle-glob.ts

@@ -1,5 +1,8 @@
 import { workspaceRoot } from '../../utils/workspace-root';
-import { globWithWorkspaceContext } from '../../utils/workspace-context';
+import {
+  globWithWorkspaceContext,
+  multiGlobWithWorkspaceContext,
+} from '../../utils/workspace-context';
 import { HandlerResult } from './server';
 
 export async function handleGlob(
@@ -12,3 +15,18 @@ export async function handleGlob(
     description: 'handleGlob',
   };
 }
+
+export async function handleMultiGlob(
+  globs: string[],
+  exclude?: string[]
+): Promise<HandlerResult> {
+  const files = await multiGlobWithWorkspaceContext(
+    workspaceRoot,
+    globs,
+    exclude
+  );
+  return {
+    response: JSON.stringify(files),
+    description: 'handleMultiGlob',
+  };
+}

packages/nx/src/daemon/server/server.ts

@@ -55,8 +55,13 @@ import {
   watchOutputFiles,
   watchWorkspace,
 } from './watcher';
-import { handleGlob } from './handle-glob';
-import { GLOB, isHandleGlobMessage } from '../message-types/glob';
+import { handleGlob, handleMultiGlob } from './handle-glob';
+import {
+  GLOB,
+  isHandleGlobMessage,
+  isHandleMultiGlobMessage,
+  MULTI_GLOB,
+} from '../message-types/glob';
 import {
   GET_NX_WORKSPACE_FILES,
   isHandleNxWorkspaceFilesMessage,
@@ -239,6 +244,10 @@ async function handleMessage(socket, data: string) {
     await handleResult(socket, GLOB, () =>
       handleGlob(payload.globs, payload.exclude)
     );
+  } else if (isHandleMultiGlobMessage(payload)) {
+    await handleResult(socket, MULTI_GLOB, () =>
+      handleMultiGlob(payload.globs, payload.exclude)
+    );
   } else if (isHandleNxWorkspaceFilesMessage(payload)) {
     await handleResult(socket, GET_NX_WORKSPACE_FILES, () =>
       handleNxWorkspaceFiles(payload.projectRootMap)

packages/nx/src/native/index.d.ts

@@ -93,6 +93,13 @@ export declare class WorkspaceContext {
   constructor(workspaceRoot: string, cacheDir: string)
   getWorkspaceFiles(projectRootMap: Record<string, string>): NxWorkspaceFiles
   glob(globs: Array<string>, exclude?: Array<string> | undefined | null): Array<string>
+  /**
+   * Performs multiple glob pattern matches against workspace files in parallel
+   * @returns An array of arrays, where each inner array contains the file paths
+   * that matched the corresponding glob pattern in the input. The outer array maintains the same order
+   * as the input globs.
+   */
+  multiGlob(globs: Array<string>, exclude?: Array<string> | undefined | null): Array<Array<string>>
   hashFilesMatchingGlob(globs: Array<string>, exclude?: Array<string> | undefined | null): string
   incrementalUpdate(updatedFiles: Array<string>, deletedFiles: Array<string>): Record<string, string>
   updateProjectFiles(projectRootMappings: ProjectRootMappings, projectFiles: ExternalObject<ProjectFiles>, globalFiles: ExternalObject<Array<FileData>>, updatedFiles: Record<string, string>, deletedFiles: Array<string>): UpdatedWorkspaceFiles

packages/nx/src/native/workspace/context.rs

@@ -231,6 +231,28 @@ impl WorkspaceContext {
         Ok(globbed_files.map(|file| file.file.to_owned()).collect())
     }
 
+    /// Performs multiple glob pattern matches against workspace files in parallel
+    /// @returns An array of arrays, where each inner array contains the file paths
+    /// that matched the corresponding glob pattern in the input. The outer array maintains the same order
+    /// as the input globs.
+    #[napi]
+    pub fn multi_glob(
+        &self,
+        globs: Vec<String>,
+        exclude: Option<Vec<String>>,
+    ) -> napi::Result<Vec<Vec<String>>> {
+        let file_data = self.all_file_data();
+
+        globs
+            .into_iter()
+            .map(|glob| {
+                let globbed_files =
+                    config_files::glob_files(&file_data, vec![glob], exclude.clone())?;
+                Ok(globbed_files.map(|file| file.file.to_owned()).collect())
+            })
+            .collect()
+    }
+
     #[napi]
     pub fn hash_files_matching_glob(
         &self,

packages/nx/src/project-graph/affected/locators/project-glob-changes.ts

@@ -3,7 +3,7 @@ import { minimatch } from 'minimatch';
 import { workspaceRoot } from '../../../utils/workspace-root';
 import { join } from 'path';
 import { existsSync } from 'fs';
-import { configurationGlobs } from '../../utils/retrieve-workspace-files';
+import { getGlobPatternsOfPlugins } from '../../utils/retrieve-workspace-files';
 import { combineGlobPatterns } from '../../../utils/globs';
 import { getPlugins } from '../../plugins/get-plugins';
 
@@ -20,8 +20,8 @@ export const getTouchedProjectsFromProjectGlobChanges: TouchedProjectLocator =
           'package.json',
         ]);
       }
-      const plugins = await getPlugins();
-      return combineGlobPatterns(configurationGlobs(plugins));
+      const plugins = (await getPlugins()).filter((p) => !!p.createNodes);
+      return combineGlobPatterns(getGlobPatternsOfPlugins(plugins));
     })();
 
     const touchedProjects = new Set<string>();

packages/nx/src/project-graph/utils/project-configuration-utils.spec.ts

@@ -5,7 +5,7 @@ import {
 import {
   ConfigurationSourceMaps,
   SourceInformation,
-  createProjectConfigurations,
+  createProjectConfigurationsWithPlugins,
   isCompatibleTarget,
   mergeProjectConfigurationIntoRootMap,
   mergeTargetConfigurations,
@@ -1679,16 +1679,17 @@ describe('project-configuration-utils', () => {
     };
 
     it('should create nodes for files matching included patterns only', async () => {
-      const projectConfigurations = await createProjectConfigurations(
-        undefined,
-        {},
-        ['libs/a/project.json', 'libs/b/project.json'],
-        [
-          new LoadedNxPlugin(fakeTagPlugin, {
-            plugin: fakeTagPlugin.name,
-          }),
-        ]
-      );
+      const projectConfigurations =
+        await createProjectConfigurationsWithPlugins(
+          undefined,
+          {},
+          [['libs/a/project.json', 'libs/b/project.json']],
+          [
+            new LoadedNxPlugin(fakeTagPlugin, {
+              plugin: fakeTagPlugin.name,
+            }),
+          ]
+        );
 
       expect(projectConfigurations.projects).toEqual({
         'libs/a': {
@@ -1705,17 +1706,18 @@ describe('project-configuration-utils', () => {
     });
 
     it('should create nodes for files matching included patterns only', async () => {
-      const projectConfigurations = await createProjectConfigurations(
-        undefined,
-        {},
-        ['libs/a/project.json', 'libs/b/project.json'],
-        [
-          new LoadedNxPlugin(fakeTagPlugin, {
-            plugin: fakeTagPlugin.name,
-            include: ['libs/a/**'],
-          }),
-        ]
-      );
+      const projectConfigurations =
+        await createProjectConfigurationsWithPlugins(
+          undefined,
+          {},
+          [['libs/a/project.json', 'libs/b/project.json']],
+          [
+            new LoadedNxPlugin(fakeTagPlugin, {
+              plugin: fakeTagPlugin.name,
+              include: ['libs/a/**'],
+            }),
+          ]
+        );
 
       expect(projectConfigurations.projects).toEqual({
         'libs/a': {
@@ -1727,17 +1729,18 @@ describe('project-configuration-utils', () => {
     });
 
     it('should not create nodes for files matching excluded patterns', async () => {
-      const projectConfigurations = await createProjectConfigurations(
-        undefined,
-        {},
-        ['libs/a/project.json', 'libs/b/project.json'],
-        [
-          new LoadedNxPlugin(fakeTagPlugin, {
-            plugin: fakeTagPlugin.name,
-            exclude: ['libs/b/**'],
-          }),
-        ]
-      );
+      const projectConfigurations =
+        await createProjectConfigurationsWithPlugins(
+          undefined,
+          {},
+          [['libs/a/project.json', 'libs/b/project.json']],
+          [
+            new LoadedNxPlugin(fakeTagPlugin, {
+              plugin: fakeTagPlugin.name,
+              exclude: ['libs/b/**'],
+            }),
+          ]
+        );
 
       expect(projectConfigurations.projects).toEqual({
         'libs/a': {
@@ -1749,10 +1752,10 @@ describe('project-configuration-utils', () => {
     });
 
     it('should normalize targets', async () => {
-      const { projects } = await createProjectConfigurations(
+      const { projects } = await createProjectConfigurationsWithPlugins(
         undefined,
         {},
-        ['libs/a/project.json'],
+        [['libs/a/project.json'], ['libs/a/project.json']],
         [
           new LoadedNxPlugin(fakeTargetsPlugin, 'fake-targets-plugin'),
           new LoadedNxPlugin(fakeTagPlugin, 'fake-tag-plugin'),
@@ -1771,10 +1774,10 @@ describe('project-configuration-utils', () => {
     });
 
     it('should validate that project names are unique', async () => {
-      const error = await createProjectConfigurations(
+      const error = await createProjectConfigurationsWithPlugins(
         undefined,
         {},
-        ['libs/a/project.json', 'libs/b/project.json', 'libs/c/project.json'],
+        [['libs/a/project.json', 'libs/b/project.json', 'libs/c/project.json']],
         [new LoadedNxPlugin(sameNamePlugin, 'same-name-plugin')]
       ).catch((e) => e);
       const isErrorType = isProjectConfigurationsError(error);
@@ -1795,10 +1798,10 @@ describe('project-configuration-utils', () => {
     });
 
     it('should validate that projects have a name', async () => {
-      const error = await createProjectConfigurations(
+      const error = await createProjectConfigurationsWithPlugins(
         undefined,
         {},
-        ['libs/a/project.json', 'libs/b/project.json', 'libs/c/project.json'],
+        [['libs/a/project.json', 'libs/b/project.json', 'libs/c/project.json']],
         [new LoadedNxPlugin(fakeTargetsPlugin, 'fake-targets-plugin')]
       ).catch((e) => e);
       const isErrorType = isProjectConfigurationsError(error);
@@ -1816,10 +1819,10 @@ describe('project-configuration-utils', () => {
     });
 
     it('should correctly set source maps', async () => {
-      const { sourceMaps } = await createProjectConfigurations(
+      const { sourceMaps } = await createProjectConfigurationsWithPlugins(
         undefined,
         {},
-        ['libs/a/project.json'],
+        [['libs/a/project.json'], ['libs/a/project.json']],
         [
           new LoadedNxPlugin(fakeTargetsPlugin, 'fake-targets-plugin'),
           new LoadedNxPlugin(fakeTagPlugin, 'fake-tag-plugin'),

packages/nx/src/project-graph/utils/project-configuration-utils.ts

@@ -319,10 +319,10 @@ export type ConfigurationResult = {
  * @param workspaceFiles A list of non-ignored workspace files
  * @param plugins The plugins that should be used to infer project configuration
  */
-export async function createProjectConfigurations(
+export async function createProjectConfigurationsWithPlugins(
   root: string = workspaceRoot,
   nxJson: NxJsonConfiguration,
-  projectFiles: string[], // making this parameter allows devkit to pick up newly created projects
+  projectFiles: string[][], // making this parameter allows devkit to pick up newly created projects
   plugins: LoadedNxPlugin[]
 ): Promise<ConfigurationResult> {
   performance.mark('build-project-configs:start');
@@ -386,7 +386,7 @@ export async function createProjectConfigurations(
     }
 
     const matchingConfigFiles: string[] = findMatchingConfigFiles(
-      projectFiles,
+      projectFiles[index],
       pattern,
       include,
       exclude
@@ -439,15 +439,15 @@ export async function createProjectConfigurations(
         externalNodes,
         projectRootMap: rootMap,
         sourceMaps: configurationSourceMaps,
-        matchingProjectFiles: projectFiles,
+        matchingProjectFiles: projectFiles.flat(),
       };
     } else {
       throw new ProjectConfigurationsError(errors, {
         projects: projectRootMap,
         externalNodes,
         projectRootMap: rootMap,
         sourceMaps: configurationSourceMaps,
-        matchingProjectFiles: projectFiles,
+        matchingProjectFiles: projectFiles.flat(),
       });
     }
   });