feat(core): add maxCacheSize option to limit local artifact size (#29654)

<!-- Please make sure you have read the submission guidelines before
posting an PR -->
<!--
https://github.com/nrwl/nx/blob/master/CONTRIBUTING.md#-submitting-a-pr
-->

<!-- Please make sure that your commit message follows our format -->
<!-- Example: `fix(nx): must begin with lowercase` -->

<!-- If this is a particularly complex change or feature addition, you
can request a dedicated Nx release for this pull request branch. Mention
someone from the Nx team or the `@nrwl/nx-pipelines-reviewers` and they
will confirm if the PR warrants its own release for testing purposes,
and generate it for you if appropriate. -->

## Current Behavior
Cache artifacts are removed based on age at a random interval. There is
not a way to set a max size for the cache, so it can grow quite large in
certain repos

## Expected Behavior
Cache size can be controlled via `maxCacheSize` in `nx.json`. Cache
artifacts are removed based on usage until the limit has been reached.

## Related Issue(s)
<!-- Please link the issue being fixed so it gets closed when this is
merged. -->

Fixes #

---------

Co-authored-by: FrozenPandaz <[email protected]>

Author: AgentEnder

Cargo.lock

@@ -28,6 +28,7 @@ Nx.json configuration
 - [generators](../../devkit/documents/NxJsonConfiguration#generators): Object
 - [implicitDependencies](../../devkit/documents/NxJsonConfiguration#implicitdependencies): ImplicitDependencyEntry<T>
 - [installation](../../devkit/documents/NxJsonConfiguration#installation): NxInstallationConfiguration
+- [maxCacheSize](../../devkit/documents/NxJsonConfiguration#maxcachesize): string
 - [namedInputs](../../devkit/documents/NxJsonConfiguration#namedinputs): Object
 - [neverConnectToCloud](../../devkit/documents/NxJsonConfiguration#neverconnecttocloud): boolean
 - [nxCloudAccessToken](../../devkit/documents/NxJsonConfiguration#nxcloudaccesstoken): string
@@ -162,6 +163,14 @@ useful for workspaces that don't have a root package.json + node_modules.
 
 ---
 
+### maxCacheSize
+
+• `Optional` **maxCacheSize**: `string`
+
+Sets the maximum size of the local cache. Accepts a number followed by a unit (e.g. 100MB). Accepted units are B, KB, MB, and GB.
+
+---
+
 ### namedInputs
 
 • `Optional` **namedInputs**: `Object`

docs/generated/devkit/NxJsonConfiguration.md

@@ -26,6 +26,7 @@ use ProjectsConfigurations or NxJsonConfiguration
 - [generators](../../devkit/documents/Workspace#generators): Object
 - [implicitDependencies](../../devkit/documents/Workspace#implicitdependencies): ImplicitDependencyEntry<string[] | "\*">
 - [installation](../../devkit/documents/Workspace#installation): NxInstallationConfiguration
+- [maxCacheSize](../../devkit/documents/Workspace#maxcachesize): string
 - [namedInputs](../../devkit/documents/Workspace#namedinputs): Object
 - [neverConnectToCloud](../../devkit/documents/Workspace#neverconnecttocloud): boolean
 - [nxCloudAccessToken](../../devkit/documents/Workspace#nxcloudaccesstoken): string
@@ -202,6 +203,18 @@ useful for workspaces that don't have a root package.json + node_modules.
 
 ---
 
+### maxCacheSize
+
+• `Optional` **maxCacheSize**: `string`
+
+Sets the maximum size of the local cache. Accepts a number followed by a unit (e.g. 100MB). Accepted units are B, KB, MB, and GB.
+
+#### Inherited from
+
+[NxJsonConfiguration](../../devkit/documents/NxJsonConfiguration).[maxCacheSize](../../devkit/documents/NxJsonConfiguration#maxcachesize)
+
+---
+
 ### namedInputs
 
 • `Optional` **namedInputs**: `Object`

docs/generated/devkit/Workspace.md

@@ -11,6 +11,9 @@ import {
   updateFile,
   updateJson,
 } from '@nx/e2e/utils';
+
+import { readdir, stat } from 'fs/promises';
+
 import { join } from 'path';
 
 describe('cache', () => {
@@ -361,6 +364,66 @@ describe('cache', () => {
     expect(fourthRun).toContain('read the output from the cache');
   }, 120000);
 
+  it('should evict cache if larger than max cache size', async () => {
+    runCLI('reset');
+    updateJson(`nx.json`, (c) => {
+      c.maxCacheSize = '500KB';
+      return c;
+    });
+
+    const lib = uniq('cache-size');
+
+    updateFile(
+      `tools/copy.js`,
+      'require("fs").cpSync(process.argv[2], process.argv[3], { recursive: true, force: true });'
+    );
+    updateFile(
+      `libs/${lib}/project.json`,
+      JSON.stringify({
+        name: lib,
+        targets: {
+          write: {
+            cache: true,
+            command: 'node tools/copy.js {projectRoot}/src dist/{projectRoot}',
+            inputs: ['{projectRoot}/hash.txt'],
+            outputs: ['{workspaceRoot}/dist/{projectRoot}'],
+          },
+        },
+      })
+    );
+    // 100KB file
+    updateFile(`libs/${lib}/src/data.txt`, 'a'.repeat(100 * 1024));
+    for (let i = 0; i < 10; ++i) {
+      updateFile(`libs/${lib}/hash.txt`, i.toString());
+      runCLI(`write ${lib}`);
+    }
+
+    // Expect that size of cache artifacts in cacheDir is less than 1MB
+    const cacheDir = tmpProjPath('.nx/cache');
+    const cacheFiles = listFiles('.nx/cache');
+    let cacheEntries = 0;
+    let cacheEntriesSize = 0;
+    for (const file of cacheFiles) {
+      if (file.match(/^[0-9]+$/)) {
+        cacheEntries += 1;
+        cacheEntriesSize += await dirSize(join(cacheDir, file));
+        console.log(
+          'Checked cache entry',
+          file,
+          'size',
+          cacheEntriesSize,
+          'total entries',
+          cacheEntries
+        );
+      }
+    }
+    console.log('Cache entries:', cacheEntries);
+    console.log('Cache size:', cacheEntriesSize);
+    expect(cacheEntries).toBeGreaterThan(1);
+    expect(cacheEntries).toBeLessThan(10);
+    expect(cacheEntriesSize).toBeLessThanOrEqual(500 * 1024);
+  });
+
   function expectCached(
     actualOutput: string,
     expectedCachedProjects: string[]
@@ -404,3 +467,27 @@ describe('cache', () => {
     expect(matchingProjects).toEqual(expectedProjects);
   }
 });
+
+const dirSize = async (dir) => {
+  const files = await readdir(dir, { withFileTypes: true });
+
+  const paths = files.map(async (file) => {
+    const path = join(dir, file.name);
+
+    if (file.isDirectory()) return await dirSize(path);
+
+    if (file.isFile()) {
+      const { size } = await stat(path);
+
+      return size;
+    }
+
+    console.log('Unknown file type', path);
+
+    return 0;
+  });
+
+  return (await Promise.all(paths))
+    .flat(Infinity)
+    .reduce((i, size) => i + size, 0);
+};

e2e/nx/src/cache.test.ts

@@ -43,7 +43,7 @@ swc_common = "0.31.16"
 swc_ecma_parser = { version = "0.137.1", features = ["typescript"] }
 swc_ecma_visit = "0.93.0"
 swc_ecma_ast = "0.107.0"
-
+rand = "0.9.0"
 [target.'cfg(windows)'.dependencies]
 winapi = { version = "0.3", features = ["fileapi"] }
 

packages/nx/Cargo.toml

@@ -82,6 +82,7 @@ export const allowedWorkspaceExtensions = [
   'neverConnectToCloud',
   'sync',
   'useLegacyCache',
+  'maxCacheSize',
 ] as const;
 
 if (!patched) {

packages/nx/src/adapter/compat.ts

@@ -544,6 +544,11 @@ export interface NxJsonConfiguration<T = '*' | string[]> {
    * Use the legacy file system cache instead of the db cache
    */
   useLegacyCache?: boolean;
+
+  /**
+   * Sets the maximum size of the local cache. Accepts a number followed by a unit (e.g. 100MB). Accepted units are B, KB, MB, and GB.
+   */
+  maxCacheSize?: string;
 }
 
 export type PluginConfiguration = string | ExpandedPluginConfiguration;