Merge pull request #305 from MaksimZhukov/mazhuk/adding-support-for-monorepos

Support caching for mono repos and repositories with complex structure

.github/workflows/e2e-cache.yml

@@ -101,6 +101,8 @@ jobs:
   node-yarn2-depencies-caching:
     name: Test yarn 2 (Node ${{ matrix.node-version}}, ${{ matrix.os }})
     runs-on: ${{ matrix.os }}
+    env:
+      YARN_ENABLE_IMMUTABLE_INSTALLS: false
     strategy:
       fail-fast: false
       matrix:

README.md

@@ -39,9 +39,13 @@ major versions: `12`, `14`, `16`
 more specific versions: `10.15`, `14.2.0`, `16.3.0`  
 nvm lts syntax: `lts/erbium`, `lts/fermium`, `lts/*`  
 
-### Caching packages dependencies
+## Caching packages dependencies
 
-The action has a built-in functionality for caching and restoring npm/yarn dependencies. Supported package managers are `npm`, `yarn`, `pnpm`. The `cache` input is optional, and caching is turned off by default.
+The action has a built-in functionality for caching and restoring dependencies. It uses [actions/cache](https://github.com/actions/cache) under hood for caching dependencies but requires less configuration settings. Supported package managers are `npm`, `yarn`, `pnpm` (v6.10+). The `cache` input is optional, and caching is turned off by default.
+
+The action defaults to search for the dependency file (`package-lock.json` or `yarn.lock`) in the repository root, and uses its hash as a part of the cache key. Use `cache-dependency-path` for cases when multiple dependency files are used, or they are located in different subdirectories.
+
+See the examples of using cache for `yarn` / `pnpm` and  `cache-dependency-path` input in the [Advanced usage](docs/advanced-usage.md#caching-packages-dependencies) guide.
 
 **Caching npm dependencies:**
 ```yaml
@@ -55,44 +59,20 @@ steps:
 - run: npm test
 ```
 
-**Caching yarn dependencies:**
+**Caching npm dependencies in monorepos:**
 ```yaml
 steps:
 - uses: actions/checkout@v2
 - uses: actions/setup-node@v2
   with:
     node-version: '14'
-    cache: 'yarn'
-- run: yarn install
-- run: yarn test
-```
-Yarn caching handles both yarn versions: 1 or 2.
-
-**Caching pnpm (v6.10+) dependencies:**
-```yaml
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
-
-# NOTE: pnpm caching support requires pnpm version >= 6.10.0
-
-steps:
-- uses: actions/checkout@v2
-- uses: pnpm/action-setup@646cdf48217256a3d0b80361c5a50727664284f2
-  with:
-    version: 6.10.0
-- uses: actions/setup-node@v2
-  with:
-    node-version: '14'
-    cache: 'pnpm'
-- run: pnpm install
-- run: pnpm test
+    cache: 'npm'
+    cache-dependency-path: subdir/package-lock.json
+- run: npm install
+- run: npm test
 ```
 
-> At the moment, only `lock` files in the project root are supported.
-
-### Matrix Testing:
+## Matrix Testing:
 ```yaml
 jobs:
   build:
@@ -114,10 +94,11 @@ jobs:
 
 1. [Check latest version](docs/advanced-usage.md#check-latest-version)
 2. [Using different architectures](docs/advanced-usage.md#architecture)
-3. [Using multiple operating systems and architectures](docs/advanced-usage.md#multiple-operating-systems-and-architectures)
-4. [Publishing to npmjs and GPR with npm](docs/advanced-usage.md#publish-to-npmjs-and-gpr-with-npm)
-5. [Publishing to npmjs and GPR with yarn](docs/advanced-usage.md#publish-to-npmjs-and-gpr-with-yarn)
-6. [Using private packages](docs/advanced-usage.md#use-private-packages)
+3. [Caching packages dependencies](docs/advanced-usage.md#caching-packages-dependencies)
+4. [Using multiple operating systems and architectures](docs/advanced-usage.md#multiple-operating-systems-and-architectures)
+5. [Publishing to npmjs and GPR with npm](docs/advanced-usage.md#publish-to-npmjs-and-gpr-with-npm)
+6. [Publishing to npmjs and GPR with yarn](docs/advanced-usage.md#publish-to-npmjs-and-gpr-with-yarn)
+7. [Using private packages](docs/advanced-usage.md#use-private-packages)
 
 # License
 

action.yml

@@ -21,6 +21,8 @@ inputs:
     default: ${{ github.token }}
   cache:
     description: 'Used to specify a package manager for caching in the default directory. Supported values: npm, yarn, pnpm'
+  cache-dependency-path:
+    description: 'Used to specify the path to a dependency file: package-lock.json, yarn.lock, etc. Supports wildcards or a list of file names for caching multiple dependencies.'
 # TODO: add input to control forcing to pull from cloud or dist. 
 #       escape valve for someone having issues or needing the absolute latest which isn't cached yet
 # Deprecated option, do not use. Will not be supported after October 1, 2019

dist/setup/index.js

@@ -6894,7 +6894,8 @@ function run() {
                 if (isGhes()) {
                     throw new Error('Caching is not supported on GHES');
                 }
-                yield cache_restore_1.restoreCache(cache);
+                const cacheDependencyPath = core.getInput('cache-dependency-path');
+                yield cache_restore_1.restoreCache(cache, cacheDependencyPath);
             }
             const matchersPath = path.join(__dirname, '../..', '.github');
             core.info(`##[add-matcher]${path.join(matchersPath, 'tsc.json')}`);
@@ -44655,15 +44656,20 @@ const path_1 = __importDefault(__webpack_require__(622));
 const fs_1 = __importDefault(__webpack_require__(747));
 const constants_1 = __webpack_require__(196);
 const cache_utils_1 = __webpack_require__(570);
-exports.restoreCache = (packageManager) => __awaiter(void 0, void 0, void 0, function* () {
+exports.restoreCache = (packageManager, cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () {
     const packageManagerInfo = yield cache_utils_1.getPackageManagerInfo(packageManager);
     if (!packageManagerInfo) {
         throw new Error(`Caching for '${packageManager}' is not supported`);
     }
     const platform = process.env.RUNNER_OS;
     const cachePath = yield cache_utils_1.getCacheDirectoryPath(packageManagerInfo, packageManager);
-    const lockFilePath = findLockFile(packageManagerInfo);
+    const lockFilePath = cacheDependencyPath
+        ? cacheDependencyPath
+        : findLockFile(packageManagerInfo);
     const fileHash = yield glob.hashFiles(lockFilePath);
+    if (!fileHash) {
+        throw new Error('Some specified paths were not resolved, unable to cache dependencies.');
+    }
     const primaryKey = `node-cache-${platform}-${packageManager}-${fileHash}`;
     core.debug(`primary key is ${primaryKey}`);
     core.saveState(constants_1.State.CachePrimaryKey, primaryKey);

docs/adrs/0000-caching-dependencies.md

@@ -1,7 +1,7 @@
 # 0. Caching dependencies
 Date: 2021-05-21
 
-Status: Proposed
+Status: Accepted
 
 # Context
 `actions/setup-node` is the 2nd most popular action in GitHub Actions. A lot of customers use it in conjunction with [actions/cache](https://github.com/actions/cache) to speed up dependencies installation.  

docs/adrs/0001-support-caching-deps-for-monorepos.md

@@ -0,0 +1,48 @@
+# 0. Support caching dependencies for mono repos
+Date: 2021-07-13
+
+Status: Proposed
+
+## Context
+Currently, `actions/setup-node` supports caching dependencies for Npm and Yarn package managers.  
+For the first iteration, we have decided to not support cases where `package-lock.json` / `yarn.lock` are located outside of repository root.  
+Current implementation searches the following file patterns in the repository root: `package-lock.json`, `yarn.lock` (in order of resolving priorities)
+
+Obviously, it made build-in caching unusable for mono-repos and repos with complex structure.  
+We would like to revisit this decision and add customization for dependencies lock file location.
+
+## Proposal
+We have the following options:
+1. Allow to specify  directory where `package-lock.json` or `yarn.lock` are located
+2. Allow to specify path to the dependencies lock file (including directory path and filename)
+
+The second option looks more generic because it allows to:
+- specify multiple dependencies files using file patterns like `**/package-lock.json` ([one of recommended approaches in actions/cache](https://github.com/actions/cache/blob/main/examples.md#macos-and-ubuntu))
+- specify custom dependencies files like `src/npm-shrinkwrap.json`
+- change default resolving priority if both `yarn.lock` and `package-lock.json` exist in repository
+
+## Decision
+
+Add `cache-dependency-path` input that will accept path (relative to repository root) to dependencies lock file.  
+If provided path contains wildcards, the action will search all maching files and calculate common hash like `${{ hashFiles('**/package-lock.json') }}` YAML construction does.  
+The hash of provided matched files will be used as a part of cache key.
+
+Yaml examples:
+```yml
+steps:
+- uses: actions/checkout@v2
+- uses: actions/setup-node@v2
+  with:
+    node-version: 14
+    cache: npm
+    cache-dependency-path: 'sub-project/package-lock.json'
+```
+```yml
+steps:
+- uses: actions/checkout@v2
+- uses: actions/setup-node@v2
+  with:
+    node-version: 14
+    cache: yarn
+    cache-dependency-path: 'sub-project/**/yarn.lock'
+```

docs/advanced-usage.md

@@ -1,6 +1,6 @@
 # Advanced usage
 
-### Check latest version:
+## Check latest version
 
 The `check-latest` flag defaults to `false`. When set to `false`, the action will first check the local cache for a semver match. If unable to find a specific version in the cache, the action will attempt to download a version of Node.js. It will pull LTS versions from [node-versions releases](https://github.com/actions/node-versions/releases) and on miss or failure will fall back to the previous behavior of downloading directly from [node dist](https://nodejs.org/dist/). Use the default or set `check-latest` to `false` if you prefer stability and if you want to ensure a specific version of Node.js is always used.
 
@@ -19,7 +19,7 @@ steps:
 - run: npm test
 ```
 
-### Architecture:
+## Architecture
 
 You can use any of the [supported operating systems](https://docs.github.com/en/actions/reference/virtual-environments-for-github-hosted-runners), and the compatible `architecture` can be selected using `architecture`. Values are `x86`, `x64`, `arm64`, `armv6l`, `armv7l`, `ppc64le`, `s390x` (not all of the architectures are available on all platforms).
 
@@ -39,7 +39,73 @@ jobs:
       - run: npm test
 ```
 
-### Multiple Operating Systems and Architectures:
+## Caching packages dependencies
+The action follows [actions/cache](https://github.com/actions/cache/blob/main/examples.md#node---npm) guidelines, and caches global cache on the machine instead of `node_modules`, so cache can be reused between different Node.js versions.
+
+**Caching yarn dependencies:**  
+Yarn caching handles both yarn versions: 1 or 2.
+```yaml
+steps:
+- uses: actions/checkout@v2
+- uses: actions/setup-node@v2
+  with:
+    node-version: '14'
+    cache: 'yarn'
+- run: yarn install
+- run: yarn test
+```
+
+**Caching pnpm (v6.10+) dependencies:**
+```yaml
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# NOTE: pnpm caching support requires pnpm version >= 6.10.0
+
+steps:
+- uses: actions/checkout@v2
+- uses: pnpm/action-setup@646cdf48217256a3d0b80361c5a50727664284f2
+  with:
+    version: 6.10.0
+- uses: actions/setup-node@v2
+  with:
+    node-version: '14'
+    cache: 'pnpm'
+- run: pnpm install
+- run: pnpm test
+```
+
+**Using wildcard patterns to cache dependencies**
+```yaml
+steps:
+- uses: actions/checkout@v2
+- uses: actions/setup-node@v2
+  with:
+    node-version: '14'
+    cache: 'npm'
+    cache-dependency-path: '**/package-lock.json'
+- run: npm install
+- run: npm test
+```
+
+**Using a list of file paths to cache dependencies**
+```yaml
+steps:
+- uses: actions/checkout@v2
+- uses: actions/setup-node@v2
+  with:
+    node-version: '14'
+    cache: 'npm'
+    cache-dependency-path: |
+      server/app/package-lock.json
+      frontend/app/package-lock.json
+- run: npm install
+- run: npm test
+```
+
+## Multiple Operating Systems and Architectures
 
 ```yaml
 jobs:
@@ -74,7 +140,7 @@ jobs:
       - run: npm test
 ```
 
-### Publish to npmjs and GPR with npm:
+## Publish to npmjs and GPR with npm
 ```yaml
 steps:
 - uses: actions/checkout@v2
@@ -94,7 +160,7 @@ steps:
     NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 ```
 
-### Publish to npmjs and GPR with yarn:
+## Publish to npmjs and GPR with yarn
 ```yaml
 steps:
 - uses: actions/checkout@v2
@@ -114,7 +180,7 @@ steps:
     NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 ```
 
-### Use private packages:
+## Use private packages
 ```yaml
 steps:
 - uses: actions/checkout@v2

src/cache-restore.ts

@@ -11,7 +11,10 @@ import {
   PackageManagerInfo
 } from './cache-utils';
 
-export const restoreCache = async (packageManager: string) => {
+export const restoreCache = async (
+  packageManager: string,
+  cacheDependencyPath?: string
+) => {
   const packageManagerInfo = await getPackageManagerInfo(packageManager);
   if (!packageManagerInfo) {
     throw new Error(`Caching for '${packageManager}' is not supported`);
@@ -22,9 +25,17 @@ export const restoreCache = async (packageManager: string) => {
     packageManagerInfo,
     packageManager
   );
-  const lockFilePath = findLockFile(packageManagerInfo);
+  const lockFilePath = cacheDependencyPath
+    ? cacheDependencyPath
+    : findLockFile(packageManagerInfo);
   const fileHash = await glob.hashFiles(lockFilePath);
 
+  if (!fileHash) {
+    throw new Error(
+      'Some specified paths were not resolved, unable to cache dependencies.'
+    );
+  }
+
   const primaryKey = `node-cache-${platform}-${packageManager}-${fileHash}`;
   core.debug(`primary key is ${primaryKey}`);
 

src/main.ts

@@ -51,7 +51,8 @@ export async function run() {
       if (isGhes()) {
         throw new Error('Caching is not supported on GHES');
       }
-      await restoreCache(cache);
+      const cacheDependencyPath = core.getInput('cache-dependency-path');
+      await restoreCache(cache, cacheDependencyPath);
     }
 
     const matchersPath = path.join(__dirname, '../..', '.github');