Deprecates implicit cache keys (#6083)

**What's the problem this PR addresses?**

The lockfile has a very small optimization feature: instead of repeating
the cache key across all checksums, it keeps a global cache key at the
top of the file, which it uses everytime a checksum is missing a cache
key. This saves a few bytes.

However, I found out this behaviour causes breakages under some
situations:

- Alice creates a feature branch from main and add a new dependency
`foo`
- Bob upgrades Yarn to a new version that compiles zip files differently
- Alice merges the new main into her feature branch
- Git merges the two lockfiles together; let's say there are no
conflicts
- The `foo` entry added by Alice has no cache key, since it was added at
a time its cache key matched the global one
- However it doesn't match the new global one, which was updated at the
top of the file

**How did you fix it?**

The checksum cache key is now always set.

**Checklist**
<!--- Don't worry if you miss something, chores are automatically
tested. -->
<!--- This checklist exists to help you remember doing the chores when
you submit a PR. -->
<!--- Put an `x` in all the boxes that apply. -->
- [x] I have read the [Contributing
Guide](https://yarnpkg.com/advanced/contributing).

<!-- See
https://yarnpkg.com/advanced/contributing#preparing-your-pr-to-be-released
for more details. -->
<!-- Check with `yarn version check` and fix with `yarn version check
-i` -->
- [x] I have set the packages that need to be released for my changes to
be effective.

<!-- The "Testing chores" workflow validates that your PR follows our
guidelines. -->
<!-- If it doesn't pass, click on it to see details as to what your PR
might be missing. -->
- [x] I will check that all automated PR checks pass before the PR gets
reviewed.

Author: arcanis

.pnp.cjs

@@ -0,0 +1,38 @@
+releases:
+  "@yarnpkg/cli": patch
+  "@yarnpkg/core": patch
+  "@yarnpkg/libzip": patch
+
+declined:
+  - "@yarnpkg/plugin-compat"
+  - "@yarnpkg/plugin-constraints"
+  - "@yarnpkg/plugin-dlx"
+  - "@yarnpkg/plugin-essentials"
+  - "@yarnpkg/plugin-exec"
+  - "@yarnpkg/plugin-file"
+  - "@yarnpkg/plugin-git"
+  - "@yarnpkg/plugin-github"
+  - "@yarnpkg/plugin-http"
+  - "@yarnpkg/plugin-init"
+  - "@yarnpkg/plugin-interactive-tools"
+  - "@yarnpkg/plugin-link"
+  - "@yarnpkg/plugin-nm"
+  - "@yarnpkg/plugin-npm"
+  - "@yarnpkg/plugin-npm-cli"
+  - "@yarnpkg/plugin-pack"
+  - "@yarnpkg/plugin-patch"
+  - "@yarnpkg/plugin-pnp"
+  - "@yarnpkg/plugin-pnpm"
+  - "@yarnpkg/plugin-stage"
+  - "@yarnpkg/plugin-typescript"
+  - "@yarnpkg/plugin-version"
+  - "@yarnpkg/plugin-workspace-tools"
+  - vscode-zipfs
+  - "@yarnpkg/builder"
+  - "@yarnpkg/doctor"
+  - "@yarnpkg/extensions"
+  - "@yarnpkg/fslib"
+  - "@yarnpkg/nm"
+  - "@yarnpkg/pnp"
+  - "@yarnpkg/pnpify"
+  - "@yarnpkg/sdks"

.yarn/versions/c9849542.yml

@@ -1,3 +1,4 @@
+import {execUtils}            from '@yarnpkg/core';
 import {Filename, ppath, xfs} from '@yarnpkg/fslib';
 import {tests}                from 'pkg-tests-core';
 
@@ -198,6 +199,71 @@ describe(`Cache`, () => {
     }),
   );
 
+  test(`it should not confuse the cache key when merging an upgraded cache key branch into a feature branch`, makeTemporaryEnv({
+    dependencies: {
+      [`depA`]: `npm:[email protected]`,
+      [`depB`]: `npm:[email protected]`,
+      [`depC`]: `npm:[email protected]`,
+    },
+  }, async ({path, run, source}) => {
+    await run(`install`);
+
+    // First we create the main branch; it contains a bunch of various dependencies
+
+    await execUtils.execvp(`git`, [`init`], {cwd: path});
+
+    await execUtils.execvp(`git`, [`add`, `.`], {cwd: path});
+    await execUtils.execvp(`git`, [`commit`, `-m`, `test`], {cwd: path});
+
+    // We now create a new feature branch derived from the base one, and we add a new dependency
+
+    await execUtils.execvp(`git`, [`checkout`, `-b`, `feature`], {cwd: path});
+
+    await run(`add`, `depX@npm:[email protected]`);
+
+    await execUtils.execvp(`git`, [`add`, `.`], {cwd: path});
+    await execUtils.execvp(`git`, [`commit`, `-m`, `new dep`], {cwd: path});
+
+    // Meanwhile, the base branch is updated with a new Yarn version:
+    //
+    // - we add an extra byte at the end of each file in the cache, to simulate a change in how zip archives are generated
+    // - we run an install with a new cache version and we persist the new checksums
+
+    await execUtils.execvp(`git`, [`checkout`, `-`], {cwd: path});
+
+    await run(`install`, {
+      cacheVersionOverride: `2`,
+      zipDataEpilogue: `<arbitrary data>`,
+    });
+
+    //throw new Error(`lol`);
+
+    await execUtils.execvp(`git`, [`add`, `.`], {cwd: path});
+    await execUtils.execvp(`git`, [`commit`, `-m`, `fake upgrade`], {cwd: path});
+
+    // Going back to our feature branch, we now merge the updated base branch into it
+
+    await execUtils.execvp(`git`, [`checkout`, `-`], {cwd: path});
+
+    await execUtils.execvp(`git`, [`merge`, `-`], {cwd: path});
+
+    // Running an install should work fine
+
+    await run(`install`, {
+      cacheVersionOverride: `2`,
+      zipDataEpilogue: `<arbitrary data>`,
+    });
+
+    // However, once we remove the cache and try to reinstall, we used to get an error because the wrong cache key had been encoded in the lockfile
+
+    await xfs.removePromise(ppath.join(path, `.yarn/cache`));
+
+    await run(`install`, {
+      cacheVersionOverride: `2`,
+      zipDataEpilogue: `<arbitrary data>`,
+    });
+  }));
+
   test(
     `it should ignore changes in the cache compression, if cacheMigrationMode=required-only`,
     makeTemporaryEnv({

packages/acceptance-tests/pkg-tests-specs/sources/features/cache.test.ts

@@ -69,6 +69,7 @@ const IGNORED_ENV_VARIABLES = new Set([
   `injectNpmUser`,
   `injectNpmPassword`,
   `injectNpm2FaToken`,
+  `zipDataEpilogue`,
   `cacheCheckpointOverride`,
   `cacheVersionOverride`,
   `lockfileVersionOverride`,

packages/yarnpkg-core/sources/Configuration.ts

@@ -1969,23 +1969,6 @@ export class Project {
 
       manifest.bin = new Map(pkg.bin);
 
-      let entryChecksum: string | undefined;
-      const checksum = this.storedChecksums.get(pkg.locatorHash);
-      if (typeof checksum !== `undefined`) {
-        const cacheKeyIndex = checksum.indexOf(`/`);
-        if (cacheKeyIndex === -1)
-          throw new Error(`Assertion failed: Expected the checksum to reference its cache key`);
-
-        const entryCacheKey = checksum.slice(0, cacheKeyIndex);
-        const hash = checksum.slice(cacheKeyIndex + 1);
-
-        if (entryCacheKey === cacheKey) {
-          entryChecksum = hash;
-        } else {
-          entryChecksum = checksum;
-        }
-      }
-
       optimizedLockfile[key] = {
         ...manifest.exportTo({}, {
           compatibilityMode: false,
@@ -1994,7 +1977,7 @@ export class Project {
         linkType: pkg.linkType.toLowerCase(),
 
         resolution: structUtils.stringifyLocator(pkg),
-        checksum: entryChecksum,
+        checksum: this.storedChecksums.get(pkg.locatorHash),
 
         conditions: pkg.conditions || undefined,
       };

packages/yarnpkg-core/sources/Project.ts

@@ -295,8 +295,12 @@ export class ZipFS extends BasePortableFakeFS {
         else if (rc > size)
           throw new Error(`Overread`);
 
-        const memory = this.libzip.HEAPU8.subarray(buffer, buffer + size);
-        return Buffer.from(memory);
+        let result = Buffer.from(this.libzip.HEAPU8.subarray(buffer, buffer + size));
+
+        if (process.env.YARN_IS_TEST_ENV && process.env.YARN_ZIP_DATA_EPILOGUE)
+          result = Buffer.concat([result, Buffer.from(process.env.YARN_ZIP_DATA_EPILOGUE)]);
+
+        return result;
       } finally {
         this.libzip.free(buffer);
       }