From c84ba89f322b3a72ec3a28ca7bd00d07a361ec62 Mon Sep 17 00:00:00 2001 From: Stella Huang Date: Thu, 25 Jun 2026 10:37:18 -0700 Subject: [PATCH 1/2] feat(inline-scripts): add cache key hash utility (PEP 723 PR 1/3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure utility for computing a deterministic cache key for a PEP 723 inline-script environment, given the script's declared dependencies and the chosen Python interpreter path. First foundation PR for inline-script env support; no behavior impact on its own -- nothing in the extension calls these helpers yet. What is in: - src/common/inlineScriptCacheKey.ts - normalizeDependency(dep) -- folds common variants of the same PEP 723 requirement so trivial edits don't fragment the cache: * PEP 503 name normalization on the project name AND on each extra: lowercase, then collapse runs of [._-] to a single -. So `Django` ≡ `django`, `Flask-Login` ≡ `flask_login`, `requests[Socks]` ≡ `requests[socks]`, `pkg[socks_5]` ≡ `pkg[socks-5]`. * Extras are deduplicated and sorted alphabetically after canonicalization, so `requests[security,socks]` and `requests[socks,security]` hash to the same key. * Whitespace around comparison operators is stripped, so `requests <3` ≡ `requests<3`. Not a full PEP 508 parser -- only folds the superficial edits users are likely to make. Pinned-behavior tests document what is and isn't folded. - normalizeInterpreterPath(path) -- wraps the existing normalizePath helper so case/separator differences on Windows do not fragment the cache. POSIX paths pass through unchanged. - computeCacheKey({ dependencies, interpreterPath }) -- normalizes and dedupes deps, sorts, builds a versioned labelled-line payload, hashes with SHA-256, and truncates to 16 hex characters (64 bits -- well below practical collision risk for a per-user cache). - src/test/common/inlineScriptCacheKey.unit.test.ts -- 37 unit tests covering the three exports, including pinned-behavior tests for asymmetries the docstrings call out (trailing whitespace in paths fragments) and explicit canonicalization tests for extras (PEP 503 folding, sort, dedup, whitespace tolerance, empty-bracket collapse, composition with version specifiers). Caller contract The interpreter path must be absolute and symlink-resolved before calling computeCacheKey. The function does no I/O, so symlink canonicalization is the caller's responsibility -- a later PR will do that at the resolution site. Design context Implements the cache-key half of PR 1 in the Phase 1 roadmap of pep723_design_questions.md. Deps-keyed + interpreter-path-keyed matches the pipx-style choice in Q4. The PEP 503 extras handling was added in response to reviewer feedback on the design doc. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/common/inlineScriptCacheKey.ts | 103 ++++++ .../common/inlineScriptCacheKey.unit.test.ts | 315 ++++++++++++++++++ 2 files changed, 418 insertions(+) create mode 100644 src/common/inlineScriptCacheKey.ts create mode 100644 src/test/common/inlineScriptCacheKey.unit.test.ts diff --git a/src/common/inlineScriptCacheKey.ts b/src/common/inlineScriptCacheKey.ts new file mode 100644 index 00000000..02042d03 --- /dev/null +++ b/src/common/inlineScriptCacheKey.ts @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { createHash } from 'crypto'; +import { normalizePath } from './utils/pathUtils'; + +/** Length, in hex chars, of the cache key returned by {@link computeCacheKey}. 16 = 64 bits of SHA-256; fixed-length and filesystem-safe. */ +export const CACHE_KEY_HEX_LENGTH = 16; + +/** + * Inputs that participate in the cache key. + * + * **Caller contract**: `interpreterPath` must be an absolute path that the + * caller has already resolved through symlinks (e.g. via `fs.realpath()`). + * This function does no I/O. Two string-distinct paths that point to the + * same physical executable will produce two different cache keys. + */ +export interface CacheKeyInputs { + readonly dependencies: readonly string[]; + readonly interpreterPath: string; +} + +function normalizePep503Name(name: string): string { + return name.replace(/[-_.]+/g, '-').toLowerCase(); +} + +function normalizeExtras(inner: string): string { + const items = inner + .split(',') + .map((e) => e.trim()) + .filter((e) => e.length > 0) + .map((e) => normalizePep503Name(e)); + const deduped = Array.from(new Set(items)).sort(); + return deduped.length > 0 ? `[${deduped.join(',')}]` : ''; +} + +/** + * Canonicalize a PEP 723 dependency entry so common variants of the same + * requirement produce identical strings. Not a full PEP 508 parser — only + * folds the superficial edits users are likely to make: + * + * "Requests" → "requests" + * "Flask-Login" → "flask-login" + * "requests <3" → "requests<3" + * "requests <3, !=2.0" → "requests<3,!=2.0" + * "Requests[Security,Socks]" → "requests[security,socks]" + * "requests[socks,security]" → "requests[security,socks]" + * "requests[security,Security]" → "requests[security]" + * + * Extras (per PEP 503) are lowercased, separator-folded ([._-]+ → -), + * deduplicated, and sorted alphabetically — same canonical form pip and + * uv use for the project name, applied to each extra. + */ +export function normalizeDependency(dep: string): string { + const trimmed = dep.trim(); + if (trimmed.length === 0) { + return ''; + } + + const nameMatch = trimmed.match(/^[A-Za-z0-9_.-]+/); + if (!nameMatch) { + // URL/VCS spec or other malformed entry — return trimmed so the + // hash stays deterministic without pretending to parse it. + return trimmed; + } + const name = normalizePep503Name(nameMatch[0]); + let rest = trimmed.slice(nameMatch[0].length); + + let extras = ''; + const extrasMatch = rest.match(/^\s*\[([^\]]*)\]/); + if (extrasMatch) { + extras = normalizeExtras(extrasMatch[1]); + rest = rest.slice(extrasMatch[0].length); + } + + const compactedRest = rest + .replace(/\s+/g, ' ') + .replace(/\s*([<>=!~]=?)\s*/g, '$1') + .trim(); + + return `${name}${extras}${compactedRest}`; +} + +export function normalizeInterpreterPath(interpreterPath: string): string { + return normalizePath(interpreterPath); +} + +/** + * Compute the deterministic cache key for an inline-script env. The + * payload uses a versioned, labelled-line shape — any future hashed input + * MUST extend this shape rather than appending a new separator, or + * existing hashes break silently. + */ +export function computeCacheKey(inputs: CacheKeyInputs): string { + const normalizedDeps = Array.from( + new Set(inputs.dependencies.map((d) => normalizeDependency(d)).filter((d) => d.length > 0)), + ).sort(); + const normalizedInterpreter = normalizeInterpreterPath(inputs.interpreterPath); + + const payload = ['v1', `interpreter=${normalizedInterpreter}`, `deps=${normalizedDeps.join('\n ')}`].join('\n'); + + return createHash('sha256').update(payload, 'utf8').digest('hex').slice(0, CACHE_KEY_HEX_LENGTH); +} diff --git a/src/test/common/inlineScriptCacheKey.unit.test.ts b/src/test/common/inlineScriptCacheKey.unit.test.ts new file mode 100644 index 00000000..ce0e72c8 --- /dev/null +++ b/src/test/common/inlineScriptCacheKey.unit.test.ts @@ -0,0 +1,315 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import assert from 'assert'; +import * as sinon from 'sinon'; +import { + CACHE_KEY_HEX_LENGTH, + computeCacheKey, + normalizeDependency, + normalizeInterpreterPath, +} from '../../common/inlineScriptCacheKey'; +import * as platformUtils from '../../common/utils/platformUtils'; + +suite('inlineScriptCacheKey', () => { + let isWindowsStub: sinon.SinonStub; + + teardown(() => { + sinon.restore(); + }); + + suite('normalizeDependency', () => { + test('bare package name is returned lowercased', () => { + assert.strictEqual(normalizeDependency('requests'), 'requests'); + assert.strictEqual(normalizeDependency('Requests'), 'requests'); + assert.strictEqual(normalizeDependency('REQUESTS'), 'requests'); + }); + + test('PEP 503 name normalization collapses -_. variants', () => { + assert.strictEqual(normalizeDependency('Flask-Login'), 'flask-login'); + assert.strictEqual(normalizeDependency('flask_login'), 'flask-login'); + assert.strictEqual(normalizeDependency('flask.login'), 'flask-login'); + assert.strictEqual(normalizeDependency('Flask__Login'), 'flask-login'); + assert.strictEqual(normalizeDependency('flask-_-login'), 'flask-login'); + }); + + test('whitespace around comparison operators is stripped', () => { + assert.strictEqual(normalizeDependency('requests<3'), 'requests<3'); + assert.strictEqual(normalizeDependency('requests <3'), 'requests<3'); + assert.strictEqual(normalizeDependency('requests < 3'), 'requests<3'); + assert.strictEqual(normalizeDependency('requests < 3'), 'requests<3'); + }); + + test('all two-char comparison operators are stripped', () => { + assert.strictEqual(normalizeDependency('requests >= 2'), 'requests>=2'); + assert.strictEqual(normalizeDependency('requests <= 2'), 'requests<=2'); + assert.strictEqual(normalizeDependency('requests == 2'), 'requests==2'); + assert.strictEqual(normalizeDependency('requests != 2'), 'requests!=2'); + assert.strictEqual(normalizeDependency('requests ~= 2'), 'requests~=2'); + }); + + test('multi-clause version specifiers normalize each clause', () => { + assert.strictEqual(normalizeDependency('requests <3, !=2.0'), 'requests<3,!=2.0'); + // Space before `,` survives — we only fold around comparison operators. Deliberate. + assert.strictEqual(normalizeDependency('requests >= 1.0 , < 3.0'), 'requests>=1.0 ,<3.0'); + }); + + test('PEP 440 arbitrary-equality `===` is also folded', () => { + assert.strictEqual(normalizeDependency('requests === 1.0.dev0'), 'requests===1.0.dev0'); + assert.strictEqual(normalizeDependency('requests=== 1.0.dev0'), 'requests===1.0.dev0'); + }); + + test('extras section is preserved and leading whitespace before [ is stripped', () => { + assert.strictEqual(normalizeDependency('requests[security]'), 'requests[security]'); + assert.strictEqual(normalizeDependency('requests [security]'), 'requests[security]'); + assert.strictEqual(normalizeDependency('Requests [security,socks]'), 'requests[security,socks]'); + }); + + test('extras are PEP 503 normalized (lowercase + separator folding)', () => { + assert.strictEqual(normalizeDependency('requests[Socks]'), 'requests[socks]'); + assert.strictEqual(normalizeDependency('requests[SOCKS]'), 'requests[socks]'); + assert.strictEqual(normalizeDependency('pkg[socks_5]'), 'pkg[socks-5]'); + assert.strictEqual(normalizeDependency('pkg[socks.5]'), 'pkg[socks-5]'); + assert.strictEqual(normalizeDependency('pkg[Socks_5]'), 'pkg[socks-5]'); + }); + + test('extras are sorted alphabetically', () => { + assert.strictEqual(normalizeDependency('requests[socks,security]'), 'requests[security,socks]'); + assert.strictEqual(normalizeDependency('pkg[c,b,a]'), 'pkg[a,b,c]'); + }); + + test('extras are deduplicated after PEP 503 normalization', () => { + assert.strictEqual(normalizeDependency('requests[socks,Socks]'), 'requests[socks]'); + assert.strictEqual(normalizeDependency('pkg[socks_5,Socks-5,SOCKS.5]'), 'pkg[socks-5]'); + }); + + test('whitespace inside the extras list is tolerated', () => { + assert.strictEqual(normalizeDependency('requests[ security , socks ]'), 'requests[security,socks]'); + assert.strictEqual(normalizeDependency('requests[security , socks]'), 'requests[security,socks]'); + }); + + test('empty extras block normalizes to no extras', () => { + assert.strictEqual(normalizeDependency('pkg[]'), 'pkg'); + assert.strictEqual(normalizeDependency('pkg[ ]'), 'pkg'); + assert.strictEqual(normalizeDependency('pkg[,,]'), 'pkg'); + }); + + test('extras combine cleanly with a version specifier', () => { + assert.strictEqual(normalizeDependency('Requests[Socks] >= 2'), 'requests[socks]>=2'); + assert.strictEqual(normalizeDependency('requests[socks,security]<3'), 'requests[security,socks]<3'); + }); + + test('marker section after `;` is preserved but operator spaces inside it are stripped', () => { + assert.strictEqual( + normalizeDependency("requests >= 2 ; python_version >= '3.10'"), + "requests>=2 ; python_version>='3.10'", + ); + }); + + test('empty and whitespace-only entries normalize to empty string', () => { + assert.strictEqual(normalizeDependency(''), ''); + assert.strictEqual(normalizeDependency(' '), ''); + assert.strictEqual(normalizeDependency('\t\n'), ''); + }); + + test('URL/VCS specifiers are returned trimmed without further folding', () => { + const input = 'requests @ git+https://github.com/psf/requests@v2.31.0'; + const result = normalizeDependency(input); + assert.strictEqual(result, normalizeDependency(input), 'must be idempotent'); + assert.ok(result.includes('requests'), 'leading name should still appear'); + }); + + test('leading and trailing whitespace is stripped', () => { + assert.strictEqual(normalizeDependency(' requests '), 'requests'); + assert.strictEqual(normalizeDependency('\trequests<3\n'), 'requests<3'); + }); + + test('idempotent: normalizing a normalized entry produces the same string', () => { + const samples = [ + 'requests', + 'flask-login', + 'requests<3', + 'requests[security]', + 'requests[security,socks]', + "requests>=2 ; python_version>='3.10'", + ]; + for (const s of samples) { + assert.strictEqual(normalizeDependency(s), s, `not idempotent for ${JSON.stringify(s)}`); + } + }); + }); + + suite('normalizeInterpreterPath', () => { + test('on POSIX, returns the input unchanged', () => { + isWindowsStub = sinon.stub(platformUtils, 'isWindows').returns(false); + assert.strictEqual(normalizeInterpreterPath('/usr/bin/python3'), '/usr/bin/python3'); + assert.strictEqual(normalizeInterpreterPath('/Users/Me/.venv/bin/python'), '/Users/Me/.venv/bin/python'); + }); + + test('on Windows, lowercases and converts backslashes to forward slashes', () => { + isWindowsStub = sinon.stub(platformUtils, 'isWindows').returns(true); + assert.strictEqual(normalizeInterpreterPath('C:\\Python313\\python.exe'), 'c:/python313/python.exe'); + assert.strictEqual(normalizeInterpreterPath('C:/Python313/python.exe'), 'c:/python313/python.exe'); + assert.strictEqual(normalizeInterpreterPath('c:\\python313\\python.exe'), 'c:/python313/python.exe'); + }); + }); + + suite('computeCacheKey', () => { + const interpreter = '/usr/bin/python3'; + + setup(() => { + // POSIX default so path-canonicalization assertions don't depend on the host platform. + isWindowsStub = sinon.stub(platformUtils, 'isWindows').returns(false); + }); + + test('output is exactly CACHE_KEY_HEX_LENGTH lowercase hex characters', () => { + const key = computeCacheKey({ dependencies: ['requests'], interpreterPath: interpreter }); + assert.match(key, /^[0-9a-f]+$/, 'must be lowercase hex'); + assert.strictEqual(key.length, CACHE_KEY_HEX_LENGTH); + }); + + test('identical inputs produce identical keys', () => { + const a = computeCacheKey({ dependencies: ['requests', 'rich'], interpreterPath: interpreter }); + const b = computeCacheKey({ dependencies: ['requests', 'rich'], interpreterPath: interpreter }); + assert.strictEqual(a, b); + }); + + test('reordering dependencies does not change the key', () => { + const a = computeCacheKey({ dependencies: ['requests', 'rich'], interpreterPath: interpreter }); + const b = computeCacheKey({ dependencies: ['rich', 'requests'], interpreterPath: interpreter }); + assert.strictEqual(a, b); + }); + + test('trivial whitespace differences inside a version spec do not change the key', () => { + const a = computeCacheKey({ dependencies: ['requests <3'], interpreterPath: interpreter }); + const b = computeCacheKey({ dependencies: ['requests<3'], interpreterPath: interpreter }); + const c = computeCacheKey({ dependencies: ['requests < 3'], interpreterPath: interpreter }); + assert.strictEqual(a, b); + assert.strictEqual(b, c); + }); + + test('case differences in the package name do not change the key', () => { + const a = computeCacheKey({ dependencies: ['Requests'], interpreterPath: interpreter }); + const b = computeCacheKey({ dependencies: ['requests'], interpreterPath: interpreter }); + assert.strictEqual(a, b); + }); + + test('PEP 503 separator differences in the package name do not change the key', () => { + const a = computeCacheKey({ dependencies: ['Flask-Login'], interpreterPath: interpreter }); + const b = computeCacheKey({ dependencies: ['flask_login'], interpreterPath: interpreter }); + const c = computeCacheKey({ dependencies: ['flask.login'], interpreterPath: interpreter }); + assert.strictEqual(a, b); + assert.strictEqual(b, c); + }); + + test('duplicate dependencies after normalization do not change the key', () => { + const a = computeCacheKey({ dependencies: ['requests'], interpreterPath: interpreter }); + const b = computeCacheKey({ dependencies: ['requests', 'Requests'], interpreterPath: interpreter }); + const c = computeCacheKey({ + dependencies: ['requests', 'requests', 'requests'], + interpreterPath: interpreter, + }); + assert.strictEqual(a, b); + assert.strictEqual(b, c); + }); + + test('empty dependency entries are dropped before hashing', () => { + const a = computeCacheKey({ dependencies: ['requests'], interpreterPath: interpreter }); + const b = computeCacheKey({ + dependencies: ['requests', '', ' '], + interpreterPath: interpreter, + }); + assert.strictEqual(a, b); + }); + + test('adding a dependency changes the key', () => { + const a = computeCacheKey({ dependencies: ['requests'], interpreterPath: interpreter }); + const b = computeCacheKey({ dependencies: ['requests', 'rich'], interpreterPath: interpreter }); + assert.notStrictEqual(a, b); + }); + + test('removing a dependency changes the key', () => { + const a = computeCacheKey({ dependencies: ['requests', 'rich'], interpreterPath: interpreter }); + const b = computeCacheKey({ dependencies: ['requests'], interpreterPath: interpreter }); + assert.notStrictEqual(a, b); + }); + + test('changing a version pin changes the key', () => { + const a = computeCacheKey({ dependencies: ['requests<3'], interpreterPath: interpreter }); + const b = computeCacheKey({ dependencies: ['requests<4'], interpreterPath: interpreter }); + assert.notStrictEqual(a, b); + }); + + test('changing the interpreter path changes the key', () => { + const a = computeCacheKey({ dependencies: ['requests'], interpreterPath: '/usr/bin/python3' }); + const b = computeCacheKey({ dependencies: ['requests'], interpreterPath: '/opt/python/python3' }); + assert.notStrictEqual(a, b); + }); + + test('empty dependency list is valid and deterministic', () => { + const a = computeCacheKey({ dependencies: [], interpreterPath: interpreter }); + const b = computeCacheKey({ dependencies: [], interpreterPath: interpreter }); + assert.strictEqual(a, b); + assert.match(a, /^[0-9a-f]+$/); + assert.strictEqual(a.length, CACHE_KEY_HEX_LENGTH); + }); + + test('on Windows, path case and separator style do not change the key', () => { + isWindowsStub.restore(); + isWindowsStub = sinon.stub(platformUtils, 'isWindows').returns(true); + const a = computeCacheKey({ + dependencies: ['requests'], + interpreterPath: 'C:\\Python313\\python.exe', + }); + const b = computeCacheKey({ + dependencies: ['requests'], + interpreterPath: 'c:/python313/python.exe', + }); + assert.strictEqual(a, b); + }); + + test('trailing whitespace in interpreter path produces a different key (pinned behavior)', () => { + // Pinned: callers must hand us a trimmed path. We don't silently + // trim — that would invalidate every cached env on upgrade. + const a = computeCacheKey({ dependencies: ['requests'], interpreterPath: '/usr/bin/python3' }); + const b = computeCacheKey({ dependencies: ['requests'], interpreterPath: '/usr/bin/python3 ' }); + assert.notStrictEqual(a, b); + }); + + test('extras order and casing do not change the key', () => { + // PEP 503 canonicalization is applied to each extra (lowercase, + // [._-]+ → -), then they are deduped and sorted alphabetically + // — same canonical form pip / uv use for the project name, + // applied to each extra. Without this the cache would fragment + // on a trivial copy-paste edit. + const a = computeCacheKey({ + dependencies: ['requests[security,socks]'], + interpreterPath: interpreter, + }); + const b = computeCacheKey({ + dependencies: ['requests[socks,security]'], + interpreterPath: interpreter, + }); + const c = computeCacheKey({ + dependencies: ['requests[Socks,Security]'], + interpreterPath: interpreter, + }); + const d = computeCacheKey({ + dependencies: ['requests[socks_5,Security]'], + interpreterPath: interpreter, + }); + const dPrime = computeCacheKey({ + dependencies: ['requests[security,SOCKS-5]'], + interpreterPath: interpreter, + }); + assert.strictEqual(a, b); + assert.strictEqual(b, c); + assert.strictEqual(d, dPrime); + }); + + test('output is filesystem-safe (no special chars)', () => { + const key = computeCacheKey({ dependencies: ['requests'], interpreterPath: interpreter }); + assert.doesNotMatch(key, /[/\\:*?"<>|]/, 'must not contain reserved filename characters'); + }); + }); +}); From 67425240cf3de8827b83745692972846667cdaac Mon Sep 17 00:00:00 2001 From: Stella Huang Date: Mon, 20 Jul 2026 15:13:03 -0700 Subject: [PATCH 2/2] refactor(inline-scripts): reuse package name normalization --- src/common/inlineScriptCacheKey.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/common/inlineScriptCacheKey.ts b/src/common/inlineScriptCacheKey.ts index 02042d03..13c94646 100644 --- a/src/common/inlineScriptCacheKey.ts +++ b/src/common/inlineScriptCacheKey.ts @@ -2,6 +2,7 @@ // Licensed under the MIT License. import { createHash } from 'crypto'; +import { normalizePackageName } from '../managers/builtin/utils'; import { normalizePath } from './utils/pathUtils'; /** Length, in hex chars, of the cache key returned by {@link computeCacheKey}. 16 = 64 bits of SHA-256; fixed-length and filesystem-safe. */ @@ -20,16 +21,12 @@ export interface CacheKeyInputs { readonly interpreterPath: string; } -function normalizePep503Name(name: string): string { - return name.replace(/[-_.]+/g, '-').toLowerCase(); -} - function normalizeExtras(inner: string): string { const items = inner .split(',') .map((e) => e.trim()) .filter((e) => e.length > 0) - .map((e) => normalizePep503Name(e)); + .map((e) => normalizePackageName(e)); const deduped = Array.from(new Set(items)).sort(); return deduped.length > 0 ? `[${deduped.join(',')}]` : ''; } @@ -63,7 +60,7 @@ export function normalizeDependency(dep: string): string { // hash stays deterministic without pretending to parse it. return trimmed; } - const name = normalizePep503Name(nameMatch[0]); + const name = normalizePackageName(nameMatch[0]); let rest = trimmed.slice(nameMatch[0].length); let extras = '';