From 0885b53abaa0c807d212cde5395c95a5cb2b369b Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Sat, 6 Jul 2024 02:37:36 +0100 Subject: [PATCH 01/26] feat: introduced initial parser version (not finished) --- bin/.gitkeep | 1 + package-lock.json | 11 +- package.json | 3 + src/constants.mjs | 293 +++++++++++++++++++++++++++++++++++++++++++ src/metadata.mjs | 115 +++++++++++++++++ src/parser.mjs | 72 +++++++++++ src/queries.mjs | 122 ++++++++++++++++++ src/types.d.ts | 49 ++++++++ src/utils/parser.mjs | 212 +++++++++++++++++++++++++++++++ 9 files changed, 874 insertions(+), 4 deletions(-) create mode 100644 bin/.gitkeep create mode 100644 src/constants.mjs create mode 100644 src/metadata.mjs create mode 100644 src/parser.mjs create mode 100644 src/queries.mjs create mode 100644 src/types.d.ts create mode 100644 src/utils/parser.mjs diff --git a/bin/.gitkeep b/bin/.gitkeep new file mode 100644 index 00000000..f935021a --- /dev/null +++ b/bin/.gitkeep @@ -0,0 +1 @@ +!.gitignore diff --git a/package-lock.json b/package-lock.json index c60a6c97..2afa8a4d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,9 @@ "requires": true, "packages": { "": { + "dependencies": { + "yaml": "^2.4.5" + }, "devDependencies": { "@eslint/js": "^9.6.0", "eslint": "^9.6.0", @@ -1741,10 +1744,10 @@ } }, "node_modules/yaml": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.4.2.tgz", - "integrity": "sha512-B3VqDZ+JAg1nZpaEmWtTXUlBneoGx6CPM9b0TENK6aoSu5t73dItudwdgmi6tHlIZZId4dZ9skcAQ2UbcyAeVA==", - "dev": true, + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.4.5.tgz", + "integrity": "sha512-aBx2bnqDzVOyNKfsysjA2ms5ZlnjSAW2eG3/L5G/CSujfjLJTJsEw1bGw8kCf04KodQWk1pxlGnZ56CRxiawmg==", + "license": "ISC", "bin": { "yaml": "bin.mjs" }, diff --git a/package.json b/package.json index 8f0f9d9a..e6c9e0db 100644 --- a/package.json +++ b/package.json @@ -12,5 +12,8 @@ "husky": "^9.0.11", "lint-staged": "^15.2.7", "prettier": "3.2.5" + }, + "dependencies": { + "yaml": "^2.4.5" } } diff --git a/src/constants.mjs b/src/constants.mjs new file mode 100644 index 00000000..779c320c --- /dev/null +++ b/src/constants.mjs @@ -0,0 +1,293 @@ +'use strict'; + +// This is the base-path of API docs within the https://nodejs.org Wesite +export const DOC_WEB_BASE_PATH = '/docs/'; + +// This is the base URL of the MDN Web documentation +export const DOC_MDN_BASE_URL = 'https://developer.mozilla.org/en-US/docs/Web/'; + +// This is the base URL for the MDN JavaScript documentation +export const DOC_MDN_BASE_URL_JS = `${DOC_MDN_BASE_URL}JavaScript/`; + +// This is the base URL for the MDN JavaScript Primitives documentation +export const DOC_MDN_BASE_URL_JS_PRIMITIVES = `${DOC_MDN_BASE_URL_JS}Data_structures`; + +// This is the base URL for the MDN JavaScript Global Objects documentation +export const DOC_MDN_BASE_URL_JS_GLOBALS = `${DOC_MDN_BASE_URL_JS}Reference/Global_Objects/`; + +// These are YAML keys from the Markdown YAML Metadata that should always be Arrays +export const DOC_API_YAML_KEYS_ARRAYS = [ + 'added', + 'napiVersion', + 'deprecated', + 'removed', + 'introduced_in', +]; + +// These are YAML keys from the Markdown YAML Metadata that should be +// removed and appended to the `update` key +export const DOC_API_YAML_KEYS_UPDATE = [ + 'added', + 'removed', + 'deprecated', + 'introduced_in', + 'napiVersion', +]; + +// These are the YAML Types that should be considered as Navigation Entries +export const DOC_API_YAML_KEYS_NAVIGATION = ['class', 'module', 'global']; + +export const DOC_API_SLUGS_REPLACEMENTS = [ + { to: 'nodejs', from: /node.js/ }, // Replace node.js + { to: '-and-', from: /&/ }, // Replace & + { to: '-', from: /[/_,:;\\ ]/g }, // Replace /_,:;\. and whitespace +]; + +export const DOC_TYPES_MAPPING_PRIMITIVES = { + boolean: 'Boolean', + integer: 'Number', // Not a primitive, used for clarification. + null: 'Null', + number: 'Number', + string: 'String', + symbol: 'Symbol', + undefined: 'Undefined', +}; + +export const DOC_TYPES_MAPPING_GLOBALS = { + AggregateError: 'AggregateError', + Array: 'Array', + ArrayBuffer: 'ArrayBuffer', + DataView: 'DataView', + Date: 'Date', + Error: 'Error', + EvalError: 'EvalError', + Function: 'Function', + Map: 'Map', + Object: 'Object', + Promise: 'Promise', + RangeError: 'RangeError', + ReferenceError: 'ReferenceError', + RegExp: 'RegExp', + Set: 'Set', + SharedArrayBuffer: 'SharedArrayBuffer', + SyntaxError: 'SyntaxError', + TypeError: 'TypeError', + TypedArray: 'TypedArray', + URIError: 'URIError', + Uint8Array: 'Uint8Array', + bigint: 'BigInt', + 'WebAssembly.Instance': 'WebAssembly/Instance', +}; + +export const DOC_TYPES_MAPPING_NODE_MODULES = { + AbortController: 'globals.html#abortcontroller', + AbortSignal: 'globals.html#abortsignal', + + Blob: 'buffer.html#blob', + + BroadcastChannel: 'worker_threads.html#broadcastchannel-extends-eventtarget', + + AsyncHook: 'async_hooks.html#async_hookscreatehookcallbacks', + AsyncResource: 'async_hooks.html#asyncresource', + + 'brotli options': 'zlib.html#brotlioptions', + + Buffer: 'buffer.html#buffer', + + ChildProcess: 'child_process.html#childprocess', + + 'cluster.Worker': 'cluster.html#worker', + + Cipher: 'crypto.html#cipher', + Decipher: 'crypto.html#decipher', + DiffieHellman: 'crypto.html#diffiehellman', + DiffieHellmanGroup: 'crypto.html#diffiehellmangroup', + ECDH: 'crypto.html#ecdh', + Hash: 'crypto.html#hash', + Hmac: 'crypto.html#hmac', + KeyObject: 'crypto.html#keyobject', + Sign: 'crypto.html#sign', + Verify: 'crypto.html#verify', + 'crypto.constants': 'crypto.html#cryptoconstants', + + CryptoKey: 'webcrypto.html#cryptokey', + CryptoKeyPair: 'webcrypto.html#cryptokeypair', + Crypto: 'webcrypto.html#crypto', + SubtleCrypto: 'webcrypto.html#subtlecrypto', + RsaOaepParams: 'webcrypto.html#rsaoaepparams', + AlgorithmIdentifier: 'webcrypto.html#algorithmidentifier', + AesCtrParams: 'webcrypto.html#aesctrparams', + AesCbcParams: 'webcrypto.html#aescbcparams', + AesGcmParams: 'webcrypto.html#aesgcmparams', + EcdhKeyDeriveParams: 'webcrypto.html#ecdhkeyderiveparams', + HkdfParams: 'webcrypto.html#hkdfparams', + Pbkdf2Params: 'webcrypto.html#pbkdf2params', + HmacKeyGenParams: 'webcrypto.html#hmackeygenparams', + AesKeyGenParams: 'webcrypto.html#aeskeygenparams', + RsaHashedKeyGenParams: 'webcrypto.html#rsahashedkeygenparams', + EcKeyGenParams: 'webcrypto.html#eckeygenparams', + RsaHashedImportParams: 'webcrypto.html#rsahashedimportparams', + EcKeyImportParams: 'webcrypto.html#eckeyimportparams', + HmacImportParams: 'webcrypto.html#hmacimportparams', + EcdsaParams: 'webcrypto.html#ecdsaparams', + RsaPssParams: 'webcrypto.html#rsapssparams', + Ed448Params: 'webcrypto.html#ed448params', + + 'dgram.Socket': 'dgram.html#dgramsocket', + + Channel: 'diagnostics_channel.html#channel', + + Domain: 'domain.html#domain', + + 'errors.Error': 'errors.html#error', + + 'import.meta': 'esm.html#importmeta', + + EventEmitter: 'events.html#eventemitter', + EventTarget: 'events.html#eventtarget', + Event: 'events.html#event', + CustomEvent: 'events.html#customevent', + EventListener: 'events.html#listener', + + FileHandle: 'fs.html#filehandle', + 'fs.Dir': 'fs.html#fsdir', + 'fs.Dirent': 'fs.html#fsdirent', + 'fs.FSWatcher': 'fs.html#fsfswatcher', + 'fs.ReadStream': 'fs.html#fsreadstream', + 'fs.Stats': 'fs.html#fsstats', + 'fs.StatWatcher': 'fs.html#fsstatwatcher', + 'fs.WriteStream': 'fs.html#fswritestream', + + 'http.Agent': 'http.html#httpagent', + 'http.ClientRequest': 'http.html#httpclientrequest', + 'http.IncomingMessage': 'http.html#httpincomingmessage', + 'http.OutgoingMessage': 'http.html#httpoutgoingmessage', + 'http.Server': 'http.html#httpserver', + 'http.ServerResponse': 'http.html#httpserverresponse', + + ClientHttp2Session: 'http2.html#clienthttp2session', + ClientHttp2Stream: 'http2.html#clienthttp2stream', + 'HTTP/2 Headers Object': 'http2.html#headers-object', + 'HTTP/2 Settings Object': 'http2.html#settings-object', + 'http2.Http2ServerRequest': 'http2.html#http2http2serverrequest', + 'http2.Http2ServerResponse': 'http2.html#http2http2serverresponse', + Http2SecureServer: 'http2.html#http2secureserver', + Http2Server: 'http2.html#http2server', + Http2Session: 'http2.html#http2session', + Http2Stream: 'http2.html#http2stream', + ServerHttp2Stream: 'http2.html#serverhttp2stream', + ServerHttp2Session: 'http2.html#serverhttp2session', + + 'https.Server': 'https.html#httpsserver', + + module: 'modules.html#the-module-object', + + 'module.SourceMap': 'module.html#modulesourcemap', + + require: 'modules.html#requireid', + + Handle: 'net.html#serverlistenhandle-backlog-callback', + 'net.BlockList': 'net.html#netblocklist', + 'net.Server': 'net.html#netserver', + 'net.Socket': 'net.html#netsocket', + 'net.SocketAddress': 'net.html#netsocketaddress', + + NodeEventTarget: 'events.html#nodeeventtarget', + + 'os.constants.dlopen': 'os.html#dlopen-constants', + + Histogram: 'perf_hooks.html#histogram', + IntervalHistogram: 'perf_hooks.html#intervalhistogram-extends-histogram', + RecordableHistogram: 'perf_hooks.html#recordablehistogram-extends-histogram', + PerformanceEntry: 'perf_hooks.html#performanceentry', + PerformanceNodeTiming: 'perf_hooks.html#performancenodetiming', + PerformanceObserver: 'perf_hooks.html#perf_hooksperformanceobserver', + PerformanceObserverEntryList: 'perf_hooks.html#performanceobserverentrylist', + + 'readline.Interface': 'readline.html#readlineinterface', + 'readline.InterfaceConstructor': 'readline.html#interfaceconstructor', + 'readlinePromises.Interface': 'readline.html#readlinepromisesinterface', + + 'repl.REPLServer': 'repl.html#replserver', + + Stream: 'stream.html#stream', + 'stream.Duplex': 'stream.html#streamduplex', + Duplex: 'stream.html#streamduplex', + 'stream.Readable': 'stream.html#streamreadable', + Readable: 'stream.html#streamreadable', + 'stream.Transform': 'stream.html#streamtransform', + Transform: 'stream.html#streamtransform', + 'stream.Writable': 'stream.html#streamwritable', + Writable: 'stream.html#streamwritable', + + Immediate: 'timers.html#immediate', + Timeout: 'timers.html#timeout', + Timer: 'timers.html#timers', + + TapStream: 'test.html#tapstream', + + 'tls.SecureContext': 'tls.html#tlscreatesecurecontextoptions', + 'tls.Server': 'tls.html#tlsserver', + 'tls.TLSSocket': 'tls.html#tlstlssocket', + + Tracing: 'tracing.html#tracing-object', + + URL: 'url.html#the-whatwg-url-api', + URLSearchParams: 'url.html#urlsearchparams', + + 'vm.Module': 'vm.html#vmmodule', + 'vm.Script': 'vm.html#vmscript', + 'vm.SourceTextModule': 'vm.html#vmsourcetextmodule', + + MessagePort: 'worker_threads.html#messageport', + Worker: 'worker_threads.html#worker', + + X509Certificate: 'crypto.html#x509certificate', + + 'zlib options': 'zlib.html#options', + + ReadableStream: 'webstreams.html#readablestream', + ReadableStreamDefaultReader: 'webstreams.html#readablestreamdefaultreader', + ReadableStreamBYOBReader: 'webstreams.html#readablestreambyobreader', + ReadableStreamDefaultController: + 'webstreams.html#readablestreamdefaultcontroller', + ReadableByteStreamController: 'webstreams.html#readablebytestreamcontroller', + ReadableStreamBYOBRequest: 'webstreams.html#readablestreambyobrequest', + WritableStream: 'webstreams.html#writablestream', + WritableStreamDefaultWriter: 'webstreams.html#writablestreamdefaultwriter', + WritableStreamDefaultController: + 'webstreams.html#writablestreamdefaultcontroller', + TransformStream: 'webstreams.html#transformstream', + TransformStreamDefaultController: + 'webstreams.html#transformstreamdefaultcontroller', + ByteLengthQueuingStrategy: 'webstreams.html#bytelengthqueuingstrategy', + CountQueuingStrategy: 'webstreams.html#countqueuingstrategy', + TextEncoderStream: 'webstreams.html#textencoderstream', + TextDecoderStream: 'webstreams.html#textdecoderstream', +}; + +export const DOC_TYPES_MAPPING_OTHER = { + any: `${DOC_MDN_BASE_URL_JS_PRIMITIVES}#Data_types`, + this: `${DOC_MDN_BASE_URL_JS}Reference/Operators/this`, + + ArrayBufferView: + 'https://developer.mozilla.org/en-US/docs/Web/API/ArrayBufferView', + + AsyncIterator: 'https://tc39.github.io/ecma262/#sec-asynciterator-interface', + AsyncIterable: 'https://tc39.github.io/ecma262/#sec-asynciterable-interface', + AsyncFunction: 'https://tc39.es/ecma262/#sec-async-function-constructor', + + 'Module Namespace Object': + 'https://tc39.github.io/ecma262/#sec-module-namespace-exotic-objects', + + AsyncGeneratorFunction: + 'https://tc39.es/proposal-async-iteration/#sec-asyncgeneratorfunction-constructor', + + Iterable: `${DOC_MDN_BASE_URL_JS}Reference/Iteration_protocols#The_iterable_protocol`, + Iterator: `${DOC_MDN_BASE_URL_JS}Reference/Iteration_protocols#The_iterator_protocol`, + + FormData: `${DOC_MDN_BASE_URL}API/FormData`, + Headers: `${DOC_MDN_BASE_URL}/API/Headers`, + Response: `${DOC_MDN_BASE_URL}/API/Response`, + Request: `${DOC_MDN_BASE_URL}/API/Request`, +}; diff --git a/src/metadata.mjs b/src/metadata.mjs new file mode 100644 index 00000000..0fbab576 --- /dev/null +++ b/src/metadata.mjs @@ -0,0 +1,115 @@ +'use strict'; + +import { + DOC_API_YAML_KEYS_NAVIGATION, + DOC_WEB_BASE_PATH, +} from './constants.mjs'; +import { stripHeadingPrefix, titleToSlug } from './utils/parser.mjs'; + +/** + * This method allows us to handle creation of Navigation Entries + * for a given file within the API documentation + * + * This can be used disconnected with a specific file, and can be aggregated + * to many files to create a full Navigation for a given version of the API + * + * @param {import('./types').ApiDocMetadata} + */ +const createMetadata = ({ version, name }) => { + const navigationMetadataEntries = []; + + return { + /** + * Retrieves all Navigation Entries generated in a said file + * + * @returns {Array} + */ + getNavigationEntries: () => navigationMetadataEntries, + newMetadataEntry: () => { + const internalMetadata = { + type: undefined, + heading: undefined, + properties: {}, + }; + + return { + /** + * The way how this works is that once we're iterating over each Section within a file + * Section = a Paragraph (or in other words chunks of text separated by two line breaks) + * We use our utilities to identify the type of the Section if possible + * And attempts to find pieces of either the current section or previous sections that are Headings. + * Within the current Doc specification the Type Metadata could be at least two leves after the heading + * + * @param {string} type The API YAML Metadata Type to add to the iteration + */ + setType: type => { + internalMetadata.type = type; + }, + /** + * Set the Heading Line of a given Metadata + * + * @param {Array} lines The current Heading Lines + */ + setHeading: lines => { + internalMetadata.heading = lines.split('\n', 1)[0]; + }, + /** + * Set the Metadata (from YAML if exists) properties to the current Metadata Entry + * + * @param {Record} properties Extra Metadata Properties to be defined + */ + setProperties: properties => { + internalMetadata.properties = properties; + }, + /** + * Generates Navigation Entries for the current Navigation Creator + * and pushes them to the Navigation Entries for the current API file + * + * @param {string} content The content of the current Metadata Entry + * @returns {import('./types').ApiDocMetadataEntry} The locally created Metadata Entries + */ + create: content => { + // We want to replace any prefix the Heading might have + const title = stripHeadingPrefix(internalMetadata.heading); + + // This is the ID of a certain Navigation Entry, which allows us to anchor + // a certain navigation section to a page ad the exact point of the page (scroll) + // This is useful for classes, globals and other type of YAML entries, as they reside + // within a module (page) and we want to link to them directly + // If the YAML entry is a "module" (aka an API doc file), we don't want to add a hash + // as the module starts within the page itself + const slugHash = + internalMetadata.type !== 'module' ? `#${titleToSlug(title)}` : ''; + + const metadataEntry = { + // Prepends all imported Properties + ...internalMetadata.properties, + // The API file name + name, + // The Content of an API Section + content, + // The unique key of the API Section + key: `${name}/${slugHash}`, + // The metadata type of the API Section + type: internalMetadata.type, + // The path/slug of the API Section + slug: `${DOC_WEB_BASE_PATH}${version}/${name}/${slugHash}`, + // Sanitizes the Heading by replacing certain characters + // and if the heading has parentheses, uses the portion before the parenthesis + title: title.split('(')[0].replace(/[^\w\- ]+/g, ''), + }; + + // If this metadata type matches certain predefined types + // We include it as a Navigation Entry for this API file + if (DOC_API_YAML_KEYS_NAVIGATION.includes(metadataEntry.type)) { + navigationMetadataEntries.push(metadataEntry); + } + + return metadataEntry; + }, + }; + }, + }; +}; + +export default createMetadata; diff --git a/src/parser.mjs b/src/parser.mjs new file mode 100644 index 00000000..582834bc --- /dev/null +++ b/src/parser.mjs @@ -0,0 +1,72 @@ +'use strict'; + +import createMetadata from './metadata.mjs'; +import createQueries from './queries.mjs'; +import { calculateCodeBlockIntersection } from './utils/parser.mjs'; + +/** + * Creates an API Doc Parser for a given Markdown API Doc + * (this requires already parsed Node.js release data, the API doc file to be loaded, and etc) + * + * @param {import('./types.d.ts').ApiDocMetadata} fileMetadata API Doc Metadata + * @param {string} markdownContent + */ +const getParser = (fileMetadata, markdownContent) => { + const { newMetadataEntry, getNavigationEntries: getNavigation } = + createMetadata(fileMetadata); + + const parseMetadata = () => { + const markdownSections = markdownContent.split('\n\n#'); + + const isCurrentLinesCodeBlock = calculateCodeBlockIntersection(); + + const parsedSections = markdownSections.map(section => { + const apiEntryMetadata = newMetadataEntry(); + + const { + markdownFootUrls, + addHeadingLevel, + normalizeLinks, + normalizeTypes, + normalizeHeading, + parseYAML, + } = createQueries(apiEntryMetadata, fileMetadata); + + const parsedLines = section.split('\n\n').map(lines => { + // This verifies if the current lines are part of a multi-line code block + // This allows us to simply ignore any modification during this time + if (isCurrentLinesCodeBlock(lines)) { + return lines; + } + + // Line Block starts is a Heading + if (lines.startsWith('#')) { + apiEntryMetadata.setHeading(lines); + + return lines + .replace(createQueries.QUERIES.addHeadingLevel, addHeadingLevel) + .replace(createQueries.QUERIES.normalizeHeading, normalizeHeading); + } + + // Line Block is a YAML Metadata + if (lines.startsWith('/, + // ReGeX for replacing the Stability Index with a MDX Component + // @TODO: Only used for the MDX Generator + stabilityIndex: /^> (.*:)\s*(\d)([\s\S]*)/, +}; + +export default createQueries; diff --git a/src/types.d.ts b/src/types.d.ts new file mode 100644 index 00000000..98c5e20c --- /dev/null +++ b/src/types.d.ts @@ -0,0 +1,49 @@ +export interface ApiDocMetadata { + // The name of the API Doc file without the file extenson (basename) + name: string; + // The Node.js release line (major) targeted for this API doc tooling + // this version includes the `v` prefix (i.e.: v18) + version: string; + // The Node.js version targeted for this API doc tooling + // this version includes the `v` prefix (i.e.: v18.0.0) + fullVersion: string; +} + +export interface ApiDocMetadataChange { + // The Node.js version or versions where said change was introduced simultaneously + version: string | string[]; + // The GitHub PR URL of said change + 'pr-url': string; + // The description of said change + description: string; +} + +export interface ApiDocMetadataUpdate { + // The type of the API Doc Metadata update + type: 'added' | 'removed' | 'deprecated' | 'introduced_in' | 'napiVersion'; + // The Node.js version or versions where said metadata stability index changed + version: string[]; +} + +export interface ApiDocRawMetadataEntry { + type?: string; + name?: string; + source_link?: string; + update?: ApiDocMetadataUpdate; + changes?: ApiDocMetadataChange[]; +} + +export interface ApiDocMetadataEntry extends ApiDocRawMetadataEntry { + // The unique key of a given metadata entry + key: string; + // The unique slug of a Heading/Navigation Entry which is linkable through an anchor + slug: string; + // Human Readable Title (Plain Text) for a Navigation Entry + title: string; + // The API YAML Metadata Type for said entry + type: string; + // The name of the API Doc fle without the file extension (basename) + name: string; + // The parsed Markdown content of a Navigation Entry + content: string; +} diff --git a/src/utils/parser.mjs b/src/utils/parser.mjs new file mode 100644 index 00000000..40cc5203 --- /dev/null +++ b/src/utils/parser.mjs @@ -0,0 +1,212 @@ +'use strict'; + +import yaml from 'yaml'; + +import { + DOC_API_SLUGS_REPLACEMENTS, + DOC_API_YAML_KEYS_ARRAYS, + DOC_API_YAML_KEYS_UPDATE, + DOC_MDN_BASE_URL_JS_GLOBALS, + DOC_MDN_BASE_URL_JS_PRIMITIVES, + DOC_TYPES_MAPPING_GLOBALS, + DOC_TYPES_MAPPING_NODE_MODULES, + DOC_TYPES_MAPPING_OTHER, + DOC_TYPES_MAPPING_PRIMITIVES, + DOC_WEB_BASE_PATH, +} from '../constants.mjs'; + +/** + * All YAML values are treated as an array for consistency + * since they get rendered within a table of changelog + * + * @param {string|Array} value YAML metadata entry value + * @returns {Array} YAML metadata entry value as an Array + */ +export const yamlValueToArray = value => + Array.isArray(value) ? value : [value]; + +/** + * Strips the Markdown Heading Prefix and the API Doc Section Heading Prefix + * of a given Heading + * + * @param {string} heading An API doc section Heading + * @returns {string} A heading without the heading prefix + */ +export const stripHeadingPrefix = heading => + heading.replace(/^#{1,5} /i, '').replace(/^(Modules|Class|Event): /, ''); + +/** + * Transforms a page title into an unique slug (that can be used as an ID, anchor, or URL) + * that follows slug conventions defined by the Node.js project + * + * @param {string} title The title to be parsed into a slug + * @returns {string} A title parsed into a page slug + */ +export const transformTitleToSlug = title => { + let slug = title.toLowerCase().trim(); + + DOC_API_SLUGS_REPLACEMENTS.forEach(set => { + slug = slug.replace(set.from, set.to); + }); + + return ( + slug + // This escape is actually needed here + // eslint-disable-next-line no-useless-escape + .replace(/[^\w\-]+/g, '') // Remove any non word characters + .replace(/--+/g, '-') // Replace multiple hyphens with single + .replace(/^-/, '') // Remove any leading hyphen + .replace(/-$/, '') // Remove any trailing hyphen + ); +}; + +/** + * This method replaces plain text Types within the Markdown content into Markdown links + * that link to the actual relevant reference for such type (either internal or external link) + * + * @param {import('../types.d.ts').ApiDocMetadata} apiMetadata API Doc Metadata + * @param {string} type The plain type to be transformed into a Markdown Link + * @returns {string} The Markdown Link as a string (formatted in Markdown) + */ +export const transformTypeToReferenceLink = (apiMetadata, type) => { + const typeInput = type.replace('{', '').replace('}', ''); + + const typePieces = typeInput.split('|').map(piece => { + // This is the content to render as the text of the Markdown Link + const trimmedPiece = piece.trim(); + + // This is what we will compare against the API Types Mappings + const lookupPiece = trimmedPiece.replace(/(?:\[])+$/, ''); + + // This is what we return on the Array map uppon a hit, this function + // is just for making this formatting reusable + const formatToMarkdownLink = result => `[\`${trimmedPiece}\`](${result})`; + + // Transform JS Primitive Type references into Markdown Links (MDN) + if (lookupPiece.toLowerCase() in DOC_TYPES_MAPPING_PRIMITIVES) { + const typeValue = DOC_TYPES_MAPPING_PRIMITIVES[lookupPiece.toLowerCase()]; + + const typeLink = `${DOC_MDN_BASE_URL_JS_PRIMITIVES}#${typeValue}_type`; + + return formatToMarkdownLink(typeLink); + } + + // Transforms JS Global Type references into Markdown Links (MDN) + if (lookupPiece in DOC_TYPES_MAPPING_GLOBALS) { + const typeLink = `${DOC_MDN_BASE_URL_JS_GLOBALS}${lookupPiece}`; + + return formatToMarkdownLink(typeLink); + } + + // Transform other external Web/JavaScript Type references into Markdown Links + // to diverse different external websites. These already are formatted as links + if (lookupPiece in DOC_TYPES_MAPPING_OTHER) { + const typeValueAsLink = DOC_TYPES_MAPPING_NODE_MODULES[lookupPiece]; + + return formatToMarkdownLink(typeValueAsLink); + } + + // Transform Node.js Type/Module references into Markdown Links + // that refer to other API Docs pages within the Node.js API docs + if (lookupPiece in DOC_TYPES_MAPPING_NODE_MODULES) { + const typeValue = DOC_TYPES_MAPPING_NODE_MODULES[lookupPiece]; + + const typeLink = `${DOC_WEB_BASE_PATH}${apiMetadata.version}/${typeValue}`; + + return formatToMarkdownLink(typeLink); + } + + return undefined; + }); + + // Filter out pieces that we failed to Map and then join the valid ones + // into different links separated by a `|` + const markdownLinks = typePieces.filter(Boolean).join(' | '); + + // Return the replaced links or the original content if they all failed to be replaced + // Note that if some failed to get replaced, only the valid ones will be returned + // NOTE: Based on the original code, we don't seem to care when we fail specific entries to be replaced + // although I believe this should be revisited and either show the original type content or show a warning + return markdownLinks || typeInput; +}; + +/** + * Parses Markdown YAML source into a JavaScript object containing all the metadata + * (this is forwarded to the parser so it knows what to do with said metadata) + * + * @param {ReturnType['newMetadataEntry']>} apiEntryMetadata The current Navigation instance + * @param {string} yamlString The YAML string to be parsed + * @returns {import('../types.d.ts').ApiDocRawMetadataEntry} + */ +export const parseYAMLIntoMetadata = (apiEntryMetadata, yamlString) => { + const cleanContent = yamlString.replace(/YAML| YAML/, ''); + + const replacedContent = cleanContent + // special validations for some non-cool formatted properties + // of the docs schema + .replace('introduced_in=', 'introduced_in: ') + .replace('source_link=', 'source_link: ') + .replace('type=', 'type: ') + .replace('name=', 'name: '); + + // Ensures that the parsed YAML is an object, because even if it is not + // i.e. a plain string or an array, it will simply not result into anything + const parsedYaml = Object(yaml.parse(replacedContent)); + + // This cleans up the YAML metadata into something more standardized + Object.keys(parsedYaml).forEach(key => { + // Some entries should always be Array for the sake of consistency + if (DOC_API_YAML_KEYS_ARRAYS.includes(key)) { + parsedYaml[key] = yamlValueToArray(parsedYaml[key]); + } + + if (DOC_API_YAML_KEYS_UPDATE.includes(key)) { + // We transform some entries in a standardized "updates" field + parsedYaml.update = { type: key, version: parsedYaml[key] }; + + delete parsedYaml[key]; + } + }); + + // If there is a `type` key we set the current metadata type to this type + if (parsedYaml.type && parsedYaml.type.length) { + apiEntryMetadata.setType(parsedYaml.type); + } + + apiEntryMetadata.setProperties(parsedYaml); + + const stringifiedYaml = JSON.stringify(parsedYaml); + + return ``; +}; + +/** + * This method allows us to keep track if we are intersecting code blocks + * and allows us to skip API Doc Parsing whilst we are inside a code block + * and prevents other issues from happening + */ +export const calculateCodeBlockIntersection = () => { + let isIntersecting = false; + + return lines => { + const linesStartWithCodeBlock = lines.startsWith('```'); + const linesEndsWithCodeBlock = lines.endsWith('```'); + + if (linesStartWithCodeBlock) { + isIntersecting = true; + } + + if (isIntersecting === true) { + // This means we're currently iterating inside a code block + // We should ignore parsing all lines until we reach the + // end of the code block + if (linesEndsWithCodeBlock) { + // If we have a ending code block, stop ignoring the code loop + // and go back to normal business starting the next block + isIntersecting = false; + } + } + + return isIntersecting; + }; +}; From 04f73730c9196b0bf903447fb7190d2f6bba1ba3 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Sat, 6 Jul 2024 06:28:43 +0100 Subject: [PATCH 02/26] chore: continued iterating on the source (but now with unified) --- package-lock.json | 1177 +++++++++++++++++++++++++++++++++++++++++- package.json | 6 + src/metadata.mjs | 40 +- src/parser.mjs | 104 ++-- src/queries.mjs | 126 ++--- src/utils/parser.mjs | 51 +- 6 files changed, 1324 insertions(+), 180 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2afa8a4d..63ff6319 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,10 +5,16 @@ "packages": { "": { "dependencies": { + "remark": "^15.0.1", + "remark-gfm": "^4.0.0", + "unist-util-remove": "^4.0.0", + "unist-util-select": "^5.1.0", + "unist-util-visit": "^5.0.0", "yaml": "^2.4.5" }, "devDependencies": { "@eslint/js": "^9.6.0", + "@types/node": "^20.14.10", "eslint": "^9.6.0", "eslint-config-prettier": "^9.1.0", "globals": "^15.8.0", @@ -181,6 +187,46 @@ "node": ">= 8" } }, + "node_modules/@types/debug": { + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", + "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/ms": { + "version": "0.7.34", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz", + "integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "20.14.10", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.10.tgz", + "integrity": "sha512-MdiXf+nDuMvY0gJKxyfZ7/6UFsETO7mGKF54MVD/ekJS6HdFtpZFBgrh6Pseu64XTb2MLyFPlbW6hj8HYRQNOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/unist": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.2.tgz", + "integrity": "sha512-dqId9J8K/vGi5Zr7oo212BGii5m3q5Hxlkwy3WpYuKPklmBEvsbMYYyLxAQpSffdLl/gdW0XUpKWFvYmyoWCoQ==", + "license": "MIT" + }, "node_modules/acorn": { "version": "8.12.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz", @@ -260,12 +306,28 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true }, + "node_modules/bail": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz", + "integrity": "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "license": "ISC" + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -297,6 +359,16 @@ "node": ">=6" } }, + "node_modules/ccount": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", + "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -313,6 +385,16 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/cli-cursor": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-4.0.0.tgz", @@ -397,11 +479,26 @@ "node": ">= 8" } }, + "node_modules/css-selector-parser": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/css-selector-parser/-/css-selector-parser-3.0.5.tgz", + "integrity": "sha512-3itoDFbKUNx1eKmVpYMFyqKX04Ww9osZ+dLgrk6GEv6KMVeXUhUnp4I5X+evw+u3ZxVU6RFXSSRxlTeMh8bA+g==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/mdevils" + }, + { + "type": "patreon", + "url": "https://patreon.com/mdevils" + } + ], + "license": "MIT" + }, "node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, "dependencies": { "ms": "2.1.2" }, @@ -414,12 +511,47 @@ } } }, + "node_modules/decode-named-character-reference": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz", + "integrity": "sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==", + "license": "MIT", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/emoji-regex": { "version": "10.3.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.3.0.tgz", @@ -617,6 +749,12 @@ "url": "https://github.com/sindresorhus/execa?sponsor=1" } }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -869,6 +1007,18 @@ "node": ">=8" } }, + "node_modules/is-plain-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-stream": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz", @@ -1117,12 +1267,795 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, + "node_modules/longest-streak": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", + "integrity": "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/markdown-table": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.3.tgz", + "integrity": "sha512-Z1NL3Tb1M9wH4XESsCDEksWoKTdlUafKc4pt0GRwjUyXaCFZ+dc3g2erqB6zm3szA2IUSi7VnPI+o/9jnxh9hw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/mdast-util-find-and-replace": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.1.tgz", + "integrity": "sha512-SG21kZHGC3XRTSUhtofZkBzZTJNM5ecCi0SK2IMKmSXR8vO3peL+kb1O0z7Zl83jKtutG4k5Wv/W7V3/YHvzPA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "escape-string-regexp": "^5.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-find-and-replace/node_modules/escape-string-regexp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mdast-util-from-markdown": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.1.tgz", + "integrity": "sha512-aJEUyzZ6TzlsX2s5B4Of7lN7EQtAxvtradMMglCQDyaTFgse6CmtmdJ15ElnVRlCg1vpNyVtbem0PWzlNieZsA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark": "^4.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-3.0.0.tgz", + "integrity": "sha512-dgQEX5Amaq+DuUqf26jJqSK9qgixgd6rYDHAv4aTBuA92cTknZlKpPfa86Z/s8Dj8xsAQpFfBmPUHWJBWqS4Bw==", + "license": "MIT", + "dependencies": { + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-gfm-autolink-literal": "^2.0.0", + "mdast-util-gfm-footnote": "^2.0.0", + "mdast-util-gfm-strikethrough": "^2.0.0", + "mdast-util-gfm-table": "^2.0.0", + "mdast-util-gfm-task-list-item": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-autolink-literal": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-2.0.0.tgz", + "integrity": "sha512-FyzMsduZZHSc3i0Px3PQcBT4WJY/X/RCtEJKuybiC6sjPqLv7h1yqAkmILZtuxMSsUyaLUWNp71+vQH2zqp5cg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "ccount": "^2.0.0", + "devlop": "^1.0.0", + "mdast-util-find-and-replace": "^3.0.0", + "micromark-util-character": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-footnote": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-2.0.0.tgz", + "integrity": "sha512-5jOT2boTSVkMnQ7LTrd6n/18kqwjmuYqo7JUPe+tRCY6O7dAuTFMtTPauYYrMPpox9hlN0uOx/FL8XvEfG9/mQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-strikethrough": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-2.0.0.tgz", + "integrity": "sha512-mKKb915TF+OC5ptj5bJ7WFRPdYtuHv0yTRxK2tJvi+BDqbkiG7h7u/9SI89nRAYcmap2xHQL9D+QG/6wSrTtXg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-table": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-2.0.0.tgz", + "integrity": "sha512-78UEvebzz/rJIxLvE7ZtDd/vIQ0RHv+3Mh5DR96p7cS7HsBhYIICDBCu8csTNWNO6tBWfqXPWekRuj2FNOGOZg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "markdown-table": "^3.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-task-list-item": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-2.0.0.tgz", + "integrity": "sha512-IrtvNvjxC1o06taBAVJznEnkiHxLFTzgonUdy8hzFVeDun0uTjxxrRGVaNFqkU1wJR3RBPEfsxmU6jDWPofrTQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-phrasing": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-4.1.0.tgz", + "integrity": "sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-markdown": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-2.1.0.tgz", + "integrity": "sha512-SR2VnIEdVNCJbP6y7kVTJgPLifdr8WEU440fQec7qHoHOUz/oJ2jmNRqdDQ3rbiStOXb2mCDGTuwsK5OPUgYlQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "longest-streak": "^3.0.0", + "mdast-util-phrasing": "^4.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark-util-decode-string": "^2.0.0", + "unist-util-visit": "^5.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-4.0.0.tgz", + "integrity": "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", "dev": true }, + "node_modules/micromark": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.0.tgz", + "integrity": "sha512-o/sd0nMof8kYff+TqcDx3VSrgBTcZpSvYcAHIfHhv5VAuNmisCxjhx6YmxS8PFEpb9z5WKWKPdzf0jM23ro3RQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.1.tgz", + "integrity": "sha512-CUQyKr1e///ZODyD1U3xit6zXwy1a8q2a1S1HKtIlmgvurrEpaw/Y9y6KSIbF8P59cn/NjzHyO+Q2fAyYLQrAA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-gfm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-3.0.0.tgz", + "integrity": "sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==", + "license": "MIT", + "dependencies": { + "micromark-extension-gfm-autolink-literal": "^2.0.0", + "micromark-extension-gfm-footnote": "^2.0.0", + "micromark-extension-gfm-strikethrough": "^2.0.0", + "micromark-extension-gfm-table": "^2.0.0", + "micromark-extension-gfm-tagfilter": "^2.0.0", + "micromark-extension-gfm-task-list-item": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-autolink-literal": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-2.1.0.tgz", + "integrity": "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==", + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-strikethrough": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-2.1.0.tgz", + "integrity": "sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-table": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-2.1.0.tgz", + "integrity": "sha512-Ub2ncQv+fwD70/l4ou27b4YzfNaCJOvyX4HxXU15m7mpYY+rjuWzsLIPZHJL253Z643RpbcP1oeIJlQ/SKW67g==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-tagfilter": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-2.0.0.tgz", + "integrity": "sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg==", + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-task-list-item": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-2.1.0.tgz", + "integrity": "sha512-qIBZhqxqI6fjLDYFTBIa4eivDMnP+OZqsNwmQ3xNLE4Cxwc+zfQEfbs6tzAo2Hjq+bh6q5F+Z8/cksrLFYWQQw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.0.tgz", + "integrity": "sha512-j9DGrQLm/Uhl2tCzcbLhy5kXsgkHUrjJHg4fFAeoMRwJmJerT9aw4FEhIbZStWN8A3qMwOp1uzHr4UL8AInxtA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.0.tgz", + "integrity": "sha512-RR3i96ohZGde//4WSe/dJsxOX6vxIg9TimLAS3i4EhBAFx8Sm5SmqVfR8E87DPSR31nEAjZfbt91OMZWcNgdZw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.0.tgz", + "integrity": "sha512-TKr+LIDX2pkBJXFLzpyPyljzYK3MtmllMUMODTQJIUfDGncESaqB90db9IAUcz4AZAJFdd8U9zOp9ty1458rxg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.0.tgz", + "integrity": "sha512-jY8CSxmpWLOxS+t8W+FG3Xigc0RDQA9bKMY/EwILvsesiRniiVMejYTE4wumNc2f4UbAa4WsHqe3J1QS1sli+A==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.0.tgz", + "integrity": "sha512-28kbwaBjc5yAI1XadbdPYHX/eDnqaUFVikLwrO7FDnKG7lpgxnvk/XGRhX/PN0mOZ+dBSZ+LgunHS+6tYQAzhA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.0.tgz", + "integrity": "sha512-KvOVV+X1yLBfs9dCBSopq/+G1PcgT3lAK07mC4BzXi5E7ahzMAF8oIupDDJ6mievI6F+lAATkbQQlQixJfT3aQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.0.tgz", + "integrity": "sha512-anK8SWmNphkXdaKgz5hJvGa7l00qmcaUQoMYsBwDlSKFKjc6gjGXPDw3FNL3Nbwq5L8gE+RCbGqTw49FK5Qyvg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.0.tgz", + "integrity": "sha512-S0ze2R9GH+fu41FA7pbSqNWObo/kzwf8rN/+IGlW/4tC6oACOs8B++bh+i9bVyNnwCcuksbFwsBme5OCKXCwIw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.0.tgz", + "integrity": "sha512-vZZio48k7ON0fVS3CUgFatWHoKbbLTK/rT7pzpJ4Bjp5JjkZeasRfrS9wsBdDJK2cJLHMckXZdzPSSr1B8a4oQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.1.tgz", + "integrity": "sha512-bmkNc7z8Wn6kgjZmVHOX3SowGmVdhYS7yBpMnuMnPzDq/6xwVA604DuOXMZTO1lvq01g+Adfa0pE2UKGlxL1XQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-2.0.0.tgz", + "integrity": "sha512-r4Sc6leeUTn3P6gk20aFMj2ntPwn6qpDZqWvYmAG6NgvFTIlj4WtrAudLi65qYoaGdXYViXYw2pkmn7QnIFasA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.0.tgz", + "integrity": "sha512-pS+ROfCXAGLWCOc8egcBvT0kf27GoWMqtdarNfDcjb6YLuV5cM3ioG45Ys2qOVqeqSbjaKg72vU+Wby3eddPsA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.0.tgz", + "integrity": "sha512-xNn4Pqkj2puRhKdKTm8t1YHC/BAjx6CEwRFXntTaRf/x16aqka6ouVoutm+QdkISTlT7e2zU7U4ZdlDLJd2Mcw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.0.tgz", + "integrity": "sha512-2xhYT0sfo85FMrUPtHcPo2rrp1lwbDEEzpx7jiH2xXJLqBuy4H0GgXk5ToU8IEwoROtXuL8ND0ttVa4rNqYK3w==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.0.tgz", + "integrity": "sha512-6KU6qO7DZ7GJkaCgwBNtplXCvGkJToU86ybBAUdavvgsCiG8lSSvYxr9MhwmQ+udpzywHsl4RpGJsYWG1pDOcA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.0.tgz", + "integrity": "sha512-WhYv5UEcZrbAtlsnPuChHUAsu/iBPOVaEVsntLBIdpibO0ddy8OzavZz3iL2xVvBZOpolujSliP65Kq0/7KIYw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.0.1.tgz", + "integrity": "sha512-jZNtiFl/1aY73yS3UGQkutD0UbhTt68qnRpw2Pifmz5wV9h8gOVsN70v+Lq/f1rKaU/W8pxRe8y8Q9FX1AOe1Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.0.tgz", + "integrity": "sha512-8JZt9ElZ5kyTnO94muPxIGS8oyElRJaiJO8EzV6ZSyGQ1Is8xwl4Q45qU5UOg+bGH4AikWziz0iN4sFLWs8PGw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.0.tgz", + "integrity": "sha512-oNh6S2WMHWRZrmutsRmDDfkzKtxF+bc2VxLC9dvtrDIRFln627VsFP6fLMgTryGDljgLPjkrzQSDcPrjPyDJ5w==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, "node_modules/micromatch": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", @@ -1163,8 +2096,7 @@ "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/natural-compare": { "version": "1.4.0", @@ -1199,6 +2131,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, "node_modules/onetime": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", @@ -1368,6 +2312,71 @@ } ] }, + "node_modules/remark": { + "version": "15.0.1", + "resolved": "https://registry.npmjs.org/remark/-/remark-15.0.1.tgz", + "integrity": "sha512-Eht5w30ruCXgFmxVUSlNWQ9iiimq07URKeFS3hNc8cUWy1llX4KDWfyEDZRycMc+znsN9Ux5/tJ/BFdgdOwA3A==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-gfm": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-4.0.0.tgz", + "integrity": "sha512-U92vJgBPkbw4Zfu/IiW2oTZLSL3Zpv+uI7My2eq8JxKgqraFdU8YUGicEJCEgSbeaG+QDFqIcwwfMTOEelPxuA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-gfm": "^3.0.0", + "micromark-extension-gfm": "^3.0.0", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-parse": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", + "integrity": "sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-stringify": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-11.0.0.tgz", + "integrity": "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-to-markdown": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -1642,6 +2651,16 @@ "node": ">=8.0" } }, + "node_modules/trough": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/trough/-/trough-2.2.0.tgz", + "integrity": "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -1654,6 +2673,119 @@ "node": ">= 0.8.0" } }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true, + "license": "MIT" + }, + "node_modules/unified": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", + "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "bail": "^2.0.0", + "devlop": "^1.0.0", + "extend": "^3.0.0", + "is-plain-obj": "^4.0.0", + "trough": "^2.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-is": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.0.tgz", + "integrity": "sha512-2qCTHimwdxLfz+YzdGfkqNlH0tLi9xjTnHddPmJwtIG9MGsdbutfTc4P+haPD7l7Cjxf/WZj+we5qfVPvvxfYw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-remove": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-remove/-/unist-util-remove-4.0.0.tgz", + "integrity": "sha512-b4gokeGId57UVRX/eVKej5gXqGlc9+trkORhFJpu9raqZkZhU0zm8Doi05+HaiBsMEIJowL+2WtQ5ItjsngPXg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-select": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/unist-util-select/-/unist-util-select-5.1.0.tgz", + "integrity": "sha512-4A5mfokSHG/rNQ4g7gSbdEs+H586xyd24sdJqF1IWamqrLHvYb+DH48fzxowyOhOfK7YSqX+XlCojAyuuyyT2A==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "css-selector-parser": "^3.0.0", + "devlop": "^1.1.0", + "nth-check": "^2.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-stringify-position": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", + "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.0.0.tgz", + "integrity": "sha512-MR04uvD+07cwl/yhVuVWAtw+3GOR/knlL55Nd/wAdblk27GCVt3lqpTivy/tkJcZoNPzTwS1Y+KMojlLDhoTzg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-parents": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-6.0.1.tgz", + "integrity": "sha512-L/PqWzfTP9lzzEa6CKs0k2nARxTdZduw3zyh8d2NVBnsyvHjSX4TWse388YrrQKbvI8w20fGjGlhgT96WwKykw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", @@ -1663,6 +2795,35 @@ "punycode": "^2.1.0" } }, + "node_modules/vfile": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.1.tgz", + "integrity": "sha512-1bYqc7pt6NIADBJ98UiG0Bn/CHIVOoZ/IyEkqIruLg0mE1BKzkOXY2D6CSqQIcKqgadppE5lrxgWXJmXd7zZJw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-stringify-position": "^4.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-message": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.2.tgz", + "integrity": "sha512-jRDZ1IMLttGj41KcZvlrYAaI3CfqpLpfpf+Mfig13viT6NKvRzWZ+lXz0Y5D60w6uJIBAOGq9mSHf0gktF0duw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -1766,6 +2927,16 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zwitch": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", + "integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } } } } diff --git a/package.json b/package.json index e6c9e0db..b529dc4a 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ }, "devDependencies": { "@eslint/js": "^9.6.0", + "@types/node": "^20.14.10", "eslint": "^9.6.0", "eslint-config-prettier": "^9.1.0", "globals": "^15.8.0", @@ -14,6 +15,11 @@ "prettier": "3.2.5" }, "dependencies": { + "remark": "^15.0.1", + "remark-gfm": "^4.0.0", + "unist-util-remove": "^4.0.0", + "unist-util-select": "^5.1.0", + "unist-util-visit": "^5.0.0", "yaml": "^2.4.5" } } diff --git a/src/metadata.mjs b/src/metadata.mjs index 0fbab576..32672114 100644 --- a/src/metadata.mjs +++ b/src/metadata.mjs @@ -4,7 +4,7 @@ import { DOC_API_YAML_KEYS_NAVIGATION, DOC_WEB_BASE_PATH, } from './constants.mjs'; -import { stripHeadingPrefix, titleToSlug } from './utils/parser.mjs'; +import { stripHeadingPrefix, transformTitleToSlug } from './utils/parser.mjs'; /** * This method allows us to handle creation of Navigation Entries @@ -60,6 +60,10 @@ const createMetadata = ({ version, name }) => { */ setProperties: properties => { internalMetadata.properties = properties; + + if (properties.type) { + internalMetadata.type = properties.type; + } }, /** * Generates Navigation Entries for the current Navigation Creator @@ -79,24 +83,38 @@ const createMetadata = ({ version, name }) => { // If the YAML entry is a "module" (aka an API doc file), we don't want to add a hash // as the module starts within the page itself const slugHash = - internalMetadata.type !== 'module' ? `#${titleToSlug(title)}` : ''; + internalMetadata.type !== 'module' + ? `#${transformTitleToSlug(title)}` + : ''; + + const { + type: yamlType, + name: yamlName, + source_link, + update, + changes = [], + } = internalMetadata.properties; const metadataEntry = { - // Prepends all imported Properties - ...internalMetadata.properties, - // The API file name - name, - // The Content of an API Section - content, // The unique key of the API Section - key: `${name}/${slugHash}`, + key: `${name}${slugHash}`, // The metadata type of the API Section - type: internalMetadata.type, + type: yamlType || internalMetadata.type, + // The API file name + name: yamlName || name, // The path/slug of the API Section - slug: `${DOC_WEB_BASE_PATH}${version}/${name}/${slugHash}`, + slug: `${DOC_WEB_BASE_PATH}${version}/${name}.html${slugHash}`, // Sanitizes the Heading by replacing certain characters // and if the heading has parentheses, uses the portion before the parenthesis title: title.split('(')[0].replace(/[^\w\- ]+/g, ''), + // The Source Link of said API Section + source_link, + // The latest update to an API Section + update, + // The full-changeset to an API Section + changes, + // The Content of an API Section + content, }; // If this metadata type matches certain predefined types diff --git a/src/parser.mjs b/src/parser.mjs index 582834bc..91c8d51b 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -1,8 +1,16 @@ 'use strict'; +import { remark } from 'remark'; +import remarkGfm from 'remark-gfm'; + +import { visit } from 'unist-util-visit'; +import { remove } from 'unist-util-remove'; +import { selectAll } from 'unist-util-select'; + import createMetadata from './metadata.mjs'; import createQueries from './queries.mjs'; -import { calculateCodeBlockIntersection } from './utils/parser.mjs'; + +const getRemarkParser = () => remark().use(remarkGfm); /** * Creates an API Doc Parser for a given Markdown API Doc @@ -11,56 +19,70 @@ import { calculateCodeBlockIntersection } from './utils/parser.mjs'; * @param {import('./types.d.ts').ApiDocMetadata} fileMetadata API Doc Metadata * @param {string} markdownContent */ -const getParser = (fileMetadata, markdownContent) => { +const createParser = (fileMetadata, markdownContent) => { const { newMetadataEntry, getNavigationEntries: getNavigation } = createMetadata(fileMetadata); const parseMetadata = () => { const markdownSections = markdownContent.split('\n\n#'); - const isCurrentLinesCodeBlock = calculateCodeBlockIntersection(); - const parsedSections = markdownSections.map(section => { const apiEntryMetadata = newMetadataEntry(); - const { - markdownFootUrls, - addHeadingLevel, - normalizeLinks, - normalizeTypes, - normalizeHeading, - parseYAML, - } = createQueries(apiEntryMetadata, fileMetadata); - - const parsedLines = section.split('\n\n').map(lines => { - // This verifies if the current lines are part of a multi-line code block - // This allows us to simply ignore any modification during this time - if (isCurrentLinesCodeBlock(lines)) { - return lines; - } - - // Line Block starts is a Heading - if (lines.startsWith('#')) { - apiEntryMetadata.setHeading(lines); - - return lines - .replace(createQueries.QUERIES.addHeadingLevel, addHeadingLevel) - .replace(createQueries.QUERIES.normalizeHeading, normalizeHeading); - } - - // Line Block is a YAML Metadata - if (lines.startsWith('/, // ReGeX for replacing the Stability Index with a MDX Component // @TODO: Only used for the MDX Generator stabilityIndex: /^> (.*:)\s*(\d)([\s\S]*)/, + // ReGeX for retrieving the inner content from a YAML block + yamlInnerContent: /^/, +}; + +createQueries.TESTS = { + isYamlNode: ({ type, value }) => + type === 'html' && createQueries.QUERIES.yamlInnerContent.test(value), + isTextWithType: ({ type, value }) => + type === 'text' && createQueries.QUERIES.normalizeTypes.test(value), + isMarkdownFootUrl: ({ type, value }) => + type === 'link' && createQueries.QUERIES.markdownFootUrls.test(value), + isHeadingNode: ({ type, depth }) => + type === 'heading' && depth >= 1 && depth <= 6, }; export default createQueries; diff --git a/src/utils/parser.mjs b/src/utils/parser.mjs index 40cc5203..f482fcdb 100644 --- a/src/utils/parser.mjs +++ b/src/utils/parser.mjs @@ -134,14 +134,11 @@ export const transformTypeToReferenceLink = (apiMetadata, type) => { * Parses Markdown YAML source into a JavaScript object containing all the metadata * (this is forwarded to the parser so it knows what to do with said metadata) * - * @param {ReturnType['newMetadataEntry']>} apiEntryMetadata The current Navigation instance * @param {string} yamlString The YAML string to be parsed - * @returns {import('../types.d.ts').ApiDocRawMetadataEntry} + * @returns {import('../types.d.ts').ApiDocRawMetadataEntry} The parsed YAML Metadata */ -export const parseYAMLIntoMetadata = (apiEntryMetadata, yamlString) => { - const cleanContent = yamlString.replace(/YAML| YAML/, ''); - - const replacedContent = cleanContent +export const parseYAMLIntoMetadata = yamlString => { + const replacedContent = yamlString // special validations for some non-cool formatted properties // of the docs schema .replace('introduced_in=', 'introduced_in: ') @@ -168,45 +165,5 @@ export const parseYAMLIntoMetadata = (apiEntryMetadata, yamlString) => { } }); - // If there is a `type` key we set the current metadata type to this type - if (parsedYaml.type && parsedYaml.type.length) { - apiEntryMetadata.setType(parsedYaml.type); - } - - apiEntryMetadata.setProperties(parsedYaml); - - const stringifiedYaml = JSON.stringify(parsedYaml); - - return ``; -}; - -/** - * This method allows us to keep track if we are intersecting code blocks - * and allows us to skip API Doc Parsing whilst we are inside a code block - * and prevents other issues from happening - */ -export const calculateCodeBlockIntersection = () => { - let isIntersecting = false; - - return lines => { - const linesStartWithCodeBlock = lines.startsWith('```'); - const linesEndsWithCodeBlock = lines.endsWith('```'); - - if (linesStartWithCodeBlock) { - isIntersecting = true; - } - - if (isIntersecting === true) { - // This means we're currently iterating inside a code block - // We should ignore parsing all lines until we reach the - // end of the code block - if (linesEndsWithCodeBlock) { - // If we have a ending code block, stop ignoring the code loop - // and go back to normal business starting the next block - isIntersecting = false; - } - } - - return isIntersecting; - }; + return parsedYaml; }; From 5a9c7cf156ca8928d76878d87b01e8bf49b57d5e Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Sat, 6 Jul 2024 14:43:20 +0100 Subject: [PATCH 03/26] chore: finished initial loader/parser implementation --- package-lock.json | 1 + package.json | 1 + src/loader.mjs | 26 ++++++++++++ src/metadata.mjs | 96 ++++++++++++++++++------------------------- src/parser.mjs | 98 +++++++++++++++++++++++++++++--------------- src/queries.mjs | 46 ++++----------------- src/types.d.ts | 47 ++++++++++++--------- src/utils/parser.mjs | 97 ++++++++++++++++++++++++++++++++++--------- src/utils/unist.mjs | 33 +++++++++++++++ 9 files changed, 279 insertions(+), 166 deletions(-) create mode 100644 src/loader.mjs create mode 100644 src/utils/unist.mjs diff --git a/package-lock.json b/package-lock.json index 63ff6319..2291a4ee 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "unist-util-remove": "^4.0.0", "unist-util-select": "^5.1.0", "unist-util-visit": "^5.0.0", + "vfile": "^6.0.1", "yaml": "^2.4.5" }, "devDependencies": { diff --git a/package.json b/package.json index b529dc4a..a1faa289 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "unist-util-remove": "^4.0.0", "unist-util-select": "^5.1.0", "unist-util-visit": "^5.0.0", + "vfile": "^6.0.1", "yaml": "^2.4.5" } } diff --git a/src/loader.mjs b/src/loader.mjs new file mode 100644 index 00000000..61c5627e --- /dev/null +++ b/src/loader.mjs @@ -0,0 +1,26 @@ +'use strict'; + +import { readFile } from 'node:fs/promises'; +import { extname } from 'node:path'; +import { VFile } from 'vfile'; + +const createLoader = () => { + /** + * Loads an API Doc file and transforms it into a VFile + * + * @param {string} path The API Doc Path + */ + const loadFile = async path => { + if (extname(path) !== '.md') { + throw new Error('API Doc file must be a Markdown file'); + } + + const apiDocFile = await readFile(path, 'utf-8'); + + return new VFile({ path: path, value: apiDocFile }); + }; + + return { loadFile }; +}; + +export default createLoader; diff --git a/src/metadata.mjs b/src/metadata.mjs index 32672114..baef15c2 100644 --- a/src/metadata.mjs +++ b/src/metadata.mjs @@ -1,10 +1,7 @@ 'use strict'; -import { - DOC_API_YAML_KEYS_NAVIGATION, - DOC_WEB_BASE_PATH, -} from './constants.mjs'; -import { stripHeadingPrefix, transformTitleToSlug } from './utils/parser.mjs'; +import { DOC_API_YAML_KEYS_NAVIGATION } from './constants.mjs'; +import { stringToSlug } from './utils/parser.mjs'; /** * This method allows us to handle creation of Navigation Entries @@ -12,51 +9,41 @@ import { stripHeadingPrefix, transformTitleToSlug } from './utils/parser.mjs'; * * This can be used disconnected with a specific file, and can be aggregated * to many files to create a full Navigation for a given version of the API - * - * @param {import('./types').ApiDocMetadata} */ -const createMetadata = ({ version, name }) => { +const createMetadata = () => { const navigationMetadataEntries = []; return { /** * Retrieves all Navigation Entries generated in a said file * - * @returns {Array} + * @returns {Array} */ getNavigationEntries: () => navigationMetadataEntries, newMetadataEntry: () => { const internalMetadata = { - type: undefined, - heading: undefined, + heading: { + text: undefined, + type: undefined, + name: undefined, + depth: -1, + }, properties: {}, }; return { - /** - * The way how this works is that once we're iterating over each Section within a file - * Section = a Paragraph (or in other words chunks of text separated by two line breaks) - * We use our utilities to identify the type of the Section if possible - * And attempts to find pieces of either the current section or previous sections that are Headings. - * Within the current Doc specification the Type Metadata could be at least two leves after the heading - * - * @param {string} type The API YAML Metadata Type to add to the iteration - */ - setType: type => { - internalMetadata.type = type; - }, /** * Set the Heading Line of a given Metadata * - * @param {Array} lines The current Heading Lines + * @param {import('./types.d.ts').HeadingMetadataEntry} heading The new Heading Metadata */ - setHeading: lines => { - internalMetadata.heading = lines.split('\n', 1)[0]; + setHeading: heading => { + internalMetadata.heading = heading; }, /** * Set the Metadata (from YAML if exists) properties to the current Metadata Entry * - * @param {Record} properties Extra Metadata Properties to be defined + * @param {import('./types.d.ts').ApiDocRawMetadataEntry} properties Extra Metadata Properties to be defined */ setProperties: properties => { internalMetadata.properties = properties; @@ -69,61 +56,56 @@ const createMetadata = ({ version, name }) => { * Generates Navigation Entries for the current Navigation Creator * and pushes them to the Navigation Entries for the current API file * - * @param {string} content The content of the current Metadata Entry + * @param {string} apiDoc The name of the API Doc + * @param {import('vfile').VFile} section The content of the current Metadata Entry * @returns {import('./types').ApiDocMetadataEntry} The locally created Metadata Entries */ - create: content => { - // We want to replace any prefix the Heading might have - const title = stripHeadingPrefix(internalMetadata.heading); - + create: (apiDoc, section) => { // This is the ID of a certain Navigation Entry, which allows us to anchor // a certain navigation section to a page ad the exact point of the page (scroll) // This is useful for classes, globals and other type of YAML entries, as they reside // within a module (page) and we want to link to them directly - // If the YAML entry is a "module" (aka an API doc file), we don't want to add a hash - // as the module starts within the page itself - const slugHash = - internalMetadata.type !== 'module' - ? `#${transformTitleToSlug(title)}` - : ''; + const slugHash = `#${stringToSlug(internalMetadata.heading.text)}`; const { type: yamlType, name: yamlName, source_link, - update, + updates = [], changes = [], } = internalMetadata.properties; - const metadataEntry = { - // The unique key of the API Section - key: `${name}${slugHash}`, - // The metadata type of the API Section - type: yamlType || internalMetadata.type, - // The API file name - name: yamlName || name, + // We override the type of the heading if we have a YAML type + internalMetadata.heading.type = + yamlType || internalMetadata.heading.type; + + const navigationEntry = { + // The API file Basename (without the Extension) + api: yamlName || apiDoc, // The path/slug of the API Section - slug: `${DOC_WEB_BASE_PATH}${version}/${name}.html${slugHash}`, - // Sanitizes the Heading by replacing certain characters - // and if the heading has parentheses, uses the portion before the parenthesis - title: title.split('(')[0].replace(/[^\w\- ]+/g, ''), + slug: `${apiDoc}.html${slugHash}`, // The Source Link of said API Section - source_link, + sourceLink: source_link, // The latest update to an API Section - update, + updates, // The full-changeset to an API Section changes, - // The Content of an API Section - content, + // The Heading Metadata + heading: internalMetadata.heading, }; // If this metadata type matches certain predefined types // We include it as a Navigation Entry for this API file - if (DOC_API_YAML_KEYS_NAVIGATION.includes(metadataEntry.type)) { - navigationMetadataEntries.push(metadataEntry); + if (DOC_API_YAML_KEYS_NAVIGATION.includes(navigationEntry.type)) { + navigationMetadataEntries.push(navigationEntry); } - return metadataEntry; + // A metadata entry is all the metadata we have about a certain API Section + // with the content being a VFile (Virtual File) containing the Markdown content + section.data = navigationEntry; + + // Returns the updated VFile with the extra Metadata + return section; }, }; }, diff --git a/src/parser.mjs b/src/parser.mjs index 91c8d51b..38b7f94b 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -10,34 +10,62 @@ import { selectAll } from 'unist-util-select'; import createMetadata from './metadata.mjs'; import createQueries from './queries.mjs'; +import { transformNodesToRaw } from './utils/unist.mjs'; +import { parseHeadingIntoMetadata } from './utils/parser.mjs'; + +// Creates a Remark Parser with all Plugins needed for our API Docs const getRemarkParser = () => remark().use(remarkGfm); /** * Creates an API Doc Parser for a given Markdown API Doc * (this requires already parsed Node.js release data, the API doc file to be loaded, and etc) - * - * @param {import('./types.d.ts').ApiDocMetadata} fileMetadata API Doc Metadata - * @param {string} markdownContent */ -const createParser = (fileMetadata, markdownContent) => { +const createParser = () => { + const { getReferenceLink, parseYAML } = createQueries(); + const { newMetadataEntry, getNavigationEntries: getNavigation } = - createMetadata(fileMetadata); + createMetadata(); + + /** + * Parses a given API Doc Metadata file into a list of Metadata Entries + * + * @param {import('vfile').VFile} apiDoc + */ + const parseMetadata = async apiDoc => { + // Does extra modification at the root level of the API Doc File + const apiDocRootParser = getRemarkParser().use(() => { + return tree => { + const definitions = selectAll('definition', tree); + + // Handles Link References + visit(tree, createQueries.UNIST_TESTS.isLinkReference, node => { + const definition = definitions.find( + ({ identifier }) => identifier === node.identifier + ); + + node.type = 'link'; + node.url = definition.url; + }); + + // Removes the Definitions from the Tree + remove(tree, definitions); + }; + }); - const parseMetadata = () => { - const markdownSections = markdownContent.split('\n\n#'); + const parsedRoot = await apiDocRootParser.process(apiDoc); - const parsedSections = markdownSections.map(section => { - const apiEntryMetadata = newMetadataEntry(); + // Gathers each Markdown Section within the API Doc + // by separating chunks of the source (root) by heading separators + const markdownSections = parsedRoot.toString().split('\n\n#'); - const { getReferenceLink, getHeadingType, parseYAML } = createQueries( - apiEntryMetadata, - fileMetadata - ); + // Parses each Markdown Section into a Metadata Entry + const parsedSections = markdownSections.map(async sectionSource => { + const apiEntryMetadata = newMetadataEntry(); - const markdownParser = getRemarkParser().use(() => { - return tree => { + const apiDocSectionParser = getRemarkParser().use(() => { + return (tree, section) => { // Handles YAML Metadata - visit(tree, createQueries.TESTS.isYamlNode, node => { + visit(tree, createQueries.UNIST_TESTS.isYamlNode, node => { const metadata = parseYAML(node.value); apiEntryMetadata.setProperties(metadata); @@ -46,43 +74,45 @@ const createParser = (fileMetadata, markdownContent) => { }); // Handles Markdown Headings - visit(tree, createQueries.TESTS.isHeadingNode, node => { - const heading = node.children.map(({ value }) => value).join(''); + visit(tree, createQueries.UNIST_TESTS.isHeadingNode, node => { + const heading = transformNodesToRaw(node.children, section); + + const parsedHeading = parseHeadingIntoMetadata( + heading, + node.depth + 1 + ); - apiEntryMetadata.setHeading(heading); - apiEntryMetadata.setType(getHeadingType(heading, node.depth + 1)); + apiEntryMetadata.setHeading(parsedHeading); remove(tree, node); }); // Handles API Type References transformation into Links - visit(tree, createQueries.TESTS.isTextWithType, node => { - node.value = node.value.replace( + visit(tree, createQueries.UNIST_TESTS.isTextWithType, node => { + const parsedReference = node.value.replace( createQueries.QUERIES.normalizeTypes, getReferenceLink ); + + node.type = 'html'; + node.value = parsedReference; }); // Handles Normalisation of Markdown URLs - visit(tree, createQueries.TESTS.isMarkdownFootUrl, node => { + visit(tree, createQueries.UNIST_TESTS.isMarkdownUrl, node => { node.url = node.url.replace( - createQueries.QUERIES.markdownFootUrls, + createQueries.QUERIES.markdownUrl, (_, filename, hash = '') => `${filename}.html${hash}` ); }); - - // Moves Definitions to the Root - visit(tree, 'root', node => { - const definitions = selectAll('definition', tree); - - node.children.concat(definitions); - - remove(tree, definitions); - }); }; }); - return apiEntryMetadata.create(markdownParser.processSync(section)); + // Process the Markdown Section into a VFile with the processed data + const parsedSection = await apiDocSectionParser.process(sectionSource); + + // Creates a Metadata Entry (VFile) and returns it + return apiEntryMetadata.create(apiDoc.stem, parsedSection); }); return parsedSections; diff --git a/src/queries.mjs b/src/queries.mjs index 3ee012f2..f76f0e2a 100644 --- a/src/queries.mjs +++ b/src/queries.mjs @@ -5,10 +5,8 @@ import * as parserUtils from './utils/parser.mjs'; /** * Creates an instance of the Query Manager, which allows to do multiple sort * of metadata and content metadata manipulation within an API Doc - * - * @param {import('./types.d.ts').ApiDocMetadata} fileMetadata The current top-level API metadata */ -const createQueries = fileMetadata => { +const createQueries = () => { /** * Transforms plain reference to Web/JavaScript/Node.js types * into Markdown links containing the proper reference to said types @@ -16,35 +14,7 @@ const createQueries = fileMetadata => { * @param {string} source The type source */ const getReferenceLink = source => - parserUtils.transformTypeToReferenceLink(fileMetadata, source); - - /** - * Retrieves the Heading Type of a Sub Heading - * - * @param {string} heading The Heading Inner Content - * @param {number} depth The Heading Depth - */ - const getHeadingType = (heading, depth) => { - if (depth === 1) { - return 'module'; - } - - if (depth >= 2 && depth <= 4) { - if (heading.startsWith('Class:')) { - return 'class'; - } - - if (heading.startsWith('Event:')) { - return 'event'; - } - - if (heading.startsWith('Static method:')) { - return 'classMethod'; - } - - return heading.includes('(') ? 'method' : 'property'; - } - }; + parserUtils.transformTypeToReferenceLink(source); /** * Sanitizes the YAML source by returning the inner YAML content @@ -61,13 +31,13 @@ const createQueries = fileMetadata => { return parserUtils.parseYAMLIntoMetadata(sanitizedString); }; - return { getReferenceLink, getHeadingType, parseYAML }; + return { getReferenceLink, parseYAML }; }; // This defines the actual REGEX Queries createQueries.QUERIES = { // Fixes the references to Markdown pages into the API documentation - markdownFootUrls: /^(?![+a-z]+:)([^#?]+)\.md(#.+)?$/i, + markdownUrl: /^(?![+a-z]+:)([^#?]+)\.md(#.+)?$/i, // ReGeX to match the {Type} (Structure Type metadatas) // eslint-disable-next-line no-useless-escape normalizeTypes: /(\{|<)(?! )[a-z0-9.| \n\[\]\\]+(?! )(\}|>)/gim, @@ -78,15 +48,17 @@ createQueries.QUERIES = { yamlInnerContent: /^/, }; -createQueries.TESTS = { +createQueries.UNIST_TESTS = { isYamlNode: ({ type, value }) => type === 'html' && createQueries.QUERIES.yamlInnerContent.test(value), isTextWithType: ({ type, value }) => type === 'text' && createQueries.QUERIES.normalizeTypes.test(value), - isMarkdownFootUrl: ({ type, value }) => - type === 'link' && createQueries.QUERIES.markdownFootUrls.test(value), + isMarkdownUrl: ({ type, url }) => + type === 'link' && createQueries.QUERIES.markdownUrl.test(url), isHeadingNode: ({ type, depth }) => type === 'heading' && depth >= 1 && depth <= 6, + isLinkReference: ({ type, identifier }) => + type === 'linkReference' && !!identifier, }; export default createQueries; diff --git a/src/types.d.ts b/src/types.d.ts index 98c5e20c..4f6dcc8e 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -1,12 +1,17 @@ -export interface ApiDocMetadata { - // The name of the API Doc file without the file extenson (basename) +import { VFile } from 'vfile'; + +export interface HeadingMetadataEntry { + type: + | 'event' + | 'method' + | 'property' + | 'class' + | 'module' + | 'classMethod' + | 'ctor'; + text: string; name: string; - // The Node.js release line (major) targeted for this API doc tooling - // this version includes the `v` prefix (i.e.: v18) - version: string; - // The Node.js version targeted for this API doc tooling - // this version includes the `v` prefix (i.e.: v18.0.0) - fullVersion: string; + depth: number; } export interface ApiDocMetadataChange { @@ -29,21 +34,25 @@ export interface ApiDocRawMetadataEntry { type?: string; name?: string; source_link?: string; - update?: ApiDocMetadataUpdate; + updates?: ApiDocMetadataUpdate[]; changes?: ApiDocMetadataChange[]; } -export interface ApiDocMetadataEntry extends ApiDocRawMetadataEntry { - // The unique key of a given metadata entry - key: string; +export interface ApiDocNavigationEntry { + // The name of the API Doc fle without the file extension (basename) + api: string; // The unique slug of a Heading/Navigation Entry which is linkable through an anchor slug: string; - // Human Readable Title (Plain Text) for a Navigation Entry - title: string; - // The API YAML Metadata Type for said entry - type: string; - // The name of the API Doc fle without the file extension (basename) - name: string; + // The GitHub URL to the source of the API Entry + sourceLink: string | undefined; + // Any updates to the API Doc Metadata + updates: ApiDocMetadataUpdate[]; + // Any changes to the API Doc Metadata + changes: ApiDocMetadataChange[]; // The parsed Markdown content of a Navigation Entry - content: string; + heading: HeadingMetadataEntry; +} + +export interface ApiDocMetadataEntry extends VFile { + data: ApiDocNavigationEntry; } diff --git a/src/utils/parser.mjs b/src/utils/parser.mjs index f482fcdb..2ff40dca 100644 --- a/src/utils/parser.mjs +++ b/src/utils/parser.mjs @@ -12,7 +12,6 @@ import { DOC_TYPES_MAPPING_NODE_MODULES, DOC_TYPES_MAPPING_OTHER, DOC_TYPES_MAPPING_PRIMITIVES, - DOC_WEB_BASE_PATH, } from '../constants.mjs'; /** @@ -25,16 +24,6 @@ import { export const yamlValueToArray = value => Array.isArray(value) ? value : [value]; -/** - * Strips the Markdown Heading Prefix and the API Doc Section Heading Prefix - * of a given Heading - * - * @param {string} heading An API doc section Heading - * @returns {string} A heading without the heading prefix - */ -export const stripHeadingPrefix = heading => - heading.replace(/^#{1,5} /i, '').replace(/^(Modules|Class|Event): /, ''); - /** * Transforms a page title into an unique slug (that can be used as an ID, anchor, or URL) * that follows slug conventions defined by the Node.js project @@ -42,7 +31,7 @@ export const stripHeadingPrefix = heading => * @param {string} title The title to be parsed into a slug * @returns {string} A title parsed into a page slug */ -export const transformTitleToSlug = title => { +export const stringToSlug = title => { let slug = title.toLowerCase().trim(); DOC_API_SLUGS_REPLACEMENTS.forEach(set => { @@ -64,11 +53,10 @@ export const transformTitleToSlug = title => { * This method replaces plain text Types within the Markdown content into Markdown links * that link to the actual relevant reference for such type (either internal or external link) * - * @param {import('../types.d.ts').ApiDocMetadata} apiMetadata API Doc Metadata * @param {string} type The plain type to be transformed into a Markdown Link * @returns {string} The Markdown Link as a string (formatted in Markdown) */ -export const transformTypeToReferenceLink = (apiMetadata, type) => { +export const transformTypeToReferenceLink = type => { const typeInput = type.replace('{', '').replace('}', ''); const typePieces = typeInput.split('|').map(piece => { @@ -109,11 +97,9 @@ export const transformTypeToReferenceLink = (apiMetadata, type) => { // Transform Node.js Type/Module references into Markdown Links // that refer to other API Docs pages within the Node.js API docs if (lookupPiece in DOC_TYPES_MAPPING_NODE_MODULES) { - const typeValue = DOC_TYPES_MAPPING_NODE_MODULES[lookupPiece]; - - const typeLink = `${DOC_WEB_BASE_PATH}${apiMetadata.version}/${typeValue}`; + const typeValueAsLink = DOC_TYPES_MAPPING_NODE_MODULES[lookupPiece]; - return formatToMarkdownLink(typeLink); + return formatToMarkdownLink(typeValueAsLink); } return undefined; @@ -159,7 +145,7 @@ export const parseYAMLIntoMetadata = yamlString => { if (DOC_API_YAML_KEYS_UPDATE.includes(key)) { // We transform some entries in a standardized "updates" field - parsedYaml.update = { type: key, version: parsedYaml[key] }; + parsedYaml.updates = [{ type: key, version: parsedYaml[key] }]; delete parsedYaml[key]; } @@ -167,3 +153,76 @@ export const parseYAMLIntoMetadata = yamlString => { return parsedYaml; }; + +/** + * Parses a raw Heading String into Heading Metadata + * + * @param {string} heading The raw Heading Text + * @param {number} depth The depth of the heading + * @returns {import('../types.d.ts').HeadingMetadataEntry} Parsed Heading Entry + */ +export const parseHeadingIntoMetadata = (heading, depth) => { + const r = String.raw; + + const eventPrefix = '^Event: +'; + const classPrefix = '^Class: +'; + const ctorPrefix = '^(?:Constructor: +)?`?new +'; + const classMethodPrefix = '^Static method: +'; + const maybeClassPropertyPrefix = '(?:Class property: +)?'; + + const maybeQuote = '[\'"]?'; + const notQuotes = '[^\'"]+'; + + const maybeBacktick = '`?'; + + // To include constructs like `readable\[Symbol.asyncIterator\]()` + // or `readable.\_read(size)` (with Markdown escapes). + const simpleId = r`(?:(?:\\?_)+|\b)\w+\b`; + const computedId = r`\\?\[[\w\.]+\\?\]`; + const id = `(?:${simpleId}|${computedId})`; + const classId = r`[A-Z]\w+`; + + const ancestors = r`(?:${id}\.?)+`; + const maybeAncestors = r`(?:${id}\.?)*`; + + const callWithParams = r`\([^)]*\)`; + + const maybeExtends = `(?: +extends +${maybeAncestors}${classId})?`; + + const headingTypes = [ + { + type: 'event', + test: `${eventPrefix}${maybeBacktick}${maybeQuote}(${notQuotes})${maybeQuote}${maybeBacktick}$`, + }, + { + type: 'class', + test: `${classPrefix}${maybeBacktick}(${maybeAncestors}${classId})${maybeExtends}${maybeBacktick}$`, + }, + { + type: 'ctor', + test: `${ctorPrefix}(${maybeAncestors}${classId})${callWithParams}${maybeBacktick}$`, + }, + { + type: 'classMethod', + test: `${classMethodPrefix}${maybeBacktick}${maybeAncestors}(${id})${callWithParams}${maybeBacktick}$`, + }, + { + type: 'method', + test: `^${maybeBacktick}${maybeAncestors}(${id})${callWithParams}${maybeBacktick}$`, + }, + { + type: 'property', + test: `^${maybeClassPropertyPrefix}${maybeBacktick}${ancestors}(${id})${maybeBacktick}$`, + }, + ]; + + for (const headingType of headingTypes) { + const result = heading.match(new RegExp(headingType.test)); + + if (result && result.length >= 2) { + return { text: heading, type: headingType.type, name: result[1], depth }; + } + } + + return { text: heading, type: 'module', name: heading, depth }; +}; diff --git a/src/utils/unist.mjs b/src/utils/unist.mjs new file mode 100644 index 00000000..1a1f333c --- /dev/null +++ b/src/utils/unist.mjs @@ -0,0 +1,33 @@ +'use strict'; + +/** + * This utility allows us to join children Nodes into one + * and do extra parsing within them + * + * @param {import('unist').Node} nodes Nodes to parsed and joined + * @param {import('vfile').VFile} source The source VFile + * @returns {string} The parsed and joined nodes as a string + */ +export const transformNodesToRaw = (nodes, source) => { + const mappedChildren = nodes.map(node => { + if (node.type === 'inlineCode') { + return `\`${node.value}\``; + } + + if (node.type === 'strong') { + return `**${transformNodesToRaw(node.children, source)}**`; + } + + if (node.type === 'emphasis') { + return `_${transformNodesToRaw(node.children, source)}_`; + } + + if (node.children) { + return transformNodesToRaw(node.children, source); + } + + return node.value; + }); + + return mappedChildren.join(''); +}; From 1cd2202945a64a8067ee88429b6e0c3d5d2d2cc2 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Sat, 6 Jul 2024 15:23:31 +0100 Subject: [PATCH 04/26] chore: stability index metadata --- src/metadata.mjs | 24 +++++---- src/parser.mjs | 53 ++++++++------------ src/queries.mjs | 119 ++++++++++++++++++++++++++++++++++++++------ src/types.d.ts | 8 +++ src/utils/unist.mjs | 9 ++-- 5 files changed, 149 insertions(+), 64 deletions(-) diff --git a/src/metadata.mjs b/src/metadata.mjs index baef15c2..da5825b8 100644 --- a/src/metadata.mjs +++ b/src/metadata.mjs @@ -43,14 +43,13 @@ const createMetadata = () => { /** * Set the Metadata (from YAML if exists) properties to the current Metadata Entry * - * @param {import('./types.d.ts').ApiDocRawMetadataEntry} properties Extra Metadata Properties to be defined + * @param {Partial} properties Extra Metadata Properties to be defined */ - setProperties: properties => { - internalMetadata.properties = properties; - - if (properties.type) { - internalMetadata.type = properties.type; - } + updateProperties: properties => { + internalMetadata.properties = { + ...internalMetadata.properties, + ...properties, + }; }, /** * Generates Navigation Entries for the current Navigation Creator @@ -68,20 +67,21 @@ const createMetadata = () => { const slugHash = `#${stringToSlug(internalMetadata.heading.text)}`; const { - type: yamlType, - name: yamlName, + type: yaml_type, + name: yaml_name, source_link, + stability_index, updates = [], changes = [], } = internalMetadata.properties; // We override the type of the heading if we have a YAML type internalMetadata.heading.type = - yamlType || internalMetadata.heading.type; + yaml_type || internalMetadata.heading.type; const navigationEntry = { // The API file Basename (without the Extension) - api: yamlName || apiDoc, + api: yaml_name || apiDoc, // The path/slug of the API Section slug: `${apiDoc}.html${slugHash}`, // The Source Link of said API Section @@ -92,6 +92,8 @@ const createMetadata = () => { changes, // The Heading Metadata heading: internalMetadata.heading, + // The Stability Index of the API Section + stability: stability_index, }; // If this metadata type matches certain predefined types diff --git a/src/parser.mjs b/src/parser.mjs index 38b7f94b..5be44c2a 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -10,9 +10,6 @@ import { selectAll } from 'unist-util-select'; import createMetadata from './metadata.mjs'; import createQueries from './queries.mjs'; -import { transformNodesToRaw } from './utils/unist.mjs'; -import { parseHeadingIntoMetadata } from './utils/parser.mjs'; - // Creates a Remark Parser with all Plugins needed for our API Docs const getRemarkParser = () => remark().use(remarkGfm); @@ -21,8 +18,6 @@ const getRemarkParser = () => remark().use(remarkGfm); * (this requires already parsed Node.js release data, the API doc file to be loaded, and etc) */ const createParser = () => { - const { getReferenceLink, parseYAML } = createQueries(); - const { newMetadataEntry, getNavigationEntries: getNavigation } = createMetadata(); @@ -37,14 +32,11 @@ const createParser = () => { return tree => { const definitions = selectAll('definition', tree); + const { updateLinkReference } = createQueries(undefined, definitions); + // Handles Link References visit(tree, createQueries.UNIST_TESTS.isLinkReference, node => { - const definition = definitions.find( - ({ identifier }) => identifier === node.identifier - ); - - node.type = 'link'; - node.url = definition.url; + updateLinkReference(node); }); // Removes the Definitions from the Tree @@ -62,48 +54,45 @@ const createParser = () => { const parsedSections = markdownSections.map(async sectionSource => { const apiEntryMetadata = newMetadataEntry(); + const { + addYAMLMetadata, + updateTypeToReferenceLink, + addHeadingMetadata, + updateMarkdownLink, + addStabilityIndexMeta, + } = createQueries(apiEntryMetadata); + const apiDocSectionParser = getRemarkParser().use(() => { - return (tree, section) => { + return tree => { // Handles YAML Metadata visit(tree, createQueries.UNIST_TESTS.isYamlNode, node => { - const metadata = parseYAML(node.value); - - apiEntryMetadata.setProperties(metadata); + addYAMLMetadata(node); remove(tree, node); }); // Handles Markdown Headings visit(tree, createQueries.UNIST_TESTS.isHeadingNode, node => { - const heading = transformNodesToRaw(node.children, section); + addHeadingMetadata(node); - const parsedHeading = parseHeadingIntoMetadata( - heading, - node.depth + 1 - ); + remove(tree, node); + }); - apiEntryMetadata.setHeading(parsedHeading); + // Handles Stability Indexes + visit(tree, createQueries.UNIST_TESTS.isStabilityIndex, node => { + addStabilityIndexMeta(node); remove(tree, node); }); // Handles API Type References transformation into Links visit(tree, createQueries.UNIST_TESTS.isTextWithType, node => { - const parsedReference = node.value.replace( - createQueries.QUERIES.normalizeTypes, - getReferenceLink - ); - - node.type = 'html'; - node.value = parsedReference; + updateTypeToReferenceLink(node); }); // Handles Normalisation of Markdown URLs visit(tree, createQueries.UNIST_TESTS.isMarkdownUrl, node => { - node.url = node.url.replace( - createQueries.QUERIES.markdownUrl, - (_, filename, hash = '') => `${filename}.html${hash}` - ); + updateMarkdownLink(node); }); }; }); diff --git a/src/queries.mjs b/src/queries.mjs index f76f0e2a..012de17f 100644 --- a/src/queries.mjs +++ b/src/queries.mjs @@ -1,37 +1,120 @@ 'use strict'; import * as parserUtils from './utils/parser.mjs'; +import * as unistUtils from './utils/unist.mjs'; /** * Creates an instance of the Query Manager, which allows to do multiple sort * of metadata and content metadata manipulation within an API Doc + * + * @param {ReturnType['newMetadataEntry']>} apiEntryMetadata The API Entry Metadata + * @param {Array} definitions The Definitions of the API Doc */ -const createQueries = () => { +const createQueries = (apiEntryMetadata = undefined, definitions = []) => { + /** + * Sanitizes the YAML source by returning the inner YAML content + * and then parsing it into an API Metadata object and updating the current Metadata + * + * @param {import('unist').Node} node The YAML Node + */ + const addYAMLMetadata = node => { + const sanitizedString = node.value.replace( + createQueries.QUERIES.yamlInnerContent, + (_, __, inner) => inner + ); + + const metadata = parserUtils.parseYAMLIntoMetadata(sanitizedString); + + apiEntryMetadata.updateProperties(metadata); + }; + /** * Transforms plain reference to Web/JavaScript/Node.js types * into Markdown links containing the proper reference to said types * - * @param {string} source The type source + * @param {import('unist').Node} node The Type Node */ - const getReferenceLink = source => - parserUtils.transformTypeToReferenceLink(source); + const updateTypeToReferenceLink = node => { + const parsedReference = node.value.replace( + createQueries.QUERIES.normalizeTypes, + parserUtils.transformTypeToReferenceLink + ); + + node.type = 'html'; + node.value = parsedReference; + }; /** - * Sanitizes the YAML source by returning the inner YAML content - * and then parsing it into an API Metadata object + * Parse a Heading Node into Metadata and updates the current Metadata * - * @param {string} yamlString The YAML Code Block + * @param {import('unist').Node} node The Heading Node */ - const parseYAML = yamlString => { - const sanitizedString = yamlString.replace( - createQueries.QUERIES.yamlInnerContent, - (_, __, inner) => inner + const addHeadingMetadata = node => { + const heading = unistUtils.transformNodesToString(node.children); + + const parsedHeading = parserUtils.parseHeadingIntoMetadata( + heading, + node.depth + 1 + ); + + apiEntryMetadata.setHeading(parsedHeading); + }; + + /** + * Updates a Markdown Link into a HTML Link for API Docs + * + * @param {import('unist').Node} node Thead Link Node + */ + const updateMarkdownLink = node => { + node.url = node.url.replace( + createQueries.QUERIES.markdownUrl, + (_, filename, hash = '') => `${filename}.html${hash}` + ); + }; + + /** + * Updates a Markdown Link Reference into an actual Link to the Definition + * + * @param {import('unist').Node} node Thead Link Reference Node + */ + const updateLinkReference = node => { + const definition = definitions.find( + ({ identifier }) => identifier === node.identifier + ); + + node.type = 'link'; + node.url = definition.url; + }; + + /** + * Parses a Stability Index Entry and updates the current Metadata + * + * @param {import('unist').Node} node Thead Link Reference Node + */ + const addStabilityIndexMeta = node => { + const stabilityIndexString = unistUtils.transformNodesToString( + node.children[0].children ); - return parserUtils.parseYAMLIntoMetadata(sanitizedString); + const stabilityIndex = + createQueries.QUERIES.stabilityIndex.exec(stabilityIndexString); + + apiEntryMetadata.updateProperties({ + stability_index: { + index: Number(stabilityIndex[1]), + description: stabilityIndex[2].replaceAll('\n', ' ').trim(), + }, + }); }; - return { getReferenceLink, parseYAML }; + return { + addYAMLMetadata, + updateTypeToReferenceLink, + addHeadingMetadata, + updateMarkdownLink, + updateLinkReference, + addStabilityIndexMeta, + }; }; // This defines the actual REGEX Queries @@ -41,9 +124,8 @@ createQueries.QUERIES = { // ReGeX to match the {Type} (Structure Type metadatas) // eslint-disable-next-line no-useless-escape normalizeTypes: /(\{|<)(?! )[a-z0-9.| \n\[\]\\]+(?! )(\}|>)/gim, - // ReGeX for replacing the Stability Index with a MDX Component - // @TODO: Only used for the MDX Generator - stabilityIndex: /^> (.*:)\s*(\d)([\s\S]*)/, + // ReGeX for handling Stability Indexes Metadata + stabilityIndex: /^Stability: ([0-5])(?:\s*-\s*)?(.*)$/s, // ReGeX for retrieving the inner content from a YAML block yamlInnerContent: /^/, }; @@ -59,6 +141,11 @@ createQueries.UNIST_TESTS = { type === 'heading' && depth >= 1 && depth <= 6, isLinkReference: ({ type, identifier }) => type === 'linkReference' && !!identifier, + isStabilityIndex: ({ type, children }) => + type === 'blockquote' && + createQueries.QUERIES.stabilityIndex.test( + unistUtils.transformNodesToString(children) + ), }; export default createQueries; diff --git a/src/types.d.ts b/src/types.d.ts index 4f6dcc8e..d5b4770f 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -1,5 +1,10 @@ import { VFile } from 'vfile'; +export interface StabilityIndexMetadataEntry { + index: number; + description: string; +} + export interface HeadingMetadataEntry { type: | 'event' @@ -36,6 +41,7 @@ export interface ApiDocRawMetadataEntry { source_link?: string; updates?: ApiDocMetadataUpdate[]; changes?: ApiDocMetadataChange[]; + stability_index?: StabilityIndexMetadataEntry; } export interface ApiDocNavigationEntry { @@ -51,6 +57,8 @@ export interface ApiDocNavigationEntry { changes: ApiDocMetadataChange[]; // The parsed Markdown content of a Navigation Entry heading: HeadingMetadataEntry; + // The API Doc Metadata Entry Stability Index if exists + stability: StabilityIndexMetadataEntry | undefined; } export interface ApiDocMetadataEntry extends VFile { diff --git a/src/utils/unist.mjs b/src/utils/unist.mjs index 1a1f333c..3aff0730 100644 --- a/src/utils/unist.mjs +++ b/src/utils/unist.mjs @@ -5,25 +5,24 @@ * and do extra parsing within them * * @param {import('unist').Node} nodes Nodes to parsed and joined - * @param {import('vfile').VFile} source The source VFile * @returns {string} The parsed and joined nodes as a string */ -export const transformNodesToRaw = (nodes, source) => { +export const transformNodesToString = nodes => { const mappedChildren = nodes.map(node => { if (node.type === 'inlineCode') { return `\`${node.value}\``; } if (node.type === 'strong') { - return `**${transformNodesToRaw(node.children, source)}**`; + return `**${transformNodesToString(node.children)}**`; } if (node.type === 'emphasis') { - return `_${transformNodesToRaw(node.children, source)}_`; + return `_${transformNodesToString(node.children)}_`; } if (node.children) { - return transformNodesToRaw(node.children, source); + return transformNodesToString(node.children); } return node.value; From 842e1c27e16068dc6564f76b455c265b07eb5948 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Sat, 6 Jul 2024 15:30:46 +0100 Subject: [PATCH 05/26] chore: codespell --- .github/workflows/codespell.yml | 2 ++ src/constants.mjs | 2 +- src/types.d.ts | 2 +- src/utils/parser.mjs | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 259e0051..010314af 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -17,3 +17,5 @@ jobs: - uses: codespell-project/actions-codespell@94259cd8be02ad2903ba34a22d9c13de21a74461 # v2.0 with: ignore_words_list: crate,raison + exclude_file: .gitignore + skip: package-lock.json diff --git a/src/constants.mjs b/src/constants.mjs index 779c320c..f5db1150 100644 --- a/src/constants.mjs +++ b/src/constants.mjs @@ -1,6 +1,6 @@ 'use strict'; -// This is the base-path of API docs within the https://nodejs.org Wesite +// This is the base-path of API docs within the https://nodejs.org Website export const DOC_WEB_BASE_PATH = '/docs/'; // This is the base URL of the MDN Web documentation diff --git a/src/types.d.ts b/src/types.d.ts index d5b4770f..c43b0e9c 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -45,7 +45,7 @@ export interface ApiDocRawMetadataEntry { } export interface ApiDocNavigationEntry { - // The name of the API Doc fle without the file extension (basename) + // The name of the API Doc file without the file extension (basename) api: string; // The unique slug of a Heading/Navigation Entry which is linkable through an anchor slug: string; diff --git a/src/utils/parser.mjs b/src/utils/parser.mjs index 2ff40dca..dbbd6c8e 100644 --- a/src/utils/parser.mjs +++ b/src/utils/parser.mjs @@ -66,7 +66,7 @@ export const transformTypeToReferenceLink = type => { // This is what we will compare against the API Types Mappings const lookupPiece = trimmedPiece.replace(/(?:\[])+$/, ''); - // This is what we return on the Array map uppon a hit, this function + // This is what we return on the Array map upon a hit, this function // is just for making this formatting reusable const formatToMarkdownLink = result => `[\`${trimmedPiece}\`](${result})`; From 2e287bc10eefdfa493faebcff70f53b72a2fb433 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Sat, 6 Jul 2024 15:57:53 +0100 Subject: [PATCH 06/26] chore: minor cleanup for heading types --- src/utils/parser.mjs | 42 ++++++++++++------------------------------ 1 file changed, 12 insertions(+), 30 deletions(-) diff --git a/src/utils/parser.mjs b/src/utils/parser.mjs index dbbd6c8e..17f4d1f6 100644 --- a/src/utils/parser.mjs +++ b/src/utils/parser.mjs @@ -189,38 +189,20 @@ export const parseHeadingIntoMetadata = (heading, depth) => { const maybeExtends = `(?: +extends +${maybeAncestors}${classId})?`; - const headingTypes = [ - { - type: 'event', - test: `${eventPrefix}${maybeBacktick}${maybeQuote}(${notQuotes})${maybeQuote}${maybeBacktick}$`, - }, - { - type: 'class', - test: `${classPrefix}${maybeBacktick}(${maybeAncestors}${classId})${maybeExtends}${maybeBacktick}$`, - }, - { - type: 'ctor', - test: `${ctorPrefix}(${maybeAncestors}${classId})${callWithParams}${maybeBacktick}$`, - }, - { - type: 'classMethod', - test: `${classMethodPrefix}${maybeBacktick}${maybeAncestors}(${id})${callWithParams}${maybeBacktick}$`, - }, - { - type: 'method', - test: `^${maybeBacktick}${maybeAncestors}(${id})${callWithParams}${maybeBacktick}$`, - }, - { - type: 'property', - test: `^${maybeClassPropertyPrefix}${maybeBacktick}${ancestors}(${id})${maybeBacktick}$`, - }, - ]; - - for (const headingType of headingTypes) { - const result = heading.match(new RegExp(headingType.test)); + const headingTypes = { + event: `${eventPrefix}${maybeBacktick}${maybeQuote}(${notQuotes})${maybeQuote}${maybeBacktick}$`, + class: `${classPrefix}${maybeBacktick}(${maybeAncestors}${classId})${maybeExtends}${maybeBacktick}$`, + ctor: `${ctorPrefix}(${maybeAncestors}${classId})${callWithParams}${maybeBacktick}$`, + classMethod: `${classMethodPrefix}${maybeBacktick}${maybeAncestors}(${id})${callWithParams}${maybeBacktick}$`, + method: `^${maybeBacktick}${maybeAncestors}(${id})${callWithParams}${maybeBacktick}$`, + property: `^${maybeClassPropertyPrefix}${maybeBacktick}${ancestors}(${id})${maybeBacktick}$`, + }; + + for (const headingType in headingTypes) { + const result = heading.match(new RegExp(headingTypes[headingType])); if (result && result.length >= 2) { - return { text: heading, type: headingType.type, name: result[1], depth }; + return { text: heading, type: headingType, name: result[1], depth }; } } From a41308ee79a94a8113f817971ae6438ba41ebde1 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Sat, 6 Jul 2024 18:59:28 +0100 Subject: [PATCH 07/26] chore: updated logic to support a glob --- package-lock.json | 343 +++++++++++++++++++++++++++++++++++++++++++--- package.json | 1 + src/loader.mjs | 27 ++-- src/parser.mjs | 26 +++- 4 files changed, 367 insertions(+), 30 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2291a4ee..436a181f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,6 +5,7 @@ "packages": { "": { "dependencies": { + "glob": "^10.4.3", "remark": "^15.0.1", "remark-gfm": "^4.0.0", "unist-util-remove": "^4.0.0", @@ -153,6 +154,102 @@ "url": "https://github.com/sponsors/nzakas" } }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "license": "MIT" + }, + "node_modules/@isaacs/cliui/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@isaacs/cliui/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -188,6 +285,16 @@ "node": ">= 8" } }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, "node_modules/@types/debug": { "version": "4.1.12", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", @@ -281,7 +388,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, "engines": { "node": ">=8" } @@ -290,7 +396,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "dependencies": { "color-convert": "^2.0.1" }, @@ -320,8 +425,7 @@ "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "node_modules/boolbase": { "version": "1.0.0", @@ -431,7 +535,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "dependencies": { "color-name": "~1.1.4" }, @@ -442,8 +545,7 @@ "node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "node_modules/colorette": { "version": "2.0.20", @@ -470,7 +572,6 @@ "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "dev": true, "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -553,6 +654,12 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "license": "MIT" + }, "node_modules/emoji-regex": { "version": "10.3.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.3.0.tgz", @@ -842,6 +949,22 @@ "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", "dev": true }, + "node_modules/foreground-child": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.2.1.tgz", + "integrity": "sha512-PXUUyLqrR2XCWICfv6ukppP96sdFwWbNEnfEMt7jNsISjMsvaLNinAHNDYyvkyU+SZG2BTSbT5NjG+vZslfGTA==", + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/get-east-asian-width": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.2.0.tgz", @@ -866,6 +989,29 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/glob": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.3.tgz", + "integrity": "sha512-Q38SGlYRpVtDBPSWEylRyctn7uDeTp4NQERTLiCT1FqA9JXPYWqAVmQU6qh4r/zMM5ehxTcbaO8EjhWnvEhmyg==", + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -878,6 +1024,30 @@ "node": ">=10.13.0" } }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/globals": { "version": "15.8.0", "resolved": "https://registry.npmjs.org/globals/-/globals-15.8.0.tgz", @@ -1035,8 +1205,25 @@ "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + }, + "node_modules/jackspeak": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.1.tgz", + "integrity": "sha512-U23pQPDnmYybVkYjObcuYMk43VRlMLLqLI+RdZy8s8WV8WsxO9SnqSroKaluuvcNOdCAlauKszDwd+umbot5Mg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } }, "node_modules/js-yaml": { "version": "4.1.0", @@ -1278,6 +1465,15 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/lru-cache": { + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.3.1.tgz", + "integrity": "sha512-9/8QXrtbGeMB6LxwQd4x1tIMnsmUxMvIH/qWGsccz6bt9Uln3S+sgAaqfQNhbGA8ufzs2fHuP/yqapGgP9Hh2g==", + "license": "ISC", + "engines": { + "node": ">=18" + } + }, "node_modules/markdown-table": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.3.tgz", @@ -2094,6 +2290,15 @@ "node": "*" } }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "license": "ISC", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -2206,6 +2411,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.0.tgz", + "integrity": "sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==", + "license": "BlueOak-1.0.0" + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -2231,11 +2442,26 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true, "engines": { "node": ">=8" } }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", @@ -2476,7 +2702,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, "dependencies": { "shebang-regex": "^3.0.0" }, @@ -2488,7 +2713,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true, "engines": { "node": ">=8" } @@ -2497,7 +2721,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "dev": true, "engines": { "node": ">=14" }, @@ -2559,6 +2782,36 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/string-width/node_modules/ansi-regex": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", @@ -2590,7 +2843,19 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" }, @@ -2829,7 +3094,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, "dependencies": { "isexe": "^2.0.0" }, @@ -2866,6 +3130,53 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/wrap-ansi/node_modules/ansi-regex": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", diff --git a/package.json b/package.json index a1faa289..f9e4bbf3 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "prettier": "3.2.5" }, "dependencies": { + "glob": "^10.4.3", "remark": "^15.0.1", "remark-gfm": "^4.0.0", "unist-util-remove": "^4.0.0", diff --git a/src/loader.mjs b/src/loader.mjs index 61c5627e..f73aa78d 100644 --- a/src/loader.mjs +++ b/src/loader.mjs @@ -1,26 +1,33 @@ 'use strict'; -import { readFile } from 'node:fs/promises'; +import { glob } from 'glob'; +import { readFileSync } from 'node:fs'; import { extname } from 'node:path'; import { VFile } from 'vfile'; const createLoader = () => { /** - * Loads an API Doc file and transforms it into a VFile + * Loads API Doc files and transforms it into VFiles * - * @param {string} path The API Doc Path + * @param {string} path A glob/path for API docs to be loaded */ - const loadFile = async path => { - if (extname(path) !== '.md') { - throw new Error('API Doc file must be a Markdown file'); - } + const loadFiles = async path => { + const resolvedFiles = await glob(path); - const apiDocFile = await readFile(path, 'utf-8'); + return resolvedFiles.map(filePath => { + const fileExtension = extname(filePath); - return new VFile({ path: path, value: apiDocFile }); + if (fileExtension === '.md') { + const fileContent = readFileSync(filePath, 'utf-8'); + + return new VFile({ path: filePath, value: fileContent }); + } + + throw new Error(`File ${filePath} is not a Markdown file`); + }); }; - return { loadFile }; + return { loadFiles }; }; export default createLoader; diff --git a/src/parser.mjs b/src/parser.mjs index 5be44c2a..eddfe58f 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -26,7 +26,7 @@ const createParser = () => { * * @param {import('vfile').VFile} apiDoc */ - const parseMetadata = async apiDoc => { + const parseApiDoc = async apiDoc => { // Does extra modification at the root level of the API Doc File const apiDocRootParser = getRemarkParser().use(() => { return tree => { @@ -51,7 +51,7 @@ const createParser = () => { const markdownSections = parsedRoot.toString().split('\n\n#'); // Parses each Markdown Section into a Metadata Entry - const parsedSections = markdownSections.map(async sectionSource => { + const parsedSections = markdownSections.map(sectionSource => { const apiEntryMetadata = newMetadataEntry(); const { @@ -98,7 +98,7 @@ const createParser = () => { }); // Process the Markdown Section into a VFile with the processed data - const parsedSection = await apiDocSectionParser.process(sectionSource); + const parsedSection = apiDocSectionParser.processSync(sectionSource); // Creates a Metadata Entry (VFile) and returns it return apiEntryMetadata.create(apiDoc.stem, parsedSection); @@ -107,7 +107,25 @@ const createParser = () => { return parsedSections; }; - return { getNavigation, parseMetadata }; + /** + * This method allows to parse multiple API Doc Files at once + * and it simply wraps parseApiDoc with the given API Docs + * + * @param {Array} apiDocs + */ + const parseApiDocs = async apiDocs => { + const apiMetadataEntries = []; + + for (const apiDoc of apiDocs) { + const parsedEntries = await parseApiDoc(apiDoc); + + apiMetadataEntries.push(...parsedEntries); + } + + return apiMetadataEntries; + }; + + return { getNavigation, parseApiDocs }; }; export default createParser; From b459661c1da7971ea2130039c6571e367e2e43b0 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Sun, 7 Jul 2024 02:37:50 +0100 Subject: [PATCH 08/26] chore.: self review and code cleanup --- package-lock.json | 7 +++ package.json | 1 + src/constants.mjs | 46 +++++++++++++- src/loader.mjs | 9 +++ src/metadata.mjs | 25 +++++--- src/parser.mjs | 9 ++- src/utils/parser.mjs | 137 +++++++++++------------------------------- src/utils/slugger.mjs | 39 ++++++++++++ src/utils/unist.mjs | 2 +- 9 files changed, 159 insertions(+), 116 deletions(-) create mode 100644 src/utils/slugger.mjs diff --git a/package-lock.json b/package-lock.json index 436a181f..9d33e262 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,6 +5,7 @@ "packages": { "": { "dependencies": { + "github-slugger": "^2.0.0", "glob": "^10.4.3", "remark": "^15.0.1", "remark-gfm": "^4.0.0", @@ -989,6 +990,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/github-slugger": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/github-slugger/-/github-slugger-2.0.0.tgz", + "integrity": "sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw==", + "license": "ISC" + }, "node_modules/glob": { "version": "10.4.3", "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.3.tgz", diff --git a/package.json b/package.json index f9e4bbf3..6d6099c0 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "prettier": "3.2.5" }, "dependencies": { + "github-slugger": "^2.0.0", "glob": "^10.4.3", "remark": "^15.0.1", "remark-gfm": "^4.0.0", diff --git a/src/constants.mjs b/src/constants.mjs index f5db1150..0795c3da 100644 --- a/src/constants.mjs +++ b/src/constants.mjs @@ -37,12 +37,44 @@ export const DOC_API_YAML_KEYS_UPDATE = [ // These are the YAML Types that should be considered as Navigation Entries export const DOC_API_YAML_KEYS_NAVIGATION = ['class', 'module', 'global']; +// These are string replacements specific to Node.js API Docs for Anchor IDs export const DOC_API_SLUGS_REPLACEMENTS = [ - { to: 'nodejs', from: /node.js/ }, // Replace node.js - { to: '-and-', from: /&/ }, // Replace & - { to: '-', from: /[/_,:;\\ ]/g }, // Replace /_,:;\. and whitespace + { from: /node.js/i, to: 'nodejs' }, // Replace Node.js + { from: /&/, to: '-and-' }, // Replace & + { from: /[/_,:;\\ ]/g, to: '-' }, // Replace /_,:;\. and whitespace + { from: /--+/g, to: '-' }, // Replace multiple hyphens with single + { from: /^-/, to: '' }, // Remove any leading hyphen + { from: /-$/, to: '' }, // Remove any trailing hyphen ]; +// These are regular expressions used to determine if a given Markdown heading +// is a specific type of API Doc entry (e.g., Event, Class, Method, etc) +// and to extract the inner content of said Heading to be used as the API Doc Entry Name +export const DOC_API_HEADING_TYPES = [ + { type: 'method', regex: /^`?([A-Z]\w+(?:\.[A-Z]\w+)*\.\w+)\([^)]*\)`?$/i }, + { type: 'event', regex: /^Event: +`?['"]?([^'"]+)['"]?`?$/i }, + { + type: 'class', + regex: + /^Class: +`?([A-Z]\w+(?:\.[A-Z]\w+)*(?: +extends +[A-Z]\w+(?:\.[A-Z]\w+)*)?)`?$/i, + }, + { + type: 'ctor', + regex: /^(?:Constructor: +)?`?new +([A-Z]\w+(?:\.[A-Z]\w+)*)\([^)]*\)`?$/i, + }, + { + type: 'classMethod', + regex: /^Static method: +`?([A-Z]\w+(?:\.[A-Z]\w+)*\.\w+)\([^)]*\)`?$/i, + }, + { + type: 'property', + regex: /^(?:Class property: +)?`?([A-Z]\w+(?:\.[A-Z]\w+)*\.\w+)`?$/i, + }, +]; + +// This is a mapping for types within the Markdown content and their respective +// JavaScript primitive types within the MDN JavaScript Docs +// @see DOC_MDN_BASE_URL_JS_PRIMITIVES export const DOC_TYPES_MAPPING_PRIMITIVES = { boolean: 'Boolean', integer: 'Number', // Not a primitive, used for clarification. @@ -53,6 +85,9 @@ export const DOC_TYPES_MAPPING_PRIMITIVES = { undefined: 'Undefined', }; +// This is a mapping for types within the Markdown content and their respective +// JavaScript globals types within the MDN JavaScript Docs +// @see DOC_MDN_BASE_URL_JS_GLOBALS export const DOC_TYPES_MAPPING_GLOBALS = { AggregateError: 'AggregateError', Array: 'Array', @@ -79,6 +114,9 @@ export const DOC_TYPES_MAPPING_GLOBALS = { 'WebAssembly.Instance': 'WebAssembly/Instance', }; +// This is a mapping for types within the Markdown content and their respective +// Node.js types within the Node.js API Docs (refers to a different API Doc Page) +// @note These hashes are generated with the GitHub Slugger export const DOC_TYPES_MAPPING_NODE_MODULES = { AbortController: 'globals.html#abortcontroller', AbortSignal: 'globals.html#abortsignal', @@ -266,6 +304,8 @@ export const DOC_TYPES_MAPPING_NODE_MODULES = { TextDecoderStream: 'webstreams.html#textdecoderstream', }; +// This is a mapping for miscellaneous types within the Markdown content and their respective +// external reference on appropriate 3rd-party vendors/documentation sites. export const DOC_TYPES_MAPPING_OTHER = { any: `${DOC_MDN_BASE_URL_JS_PRIMITIVES}#Data_types`, this: `${DOC_MDN_BASE_URL_JS}Reference/Operators/this`, diff --git a/src/loader.mjs b/src/loader.mjs index f73aa78d..3fb891d8 100644 --- a/src/loader.mjs +++ b/src/loader.mjs @@ -5,11 +5,20 @@ import { readFileSync } from 'node:fs'; import { extname } from 'node:path'; import { VFile } from 'vfile'; +/** + * This method creates a simple abstract "Loader", which technically + * could be used for different things, but here we want to use it to load + * Markdown files and transform them into VFiles + */ const createLoader = () => { /** * Loads API Doc files and transforms it into VFiles * * @param {string} path A glob/path for API docs to be loaded + * The input string can be a simple path (relative or absolute) + * The input string can also be any allowed glob string + * + * @see https://code.visualstudio.com/docs/editor/glob-patterns */ const loadFiles = async path => { const resolvedFiles = await glob(path); diff --git a/src/metadata.mjs b/src/metadata.mjs index da5825b8..902f40e2 100644 --- a/src/metadata.mjs +++ b/src/metadata.mjs @@ -1,16 +1,17 @@ 'use strict'; import { DOC_API_YAML_KEYS_NAVIGATION } from './constants.mjs'; -import { stringToSlug } from './utils/parser.mjs'; /** - * This method allows us to handle creation of Navigation Entries - * for a given file within the API documentation + * This method allows us to handle creation of Metadata Entries + * çwithin the current scope of API Docs being parsed * * This can be used disconnected with a specific file, and can be aggregated * to many files to create a full Navigation for a given version of the API + * + * @param {InstanceType} slugger A GitHub Slugger */ -const createMetadata = () => { +const createMetadata = slugger => { const navigationMetadataEntries = []; return { @@ -21,6 +22,8 @@ const createMetadata = () => { */ getNavigationEntries: () => navigationMetadataEntries, newMetadataEntry: () => { + // This holds a temporary buffer of raw metadata before being + // transformed into NavigationEntries and Metadata Entries const internalMetadata = { heading: { text: undefined, @@ -33,7 +36,7 @@ const createMetadata = () => { return { /** - * Set the Heading Line of a given Metadata + * Set the Heading of a given Metadata * * @param {import('./types.d.ts').HeadingMetadataEntry} heading The new Heading Metadata */ @@ -42,6 +45,8 @@ const createMetadata = () => { }, /** * Set the Metadata (from YAML if exists) properties to the current Metadata Entry + * itI also allows for extra data (such as Stability Index) and miscellaneous data to be set + * although it'd be best to only set ones from {ApiDocRawMetadataEntry} * * @param {Partial} properties Extra Metadata Properties to be defined */ @@ -52,8 +57,12 @@ const createMetadata = () => { }; }, /** - * Generates Navigation Entries for the current Navigation Creator - * and pushes them to the Navigation Entries for the current API file + * Generates a new Navigation Entry and pushes them to the internal collection + * of Navigation Entries, and returns a MetadataEntry which is then used by the Parser + * and forwarded to any relevant generator. + * + * The Navigation Entries has a dedicated separate method as it can be manipulated + * outside of the scope of the generation of the content * * @param {string} apiDoc The name of the API Doc * @param {import('vfile').VFile} section The content of the current Metadata Entry @@ -64,7 +73,7 @@ const createMetadata = () => { // a certain navigation section to a page ad the exact point of the page (scroll) // This is useful for classes, globals and other type of YAML entries, as they reside // within a module (page) and we want to link to them directly - const slugHash = `#${stringToSlug(internalMetadata.heading.text)}`; + const slugHash = `#${slugger.slug(internalMetadata.heading.text)}`; const { type: yaml_type, diff --git a/src/parser.mjs b/src/parser.mjs index eddfe58f..d287b836 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -10,6 +10,8 @@ import { selectAll } from 'unist-util-select'; import createMetadata from './metadata.mjs'; import createQueries from './queries.mjs'; +import { createNodeSlugger } from './utils/slugger.mjs'; + // Creates a Remark Parser with all Plugins needed for our API Docs const getRemarkParser = () => remark().use(remarkGfm); @@ -18,8 +20,10 @@ const getRemarkParser = () => remark().use(remarkGfm); * (this requires already parsed Node.js release data, the API doc file to be loaded, and etc) */ const createParser = () => { + const slugger = createNodeSlugger(); + const { newMetadataEntry, getNavigationEntries: getNavigation } = - createMetadata(); + createMetadata(slugger); /** * Parses a given API Doc Metadata file into a list of Metadata Entries @@ -27,6 +31,9 @@ const createParser = () => { * @param {import('vfile').VFile} apiDoc */ const parseApiDoc = async apiDoc => { + // Resets the Slugger as we are parsing a new API Doc file + slugger.reset(); + // Does extra modification at the root level of the API Doc File const apiDocRootParser = getRemarkParser().use(() => { return tree => { diff --git a/src/utils/parser.mjs b/src/utils/parser.mjs index 17f4d1f6..a4f2166f 100644 --- a/src/utils/parser.mjs +++ b/src/utils/parser.mjs @@ -3,7 +3,7 @@ import yaml from 'yaml'; import { - DOC_API_SLUGS_REPLACEMENTS, + DOC_API_HEADING_TYPES, DOC_API_YAML_KEYS_ARRAYS, DOC_API_YAML_KEYS_UPDATE, DOC_MDN_BASE_URL_JS_GLOBALS, @@ -14,41 +14,6 @@ import { DOC_TYPES_MAPPING_PRIMITIVES, } from '../constants.mjs'; -/** - * All YAML values are treated as an array for consistency - * since they get rendered within a table of changelog - * - * @param {string|Array} value YAML metadata entry value - * @returns {Array} YAML metadata entry value as an Array - */ -export const yamlValueToArray = value => - Array.isArray(value) ? value : [value]; - -/** - * Transforms a page title into an unique slug (that can be used as an ID, anchor, or URL) - * that follows slug conventions defined by the Node.js project - * - * @param {string} title The title to be parsed into a slug - * @returns {string} A title parsed into a page slug - */ -export const stringToSlug = title => { - let slug = title.toLowerCase().trim(); - - DOC_API_SLUGS_REPLACEMENTS.forEach(set => { - slug = slug.replace(set.from, set.to); - }); - - return ( - slug - // This escape is actually needed here - // eslint-disable-next-line no-useless-escape - .replace(/[^\w\-]+/g, '') // Remove any non word characters - .replace(/--+/g, '-') // Replace multiple hyphens with single - .replace(/^-/, '') // Remove any leading hyphen - .replace(/-$/, '') // Remove any trailing hyphen - ); -}; - /** * This method replaces plain text Types within the Markdown content into Markdown links * that link to the actual relevant reference for such type (either internal or external link) @@ -59,50 +24,48 @@ export const stringToSlug = title => { export const transformTypeToReferenceLink = type => { const typeInput = type.replace('{', '').replace('}', ''); - const typePieces = typeInput.split('|').map(piece => { - // This is the content to render as the text of the Markdown Link - const trimmedPiece = piece.trim(); - - // This is what we will compare against the API Types Mappings - const lookupPiece = trimmedPiece.replace(/(?:\[])+$/, ''); - - // This is what we return on the Array map upon a hit, this function - // is just for making this formatting reusable - const formatToMarkdownLink = result => `[\`${trimmedPiece}\`](${result})`; - + /** + * Handles the Mapping (if there's a match) of the input text + * into the reference type from the API Docs + * + * @param {string} lookupPiece + */ + const transformType = lookupPiece => { // Transform JS Primitive Type references into Markdown Links (MDN) if (lookupPiece.toLowerCase() in DOC_TYPES_MAPPING_PRIMITIVES) { const typeValue = DOC_TYPES_MAPPING_PRIMITIVES[lookupPiece.toLowerCase()]; - const typeLink = `${DOC_MDN_BASE_URL_JS_PRIMITIVES}#${typeValue}_type`; - - return formatToMarkdownLink(typeLink); + return `${DOC_MDN_BASE_URL_JS_PRIMITIVES}#${typeValue}_type`; } // Transforms JS Global Type references into Markdown Links (MDN) if (lookupPiece in DOC_TYPES_MAPPING_GLOBALS) { - const typeLink = `${DOC_MDN_BASE_URL_JS_GLOBALS}${lookupPiece}`; - - return formatToMarkdownLink(typeLink); + return `${DOC_MDN_BASE_URL_JS_GLOBALS}${lookupPiece}`; } // Transform other external Web/JavaScript Type references into Markdown Links // to diverse different external websites. These already are formatted as links if (lookupPiece in DOC_TYPES_MAPPING_OTHER) { - const typeValueAsLink = DOC_TYPES_MAPPING_NODE_MODULES[lookupPiece]; - - return formatToMarkdownLink(typeValueAsLink); + return DOC_TYPES_MAPPING_NODE_MODULES[lookupPiece]; } // Transform Node.js Type/Module references into Markdown Links // that refer to other API Docs pages within the Node.js API docs if (lookupPiece in DOC_TYPES_MAPPING_NODE_MODULES) { - const typeValueAsLink = DOC_TYPES_MAPPING_NODE_MODULES[lookupPiece]; - - return formatToMarkdownLink(typeValueAsLink); + return DOC_TYPES_MAPPING_NODE_MODULES[lookupPiece]; } return undefined; + }; + + const typePieces = typeInput.split('|').map(piece => { + // This is the content to render as the text of the Markdown Link + const trimmedPiece = piece.trim(); + + // This is what we will compare against the API Types Mappings + const result = transformType(trimmedPiece.replace(/(?:\[])+$/, '')); + + return result ? `[\`${trimmedPiece}\`](${result})` : undefined; }); // Filter out pieces that we failed to Map and then join the valid ones @@ -140,11 +103,12 @@ export const parseYAMLIntoMetadata = yamlString => { Object.keys(parsedYaml).forEach(key => { // Some entries should always be Array for the sake of consistency if (DOC_API_YAML_KEYS_ARRAYS.includes(key)) { - parsedYaml[key] = yamlValueToArray(parsedYaml[key]); + parsedYaml[key] = [parsedYaml[key]].flat(); } + // We transform some entries in a standardized "updates" field + // and then remove them from the parsedYaml if (DOC_API_YAML_KEYS_UPDATE.includes(key)) { - // We transform some entries in a standardized "updates" field parsedYaml.updates = [{ type: key, version: parsedYaml[key] }]; delete parsedYaml[key]; @@ -155,54 +119,21 @@ export const parseYAMLIntoMetadata = yamlString => { }; /** - * Parses a raw Heading String into Heading Metadata + * Parses a raw Heading string into Heading Metadata * * @param {string} heading The raw Heading Text * @param {number} depth The depth of the heading * @returns {import('../types.d.ts').HeadingMetadataEntry} Parsed Heading Entry */ export const parseHeadingIntoMetadata = (heading, depth) => { - const r = String.raw; - - const eventPrefix = '^Event: +'; - const classPrefix = '^Class: +'; - const ctorPrefix = '^(?:Constructor: +)?`?new +'; - const classMethodPrefix = '^Static method: +'; - const maybeClassPropertyPrefix = '(?:Class property: +)?'; - - const maybeQuote = '[\'"]?'; - const notQuotes = '[^\'"]+'; - - const maybeBacktick = '`?'; - - // To include constructs like `readable\[Symbol.asyncIterator\]()` - // or `readable.\_read(size)` (with Markdown escapes). - const simpleId = r`(?:(?:\\?_)+|\b)\w+\b`; - const computedId = r`\\?\[[\w\.]+\\?\]`; - const id = `(?:${simpleId}|${computedId})`; - const classId = r`[A-Z]\w+`; - - const ancestors = r`(?:${id}\.?)+`; - const maybeAncestors = r`(?:${id}\.?)*`; - - const callWithParams = r`\([^)]*\)`; - - const maybeExtends = `(?: +extends +${maybeAncestors}${classId})?`; - - const headingTypes = { - event: `${eventPrefix}${maybeBacktick}${maybeQuote}(${notQuotes})${maybeQuote}${maybeBacktick}$`, - class: `${classPrefix}${maybeBacktick}(${maybeAncestors}${classId})${maybeExtends}${maybeBacktick}$`, - ctor: `${ctorPrefix}(${maybeAncestors}${classId})${callWithParams}${maybeBacktick}$`, - classMethod: `${classMethodPrefix}${maybeBacktick}${maybeAncestors}(${id})${callWithParams}${maybeBacktick}$`, - method: `^${maybeBacktick}${maybeAncestors}(${id})${callWithParams}${maybeBacktick}$`, - property: `^${maybeClassPropertyPrefix}${maybeBacktick}${ancestors}(${id})${maybeBacktick}$`, - }; - - for (const headingType in headingTypes) { - const result = heading.match(new RegExp(headingTypes[headingType])); - - if (result && result.length >= 2) { - return { text: heading, type: headingType, name: result[1], depth }; + for (const { type, regex } of DOC_API_HEADING_TYPES) { + // Attempts to get a match from one of the Heading Tyoes, if a match is found + // we use that type as the heading type, and extract the regex expression match group + // which should be the inner "plain" heading content (or the title of the heading for navigation) + const [, innerHeading] = heading.match(regex) ?? []; + + if (innerHeading && innerHeading.length) { + return { text: heading, type, name: innerHeading, depth }; } } diff --git a/src/utils/slugger.mjs b/src/utils/slugger.mjs new file mode 100644 index 00000000..fc0eb933 --- /dev/null +++ b/src/utils/slugger.mjs @@ -0,0 +1,39 @@ +'use strict'; + +import GitHubSlugger from 'github-slugger'; + +import { DOC_API_SLUGS_REPLACEMENTS } from '../constants.mjs'; + +/** + * Creates a modified version of the GitHub Slugger + * + * @returns {InstanceType} The modified GitHub Slugger + */ +export const createNodeSlugger = () => { + const slugger = new GitHubSlugger(); + + return { + /** + * Creates a new slug based on the provided string + * + * @param {string} title The title to be parsed into a slug + */ + slug: title => { + // Applies certain string replacements that are specific + // to the way how Node.js generates slugs/anchor IDs + return DOC_API_SLUGS_REPLACEMENTS.reduce( + (piece, { from, to }) => piece.replace(from, to), + // Slugify the title using GitHub Slugger + slugger.slug(title) + ); + }, + + /** + * Resets the cache of the Slugger, preventing repeated slugs + * to be marked as repeated. + * + * @returns {void} + */ + reset: () => slugger.reset(), + }; +}; diff --git a/src/utils/unist.mjs b/src/utils/unist.mjs index 3aff0730..ac41c4b7 100644 --- a/src/utils/unist.mjs +++ b/src/utils/unist.mjs @@ -2,7 +2,7 @@ /** * This utility allows us to join children Nodes into one - * and do extra parsing within them + * and transfor them back to what their source would look like * * @param {import('unist').Node} nodes Nodes to parsed and joined * @returns {string} The parsed and joined nodes as a string From 863c49817ab1497c4f3fac46674c34c7cf4b4840 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Sun, 7 Jul 2024 02:43:22 +0100 Subject: [PATCH 09/26] chore: minor import cleanup --- src/queries.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queries.mjs b/src/queries.mjs index 012de17f..f0db2742 100644 --- a/src/queries.mjs +++ b/src/queries.mjs @@ -7,7 +7,7 @@ import * as unistUtils from './utils/unist.mjs'; * Creates an instance of the Query Manager, which allows to do multiple sort * of metadata and content metadata manipulation within an API Doc * - * @param {ReturnType['newMetadataEntry']>} apiEntryMetadata The API Entry Metadata + * @param {ReturnType['newMetadataEntry']>} apiEntryMetadata The API Entry Metadata * @param {Array} definitions The Definitions of the API Doc */ const createQueries = (apiEntryMetadata = undefined, definitions = []) => { From 1c2512ba74b4ae3f6dfb448308a3d41f45655bc0 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Sun, 7 Jul 2024 13:36:11 +0100 Subject: [PATCH 10/26] fix: heading issue + improved docs and code --- src/constants.mjs | 23 +++++++++++--------- src/metadata.mjs | 50 +++++++++++++++++++++---------------------- src/parser.mjs | 51 ++++++++++++++++++++++++++------------------ src/queries.mjs | 2 +- src/utils/parser.mjs | 37 ++++++++++++++++---------------- 5 files changed, 87 insertions(+), 76 deletions(-) diff --git a/src/constants.mjs b/src/constants.mjs index 0795c3da..9028517f 100644 --- a/src/constants.mjs +++ b/src/constants.mjs @@ -9,13 +9,16 @@ export const DOC_MDN_BASE_URL = 'https://developer.mozilla.org/en-US/docs/Web/'; // This is the base URL for the MDN JavaScript documentation export const DOC_MDN_BASE_URL_JS = `${DOC_MDN_BASE_URL}JavaScript/`; -// This is the base URL for the MDN JavaScript Primitives documentation +// This is the base URL for the MDN JavaScript primitives documentation export const DOC_MDN_BASE_URL_JS_PRIMITIVES = `${DOC_MDN_BASE_URL_JS}Data_structures`; -// This is the base URL for the MDN JavaScript Global Objects documentation +// This is the base URL for the MDN JavaScript global objects documentation export const DOC_MDN_BASE_URL_JS_GLOBALS = `${DOC_MDN_BASE_URL_JS}Reference/Global_Objects/`; -// These are YAML keys from the Markdown YAML Metadata that should always be Arrays +// Characters used to split each section within an API Doc file +export const DOC_API_SECTION_SEPARATOR = '\n\n#'; + +// These are YAML keys from the Markdown YAML Metadata that should always be arrays export const DOC_API_YAML_KEYS_ARRAYS = [ 'added', 'napiVersion', @@ -24,7 +27,7 @@ export const DOC_API_YAML_KEYS_ARRAYS = [ 'introduced_in', ]; -// These are YAML keys from the Markdown YAML Metadata that should be +// These are YAML keys from the Markdown YAML metadata that should be // removed and appended to the `update` key export const DOC_API_YAML_KEYS_UPDATE = [ 'added', @@ -34,10 +37,10 @@ export const DOC_API_YAML_KEYS_UPDATE = [ 'napiVersion', ]; -// These are the YAML Types that should be considered as Navigation Entries +// These are the YAML types that should be considered as navigation entries export const DOC_API_YAML_KEYS_NAVIGATION = ['class', 'module', 'global']; -// These are string replacements specific to Node.js API Docs for Anchor IDs +// These are string replacements specific to Node.js API docs for anchor IDs export const DOC_API_SLUGS_REPLACEMENTS = [ { from: /node.js/i, to: 'nodejs' }, // Replace Node.js { from: /&/, to: '-and-' }, // Replace & @@ -49,7 +52,7 @@ export const DOC_API_SLUGS_REPLACEMENTS = [ // These are regular expressions used to determine if a given Markdown heading // is a specific type of API Doc entry (e.g., Event, Class, Method, etc) -// and to extract the inner content of said Heading to be used as the API Doc Entry Name +// and to extract the inner content of said Heading to be used as the API doc entry name export const DOC_API_HEADING_TYPES = [ { type: 'method', regex: /^`?([A-Z]\w+(?:\.[A-Z]\w+)*\.\w+)\([^)]*\)`?$/i }, { type: 'event', regex: /^Event: +`?['"]?([^'"]+)['"]?`?$/i }, @@ -73,7 +76,7 @@ export const DOC_API_HEADING_TYPES = [ ]; // This is a mapping for types within the Markdown content and their respective -// JavaScript primitive types within the MDN JavaScript Docs +// JavaScript primitive types within the MDN JavaScript docs // @see DOC_MDN_BASE_URL_JS_PRIMITIVES export const DOC_TYPES_MAPPING_PRIMITIVES = { boolean: 'Boolean', @@ -86,7 +89,7 @@ export const DOC_TYPES_MAPPING_PRIMITIVES = { }; // This is a mapping for types within the Markdown content and their respective -// JavaScript globals types within the MDN JavaScript Docs +// JavaScript globals types within the MDN JavaScript docs // @see DOC_MDN_BASE_URL_JS_GLOBALS export const DOC_TYPES_MAPPING_GLOBALS = { AggregateError: 'AggregateError', @@ -115,7 +118,7 @@ export const DOC_TYPES_MAPPING_GLOBALS = { }; // This is a mapping for types within the Markdown content and their respective -// Node.js types within the Node.js API Docs (refers to a different API Doc Page) +// Node.js types within the Node.js API docs (refers to a different API doc page) // @note These hashes are generated with the GitHub Slugger export const DOC_TYPES_MAPPING_NODE_MODULES = { AbortController: 'globals.html#abortcontroller', diff --git a/src/metadata.mjs b/src/metadata.mjs index 902f40e2..c6b88efb 100644 --- a/src/metadata.mjs +++ b/src/metadata.mjs @@ -3,8 +3,8 @@ import { DOC_API_YAML_KEYS_NAVIGATION } from './constants.mjs'; /** - * This method allows us to handle creation of Metadata Entries - * çwithin the current scope of API Docs being parsed + * This method allows us to handle creation of Metadata entries + * within the current scope of API docs being parsed * * This can be used disconnected with a specific file, and can be aggregated * to many files to create a full Navigation for a given version of the API @@ -16,14 +16,14 @@ const createMetadata = slugger => { return { /** - * Retrieves all Navigation Entries generated in a said file + * Retrieves all Navigation entries generated in a said file * * @returns {Array} */ getNavigationEntries: () => navigationMetadataEntries, newMetadataEntry: () => { // This holds a temporary buffer of raw metadata before being - // transformed into NavigationEntries and Metadata Entries + // transformed into NavigationEntries and MetadataEntries const internalMetadata = { heading: { text: undefined, @@ -38,17 +38,17 @@ const createMetadata = slugger => { /** * Set the Heading of a given Metadata * - * @param {import('./types.d.ts').HeadingMetadataEntry} heading The new Heading Metadata + * @param {import('./types.d.ts').HeadingMetadataEntry} heading The new heading metadata */ setHeading: heading => { internalMetadata.heading = heading; }, /** - * Set the Metadata (from YAML if exists) properties to the current Metadata Entry + * Set the Metadata (from YAML if exists) properties to the current Metadata entry * itI also allows for extra data (such as Stability Index) and miscellaneous data to be set * although it'd be best to only set ones from {ApiDocRawMetadataEntry} * - * @param {Partial} properties Extra Metadata Properties to be defined + * @param {Partial} properties Extra Metadata properties to be defined */ updateProperties: properties => { internalMetadata.properties = { @@ -57,19 +57,19 @@ const createMetadata = slugger => { }; }, /** - * Generates a new Navigation Entry and pushes them to the internal collection - * of Navigation Entries, and returns a MetadataEntry which is then used by the Parser + * Generates a new Navigation entry and pushes them to the internal collection + * of Navigation entries, and returns a MetadataEntry which is then used by the parser * and forwarded to any relevant generator. * - * The Navigation Entries has a dedicated separate method as it can be manipulated - * outside of the scope of the generation of the content + * The Navigation entries has a dedicated separate method for retrieval + * as it can be manipulated outside of the scope of the generation of the content * - * @param {string} apiDoc The name of the API Doc - * @param {import('vfile').VFile} section The content of the current Metadata Entry - * @returns {import('./types').ApiDocMetadataEntry} The locally created Metadata Entries + * @param {string} apiDoc The name of the API doc + * @param {import('vfile').VFile} section The content of the current Metadata entry + * @returns {import('./types').ApiDocMetadataEntry} The locally created Metadata entries */ create: (apiDoc, section) => { - // This is the ID of a certain Navigation Entry, which allows us to anchor + // This is the ID of a certain Navigation entry, which allows us to anchor // a certain navigation section to a page ad the exact point of the page (scroll) // This is useful for classes, globals and other type of YAML entries, as they reside // within a module (page) and we want to link to them directly @@ -89,33 +89,33 @@ const createMetadata = slugger => { yaml_type || internalMetadata.heading.type; const navigationEntry = { - // The API file Basename (without the Extension) + // The API file basename (without the extension) api: yaml_name || apiDoc, - // The path/slug of the API Section + // The path/slug of the API section slug: `${apiDoc}.html${slugHash}`, - // The Source Link of said API Section + // The source link of said API section sourceLink: source_link, - // The latest update to an API Section + // The latest update to an API section updates, - // The full-changeset to an API Section + // The full-changeset to an API section changes, - // The Heading Metadata + // The Heading metadata heading: internalMetadata.heading, - // The Stability Index of the API Section + // The Stability Index of the API section stability: stability_index, }; // If this metadata type matches certain predefined types - // We include it as a Navigation Entry for this API file + // We include it as a Navigation entry for this API file if (DOC_API_YAML_KEYS_NAVIGATION.includes(navigationEntry.type)) { navigationMetadataEntries.push(navigationEntry); } - // A metadata entry is all the metadata we have about a certain API Section + // A metadata entry is all the metadata we have about a certain API section // with the content being a VFile (Virtual File) containing the Markdown content section.data = navigationEntry; - // Returns the updated VFile with the extra Metadata + // Returns the updated VFile with the extra metadata return section; }, }; diff --git a/src/parser.mjs b/src/parser.mjs index d287b836..6bbe50fe 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -11,13 +11,13 @@ import createMetadata from './metadata.mjs'; import createQueries from './queries.mjs'; import { createNodeSlugger } from './utils/slugger.mjs'; +import { DOC_API_SECTION_SEPARATOR } from './constants.mjs'; -// Creates a Remark Parser with all Plugins needed for our API Docs +// Creates a Remark parser with all Plugins needed for our API docs const getRemarkParser = () => remark().use(remarkGfm); /** - * Creates an API Doc Parser for a given Markdown API Doc - * (this requires already parsed Node.js release data, the API doc file to be loaded, and etc) + * Creates an API doc parser for a given Markdown API doc file */ const createParser = () => { const slugger = createNodeSlugger(); @@ -26,15 +26,15 @@ const createParser = () => { createMetadata(slugger); /** - * Parses a given API Doc Metadata file into a list of Metadata Entries + * Parses a given API doc metadata file into a list of Metadata entries * * @param {import('vfile').VFile} apiDoc */ const parseApiDoc = async apiDoc => { - // Resets the Slugger as we are parsing a new API Doc file + // Resets the Slugger as we are parsing a new API doc file slugger.reset(); - // Does extra modification at the root level of the API Doc File + // Does extra modification at the root level of the API doc File const apiDocRootParser = getRemarkParser().use(() => { return tree => { const definitions = selectAll('definition', tree); @@ -46,19 +46,22 @@ const createParser = () => { updateLinkReference(node); }); - // Removes the Definitions from the Tree remove(tree, definitions); }; }); const parsedRoot = await apiDocRootParser.process(apiDoc); - // Gathers each Markdown Section within the API Doc + // Gathers each Markdown Section within the API doc // by separating chunks of the source (root) by heading separators - const markdownSections = parsedRoot.toString().split('\n\n#'); - - // Parses each Markdown Section into a Metadata Entry - const parsedSections = markdownSections.map(sectionSource => { + // @TODO: Remove the splitting of the doc file, do a single traversal via `unified` + // and determine beginning/end of sections based on encounter of a Heading Node + const markdownSections = String(parsedRoot).split( + DOC_API_SECTION_SEPARATOR + ); + + // Parses each Markdown section into a Metadata entry + const parsedSections = markdownSections.map((section, index) => { const apiEntryMetadata = newMetadataEntry(); const { @@ -71,7 +74,7 @@ const createParser = () => { const apiDocSectionParser = getRemarkParser().use(() => { return tree => { - // Handles YAML Metadata + // Handles YAML metadata visit(tree, createQueries.UNIST_TESTS.isYamlNode, node => { addYAMLMetadata(node); @@ -92,22 +95,28 @@ const createParser = () => { remove(tree, node); }); - // Handles API Type References transformation into Links + // Handles API type references transformation into links visit(tree, createQueries.UNIST_TESTS.isTextWithType, node => { updateTypeToReferenceLink(node); }); - // Handles Normalisation of Markdown URLs + // Handles normalisation of Markdown URLs visit(tree, createQueries.UNIST_TESTS.isMarkdownUrl, node => { updateMarkdownLink(node); }); }; }); - // Process the Markdown Section into a VFile with the processed data - const parsedSection = apiDocSectionParser.processSync(sectionSource); + // Process the Markdown section into a VFile with the processed data + const parsedSection = apiDocSectionParser.processSync( + // We add the `#` back to the beginning of the section source + // since we split the document by the `DOC_API_SECTION_SEPARATOR`. + // The only exception is the first entry, as it is not preceded by two blank lines + // as shown on `DOC_API_SECTION_SEPARATOR` + index > 0 ? `#${section}` : section + ); - // Creates a Metadata Entry (VFile) and returns it + // Creates a Metadata entry (VFile) and returns it return apiEntryMetadata.create(apiDoc.stem, parsedSection); }); @@ -115,10 +124,10 @@ const createParser = () => { }; /** - * This method allows to parse multiple API Doc Files at once - * and it simply wraps parseApiDoc with the given API Docs + * This method allows to parse multiple API doc files at once + * and it simply wraps parseApiDoc with the given API docs * - * @param {Array} apiDocs + * @param {Array} apiDocs List of API doc files to be parsed */ const parseApiDocs = async apiDocs => { const apiMetadataEntries = []; diff --git a/src/queries.mjs b/src/queries.mjs index f0db2742..cd468906 100644 --- a/src/queries.mjs +++ b/src/queries.mjs @@ -54,7 +54,7 @@ const createQueries = (apiEntryMetadata = undefined, definitions = []) => { const parsedHeading = parserUtils.parseHeadingIntoMetadata( heading, - node.depth + 1 + node.depth ); apiEntryMetadata.setHeading(parsedHeading); diff --git a/src/utils/parser.mjs b/src/utils/parser.mjs index a4f2166f..41caf3f8 100644 --- a/src/utils/parser.mjs +++ b/src/utils/parser.mjs @@ -18,39 +18,39 @@ import { * This method replaces plain text Types within the Markdown content into Markdown links * that link to the actual relevant reference for such type (either internal or external link) * - * @param {string} type The plain type to be transformed into a Markdown Link - * @returns {string} The Markdown Link as a string (formatted in Markdown) + * @param {string} type The plain type to be transformed into a Markdown link + * @returns {string} The Markdown link as a string (formatted in Markdown) */ export const transformTypeToReferenceLink = type => { const typeInput = type.replace('{', '').replace('}', ''); /** - * Handles the Mapping (if there's a match) of the input text - * into the reference type from the API Docs + * Handles the mapping (if there's a match) of the input text + * into the reference type from the API docs * * @param {string} lookupPiece */ const transformType = lookupPiece => { - // Transform JS Primitive Type references into Markdown Links (MDN) + // Transform JS primitive type references into Markdown links (MDN) if (lookupPiece.toLowerCase() in DOC_TYPES_MAPPING_PRIMITIVES) { const typeValue = DOC_TYPES_MAPPING_PRIMITIVES[lookupPiece.toLowerCase()]; return `${DOC_MDN_BASE_URL_JS_PRIMITIVES}#${typeValue}_type`; } - // Transforms JS Global Type references into Markdown Links (MDN) + // Transforms JS Global type references into Markdown links (MDN) if (lookupPiece in DOC_TYPES_MAPPING_GLOBALS) { return `${DOC_MDN_BASE_URL_JS_GLOBALS}${lookupPiece}`; } - // Transform other external Web/JavaScript Type references into Markdown Links + // Transform other external Web/JavaScript type references into Markdown links // to diverse different external websites. These already are formatted as links if (lookupPiece in DOC_TYPES_MAPPING_OTHER) { return DOC_TYPES_MAPPING_NODE_MODULES[lookupPiece]; } - // Transform Node.js Type/Module references into Markdown Links - // that refer to other API Docs pages within the Node.js API docs + // Transform Node.js type/module references into Markdown links + // that refer to other API docs pages within the Node.js API docs if (lookupPiece in DOC_TYPES_MAPPING_NODE_MODULES) { return DOC_TYPES_MAPPING_NODE_MODULES[lookupPiece]; } @@ -59,16 +59,16 @@ export const transformTypeToReferenceLink = type => { }; const typePieces = typeInput.split('|').map(piece => { - // This is the content to render as the text of the Markdown Link + // This is the content to render as the text of the Markdown link const trimmedPiece = piece.trim(); - // This is what we will compare against the API Types Mappings + // This is what we will compare against the API types mappings const result = transformType(trimmedPiece.replace(/(?:\[])+$/, '')); return result ? `[\`${trimmedPiece}\`](${result})` : undefined; }); - // Filter out pieces that we failed to Map and then join the valid ones + // Filter out pieces that we failed to map and then join the valid ones // into different links separated by a `|` const markdownLinks = typePieces.filter(Boolean).join(' | '); @@ -84,12 +84,11 @@ export const transformTypeToReferenceLink = type => { * (this is forwarded to the parser so it knows what to do with said metadata) * * @param {string} yamlString The YAML string to be parsed - * @returns {import('../types.d.ts').ApiDocRawMetadataEntry} The parsed YAML Metadata + * @returns {import('../types.d.ts').ApiDocRawMetadataEntry} The parsed YAML metadata */ export const parseYAMLIntoMetadata = yamlString => { const replacedContent = yamlString - // special validations for some non-cool formatted properties - // of the docs schema + // special validations for some non-cool formatted properties of the docs schema .replace('introduced_in=', 'introduced_in: ') .replace('source_link=', 'source_link: ') .replace('type=', 'type: ') @@ -119,15 +118,15 @@ export const parseYAMLIntoMetadata = yamlString => { }; /** - * Parses a raw Heading string into Heading Metadata + * Parses a raw Heading string into Heading metadata * - * @param {string} heading The raw Heading Text + * @param {string} heading The raw Heading text * @param {number} depth The depth of the heading - * @returns {import('../types.d.ts').HeadingMetadataEntry} Parsed Heading Entry + * @returns {import('../types.d.ts').HeadingMetadataEntry} Parsed Heading entry */ export const parseHeadingIntoMetadata = (heading, depth) => { for (const { type, regex } of DOC_API_HEADING_TYPES) { - // Attempts to get a match from one of the Heading Tyoes, if a match is found + // Attempts to get a match from one of the heading types, if a match is found // we use that type as the heading type, and extract the regex expression match group // which should be the inner "plain" heading content (or the title of the heading for navigation) const [, innerHeading] = heading.match(regex) ?? []; From 6c45c78d4d757b57208fa869f3e8909d5fd59fc2 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Sun, 7 Jul 2024 20:10:56 +0100 Subject: [PATCH 11/26] refactor: cleanup and simplify unist visit process --- src/constants.mjs | 2 +- src/parser.mjs | 216 ++++++++++++++++++++++++++-------------------- src/queries.mjs | 33 ++++--- 3 files changed, 140 insertions(+), 111 deletions(-) diff --git a/src/constants.mjs b/src/constants.mjs index 9028517f..8ecada77 100644 --- a/src/constants.mjs +++ b/src/constants.mjs @@ -16,7 +16,7 @@ export const DOC_MDN_BASE_URL_JS_PRIMITIVES = `${DOC_MDN_BASE_URL_JS}Data_struct export const DOC_MDN_BASE_URL_JS_GLOBALS = `${DOC_MDN_BASE_URL_JS}Reference/Global_Objects/`; // Characters used to split each section within an API Doc file -export const DOC_API_SECTION_SEPARATOR = '\n\n#'; +export const DOC_API_SECTION_SEPARATOR = /^#{1,4} .*/gm; // These are YAML keys from the Markdown YAML Metadata that should always be arrays export const DOC_API_YAML_KEYS_ARRAYS = [ diff --git a/src/parser.mjs b/src/parser.mjs index 6bbe50fe..77cd2e09 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -3,7 +3,7 @@ import { remark } from 'remark'; import remarkGfm from 'remark-gfm'; -import { visit } from 'unist-util-visit'; +import { SKIP, visit } from 'unist-util-visit'; import { remove } from 'unist-util-remove'; import { selectAll } from 'unist-util-select'; @@ -11,19 +11,26 @@ import createMetadata from './metadata.mjs'; import createQueries from './queries.mjs'; import { createNodeSlugger } from './utils/slugger.mjs'; -import { DOC_API_SECTION_SEPARATOR } from './constants.mjs'; - -// Creates a Remark parser with all Plugins needed for our API docs -const getRemarkParser = () => remark().use(remarkGfm); +import { DOC_API_SECTION_SEPARATOR as DOC_API_ENTRY_SEPARATOR } from './constants.mjs'; +import { VFile } from 'vfile'; /** * Creates an API doc parser for a given Markdown API doc file */ const createParser = () => { - const slugger = createNodeSlugger(); + const nodeSlugger = createNodeSlugger(); const { newMetadataEntry, getNavigationEntries: getNavigation } = - createMetadata(slugger); + createMetadata(nodeSlugger); + + const { + updateLinkReference, + updateTypeToReferenceLink, + updateMarkdownLink, + addYAMLMetadata, + addHeadingMetadata, + addStabilityIndexMeta, + } = createQueries(); /** * Parses a given API doc metadata file into a list of Metadata entries @@ -31,96 +38,128 @@ const createParser = () => { * @param {import('vfile').VFile} apiDoc */ const parseApiDoc = async apiDoc => { + /** + * This holds references to all the Metadata entries for a given file + * this is used so we can traverse the AST tree and keep mutating things + * and then stringify the whole api doc file at once without creating sub traversals + * + * Then once we have the whole file parsed, we can split the resulting string into sections + * and seal the Metadata Entries (`.create()`) and return the result to the caller of parae. + * + * @type {Array['newMetadataEntry']>} + */ + const metadataCollection = []; + + // This allows us to get the reference to the current Metadata entry + const getCurrentMetadataEntry = () => + metadataCollection[metadataCollection.length - 1]; + + // This allows us to create an retrieve a new Metadata entry + const createMetadataEntry = () => { + const metadataEntry = newMetadataEntry(); + + metadataCollection.push(metadataEntry); + + return metadataEntry; + }; + // Resets the Slugger as we are parsing a new API doc file - slugger.reset(); + nodeSlugger.reset(); + + // Creates a new Remark processor with GFM (GitHub Flavoured Markdown) support + const apiDocProcessor = remark().use(remarkGfm); - // Does extra modification at the root level of the API doc File - const apiDocRootParser = getRemarkParser().use(() => { + // Adds a transformer for handling link references and Markdown definitions + apiDocProcessor.use(() => { return tree => { + // Get all Markdown Footnote definitions from the tree const definitions = selectAll('definition', tree); - const { updateLinkReference } = createQueries(undefined, definitions); - // Handles Link References visit(tree, createQueries.UNIST_TESTS.isLinkReference, node => { - updateLinkReference(node); + updateLinkReference(node, definitions); + + return SKIP; }); + // Removes all the original definitions from the tree as they are not needed + // anymore, since all link references got updated to be plain links remove(tree, definitions); }; }); - const parsedRoot = await apiDocRootParser.process(apiDoc); - - // Gathers each Markdown Section within the API doc - // by separating chunks of the source (root) by heading separators - // @TODO: Remove the splitting of the doc file, do a single traversal via `unified` - // and determine beginning/end of sections based on encounter of a Heading Node - const markdownSections = String(parsedRoot).split( - DOC_API_SECTION_SEPARATOR - ); - - // Parses each Markdown section into a Metadata entry - const parsedSections = markdownSections.map((section, index) => { - const apiEntryMetadata = newMetadataEntry(); - - const { - addYAMLMetadata, - updateTypeToReferenceLink, - addHeadingMetadata, - updateMarkdownLink, - addStabilityIndexMeta, - } = createQueries(apiEntryMetadata); - - const apiDocSectionParser = getRemarkParser().use(() => { - return tree => { - // Handles YAML metadata - visit(tree, createQueries.UNIST_TESTS.isYamlNode, node => { - addYAMLMetadata(node); - - remove(tree, node); - }); - - // Handles Markdown Headings - visit(tree, createQueries.UNIST_TESTS.isHeadingNode, node => { - addHeadingMetadata(node); - - remove(tree, node); - }); - - // Handles Stability Indexes - visit(tree, createQueries.UNIST_TESTS.isStabilityIndex, node => { - addStabilityIndexMeta(node); - - remove(tree, node); - }); - - // Handles API type references transformation into links - visit(tree, createQueries.UNIST_TESTS.isTextWithType, node => { - updateTypeToReferenceLink(node); - }); - - // Handles normalisation of Markdown URLs - visit(tree, createQueries.UNIST_TESTS.isMarkdownUrl, node => { - updateMarkdownLink(node); - }); - }; - }); - - // Process the Markdown section into a VFile with the processed data - const parsedSection = apiDocSectionParser.processSync( - // We add the `#` back to the beginning of the section source - // since we split the document by the `DOC_API_SECTION_SEPARATOR`. - // The only exception is the first entry, as it is not preceded by two blank lines - // as shown on `DOC_API_SECTION_SEPARATOR` - index > 0 ? `#${section}` : section - ); - - // Creates a Metadata entry (VFile) and returns it - return apiEntryMetadata.create(apiDoc.stem, parsedSection); + // Adds a transformer for normalising the Markdown file with numerous actions + // that are required for the API docs to be correctly parsed + apiDocProcessor.use(() => { + return tree => { + // Handles API type references transformation into links + visit(tree, createQueries.UNIST_TESTS.isTextWithType, node => { + updateTypeToReferenceLink(node); + + return SKIP; + }); + + // Handles normalisation of Markdown URLs + visit(tree, createQueries.UNIST_TESTS.isMarkdownUrl, node => { + updateMarkdownLink(node); + + return SKIP; + }); + }; + }); + + // Adds a transformer for iterating through the Markdown file + // and going over the sections and creating metadata entries by traversing the tree + // A new metadata entry is created once a new Heading is found (from level 1 to 6) + apiDocProcessor.use(() => { + return tree => { + // Handles Markdown Headings + visit(tree, createQueries.UNIST_TESTS.isHeadingNode, node => { + addHeadingMetadata(node, createMetadataEntry()); + + return SKIP; + }); + + // Handles Stability Indexes + visit(tree, createQueries.UNIST_TESTS.isStabilityIndex, node => { + addStabilityIndexMeta(node, getCurrentMetadataEntry()); + + remove(tree, node); + + return SKIP; + }); + + // Handles YAML metadata + visit(tree, createQueries.UNIST_TESTS.isYamlNode, node => { + addYAMLMetadata(node, getCurrentMetadataEntry()); + + remove(tree, node); + + return SKIP; + }); + }; }); - return parsedSections; + const parsedApiDoc = await apiDocProcessor.process(apiDoc); + + /** + * Splits the parsed API doc file into sections (a section is defined by when a heading is found) + * We consider up to level 4 headings as sections, and we split the file into sections + * + * @type {Array} + */ + const [, ...sections] = String(parsedApiDoc).split(DOC_API_ENTRY_SEPARATOR); + + // We iterate the Markdown sections and retrieve the parsed Metadata entries from when Unified + // was iterating through the tree and then seal (create) the metadata entry for each section + return sections.map((section, index) => { + // This creates a new VFile with the top-level metadata of the API doc file + // and the trimmed content of the section (which excludes the header, since it is already within the Metadata) + const apiEntryFile = new VFile({ ...apiDoc, value: section.trim() }); + + // Seals the Metadata entry with the actual content of the section + return metadataCollection[index].create(apiDoc.stem, apiEntryFile); + }); }; /** @@ -129,17 +168,8 @@ const createParser = () => { * * @param {Array} apiDocs List of API doc files to be parsed */ - const parseApiDocs = async apiDocs => { - const apiMetadataEntries = []; - - for (const apiDoc of apiDocs) { - const parsedEntries = await parseApiDoc(apiDoc); - - apiMetadataEntries.push(...parsedEntries); - } - - return apiMetadataEntries; - }; + const parseApiDocs = apiDocs => + Promise.all(apiDocs.map(parseApiDoc)).then(entries => entries.flat()); return { getNavigation, parseApiDocs }; }; diff --git a/src/queries.mjs b/src/queries.mjs index cd468906..7a5d9948 100644 --- a/src/queries.mjs +++ b/src/queries.mjs @@ -1,23 +1,21 @@ 'use strict'; import * as parserUtils from './utils/parser.mjs'; -import * as unistUtils from './utils/unist.mjs'; +import { transformNodesToString } from './utils/unist.mjs'; /** * Creates an instance of the Query Manager, which allows to do multiple sort * of metadata and content metadata manipulation within an API Doc - * - * @param {ReturnType['newMetadataEntry']>} apiEntryMetadata The API Entry Metadata - * @param {Array} definitions The Definitions of the API Doc */ -const createQueries = (apiEntryMetadata = undefined, definitions = []) => { +const createQueries = () => { /** * Sanitizes the YAML source by returning the inner YAML content * and then parsing it into an API Metadata object and updating the current Metadata * * @param {import('unist').Node} node The YAML Node + * @param {ReturnType['newMetadataEntry']>} apiEntryMetadata The API entry Metadata */ - const addYAMLMetadata = node => { + const addYAMLMetadata = (node, apiEntryMetadata) => { const sanitizedString = node.value.replace( createQueries.QUERIES.yamlInnerContent, (_, __, inner) => inner @@ -48,9 +46,10 @@ const createQueries = (apiEntryMetadata = undefined, definitions = []) => { * Parse a Heading Node into Metadata and updates the current Metadata * * @param {import('unist').Node} node The Heading Node + * @param {ReturnType['newMetadataEntry']>} apiEntryMetadata The API entry Metadata */ - const addHeadingMetadata = node => { - const heading = unistUtils.transformNodesToString(node.children); + const addHeadingMetadata = (node, apiEntryMetadata) => { + const heading = transformNodesToString(node.children); const parsedHeading = parserUtils.parseHeadingIntoMetadata( heading, @@ -76,8 +75,9 @@ const createQueries = (apiEntryMetadata = undefined, definitions = []) => { * Updates a Markdown Link Reference into an actual Link to the Definition * * @param {import('unist').Node} node Thead Link Reference Node + * @param {Array} definitions The Definitions of the API Doc */ - const updateLinkReference = node => { + const updateLinkReference = (node, definitions) => { const definition = definitions.find( ({ identifier }) => identifier === node.identifier ); @@ -90,9 +90,10 @@ const createQueries = (apiEntryMetadata = undefined, definitions = []) => { * Parses a Stability Index Entry and updates the current Metadata * * @param {import('unist').Node} node Thead Link Reference Node + * @param {ReturnType['newMetadataEntry']>} apiEntryMetadata The API entry Metadata */ - const addStabilityIndexMeta = node => { - const stabilityIndexString = unistUtils.transformNodesToString( + const addStabilityIndexMeta = (node, apiEntryMetadata) => { + const stabilityIndexString = transformNodesToString( node.children[0].children ); @@ -131,6 +132,9 @@ createQueries.QUERIES = { }; createQueries.UNIST_TESTS = { + isStabilityIndex: ({ type, children }) => + type === 'blockquote' && + createQueries.QUERIES.stabilityIndex.test(transformNodesToString(children)), isYamlNode: ({ type, value }) => type === 'html' && createQueries.QUERIES.yamlInnerContent.test(value), isTextWithType: ({ type, value }) => @@ -138,14 +142,9 @@ createQueries.UNIST_TESTS = { isMarkdownUrl: ({ type, url }) => type === 'link' && createQueries.QUERIES.markdownUrl.test(url), isHeadingNode: ({ type, depth }) => - type === 'heading' && depth >= 1 && depth <= 6, + type === 'heading' && depth >= 1 && depth <= 4, isLinkReference: ({ type, identifier }) => type === 'linkReference' && !!identifier, - isStabilityIndex: ({ type, children }) => - type === 'blockquote' && - createQueries.QUERIES.stabilityIndex.test( - unistUtils.transformNodesToString(children) - ), }; export default createQueries; From d35e974f48c3330ec9c2cdb1a20bf1030eeb041a Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Sun, 7 Jul 2024 21:13:52 +0100 Subject: [PATCH 12/26] fix: missing metadata --- src/parser.mjs | 47 ++++++++++++++++++++--------------------------- src/queries.mjs | 4 ++-- 2 files changed, 22 insertions(+), 29 deletions(-) diff --git a/src/parser.mjs b/src/parser.mjs index 77cd2e09..38c3e032 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -29,7 +29,7 @@ const createParser = () => { updateMarkdownLink, addYAMLMetadata, addHeadingMetadata, - addStabilityIndexMeta, + addStabilityIndexMetadata, } = createQueries(); /** @@ -50,19 +50,6 @@ const createParser = () => { */ const metadataCollection = []; - // This allows us to get the reference to the current Metadata entry - const getCurrentMetadataEntry = () => - metadataCollection[metadataCollection.length - 1]; - - // This allows us to create an retrieve a new Metadata entry - const createMetadataEntry = () => { - const metadataEntry = newMetadataEntry(); - - metadataCollection.push(metadataEntry); - - return metadataEntry; - }; - // Resets the Slugger as we are parsing a new API doc file nodeSlugger.reset(); @@ -115,25 +102,31 @@ const createParser = () => { return tree => { // Handles Markdown Headings visit(tree, createQueries.UNIST_TESTS.isHeadingNode, node => { - addHeadingMetadata(node, createMetadataEntry()); + // Creates a new Metadata entry for the API doc file + const apiEntryMetadata = newMetadataEntry(); - return SKIP; - }); + addHeadingMetadata(node, apiEntryMetadata); - // Handles Stability Indexes - visit(tree, createQueries.UNIST_TESTS.isStabilityIndex, node => { - addStabilityIndexMeta(node, getCurrentMetadataEntry()); + // Handles Stability Indexes + visit(tree, createQueries.UNIST_TESTS.isStabilityIndex, node => { + addStabilityIndexMetadata(node, apiEntryMetadata); - remove(tree, node); + remove(tree, node); - return SKIP; - }); + return SKIP; + }); + + // Handles YAML metadata + visit(tree, createQueries.UNIST_TESTS.isYamlNode, node => { + addYAMLMetadata(node, apiEntryMetadata); + + remove(tree, node); - // Handles YAML metadata - visit(tree, createQueries.UNIST_TESTS.isYamlNode, node => { - addYAMLMetadata(node, getCurrentMetadataEntry()); + return SKIP; + }); - remove(tree, node); + // Pushes them to the Metadata collection + metadataCollection.push(apiEntryMetadata); return SKIP; }); diff --git a/src/queries.mjs b/src/queries.mjs index 7a5d9948..35f9e176 100644 --- a/src/queries.mjs +++ b/src/queries.mjs @@ -92,7 +92,7 @@ const createQueries = () => { * @param {import('unist').Node} node Thead Link Reference Node * @param {ReturnType['newMetadataEntry']>} apiEntryMetadata The API entry Metadata */ - const addStabilityIndexMeta = (node, apiEntryMetadata) => { + const addStabilityIndexMetadata = (node, apiEntryMetadata) => { const stabilityIndexString = transformNodesToString( node.children[0].children ); @@ -114,7 +114,7 @@ const createQueries = () => { addHeadingMetadata, updateMarkdownLink, updateLinkReference, - addStabilityIndexMeta, + addStabilityIndexMetadata, }; }; From 551e85a7dc2b3edf5774202de41ea41179ab9057 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Mon, 8 Jul 2024 02:18:56 +0100 Subject: [PATCH 13/26] fix: fixed and refactored the code --- package-lock.json | 44 +++++++++++ package.json | 3 + src/constants.mjs | 6 -- src/loader.mjs | 3 +- src/metadata.mjs | 176 +++++++++++++++++++------------------------ src/parser.mjs | 104 +++++++++++++++---------- src/queries.mjs | 8 +- src/utils/parser.mjs | 21 ++++++ 8 files changed, 218 insertions(+), 147 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9d33e262..a69146c4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,9 @@ "glob": "^10.4.3", "remark": "^15.0.1", "remark-gfm": "^4.0.0", + "unist-util-find-after": "^5.0.0", + "unist-util-find-all-after": "^5.0.0", + "unist-util-position": "^5.0.0", "unist-util-remove": "^4.0.0", "unist-util-select": "^5.1.0", "unist-util-visit": "^5.0.0", @@ -2972,6 +2975,34 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/unist-util-find-after": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-find-after/-/unist-util-find-after-5.0.0.tgz", + "integrity": "sha512-amQa0Ep2m6hE2g72AugUItjbuM8X8cGQnFoHk0pGfrFeT9GZhzN5SW8nRsiGKK7Aif4CrACPENkA6P/Lw6fHGQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-find-all-after": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-find-all-after/-/unist-util-find-all-after-5.0.0.tgz", + "integrity": "sha512-nGmOYvTSdGcI4RvrUNfe0mOsqqbbJOtqCQsppsY9KZjmv3nwM3YRgNBwFPdZ8Y+iv9Z/2PDjR9u6u+uK62XTTg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/unist-util-is": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.0.tgz", @@ -2985,6 +3016,19 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/unist-util-position": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-position/-/unist-util-position-5.0.0.tgz", + "integrity": "sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/unist-util-remove": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/unist-util-remove/-/unist-util-remove-4.0.0.tgz", diff --git a/package.json b/package.json index 6d6099c0..ca309353 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,9 @@ "glob": "^10.4.3", "remark": "^15.0.1", "remark-gfm": "^4.0.0", + "unist-util-find-after": "^5.0.0", + "unist-util-find-all-after": "^5.0.0", + "unist-util-position": "^5.0.0", "unist-util-remove": "^4.0.0", "unist-util-select": "^5.1.0", "unist-util-visit": "^5.0.0", diff --git a/src/constants.mjs b/src/constants.mjs index 8ecada77..34ef83c8 100644 --- a/src/constants.mjs +++ b/src/constants.mjs @@ -15,9 +15,6 @@ export const DOC_MDN_BASE_URL_JS_PRIMITIVES = `${DOC_MDN_BASE_URL_JS}Data_struct // This is the base URL for the MDN JavaScript global objects documentation export const DOC_MDN_BASE_URL_JS_GLOBALS = `${DOC_MDN_BASE_URL_JS}Reference/Global_Objects/`; -// Characters used to split each section within an API Doc file -export const DOC_API_SECTION_SEPARATOR = /^#{1,4} .*/gm; - // These are YAML keys from the Markdown YAML Metadata that should always be arrays export const DOC_API_YAML_KEYS_ARRAYS = [ 'added', @@ -37,9 +34,6 @@ export const DOC_API_YAML_KEYS_UPDATE = [ 'napiVersion', ]; -// These are the YAML types that should be considered as navigation entries -export const DOC_API_YAML_KEYS_NAVIGATION = ['class', 'module', 'global']; - // These are string replacements specific to Node.js API docs for anchor IDs export const DOC_API_SLUGS_REPLACEMENTS = [ { from: /node.js/i, to: 'nodejs' }, // Replace Node.js diff --git a/src/loader.mjs b/src/loader.mjs index 3fb891d8..f71450f7 100644 --- a/src/loader.mjs +++ b/src/loader.mjs @@ -1,8 +1,9 @@ 'use strict'; -import { glob } from 'glob'; import { readFileSync } from 'node:fs'; import { extname } from 'node:path'; + +import { glob } from 'glob'; import { VFile } from 'vfile'; /** diff --git a/src/metadata.mjs b/src/metadata.mjs index c6b88efb..2e9a4295 100644 --- a/src/metadata.mjs +++ b/src/metadata.mjs @@ -1,7 +1,5 @@ 'use strict'; -import { DOC_API_YAML_KEYS_NAVIGATION } from './constants.mjs'; - /** * This method allows us to handle creation of Metadata entries * within the current scope of API docs being parsed @@ -12,113 +10,95 @@ import { DOC_API_YAML_KEYS_NAVIGATION } from './constants.mjs'; * @param {InstanceType} slugger A GitHub Slugger */ const createMetadata = slugger => { - const navigationMetadataEntries = []; + // This holds a temporary buffer of raw metadata before being + // transformed into NavigationEntries and MetadataEntries + const internalMetadata = { + heading: { + text: undefined, + type: undefined, + name: undefined, + depth: -1, + }, + properties: {}, + }; return { /** - * Retrieves all Navigation entries generated in a said file + * Set the Heading of a given Metadata + * + * @param {import('./types.d.ts').HeadingMetadataEntry} heading The new heading metadata + */ + setHeading: heading => { + internalMetadata.heading = heading; + }, + /** + * Set the Metadata (from YAML if exists) properties to the current Metadata entry + * itI also allows for extra data (such as Stability Index) and miscellaneous data to be set + * although it'd be best to only set ones from {ApiDocRawMetadataEntry} * - * @returns {Array} + * @param {Partial} properties Extra Metadata properties to be defined */ - getNavigationEntries: () => navigationMetadataEntries, - newMetadataEntry: () => { - // This holds a temporary buffer of raw metadata before being - // transformed into NavigationEntries and MetadataEntries - const internalMetadata = { - heading: { - text: undefined, - type: undefined, - name: undefined, - depth: -1, - }, - properties: {}, + updateProperties: properties => { + internalMetadata.properties = { + ...internalMetadata.properties, + ...properties, }; + }, + /** + * Generates a new Navigation entry and pushes them to the internal collection + * of Navigation entries, and returns a MetadataEntry which is then used by the parser + * and forwarded to any relevant generator. + * + * The Navigation entries has a dedicated separate method for retrieval + * as it can be manipulated outside of the scope of the generation of the content + * + * @param {string} apiDoc The name of the API doc + * @param {import('vfile').VFile} section The content of the current Metadata entry + * @returns {import('./types').ApiDocMetadataEntry} The locally created Metadata entries + */ + create: (apiDoc, section) => { + // This is the ID of a certain Navigation entry, which allows us to anchor + // a certain navigation section to a page ad the exact point of the page (scroll) + // This is useful for classes, globals and other type of YAML entries, as they reside + // within a module (page) and we want to link to them directly + const slugHash = `#${slugger.slug(internalMetadata.heading.text)}`; - return { - /** - * Set the Heading of a given Metadata - * - * @param {import('./types.d.ts').HeadingMetadataEntry} heading The new heading metadata - */ - setHeading: heading => { - internalMetadata.heading = heading; - }, - /** - * Set the Metadata (from YAML if exists) properties to the current Metadata entry - * itI also allows for extra data (such as Stability Index) and miscellaneous data to be set - * although it'd be best to only set ones from {ApiDocRawMetadataEntry} - * - * @param {Partial} properties Extra Metadata properties to be defined - */ - updateProperties: properties => { - internalMetadata.properties = { - ...internalMetadata.properties, - ...properties, - }; - }, - /** - * Generates a new Navigation entry and pushes them to the internal collection - * of Navigation entries, and returns a MetadataEntry which is then used by the parser - * and forwarded to any relevant generator. - * - * The Navigation entries has a dedicated separate method for retrieval - * as it can be manipulated outside of the scope of the generation of the content - * - * @param {string} apiDoc The name of the API doc - * @param {import('vfile').VFile} section The content of the current Metadata entry - * @returns {import('./types').ApiDocMetadataEntry} The locally created Metadata entries - */ - create: (apiDoc, section) => { - // This is the ID of a certain Navigation entry, which allows us to anchor - // a certain navigation section to a page ad the exact point of the page (scroll) - // This is useful for classes, globals and other type of YAML entries, as they reside - // within a module (page) and we want to link to them directly - const slugHash = `#${slugger.slug(internalMetadata.heading.text)}`; - - const { - type: yaml_type, - name: yaml_name, - source_link, - stability_index, - updates = [], - changes = [], - } = internalMetadata.properties; - - // We override the type of the heading if we have a YAML type - internalMetadata.heading.type = - yaml_type || internalMetadata.heading.type; + const { + type: yaml_type, + name: yaml_name, + source_link, + stability_index, + updates = [], + changes = [], + } = internalMetadata.properties; - const navigationEntry = { - // The API file basename (without the extension) - api: yaml_name || apiDoc, - // The path/slug of the API section - slug: `${apiDoc}.html${slugHash}`, - // The source link of said API section - sourceLink: source_link, - // The latest update to an API section - updates, - // The full-changeset to an API section - changes, - // The Heading metadata - heading: internalMetadata.heading, - // The Stability Index of the API section - stability: stability_index, - }; + // We override the type of the heading if we have a YAML type + internalMetadata.heading.type = + yaml_type || internalMetadata.heading.type; - // If this metadata type matches certain predefined types - // We include it as a Navigation entry for this API file - if (DOC_API_YAML_KEYS_NAVIGATION.includes(navigationEntry.type)) { - navigationMetadataEntries.push(navigationEntry); - } + const navigationEntry = { + // The API file basename (without the extension) + api: yaml_name || apiDoc, + // The path/slug of the API section + slug: `${apiDoc}.html${slugHash}`, + // The source link of said API section + sourceLink: source_link, + // The latest update to an API section + updates, + // The full-changeset to an API section + changes, + // The Heading metadata + heading: internalMetadata.heading, + // The Stability Index of the API section + stability: stability_index, + }; - // A metadata entry is all the metadata we have about a certain API section - // with the content being a VFile (Virtual File) containing the Markdown content - section.data = navigationEntry; + // A metadata entry is all the metadata we have about a certain API section + // with the content being a VFile (Virtual File) containing the Markdown content + section.data = navigationEntry; - // Returns the updated VFile with the extra metadata - return section; - }, - }; + // Returns the updated VFile with the extra metadata + return section; }, }; }; diff --git a/src/parser.mjs b/src/parser.mjs index 38c3e032..2f0497b9 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -1,28 +1,29 @@ 'use strict'; +import { VFile } from 'vfile'; + import { remark } from 'remark'; import remarkGfm from 'remark-gfm'; import { SKIP, visit } from 'unist-util-visit'; import { remove } from 'unist-util-remove'; import { selectAll } from 'unist-util-select'; +import { findAfter } from 'unist-util-find-after'; +import { findAllAfter } from 'unist-util-find-all-after'; import createMetadata from './metadata.mjs'; import createQueries from './queries.mjs'; import { createNodeSlugger } from './utils/slugger.mjs'; -import { DOC_API_SECTION_SEPARATOR as DOC_API_ENTRY_SEPARATOR } from './constants.mjs'; -import { VFile } from 'vfile'; +import { callIfBefore } from './utils/parser.mjs'; + +// Characters used to split each section within an API Doc file +const DOC_API_ENTRY_SEPARATOR = /^#{1,4} .*/gm; /** * Creates an API doc parser for a given Markdown API doc file */ const createParser = () => { - const nodeSlugger = createNodeSlugger(); - - const { newMetadataEntry, getNavigationEntries: getNavigation } = - createMetadata(nodeSlugger); - const { updateLinkReference, updateTypeToReferenceLink, @@ -46,12 +47,12 @@ const createParser = () => { * Then once we have the whole file parsed, we can split the resulting string into sections * and seal the Metadata Entries (`.create()`) and return the result to the caller of parae. * - * @type {Array['newMetadataEntry']>} + * @type {Array} */ const metadataCollection = []; - // Resets the Slugger as we are parsing a new API doc file - nodeSlugger.reset(); + // Creates an instance of the Node slugger per API doc file + const nodeSlugger = createNodeSlugger(); // Creates a new Remark processor with GFM (GitHub Flavoured Markdown) support const apiDocProcessor = remark().use(remarkGfm); @@ -60,18 +61,18 @@ const createParser = () => { apiDocProcessor.use(() => { return tree => { // Get all Markdown Footnote definitions from the tree - const definitions = selectAll('definition', tree); + const markdownDefinitions = selectAll('definition', tree); // Handles Link References - visit(tree, createQueries.UNIST_TESTS.isLinkReference, node => { - updateLinkReference(node, definitions); + visit(tree, createQueries.UNIST.isLinkReference, node => { + updateLinkReference(node, markdownDefinitions); return SKIP; }); // Removes all the original definitions from the tree as they are not needed // anymore, since all link references got updated to be plain links - remove(tree, definitions); + remove(tree, markdownDefinitions); }; }); @@ -80,14 +81,14 @@ const createParser = () => { apiDocProcessor.use(() => { return tree => { // Handles API type references transformation into links - visit(tree, createQueries.UNIST_TESTS.isTextWithType, node => { + visit(tree, createQueries.UNIST.isTextWithType, node => { updateTypeToReferenceLink(node); return SKIP; }); // Handles normalisation of Markdown URLs - visit(tree, createQueries.UNIST_TESTS.isMarkdownUrl, node => { + visit(tree, createQueries.UNIST.isMarkdownUrl, node => { updateMarkdownLink(node); return SKIP; @@ -100,32 +101,58 @@ const createParser = () => { // A new metadata entry is created once a new Heading is found (from level 1 to 6) apiDocProcessor.use(() => { return tree => { - // Handles Markdown Headings - visit(tree, createQueries.UNIST_TESTS.isHeadingNode, node => { - // Creates a new Metadata entry for the API doc file - const apiEntryMetadata = newMetadataEntry(); + visit(tree, createQueries.UNIST.isHeadingNode, node => { + // Creates a new Metadata entry for the current API doc file + const apiEntryMetadata = createMetadata(nodeSlugger); + // Adds the Metadata of the current Heading Node to the Metadata entry addHeadingMetadata(node, apiEntryMetadata); - // Handles Stability Indexes - visit(tree, createQueries.UNIST_TESTS.isStabilityIndex, node => { - addStabilityIndexMetadata(node, apiEntryMetadata); - - remove(tree, node); - - return SKIP; - }); - - // Handles YAML metadata - visit(tree, createQueries.UNIST_TESTS.isYamlNode, node => { - addYAMLMetadata(node, apiEntryMetadata); - - remove(tree, node); - - return SKIP; + // We retrieve the immediate next Heading if it exists + // This is used for ensuring that we don't include items that would + // belong only to the next heading to the current Heading metadata + callIfBefore(findAfter(tree, node, 'heading'), node, next => { + // Gets the next available Stability Index Node (if any) + // from the current Heading Node, and then adds it to the current Metadata + // if the found Stability Index Node is before the next Heading Node + // we then add it to the current Metadata + const stabilityIndexNodes = findAllAfter( + tree, + node, + createQueries.UNIST.isStabilityIndex + ); + + // Gets the next available YAML Node (if any) + // from the current Heading Node, and then adds it to the current Metadata + // if the found YAML Node is before the next Heading Node + // we then add it to the current Metadata + const yamlMetadataNodes = findAllAfter( + tree, + node, + createQueries.UNIST.isYamlNode + ); + + stabilityIndexNodes.forEach(stabilityIndexNode => { + callIfBefore(next, stabilityIndexNode, () => { + // Adds the Stability Index Metadata to the current Metadata entry + addStabilityIndexMetadata(stabilityIndexNode, apiEntryMetadata); + + remove(tree, stabilityIndexNode); + }); + }); + + yamlMetadataNodes.forEach(yamlMetadataNode => { + callIfBefore(next, yamlMetadataNode, () => { + // Adds the YAML Metadata to the current Metadata entry + addYAMLMetadata(yamlMetadataNode, apiEntryMetadata); + + remove(tree, yamlMetadataNode); + }); + }); }); - // Pushes them to the Metadata collection + // After all is processed for the current Heading we proceed to push it + // to the Metadata collection, so that it can be sealed later on metadataCollection.push(apiEntryMetadata); return SKIP; @@ -133,6 +160,7 @@ const createParser = () => { }; }); + // Processes the API doc file and returns the parsed API doc const parsedApiDoc = await apiDocProcessor.process(apiDoc); /** @@ -164,7 +192,7 @@ const createParser = () => { const parseApiDocs = apiDocs => Promise.all(apiDocs.map(parseApiDoc)).then(entries => entries.flat()); - return { getNavigation, parseApiDocs }; + return { parseApiDocs }; }; export default createParser; diff --git a/src/queries.mjs b/src/queries.mjs index 35f9e176..26f36964 100644 --- a/src/queries.mjs +++ b/src/queries.mjs @@ -13,7 +13,7 @@ const createQueries = () => { * and then parsing it into an API Metadata object and updating the current Metadata * * @param {import('unist').Node} node The YAML Node - * @param {ReturnType['newMetadataEntry']>} apiEntryMetadata The API entry Metadata + * @param {ReturnType} apiEntryMetadata The API entry Metadata */ const addYAMLMetadata = (node, apiEntryMetadata) => { const sanitizedString = node.value.replace( @@ -46,7 +46,7 @@ const createQueries = () => { * Parse a Heading Node into Metadata and updates the current Metadata * * @param {import('unist').Node} node The Heading Node - * @param {ReturnType['newMetadataEntry']>} apiEntryMetadata The API entry Metadata + * @param {ReturnType} apiEntryMetadata The API entry Metadata */ const addHeadingMetadata = (node, apiEntryMetadata) => { const heading = transformNodesToString(node.children); @@ -90,7 +90,7 @@ const createQueries = () => { * Parses a Stability Index Entry and updates the current Metadata * * @param {import('unist').Node} node Thead Link Reference Node - * @param {ReturnType['newMetadataEntry']>} apiEntryMetadata The API entry Metadata + * @param {ReturnType} apiEntryMetadata The API entry Metadata */ const addStabilityIndexMetadata = (node, apiEntryMetadata) => { const stabilityIndexString = transformNodesToString( @@ -131,7 +131,7 @@ createQueries.QUERIES = { yamlInnerContent: /^/, }; -createQueries.UNIST_TESTS = { +createQueries.UNIST = { isStabilityIndex: ({ type, children }) => type === 'blockquote' && createQueries.QUERIES.stabilityIndex.test(transformNodesToString(children)), diff --git a/src/utils/parser.mjs b/src/utils/parser.mjs index 41caf3f8..6b7464be 100644 --- a/src/utils/parser.mjs +++ b/src/utils/parser.mjs @@ -2,6 +2,8 @@ import yaml from 'yaml'; +import { pointEnd, pointStart } from 'unist-util-position'; + import { DOC_API_HEADING_TYPES, DOC_API_YAML_KEYS_ARRAYS, @@ -138,3 +140,22 @@ export const parseHeadingIntoMetadata = (heading, depth) => { return { text: heading, type: 'module', name: heading, depth }; }; + +/** + * This method is an utility that allows us to conditionally invoke/call a callback + * based on test conditions related to a Node's position relative to another one + * being before or not the other Node + * + * @param {import('unist').Node | undefined} nodeA The Node to be used as a position reference to check against + * the other Node. If the other Node is before this one, the callback will be called. + * @param {import('unist').Node | undefined} nodeB The Node to be checked against the position of the first Node + * @param {(nodeA: import('unist').Node, nodeB: import('unist').Node) => void} callback The callback to be called + */ +export const callIfBefore = (nodeA, nodeB, callback) => { + const positionA = pointEnd(nodeA); + const positionB = pointStart(nodeB); + + if (positionA && positionB && positionA.line > positionB.line) { + callback(nodeA, nodeB); + } +}; From a009ec48fcc4ee51b8f933303d048fd3c0edf103 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Mon, 8 Jul 2024 04:29:59 +0100 Subject: [PATCH 14/26] chore: deep optimization and fix of edge cases --- package-lock.json | 14 ++++++ package.json | 1 + src/parser.mjs | 115 ++++++++++++++++++++++++++++++------------- src/utils/parser.mjs | 21 -------- src/utils/unist.mjs | 23 ++++++++- 5 files changed, 118 insertions(+), 56 deletions(-) diff --git a/package-lock.json b/package-lock.json index a69146c4..d9fca624 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "glob": "^10.4.3", "remark": "^15.0.1", "remark-gfm": "^4.0.0", + "unist-builder": "^4.0.0", "unist-util-find-after": "^5.0.0", "unist-util-find-all-after": "^5.0.0", "unist-util-position": "^5.0.0", @@ -2975,6 +2976,19 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/unist-builder": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-builder/-/unist-builder-4.0.0.tgz", + "integrity": "sha512-wmRFnH+BLpZnTKpc5L7O67Kac89s9HMrtELpnNaE6TAobq5DTZZs5YaTQfAZBA9bFPECx2uVAPO31c+GVug8mg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/unist-util-find-after": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/unist-util-find-after/-/unist-util-find-after-5.0.0.tgz", diff --git a/package.json b/package.json index ca309353..538cf14f 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "glob": "^10.4.3", "remark": "^15.0.1", "remark-gfm": "^4.0.0", + "unist-builder": "^4.0.0", "unist-util-find-after": "^5.0.0", "unist-util-find-all-after": "^5.0.0", "unist-util-position": "^5.0.0", diff --git a/src/parser.mjs b/src/parser.mjs index 2f0497b9..ded93a9d 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -5,9 +5,10 @@ import { VFile } from 'vfile'; import { remark } from 'remark'; import remarkGfm from 'remark-gfm'; -import { SKIP, visit } from 'unist-util-visit'; +import { u } from 'unist-builder'; import { remove } from 'unist-util-remove'; import { selectAll } from 'unist-util-select'; +import { SKIP, visit } from 'unist-util-visit'; import { findAfter } from 'unist-util-find-after'; import { findAllAfter } from 'unist-util-find-all-after'; @@ -15,10 +16,7 @@ import createMetadata from './metadata.mjs'; import createQueries from './queries.mjs'; import { createNodeSlugger } from './utils/slugger.mjs'; -import { callIfBefore } from './utils/parser.mjs'; - -// Characters used to split each section within an API Doc file -const DOC_API_ENTRY_SEPARATOR = /^#{1,4} .*/gm; +import { callIfBefore } from './utils/unist.mjs'; /** * Creates an API doc parser for a given Markdown API doc file @@ -47,7 +45,7 @@ const createParser = () => { * Then once we have the whole file parsed, we can split the resulting string into sections * and seal the Metadata Entries (`.create()`) and return the result to the caller of parae. * - * @type {Array} + * @type {Array} */ const metadataCollection = []; @@ -59,6 +57,11 @@ const createParser = () => { // Adds a transformer for handling link references and Markdown definitions apiDocProcessor.use(() => { + /** + * This transformer is responsible for handling link references and Markdown definitions + * + * @param {import('unist').Parent} tree The root AST tree for the API doc file + */ return tree => { // Get all Markdown Footnote definitions from the tree const markdownDefinitions = selectAll('definition', tree); @@ -79,6 +82,11 @@ const createParser = () => { // Adds a transformer for normalising the Markdown file with numerous actions // that are required for the API docs to be correctly parsed apiDocProcessor.use(() => { + /** + * This transformer is responsible for normalising the Markdown file + * + * @param {import('unist').Parent} tree The root AST tree for the API doc file + */ return tree => { // Handles API type references transformation into links visit(tree, createQueries.UNIST.isTextWithType, node => { @@ -100,8 +108,13 @@ const createParser = () => { // and going over the sections and creating metadata entries by traversing the tree // A new metadata entry is created once a new Heading is found (from level 1 to 6) apiDocProcessor.use(() => { + /** + * Iterates through the AST tree and creates Metadata entries for each API doc section + * + * @param {import('unist').Parent} tree The root AST tree for the API doc file + */ return tree => { - visit(tree, createQueries.UNIST.isHeadingNode, node => { + visit(tree, createQueries.UNIST.isHeadingNode, (node, index) => { // Creates a new Metadata entry for the current API doc file const apiEntryMetadata = createMetadata(nodeSlugger); @@ -111,7 +124,10 @@ const createParser = () => { // We retrieve the immediate next Heading if it exists // This is used for ensuring that we don't include items that would // belong only to the next heading to the current Heading metadata - callIfBefore(findAfter(tree, node, 'heading'), node, next => { + // Note that if there is no next heading, we use the current node as the next one + const nextHeadingNode = findAfter(tree, index, 'heading') ?? node; + + callIfBefore(nextHeadingNode, node, next => { // Gets the next available Stability Index Node (if any) // from the current Heading Node, and then adds it to the current Metadata // if the found Stability Index Node is before the next Heading Node @@ -133,7 +149,11 @@ const createParser = () => { ); stabilityIndexNodes.forEach(stabilityIndexNode => { - callIfBefore(next, stabilityIndexNode, () => { + // If the next heading === current one, then we use the stabilityIndex + // itself as the check, since this means we reaching the end of the document + const toCheckAgainst = node === next ? stabilityIndexNode : next; + + callIfBefore(toCheckAgainst, stabilityIndexNode, () => { // Adds the Stability Index Metadata to the current Metadata entry addStabilityIndexMetadata(stabilityIndexNode, apiEntryMetadata); @@ -142,45 +162,69 @@ const createParser = () => { }); yamlMetadataNodes.forEach(yamlMetadataNode => { - callIfBefore(next, yamlMetadataNode, () => { + // If the next heading === current one, then we use the yamlMetadataNode + // itself as the check, since this means we reaching the end of the document + const toCheckAgainst = node === next ? yamlMetadataNode : next; + + callIfBefore(toCheckAgainst, yamlMetadataNode, () => { // Adds the YAML Metadata to the current Metadata entry addYAMLMetadata(yamlMetadataNode, apiEntryMetadata); remove(tree, yamlMetadataNode); }); }); - }); - // After all is processed for the current Heading we proceed to push it - // to the Metadata collection, so that it can be sealed later on - metadataCollection.push(apiEntryMetadata); + // This is the cutover index of the subtree that we should get + // of all the Nodes within the AST tree that belong to this section + // If `next` is equals the current heading, it means ther's no next heading + // and we are reaching the end of the document, hence the cutover should be the end of + // the document itself, + // Note.: This index needs to be retrieved after any modification to the tree occurs, + // otherwise the index will be off (out of sync with the tree) + const cutoverIndex = + node === next + ? tree.children.length - 1 + : tree.children.indexOf(next); + + // Retrieves all the Nodes that should belong to the current API doc section + const sectionNodes = tree.children.slice(index + 1, cutoverIndex); + + // The stringified (back to Markdown) content of the section + const sectionParsedContent = apiDocProcessor.stringify( + // Creates a subtree based on the section Nodes + u('root', sectionNodes), + apiDoc + ); + + // We seal and create the API doc entry Metadata and push them to the collection + // Creates the API doc entry Metadata and pushes it to the collection + const parsedApiEntryMetadata = apiEntryMetadata.create( + apiDoc.stem, + // Creates a VFile for the current section content + new VFile({ ...apiDoc, value: sectionParsedContent }) + ); + + // We push the parsed API doc entry Metadata to the collection + metadataCollection.push(parsedApiEntryMetadata); + + // Finally, we remove the Heading from the tree as it's no longer needed + remove(tree, node); + }); return SKIP; }); }; }); - // Processes the API doc file and returns the parsed API doc - const parsedApiDoc = await apiDocProcessor.process(apiDoc); + console.log(`[parser] parsing: ${apiDoc.stem}`); - /** - * Splits the parsed API doc file into sections (a section is defined by when a heading is found) - * We consider up to level 4 headings as sections, and we split the file into sections - * - * @type {Array} - */ - const [, ...sections] = String(parsedApiDoc).split(DOC_API_ENTRY_SEPARATOR); + // Parses the API doc into an AST tree using `unified` and `remark` + const apiDocTree = apiDocProcessor.parse(apiDoc); - // We iterate the Markdown sections and retrieve the parsed Metadata entries from when Unified - // was iterating through the tree and then seal (create) the metadata entry for each section - return sections.map((section, index) => { - // This creates a new VFile with the top-level metadata of the API doc file - // and the trimmed content of the section (which excludes the header, since it is already within the Metadata) - const apiEntryFile = new VFile({ ...apiDoc, value: section.trim() }); + // Applies the AST transformers defined before to the API doc tree + await apiDocProcessor.run(apiDocTree); - // Seals the Metadata entry with the actual content of the section - return metadataCollection[index].create(apiDoc.stem, apiEntryFile); - }); + return metadataCollection; }; /** @@ -189,8 +233,11 @@ const createParser = () => { * * @param {Array} apiDocs List of API doc files to be parsed */ - const parseApiDocs = apiDocs => - Promise.all(apiDocs.map(parseApiDoc)).then(entries => entries.flat()); + const parseApiDocs = async apiDocs => { + const apiDocsParsedPromise = await Promise.all(apiDocs.map(parseApiDoc)); + + return apiDocsParsedPromise.flat(); + }; return { parseApiDocs }; }; diff --git a/src/utils/parser.mjs b/src/utils/parser.mjs index 6b7464be..41caf3f8 100644 --- a/src/utils/parser.mjs +++ b/src/utils/parser.mjs @@ -2,8 +2,6 @@ import yaml from 'yaml'; -import { pointEnd, pointStart } from 'unist-util-position'; - import { DOC_API_HEADING_TYPES, DOC_API_YAML_KEYS_ARRAYS, @@ -140,22 +138,3 @@ export const parseHeadingIntoMetadata = (heading, depth) => { return { text: heading, type: 'module', name: heading, depth }; }; - -/** - * This method is an utility that allows us to conditionally invoke/call a callback - * based on test conditions related to a Node's position relative to another one - * being before or not the other Node - * - * @param {import('unist').Node | undefined} nodeA The Node to be used as a position reference to check against - * the other Node. If the other Node is before this one, the callback will be called. - * @param {import('unist').Node | undefined} nodeB The Node to be checked against the position of the first Node - * @param {(nodeA: import('unist').Node, nodeB: import('unist').Node) => void} callback The callback to be called - */ -export const callIfBefore = (nodeA, nodeB, callback) => { - const positionA = pointEnd(nodeA); - const positionB = pointStart(nodeB); - - if (positionA && positionB && positionA.line > positionB.line) { - callback(nodeA, nodeB); - } -}; diff --git a/src/utils/unist.mjs b/src/utils/unist.mjs index ac41c4b7..82757ef9 100644 --- a/src/utils/unist.mjs +++ b/src/utils/unist.mjs @@ -1,10 +1,12 @@ 'use strict'; +import { pointEnd, pointStart } from 'unist-util-position'; + /** * This utility allows us to join children Nodes into one * and transfor them back to what their source would look like * - * @param {import('unist').Node} nodes Nodes to parsed and joined + * @param {Array} nodes Nodes to parsed and joined * @returns {string} The parsed and joined nodes as a string */ export const transformNodesToString = nodes => { @@ -30,3 +32,22 @@ export const transformNodesToString = nodes => { return mappedChildren.join(''); }; + +/** + * This method is an utility that allows us to conditionally invoke/call a callback + * based on test conditions related to a Node's position relative to another one + * being before or not the other Node + * + * @param {import('unist').Node | undefined} nodeA The Node to be used as a position reference to check against + * the other Node. If the other Node is before this one, the callback will be called. + * @param {import('unist').Node | undefined} nodeB The Node to be checked against the position of the first Node + * @param {(nodeA: import('unist').Node, nodeB: import('unist').Node) => void} callback The callback to be called + */ +export const callIfBefore = (nodeA, nodeB, callback) => { + const positionA = pointEnd(nodeA); + const positionB = pointStart(nodeB); + + if (positionA && positionB && positionA.line >= positionB.line) { + callback(nodeA, nodeB); + } +}; From a16f029f5e2ecfadfefcd437f00f25a1e7c0013d Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Mon, 8 Jul 2024 04:30:49 +0100 Subject: [PATCH 15/26] chore: remove console log --- src/parser.mjs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/parser.mjs b/src/parser.mjs index ded93a9d..75e5d1b9 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -216,8 +216,6 @@ const createParser = () => { }; }); - console.log(`[parser] parsing: ${apiDoc.stem}`); - // Parses the API doc into an AST tree using `unified` and `remark` const apiDocTree = apiDocProcessor.parse(apiDoc); From d46223867f7e4358abce9d4fd5a0856d1b2b44b5 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Mon, 8 Jul 2024 12:24:00 +0100 Subject: [PATCH 16/26] chore: final cleanup and optimization --- package-lock.json | 15 ---- package.json | 1 - src/loader.mjs | 12 ++-- src/parser.mjs | 165 ++++++++++++++++++++------------------------ src/utils/unist.mjs | 2 +- 5 files changed, 80 insertions(+), 115 deletions(-) diff --git a/package-lock.json b/package-lock.json index d9fca624..488f16ee 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,6 @@ "remark-gfm": "^4.0.0", "unist-builder": "^4.0.0", "unist-util-find-after": "^5.0.0", - "unist-util-find-all-after": "^5.0.0", "unist-util-position": "^5.0.0", "unist-util-remove": "^4.0.0", "unist-util-select": "^5.1.0", @@ -3003,20 +3002,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/unist-util-find-all-after": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/unist-util-find-all-after/-/unist-util-find-all-after-5.0.0.tgz", - "integrity": "sha512-nGmOYvTSdGcI4RvrUNfe0mOsqqbbJOtqCQsppsY9KZjmv3nwM3YRgNBwFPdZ8Y+iv9Z/2PDjR9u6u+uK62XTTg==", - "license": "MIT", - "dependencies": { - "@types/unist": "^3.0.0", - "unist-util-is": "^6.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, "node_modules/unist-util-is": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.0.tgz", diff --git a/package.json b/package.json index 538cf14f..6b10a4a7 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,6 @@ "remark-gfm": "^4.0.0", "unist-builder": "^4.0.0", "unist-util-find-after": "^5.0.0", - "unist-util-find-all-after": "^5.0.0", "unist-util-position": "^5.0.0", "unist-util-remove": "^4.0.0", "unist-util-select": "^5.1.0", diff --git a/src/loader.mjs b/src/loader.mjs index f71450f7..4490c6c7 100644 --- a/src/loader.mjs +++ b/src/loader.mjs @@ -1,9 +1,9 @@ 'use strict'; -import { readFileSync } from 'node:fs'; import { extname } from 'node:path'; +import { readFile } from 'node:fs/promises'; -import { glob } from 'glob'; +import { globSync } from 'glob'; import { VFile } from 'vfile'; /** @@ -21,14 +21,14 @@ const createLoader = () => { * * @see https://code.visualstudio.com/docs/editor/glob-patterns */ - const loadFiles = async path => { - const resolvedFiles = await glob(path); + const loadFiles = path => { + const resolvedFiles = globSync(path); - return resolvedFiles.map(filePath => { + return resolvedFiles.map(async filePath => { const fileExtension = extname(filePath); if (fileExtension === '.md') { - const fileContent = readFileSync(filePath, 'utf-8'); + const fileContent = await readFile(filePath, 'utf-8'); return new VFile({ path: filePath, value: fileContent }); } diff --git a/src/parser.mjs b/src/parser.mjs index 75e5d1b9..0654fe4e 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -10,13 +10,11 @@ import { remove } from 'unist-util-remove'; import { selectAll } from 'unist-util-select'; import { SKIP, visit } from 'unist-util-visit'; import { findAfter } from 'unist-util-find-after'; -import { findAllAfter } from 'unist-util-find-all-after'; import createMetadata from './metadata.mjs'; import createQueries from './queries.mjs'; import { createNodeSlugger } from './utils/slugger.mjs'; -import { callIfBefore } from './utils/unist.mjs'; /** * Creates an API doc parser for a given Markdown API doc file @@ -34,9 +32,13 @@ const createParser = () => { /** * Parses a given API doc metadata file into a list of Metadata entries * - * @param {import('vfile').VFile} apiDoc + * @param {import('vfile').VFile | Promise} apiDoc */ const parseApiDoc = async apiDoc => { + // We allow the API doc VFile to be a Promise of a VFile also, + // hence we want to ensure that it first resolves before we pass it to the parser + const resolvedApiDoc = await Promise.resolve(apiDoc); + /** * This holds references to all the Metadata entries for a given file * this is used so we can traverse the AST tree and keep mutating things @@ -114,110 +116,85 @@ const createParser = () => { * @param {import('unist').Parent} tree The root AST tree for the API doc file */ return tree => { - visit(tree, createQueries.UNIST.isHeadingNode, (node, index) => { + visit(tree, createQueries.UNIST.isHeadingNode, (headingNode, index) => { // Creates a new Metadata entry for the current API doc file const apiEntryMetadata = createMetadata(nodeSlugger); // Adds the Metadata of the current Heading Node to the Metadata entry - addHeadingMetadata(node, apiEntryMetadata); + addHeadingMetadata(headingNode, apiEntryMetadata); // We retrieve the immediate next Heading if it exists // This is used for ensuring that we don't include items that would // belong only to the next heading to the current Heading metadata // Note that if there is no next heading, we use the current node as the next one - const nextHeadingNode = findAfter(tree, index, 'heading') ?? node; - - callIfBefore(nextHeadingNode, node, next => { - // Gets the next available Stability Index Node (if any) - // from the current Heading Node, and then adds it to the current Metadata - // if the found Stability Index Node is before the next Heading Node - // we then add it to the current Metadata - const stabilityIndexNodes = findAllAfter( - tree, - node, - createQueries.UNIST.isStabilityIndex - ); - - // Gets the next available YAML Node (if any) - // from the current Heading Node, and then adds it to the current Metadata - // if the found YAML Node is before the next Heading Node - // we then add it to the current Metadata - const yamlMetadataNodes = findAllAfter( - tree, - node, - createQueries.UNIST.isYamlNode - ); - - stabilityIndexNodes.forEach(stabilityIndexNode => { - // If the next heading === current one, then we use the stabilityIndex - // itself as the check, since this means we reaching the end of the document - const toCheckAgainst = node === next ? stabilityIndexNode : next; - - callIfBefore(toCheckAgainst, stabilityIndexNode, () => { - // Adds the Stability Index Metadata to the current Metadata entry - addStabilityIndexMetadata(stabilityIndexNode, apiEntryMetadata); - - remove(tree, stabilityIndexNode); - }); - }); - - yamlMetadataNodes.forEach(yamlMetadataNode => { - // If the next heading === current one, then we use the yamlMetadataNode - // itself as the check, since this means we reaching the end of the document - const toCheckAgainst = node === next ? yamlMetadataNode : next; - - callIfBefore(toCheckAgainst, yamlMetadataNode, () => { - // Adds the YAML Metadata to the current Metadata entry - addYAMLMetadata(yamlMetadataNode, apiEntryMetadata); - - remove(tree, yamlMetadataNode); - }); - }); - - // This is the cutover index of the subtree that we should get - // of all the Nodes within the AST tree that belong to this section - // If `next` is equals the current heading, it means ther's no next heading - // and we are reaching the end of the document, hence the cutover should be the end of - // the document itself, - // Note.: This index needs to be retrieved after any modification to the tree occurs, - // otherwise the index will be off (out of sync with the tree) - const cutoverIndex = - node === next - ? tree.children.length - 1 - : tree.children.indexOf(next); - - // Retrieves all the Nodes that should belong to the current API doc section - const sectionNodes = tree.children.slice(index + 1, cutoverIndex); - - // The stringified (back to Markdown) content of the section - const sectionParsedContent = apiDocProcessor.stringify( - // Creates a subtree based on the section Nodes - u('root', sectionNodes), - apiDoc - ); - - // We seal and create the API doc entry Metadata and push them to the collection - // Creates the API doc entry Metadata and pushes it to the collection - const parsedApiEntryMetadata = apiEntryMetadata.create( - apiDoc.stem, - // Creates a VFile for the current section content - new VFile({ ...apiDoc, value: sectionParsedContent }) - ); - - // We push the parsed API doc entry Metadata to the collection - metadataCollection.push(parsedApiEntryMetadata); - - // Finally, we remove the Heading from the tree as it's no longer needed - remove(tree, node); + const nextHeadingNode = + findAfter(tree, index, createQueries.UNIST.isHeadingNode) ?? + headingNode; + + // This is the cutover index of the subtree that we should get + // of all the Nodes within the AST tree that belong to this section + // If `next` is equals the current heading, it means ther's no next heading + // and we are reaching the end of the document, hence the cutover should be the end of + // the document itself, + // Note.: This index needs to be retrieved after any modification to the tree occurs, + // otherwise the index will be off (out of sync with the tree) + const stop = + headingNode === nextHeadingNode + ? tree.children.length - 1 + : tree.children.indexOf(nextHeadingNode); + + // Retrieves all the Nodes that should belong to the current API doc section + // `index + 1` is used to skip the current Heading Node + const subtree = u('root', tree.children.slice(index + 1, stop)); + + // Visits all Stability Index Nodes from the current subtree if there's any + // and then apply the Stability Index Metadata to the current Metadata entry + visit(subtree, createQueries.UNIST.isStabilityIndex, node => { + // Adds the Stability Index Metadata to the current Metadata entry + addStabilityIndexMetadata(node, apiEntryMetadata); + + return SKIP; }); + // Visits all YAML Nodes from the current subtree if there's any + // and then apply the YAML Metadata to the current Metadata entry + visit(subtree, createQueries.UNIST.isYamlNode, node => { + // Adds the YAML Metadata to the current Metadata entry + addYAMLMetadata(node, apiEntryMetadata); + + return SKIP; + }); + + // Removes already parsed items from the subtree so that they aren't included in the final content + remove(subtree, [ + createQueries.UNIST.isStabilityIndex, + createQueries.UNIST.isYamlNode, + ]); + + // The stringified (back to Markdown) content of the section + const parsedSection = apiDocProcessor.stringify( + subtree, + resolvedApiDoc + ); + + // We seal and create the API doc entry Metadata and push them to the collection + // Creates the API doc entry Metadata and pushes it to the collection + const parsedApiEntryMetadata = apiEntryMetadata.create( + resolvedApiDoc.stem, + // Creates a VFile for the current section content + new VFile({ ...resolvedApiDoc, value: parsedSection }) + ); + + // We push the parsed API doc entry Metadata to the collection + metadataCollection.push(parsedApiEntryMetadata); + return SKIP; }); }; }); // Parses the API doc into an AST tree using `unified` and `remark` - const apiDocTree = apiDocProcessor.parse(apiDoc); + const apiDocTree = apiDocProcessor.parse(resolvedApiDoc); // Applies the AST transformers defined before to the API doc tree await apiDocProcessor.run(apiDocTree); @@ -229,15 +206,19 @@ const createParser = () => { * This method allows to parse multiple API doc files at once * and it simply wraps parseApiDoc with the given API docs * - * @param {Array} apiDocs List of API doc files to be parsed + * @param {Array>} apiDocs List of API doc files to be parsed */ const parseApiDocs = async apiDocs => { + // We do a Promise.all, to ensure that each API doc is resolved asynchronously + // but all need to be resolved first before we return the result to the caller const apiDocsParsedPromise = await Promise.all(apiDocs.map(parseApiDoc)); + // Since each individual API doc has a collection of Metadata entries, + // we flatten the resulting array to combine all entries into a single collection return apiDocsParsedPromise.flat(); }; - return { parseApiDocs }; + return { parseApiDocs, parseApiDoc }; }; export default createParser; diff --git a/src/utils/unist.mjs b/src/utils/unist.mjs index 82757ef9..d4e32134 100644 --- a/src/utils/unist.mjs +++ b/src/utils/unist.mjs @@ -47,7 +47,7 @@ export const callIfBefore = (nodeA, nodeB, callback) => { const positionA = pointEnd(nodeA); const positionB = pointStart(nodeB); - if (positionA && positionB && positionA.line >= positionB.line) { + if (positionA && positionB && positionA.line > positionB.line) { callback(nodeA, nodeB); } }; From edafe6fcf1ef0426a04045a2ed35f12d23631586 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Mon, 8 Jul 2024 19:01:18 +0100 Subject: [PATCH 17/26] chore: simplification of parser and separation of concerns --- src/loader.mjs | 4 +- src/metadata.mjs | 8 +- src/parser.mjs | 289 +++++++++++++++++++++++------------------------ 3 files changed, 148 insertions(+), 153 deletions(-) diff --git a/src/loader.mjs b/src/loader.mjs index 4490c6c7..83c3ae96 100644 --- a/src/loader.mjs +++ b/src/loader.mjs @@ -28,9 +28,9 @@ const createLoader = () => { const fileExtension = extname(filePath); if (fileExtension === '.md') { - const fileContent = await readFile(filePath, 'utf-8'); + const fileBuffer = await readFile(filePath); - return new VFile({ path: filePath, value: fileContent }); + return new VFile({ path: filePath, value: fileBuffer }); } throw new Error(`File ${filePath} is not a Markdown file`); diff --git a/src/metadata.mjs b/src/metadata.mjs index 2e9a4295..e95176dc 100644 --- a/src/metadata.mjs +++ b/src/metadata.mjs @@ -76,7 +76,9 @@ const createMetadata = slugger => { internalMetadata.heading.type = yaml_type || internalMetadata.heading.type; - const navigationEntry = { + // A metadata entry is all the metadata we have about a certain API section + // with the content being a VFile (Virtual File) containing the Markdown content + section.data = { // The API file basename (without the extension) api: yaml_name || apiDoc, // The path/slug of the API section @@ -93,10 +95,6 @@ const createMetadata = slugger => { stability: stability_index, }; - // A metadata entry is all the metadata we have about a certain API section - // with the content being a VFile (Virtual File) containing the Markdown content - section.data = navigationEntry; - // Returns the updated VFile with the extra metadata return section; }, diff --git a/src/parser.mjs b/src/parser.mjs index 0654fe4e..2cc4302e 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -16,10 +16,17 @@ import createQueries from './queries.mjs'; import { createNodeSlugger } from './utils/slugger.mjs'; +// Retrieves an unfrozen instance of the Remark processor with GFM support +const getRemarkProcessor = () => remark().use(remarkGfm); + /** * Creates an API doc parser for a given Markdown API doc file */ const createParser = () => { + // Creates an instance of the Remark processor with GFM support + // which is used for stringifying the AST tree back to Markdown + const defaultRemarkGfmProcessor = getRemarkProcessor(); + const { updateLinkReference, updateTypeToReferenceLink, @@ -29,16 +36,138 @@ const createParser = () => { addStabilityIndexMetadata, } = createQueries(); + /** + * This creates a Unified Plugin (Transformer) for parsing the source API doc file + * with numerous transformations into API doc Metadata entries. + * + * This transformer iterates on several sort of AST Nodes and applies transformations + * and then grabs a subtree of Nodes (groupped by a Heading Node) and then stringifies them into a VFile + * and finally creates a Metadata entry for the given section. + * + * @type {import('unified').Plugin<[{ + * apiDoc: import('vfile').VFile, + * metadatas: Array, + * slugger: ReturnType + * }]>} + */ + const apiDocTransformer = ({ apiDoc, metadatas, slugger }) => { + /** + * Iterates through the AST tree and creates Metadata entries for each API doc section + * + * @param {import('unist').Parent} tree The root AST tree for the API doc file + */ + return tree => { + // Get all Markdown Footnote definitions from the tree + const markdownDefinitions = selectAll('definition', tree); + + // Handles Link References + visit(tree, createQueries.UNIST.isLinkReference, node => { + updateLinkReference(node, markdownDefinitions); + + return SKIP; + }); + + // Removes all the original definitions from the tree as they are not needed + // anymore, since all link references got updated to be plain links + remove(tree, markdownDefinitions); + + // Handles API type references transformation into links + visit(tree, createQueries.UNIST.isTextWithType, node => { + updateTypeToReferenceLink(node); + + return SKIP; + }); + + // Handles normalisation of Markdown URLs + visit(tree, createQueries.UNIST.isMarkdownUrl, node => { + updateMarkdownLink(node); + + return SKIP; + }); + + visit(tree, createQueries.UNIST.isHeadingNode, (headingNode, index) => { + // Creates a new Metadata entry for the current API doc file + const apiEntryMetadata = createMetadata(slugger); + + // Adds the Metadata of the current Heading Node to the Metadata entry + addHeadingMetadata(headingNode, apiEntryMetadata); + + // We retrieve the immediate next Heading if it exists + // This is used for ensuring that we don't include items that would + // belong only to the next heading to the current Heading metadata + // Note that if there is no next heading, we use the current node as the next one + const nextHeadingNode = + findAfter(tree, index, createQueries.UNIST.isHeadingNode) ?? + headingNode; + + // This is the cutover index of the subtree that we should get + // of all the Nodes within the AST tree that belong to this section + // If `next` is equals the current heading, it means ther's no next heading + // and we are reaching the end of the document, hence the cutover should be the end of + // the document itself, + // Note.: This index needs to be retrieved after any modification to the tree occurs, + // otherwise the index will be off (out of sync with the tree) + const stop = + headingNode === nextHeadingNode + ? tree.children.length - 1 + : tree.children.indexOf(nextHeadingNode); + + // Retrieves all the Nodes that should belong to the current API doc section + // `index + 1` is used to skip the current Heading Node + const subtree = u('root', tree.children.slice(index + 1, stop)); + + // Visits all Stability Index Nodes from the current subtree if there's any + // and then apply the Stability Index Metadata to the current Metadata entry + visit(subtree, createQueries.UNIST.isStabilityIndex, node => { + // Adds the Stability Index Metadata to the current Metadata entry + addStabilityIndexMetadata(node, apiEntryMetadata); + + return SKIP; + }); + + // Visits all YAML Nodes from the current subtree if there's any + // and then apply the YAML Metadata to the current Metadata entry + visit(subtree, createQueries.UNIST.isYamlNode, node => { + // Adds the YAML Metadata to the current Metadata entry + addYAMLMetadata(node, apiEntryMetadata); + + return SKIP; + }); + + // Removes already parsed items from the subtree so that they aren't included in the final content + remove(subtree, [ + createQueries.UNIST.isStabilityIndex, + createQueries.UNIST.isYamlNode, + ]); + + // The stringified (back to Markdown) content of the section + const parsedSection = defaultRemarkGfmProcessor.stringify( + subtree, + apiDoc + ); + + // We seal and create the API doc entry Metadata and push them to the collection + // Creates the API doc entry Metadata and pushes it to the collection + const parsedApiEntryMetadata = apiEntryMetadata.create( + apiDoc.stem, + // Creates a VFile for the current section content + new VFile({ ...apiDoc, value: parsedSection }) + ); + + // We push the parsed API doc entry Metadata to the collection + metadatas.push(parsedApiEntryMetadata); + + return SKIP; + }); + }; + }; + /** * Parses a given API doc metadata file into a list of Metadata entries * * @param {import('vfile').VFile | Promise} apiDoc */ const parseApiDoc = async apiDoc => { - // We allow the API doc VFile to be a Promise of a VFile also, - // hence we want to ensure that it first resolves before we pass it to the parser - const resolvedApiDoc = await Promise.resolve(apiDoc); - /** * This holds references to all the Metadata entries for a given file * this is used so we can traverse the AST tree and keep mutating things @@ -51,146 +180,15 @@ const createParser = () => { */ const metadataCollection = []; - // Creates an instance of the Node slugger per API doc file - const nodeSlugger = createNodeSlugger(); + // We allow the API doc VFile to be a Promise of a VFile also, + // hence we want to ensure that it first resolves before we pass it to the parser + const resolvedApiDoc = await Promise.resolve(apiDoc); // Creates a new Remark processor with GFM (GitHub Flavoured Markdown) support - const apiDocProcessor = remark().use(remarkGfm); - - // Adds a transformer for handling link references and Markdown definitions - apiDocProcessor.use(() => { - /** - * This transformer is responsible for handling link references and Markdown definitions - * - * @param {import('unist').Parent} tree The root AST tree for the API doc file - */ - return tree => { - // Get all Markdown Footnote definitions from the tree - const markdownDefinitions = selectAll('definition', tree); - - // Handles Link References - visit(tree, createQueries.UNIST.isLinkReference, node => { - updateLinkReference(node, markdownDefinitions); - - return SKIP; - }); - - // Removes all the original definitions from the tree as they are not needed - // anymore, since all link references got updated to be plain links - remove(tree, markdownDefinitions); - }; - }); - - // Adds a transformer for normalising the Markdown file with numerous actions - // that are required for the API docs to be correctly parsed - apiDocProcessor.use(() => { - /** - * This transformer is responsible for normalising the Markdown file - * - * @param {import('unist').Parent} tree The root AST tree for the API doc file - */ - return tree => { - // Handles API type references transformation into links - visit(tree, createQueries.UNIST.isTextWithType, node => { - updateTypeToReferenceLink(node); - - return SKIP; - }); - - // Handles normalisation of Markdown URLs - visit(tree, createQueries.UNIST.isMarkdownUrl, node => { - updateMarkdownLink(node); - - return SKIP; - }); - }; - }); - - // Adds a transformer for iterating through the Markdown file - // and going over the sections and creating metadata entries by traversing the tree - // A new metadata entry is created once a new Heading is found (from level 1 to 6) - apiDocProcessor.use(() => { - /** - * Iterates through the AST tree and creates Metadata entries for each API doc section - * - * @param {import('unist').Parent} tree The root AST tree for the API doc file - */ - return tree => { - visit(tree, createQueries.UNIST.isHeadingNode, (headingNode, index) => { - // Creates a new Metadata entry for the current API doc file - const apiEntryMetadata = createMetadata(nodeSlugger); - - // Adds the Metadata of the current Heading Node to the Metadata entry - addHeadingMetadata(headingNode, apiEntryMetadata); - - // We retrieve the immediate next Heading if it exists - // This is used for ensuring that we don't include items that would - // belong only to the next heading to the current Heading metadata - // Note that if there is no next heading, we use the current node as the next one - const nextHeadingNode = - findAfter(tree, index, createQueries.UNIST.isHeadingNode) ?? - headingNode; - - // This is the cutover index of the subtree that we should get - // of all the Nodes within the AST tree that belong to this section - // If `next` is equals the current heading, it means ther's no next heading - // and we are reaching the end of the document, hence the cutover should be the end of - // the document itself, - // Note.: This index needs to be retrieved after any modification to the tree occurs, - // otherwise the index will be off (out of sync with the tree) - const stop = - headingNode === nextHeadingNode - ? tree.children.length - 1 - : tree.children.indexOf(nextHeadingNode); - - // Retrieves all the Nodes that should belong to the current API doc section - // `index + 1` is used to skip the current Heading Node - const subtree = u('root', tree.children.slice(index + 1, stop)); - - // Visits all Stability Index Nodes from the current subtree if there's any - // and then apply the Stability Index Metadata to the current Metadata entry - visit(subtree, createQueries.UNIST.isStabilityIndex, node => { - // Adds the Stability Index Metadata to the current Metadata entry - addStabilityIndexMetadata(node, apiEntryMetadata); - - return SKIP; - }); - - // Visits all YAML Nodes from the current subtree if there's any - // and then apply the YAML Metadata to the current Metadata entry - visit(subtree, createQueries.UNIST.isYamlNode, node => { - // Adds the YAML Metadata to the current Metadata entry - addYAMLMetadata(node, apiEntryMetadata); - - return SKIP; - }); - - // Removes already parsed items from the subtree so that they aren't included in the final content - remove(subtree, [ - createQueries.UNIST.isStabilityIndex, - createQueries.UNIST.isYamlNode, - ]); - - // The stringified (back to Markdown) content of the section - const parsedSection = apiDocProcessor.stringify( - subtree, - resolvedApiDoc - ); - - // We seal and create the API doc entry Metadata and push them to the collection - // Creates the API doc entry Metadata and pushes it to the collection - const parsedApiEntryMetadata = apiEntryMetadata.create( - resolvedApiDoc.stem, - // Creates a VFile for the current section content - new VFile({ ...resolvedApiDoc, value: parsedSection }) - ); - - // We push the parsed API doc entry Metadata to the collection - metadataCollection.push(parsedApiEntryMetadata); - - return SKIP; - }); - }; + const apiDocProcessor = getRemarkProcessor().use(apiDocTransformer, { + apiDoc: resolvedApiDoc, + metadatas: metadataCollection, + slugger: createNodeSlugger(), }); // Parses the API doc into an AST tree using `unified` and `remark` @@ -199,6 +197,7 @@ const createParser = () => { // Applies the AST transformers defined before to the API doc tree await apiDocProcessor.run(apiDocTree); + // Returns the Metadata entries for the given API doc file return metadataCollection; }; @@ -211,11 +210,9 @@ const createParser = () => { const parseApiDocs = async apiDocs => { // We do a Promise.all, to ensure that each API doc is resolved asynchronously // but all need to be resolved first before we return the result to the caller - const apiDocsParsedPromise = await Promise.all(apiDocs.map(parseApiDoc)); + const resolvedApiDocEntries = await Promise.all(apiDocs.map(parseApiDoc)); - // Since each individual API doc has a collection of Metadata entries, - // we flatten the resulting array to combine all entries into a single collection - return apiDocsParsedPromise.flat(); + return resolvedApiDocEntries.flat(); }; return { parseApiDocs, parseApiDoc }; From bf4004e3acd6cd441e0042f7fa70e080c8a2e785 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Mon, 8 Jul 2024 20:58:27 +0100 Subject: [PATCH 18/26] chore: updated docs --- src/parser.mjs | 2 +- src/utils/unist.mjs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/parser.mjs b/src/parser.mjs index 2cc4302e..ffd6c9ff 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -41,7 +41,7 @@ const createParser = () => { * with numerous transformations into API doc Metadata entries. * * This transformer iterates on several sort of AST Nodes and applies transformations - * and then grabs a subtree of Nodes (groupped by a Heading Node) and then stringifies them into a VFile + * and then grabs a subtree of Nodes (grouped by a Heading Node) and then stringifies them into a VFile * and finally creates a Metadata entry for the given section. * * @type {import('unified').Plugin<[{ diff --git a/src/utils/unist.mjs b/src/utils/unist.mjs index d4e32134..bb7ed491 100644 --- a/src/utils/unist.mjs +++ b/src/utils/unist.mjs @@ -38,6 +38,8 @@ export const transformNodesToString = nodes => { * based on test conditions related to a Node's position relative to another one * being before or not the other Node * + * NOTE: Not yet used, but probably going to be used by the JSON generator. + * * @param {import('unist').Node | undefined} nodeA The Node to be used as a position reference to check against * the other Node. If the other Node is before this one, the callback will be called. * @param {import('unist').Node | undefined} nodeB The Node to be checked against the position of the first Node From 19c4a01e12bd158b9937c16ee1cd96dd185363b0 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Tue, 9 Jul 2024 23:12:29 +0200 Subject: [PATCH 19/26] chore: code review changes --- src/loader.mjs | 18 +++++++----------- src/metadata.mjs | 2 +- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/loader.mjs b/src/loader.mjs index 83c3ae96..b7c32a82 100644 --- a/src/loader.mjs +++ b/src/loader.mjs @@ -15,25 +15,21 @@ const createLoader = () => { /** * Loads API Doc files and transforms it into VFiles * - * @param {string} path A glob/path for API docs to be loaded + * @param {string} searchPath A glob/path for API docs to be loaded * The input string can be a simple path (relative or absolute) * The input string can also be any allowed glob string * * @see https://code.visualstudio.com/docs/editor/glob-patterns */ - const loadFiles = path => { - const resolvedFiles = globSync(path); + const loadFiles = searchPath => { + const resolvedFiles = globSync(searchPath).filter( + filePath => extname(filePath) === '.md' + ); return resolvedFiles.map(async filePath => { - const fileExtension = extname(filePath); + const fileBuffer = await readFile(filePath); - if (fileExtension === '.md') { - const fileBuffer = await readFile(filePath); - - return new VFile({ path: filePath, value: fileBuffer }); - } - - throw new Error(`File ${filePath} is not a Markdown file`); + return new VFile({ path: filePath, value: fileBuffer }); }); }; diff --git a/src/metadata.mjs b/src/metadata.mjs index e95176dc..8a8f8ff7 100644 --- a/src/metadata.mjs +++ b/src/metadata.mjs @@ -33,7 +33,7 @@ const createMetadata = slugger => { }, /** * Set the Metadata (from YAML if exists) properties to the current Metadata entry - * itI also allows for extra data (such as Stability Index) and miscellaneous data to be set + * it also allows for extra data (such as Stability Index) and miscellaneous data to be set * although it'd be best to only set ones from {ApiDocRawMetadataEntry} * * @param {Partial} properties Extra Metadata properties to be defined From 410770e5e9c07278345c4960e5c2050ec83d7cca Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Wed, 10 Jul 2024 00:15:44 +0200 Subject: [PATCH 20/26] refactor: simplified the parsing of the tree without using a transformer --- src/metadata.mjs | 29 ++++-- src/parser.mjs | 243 ++++++++++++++++++++--------------------------- src/types.d.ts | 27 +++--- 3 files changed, 137 insertions(+), 162 deletions(-) diff --git a/src/metadata.mjs b/src/metadata.mjs index 8a8f8ff7..558921a9 100644 --- a/src/metadata.mjs +++ b/src/metadata.mjs @@ -8,8 +8,9 @@ * to many files to create a full Navigation for a given version of the API * * @param {InstanceType} slugger A GitHub Slugger + * @param {ReturnType} remarkProcessor A Remark processor */ -const createMetadata = slugger => { +const createMetadata = (slugger, remarkProcessor) => { // This holds a temporary buffer of raw metadata before being // transformed into NavigationEntries and MetadataEntries const internalMetadata = { @@ -53,7 +54,7 @@ const createMetadata = slugger => { * as it can be manipulated outside of the scope of the generation of the content * * @param {string} apiDoc The name of the API doc - * @param {import('vfile').VFile} section The content of the current Metadata entry + * @param {import('unist').Parent} section An AST tree containing the Nodes of the API doc entry section * @returns {import('./types').ApiDocMetadataEntry} The locally created Metadata entries */ create: (apiDoc, section) => { @@ -76,16 +77,14 @@ const createMetadata = slugger => { internalMetadata.heading.type = yaml_type || internalMetadata.heading.type; - // A metadata entry is all the metadata we have about a certain API section - // with the content being a VFile (Virtual File) containing the Markdown content - section.data = { + const apiEntryMetadata = { // The API file basename (without the extension) api: yaml_name || apiDoc, // The path/slug of the API section slug: `${apiDoc}.html${slugHash}`, // The source link of said API section sourceLink: source_link, - // The latest update to an API section + // The latest updates to an API section updates, // The full-changeset to an API section changes, @@ -93,10 +92,24 @@ const createMetadata = slugger => { heading: internalMetadata.heading, // The Stability Index of the API section stability: stability_index, + // The AST tree of the API section + content: section, }; - // Returns the updated VFile with the extra metadata - return section; + // Returns the Metadata entry for the API doc + return { + // Appends the base Metadata entry + ...apiEntryMetadata, + + // Overrides the toJSON method to allow for custom serialization + toJSON: () => ({ + ...apiEntryMetadata, + + // We stringify the AST tree to a string + // since this is what we wanbt to render within a JSON object + content: remarkProcessor.stringify(section), + }), + }; }, }; }; diff --git a/src/parser.mjs b/src/parser.mjs index ffd6c9ff..40b062b1 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -1,7 +1,5 @@ 'use strict'; -import { VFile } from 'vfile'; - import { remark } from 'remark'; import remarkGfm from 'remark-gfm'; @@ -16,16 +14,13 @@ import createQueries from './queries.mjs'; import { createNodeSlugger } from './utils/slugger.mjs'; -// Retrieves an unfrozen instance of the Remark processor with GFM support -const getRemarkProcessor = () => remark().use(remarkGfm); - /** * Creates an API doc parser for a given Markdown API doc file */ const createParser = () => { // Creates an instance of the Remark processor with GFM support // which is used for stringifying the AST tree back to Markdown - const defaultRemarkGfmProcessor = getRemarkProcessor(); + const remarkGfmProcessor = remark().use(remarkGfm); const { updateLinkReference, @@ -36,132 +31,6 @@ const createParser = () => { addStabilityIndexMetadata, } = createQueries(); - /** - * This creates a Unified Plugin (Transformer) for parsing the source API doc file - * with numerous transformations into API doc Metadata entries. - * - * This transformer iterates on several sort of AST Nodes and applies transformations - * and then grabs a subtree of Nodes (grouped by a Heading Node) and then stringifies them into a VFile - * and finally creates a Metadata entry for the given section. - * - * @type {import('unified').Plugin<[{ - * apiDoc: import('vfile').VFile, - * metadatas: Array, - * slugger: ReturnType - * }]>} - */ - const apiDocTransformer = ({ apiDoc, metadatas, slugger }) => { - /** - * Iterates through the AST tree and creates Metadata entries for each API doc section - * - * @param {import('unist').Parent} tree The root AST tree for the API doc file - */ - return tree => { - // Get all Markdown Footnote definitions from the tree - const markdownDefinitions = selectAll('definition', tree); - - // Handles Link References - visit(tree, createQueries.UNIST.isLinkReference, node => { - updateLinkReference(node, markdownDefinitions); - - return SKIP; - }); - - // Removes all the original definitions from the tree as they are not needed - // anymore, since all link references got updated to be plain links - remove(tree, markdownDefinitions); - - // Handles API type references transformation into links - visit(tree, createQueries.UNIST.isTextWithType, node => { - updateTypeToReferenceLink(node); - - return SKIP; - }); - - // Handles normalisation of Markdown URLs - visit(tree, createQueries.UNIST.isMarkdownUrl, node => { - updateMarkdownLink(node); - - return SKIP; - }); - - visit(tree, createQueries.UNIST.isHeadingNode, (headingNode, index) => { - // Creates a new Metadata entry for the current API doc file - const apiEntryMetadata = createMetadata(slugger); - - // Adds the Metadata of the current Heading Node to the Metadata entry - addHeadingMetadata(headingNode, apiEntryMetadata); - - // We retrieve the immediate next Heading if it exists - // This is used for ensuring that we don't include items that would - // belong only to the next heading to the current Heading metadata - // Note that if there is no next heading, we use the current node as the next one - const nextHeadingNode = - findAfter(tree, index, createQueries.UNIST.isHeadingNode) ?? - headingNode; - - // This is the cutover index of the subtree that we should get - // of all the Nodes within the AST tree that belong to this section - // If `next` is equals the current heading, it means ther's no next heading - // and we are reaching the end of the document, hence the cutover should be the end of - // the document itself, - // Note.: This index needs to be retrieved after any modification to the tree occurs, - // otherwise the index will be off (out of sync with the tree) - const stop = - headingNode === nextHeadingNode - ? tree.children.length - 1 - : tree.children.indexOf(nextHeadingNode); - - // Retrieves all the Nodes that should belong to the current API doc section - // `index + 1` is used to skip the current Heading Node - const subtree = u('root', tree.children.slice(index + 1, stop)); - - // Visits all Stability Index Nodes from the current subtree if there's any - // and then apply the Stability Index Metadata to the current Metadata entry - visit(subtree, createQueries.UNIST.isStabilityIndex, node => { - // Adds the Stability Index Metadata to the current Metadata entry - addStabilityIndexMetadata(node, apiEntryMetadata); - - return SKIP; - }); - - // Visits all YAML Nodes from the current subtree if there's any - // and then apply the YAML Metadata to the current Metadata entry - visit(subtree, createQueries.UNIST.isYamlNode, node => { - // Adds the YAML Metadata to the current Metadata entry - addYAMLMetadata(node, apiEntryMetadata); - - return SKIP; - }); - - // Removes already parsed items from the subtree so that they aren't included in the final content - remove(subtree, [ - createQueries.UNIST.isStabilityIndex, - createQueries.UNIST.isYamlNode, - ]); - - // The stringified (back to Markdown) content of the section - const parsedSection = defaultRemarkGfmProcessor.stringify( - subtree, - apiDoc - ); - - // We seal and create the API doc entry Metadata and push them to the collection - // Creates the API doc entry Metadata and pushes it to the collection - const parsedApiEntryMetadata = apiEntryMetadata.create( - apiDoc.stem, - // Creates a VFile for the current section content - new VFile({ ...apiDoc, value: parsedSection }) - ); - - // We push the parsed API doc entry Metadata to the collection - metadatas.push(parsedApiEntryMetadata); - - return SKIP; - }); - }; - }; - /** * Parses a given API doc metadata file into a list of Metadata entries * @@ -184,18 +53,110 @@ const createParser = () => { // hence we want to ensure that it first resolves before we pass it to the parser const resolvedApiDoc = await Promise.resolve(apiDoc); - // Creates a new Remark processor with GFM (GitHub Flavoured Markdown) support - const apiDocProcessor = getRemarkProcessor().use(apiDocTransformer, { - apiDoc: resolvedApiDoc, - metadatas: metadataCollection, - slugger: createNodeSlugger(), - }); + // Creates a new Slugger instance for the current API doc file + const nodeSlugger = createNodeSlugger(); // Parses the API doc into an AST tree using `unified` and `remark` - const apiDocTree = apiDocProcessor.parse(resolvedApiDoc); + const tree = remarkGfmProcessor.parse(resolvedApiDoc); + + // Get all Markdown Footnote definitions from the tree + const markdownDefinitions = selectAll('definition', tree); + + // Handles Link References + visit(tree, createQueries.UNIST.isLinkReference, node => { + updateLinkReference(node, markdownDefinitions); + + return SKIP; + }); + + // Removes all the original definitions from the tree as they are not needed + // anymore, since all link references got updated to be plain links + remove(tree, markdownDefinitions); + + // Handles API type references transformation into links + visit(tree, createQueries.UNIST.isTextWithType, node => { + updateTypeToReferenceLink(node); + + return SKIP; + }); - // Applies the AST transformers defined before to the API doc tree - await apiDocProcessor.run(apiDocTree); + // Handles normalisation of Markdown URLs + visit(tree, createQueries.UNIST.isMarkdownUrl, node => { + updateMarkdownLink(node); + + return SKIP; + }); + + visit(tree, createQueries.UNIST.isHeadingNode, (headingNode, index) => { + // Creates a new Metadata entry for the current API doc file + const apiEntryMetadata = createMetadata(nodeSlugger, remarkGfmProcessor); + + // Adds the Metadata of the current Heading Node to the Metadata entry + addHeadingMetadata(headingNode, apiEntryMetadata); + + // We retrieve the immediate next Heading if it exists + // This is used for ensuring that we don't include items that would + // belong only to the next heading to the current Heading metadata + // Note that if there is no next heading, we use the current node as the next one + const nextHeadingNode = + findAfter(tree, index, createQueries.UNIST.isHeadingNode) ?? + headingNode; + + // This is the cutover index of the subtree that we should get + // of all the Nodes within the AST tree that belong to this section + // If `next` is equals the current heading, it means ther's no next heading + // and we are reaching the end of the document, hence the cutover should be the end of + // the document itself, + // Note.: This index needs to be retrieved after any modification to the tree occurs, + // otherwise the index will be off (out of sync with the tree) + const stop = + headingNode === nextHeadingNode + ? tree.children.length - 1 + : tree.children.indexOf(nextHeadingNode); + + // Retrieves all the Nodes that should belong to the current API doc section + // `index + 1` is used to skip the current Heading Node + const subtree = u('root', tree.children.slice(index + 1, stop)); + + // Visits all Stability Index Nodes from the current subtree if there's any + // and then apply the Stability Index Metadata to the current Metadata entry + visit(subtree, createQueries.UNIST.isStabilityIndex, node => { + // Adds the Stability Index Metadata to the current Metadata entry + addStabilityIndexMetadata(node, apiEntryMetadata); + + return SKIP; + }); + + // Visits all YAML Nodes from the current subtree if there's any + // and then apply the YAML Metadata to the current Metadata entry + visit(subtree, createQueries.UNIST.isYamlNode, node => { + // Adds the YAML Metadata to the current Metadata entry + addYAMLMetadata(node, apiEntryMetadata); + + return SKIP; + }); + + // Removes already parsed items from the subtree so that they aren't included in the final content + remove(subtree, [ + createQueries.UNIST.isStabilityIndex, + createQueries.UNIST.isYamlNode, + ]); + + // We seal and create the API doc entry Metadata and push them to the collection + // Creates the API doc entry Metadata and pushes it to the collection + const parsedApiEntryMetadata = apiEntryMetadata.create( + resolvedApiDoc.stem, + // Applies the AST transformations to the subtree based on the API doc entry Metadata + // Note that running the transformation on the subtree isn't costly as it is a reduced tree + // and the GFM transformations aren't that heavy + remarkGfmProcessor.runSync(subtree) + ); + + // We push the parsed API doc entry Metadata to the collection + metadataCollection.push(parsedApiEntryMetadata); + + return SKIP; + }); // Returns the Metadata entries for the given API doc file return metadataCollection; diff --git a/src/types.d.ts b/src/types.d.ts index c43b0e9c..c5ff16cf 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -1,4 +1,4 @@ -import { VFile } from 'vfile'; +import { Parent } from 'unist'; export interface StabilityIndexMetadataEntry { index: number; @@ -29,7 +29,7 @@ export interface ApiDocMetadataChange { } export interface ApiDocMetadataUpdate { - // The type of the API Doc Metadata update + // The type of the API doc Metadata update type: 'added' | 'removed' | 'deprecated' | 'introduced_in' | 'napiVersion'; // The Node.js version or versions where said metadata stability index changed version: string[]; @@ -44,23 +44,24 @@ export interface ApiDocRawMetadataEntry { stability_index?: StabilityIndexMetadataEntry; } -export interface ApiDocNavigationEntry { - // The name of the API Doc file without the file extension (basename) +export interface ApiDocMetadataEntry { + // The name of the API doc file without the file extension (basename) api: string; - // The unique slug of a Heading/Navigation Entry which is linkable through an anchor + // The unique slug of a Heading/Navigation entry which is linkable through an anchor slug: string; - // The GitHub URL to the source of the API Entry + // The GitHub URL to the source of the API entry sourceLink: string | undefined; - // Any updates to the API Doc Metadata + // Any updates to the API doc Metadata updates: ApiDocMetadataUpdate[]; - // Any changes to the API Doc Metadata + // Any changes to the API doc Metadata changes: ApiDocMetadataChange[]; // The parsed Markdown content of a Navigation Entry heading: HeadingMetadataEntry; - // The API Doc Metadata Entry Stability Index if exists + // The API doc metadata Entry Stability Index if exists stability: StabilityIndexMetadataEntry | undefined; -} - -export interface ApiDocMetadataEntry extends VFile { - data: ApiDocNavigationEntry; + // The subtree containing all Nodes of the API doc entry + content: Parent; + // String serialization of the AST tree + // @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify#tojson_behavior + toJSON: () => string; } From 1b35124286a5fd0e2179ab4a651fdc359f4bbbf6 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Wed, 10 Jul 2024 00:19:26 +0200 Subject: [PATCH 21/26] chore: renamed variables for simplicity --- src/parser.mjs | 35 +++++++++++++++++++---------------- src/queries.mjs | 2 +- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/parser.mjs b/src/parser.mjs index 40b062b1..515537df 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -3,7 +3,7 @@ import { remark } from 'remark'; import remarkGfm from 'remark-gfm'; -import { u } from 'unist-builder'; +import { u as createTree } from 'unist-builder'; import { remove } from 'unist-util-remove'; import { selectAll } from 'unist-util-select'; import { SKIP, visit } from 'unist-util-visit'; @@ -57,13 +57,13 @@ const createParser = () => { const nodeSlugger = createNodeSlugger(); // Parses the API doc into an AST tree using `unified` and `remark` - const tree = remarkGfmProcessor.parse(resolvedApiDoc); + const apiDocTree = remarkGfmProcessor.parse(resolvedApiDoc); // Get all Markdown Footnote definitions from the tree - const markdownDefinitions = selectAll('definition', tree); + const markdownDefinitions = selectAll('definition', apiDocTree); // Handles Link References - visit(tree, createQueries.UNIST.isLinkReference, node => { + visit(apiDocTree, createQueries.UNIST.isLinkReference, node => { updateLinkReference(node, markdownDefinitions); return SKIP; @@ -71,23 +71,23 @@ const createParser = () => { // Removes all the original definitions from the tree as they are not needed // anymore, since all link references got updated to be plain links - remove(tree, markdownDefinitions); + remove(apiDocTree, markdownDefinitions); // Handles API type references transformation into links - visit(tree, createQueries.UNIST.isTextWithType, node => { + visit(apiDocTree, createQueries.UNIST.isTextWithType, node => { updateTypeToReferenceLink(node); return SKIP; }); // Handles normalisation of Markdown URLs - visit(tree, createQueries.UNIST.isMarkdownUrl, node => { + visit(apiDocTree, createQueries.UNIST.isMarkdownUrl, node => { updateMarkdownLink(node); return SKIP; }); - visit(tree, createQueries.UNIST.isHeadingNode, (headingNode, index) => { + visit(apiDocTree, createQueries.UNIST.isHeading, (headingNode, index) => { // Creates a new Metadata entry for the current API doc file const apiEntryMetadata = createMetadata(nodeSlugger, remarkGfmProcessor); @@ -99,7 +99,7 @@ const createParser = () => { // belong only to the next heading to the current Heading metadata // Note that if there is no next heading, we use the current node as the next one const nextHeadingNode = - findAfter(tree, index, createQueries.UNIST.isHeadingNode) ?? + findAfter(apiDocTree, index, createQueries.UNIST.isHeading) ?? headingNode; // This is the cutover index of the subtree that we should get @@ -111,16 +111,19 @@ const createParser = () => { // otherwise the index will be off (out of sync with the tree) const stop = headingNode === nextHeadingNode - ? tree.children.length - 1 - : tree.children.indexOf(nextHeadingNode); + ? apiDocTree.children.length - 1 + : apiDocTree.children.indexOf(nextHeadingNode); // Retrieves all the Nodes that should belong to the current API doc section // `index + 1` is used to skip the current Heading Node - const subtree = u('root', tree.children.slice(index + 1, stop)); + const apiSectionTree = createTree( + 'root', + apiDocTree.children.slice(index + 1, stop) + ); // Visits all Stability Index Nodes from the current subtree if there's any // and then apply the Stability Index Metadata to the current Metadata entry - visit(subtree, createQueries.UNIST.isStabilityIndex, node => { + visit(apiSectionTree, createQueries.UNIST.isStabilityIndex, node => { // Adds the Stability Index Metadata to the current Metadata entry addStabilityIndexMetadata(node, apiEntryMetadata); @@ -129,7 +132,7 @@ const createParser = () => { // Visits all YAML Nodes from the current subtree if there's any // and then apply the YAML Metadata to the current Metadata entry - visit(subtree, createQueries.UNIST.isYamlNode, node => { + visit(apiSectionTree, createQueries.UNIST.isYamlNode, node => { // Adds the YAML Metadata to the current Metadata entry addYAMLMetadata(node, apiEntryMetadata); @@ -137,7 +140,7 @@ const createParser = () => { }); // Removes already parsed items from the subtree so that they aren't included in the final content - remove(subtree, [ + remove(apiSectionTree, [ createQueries.UNIST.isStabilityIndex, createQueries.UNIST.isYamlNode, ]); @@ -149,7 +152,7 @@ const createParser = () => { // Applies the AST transformations to the subtree based on the API doc entry Metadata // Note that running the transformation on the subtree isn't costly as it is a reduced tree // and the GFM transformations aren't that heavy - remarkGfmProcessor.runSync(subtree) + remarkGfmProcessor.runSync(apiSectionTree) ); // We push the parsed API doc entry Metadata to the collection diff --git a/src/queries.mjs b/src/queries.mjs index 26f36964..28c486ff 100644 --- a/src/queries.mjs +++ b/src/queries.mjs @@ -141,7 +141,7 @@ createQueries.UNIST = { type === 'text' && createQueries.QUERIES.normalizeTypes.test(value), isMarkdownUrl: ({ type, url }) => type === 'link' && createQueries.QUERIES.markdownUrl.test(url), - isHeadingNode: ({ type, depth }) => + isHeading: ({ type, depth }) => type === 'heading' && depth >= 1 && depth <= 4, isLinkReference: ({ type, identifier }) => type === 'linkReference' && !!identifier, From 44f912290fc17b04963abb22d84ca49bcf0f4efd Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Wed, 10 Jul 2024 00:27:57 +0200 Subject: [PATCH 22/26] chore: pass apidoc itself --- src/metadata.mjs | 6 +++--- src/parser.mjs | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/metadata.mjs b/src/metadata.mjs index 558921a9..9f3f5649 100644 --- a/src/metadata.mjs +++ b/src/metadata.mjs @@ -53,7 +53,7 @@ const createMetadata = (slugger, remarkProcessor) => { * The Navigation entries has a dedicated separate method for retrieval * as it can be manipulated outside of the scope of the generation of the content * - * @param {string} apiDoc The name of the API doc + * @param {import('vfile').VFile} apiDoc The API doc file being parsed * @param {import('unist').Parent} section An AST tree containing the Nodes of the API doc entry section * @returns {import('./types').ApiDocMetadataEntry} The locally created Metadata entries */ @@ -79,9 +79,9 @@ const createMetadata = (slugger, remarkProcessor) => { const apiEntryMetadata = { // The API file basename (without the extension) - api: yaml_name || apiDoc, + api: yaml_name || apiDoc.stem, // The path/slug of the API section - slug: `${apiDoc}.html${slugHash}`, + slug: `${apiDoc.stem}.html${slugHash}`, // The source link of said API section sourceLink: source_link, // The latest updates to an API section diff --git a/src/parser.mjs b/src/parser.mjs index 515537df..267c8298 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -20,7 +20,7 @@ import { createNodeSlugger } from './utils/slugger.mjs'; const createParser = () => { // Creates an instance of the Remark processor with GFM support // which is used for stringifying the AST tree back to Markdown - const remarkGfmProcessor = remark().use(remarkGfm); + const remarkProcessor = remark().use(remarkGfm); const { updateLinkReference, @@ -57,7 +57,7 @@ const createParser = () => { const nodeSlugger = createNodeSlugger(); // Parses the API doc into an AST tree using `unified` and `remark` - const apiDocTree = remarkGfmProcessor.parse(resolvedApiDoc); + const apiDocTree = remarkProcessor.parse(resolvedApiDoc); // Get all Markdown Footnote definitions from the tree const markdownDefinitions = selectAll('definition', apiDocTree); @@ -89,7 +89,7 @@ const createParser = () => { visit(apiDocTree, createQueries.UNIST.isHeading, (headingNode, index) => { // Creates a new Metadata entry for the current API doc file - const apiEntryMetadata = createMetadata(nodeSlugger, remarkGfmProcessor); + const apiEntryMetadata = createMetadata(nodeSlugger, remarkProcessor); // Adds the Metadata of the current Heading Node to the Metadata entry addHeadingMetadata(headingNode, apiEntryMetadata); @@ -148,11 +148,11 @@ const createParser = () => { // We seal and create the API doc entry Metadata and push them to the collection // Creates the API doc entry Metadata and pushes it to the collection const parsedApiEntryMetadata = apiEntryMetadata.create( - resolvedApiDoc.stem, + resolvedApiDoc, // Applies the AST transformations to the subtree based on the API doc entry Metadata // Note that running the transformation on the subtree isn't costly as it is a reduced tree // and the GFM transformations aren't that heavy - remarkGfmProcessor.runSync(apiSectionTree) + remarkProcessor.runSync(apiSectionTree) ); // We push the parsed API doc entry Metadata to the collection From b3ff2964b8e0ecdf10647b17bee73af86691e8fb Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Wed, 10 Jul 2024 00:36:32 +0200 Subject: [PATCH 23/26] chore: more doc updates --- src/parser.mjs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/parser.mjs b/src/parser.mjs index 267c8298..f7e97f4c 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -73,20 +73,26 @@ const createParser = () => { // anymore, since all link references got updated to be plain links remove(apiDocTree, markdownDefinitions); - // Handles API type references transformation into links + // Handles API type references transformation into links that point + // to the reference to a said API type visit(apiDocTree, createQueries.UNIST.isTextWithType, node => { updateTypeToReferenceLink(node); return SKIP; }); - // Handles normalisation of Markdown URLs + // Handles normalisation URLs that reference to API doc files with .md extension + // to replace the .md into .html, since the API doc files get eventually compiled as HTML visit(apiDocTree, createQueries.UNIST.isMarkdownUrl, node => { updateMarkdownLink(node); return SKIP; }); + // Handles iterating the tree and creating subtrees for each API doc entry + // where an API doc entry is defined by a Heading Node (so all elements after a Heading until the next Heading) + // and then it creates and updates a Metadata entry for each API doc entry + // and then generates the final content for each API doc entry and pushes it to the collection visit(apiDocTree, createQueries.UNIST.isHeading, (headingNode, index) => { // Creates a new Metadata entry for the current API doc file const apiEntryMetadata = createMetadata(nodeSlugger, remarkProcessor); @@ -146,7 +152,6 @@ const createParser = () => { ]); // We seal and create the API doc entry Metadata and push them to the collection - // Creates the API doc entry Metadata and pushes it to the collection const parsedApiEntryMetadata = apiEntryMetadata.create( resolvedApiDoc, // Applies the AST transformations to the subtree based on the API doc entry Metadata From ba96883b30d9ca5877b567b9369a3d7613fd1c8a Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Wed, 10 Jul 2024 00:37:29 +0200 Subject: [PATCH 24/26] chore: more doc updates --- src/parser.mjs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/parser.mjs b/src/parser.mjs index f7e97f4c..826977bf 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -90,7 +90,8 @@ const createParser = () => { }); // Handles iterating the tree and creating subtrees for each API doc entry - // where an API doc entry is defined by a Heading Node (so all elements after a Heading until the next Heading) + // where an API doc entry is defined by a Heading Node + // (so all elements after a Heading until the next Heading) // and then it creates and updates a Metadata entry for each API doc entry // and then generates the final content for each API doc entry and pushes it to the collection visit(apiDocTree, createQueries.UNIST.isHeading, (headingNode, index) => { @@ -112,9 +113,7 @@ const createParser = () => { // of all the Nodes within the AST tree that belong to this section // If `next` is equals the current heading, it means ther's no next heading // and we are reaching the end of the document, hence the cutover should be the end of - // the document itself, - // Note.: This index needs to be retrieved after any modification to the tree occurs, - // otherwise the index will be off (out of sync with the tree) + // the document itself. const stop = headingNode === nextHeadingNode ? apiDocTree.children.length - 1 From 77f98848f9c8485186a36a380fcd56bfa699b25d Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Thu, 11 Jul 2024 13:06:10 +0200 Subject: [PATCH 25/26] refactor: applied cdoe reviews and stability index as Parent --- src/metadata.mjs | 71 +++++++++++++++++++++----------------------- src/parser.mjs | 22 ++++++++++---- src/queries.mjs | 40 ++++++++++++++----------- src/types.d.ts | 16 +++++----- src/utils/parser.mjs | 1 + src/utils/unist.mjs | 27 +++++++---------- 6 files changed, 94 insertions(+), 83 deletions(-) diff --git a/src/metadata.mjs b/src/metadata.mjs index 9f3f5649..6b14fd7a 100644 --- a/src/metadata.mjs +++ b/src/metadata.mjs @@ -8,11 +8,18 @@ * to many files to create a full Navigation for a given version of the API * * @param {InstanceType} slugger A GitHub Slugger - * @param {ReturnType} remarkProcessor A Remark processor */ -const createMetadata = (slugger, remarkProcessor) => { - // This holds a temporary buffer of raw metadata before being - // transformed into NavigationEntries and MetadataEntries +const createMetadata = slugger => { + /** + * This holds a temporary buffer of raw metadata before being + * transformed into NavigationEntries and MetadataEntries + * + * @type {{ + * heading: import('./types.d.ts').HeadingMetadataEntry, + * stability: import('./types.d.ts').ApiDocMetadataEntry['stability'], + * properties: import('./types.d.ts').ApiDocRawMetadataEntry, + * }} + */ const internalMetadata = { heading: { text: undefined, @@ -20,6 +27,7 @@ const createMetadata = (slugger, remarkProcessor) => { name: undefined, depth: -1, }, + stability: undefined, properties: {}, }; @@ -32,18 +40,23 @@ const createMetadata = (slugger, remarkProcessor) => { setHeading: heading => { internalMetadata.heading = heading; }, + /** + * Set the Stability Index of a given Metadata + * + * @param {import('./types.d.ts').ApiDocMetadataEntry['stability']} stability The new stability metadata + */ + setStability: stability => { + internalMetadata.stability = stability; + }, /** * Set the Metadata (from YAML if exists) properties to the current Metadata entry * it also allows for extra data (such as Stability Index) and miscellaneous data to be set * although it'd be best to only set ones from {ApiDocRawMetadataEntry} * - * @param {Partial} properties Extra Metadata properties to be defined + * @param {import('./types.d.ts').ApiDocRawMetadataEntry} properties Extra Metadata properties to be defined */ - updateProperties: properties => { - internalMetadata.properties = { - ...internalMetadata.properties, - ...properties, - }; + setProperties: properties => { + internalMetadata.properties = properties; }, /** * Generates a new Navigation entry and pushes them to the internal collection @@ -54,8 +67,8 @@ const createMetadata = (slugger, remarkProcessor) => { * as it can be manipulated outside of the scope of the generation of the content * * @param {import('vfile').VFile} apiDoc The API doc file being parsed - * @param {import('unist').Parent} section An AST tree containing the Nodes of the API doc entry section - * @returns {import('./types').ApiDocMetadataEntry} The locally created Metadata entries + * @param {import('./types.d.ts').ApiDocMetadataEntry['content']} section An AST tree containing the Nodes of the API doc entry section + * @returns {import('./types.d.ts').ApiDocMetadataEntry} The locally created Metadata entries */ create: (apiDoc, section) => { // This is the ID of a certain Navigation entry, which allows us to anchor @@ -65,25 +78,24 @@ const createMetadata = (slugger, remarkProcessor) => { const slugHash = `#${slugger.slug(internalMetadata.heading.text)}`; const { - type: yaml_type, - name: yaml_name, - source_link, - stability_index, + type: yamlType, + name: yamlName, + source_link: sourceLink, updates = [], changes = [], } = internalMetadata.properties; // We override the type of the heading if we have a YAML type - internalMetadata.heading.type = - yaml_type || internalMetadata.heading.type; + internalMetadata.heading.type = yamlType || internalMetadata.heading.type; - const apiEntryMetadata = { + // Returns the Metadata entry for the API doc + return { // The API file basename (without the extension) - api: yaml_name || apiDoc.stem, + api: yamlName || apiDoc.stem, // The path/slug of the API section slug: `${apiDoc.stem}.html${slugHash}`, // The source link of said API section - sourceLink: source_link, + sourceLink: sourceLink, // The latest updates to an API section updates, // The full-changeset to an API section @@ -91,25 +103,10 @@ const createMetadata = (slugger, remarkProcessor) => { // The Heading metadata heading: internalMetadata.heading, // The Stability Index of the API section - stability: stability_index, + stability: internalMetadata.stability, // The AST tree of the API section content: section, }; - - // Returns the Metadata entry for the API doc - return { - // Appends the base Metadata entry - ...apiEntryMetadata, - - // Overrides the toJSON method to allow for custom serialization - toJSON: () => ({ - ...apiEntryMetadata, - - // We stringify the AST tree to a string - // since this is what we wanbt to render within a JSON object - content: remarkProcessor.stringify(section), - }), - }; }, }; }; diff --git a/src/parser.mjs b/src/parser.mjs index 826977bf..e6244474 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -96,7 +96,7 @@ const createParser = () => { // and then generates the final content for each API doc entry and pushes it to the collection visit(apiDocTree, createQueries.UNIST.isHeading, (headingNode, index) => { // Creates a new Metadata entry for the current API doc file - const apiEntryMetadata = createMetadata(nodeSlugger, remarkProcessor); + const apiEntryMetadata = createMetadata(nodeSlugger); // Adds the Metadata of the current Heading Node to the Metadata entry addHeadingMetadata(headingNode, apiEntryMetadata); @@ -129,8 +129,11 @@ const createParser = () => { // Visits all Stability Index Nodes from the current subtree if there's any // and then apply the Stability Index Metadata to the current Metadata entry visit(apiSectionTree, createQueries.UNIST.isStabilityIndex, node => { + // Retrieves the subtree of the Stability Index Node + const stabilityNodes = createTree('root', node.children[0].children); + // Adds the Stability Index Metadata to the current Metadata entry - addStabilityIndexMetadata(node, apiEntryMetadata); + addStabilityIndexMetadata(stabilityNodes, apiEntryMetadata); return SKIP; }); @@ -150,13 +153,20 @@ const createParser = () => { createQueries.UNIST.isYamlNode, ]); + // Applies the AST transformations to the subtree based on the API doc entry Metadata + // Note that running the transformation on the subtree isn't costly as it is a reduced tree + // and the GFM transformations aren't that heavy + const transformedApiSectionTree = remarkProcessor.runSync(apiSectionTree); + + // Adds the `toJSON` method to stringify the tree back to Markdown + // So that it gets serialized correctly by JSON.stringify + transformedApiSectionTree.toJSON = () => + remarkProcessor.stringify(transformedApiSectionTree); + // We seal and create the API doc entry Metadata and push them to the collection const parsedApiEntryMetadata = apiEntryMetadata.create( resolvedApiDoc, - // Applies the AST transformations to the subtree based on the API doc entry Metadata - // Note that running the transformation on the subtree isn't costly as it is a reduced tree - // and the GFM transformations aren't that heavy - remarkProcessor.runSync(apiSectionTree) + transformedApiSectionTree ); // We push the parsed API doc entry Metadata to the collection diff --git a/src/queries.mjs b/src/queries.mjs index 28c486ff..b9e2b824 100644 --- a/src/queries.mjs +++ b/src/queries.mjs @@ -21,9 +21,9 @@ const createQueries = () => { (_, __, inner) => inner ); - const metadata = parserUtils.parseYAMLIntoMetadata(sanitizedString); - - apiEntryMetadata.updateProperties(metadata); + apiEntryMetadata.setProperties( + parserUtils.parseYAMLIntoMetadata(sanitizedString) + ); }; /** @@ -89,23 +89,29 @@ const createQueries = () => { /** * Parses a Stability Index Entry and updates the current Metadata * - * @param {import('unist').Node} node Thead Link Reference Node + * @param {import('unist').Parent} node Thead Link Reference Node * @param {ReturnType} apiEntryMetadata The API entry Metadata */ const addStabilityIndexMetadata = (node, apiEntryMetadata) => { - const stabilityIndexString = transformNodesToString( - node.children[0].children - ); - - const stabilityIndex = - createQueries.QUERIES.stabilityIndex.exec(stabilityIndexString); - - apiEntryMetadata.updateProperties({ - stability_index: { - index: Number(stabilityIndex[1]), - description: stabilityIndex[2].replaceAll('\n', ' ').trim(), - }, - }); + /** + * Handles transforming the Stability Index Node into a JSON object + * including the index and a concatenated description of the index + * + * @returns {import('./types.d.ts').StabilityIndexMetadataEntry} + */ + node.toJSON = () => { + const stabilityIndexString = transformNodesToString(node.children); + + const [, index, description] = + createQueries.QUERIES.stabilityIndex.exec(stabilityIndexString); + + return { + index: Number(index), + description: description.replaceAll('\n', ' ').trim(), + }; + }; + + apiEntryMetadata.setStability(node); }; return { diff --git a/src/types.d.ts b/src/types.d.ts index c5ff16cf..bbda36a5 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -1,4 +1,10 @@ -import { Parent } from 'unist'; +import { Parent, Node } from 'unist'; + +// String serialization of the AST tree +// @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify#tojson_behavior +export interface WithJSON extends T { + toJSON: () => J; +} export interface StabilityIndexMetadataEntry { index: number; @@ -41,7 +47,6 @@ export interface ApiDocRawMetadataEntry { source_link?: string; updates?: ApiDocMetadataUpdate[]; changes?: ApiDocMetadataChange[]; - stability_index?: StabilityIndexMetadataEntry; } export interface ApiDocMetadataEntry { @@ -58,10 +63,7 @@ export interface ApiDocMetadataEntry { // The parsed Markdown content of a Navigation Entry heading: HeadingMetadataEntry; // The API doc metadata Entry Stability Index if exists - stability: StabilityIndexMetadataEntry | undefined; + stability: WithJSON | undefined; // The subtree containing all Nodes of the API doc entry - content: Parent; - // String serialization of the AST tree - // @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify#tojson_behavior - toJSON: () => string; + content: WithJSON; } diff --git a/src/utils/parser.mjs b/src/utils/parser.mjs index 41caf3f8..b7714763 100644 --- a/src/utils/parser.mjs +++ b/src/utils/parser.mjs @@ -29,6 +29,7 @@ export const transformTypeToReferenceLink = type => { * into the reference type from the API docs * * @param {string} lookupPiece + * @returns {string | undefined} The reference URL or undefined if no match was found */ const transformType = lookupPiece => { // Transform JS primitive type references into Markdown links (MDN) diff --git a/src/utils/unist.mjs b/src/utils/unist.mjs index bb7ed491..5b35288b 100644 --- a/src/utils/unist.mjs +++ b/src/utils/unist.mjs @@ -11,23 +11,18 @@ import { pointEnd, pointStart } from 'unist-util-position'; */ export const transformNodesToString = nodes => { const mappedChildren = nodes.map(node => { - if (node.type === 'inlineCode') { - return `\`${node.value}\``; + switch (node.type) { + case 'inlineCode': + return `\`${node.value}\``; + case 'strong': + return `**${transformNodesToString(node.children)}**`; + case 'emphasis': + return `_${transformNodesToString(node.children)}_`; + default: + return node.children + ? transformNodesToString(node.children) + : node.value; } - - if (node.type === 'strong') { - return `**${transformNodesToString(node.children)}**`; - } - - if (node.type === 'emphasis') { - return `_${transformNodesToString(node.children)}_`; - } - - if (node.children) { - return transformNodesToString(node.children); - } - - return node.value; }); return mappedChildren.join(''); From be1927fbf20b99bc208aaa443920ef5d7be473d9 Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Fri, 12 Jul 2024 04:04:34 +0200 Subject: [PATCH 26/26] fix: partial metadata definition --- src/metadata.mjs | 13 ++++++++++--- src/queries.mjs | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/metadata.mjs b/src/metadata.mjs index 6b14fd7a..7a02eb65 100644 --- a/src/metadata.mjs +++ b/src/metadata.mjs @@ -53,10 +53,17 @@ const createMetadata = slugger => { * it also allows for extra data (such as Stability Index) and miscellaneous data to be set * although it'd be best to only set ones from {ApiDocRawMetadataEntry} * - * @param {import('./types.d.ts').ApiDocRawMetadataEntry} properties Extra Metadata properties to be defined + * Note: A single API doc entry might have multiple YAML metadata blocks, + * meaning that this method can be called multiple times to update the properties + * and complement each set of data. + * + * @param {Partial} properties Extra Metadata properties to be defined */ - setProperties: properties => { - internalMetadata.properties = properties; + updateProperties: properties => { + internalMetadata.properties = { + ...internalMetadata.properties, + ...properties, + }; }, /** * Generates a new Navigation entry and pushes them to the internal collection diff --git a/src/queries.mjs b/src/queries.mjs index b9e2b824..d0c09b99 100644 --- a/src/queries.mjs +++ b/src/queries.mjs @@ -21,7 +21,7 @@ const createQueries = () => { (_, __, inner) => inner ); - apiEntryMetadata.setProperties( + apiEntryMetadata.updateProperties( parserUtils.parseYAMLIntoMetadata(sanitizedString) ); };