From 6297b6195ce6ae1c6d055740d26bca623efcf506 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Mon, 9 Dec 2024 09:24:57 +0000 Subject: [PATCH] Add search capabilities to docs Include "minisearch" which is a straightforward client-side search tool; and a script to generate the search index for minisearch for each version of libgit2. --- script/api-docs/generate | 1 + script/api-docs/package-lock.json | 8 +- script/api-docs/package.json | 3 +- script/api-docs/search-generator.js | 212 ++++++++++++++++++++++++++++ 4 files changed, 222 insertions(+), 2 deletions(-) create mode 100755 script/api-docs/search-generator.js diff --git a/script/api-docs/generate b/script/api-docs/generate index bc9af0cae..b150fc8e2 100755 --- a/script/api-docs/generate +++ b/script/api-docs/generate @@ -109,5 +109,6 @@ if [ "${force}" ]; then options="${options} --force" fi +node ./search-generator.js --verbose "${output_path}/api" "${output_path}/search-index" node ./docs-generator.js --verbose --jekyll-layout default "${output_path}/api" "${output_path}/reference" node ./docs-generator.js ${options} --jekyll-layout default "${output_path}/api" "${output_path}/reference" diff --git a/script/api-docs/package-lock.json b/script/api-docs/package-lock.json index bd9ca1a47..3ba22260f 100644 --- a/script/api-docs/package-lock.json +++ b/script/api-docs/package-lock.json @@ -6,7 +6,8 @@ "": { "dependencies": { "commander": "^12.1.0", - "markdown-it": "^14.1.0" + "markdown-it": "^14.1.0", + "minisearch": "^7.1.1" } }, "node_modules/argparse": { @@ -62,6 +63,11 @@ "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==" }, + "node_modules/minisearch": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/minisearch/-/minisearch-7.1.1.tgz", + "integrity": "sha512-b3YZEYCEH4EdCAtYP7OlDyx7FdPwNzuNwLQ34SfJpM9dlbBZzeXndGavTrC+VCiRWomL21SWfMc6SCKO/U2ZNw==" + }, "node_modules/punycode.js": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", diff --git a/script/api-docs/package.json b/script/api-docs/package.json index 53ae07054..67a6fe7aa 100644 --- a/script/api-docs/package.json +++ b/script/api-docs/package.json @@ -1,6 +1,7 @@ { "dependencies": { "commander": "^12.1.0", - "markdown-it": "^14.1.0" + "markdown-it": "^14.1.0", + "minisearch": "^7.1.1" } } diff --git a/script/api-docs/search-generator.js b/script/api-docs/search-generator.js new file mode 100755 index 000000000..cad73f947 --- /dev/null +++ b/script/api-docs/search-generator.js @@ -0,0 +1,212 @@ +#!/usr/bin/env node + +const markdownit = require('markdown-it'); +const { program } = require('commander'); +const minisearch = require('minisearch'); + +const path = require('node:path'); +const fs = require('node:fs/promises'); + +const linkPrefix = '/docs/reference'; + +const defaultBranch = 'main'; + +function uniqueifyId(api, nodes) { + let suffix = "", i = 1; + + while (true) { + const possibleId = `${api.kind}-${api.name}${suffix}`; + let collision = false; + + for (const item of nodes) { + if (item.id === possibleId) { + collision = true; + break; + } + } + + if (!collision) { + return possibleId; + } + + suffix = `-${++i}`; + } +} + +async function produceSearchIndex(version, apiData) { + const nodes = [ ]; + + for (const group in apiData['groups']) { + for (const name in apiData['groups'][group]['apis']) { + const api = apiData['groups'][group]['apis'][name]; + + let displayName = name; + + if (api.kind === 'macro') { + displayName = displayName.replace(/\(.*/, ''); + } + + const apiSearchData = { + id: uniqueifyId(api, nodes), + name: displayName, + group: group, + kind: api.kind + }; + + apiSearchData.description = Array.isArray(api.comment) ? + api.comment[0] : api.comment; + + let detail = ""; + + if (api.kind === 'macro') { + detail = api.value; + } + else if (api.kind === 'alias') { + detail = api.type; + } + else { + let details = undefined; + + if (api.kind === 'struct' || api.kind === 'enum') { + details = api.members; + } + else if (api.kind === 'function' || api.kind === 'callback') { + details = api.params; + } + else { + throw new Error(`unknown api type '${api.kind}'`); + } + + for (const item of details || [ ]) { + if (detail.length > 0) { + detail += ' '; + } + + detail += item.name; + + if (item.comment) { + detail += ' '; + detail += item.comment; + } + } + + if (api.kind === 'function' || api.kind === 'callback') { + if (detail.length > 0 && api.returns?.type) { + detail += ' ' + api.returns.type; + } + + if (detail.length > 0 && api.returns?.comment) { + detail += ' ' + api.returns.comment; + } + } + } + + detail = detail.replaceAll(/\s+/g, ' ') + .replaceAll(/[\"\'\`]/g, ''); + + apiSearchData.detail = detail; + + nodes.push(apiSearchData); + } + } + + const index = new minisearch({ + fields: [ 'name', 'description', 'detail' ], + storeFields: [ 'name', 'group', 'kind', 'description' ], + searchOptions: { boost: { name: 5, description: 2 } } + }); + + index.addAll(nodes); + + const filename = `${outputPath}/${version}.json`; + await fs.mkdir(outputPath, { recursive: true }); + await fs.writeFile(filename, JSON.stringify(index, null, 2)); +} + +function versionSort(a, b) { + if (a === b) { + return 0; + } + + const aVersion = a.match(/^v(\d+)(?:\.(\d+)(?:\.(\d+)(?:\.(\d+))?)?)?(?:-(.*))?$/); + const bVersion = b.match(/^v(\d+)(?:\.(\d+)(?:\.(\d+)(?:\.(\d+))?)?)?(?:-(.*))?$/); + + if (!aVersion && !bVersion) { + return a.localeCompare(b); + } + else if (aVersion && !bVersion) { + return -1; + } + else if (!aVersion && bVersion) { + return 1; + } + + for (let i = 1; i < 5; i++) { + if (!aVersion[i] && !bVersion[i]) { + break; + } + else if (aVersion[i] && !bVersion[i]) { + return 1; + } + else if (!aVersion[i] && bVersion[i]) { + return -1; + } + else if (aVersion[i] !== bVersion[i]) { + return aVersion[i] - bVersion[i]; + } + } + + if (aVersion[5] && !bVersion[5]) { + return -1; + } + else if (!aVersion[5] && bVersion[5]) { + return 1; + } + else if (aVersion[5] && bVersion[5]) { + return aVersion[5].localeCompare(bVersion[5]); + } + + return 0; +} + +program.option('--verbose') + .option('--version '); +program.parse(); + +const options = program.opts(); + +if (program.args.length != 2) { + console.error(`usage: ${path.basename(process.argv[1])} raw_api_dir output_dir`); + process.exit(1); +} + +const docsPath = program.args[0]; +const outputPath = program.args[1]; + +(async () => { + try { + const v = options.version ? options.version : + (await fs.readdir(docsPath)) + .filter(a => a.endsWith('.json')) + .map(a => a.replace(/\.json$/, '')); + + const versions = v.sort(versionSort).reverse(); + + for (const version of versions) { + if (options.verbose) { + console.log(`Reading documentation data for ${version}...`); + } + + const apiData = JSON.parse(await fs.readFile(`${docsPath}/${version}.json`)); + + if (options.verbose) { + console.log(`Creating minisearch index for ${version}...`); + } + + await produceSearchIndex(version, apiData); + } + } catch (e) { + console.error(e); + process.exit(1); + } +})();