From e99399e422a3ed8cedf45b3a9c1fd055ab63d317 Mon Sep 17 00:00:00 2001 From: David Huggins-Daines Date: Fri, 5 Jul 2024 15:41:03 -0400 Subject: [PATCH 1/2] fix: load index from alexi space --- create-index.ts | 135 ----------- package-lock.json | 594 ++++++++++++++++++++++++++++++++++++---------- package.json | 8 +- search-index.ts | 21 +- src/config.ts | 3 + src/main.ts | 21 +- 6 files changed, 505 insertions(+), 277 deletions(-) delete mode 100644 create-index.ts diff --git a/create-index.ts b/create-index.ts deleted file mode 100644 index 291f604..0000000 --- a/create-index.ts +++ /dev/null @@ -1,135 +0,0 @@ -import process from "node:process"; -import fetch from "node-fetch"; -import lunr from "lunr"; -import { ALEXI_URL } from "./src/config.js"; -import folding from "lunr-folding"; -import { readFile, writeFile, mkdir } from "node:fs/promises"; -import path from "node:path"; -import { parse, HTMLElement, TextNode } from "node-html-parser"; - -folding(lunr); // beurk - -async function fetch_alexi(url: string): Promise { - if (false) - return readFile(`../alexi/export/${url}`, "utf8"); - else { - const result = await fetch(`${ALEXI_URL}/${url}`); - if (!result.ok) { - console.error(`Failed to fetch ${ALEXI_URL}/${url}`); - process.exit(1); - } - return result.text(); - } -} - -interface Document { - url: string; - titre: string; - texte: string; -} - -interface Texte { - titre: string; - texte: string; -} - -// Seems lunr.js does not have stored fields so we have to do this -const textes: {[url: string]: Texte} = {}; - -function make_doc(url: string, titreEl: HTMLElement, html: string): Document | null { - const root = parse(html); - const bodyEl = root.querySelector("div#body"); - if (bodyEl === null || !bodyEl.textContent || !titreEl.textContent) - return null; - const titre = titreEl.textContent.trim(); - // Remove titles - for (const header of bodyEl.querySelectorAll(".header")) { - header.parentNode.removeChild(header); - } - // Swap images for alternate text - for (const img of bodyEl.querySelectorAll("img")) { - const altText = img.getAttribute("alt"); - if (!altText) - continue; - const altEl = new HTMLElement("p", {}); - altEl.appendChild(new TextNode(altText)); - img.parentNode.exchangeChild(img, altEl); - } - const texte = bodyEl.textContent; - return { - url, titre, texte - } -} - -async function crawl_alexi(builder: lunr.Builder): Promise { - /* Crawl ALEXI for things to index */ - const html = await fetch_alexi("index.html"); - const root = parse(html); - /* Gather documents, chapters, sections */ - for (const section of root.querySelectorAll("li.node")) { - /* Skip entire documents for now */ - if (section.classList.contains("Document")) - continue; - const summary = section.querySelector("summary"); - if (!summary) { - console.error(`No summary found for ${section.classNames}`); - continue; - } - const htmlLink = section.querySelector("a"); - if (!htmlLink) { - console.error("No link found in li.leaf"); - continue; - } - const htmlUrl = htmlLink.getAttribute("href"); - if (!htmlUrl) { - console.error("No href found in li.leaf"); - continue; - } - const html = await fetch_alexi(htmlUrl); - const doc = make_doc(htmlUrl, summary, html); - if (doc == null) { - console.error(`No content found in ${htmlUrl}`); - continue; - } - console.log(doc.url); - builder.add(doc); - textes[htmlUrl] = { titre: doc.titre, texte: doc.texte }; - } - /* Gather articles, other leaf nodes */ - for (const text of root.querySelectorAll("li.leaf")) { - const htmlLink = text.querySelector("a"); - if (!htmlLink) { - console.error("No link found in li.leaf"); - continue; - } - const htmlUrl = htmlLink.getAttribute("href"); - if (!htmlUrl) { - console.error("No href found in li.leaf"); - continue; - } - const html = await fetch_alexi(htmlUrl); - const doc = make_doc(htmlUrl, htmlLink, html); - if (doc == null) { - console.error(`No content found in ${htmlUrl}`); - continue; - } - console.log(doc.url); - builder.add(doc); - textes[htmlUrl] = { titre: doc.titre, texte: doc.texte }; - } -} - -await mkdir("public", { recursive: true }); - -const builder = new lunr.Builder(); -builder.ref("url"); -builder.field("titre", { boost: 2 }); -builder.field("texte"); -builder.metadataWhitelist = ['position']; // Yes this is undocumented -await crawl_alexi(builder); -const index = builder.build(); - -await writeFile(path.join("public", "index.json"), - JSON.stringify(index.toJSON())); -await writeFile(path.join("public", "textes.json"), - JSON.stringify(textes)); diff --git a/package-lock.json b/package-lock.json index 11b2199..da13be7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,16 +9,18 @@ "version": "0.3.1", "license": "MIT", "dependencies": { - "@types/debounce": "^1.2.4", "debounce": "^2.0.0", "lunr": "^2.3.9", - "lunr-folding": "^0.1.0", - "node-html-parser": "^6.1.13" + "lunr-languages": "^1.14.0", + "unidecode": "^1.1.0" }, "devDependencies": { + "@types/debounce": "^1.2.4", "@types/lunr": "^2.3.7", "@types/node": "^20.12.8", + "@types/unidecode": "^0.1.3", "node-fetch": "^3.3.2", + "tsx": "^4.16.2", "typescript": "^5.4.5", "vite": "^5.2.10" } @@ -602,7 +604,8 @@ "node_modules/@types/debounce": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/@types/debounce/-/debounce-1.2.4.tgz", - "integrity": "sha512-jBqiORIzKDOToaF63Fm//haOCHuwQuLa2202RK4MozpA6lh93eCBc+/8+wZn5OzjJt3ySdc+74SXWXB55Ewtyw==" + "integrity": "sha512-jBqiORIzKDOToaF63Fm//haOCHuwQuLa2202RK4MozpA6lh93eCBc+/8+wZn5OzjJt3ySdc+74SXWXB55Ewtyw==", + "dev": true }, "node_modules/@types/estree": { "version": "1.0.5", @@ -625,36 +628,11 @@ "undici-types": "~5.26.4" } }, - "node_modules/boolbase": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", - "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" - }, - "node_modules/css-select": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", - "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", - "dependencies": { - "boolbase": "^1.0.0", - "css-what": "^6.1.0", - "domhandler": "^5.0.2", - "domutils": "^3.0.1", - "nth-check": "^2.0.1" - }, - "funding": { - "url": "https://github.com/sponsors/fb55" - } - }, - "node_modules/css-what": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", - "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", - "engines": { - "node": ">= 6" - }, - "funding": { - "url": "https://github.com/sponsors/fb55" - } + "node_modules/@types/unidecode": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@types/unidecode/-/unidecode-0.1.3.tgz", + "integrity": "sha512-7R8zgAf8y1qq5Zif6UIXYR07MHvJIjcQM9Ym2am1YXaWdn9zJltLDwO8HpmIIjHiNT4VMGiNAw+UI9S7OM2foA==", + "dev": true }, "node_modules/data-uri-to-buffer": { "version": "4.0.1", @@ -676,68 +654,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/dom-serializer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", - "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.2", - "entities": "^4.2.0" - }, - "funding": { - "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" - } - }, - "node_modules/domelementtype": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", - "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fb55" - } - ] - }, - "node_modules/domhandler": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", - "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", - "dependencies": { - "domelementtype": "^2.3.0" - }, - "engines": { - "node": ">= 4" - }, - "funding": { - "url": "https://github.com/fb55/domhandler?sponsor=1" - } - }, - "node_modules/domutils": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", - "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", - "dependencies": { - "dom-serializer": "^2.0.0", - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3" - }, - "funding": { - "url": "https://github.com/fb55/domutils?sponsor=1" - } - }, - "node_modules/entities": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", - "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, "node_modules/esbuild": { "version": "0.20.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.20.2.tgz", @@ -825,12 +741,16 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, - "node_modules/he": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", - "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", - "bin": { - "he": "bin/he" + "node_modules/get-tsconfig": { + "version": "4.7.5", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.7.5.tgz", + "integrity": "sha512-ZCuZCnlqNzjb4QprAzXKdpp/gh6KTxSJuw3IBsPnV/7fV4NxC9ckB+vPTt8w7fJA0TaSD7c55BR47JD6MEDyDw==", + "dev": true, + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, "node_modules/lunr": { @@ -838,10 +758,10 @@ "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.9.tgz", "integrity": "sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==" }, - "node_modules/lunr-folding": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/lunr-folding/-/lunr-folding-0.1.0.tgz", - "integrity": "sha512-IWsPGKrcoowCACgfG0jc1vRz2sjjnsE5e1nN5SWGT77Adp7WiwX//4LUN6WhiMuWNptHizF04q4+gN31rPcbyg==" + "node_modules/lunr-languages": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/lunr-languages/-/lunr-languages-1.14.0.tgz", + "integrity": "sha512-hWUAb2KqM3L7J5bcrngszzISY4BxrXn/Xhbb9TTCJYEGqlR1nG67/M14sp09+PTIRklobrn57IAxcdcO/ZFyNA==" }, "node_modules/nanoid": { "version": "3.3.7", @@ -898,26 +818,6 @@ "url": "https://opencollective.com/node-fetch" } }, - "node_modules/node-html-parser": { - "version": "6.1.13", - "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-6.1.13.tgz", - "integrity": "sha512-qIsTMOY4C/dAa5Q5vsobRpOOvPfC4pB61UVW2uSwZNUp0QU/jCekTal1vMmbO0DgdHeLUJpv/ARmDqErVxA3Sg==", - "dependencies": { - "css-select": "^5.1.0", - "he": "1.2.0" - } - }, - "node_modules/nth-check": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", - "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", - "dependencies": { - "boolbase": "^1.0.0" - }, - "funding": { - "url": "https://github.com/fb55/nth-check?sponsor=1" - } - }, "node_modules/picocolors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", @@ -952,6 +852,15 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, "node_modules/rollup": { "version": "4.17.2", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.17.2.tgz", @@ -996,6 +905,431 @@ "node": ">=0.10.0" } }, + "node_modules/tsx": { + "version": "4.16.2", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.16.2.tgz", + "integrity": "sha512-C1uWweJDgdtX2x600HjaFaucXTilT7tgUZHbOE4+ypskZ1OP8CRCSDkCxG6Vya9EwaFIVagWwpaVAn5wzypaqQ==", + "dev": true, + "dependencies": { + "esbuild": "~0.21.5", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/tsx/node_modules/@esbuild/aix-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", + "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", + "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", + "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", + "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/darwin-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", + "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/darwin-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", + "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/freebsd-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", + "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/freebsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", + "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", + "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", + "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", + "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-loong64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", + "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-mips64el": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", + "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", + "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-riscv64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", + "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-s390x": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", + "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", + "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/netbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", + "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/openbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", + "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/sunos-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", + "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", + "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", + "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", + "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/esbuild": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", + "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.21.5", + "@esbuild/android-arm": "0.21.5", + "@esbuild/android-arm64": "0.21.5", + "@esbuild/android-x64": "0.21.5", + "@esbuild/darwin-arm64": "0.21.5", + "@esbuild/darwin-x64": "0.21.5", + "@esbuild/freebsd-arm64": "0.21.5", + "@esbuild/freebsd-x64": "0.21.5", + "@esbuild/linux-arm": "0.21.5", + "@esbuild/linux-arm64": "0.21.5", + "@esbuild/linux-ia32": "0.21.5", + "@esbuild/linux-loong64": "0.21.5", + "@esbuild/linux-mips64el": "0.21.5", + "@esbuild/linux-ppc64": "0.21.5", + "@esbuild/linux-riscv64": "0.21.5", + "@esbuild/linux-s390x": "0.21.5", + "@esbuild/linux-x64": "0.21.5", + "@esbuild/netbsd-x64": "0.21.5", + "@esbuild/openbsd-x64": "0.21.5", + "@esbuild/sunos-x64": "0.21.5", + "@esbuild/win32-arm64": "0.21.5", + "@esbuild/win32-ia32": "0.21.5", + "@esbuild/win32-x64": "0.21.5" + } + }, "node_modules/typescript": { "version": "5.4.5", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.5.tgz", @@ -1015,6 +1349,14 @@ "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", "dev": true }, + "node_modules/unidecode": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/unidecode/-/unidecode-1.1.0.tgz", + "integrity": "sha512-GIp57N6DVVJi8dpeIU6/leJGdv7W65ZSXFLFiNmxvexXkc0nXdqUvhA/qL9KqBKsILxMwg5MnmYNOIDJLb5JVA==", + "engines": { + "node": ">= 0.4.12" + } + }, "node_modules/vite": { "version": "5.2.10", "resolved": "https://registry.npmjs.org/vite/-/vite-5.2.10.tgz", diff --git a/package.json b/package.json index 5008334..ce09f41 100644 --- a/package.json +++ b/package.json @@ -26,17 +26,19 @@ }, "homepage": "https://github.com/dhdaines/serafim#readme", "devDependencies": { + "@types/debounce": "^1.2.4", "@types/lunr": "^2.3.7", "@types/node": "^20.12.8", + "@types/unidecode": "^0.1.3", "node-fetch": "^3.3.2", + "tsx": "^4.16.2", "typescript": "^5.4.5", "vite": "^5.2.10" }, "dependencies": { - "@types/debounce": "^1.2.4", "debounce": "^2.0.0", "lunr": "^2.3.9", - "lunr-folding": "^0.1.0", - "node-html-parser": "^6.1.13" + "lunr-languages": "^1.14.0", + "unidecode": "^1.1.0" } } diff --git a/search-index.ts b/search-index.ts index bf0ec47..49fce09 100644 --- a/search-index.ts +++ b/search-index.ts @@ -1,12 +1,23 @@ import lunr from "lunr"; import fs from "node:fs"; -import folding from "lunr-folding"; +import stemmerSupport from "lunr-languages/lunr.stemmer.support"; +import lunrFR from "lunr-languages/lunr.fr"; +import unidecode from "unidecode"; -/* UGH! This API is SO WEIRD!!! */ -folding(lunr); +stemmerSupport(lunr); +lunrFR(lunr); +lunr.Pipeline.registerFunction(token => token.update(unidecode), "unifold") const index = lunr.Index.load(JSON.parse(fs.readFileSync("public/index.json", "utf8"))); -for (const result of index.search("mobilité")) { - console.log(JSON.stringify(result)); +let i = 0; +for (const result of index.search("règlement de démolition")) { + if (i++ == 5) break; + console.log(JSON.stringify(result)); +} +console.log(); +i = 0; +for (const result of index.search("t4.2")) { + if (i++ == 5) break; + console.log(JSON.stringify(result)); } diff --git a/src/config.ts b/src/config.ts index fdb0d63..45a499e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1 +1,4 @@ export const ALEXI_URL = "https://dhdaines.github.io/alexi"; +// Unfortunately this doesn't really work for testing because URL() +// won't accept a relative URL as base (see https://github.com/whatwg/url/issues/531) +// export const ALEXI_URL = "/alexi"; diff --git a/src/main.ts b/src/main.ts index 013c2f6..318e927 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,17 +1,21 @@ import lunr from "lunr"; -import folding from "lunr-folding"; +// @ts-ignore +import stemmerSupport from "lunr-languages/lunr.stemmer.support"; +// @ts-ignore +import lunrFR from "lunr-languages/lunr.fr"; +import unidecode from "unidecode"; import debounce from "debounce"; +stemmerSupport(lunr); +lunrFR(lunr); +lunr.Pipeline.registerFunction(token => token.update(unidecode), "unifold") + import { ALEXI_URL } from "./config.ts"; -// @ts-ignore -import INDEX_URL from "/index.json?url"; -// @ts-ignore -import TEXTES_URL from "/textes.json?url"; +const INDEX_URL = `${ALEXI_URL}/_idx/index.json` +const TEXTES_URL = `${ALEXI_URL}/_idx/textes.json`; // @ts-ignore const BASE_URL = import.meta.env.BASE_URL; -folding(lunr); // beurk - interface Texte { titre: string; texte: string; @@ -66,7 +70,8 @@ class App { continue; } /* Resolve (maybe) relative image URL (will not work if source - * has a trailing slash instead of index.html) */ + * has a trailing slash instead of index.html) (will also not + * work if source is not an absolute URL) */ const url = new URL(srcAttr, source); img.setAttribute("src", url.toString()); } From 9a2eaa03c94c6047bd41ffa12b97c0c3324b5738 Mon Sep 17 00:00:00 2001 From: David Huggins-Daines Date: Fri, 5 Jul 2024 16:06:13 -0400 Subject: [PATCH 2/2] fix: plus necessaire de faire index --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 34e1557..bce34e7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,7 +31,6 @@ jobs: - name: Build run: | npm install - npm run index npx vite build --base=/serafim/ cp dist/index.html dist/404.html - name: Setup Pages