Add ES Module import/export conventions to improve the ergonomics around including other script files or code blocks.

package.json

@@ -1,60 +1,63 @@
 {
-  "name": "datacore",
-  "version": "0.1.19",
-  "description": "Reactive data engine for Obsidian.md.",
-  "main": "lib/index.js",
-  "types": "lib/index.d.ts",
-  "files": [
-    "lib/**/*"
-  ],
-  "scripts": {
-    "dev": "node esbuild.config.mjs",
-    "build": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs production",
-    "test": "yarn run jest",
-    "test-watch": "yarn run jest -i --watch --no-cache",
-    "check-format": "yarn run prettier --check src",
-    "format": "yarn run prettier --write src"
-  },
-  "keywords": [
-    "obsidian",
-    "datacore",
-    "dataview",
-    "pkm"
-  ],
-  "author": "Michael Brenan",
-  "license": "MIT",
-  "devDependencies": {
-    "@codemirror/language": "https://github.com/lishid/cm-language",
-    "@codemirror/state": "^6.0.1",
-    "@codemirror/view": "^6.0.1",
-    "@types/jest": "^27.0.1",
-    "@types/luxon": "^2.3.2",
-    "@types/node": "^16.7.13",
-    "@types/parsimmon": "^1.10.6",
-    "builtin-modules": "3.3.0",
-    "esbuild": "^0.16.11",
-    "esbuild-plugin-inline-worker": "https://github.com/mitschabaude/esbuild-plugin-inline-worker",
-    "jest": "^27.1.0",
-    "obsidian": "^1.6.6",
-    "prettier": "2.3.2",
-    "ts-jest": "^27.0.5",
-    "tslib": "^2.3.1",
-    "typescript": "^5.4.2"
-  },
-  "dependencies": {
-    "@datastructures-js/queue": "^4.2.3",
-    "@fortawesome/fontawesome-svg-core": "^6.4.0",
-    "@fortawesome/free-solid-svg-icons": "^6.4.0",
-    "@fortawesome/react-fontawesome": "^0.2.0",
-    "emoji-regex": "^10.2.1",
-    "flatqueue": "^2.0.3",
-    "localforage": "1.10.0",
-    "luxon": "^2.4.0",
-    "parsimmon": "^1.18.0",
-    "preact": "^10.17.1",
-    "react-select": "^5.8.0",
-    "sorted-btree": "^1.8.1",
-    "sucrase": "3.35.0",
-    "yaml": "^2.3.3"
-  }
+    "name": "datacore",
+    "version": "0.1.19",
+    "description": "Reactive data engine for Obsidian.md.",
+    "main": "lib/index.js",
+    "types": "lib/index.d.ts",
+    "files": [
+        "lib/**/*"
+    ],
+    "scripts": {
+        "dev": "node esbuild.config.mjs",
+        "build": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs production",
+        "test": "yarn run jest",
+        "test-watch": "yarn run jest -i --watch --no-cache",
+        "check-format": "yarn run prettier --check src",
+        "format": "yarn run prettier --write src"
+    },
+    "keywords": [
+        "obsidian",
+        "datacore",
+        "dataview",
+        "pkm"
+    ],
+    "author": "Michael Brenan",
+    "license": "MIT",
+    "devDependencies": {
+        "@codemirror/language": "https://github.com/lishid/cm-language",
+        "@codemirror/state": "^6.0.1",
+        "@codemirror/view": "^6.0.1",
+        "@types/jest": "^27.0.1",
+        "@types/luxon": "^2.3.2",
+        "@types/node": "^16.7.13",
+        "@types/parsimmon": "^1.10.6",
+        "@types/path-browserify": "^1.0.3",
+        "builtin-modules": "3.3.0",
+        "esbuild": "^0.16.11",
+        "esbuild-plugin-inline-worker": "https://github.com/mitschabaude/esbuild-plugin-inline-worker",
+        "jest": "^27.1.0",
+        "obsidian": "^1.6.6",
+        "prettier": "2.3.2",
+        "ts-jest": "^27.0.5",
+        "ts-node": "^10.9.2",
+        "tslib": "^2.3.1",
+        "typescript": "^5.4.2"
+    },
+    "dependencies": {
+        "@datastructures-js/queue": "^4.2.3",
+        "@fortawesome/fontawesome-svg-core": "^6.4.0",
+        "@fortawesome/free-solid-svg-icons": "^6.4.0",
+        "@fortawesome/react-fontawesome": "^0.2.0",
+        "emoji-regex": "^10.2.1",
+        "flatqueue": "^2.0.3",
+        "localforage": "1.10.0",
+        "luxon": "^2.4.0",
+        "parsimmon": "^1.18.0",
+        "path-browserify": "^1.0.1",
+        "preact": "^10.17.1",
+        "react-select": "^5.8.0",
+        "sorted-btree": "^1.8.1",
+        "sucrase": "3.35.0",
+        "yaml": "^2.3.3"
+    }
 }

src/api/local-api.tsx

@@ -90,7 +90,7 @@ export class DatacoreLocalApi {
      * ```
      */
     public async require(path: string | Link): Promise<any> {
-        const result = await this.scriptCache.load(path, { dc: this });
+        const result = await this.scriptCache.load(path, this);
         return result.orElseThrow();
     }
 

src/api/script-cache.ts

@@ -3,10 +3,16 @@ import { Datastore } from "index/datastore";
 import { Result } from "./result";
 import { MarkdownCodeblock, MarkdownSection } from "index/types/markdown";
 import { Deferred, deferred } from "utils/deferred";
-import { ScriptLanguage, asyncEvalInContext, transpile } from "utils/javascript";
+import {
+    ScriptDefinition,
+    ScriptLanguage,
+    asyncEvalInContext,
+    defaultScriptLoadingContext,
+    transpile,
+} from "utils/javascript";
 import { lineRange } from "utils/normalizers";
-import { TFile } from "obsidian";
-import { Fragment, h } from "preact";
+import { normalizePath, TFile } from "obsidian";
+import { DatacoreLocalApi } from "./local-api";
 
 /** A script that is currently being loaded. */
 export interface LoadingScript {
@@ -56,20 +62,34 @@ export class ScriptCache {
     public constructor(private store: Datastore) {}
 
     /** Load the given script at the given path, recursively loading any subscripts as well.  */
-    public async load(path: string | Link, context: Record<string, any>): Promise<Result<any, string>> {
+    public async load(path: string | Link, api: DatacoreLocalApi): Promise<Result<any, string>> {
+        // First, attempt to resolve the script against the script roots so we cache a canonical script path as the key.
+        var linkToLoad = undefined;
+        const roots = ["", ...api.core.settings.scriptRoots];
+        for (var i = 0; i < roots.length; i++) {
+            linkToLoad = this.store.tryNormalizeLink(path, normalizePath(roots[i]));
+            if (linkToLoad) {
+                break;
+            }
+        }
+
+        const resolvedPath = linkToLoad ?? path;
+
         // Always check the cache first.
-        const key = this.pathkey(path);
+        const key = this.pathkey(resolvedPath);
         const currentScript = this.scripts.get(key);
         if (currentScript) {
-            if (currentScript.type === "loaded") return Result.success(currentScript.object);
+            if (currentScript.type === "loaded") {
+                return Result.success(currentScript.object);
+            }
 
             // TODO: If we try to load an already-loading script, we are almost certainly doing something
             // weird. Either the caller is not `await`-ing the load and loading multiple times, OR
             // we are in a `require()` loop. Either way, we'll error out for now since we can't handle
             // either case currently.
             return Result.failure(
-                `Failed to import script "${path.toString()}", as it is in the middle of being loaded. Do you have
-                 a circular dependency in your require() calls? The currently loaded or loading scripts are: 
+                `Failed to import script "${resolvedPath.toString()}", as it is in the middle of being loaded. Do you have
+                 a circular dependency in your require() calls? The currently loaded or loading scripts are:
                  ${Array.from(this.scripts.values())
                      .map((sc) => "\t" + sc.path)
                      .join("\n")}`
@@ -80,7 +100,7 @@ export class ScriptCache {
         const deferral = deferred<Result<any, string>>();
         this.scripts.set(key, { type: "loading", promise: deferral, path: key });
 
-        const result = await this.loadUncached(path, context);
+        const result = await this.loadUncached(resolvedPath, api);
         deferral.resolve(result);
 
         if (result.successful) {
@@ -93,25 +113,30 @@ export class ScriptCache {
     }
 
     /** Load a script, directly bypassing the cache. */
-    private async loadUncached(path: string | Link, context: Record<string, any>): Promise<Result<any, string>> {
-        const maybeSource = await this.resolveSource(path);
+    private async loadUncached(scriptPath: string | Link, api: DatacoreLocalApi): Promise<Result<any, string>> {
+        var maybeSource = await this.resolveSource(scriptPath);
         if (!maybeSource.successful) return maybeSource;
 
         // Transpile to vanilla javascript first...
-        const { code, language } = maybeSource.value;
+        const scriptDefinition = maybeSource.value;
         let basic;
         try {
-            basic = transpile(code, language);
+            basic = transpile(scriptDefinition);
         } catch (error) {
-            return Result.failure(`Failed to import ${path.toString()} while transpiling from ${language}: ${error}`);
+            return Result.failure(
+                `Failed to import ${scriptPath.toString()} while transpiling from ${
+                    scriptDefinition.scriptLanguage
+                }: ${error}`
+            );
         }
 
         // Then finally execute the script to 'load' it.
-        const finalContext = Object.assign({ h: h, Fragment: Fragment }, context);
+        const scriptContext = defaultScriptLoadingContext(api);
         try {
-            return Result.success(await asyncEvalInContext(basic, finalContext));
+            const loadRet = (await asyncEvalInContext(basic, scriptContext)) ?? scriptContext.exports;
+            return Result.success(loadRet);
         } catch (error) {
-            return Result.failure(`Failed to execute script '${path.toString()}': ${error}`);
+            return Result.failure(`Failed to execute script '${scriptPath.toString()}': ${error}`);
         }
     }
 
@@ -122,10 +147,8 @@ export class ScriptCache {
     }
 
     /** Attempts to resolve the source to load given a path or link to a markdown section. */
-    private async resolveSource(
-        path: string | Link
-    ): Promise<Result<{ code: string; language: ScriptLanguage }, string>> {
-        const object = this.store.resolveLink(path);
+    private async resolveSource(path: string | Link, sourcePath?: string): Promise<Result<ScriptDefinition, string>> {
+        const object = this.store.resolveLink(path, sourcePath);
         if (!object) return Result.failure("Could not find a script at the given path: " + path.toString());
 
         const tfile = this.store.vault.getFileByPath(object.$file!);
@@ -137,7 +160,7 @@ export class ScriptCache {
 
             try {
                 const code = await this.store.vault.cachedRead(tfile);
-                return Result.success({ code, language });
+                return Result.success({ scriptFile: tfile, scriptLanguage: language, scriptSource: code });
             } catch (error) {
                 return Result.failure("Failed to load javascript/typescript source file: " + error);
             }
@@ -158,7 +181,11 @@ export class ScriptCache {
                 ScriptCache.SCRIPT_LANGUAGES[
                     maybeBlock.$languages.find((lang) => lang.toLocaleLowerCase() in ScriptCache.SCRIPT_LANGUAGES)!
                 ];
-            return (await this.readCodeblock(tfile, maybeBlock)).map((code) => ({ code, language }));
+            return (await this.readCodeblock(tfile, maybeBlock)).map((code) => ({
+                scriptFile: tfile,
+                scriptSource: code,
+                scriptLanguage: language,
+            }));
         } else if (object instanceof MarkdownCodeblock) {
             const maybeLanguage = object.$languages.find(
                 (lang) => lang.toLocaleLowerCase() in ScriptCache.SCRIPT_LANGUAGES
@@ -167,7 +194,11 @@ export class ScriptCache {
                 return Result.failure(`The codeblock referenced by '${path}' is not a JS/TS codeblock.`);
 
             const language = ScriptCache.SCRIPT_LANGUAGES[maybeLanguage];
-            return (await this.readCodeblock(tfile, object)).map((code) => ({ code, language }));
+            return (await this.readCodeblock(tfile, object)).map((code) => ({
+                scriptFile: tfile,
+                scriptSource: code,
+                scriptLanguage: language,
+            }));
         }
 
         return Result.failure(`Cannot import '${path.toString()}: not a JS/TS file or codeblock reference.`);

src/index/datastore.ts

@@ -4,7 +4,7 @@ import { FolderIndex } from "index/storage/folder";
 import { InvertedIndex } from "index/storage/inverted";
 import { IndexPrimitive, IndexQuery, IndexSource } from "index/types/index-query";
 import { Indexable, LINKABLE_TYPE, LINKBEARING_TYPE, TAGGABLE_TYPE } from "index/types/indexable";
-import { MetadataCache, Vault } from "obsidian";
+import { MetadataCache, normalizePath, Vault } from "obsidian";
 import { MarkdownPage } from "./types/markdown";
 import { extractSubtags, normalizeHeaderForLink } from "utils/normalizers";
 import FlatQueue from "flatqueue";
@@ -14,6 +14,7 @@ import { IndexResolver, execute, optimizeQuery } from "index/storage/query-execu
 import { Result } from "api/result";
 import { Evaluator } from "expression/evaluator";
 import { Settings } from "settings";
+import path from "path-browserify";
 
 /** Central, index storage for datacore values. */
 export class Datastore {
@@ -266,15 +267,44 @@ export class Datastore {
         this.revision++;
     }
 
-    /** Find the corresponding object for a given link. */
-    public resolveLink(rawLink: string | Link, sourcePath?: string): Indexable | undefined {
+    public tryNormalizeLink(rawLink: string | Link, sourcePath?: string): Link | undefined {
         let link = typeof rawLink === "string" ? Link.parseInner(rawLink) : rawLink;
-
         if (sourcePath) {
             const linkdest = this.metadataCache.getFirstLinkpathDest(link.path, sourcePath);
             if (linkdest) link = link.withPath(linkdest.path);
         }
 
+        if (this.objects.has(link.path)) {
+            return link;
+        }
+
+        const normalizedSourcePath = normalizePath(sourcePath ?? "/");
+        const normalizedModuleParentDir = normalizePath(path.join(normalizedSourcePath, path.dirname(link.path)));
+        const resolvedModuleFile = this.folder
+            .getExact(normalizedModuleParentDir, (childPath) => {
+                const moduleBasename = path.basename(link.path);
+                const childFileBasename = path.basename(childPath);
+                return childFileBasename === moduleBasename || childFileBasename.startsWith(`${moduleBasename}.`);
+            })
+            .values()
+            .next()?.value;
+        if (resolvedModuleFile) {
+            return link.withPath(resolvedModuleFile);
+        }
+
+        return undefined;
+    }
+
+    public normalizeLink(rawLink: string | Link, sourcePath?: string): Link {
+        return (
+            this.tryNormalizeLink(rawLink, sourcePath) ??
+            (typeof rawLink === "string" ? Link.parseInner(rawLink) : rawLink)
+        );
+    }
+
+    /** Find the corresponding object for a given link. */
+    public resolveLink(rawLink: string | Link, sourcePath?: string): Indexable | undefined {
+        const link = this.normalizeLink(rawLink, sourcePath);
         const file = this.objects.get(link.path);
         if (!file) return undefined;
 
@@ -288,8 +318,9 @@ export class Datastore {
                 (sec) => normalizeHeaderForLink(sec.$title) == link.subpath || sec.$title == link.subpath
             );
 
-            if (section) return section;
-            else return undefined;
+            if (section) {
+                return section;
+            } else return undefined;
         } else if (link.type === "block") {
             for (const section of file.$sections) {
                 const block = section.$blocks.find((bl) => bl.$blockId === link.subpath);

src/index/types/json/common.ts

@@ -61,7 +61,7 @@ export namespace JsonConversion {
             const type = json["$_type"];
             switch (type) {
                 case "date":
-                    return normalizer(DateTime.fromISO(json.value));
+                    return normalizer(DateTime.fromISO(json.value, { setZone: true }));
                 case "duration":
                     return normalizer(Duration.fromISO(json.value));
                 case "link":