From 07e43f1cbf97741a0c077e81864cea7924863b46 Mon Sep 17 00:00:00 2001 From: noah Date: Mon, 3 Jun 2024 14:38:41 -0700 Subject: [PATCH] Layered rendering util (#24) * well its a start * apostrophy * mostly just move ReglLayer2D over to packages, and make a minor change (that should hopefully be less surprising) to long-running-frame lifecycle callbacks * bump the version --- apps/layers/src/demo.ts | 3 +- apps/layers/src/types.ts | 2 +- packages/scatterbrain/package.json | 9 ++--- packages/scatterbrain/src/index.ts | 2 ++ .../scatterbrain/src/layers/buffer-pair.ts | 0 .../scatterbrain/src/layers/layer-2D.ts | 34 +++++++++---------- packages/scatterbrain/src/layers/types.ts | 23 +++++++++++++ packages/scatterbrain/src/render-queue.ts | 16 ++++++--- packages/scatterbrain/tsconfig.json | 1 + pnpm-lock.yaml | 9 +++++ 10 files changed, 69 insertions(+), 30 deletions(-) rename apps/common/src/bufferPair.ts => packages/scatterbrain/src/layers/buffer-pair.ts (100%) rename apps/layers/src/layer.ts => packages/scatterbrain/src/layers/layer-2D.ts (72%) create mode 100644 packages/scatterbrain/src/layers/types.ts diff --git a/apps/layers/src/demo.ts b/apps/layers/src/demo.ts index 04eb169..edba0fe 100644 --- a/apps/layers/src/demo.ts +++ b/apps/layers/src/demo.ts @@ -1,10 +1,9 @@ import { Box2D, Vec2, type box2D, type vec2 } from "@alleninstitute/vis-geometry"; import { type ColumnRequest } from "Common/loaders/scatterplot/scatterbrain-loader"; import REGL from "regl"; -import { AsyncDataCache, type FrameLifecycle, type NormalStatus } from "@alleninstitute/vis-scatterbrain"; +import { AsyncDataCache, ReglLayer2D, type FrameLifecycle, type NormalStatus } from "@alleninstitute/vis-scatterbrain"; import { buildRenderer } from "../../scatterplot/src/renderer"; import { buildImageRenderer } from "../../omezarr-viewer/src/image-renderer"; -import { ReglLayer2D } from "./layer"; import { renderDynamicGrid, renderSlide, type RenderSettings as SlideRenderSettings } from "./data-renderers/dynamicGridSlideRenderer"; import { renderGrid, renderSlice, type RenderSettings as SliceRenderSettings } from "./data-renderers/volumeSliceRenderer"; import { renderAnnotationLayer, type RenderSettings as AnnotationRenderSettings, type SimpleAnnotation } from "./data-renderers/simpleAnnotationRenderer"; diff --git a/apps/layers/src/types.ts b/apps/layers/src/types.ts index e0a1bca..7c754c7 100644 --- a/apps/layers/src/types.ts +++ b/apps/layers/src/types.ts @@ -1,6 +1,5 @@ import type { box2D } from "@alleninstitute/vis-geometry"; import type REGL from "regl"; -import type { ReglLayer2D } from "./layer"; import { type RenderSettings as SlideRenderSettings } from "./data-renderers/dynamicGridSlideRenderer"; import { type RenderSettings as SliceRenderSettings } from "./data-renderers/volumeSliceRenderer"; import { type RenderSettings as AnnotationRenderSettings, type SimpleAnnotation } from "./data-renderers/simpleAnnotationRenderer"; @@ -9,6 +8,7 @@ import type { DynamicGrid, DynamicGridSlide } from "./data-sources/scatterplot/d import type { AxisAlignedZarrSliceGrid } from "./data-sources/ome-zarr/slice-grid"; import type { RenderSettings as AnnotationGridRenderSettings, CacheContentType as GpuMesh } from "./data-renderers/annotation-renderer"; import type { AnnotationGrid } from "./data-sources/annotation/annotation-grid"; +import type { ReglLayer2D } from "@alleninstitute/vis-scatterbrain"; // note: right now, all layers should be considered 2D, and WebGL only... export type Image = { texture: REGL.Framebuffer2D diff --git a/packages/scatterbrain/package.json b/packages/scatterbrain/package.json index c965589..f53f3f3 100644 --- a/packages/scatterbrain/package.json +++ b/packages/scatterbrain/package.json @@ -1,6 +1,6 @@ { "name": "@alleninstitute/vis-scatterbrain", - "version": "0.0.2", + "version": "0.0.3", "contributors": [ { "name": "Lane Sawyer", @@ -54,7 +54,8 @@ "vitest": "^1.4.0" }, "dependencies": { - "lodash": "^4.17.21" - + "@alleninstitute/vis-geometry": "workspace:*", + "lodash": "^4.17.21", + "regl": "^2.1.0" } -} +} \ No newline at end of file diff --git a/packages/scatterbrain/src/index.ts b/packages/scatterbrain/src/index.ts index 0b2c27e..b9ea96b 100644 --- a/packages/scatterbrain/src/index.ts +++ b/packages/scatterbrain/src/index.ts @@ -1,2 +1,4 @@ export { beginLongRunningFrame } from "./render-queue"; export { AsyncDataCache } from "./dataset-cache"; +export { ReglLayer2D } from './layers/layer-2D' +export * from './layers/buffer-pair' \ No newline at end of file diff --git a/apps/common/src/bufferPair.ts b/packages/scatterbrain/src/layers/buffer-pair.ts similarity index 100% rename from apps/common/src/bufferPair.ts rename to packages/scatterbrain/src/layers/buffer-pair.ts diff --git a/apps/layers/src/layer.ts b/packages/scatterbrain/src/layers/layer-2D.ts similarity index 72% rename from apps/layers/src/layer.ts rename to packages/scatterbrain/src/layers/layer-2D.ts index 7cec024..ccb5b15 100644 --- a/apps/layers/src/layer.ts +++ b/packages/scatterbrain/src/layers/layer-2D.ts @@ -1,20 +1,14 @@ -import { Box2D, type box2D, type vec2 } from "@alleninstitute/vis-geometry"; + +// a helper to render a 2D layer, using regl +import type { Image, ImageRenderer, RenderFn } from "./types"; +import { type BufferPair, swapBuffers } from "./buffer-pair"; import type REGL from "regl"; -import { swapBuffers, type BufferPair } from "../../common/src/bufferPair"; -import type { Image } from "./types"; -import type { FrameLifecycle, NormalStatus } from "@alleninstitute/vis-scatterbrain"; -import type { Camera } from "./data-renderers/types"; -import type { buildImageRenderer } from "../../omezarr-viewer/src/image-renderer"; +import type { FrameLifecycle, RenderCallback } from "../render-queue"; +import { Box2D, type box2D, type vec2 } from '@alleninstitute/vis-geometry' -type RenderFn = - (target: REGL.Framebuffer2D | null, thing: Readonly, settings: Readonly) => FrameLifecycle; -type ImageRenderer = ReturnType -type RenderCallback = (event: { status: NormalStatus } | { status: 'error', error: unknown }) => void; type EventType = Parameters[0] -type RequiredSettings = { camera: Camera, callback: RenderCallback } -/** - * a class that makes it easy to manage rendering 2D layers using regl - */ +type RequiredSettings = { camera: { view: box2D }, callback: RenderCallback } + export class ReglLayer2D { private buffers: BufferPair; private renderFn: RenderFn @@ -23,8 +17,8 @@ export class ReglLayer2D { private renderImg: ImageRenderer constructor(regl: REGL.Regl, imgRenderer: ImageRenderer, renderFn: RenderFn, resolution: vec2) { this.buffers = { - readFrom: { texture: regl.framebuffer(...resolution), bounds: undefined }, - writeTo: { texture: regl.framebuffer(...resolution), bounds: undefined } + readFrom: { resolution, texture: regl.framebuffer(...resolution), bounds: undefined }, + writeTo: { resolution, texture: regl.framebuffer(...resolution), bounds: undefined } }; this.renderImg = imgRenderer this.regl = regl; @@ -44,7 +38,7 @@ export class ReglLayer2D { onChange(props: { readonly data: Readonly; readonly settings: Readonly - }, cancel:boolean=true) { + }, cancel: boolean = true) { if (cancel && this.runningFrame) { this.runningFrame.cancelFrame(); @@ -52,10 +46,14 @@ export class ReglLayer2D { const { readFrom, writeTo } = this.buffers; // copy our work to the prev-buffer... if (readFrom.bounds && writeTo.bounds && Box2D.intersection(readFrom.bounds, writeTo.bounds)) { + const [width, height] = writeTo.resolution; this.renderImg({ box: Box2D.toFlatArray(writeTo.bounds), img: writeTo.texture, target: readFrom.texture, + viewport: { + x: 0, y: 0, width, height + }, view: Box2D.toFlatArray(readFrom.bounds) }) } @@ -76,7 +74,7 @@ export class ReglLayer2D { case 'finished_synchronously': this.buffers = swapBuffers(this.buffers); // only erase... if we would have cancelled... - if(cancel){ + if (cancel) { this.regl.clear({ framebuffer: this.buffers.writeTo.texture, color: [0, 0, 0, 0], depth: 1 }) } this.runningFrame = null; diff --git a/packages/scatterbrain/src/layers/types.ts b/packages/scatterbrain/src/layers/types.ts new file mode 100644 index 0000000..8801f3e --- /dev/null +++ b/packages/scatterbrain/src/layers/types.ts @@ -0,0 +1,23 @@ +import type { box2D, vec2, vec4 } from '@alleninstitute/vis-geometry'; +import type { FrameLifecycle } from '../render-queue' +import type REGL from "regl"; + +export type RenderFn = + (target: REGL.Framebuffer2D | null, thing: Readonly, settings: Readonly) => FrameLifecycle; + +export type Image = { + resolution: vec2; + texture: REGL.Framebuffer2D; + bounds: box2D | undefined; // if undefined, it means we allocated the texture, but its empty and should not be used (except to fill it) +} + +type ImageRendererProps = { + target: REGL.Framebuffer2D | null; + box: vec4; + view: vec4; + viewport: REGL.BoundingBox; + img: REGL.Texture2D | REGL.Framebuffer2D; +} + +// a function which renders an axis aligned image to another axis aligned image - no funny buisness +export type ImageRenderer = (props: ImageRendererProps) => void; \ No newline at end of file diff --git a/packages/scatterbrain/src/render-queue.ts b/packages/scatterbrain/src/render-queue.ts index b3239f7..7e79dad 100644 --- a/packages/scatterbrain/src/render-queue.ts +++ b/packages/scatterbrain/src/render-queue.ts @@ -25,7 +25,7 @@ export type FrameLifecycle = { * `progress` - The frame is still running and has not finished */ export type NormalStatus = 'begun' | 'finished' | 'cancelled' | 'finished_synchronously' | 'progress'; - +export type RenderCallback = (event: { status: NormalStatus } | { status: 'error', error: unknown }) => void; /** * `beingLongRunningFrame` starts a long-running frame that will render a list of items asynchronously based on * the provided data, settings, and rendering functions. @@ -59,17 +59,23 @@ export function beginLongRunningFrame( settings: Settings, requestsForItem: (item: Item, settings: Settings, signal?: AbortSignal) => Record Promise>, render: (item: Item, settings: Settings, columns: Record) => void, - lifecycleCallback: (event: { status: NormalStatus } | { status: 'error'; error: unknown }) => void, + lifecycleCallback: RenderCallback, cacheKeyForRequest: (requestKey: string, item: Item, settings: Settings) => string = (key) => key, queueTimeBudgetMS: number = queueProcessingIntervalMS / 3 ): FrameLifecycle { const abort = new AbortController(); - const reportNormalStatus = (status: NormalStatus) => { - lifecycleCallback({ status }); - }; const queue: Item[] = []; const taskCancelCallbacks: Array<() => void> = []; + const reportNormalStatus = (status: NormalStatus) => { + // we want to report our status, however the flow of events can be confusing - + // our callers anticipate an asynchronous (long running) frame to be started, + // but there are scenarios in which the whole thing is completely synchronous + // callers who are scheduling things may be surprised that their frame finished + // before the code that handles it appears to start. thus, we make the entire lifecycle callback + // system async, to prevent surprises. + Promise.resolve().then(() => lifecycleCallback({ status })); + }; // when starting a frame, we greedily attempt to render any tasks that are already in the cache // however, if there is too much overhead (or too many tasks) we would risk hogging the main thread // thus - obey the limit (its a soft limit) diff --git a/packages/scatterbrain/tsconfig.json b/packages/scatterbrain/tsconfig.json index ac6b374..d9869b8 100644 --- a/packages/scatterbrain/tsconfig.json +++ b/packages/scatterbrain/tsconfig.json @@ -6,6 +6,7 @@ "./*" ] }, + "moduleResolution": "Bundler", "module": "ES2022", "target": "ES2022", "lib": [ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b33bf6f..0767af5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -212,9 +212,15 @@ importers: packages/scatterbrain: dependencies: + '@alleninstitute/vis-geometry': + specifier: workspace:* + version: link:../geometry lodash: specifier: ^4.17.21 version: 4.17.21 + regl: + specifier: ^2.1.0 + version: 2.1.0 devDependencies: '@parcel/packager-ts': specifier: ^2.12.0 @@ -3454,6 +3460,9 @@ packages: resolution: {integrity: sha512-W+gxAq7aQ9dJIg/XLKGcRT0cvnStFAQHPaI0pvD0U2l6IVLueUAm3nwN7lkY62zZNmlvNx6jNtE4wlbS+CyqSg==} engines: {node: '>= 12.0.0'} hasBin: true + peerDependenciesMeta: + '@parcel/core': + optional: true dependencies: '@parcel/config-default': 2.12.0(@parcel/core@2.12.0)(typescript@5.3.3) '@parcel/core': 2.12.0