Handling file streaming and uploading to asset store upon processing …

…to DZI

apps/ove-core/src/server/projects/controller.ts

@@ -4,6 +4,7 @@ import path from "path";
 import fetch from "node-fetch";
 import { env } from "../../env";
 import { nanoid } from "nanoid";
+import unzip from "unzip-stream";
 import type { Client } from "minio";
 import service from "../auth/service";
 import { readFileSync } from "atomically";
@@ -14,6 +15,7 @@ import type { PrismaClient, Project, Section } from "@prisma/client";
 import { assert, Json, raise, titleToBucketName } from "@ove/ove-utils";
 
 import "@total-typescript/ts-reset";
+import http from "http";
 
 const getProjectsForUser = async (prisma: PrismaClient, username: string) => {
   const user = await prisma.user.findUnique({
@@ -221,12 +223,12 @@ const addLatest = <T extends {
   isLatest: boolean,
   lastModified: Date
 }>(files: T[]): T[] =>
-    files.concat(files.filter(({ isLatest }) => isLatest).map(file => ({
-      ...file,
-      versionId: "latest",
-      lastModified: new Date(),
-      isLatest: false
-    })));
+  files.concat(files.filter(({ isLatest }) => isLatest).map(file => ({
+    ...file,
+    versionId: "latest",
+    lastModified: new Date(),
+    isLatest: false
+  })));
 
 const getProjectFiles = async (s3: Client, bucketName: string) => {
   const files = await S3Controller.listObjects(s3, bucketName);
@@ -595,6 +597,45 @@ const formatData = async (
   }
 };
 
+const formatDZI = async (
+  s3: Client | null,
+  bucketName: string,
+  objectName: string,
+  versionId: string
+) => {
+  if (s3 === null) return raise("No S3 store configured");
+  const url = await getPresignedGetURL(s3, bucketName, objectName, versionId);
+  if (isError(url)) return url;
+  if (env.DATA_FORMATTER === undefined) return raise("No data formatter configured");
+  const data = Json.stringify({
+    "get_url": url
+  });
+  const formatter = new URL(env.DATA_FORMATTER);
+
+  const options = {
+    host: formatter.host,
+    port: formatter.port,
+    path: formatter.pathname,
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "Content-Length": Buffer.byteLength(data)
+    }
+  };
+
+  const req = http.request(options, res => {
+    res.pipe(unzip.Parse())
+      .on("entry", async entry => {
+        const dziObjectName = `${objectName.replaceAll(/png|jpg$/, "dzi")}/${entry.path}`;
+        const entryURL = await S3Controller.getPresignedPutURL(s3, bucketName, dziObjectName);
+        await fetch(entryURL, {method: "PUT", body: entry});
+      });
+  });
+
+  req.write(data);
+  req.end();
+};
+
 const controller: Controller = {
   getProjectsForUser,
   getProject,
@@ -613,7 +654,8 @@ const controller: Controller = {
   getLayout,
   getEnv,
   getController,
-  formatData
+  formatData,
+  formatDZI
 };
 
 export default controller;

apps/ove-core/src/server/projects/router.ts

@@ -131,6 +131,8 @@ export type Controller = {
     data: string
     fileName: string
   } | OVEException>
+  formatDZI: (s3: MinioClient | null, bucketName: string, objectName: string,
+    versionId: string) => Promise<void | OVEException>
 }
 
 export const projectsRouter = router({
@@ -462,5 +464,24 @@ export const projectsRouter = router({
       logger.info(`Formatting data of type ${dataType}`);
       return safe(logger, () =>
         controller.formatData(title, dataType, data, opts));
+    }),
+  formatDZI: protectedProcedure
+    .meta({
+      openapi: {
+        method: "POST",
+        path: "/project/{bucketName}/file/{objectName}/{versionId}/format/dzi",
+        protect: true
+      }
+    })
+    .input(z.strictObject({
+      bucketName: z.string(),
+      objectName: z.string(),
+      versionId: z.string()
+    }))
+    .output(z.union([z.void(), OVEExceptionSchema]))
+    .mutation(({ input: { bucketName, objectName, versionId }, ctx }) => {
+      logger.info("Converting image file to DZI");
+      return safe(logger, () =>
+        controller.formatDZI(ctx.s3, bucketName, objectName, versionId));
     })
 });

package.json

@@ -84,6 +84,7 @@
     "@types/react-dom": "18.2.9",
     "@types/react-router-dom": "5.3.3",
     "@types/swagger-ui-express": "^4.1.3",
+    "@types/unzip-stream": "^0.3.4",
     "@types/ws": "^8.5.5",
     "@types/yamljs": "^0.2.31",
     "@typescript-eslint/eslint-plugin": "^6.18.1",
@@ -209,6 +210,7 @@
     "trpc-openapi": "^1.1.2",
     "tsconfig-paths": "^4.2.0",
     "tslib": "^2.4.1",
+    "unzip-stream": "^0.3.4",
     "ws": "^8.13.0",
     "yamljs": "^0.3.0",
     "zod": "^3.22.4",

tools/data-formatting/Dockerfile

@@ -1,57 +1,69 @@
-FROM python:3.12.4-slim-bookworm
+FROM python:3.12.4-slim
 LABEL authors="Brython Caley-Davies <bc2918@ic.ac.uk>"
 
-RUN apk add --no-cache bash
-RUN apk --no-cache add --virtual build-base autoconf automake
+RUN apt-get update && apt-get install -y \
+        pkg-config \
+        wget \
+        build-essential \
+        ninja-build \
+    && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
-ARG VIPS_VERSION=8.8.3
+RUN pip3 install meson
+
+RUN apt-get install -y --no-install-recommends \
+    glib-2.0-dev \
+    libexpat-dev \
+    librsvg2-dev \
+    libpng-dev \
+    libspng-dev \
+    libjpeg-dev \
+    libexif-dev \
+    liblcms2-dev \
+    liborc-dev \
+    libtiff5-dev \
+    libexif-dev \
+    libwebp-dev \
+    libheif-dev \
+    libpoppler-glib-dev \
+    libfftw3-dev \
+    libopenjp2-7-dev \
+    libimagequant-dev \
+    libpango1.0-dev \
+    libcgif-dev \
+    libarchive-dev \
+    libjxl-dev \
+    && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
+
+ARG VIPS_VERSION=8.15.2
 ARG VIPS_URL=https://github.com/libvips/libvips/releases/download
 
-RUN apk update && apk upgrade
+WORKDIR /usr/local/src
+
+RUN wget ${VIPS_URL}/v${VIPS_VERSION}/vips-${VIPS_VERSION}.tar.xz
 
-# basic packages libvips likes
-# we need to figure out the dependencies we need
-RUN apk add \
-    libtool \
+RUN apt-get install -y --no-install-recommends \
     bc \
-	zlib-dev \
-	expat-dev \
-	jpeg-dev \
-	openjpeg-dev \
-	tiff-dev \
-	gdk-pixbuf-dev \
-	glib-dev \
-	libjpeg-turbo-dev \
-	libexif-dev \
-	lcms2-dev \
-	fftw-dev \
-	giflib-dev \
-	libpng-dev \
-	libwebp-dev \
-    libgsf-dev \
-	libxml2-dev \
-	orc-dev \
-	poppler-dev \
-	librsvg-dev \
-	openexr-dev \
-	sqlite-dev
-
-RUN ./configure && make && make install
-
-RUN wget -O- ${VIPS_URL}/v${VIPS_VERSION}/vips-${VIPS_VERSION}.tar.gz | tar xzC /tmp
-
-WORKDIR /tmp/vips-${VIPS_VERSION}
-
-RUN ./configure --prefix=/usr --disable-static --disable-debug && make V=0 && make install
-
-RUN rm -rf /tmp/vips-${VIPS_VERSION}
-
-RUN apk del build-base autoconf automake
+    && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
+
+RUN tar xf vips-${VIPS_VERSION}.tar.xz \
+    && cd vips-${VIPS_VERSION} \
+    && rm -rf build \
+    && meson build --libdir lib -Dintrospection=disabled --buildtype release \
+    && cd build \
+    && ninja \
+    && ninja test \
+    && ninja install
+
+ENV LD_LIBRARY_PATH=/usr/local/lib
+ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
+
+RUN pip install pyvips
 
 WORKDIR /app
 
 COPY requirements.txt requirements.txt
-RUN pip3 install -r requirements.txt
+RUN pip install --upgrade pip
+RUN pip install -r requirements.txt
 
 COPY . .