Integrating DZI support into core-ui, fixing issues with file upload …

…to asset store and Dockerfile package not found errors

apps/ove-core-ui/src/pages/project-editor/file-upload/file-upload.module.scss

@@ -21,6 +21,7 @@
   & ul {
     width: 100%;
     display: flex;
+    color: white;
     flex-direction: column;
     align-items: center;
     overflow-y: scroll;
@@ -33,7 +34,7 @@
       margin-top: 0.5rem;
       display: flex;
 
-      & select {
+      .ml {
         margin-left: auto;
       }
     }

apps/ove-core-ui/src/pages/project-editor/file-upload/upload.tsx

@@ -1,9 +1,11 @@
+import { toast } from "sonner";
 import { actionColors } from "../utils";
+import { trpc } from "../../../utils/api";
 import type { FileUploadForm } from "./file-upload";
 import type { UseFormRegister } from "react-hook-form";
 import React, { FormEventHandler, useRef } from "react";
 import { type File as FileT, dataTypes } from "@ove/ove-types";
-import { Brush, Upload as UploadButton, X } from "react-bootstrap-icons";
+import { Brush, Gear, Upload as UploadButton, X } from "react-bootstrap-icons";
 
 import styles from "./file-upload.module.scss";
 
@@ -36,21 +38,32 @@ const Upload = ({
   const { ref, ...rest } = register("file", { required: true });
   const fileRef = useRef<HTMLInputElement | null>(null);
   const formRef = useRef<HTMLFormElement | null>(null);
+  const processImage = trpc.projects.formatDZI.useMutation({ retry: false });
 
   return <section
     id={styles["upload"]}>
     <header>
       <h2>Project Files</h2>
       <button onClick={closeDialog}><X size="1.25rem" /></button>
     </header>
-    <ul style={{ color: "white" }}>
+    <ul>
       {names.map((name, i) => {
         const [bucketName, fileName] = name.split("/");
+        const isImage = name.match(/.*(?:png|jpg|jpeg|PNG|JPG|JPEG)$/g) !== null;
         return <li key={name}
                    style={{ backgroundColor: colors[i % names.length] }}>
           {name}
+          {isImage ? <button className={styles.ml}
+            onClick={() => processImage.mutateAsync({
+              bucketName,
+              objectName: fileName,
+              versionId: getLatest(bucketName, fileName).version
+            })
+              .then(() => toast.info(`Converted ${fileName} to DZI`))
+              .catch(() => toast.error(`Error converting ${fileName} to DZI`))}>
+            <Gear /></button> : null}
           {/* @ts-expect-error - readOnly prop is not known on type */}
-          <select readOnly={true}
+          <select readOnly={true} className={isImage ? undefined : styles.ml}
                   value={getLatest(bucketName, fileName).version}
                   style={{ backgroundColor: colors[i % names.length] }}>
             {files.filter(file =>

apps/ove-core-ui/src/pages/project-editor/hooks.ts

@@ -521,7 +521,10 @@ export const useFiles = (projectId: string, token: string) => {
       assert(name.split("/").at(-1)) : name;
     const file = files.find(file =>
       file.name === name && file.bucketName === bucketName && file.isLatest);
-    if (!file) throw new Error("File not found");
+    if (!file) {
+      console.log(bucketName, name);
+      throw new Error("File not found");
+    }
     return file;
   };
 

apps/ove-core/src/server/projects/controller.ts

@@ -1,10 +1,12 @@
 /* global __dirname */
 
+import http from "http";
 import path from "path";
+import { File } from "buffer";
 import fetch from "node-fetch";
 import { env } from "../../env";
 import { nanoid } from "nanoid";
-import unzip from "unzip-stream";
+import unzip, { Entry } from "unzip-stream";
 import type { Client } from "minio";
 import service from "../auth/service";
 import { readFileSync } from "atomically";
@@ -15,7 +17,6 @@ import type { PrismaClient, Project, Section } from "@prisma/client";
 import { assert, Json, raise, titleToBucketName } from "@ove/ove-utils";
 
 import "@total-typescript/ts-reset";
-import http from "http";
 
 const getProjectsForUser = async (prisma: PrismaClient, username: string) => {
   const user = await prisma.user.findUnique({
@@ -231,7 +232,9 @@ const addLatest = <T extends {
   })));
 
 const getProjectFiles = async (s3: Client, bucketName: string) => {
-  const files = await S3Controller.listObjects(s3, bucketName);
+  const files = (await S3Controller.listObjects(s3, bucketName))
+    .filter(obj => !obj.name.includes("/") || obj.name.endsWith("dzi"))
+    .map(obj => ({...obj, name: obj.name.includes("/") ? obj.name.split("/")[0] : obj.name}));
   return Object.values(groupBy(files, "name"))
     .map(group => group.sort((a, b) =>
       a.lastModified.getTime() - b.lastModified.getTime())
@@ -244,7 +247,9 @@ const getProjectFiles = async (s3: Client, bucketName: string) => {
 const getGlobalFiles = async (s3: Client): ReturnType<Controller["getFiles"]> =>
   (await Promise.all(env.ASSET_STORE_CONFIG?.GLOBAL_BUCKETS
     .map(async bucket => {
-      const objects = await S3Controller.listObjects(s3, bucket);
+      const objects = (await S3Controller.listObjects(s3, bucket))
+        .filter(obj => !obj.name.includes("/") || obj.name.endsWith("dzi"))
+        .map(obj => ({...obj, name: obj.name.includes("/") ? obj.name.split("/")[0] : obj.name}));
       return Object.values(groupBy(objects, "name"))
         .map(group => group.sort((a, b) =>
           a.lastModified.getTime() - b.lastModified.getTime())
@@ -607,33 +612,50 @@ const formatDZI = async (
   const url = await getPresignedGetURL(s3, bucketName, objectName, versionId);
   if (isError(url)) return url;
   if (env.DATA_FORMATTER === undefined) return raise("No data formatter configured");
-  const data = Json.stringify({
-    "get_url": url
-  });
   const formatter = new URL(env.DATA_FORMATTER);
+  await new Promise(resolve => {
+    const data = Json.stringify({
+      "get_url": url
+    });
 
-  const options = {
-    host: formatter.host,
-    port: formatter.port,
-    path: formatter.pathname,
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      "Content-Length": Buffer.byteLength(data)
-    }
-  };
-
-  const req = http.request(options, res => {
-    res.pipe(unzip.Parse())
-      .on("entry", async entry => {
-        const dziObjectName = `${objectName.replaceAll(/png|jpg$/, "dzi")}/${entry.path}`;
-        const entryURL = await S3Controller.getPresignedPutURL(s3, bucketName, dziObjectName);
-        await fetch(entryURL, {method: "PUT", body: entry});
-      });
-  });
+    const options = {
+      host: formatter.hostname,
+      port: formatter.port,
+      path: `${formatter.pathname}/dzi`,
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Content-Length": Buffer.byteLength(data)
+      }
+    };
+
+    const req = http.request(options, res => {
+      res.pipe(unzip.Parse())
+        .on("entry", async (entry: Entry) => {
+          const dziObjectName = `${objectName.replaceAll(/(?:png|jpg|jpeg|PNG|JPEG|JPG)$/g, "dzi")}/${entry.path}`;
+          const entryURL = await S3Controller.getPresignedPutURL(s3, bucketName, dziObjectName);
+
+          const [size, chunks] = await new Promise<[number, any[]]>(resolve => {
+            let size = 0;
+            const chunks: any[] = [];
+            entry.on("data", chunk => {
+              size += chunk.length;
+              chunks.push(chunk);
+            }).on("end", () => {
+              resolve([size, chunks]);
+            });
+          });
+          await fetch(entryURL, {
+            method: "PUT",
+            body: await (new File([Buffer.from(chunks)], entry.path)).arrayBuffer(),
+            headers: { "Content-Length": `${size}` }
+          });
+        }).on("finish", resolve);
+    });
 
-  req.write(data);
-  req.end();
+    req.write(data);
+    req.end();
+  });
 };
 
 const controller: Controller = {

tools/data-formatting/Dockerfile

@@ -5,8 +5,7 @@ RUN apt-get update && apt-get install -y \
         pkg-config \
         wget \
         build-essential \
-        ninja-build \
-    && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
+        ninja-build
 
 RUN pip3 install meson
 
@@ -31,8 +30,7 @@ RUN apt-get install -y --no-install-recommends \
     libpango1.0-dev \
     libcgif-dev \
     libarchive-dev \
-    libjxl-dev \
-    && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
+    libjxl-dev
 
 ARG VIPS_VERSION=8.15.2
 ARG VIPS_URL=https://github.com/libvips/libvips/releases/download
@@ -42,8 +40,7 @@ WORKDIR /usr/local/src
 RUN wget ${VIPS_URL}/v${VIPS_VERSION}/vips-${VIPS_VERSION}.tar.xz
 
 RUN apt-get install -y --no-install-recommends \
-    bc \
-    && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
+    bc
 
 RUN tar xf vips-${VIPS_VERSION}.tar.xz \
     && cd vips-${VIPS_VERSION} \

tools/data-formatting/src/v1/controller.py

@@ -8,7 +8,7 @@
 from datetime import datetime
 from dotenv import load_dotenv
 from markdown import markdown as md
-from stream_zip import ZIP_64, stream_zip
+from stream_zip import ZIP_32, stream_zip
 from src.locks import LATEX_LOCK, MARKDOWN_LOCK
 from IPython.lib.latextools import latex_to_html
 from tempfile import NamedTemporaryFile, TemporaryDirectory
@@ -57,6 +57,6 @@ def dzi():
         continue
 
       with open(os.path.join(folder, filename), mode="rb") as f:
-        files.append((filename, datetime.now(), S_IFREG | 0o600, ZIP_64, (f.read(),)))
+        files.append((filename, datetime.now(), S_IFREG | 0o600, ZIP_32, (f.read(),)))
 
     return stream_zip(files)