Adding file upload tool for storing assets into S3 and adding DZI con…

…version support to the data-formatting tool

tools/data-formatting/Dockerfile

@@ -1,6 +1,53 @@
-FROM python:3.9-slim-buster
+FROM python:3.12.4-slim-bookworm
 LABEL authors="Brython Caley-Davies <bc2918@ic.ac.uk>"
 
+RUN apk add --no-cache bash
+RUN apk --no-cache add --virtual build-base autoconf automake
+
+ARG VIPS_VERSION=8.8.3
+ARG VIPS_URL=https://github.com/libvips/libvips/releases/download
+
+RUN apk update && apk upgrade
+
+# basic packages libvips likes
+# we need to figure out the dependencies we need
+RUN apk add \
+    libtool \
+    bc \
+	zlib-dev \
+	expat-dev \
+	jpeg-dev \
+	openjpeg-dev \
+	tiff-dev \
+	gdk-pixbuf-dev \
+	glib-dev \
+	libjpeg-turbo-dev \
+	libexif-dev \
+	lcms2-dev \
+	fftw-dev \
+	giflib-dev \
+	libpng-dev \
+	libwebp-dev \
+    libgsf-dev \
+	libxml2-dev \
+	orc-dev \
+	poppler-dev \
+	librsvg-dev \
+	openexr-dev \
+	sqlite-dev
+
+RUN ./configure && make && make install
+
+RUN wget -O- ${VIPS_URL}/v${VIPS_VERSION}/vips-${VIPS_VERSION}.tar.gz | tar xzC /tmp
+
+WORKDIR /tmp/vips-${VIPS_VERSION}
+
+RUN ./configure --prefix=/usr --disable-static --disable-debug && make V=0 && make install
+
+RUN rm -rf /tmp/vips-${VIPS_VERSION}
+
+RUN apk del build-base autoconf automake
+
 WORKDIR /app
 
 COPY requirements.txt requirements.txt

tools/data-formatting/app.py

@@ -19,5 +19,10 @@ def latex():
   return controllerV1.latex()
 
 
+@app.route("/v1/dzi", methods=["POST"])
+def dzi():
+  return controllerV1.dzi()
+
+
 if __name__ == "__main__":
   app.run()

tools/data-formatting/requirements.txt

@@ -2,4 +2,5 @@ markdown==3.6
 jupyter==1.0.0
 Flask==3.0.3
 python-dotenv==1.0.1
-matplotlib==3.9.0
+matplotlib==3.9.1
+pyvips==2.2.3

tools/data-formatting/src/v1/controller.py

@@ -25,3 +25,15 @@ def latex():
     if "$$" not in data:
       data = data.replace("$", "$$")
     return latex_to_html(data)
+
+
+def dzi():
+  data = request.get_json()
+  response = requests.get(data["get_url"])
+  if response.status_code != 200:
+    raise ConnectionError("Unable to get file")
+  with TemporaryDirectory() as folder:
+    with NamedTemporaryFile() as image_file:
+      image_file.write(response.content)
+      pyvips.Image.new_from_file(image_file.name).dzsave(folder + "/image", suffix=".png")
+      # TODO: return assets as zip file

tools/upload-file/.env.example

@@ -0,0 +1,5 @@
+ASSET_STORE_ACCESS_KEY=
+ASSET_STORE_SECRET_KEY=
+ASSET_STORE_USE_SSL=
+ASSET_STORE_END_POINT=
+ASSET_STORE_PORT=

tools/upload-file/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "upload-file",
+  "version": "1.0.0",
+  "description": "Utility for uploading files to Minio asset storage",
+  "main": "upload-file.js",
+  "scripts": {
+    "start": "node upload-file.js"
+  },
+  "dependencies": {
+    "dotenv": "latest",
+    "minio": "latest",
+    "prompt": "latest"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dsi-icl/next-ove.git"
+  },
+  "private": true
+}

tools/upload-file/upload-file.js

@@ -0,0 +1,42 @@
+const prompt = require('prompt');
+const dotenv = require('dotenv');
+const Minio = require('minio');
+
+dotenv.config();
+
+const s3 = new Minio.Client({
+  endPoint: process.env.ASSET_STORE_END_POINT,
+  port: parseInt(process.env.ASSET_STORE_PORT),
+  useSSL: process.env.ASSET_STORE_USE_SSL === 'true',
+  accessKey: process.env.ASSET_STORE_ACCESS_KEY,
+  secretKey: process.env.ASSET_STORE_SECRET_KEY
+});
+
+/** @type { () => Promise<{objectName: string, filePath: string | null | undefined, projectTitle: string}> } */
+const getDetails = () => {
+  prompt.start();
+  prompt.message = 'Enter project title, the object\'s name and the file\'s path:\n';
+  prompt.delimiter = '';
+  return prompt.get({
+    properties: {
+      projectTitle: {
+        message: 'project title:',
+        required: true
+      },
+      objectName: {
+        message: 'object name:',
+        required: true
+      },
+      filePath: {
+        message: 'file path:'
+      }
+    }
+  });
+};
+
+const run = async () => {
+  const { projectTitle, objectName, filePath } = await getDetails();
+  s3.fPutObject(projectTitle, objectName, filePath === null || filePath === undefined ? objectName : filePath).catch(console.error);
+};
+
+run().catch(console.error);