From 9bbcc7c7858f87eba6759289a273d4b7e004e6e3 Mon Sep 17 00:00:00 2001
From: sasha0552 <admin@sasha0552.org>
Date: Mon, 5 Feb 2024 16:26:18 +0000
Subject: [PATCH] Dedupe files at each dependency installation step on rocm

---
 .../scripts/1000-automatic-dependencies.sh                   | 5 +++++
 .../scripts/1000-axolotl-dependencies.sh.jinja2              | 5 +++++
 .../scripts/1000-comfyui-dependencies.sh.jinja2              | 5 +++++
 .../scripts/1000-sillytavern-extras-dependencies.sh.jinja2   | 5 +++++
 .../1000-text-generation-webui-dependencies.sh.jinja2        | 5 +++++
 .../scripts/1000-vllm-dependencies.sh.jinja2                 | 5 +++++
 packages.x86_64.jinja2                                       | 3 +++
 7 files changed, 33 insertions(+)

diff --git a/airootfs/root/customize_airootfs/scripts/1000-automatic-dependencies.sh b/airootfs/root/customize_airootfs/scripts/1000-automatic-dependencies.sh
index 5dd48af..581d939 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-automatic-dependencies.sh
+++ b/airootfs/root/customize_airootfs/scripts/1000-automatic-dependencies.sh
@@ -21,3 +21,8 @@ pushd "automatic"
   # remove installation log
   rm sdnext.log
 popd
+
+{% if ROCm %}
+# Dedupe files
+rdfind -checksum sha256 -makehardlinks true .
+{% endif %}
diff --git a/airootfs/root/customize_airootfs/scripts/1000-axolotl-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-axolotl-dependencies.sh.jinja2
index 74fb4c3..ddafa10 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-axolotl-dependencies.sh.jinja2
+++ b/airootfs/root/customize_airootfs/scripts/1000-axolotl-dependencies.sh.jinja2
@@ -37,3 +37,8 @@ pushd "axolotl"
   deactivate
 {% endif %}
 popd
+
+{% if ROCm %}
+# Dedupe files
+rdfind -checksum sha256 -makehardlinks true .
+{% endif %}
diff --git a/airootfs/root/customize_airootfs/scripts/1000-comfyui-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-comfyui-dependencies.sh.jinja2
index 767b269..84b4328 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-comfyui-dependencies.sh.jinja2
+++ b/airootfs/root/customize_airootfs/scripts/1000-comfyui-dependencies.sh.jinja2
@@ -23,3 +23,8 @@ pushd "ComfyUI"
     pip3 install -r requirements.txt
   deactivate
 popd
+
+{% if ROCm %}
+# Dedupe files
+rdfind -checksum sha256 -makehardlinks true .
+{% endif %}
diff --git a/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2
index f27537a..f88b3bb 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2
+++ b/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2
@@ -26,3 +26,8 @@ pushd "SillyTavern-Extras"
     pip3 install -r requirements-rvc.txt
   deactivate
 popd
+
+{% if ROCm %}
+# Dedupe files
+rdfind -checksum sha256 -makehardlinks true .
+{% endif %}
diff --git a/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2
index c4ece10..abcd6f4 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2
+++ b/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2
@@ -31,3 +31,8 @@ pushd "text-generation-webui"
     pip3 install -r extensions/openai/requirements.txt
   deactivate
 popd
+
+{% if ROCm %}
+# Dedupe files
+rdfind -checksum sha256 -makehardlinks true .
+{% endif %}
diff --git a/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
index ceb2644..d23fc5a 100644
--- a/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
+++ b/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
@@ -46,3 +46,8 @@ pushd "vllm"
   deactivate
 {% endif %}
 popd
+
+{% if ROCm %}
+# Dedupe files
+rdfind -checksum sha256 -makehardlinks true .
+{% endif %}
diff --git a/packages.x86_64.jinja2 b/packages.x86_64.jinja2
index 50f9592..39a654f 100644
--- a/packages.x86_64.jinja2
+++ b/packages.x86_64.jinja2
@@ -47,4 +47,7 @@ rocm-hip-sdk
 
 # Required for SillyTavern-Extras
 gtk3
+
+# File deduplication
+rdfind
 {% endif %}