Dedupe files at each dependency installation step on rocm

sasha0552 · Feb 5, 2024 · 9bbcc7c · 9bbcc7c
1 parent 3d9c1ab
commit 9bbcc7c
Show file tree

Hide file tree

Showing 7 changed files with 33 additions and 0 deletions.
diff --git a/airootfs/root/customize_airootfs/scripts/1000-automatic-dependencies.sh b/airootfs/root/customize_airootfs/scripts/1000-automatic-dependencies.sh
@@ -21,3 +21,8 @@ pushd "automatic"
   # remove installation log
   rm sdnext.log
 popd
+
+{% if ROCm %}
+# Dedupe files
+rdfind -checksum sha256 -makehardlinks true .
+{% endif %}
diff --git a/airootfs/root/customize_airootfs/scripts/1000-axolotl-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-axolotl-dependencies.sh.jinja2
@@ -37,3 +37,8 @@ pushd "axolotl"
   deactivate
 {% endif %}
 popd
+
+{% if ROCm %}
+# Dedupe files
+rdfind -checksum sha256 -makehardlinks true .
+{% endif %}
diff --git a/airootfs/root/customize_airootfs/scripts/1000-comfyui-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-comfyui-dependencies.sh.jinja2
@@ -23,3 +23,8 @@ pushd "ComfyUI"
     pip3 install -r requirements.txt
   deactivate
 popd
+
+{% if ROCm %}
+# Dedupe files
+rdfind -checksum sha256 -makehardlinks true .
+{% endif %}
diff --git a/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-sillytavern-extras-dependencies.sh.jinja2
@@ -26,3 +26,8 @@ pushd "SillyTavern-Extras"
     pip3 install -r requirements-rvc.txt
   deactivate
 popd
+
+{% if ROCm %}
+# Dedupe files
+rdfind -checksum sha256 -makehardlinks true .
+{% endif %}
diff --git a/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-text-generation-webui-dependencies.sh.jinja2
@@ -31,3 +31,8 @@ pushd "text-generation-webui"
     pip3 install -r extensions/openai/requirements.txt
   deactivate
 popd
+
+{% if ROCm %}
+# Dedupe files
+rdfind -checksum sha256 -makehardlinks true .
+{% endif %}
diff --git a/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2 b/airootfs/root/customize_airootfs/scripts/1000-vllm-dependencies.sh.jinja2
@@ -46,3 +46,8 @@ pushd "vllm"
   deactivate
 {% endif %}
 popd
+
+{% if ROCm %}
+# Dedupe files
+rdfind -checksum sha256 -makehardlinks true .
+{% endif %}
diff --git a/packages.x86_64.jinja2 b/packages.x86_64.jinja2
@@ -47,4 +47,7 @@ rocm-hip-sdk
 
 # Required for SillyTavern-Extras
 gtk3
+
+# File deduplication
+rdfind
 {% endif %}