Clean up unused attachments (#269)

app/models/clean_up_unused_attachments.rb

@@ -0,0 +1,13 @@
+module CleanUpUnusedAttachments
+  def self.perform
+    S3File
+      .attachments
+      .where('became_unused_at < ?', 24.hours.ago)
+      .where(do_not_delete_unused: false)
+      .find_each do |s3_file|
+        # Double check became_unused_at
+        s3_file.update_unused
+        s3_file.destroy! if s3_file.unused?
+      end
+  end
+end

app/models/document.rb

@@ -87,9 +87,16 @@ def update_linked_s3_files
       end
     end
 
-    self.s3_files = s3_file_ids.uniq.map do |s3_file_id|
-      owner.s3_files.find_by(id: s3_file_id)
-    end.compact
+    documents_s3_files.each do |documents_s3_file|
+      if s3_file_ids.exclude?(documents_s3_file.s3_file_id)
+        documents_s3_file.destroy
+      end
+    end
+
+    s3_file_ids.uniq.each do |s3_file_id|
+      s3_file = owner.s3_files.find_by(id: s3_file_id)
+      documents_s3_files.find_or_create_by(s3_file: s3_file) unless s3_file.nil?
+    end
   end
 
   def upsert_to_typesense(collection: self.class.typesense_collection)

app/models/documents_s3_file.rb

@@ -1,4 +1,10 @@
 class DocumentsS3File < ApplicationRecord
   belongs_to :document
   belongs_to :s3_file
+
+  after_commit :update_s3_file_unused
+
+  def update_s3_file_unused
+    s3_file.update_unused unless s3_file.destroyed?
+  end
 end

app/models/s3_file.rb

@@ -1,6 +1,6 @@
 class S3File < ApplicationRecord
   belongs_to :owner, class_name: 'User', inverse_of: :s3_files
-  belongs_to :original_project, class_name: 'Project', inverse_of: :s3_files
+  belongs_to :original_project, class_name: 'Project', inverse_of: :s3_files, optional: true
   has_many :used_as_image_in_projects, class_name: 'Project', foreign_key: 'image_id', dependent: :nullify
 
   has_many :documents_s3_files, dependent: :destroy
@@ -12,7 +12,10 @@ class S3File < ApplicationRecord
   validates :size, numericality: { only_integer: true, greater_than_or_equal_to: 0 }
   validates :content_type, presence: true
 
-  include Queryable.permit(*%i[id role filename size content_type url created_at])
+  scope :attachments, -> { where(role: 'attachment') }
+  scope :project_images, -> { where(role: 'project-image') }
+
+  include Queryable.permit(*%i[id role filename size content_type url created_at became_unused_at do_not_delete_unused])
   include Listenable
 
   INLINE_CONTENT_TYPES = %w[
@@ -75,6 +78,28 @@ def uploaded?(update_cache: true)
     uploaded_cache? || check_uploaded(update_cache: update_cache)
   end
 
+  def attachment?
+    role == 'attachment'
+  end
+
+  def project_image?
+    role == 'project-image'
+  end
+
+  def unused?
+    became_unused_at.present?
+  end
+
+  def update_unused
+    if attachment?
+      if documents_s3_files.empty?
+        update!(became_unused_at: Time.current)
+      else
+        update!(became_unused_at: nil)
+      end
+    end
+  end
+
   private
 
   def generate_url

client/components/accountModalSections/FileStorageSection.tsx

@@ -3,6 +3,7 @@ import { deleteFile as deleteFileAPI } from '~/lib/apis/file';
 import { useAppContext } from '~/lib/appContext';
 import { filesize } from '~/lib/filesize';
 import { dispatchGlobalEvent } from '~/lib/globalEvents';
+import { groupedClassNames } from '~/lib/groupedClassNames';
 import { handleDeleteFileError } from '~/lib/handleErrors';
 import { sequenceFutures, unwrapFuture } from '~/lib/monads';
 import { S3File } from '~/lib/types';
@@ -12,6 +13,7 @@ import DownloadIcon from '~/components/icons/DownloadIcon';
 import OverflowMenuIcon from '~/components/icons/OverflowMenuIcon';
 import { LoadingView } from '~/components/LoadingView';
 import { Meter } from '~/components/Meter';
+import { Tooltip } from '../Tooltip';
 
 export const FileStorageSection = () => {
   const futureQuotaUsage = useAppContext('futureQuotaUsage');
@@ -22,27 +24,6 @@ export const FileStorageSection = () => {
     files: futureFiles,
   });
 
-  const deleteFile = (file: S3File) =>
-    handleDeleteFileError(deleteFileAPI(file.id)).then(() =>
-      dispatchGlobalEvent('s3File:delete', { s3FileId: file.id })
-    );
-
-  const fileMenu = (file: S3File) => (
-    <>
-      <DropdownItem icon={DownloadIcon} as="a" href={file.url} target="_blank">
-        Download file
-      </DropdownItem>
-
-      <DropdownItem
-        icon={DeleteIcon}
-        className="children:text-red-500 dark:children:text-red-400"
-        onClick={() => deleteFile(file)}
-      >
-        Delete file
-      </DropdownItem>
-    </>
-  );
-
   return unwrapFuture(futureData, {
     pending: (
       <div className="flex pt-5">
@@ -68,42 +49,140 @@ export const FileStorageSection = () => {
 
         <div className="space-y-2">
           <h3 className="h3 select-none">Files ({files.length})</h3>
-
-          {files.map((file) => (
-            <div
-              key={file.id}
-              className="rounded-lg flex gap-3 items-center p-3 bg-plain-50/90 dark:bg-plain-900/90"
-            >
-              <div className="grow">
-                {file.filename}
-
-                <div className="text-sm text-plain-500 dark:text-plain-400">
-                  {filesize(file.size)}
-                  {file.role === 'project-image' && (
-                    <> &middot; Used as project image</>
-                  )}
-                </div>
-              </div>
-
-              <Dropdown items={fileMenu(file)} placement="bottom-end">
-                <button
-                  type="button"
-                  className="shrink-0 btn p-2 aspect-square"
-                  aria-label="File actions"
-                >
-                  <OverflowMenuIcon size="1.25em" noAriaLabel />
-                </button>
-              </Dropdown>
-            </div>
-          ))}
-
-          {files.length === 0 && (
-            <div className="text-sm text-plain-500 dark:text-plain-400">
-              No files uploaded yet
-            </div>
-          )}
+          <FileList files={files} />
         </div>
       </>
     ),
   });
 };
+
+interface FileListProps {
+  files: S3File[];
+}
+
+const FileList = ({ files }: FileListProps) => {
+  return (
+    <>
+      {files.map((file) => (
+        <FileEntry key={file.id} {...file} />
+      ))}
+
+      {files.length === 0 && (
+        <div className="text-sm text-plain-500 dark:text-plain-400">
+          No files uploaded yet
+        </div>
+      )}
+    </>
+  );
+};
+
+type Badge = {
+  text: string;
+  hint: string;
+  color: 'neutral' | 'danger';
+};
+
+const FileEntry = ({
+  id,
+  url,
+  filename,
+  size,
+  role,
+  became_unused_at: becameUnusedAt,
+  do_not_delete_unused: doNotDeleteUnused,
+}: S3File) => {
+  const friendlyRole: string = {
+    'project-image': 'Project image',
+    attachment: 'Attachment',
+  }[role];
+
+  const badges = (() => {
+    const badges: Badge[] = [];
+
+    if (becameUnusedAt) {
+      if (doNotDeleteUnused) {
+        badges.push({
+          text: 'Unused',
+          hint: 'This file is unused but will not be deleted',
+          color: 'neutral',
+        });
+      } else {
+        badges.push({
+          text: 'Unused',
+          hint: 'This file is unused and will be deleted soon',
+          color: 'danger',
+        });
+      }
+    }
+
+    return badges;
+  })();
+
+  const performDelete = () =>
+    handleDeleteFileError(deleteFileAPI(id)).then(() =>
+      dispatchGlobalEvent('s3File:delete', { s3FileId: id })
+    );
+
+  const fileMenu = (
+    <>
+      <DropdownItem icon={DownloadIcon} as="a" href={url} target="_blank">
+        Download file
+      </DropdownItem>
+
+      <DropdownItem
+        icon={DeleteIcon}
+        className="children:text-red-500 dark:children:text-red-400"
+        onClick={performDelete}
+      >
+        Delete file
+      </DropdownItem>
+    </>
+  );
+
+  return (
+    <div className="rounded-lg flex gap-3 items-center p-3 bg-plain-50/90 dark:bg-plain-900/90">
+      <div className="grow">
+        <div className="flex gap-2 items-center">
+          {filename}
+
+          {badges.map((badge) => (
+            <Badge key={badge.text} {...badge} />
+          ))}
+        </div>
+
+        <div className="text-sm text-plain-500 dark:text-plain-400">
+          {friendlyRole} &middot; {filesize(size)}
+        </div>
+      </div>
+
+      <Dropdown items={fileMenu} placement="bottom-end">
+        <button
+          type="button"
+          className="shrink-0 btn p-2 aspect-square"
+          aria-label="File actions"
+        >
+          <OverflowMenuIcon size="1.25em" noAriaLabel />
+        </button>
+      </Dropdown>
+    </div>
+  );
+};
+
+const Badge = ({ text, hint, color }: Badge) => {
+  return (
+    <Tooltip content={hint}>
+      <span
+        className={groupedClassNames({
+          base: 'text-xs rounded-sm px-1 py-0.5 select-none',
+          color: {
+            neutral: 'bg-plain-200 dark:bg-plain-600',
+            danger: 'bg-red-600 text-white',
+          }[color],
+        })}
+        tabIndex={0}
+      >
+        {text}
+      </span>
+    </Tooltip>
+  );
+};

client/lib/types.ts

@@ -78,6 +78,8 @@ export type S3File = {
   content_type: string;
   url: string;
   created_at: string;
+  became_unused_at: string | null;
+  do_not_delete_unused: boolean;
 };
 
 export type StorageQuotaUsage = {

config/clockwork.rb

@@ -9,6 +9,7 @@ module Clockwork
         isolate_errors { CleanUpUnuploadedS3Files.perform }
         isolate_errors { CleanUpUntrackedS3Objects.perform }
         isolate_errors { CleanUpBlankDocuments.perform }
+        isolate_errors { CleanUpUnusedAttachments.perform }
       end
     end
   end

db/migrate/20231125213413_add_became_unused_at_to_s3_files.rb

@@ -0,0 +1,16 @@
+class AddBecameUnusedAtToS3Files < ActiveRecord::Migration[7.0]
+  def up
+    add_column :s3_files, :became_unused_at, :timestamp, null: true
+    add_column :s3_files, :do_not_delete_unused, :boolean, null: false, default: false
+
+    # Flag all existing files as "do not delete"
+    S3File.attachments.update_all(do_not_delete_unused: true)
+
+    S3File.find_each(&:update_unused)
+  end
+
+  def down
+    remove_column :s3_files, :became_unused_at
+    remove_column :s3_files, :do_not_delete_unused
+  end
+end

db/migrate/20231125215025_add_id_to_documents_s3_files.rb

@@ -0,0 +1,5 @@
+class AddIdToDocumentsS3Files < ActiveRecord::Migration[7.0]
+  def change
+    add_column :documents_s3_files, :id, :primary_key
+  end
+end

test/factories.rb

@@ -22,13 +22,18 @@
   factory :s3_file do
     owner
     original_project
-    role { 'project-icon' }
+    role { 'attachment' }
     sequence(:s3_key) { |n| "uploads/#{n}.png" }
     sequence(:filename) { |n| "#{n}.png" }
     size { 100 }
     content_type { 'image/png' }
   end
 
+  factory :documents_s3_file do
+    document
+    s3_file
+  end
+
   factory :settings do
     user
     data { { hello: 'world' }.to_json }