fix(s3): chunked object deletion

Author: LouisHaftmann

lib/storage/drivers/s3.ts

@@ -9,6 +9,7 @@ import {
   UploadPartCommand,
 } from '@aws-sdk/client-s3'
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
+import * as R from 'remeda'
 import { z } from 'zod'
 
 import { defineStorageDriver } from '~/lib/storage/defineStorageDriver'
@@ -130,14 +131,18 @@ export const s3Driver = defineStorageDriver({
         )
       },
       async delete({ objectNames }) {
-        await s3.send(
-          new DeleteObjectsCommand({
-            Bucket: options.STORAGE_S3_BUCKET,
-            Delete: {
-              Objects: objectNames.map((name) => ({ Key: getObjectKey(name) })),
-              Quiet: true,
-            },
-          }),
+        await Promise.all(
+          R.chunk(objectNames, 1000).map((chunkedObjectNames) =>
+            s3.send(
+              new DeleteObjectsCommand({
+                Bucket: options.STORAGE_S3_BUCKET,
+                Delete: {
+                  Objects: chunkedObjectNames.map((name) => ({ Key: getObjectKey(name) })),
+                  Quiet: true,
+                },
+              }),
+            ),
+          ),
         )
       },
     }

lib/storage/index.ts

@@ -43,7 +43,7 @@ export interface Storage {
     cacheId: number | null
   }>
   pruneCaches: (olderThanDays?: number) => Promise<void>
-  pruneUploads: () => Promise<void>
+  pruneUploads: (olderThanDate: Date) => Promise<void>
 }
 
 let storage: Storage
@@ -251,14 +251,14 @@ export async function initializeStorage() {
             olderThanDays,
           })
         },
-        async pruneUploads() {
+        async pruneUploads(olderThanDate) {
           logger.debug('Prune uploads')
 
           // uploads older than 24 hours
           const uploads = await db
             .selectFrom('uploads')
             .selectAll()
-            .where('created_at', '<', new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString())
+            .where('created_at', '<', olderThanDate.toISOString())
             .execute()
 
           for (const upload of uploads) {

package.json

@@ -39,6 +39,7 @@
     "mysql2": "^3.12.0",
     "nitropack": "^2.10.4",
     "pg": "^8.13.1",
+    "remeda": "^2.21.0",
     "typescript": "^5.7.3",
     "zod": "^3.24.1"
   },

plugins/cleanup.ts

@@ -26,8 +26,10 @@ export default defineNitroPlugin(() => {
   logger.info(
     `Cleaning up dangling uploads with schedule ${colorize('blue', job.getPattern() ?? '')}${nextRun ? ` (next run: ${nextRun.toLocaleString()})` : ''}`,
   )
+  let lastRun = new Date()
   job.schedule(async () => {
     const adapter = await useStorageAdapter()
-    await adapter.pruneUploads()
+    await adapter.pruneUploads(lastRun)
+    lastRun = new Date()
   })
 })