[Storage] Remove files migration (#908)

instantdb · Feb 20, 2025 · 5332248 · 5332248
1 parent 8896f10
commit 5332248
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 86 deletions.
diff --git a/server/src/instant/admin/routes.clj b/server/src/instant/admin/routes.clj
@@ -457,9 +457,7 @@
 ;; Legacy StorageFile format that was only used by the list() endpoint
 (defn legacy-storage-file-format
   [app-id file]
-  (let [object-key (if (instant-s3/migrating?)
-                     (instant-s3/->path-object-key app-id (:path file))
-                     (instant-s3/->object-key app-id (:location-id file)))]
+  (let [object-key (instant-s3/->object-key app-id (:location-id file))]
     {:key object-key
      :name (:path file)
      :size (:size file)

diff --git a/server/src/instant/db/instaql.clj b/server/src/instant/db/instaql.clj
@@ -886,22 +886,15 @@
 
 (defn compute-$files-triples [{:keys [app-id]} join-rows]
   (when-let [[eid _ _ t] (ffirst join-rows)]
-    (let [path (some-> (find-row-by-ident-name
-                        join-rows
-                        $system-attrs
-                        "$files"
-                        "path")
-                       first
-                       (nth 2))
-          location-id (some-> (find-row-by-ident-name
+    (let [location-id (some-> (find-row-by-ident-name
                                join-rows
                                $system-attrs
                                "$files"
                                "location-id")
                               first
                               (nth 2))
           url-aid (attr-model/resolve-attr-id $system-attrs "$files" "url")
-          url (instant-s3/create-signed-download-url! app-id path location-id)]
+          url (instant-s3/create-signed-download-url! app-id location-id)]
       [[[eid url-aid url t]]])))
 
 (def compute-triples-handler

diff --git a/server/src/instant/storage/coordinator.clj b/server/src/instant/storage/coordinator.clj
@@ -57,7 +57,7 @@
   (let [deleted (app-file-model/delete-by-paths! {:app-id app-id :paths paths})
         locations (mapv :location-id deleted)
         ids (mapv :id deleted)
-        _ (instant-s3/bulk-delete-files! app-id paths locations)]
+        _ (instant-s3/bulk-delete-files! app-id locations)]
     {:ids ids}))
 
 (defn delete-file!
@@ -68,7 +68,7 @@
                                           :path path
                                           :current-user current-user}))
   (let [{:keys [id location-id]} (app-file-model/delete-by-path! {:app-id app-id :path path})
-        _ (instant-s3/delete-file! app-id path location-id)]
+        _ (instant-s3/delete-file! app-id location-id)]
     {:id id}))
 
 ;; Logic for legacy S3 upload/download URLs
@@ -111,4 +111,4 @@
                                         :path path
                                         :current-user current-user}))
   (let [{:keys [location-id]} (app-file-model/get-by-path {:app-id app-id :path path})]
-    (instant-s3/create-signed-download-url! app-id path location-id)))
+    (instant-s3/create-signed-download-url! app-id location-id)))
diff --git a/server/src/instant/storage/s3.clj b/server/src/instant/storage/s3.clj
@@ -1,45 +1,9 @@
 (ns instant.storage.s3
   (:require [clojure.string :as string]
-            [clojure.java.io :as io]
-            [instant.util.s3 :as s3-util]
-            [instant.flags :as flags])
+            [instant.util.s3 :as s3-util])
   (:import
    [java.time Duration]))
 
-;; Legacy S3 migration helpers
-;; ------------------
-(defn migrating? []
-  (-> (flags/storage-migration) :useLocationId? not))
-
-(defn ->legacy-object-key [app-id filename]
-  (str app-id "/" filename))
-
-(defn legacy-object-key->path
-  "Extract path from our S3 object keys"
-  [object-key]
-  (let [[_app-id & path] (string/split object-key #"/")]
-    (string/join "/" path)))
-
-(defn filename->bin
-  ^long [^String filename]
-  (mod (Math/abs (.hashCode filename)) 10))
-
-(defn ->path-object-key
-  "We prefix objects with an app id and bin. Combined with a filename
-  this gives us our key for each object."
-  [app-id filename]
-  (let [bin (filename->bin filename)
-        fname (if (string/starts-with? filename "/")
-                (subs filename 1)
-                filename)]
-    (str app-id "/" bin "/" fname)))
-
-(defn path-object-key->path
-  "Extract path from our S3 object keys"
-  [object-key]
-  (let [[_app-id _bin & path] (string/split object-key #"/")]
-    (string/join "/" path)))
-
 ;; S3 path manipulation
 ;; ----------------------
 (defn location-id->bin
@@ -70,19 +34,11 @@
 
 ;; Instant <> S3 integration
 ;; ----------------------
-(defn upload-file-to-s3 [{:keys [app-id path location-id] :as ctx} file]
+(defn upload-file-to-s3 [{:keys [app-id location-id] :as ctx} file]
   (when (not (instance? java.io.InputStream file))
     (throw (Exception. "Unsupported file format")))
-  (if (migrating?)
-    (let [baos (java.io.ByteArrayOutputStream.)
-          _ (io/copy file baos)
-          bytes (.toByteArray baos)
-          ctx* (assoc ctx :object-key (->object-key app-id location-id))
-          ctx-legacy* (assoc ctx :object-key (->path-object-key app-id path))]
-      (s3-util/upload-stream-to-s3 ctx-legacy* (io/input-stream bytes))
-      (s3-util/upload-stream-to-s3 ctx* (io/input-stream bytes)))
-    (let [ctx* (assoc ctx :object-key (->object-key app-id location-id))]
-      (s3-util/upload-stream-to-s3 ctx* file))))
+  (let [ctx* (assoc ctx :object-key (->object-key app-id location-id))]
+    (s3-util/upload-stream-to-s3 ctx* file)))
 
 (defn format-object [{:keys [object-metadata]}]
   (-> object-metadata
@@ -109,32 +65,16 @@
        :content-type content-type
        :content-disposition content-disposition}))))
 
-(defn delete-file! [app-id path location-id]
-  (when (migrating?)
-    (s3-util/delete-object (->path-object-key app-id path)))
+(defn delete-file! [app-id location-id]
   (when location-id
     (s3-util/delete-object (->object-key app-id location-id))))
 
-(defn bulk-delete-files! [app-id paths location-ids]
-  (when (migrating?)
-    (let [path-keys (mapv
-                     (fn [path] (->path-object-key app-id path))
-                     paths)]
-      (s3-util/delete-objects-paginated path-keys)))
+(defn bulk-delete-files! [app-id location-ids]
   (let [location-keys (mapv
                        (fn [location-id] (->object-key app-id location-id))
                        location-ids)]
     (s3-util/delete-objects-paginated location-keys)))
 
-(defn path-url [app-id filename]
-  (let [duration (Duration/ofDays 7)
-        object-key (->path-object-key app-id filename)]
-    (str (s3-util/generate-presigned-url
-          {:method :get
-           :bucket-name s3-util/default-bucket
-           :key object-key
-           :duration duration}))))
-
 (defn location-id-url [app-id location-id]
   (let [duration (Duration/ofDays 7)
         object-key (->object-key app-id location-id)]
@@ -144,11 +84,9 @@
            :key object-key
            :duration duration}))))
 
-(defn create-signed-download-url! [app-id path location-id]
-  (if (migrating?)
-    (path-url app-id path)
-    (when location-id
-      (location-id-url app-id location-id))))
+(defn create-signed-download-url! [app-id location-id]
+  (when location-id
+    (location-id-url app-id location-id)))
 
 ;; S3 Usage Metrics
 ;; These functions calculate usage by talking to S3 directly. We can use these