🐛 Adapt trim-file task to page/file data model changes.

backend/src/app/config.clj

@@ -48,7 +48,7 @@
    ;; modification in order to make the file ellegible for
    ;; trimming. The value only supports s(econds) m(inutes) and
    ;; h(ours) as time unit.
-   :file-trimming-max-age "72h"
+   :file-trimming-threshold "72h"
 
    ;; LDAP auth disabled by default. Set ldap-auth-host to enable
    ;:ldap-auth-host "ldap.mysupercompany.com"
@@ -92,7 +92,9 @@
 (s/def ::debug-humanize-transit ::us/boolean)
 (s/def ::public-uri ::us/string)
 (s/def ::backend-uri ::us/string)
+
 (s/def ::image-process-max-threads ::us/integer)
+(s/def ::file-trimming-threshold ::dt/duration)
 
 (s/def ::google-client-id ::us/string)
 (s/def ::google-client-secret ::us/string)
@@ -113,7 +115,6 @@
 (s/def ::ldap-auth-email-attribute ::us/string)
 (s/def ::ldap-auth-fullname-attribute ::us/string)
 (s/def ::ldap-auth-avatar-attribute ::us/string)
-(s/def ::file-trimming-threshold ::dt/duration)
 
 (s/def ::config
   (s/keys :opt-un [::http-server-cors
@@ -143,7 +144,7 @@
                    ::smtp-password
                    ::smtp-tls
                    ::smtp-ssl
-                   ::file-trimming-max-age
+                   ::file-trimming-threshold
                    ::debug-humanize-transit
                    ::allow-demo-users
                    ::registration-enabled

backend/src/app/tasks/trim_file.clj

@@ -9,6 +9,7 @@
 
 (ns app.tasks.trim-file
   (:require
+   [app.common.pages-migrations :as pmg]
    [app.config :as cfg]
    [app.db :as db]
    [app.tasks :as tasks]
@@ -29,7 +30,8 @@
     (bytes? data) (assoc :data (blob/decode data))))
 
 (def sql:retrieve-files-to-trim
-  "select id from file as f
+  "select f.id, f.data
+     from file as f
     where f.has_media_trimmed is false
       and f.modified_at < now() - ?::interval
     order by f.modified_at asc
@@ -40,39 +42,42 @@
   file is considered candidate when some time passes whith no
   modification."
   [conn]
-  (let [interval (:file-trimming-max-age cfg/config)]
+  (let [threshold (:file-trimming-threshold cfg/config)
-    (->> (db/exec! conn [sql:retrieve-files-to-trim interval])
+        interval  (db/interval threshold)]
-         (map :id))))
+    (db/exec! conn [sql:retrieve-files-to-trim interval])))
-
 
 (def collect-media-xf
-  (comp (map :data)
+  (comp
-        (map :objects)
+   (map :objects)
-        (mapcat vals)
+   (mapcat vals)
-        (filter #(= :image (:type %)))
+   (filter #(= :image (:type %)))
-        (map :metadata)
+   (map :metadata)
-        (map :id)))
+   (map :id)))
-
 
 (defn collect-used-media
-  [pages]
+  [data]
-  (into #{} collect-media-xf pages))
+  (-> #{}
+      (into collect-media-xf (vals (:pages-index data)))
+      (into collect-media-xf (vals (:components data)))
+      (into (keys (:media data)))))
 
 (defn process-file
-  [file-id]
+  [{:keys [id data] :as file}]
-  (log/debugf "Processing file: '%s'." file-id)
+  (log/debugf "Processing file: '%s'." id)
   (db/with-atomic [conn db/pool]
-    (let [mobjs  (db/query conn :media-object {:file-id file-id})
+    (let [mobjs  (map :id (db/query conn :media-object {:file-id id}))
-          pages  (->> (db/query conn :page {:file-id file-id})
+          data   (-> (blob/decode data)
-                      (map decode-row))
+                     (pmg/migrate-data))
-          used   (into #{} collect-media-xf pages)
+
-          unused (into #{} (comp (map :id) (remove #(contains? used %))) mobjs)]
+          used   (collect-used-media data)
+          unused (into #{} (remove #(contains? used %)) mobjs)]
 
       (log/debugf "Collected media ids: '%s'." (pr-str used))
       (log/debugf "Unused media ids: '%s'." (pr-str unused))
+
       (db/update! conn :file
                   {:has-media-trimmed true}
-                  {:id file-id})
+                  {:id id})
 
       (doseq [id unused]
         ;; TODO: add task batching

backend/src/app/util/time.clj

@@ -103,9 +103,10 @@
 (letfn [(conformer [v]
           (cond
             (duration? v) v
+
             (string? v)
             (try
-              (parse-duration v)
+              (duration v)
               (catch java.time.format.DateTimeParseException _e
                 ::s/invalid))