diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index a590eb551..78159a6d7 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -48,7 +48,7 @@ ;; modification in order to make the file ellegible for ;; trimming. The value only supports s(econds) m(inutes) and ;; h(ours) as time unit. - :file-trimming-max-age "72h" + :file-trimming-threshold "72h" ;; LDAP auth disabled by default. Set ldap-auth-host to enable ;:ldap-auth-host "ldap.mysupercompany.com" @@ -92,7 +92,9 @@ (s/def ::debug-humanize-transit ::us/boolean) (s/def ::public-uri ::us/string) (s/def ::backend-uri ::us/string) + (s/def ::image-process-max-threads ::us/integer) +(s/def ::file-trimming-threshold ::dt/duration) (s/def ::google-client-id ::us/string) (s/def ::google-client-secret ::us/string) @@ -113,7 +115,6 @@ (s/def ::ldap-auth-email-attribute ::us/string) (s/def ::ldap-auth-fullname-attribute ::us/string) (s/def ::ldap-auth-avatar-attribute ::us/string) -(s/def ::file-trimming-threshold ::dt/duration) (s/def ::config (s/keys :opt-un [::http-server-cors @@ -143,7 +144,7 @@ ::smtp-password ::smtp-tls ::smtp-ssl - ::file-trimming-max-age + ::file-trimming-threshold ::debug-humanize-transit ::allow-demo-users ::registration-enabled diff --git a/backend/src/app/tasks/trim_file.clj b/backend/src/app/tasks/trim_file.clj index e135953a4..dd16f2fd0 100644 --- a/backend/src/app/tasks/trim_file.clj +++ b/backend/src/app/tasks/trim_file.clj @@ -9,6 +9,7 @@ (ns app.tasks.trim-file (:require + [app.common.pages-migrations :as pmg] [app.config :as cfg] [app.db :as db] [app.tasks :as tasks] @@ -29,7 +30,8 @@ (bytes? data) (assoc :data (blob/decode data)))) (def sql:retrieve-files-to-trim - "select id from file as f + "select f.id, f.data + from file as f where f.has_media_trimmed is false and f.modified_at < now() - ?::interval order by f.modified_at asc @@ -40,39 +42,42 @@ file is considered candidate when some time passes whith no modification." [conn] - (let [interval (:file-trimming-max-age cfg/config)] - (->> (db/exec! conn [sql:retrieve-files-to-trim interval]) - (map :id)))) - + (let [threshold (:file-trimming-threshold cfg/config) + interval (db/interval threshold)] + (db/exec! conn [sql:retrieve-files-to-trim interval]))) (def collect-media-xf - (comp (map :data) - (map :objects) - (mapcat vals) - (filter #(= :image (:type %))) - (map :metadata) - (map :id))) - + (comp + (map :objects) + (mapcat vals) + (filter #(= :image (:type %))) + (map :metadata) + (map :id))) (defn collect-used-media - [pages] - (into #{} collect-media-xf pages)) + [data] + (-> #{} + (into collect-media-xf (vals (:pages-index data))) + (into collect-media-xf (vals (:components data))) + (into (keys (:media data))))) (defn process-file - [file-id] - (log/debugf "Processing file: '%s'." file-id) + [{:keys [id data] :as file}] + (log/debugf "Processing file: '%s'." id) (db/with-atomic [conn db/pool] - (let [mobjs (db/query conn :media-object {:file-id file-id}) - pages (->> (db/query conn :page {:file-id file-id}) - (map decode-row)) - used (into #{} collect-media-xf pages) - unused (into #{} (comp (map :id) (remove #(contains? used %))) mobjs)] + (let [mobjs (map :id (db/query conn :media-object {:file-id id})) + data (-> (blob/decode data) + (pmg/migrate-data)) + + used (collect-used-media data) + unused (into #{} (remove #(contains? used %)) mobjs)] (log/debugf "Collected media ids: '%s'." (pr-str used)) (log/debugf "Unused media ids: '%s'." (pr-str unused)) + (db/update! conn :file {:has-media-trimmed true} - {:id file-id}) + {:id id}) (doseq [id unused] ;; TODO: add task batching diff --git a/backend/src/app/util/time.clj b/backend/src/app/util/time.clj index e8d62d877..628835c33 100644 --- a/backend/src/app/util/time.clj +++ b/backend/src/app/util/time.clj @@ -103,9 +103,10 @@ (letfn [(conformer [v] (cond (duration? v) v + (string? v) (try - (parse-duration v) + (duration v) (catch java.time.format.DateTimeParseException _e ::s/invalid))