From 0647fa832a325c2aeb890a1d8b1538c00744a0ec Mon Sep 17 00:00:00 2001 From: "alonso.torres" Date: Mon, 7 Jun 2021 22:02:16 +0200 Subject: [PATCH] :sparkles: Read files info from manifest --- backend/src/app/rpc/mutations/files.clj | 36 ++++++- frontend/src/app/util/zip.cljs | 67 +++++++------ frontend/src/app/worker/export.cljs | 63 ++++++++++--- frontend/src/app/worker/import.cljs | 120 +++++++++++------------- 4 files changed, 181 insertions(+), 105 deletions(-) diff --git a/backend/src/app/rpc/mutations/files.clj b/backend/src/app/rpc/mutations/files.clj index a4f37867f..1093e9dc3 100644 --- a/backend/src/app/rpc/mutations/files.clj +++ b/backend/src/app/rpc/mutations/files.clj @@ -11,6 +11,7 @@ [app.common.pages.migrations :as pmg] [app.common.spec :as us] [app.common.uuid :as uuid] + [app.db :as db] [app.rpc.permissions :as perms] [app.rpc.queries.files :as files] @@ -18,6 +19,7 @@ [app.util.blob :as blob] [app.util.services :as sv] [app.util.time :as dt] + [clojure.spec.alpha :as s])) ;; --- Helpers & Specs @@ -43,6 +45,7 @@ (proj/check-edition-permissions! conn profile-id project-id) (create-file conn params))) + (defn create-file-role [conn {:keys [file-id profile-id role]}] (let [params {:file-id file-id @@ -51,8 +54,9 @@ (db/insert! conn :file-profile-rel)))) (defn create-file - [conn {:keys [id name project-id is-shared data] - :or {is-shared false} + [conn {:keys [id name project-id is-shared data deleted-at] + :or {is-shared false + deleted-at nil} :as params}] (let [id (or id (:id data) (uuid/next)) data (or data (cp/make-file-data id)) @@ -61,7 +65,8 @@ :project-id project-id :name name :is-shared is-shared - :data (blob/encode data)})] + :data (blob/encode data) + :deleted-at deleted-at})] (->> (assoc params :file-id id :role :owner) (create-file-role conn)) (assoc file :data data))) @@ -118,6 +123,7 @@ [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}] (db/with-atomic [conn pool] (files/check-edition-permissions! conn profile-id id) + (mark-file-deleted conn params))) (defn mark-file-deleted @@ -268,7 +274,8 @@ [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}] (db/with-atomic [conn pool] (db/xact-lock! conn id) - (let [{:keys [id] :as file} (db/get-by-id conn :file id {:for-key-share true})] + (let [{:keys [id] :as file} (db/get-by-id conn :file id {:for-key-share true + :uncheked true})] (files/check-edition-permissions! conn profile-id id) (update-file (assoc cfg :conn conn) (assoc params :file file))))) @@ -381,3 +388,24 @@ [conn project-id] (:team-id (db/get-by-id conn :project project-id {:columns [:team-id]}))) + +;; TEMPORARY FILE CREATION + +(s/def ::create-temp-file ::create-file) + +(sv/defmethod ::create-temp-file + [{:keys [pool] :as cfg} {:keys [profile-id project-id] :as params}] + (db/with-atomic [conn pool] + (proj/check-edition-permissions! conn profile-id project-id) + (create-file conn (assoc params :deleted-at (dt/in-future {:days 1}))))) + +(s/def ::make-permanent + (s/keys :req-un [::id ::profile-id])) + +(sv/defmethod ::make-permanent + [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}] + (db/with-atomic [conn pool] + (files/check-edition-permissions! conn profile-id id) + (db/update! conn :file + {:deleted-at nil} + {:id id}))) diff --git a/frontend/src/app/util/zip.cljs b/frontend/src/app/util/zip.cljs index 071a7c456..84a3759d2 100644 --- a/frontend/src/app/util/zip.cljs +++ b/frontend/src/app/util/zip.cljs @@ -10,7 +10,8 @@ ["jszip" :as zip] [app.common.data :as d] [beicon.core :as rx] - [promesa.core :as p])) + [promesa.core :as p] + [app.util.http :as http])) (defn compress-files [files] @@ -21,32 +22,44 @@ (->> (.generateAsync zobj #js {:type "blob"}) (rx/from))))) +(defn load-from-url + "Loads the data from a blob url" + [url] + (->> (http/send! + {:uri url + :response-type :blob + :method :get}) + (rx/map :body) + (rx/flat-map zip/loadAsync))) + +(defn- process-file [entry path] + (cond + (nil? entry) + (p/rejected "No file found") + + (.-dir entry) + (p/resolved {:dir path}) + + :else + (-> (.async entry "text") + (p/then #(hash-map :path path :content %))))) + +(defn get-file + "Gets a single file from the zip archive" + [zip path] + (-> (.file zip path) + (process-file path) + (rx/from))) + (defn extract-files "Creates a stream that will emit values for every file in the zip" - [file] - (rx/create - (fn [subs] - (let [process-entry - (fn [path entry] - (if (.-dir entry) - (rx/push! subs {:dir path}) - (p/then - (.async entry "text") - (fn [content] - (rx/push! subs - {:path path - :content content})))))] + [zip] + (let [promises (atom []) + get-file + (fn [path entry] + (let [current (process-file entry path)] + (swap! promises conj current)))] + (.forEach zip get-file) - (p/let [response (js/fetch file) - data (.blob response) - content (zip/loadAsync data)] - - (let [promises (atom [])] - (.forEach content - (fn [path entry] - (let [current (process-entry path entry)] - (swap! promises conj current)))) - - (p/then (p/all @promises) - #(rx/end! subs)))) - nil)))) + (->> (rx/from (p/all @promises)) + (rx/flat-map identity)))) diff --git a/frontend/src/app/worker/export.cljs b/frontend/src/app/worker/export.cljs index 1d1dcb108..26ae4a89f 100644 --- a/frontend/src/app/worker/export.cljs +++ b/frontend/src/app/worker/export.cljs @@ -10,16 +10,39 @@ [app.main.repo :as rp] [app.util.dom :as dom] [app.util.zip :as uz] + [app.util.json :as json] [app.worker.impl :as impl] [beicon.core :as rx])) +(defn create-manifest + "Creates a manifest entry for the given files" + [team-id files] + (letfn [(format-page [manifest page] + (-> manifest + (assoc (str (:id page)) + {:name (:name page)}))) + + (format-file [manifest file] + (let [name (:name file) + pages (->> (get-in file [:data :pages]) (mapv str)) + index (->> (get-in file [:data :pages-index]) (vals) + (reduce format-page {}))] + (-> manifest + (assoc (str (:id file)) + {:name name + :pages pages + :pagesIndex index}))))] + (let [manifest {:teamId (str team-id) + :files (->> (vals files) (reduce format-file {}))}] + (json/encode manifest)))) + (defn get-page-data - [{file-name :file-name {:keys [id name] :as data} :data}] + [{file-id :file-id {:keys [id name] :as data} :data}] (->> (r/render-page data) (rx/map (fn [markup] {:id id :name name - :file-name file-name + :file-id file-id :markup markup})))) (defn process-pages [file] @@ -27,30 +50,48 @@ pages-index (get-in file [:data :pages-index])] (->> pages (map #(hash-map - :file-name (:name file) + :file-id (:id file) :data (get pages-index %)))))) (defn collect-page - [coll {:keys [id file-name name markup] :as page}] - (conj coll [(str file-name "/" name ".svg") markup])) + [{:keys [id file-id markup] :as page}] + [(str file-id "/" id ".svg") markup]) (defmethod impl/handler :export-file - [{:keys [team-id files] :as message}] + [{:keys [team-id project-id files] :as message}] - (let [render-stream - (->> (rx/from (->> files (mapv :id))) + (let [files-ids (->> files (mapv :id)) + + files-stream + (->> (rx/from files-ids) (rx/merge-map #(rp/query :file {:id %})) + (rx/reduce #(assoc %1 (:id %2) %2) {}) + (rx/share)) + + manifest-stream + (->> files-stream + (rx/map #(create-manifest team-id %)) + (rx/map #(vector "manifest.json" %))) + + render-stream + (->> files-stream + (rx/flat-map vals) (rx/flat-map process-pages) (rx/observe-on :async) (rx/flat-map get-page-data) - (rx/share))] + (rx/share)) + + pages-stream + (->> render-stream + (rx/map collect-page))] (rx/merge (->> render-stream (rx/map #(hash-map :type :progress :data (str "Render " (:file-name %) " - " (:name %))))) - (->> render-stream - (rx/reduce collect-page []) + (->> (rx/merge pages-stream + manifest-stream) + (rx/reduce conj []) (rx/flat-map uz/compress-files) (rx/map #(hash-map :type :finish :data (dom/create-uri %))))))) diff --git a/frontend/src/app/worker/import.cljs b/frontend/src/app/worker/import.cljs index dfe240b8e..e2f1cd785 100644 --- a/frontend/src/app/worker/import.cljs +++ b/frontend/src/app/worker/import.cljs @@ -13,6 +13,7 @@ [app.main.repo :as rp] [app.util.http :as http] [app.util.import.parser :as cip] + [app.util.json :as json] [app.util.zip :as uz] [app.worker.impl :as impl] [beicon.core :as rx] @@ -27,7 +28,7 @@ [project-id name] (let [file-id (uuid/next)] (rp/mutation - :create-file + :create-temp-file {:id file-id :name name :project-id project-id @@ -44,17 +45,20 @@ (partition change-batch-size change-batch-size nil) (mapv vec))] - (->> (rx/from changes-batches) - (rx/merge-map - (fn [cur-changes-batch] - (rp/mutation - :update-file - {:id file-id - :session-id session-id - :revn @revn - :changes cur-changes-batch}))) + (rx/concat + (->> (rx/from changes-batches) + (rx/mapcat + (fn [cur-changes-batch] + (rp/mutation + :update-file + {:id file-id + :session-id session-id + :revn @revn + :changes cur-changes-batch}))) - (rx/tap #(reset! revn (:revn %)))))) + (rx/tap #(reset! revn (:revn %)))) + + (rp/mutation :make-permanent {:id (:id file)})))) (defn upload-media-files "Upload a image to the backend and returns its id" @@ -72,17 +76,6 @@ :is-local true})) (rx/flat-map #(rp/mutation! :upload-file-media-object %)))) -(defn parse-file-name - [dir] - (if (str/ends-with? dir "/") - (subs dir 0 (dec (count dir))) - dir)) - -(defn parse-page-name - [path] - (let [[file page] (str/split path "/")] - (str/replace page ".svg" ""))) - (defn add-shape-file [file node] @@ -137,49 +130,50 @@ (rx/of node))) (defn import-page - [file {:keys [path content]}] - (let [page-name (parse-page-name path)] - (if (cip/valid? content) - (let [nodes (->> content cip/node-seq) - file-id (:id file)] - (->> (rx/from nodes) - (rx/filter cip/shape?) - (rx/mapcat (partial resolve-images file-id)) - (rx/reduce add-shape-file (fb/add-page file page-name)) - (rx/map fb/close-page))) - (rx/empty)))) + [file [page-name content]] + (if (cip/valid? content) + (let [nodes (->> content cip/node-seq) + file-id (:id file)] + (->> (rx/from nodes) + (rx/filter cip/shape?) + (rx/mapcat (partial resolve-images file-id)) + (rx/reduce add-shape-file (fb/add-page file page-name)) + (rx/map fb/close-page))) + (rx/empty))) + +(defn get-page-path [dir-id id] + (str dir-id "/" id ".svg")) + +(defn process-page [file-id zip [page-id page-name]] + (->> (uz/get-file zip (get-page-path (d/name file-id) page-id)) + (rx/map (comp tubax/xml->clj :content)) + (rx/map #(vector page-name %)))) + +(defn process-file + [file file-id file-desc zip] + (let [index (:pagesIndex file-desc) + pages (->> (:pages file-desc) + (mapv #(vector % (get-in index [(keyword %) :name]))))] + (->> (rx/from pages) + (rx/flat-map #(process-page file-id zip %)) + (merge-reduce import-page file) + (rx/flat-map send-changes) + (rx/ignore)))) (defmethod impl/handler :import-file [{:keys [project-id files]}] - (let [extract-stream - (->> (rx/from files) - (rx/merge-map uz/extract-files)) + (let [zip-str (->> (rx/from files) + (rx/flat-map uz/load-from-url) + (rx/share))] - dir-str - (->> extract-stream - (rx/filter #(contains? % :dir)) - (rx/map :dir)) - - file-str - (->> extract-stream - (rx/filter #(not (contains? % :dir))) - (rx/map #(d/update-when % :content tubax/xml->clj)))] - - (->> dir-str - (rx/merge-map #(create-file project-id (parse-file-name %))) - (rx/merge-map - (fn [file] - (rx/concat - (->> file-str - (rx/filter #(str/starts-with? (:path %) (:name file))) - (merge-reduce import-page file) - (rx/flat-map send-changes) - (rx/catch (fn [err] - (.error js/console "ERROR" err (clj->js (.-data err))) - - ;; We delete the file when there is an error - (rp/mutation! :delete-file {:id (:id file)}))) - (rx/ignore)) - - (rx/of (select-keys file [:id :name])))))))) + (->> zip-str + (rx/flat-map #(uz/get-file % "manifest.json")) + (rx/flat-map (comp :files json/decode :content)) + (rx/with-latest-from zip-str) + (rx/flat-map + (fn [[[file-id file-desc] zip]] + (->> (create-file project-id (:name file-desc)) + (rx/flat-map #(process-file % file-id file-desc zip)) + (rx/catch (fn [err] + (.error js/console "ERROR" err (clj->js (.-data err)))))))))))