diff --git a/backend/src/app/http/debug.clj b/backend/src/app/http/debug.clj index 56642f160..4233909b4 100644 --- a/backend/src/app/http/debug.clj +++ b/backend/src/app/http/debug.clj @@ -16,7 +16,7 @@ [app.http.middleware :as mw] [app.http.session :as session] [app.rpc.commands.binfile :as binf] - [app.rpc.mutations.files :refer [create-file]] + [app.rpc.commands.files.create :refer [create-file]] [app.rpc.queries.profile :as profile] [app.util.blob :as blob] [app.util.template :as tmpl] diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index 77171d345..7b2d22bc9 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -257,6 +257,11 @@ {:name "0082-add-features-column-to-file-table" :fn (mg/resource "app/migrations/sql/0082-add-features-column-to-file-table.sql")} + {:name "0083-add-file-data-fragment-table" + :fn (mg/resource "app/migrations/sql/0083-add-file-data-fragment-table.sql")} + + {:name "0084-add-features-column-to-file-change-table" + :fn (mg/resource "app/migrations/sql/0084-add-features-column-to-file-change-table.sql")} ]) diff --git a/backend/src/app/migrations/sql/0083-add-file-data-fragment-table.sql b/backend/src/app/migrations/sql/0083-add-file-data-fragment-table.sql new file mode 100644 index 000000000..6cb817e0b --- /dev/null +++ b/backend/src/app/migrations/sql/0083-add-file-data-fragment-table.sql @@ -0,0 +1,15 @@ +CREATE TABLE file_data_fragment ( + id uuid NOT NULL, + file_id uuid NOT NULL REFERENCES file(id) ON DELETE CASCADE DEFERRABLE, + + created_at timestamptz NOT NULL DEFAULT now(), + + metadata jsonb NULL, + content bytea NOT NULL, + + PRIMARY KEY (file_id, id) +); + +ALTER TABLE file_data_fragment + ALTER COLUMN metadata SET STORAGE external, + ALTER COLUMN content SET STORAGE external; diff --git a/backend/src/app/migrations/sql/0084-add-features-column-to-file-change-table.sql b/backend/src/app/migrations/sql/0084-add-features-column-to-file-change-table.sql new file mode 100644 index 000000000..4fa0f9456 --- /dev/null +++ b/backend/src/app/migrations/sql/0084-add-features-column-to-file-change-table.sql @@ -0,0 +1,8 @@ +ALTER TABLE file_change + ADD COLUMN features text[] DEFAULT NULL; + +ALTER TABLE file_change + ALTER COLUMN features SET STORAGE external; + +ALTER TABLE file + ALTER COLUMN features SET STORAGE external; diff --git a/backend/src/app/rpc.clj b/backend/src/app/rpc.clj index 1f1274b8f..e7e39fc34 100644 --- a/backend/src/app/rpc.clj +++ b/backend/src/app/rpc.clj @@ -230,7 +230,10 @@ 'app.rpc.commands.auth 'app.rpc.commands.ldap 'app.rpc.commands.demo - 'app.rpc.commands.files) + 'app.rpc.commands.files + 'app.rpc.commands.files.update + 'app.rpc.commands.files.create + 'app.rpc.commands.files.temp) (map (partial process-method cfg)) (into {})))) diff --git a/backend/src/app/rpc/commands/binfile.clj b/backend/src/app/rpc/commands/binfile.clj index d4275b281..301366d03 100644 --- a/backend/src/app/rpc/commands/binfile.clj +++ b/backend/src/app/rpc/commands/binfile.clj @@ -17,14 +17,15 @@ [app.db :as db] [app.media :as media] [app.rpc :as-alias rpc] + [app.rpc.commands.files :as files] [app.rpc.doc :as-alias doc] - [app.rpc.queries.files :as files] [app.rpc.queries.projects :as projects] [app.storage :as sto] [app.storage.tmp :as tmp] [app.tasks.file-gc] [app.util.blob :as blob] [app.util.fressian :as fres] + [app.util.pointer-map :as pmap] [app.util.services :as sv] [app.util.time :as dt] [clojure.spec.alpha :as s] @@ -290,9 +291,11 @@ (defn- retrieve-file [pool file-id] - (->> (db/query pool :file {:id file-id}) - (map files/decode-row) - (first))) + (with-open [conn (db/open pool)] + (binding [pmap/*load-fn* (partial files/load-pointer conn file-id)] + (some-> (db/get* conn :file {:id file-id}) + (files/decode-row) + (update :data files/process-pointers deref))))) (def ^:private sql:file-media-objects "SELECT * FROM file_media_object WHERE id = ANY(?)") diff --git a/backend/src/app/rpc/commands/comments.clj b/backend/src/app/rpc/commands/comments.clj index 9871f5da3..0214e3111 100644 --- a/backend/src/app/rpc/commands/comments.clj +++ b/backend/src/app/rpc/commands/comments.clj @@ -10,8 +10,8 @@ [app.common.geom.point :as gpt] [app.common.spec :as us] [app.db :as db] + [app.rpc.commands.files :as files] [app.rpc.doc :as-alias doc] - [app.rpc.queries.files :as files] [app.rpc.queries.teams :as teams] [app.rpc.retry :as retry] [app.util.blob :as blob] diff --git a/backend/src/app/rpc/commands/files.clj b/backend/src/app/rpc/commands/files.clj index 3a21ff28d..82d2dd779 100644 --- a/backend/src/app/rpc/commands/files.clj +++ b/backend/src/app/rpc/commands/files.clj @@ -6,18 +6,312 @@ (ns app.rpc.commands.files (:require + [app.common.data :as d] + [app.common.data.macros :as dm] + [app.common.exceptions :as ex] + [app.common.geom.shapes :as gsh] + [app.common.pages.helpers :as cph] + [app.common.pages.migrations :as pmg] [app.common.spec :as us] + [app.common.types.file :as ctf] + [app.common.types.shape-tree :as ctt] [app.db :as db] + [app.db.sql :as sql] + [app.rpc :as-alias rpc] + [app.rpc.commands.files.thumbnails :as-alias thumbs] [app.rpc.doc :as-alias doc] - [app.rpc.queries.files :as files] + [app.rpc.helpers :as rpch] + [app.rpc.permissions :as perms] + [app.rpc.queries.projects :as projects] + [app.rpc.queries.share-link :refer [retrieve-share-link]] + [app.rpc.queries.teams :as teams] + [app.util.blob :as blob] + [app.util.pointer-map :as pmap] [app.util.services :as sv] - [clojure.spec.alpha :as s])) + [app.util.time :as dt] + [clojure.set :as set] + [clojure.spec.alpha :as s] + [cuerdas.core :as str])) + +;; --- FEATURES + +(def supported-features + #{"storage/objects-map" + "storage/pointer-map" + "components/v2"}) + +(def default-features #{}) + +;; --- SPECS + +(s/def ::features ::us/set-of-strings) +(s/def ::file-id ::us/uuid) +(s/def ::frame-id ::us/uuid) +(s/def ::id ::us/uuid) +(s/def ::is-shared ::us/boolean) +(s/def ::name ::us/string) +(s/def ::profile-id ::us/uuid) +(s/def ::project-id ::us/uuid) +(s/def ::search-term ::us/string) +(s/def ::team-id ::us/uuid) + +(defn decode-row + [{:keys [data changes features] :as row}] + (when row + (cond-> row + features (assoc :features (db/decode-pgarray features #{})) + changes (assoc :changes (blob/decode changes)) + data (assoc :data (blob/decode data))))) + +;; --- FILE PERMISSIONS + +(def ^:private sql:file-permissions + "select fpr.is_owner, + fpr.is_admin, + fpr.can_edit + from file_profile_rel as fpr + where fpr.file_id = ? + and fpr.profile_id = ? + union all + select tpr.is_owner, + tpr.is_admin, + tpr.can_edit + from team_profile_rel as tpr + inner join project as p on (p.team_id = tpr.team_id) + inner join file as f on (p.id = f.project_id) + where f.id = ? + and tpr.profile_id = ? + union all + select ppr.is_owner, + ppr.is_admin, + ppr.can_edit + from project_profile_rel as ppr + inner join file as f on (f.project_id = ppr.project_id) + where f.id = ? + and ppr.profile_id = ?") + +(defn retrieve-file-permissions + [conn profile-id file-id] + (when (and profile-id file-id) + (db/exec! conn [sql:file-permissions + file-id profile-id + file-id profile-id + file-id profile-id]))) + +(defn get-permissions + ([conn profile-id file-id] + (let [rows (retrieve-file-permissions conn profile-id file-id) + is-owner (boolean (some :is-owner rows)) + is-admin (boolean (some :is-admin rows)) + can-edit (boolean (some :can-edit rows))] + (when (seq rows) + {:type :membership + :is-owner is-owner + :is-admin (or is-owner is-admin) + :can-edit (or is-owner is-admin can-edit) + :can-read true + :is-logged (some? profile-id)}))) + ([conn profile-id file-id share-id] + (let [perms (get-permissions conn profile-id file-id) + ldata (retrieve-share-link conn file-id share-id)] + + ;; NOTE: in a future when share-link becomes more powerful and + ;; will allow us specify which parts of the app is available, we + ;; will probably need to tweak this function in order to expose + ;; this flags to the frontend. + (cond + (some? perms) perms + (some? ldata) {:type :share-link + :can-read true + :is-logged (some? profile-id) + :who-comment (:who-comment ldata) + :who-inspect (:who-inspect ldata)})))) + +(def has-edit-permissions? + (perms/make-edition-predicate-fn get-permissions)) + +(def has-read-permissions? + (perms/make-read-predicate-fn get-permissions)) + +(def has-comment-permissions? + (perms/make-comment-predicate-fn get-permissions)) + +(def check-edition-permissions! + (perms/make-check-fn has-edit-permissions?)) + +(def check-read-permissions! + (perms/make-check-fn has-read-permissions?)) + +;; A user has comment permissions if she has read permissions, or comment permissions +(defn check-comment-permissions! + [conn profile-id file-id share-id] + (let [can-read (has-read-permissions? conn profile-id file-id) + can-comment (has-comment-permissions? conn profile-id file-id share-id)] + (when-not (or can-read can-comment) + (ex/raise :type :not-found + :code :object-not-found + :hint "not found")))) + +;; --- HELPERS + +(defn retrieve-team-id + [conn project-id] + (:team-id (db/get-by-id conn :project project-id {:columns [:team-id]}))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; FEATURES: pointer-map +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn check-features-compatibility! + [features] + (let [not-supported (set/difference features supported-features)] + (when (seq not-supported) + (ex/raise :type :restriction + :code :features-not-supported + :feature (first not-supported) + :hint (format "features %s not supported" (str/join "," not-supported)))) + features)) + +(defn load-pointer + [conn file-id id] + (let [row (db/get conn :file-data-fragment + {:id id :file-id file-id} + {:columns [:content] + :check-deleted? false})] + (blob/decode (:content row)))) + +(defn persist-pointers! + [conn file-id] + (doseq [[id item] @pmap/*tracked*] + (when (pmap/modified? item) + (let [content (-> item deref blob/encode)] + (db/insert! conn :file-data-fragment + {:id id + :file-id file-id + :content content}))))) + +(defn process-pointers + [file update-fn] + (update file :data (fn resolve-fn [data] + (cond-> data + (contains? data :pages-index) + (update :pages-index resolve-fn) + + :always + (update-vals (fn [val] + (if (pmap/pointer-map? val) + (update-fn val) + val))))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; QUERY COMMANDS ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; --- Query: File Libraries used by a File +;; --- COMMAND QUERY: get-file (by id) + +(defn retrieve-object-thumbnails + ([conn file-id] + (let [sql (str/concat + "select object_id, data " + " from file_object_thumbnail" + " where file_id=?")] + (->> (db/exec! conn [sql file-id]) + (d/index-by :object-id :data)))) + + ([conn file-id object-ids] + (let [sql (str/concat + "select object_id, data " + " from file_object_thumbnail" + " where file_id=? and object_id = ANY(?)") + ids (db/create-array conn "text" (seq object-ids))] + (->> (db/exec! conn [sql file-id ids]) + (d/index-by :object-id :data))))) + +(defn retrieve-file + [conn id client-features] + ;; here we check if client requested features are supported + (check-features-compatibility! client-features) + + (binding [pmap/*load-fn* (partial load-pointer conn id)] + (let [file (->> (db/get-by-id conn :file id) + (decode-row) + (pmg/migrate-file)) + features (:features file) + file (cond-> file + (and (contains? client-features "components/v2") + (not (contains? features "components/v2"))) + (update :data ctf/migrate-to-components-v2) + + (and (contains? features "storage/pointer-map") + (not (contains? client-features "storage/pointer-map"))) + (process-pointers deref))] + + (when (and (contains? features "components/v2") + (not (contains? client-features "components/v2"))) + (ex/raise :type :restriction + :code :feature-mismatch + :feature "components/v2" + :hint "file has 'components/v2' feature enabled but frontend didn't specifies it")) + + file))) + +(defn get-file + [conn id features] + (let [file (retrieve-file conn id features) + thumbs (retrieve-object-thumbnails conn id)] + (assoc file :thumbnails thumbs) + #_file)) + +(s/def ::get-file + (s/keys :req-un [::profile-id ::id] + :opt-un [::features])) + + +;; TODO: this should be changed probably because thumbnails will not be included + +(sv/defmethod ::get-file + "Retrieve a file by its ID. Only authenticated users." + {::doc/added "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id id features] :as params}] + (with-open [conn (db/open pool)] + (let [perms (get-permissions conn profile-id id)] + (check-read-permissions! perms) + (-> (get-file conn id features) + (assoc :permissions perms))))) + + +;; --- COMMAND QUERY: get-project-files + +(def ^:private sql:project-files + "select f.id, + f.project_id, + f.created_at, + f.modified_at, + f.name, + f.revn, + f.is_shared + from file as f + where f.project_id = ? + and f.deleted_at is null + order by f.modified_at desc") + +(s/def ::get-project-files + (s/keys :req-un [::profile-id ::project-id])) + +(defn get-project-files + [conn project-id] + (db/exec! conn [sql:project-files project-id])) + +(sv/defmethod ::get-project-files + "Get all files for the specified project." + {::doc/added "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id project-id] :as params}] + (with-open [conn (db/open pool)] + (projects/check-read-permissions! conn profile-id project-id) + (get-project-files conn project-id))) + + +;; --- COMMAND QUERY: has-file-libraries (declare retrieve-has-file-libraries) @@ -32,7 +326,7 @@ {::doc/added "1.15.1"} [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}] (with-open [conn (db/open pool)] - (files/check-read-permissions! pool profile-id file-id) + (check-read-permissions! pool profile-id file-id) (retrieve-has-file-libraries conn params))) (def ^:private sql:has-file-libraries @@ -48,3 +342,586 @@ (let [row (db/exec-one! conn [sql:has-file-libraries file-id])] (:has-libraries row))) + +;; --- QUERY COMMAND: get-page + +(defn- prune-objects + "Given the page data and the object-id returns the page data with all + other not needed objects removed from the `:objects` data + structure." + [{:keys [objects] :as page} object-id] + (let [objects (cph/get-children-with-self objects object-id)] + (assoc page :objects (d/index-by :id objects)))) + +(defn- prune-thumbnails + "Given the page data, removes the `:thumbnail` prop from all + shapes." + [page] + (update page :objects d/update-vals #(dissoc % :thumbnail))) + +(defn get-page + [conn {:keys [file-id page-id object-id features]}] + (let [file (retrieve-file conn file-id features) + page-id (or page-id (-> file :data :pages first)) + page (dm/get-in file [:data :pages-index page-id])] + (cond-> (prune-thumbnails page) + (uuid? object-id) + (prune-objects object-id)))) + +(s/def ::page-id ::us/uuid) +(s/def ::object-id ::us/uuid) +(s/def ::get-page + (s/and + (s/keys :req-un [::profile-id ::file-id] + :opt-un [::page-id ::object-id ::features]) + (fn [obj] + (if (contains? obj :object-id) + (contains? obj :page-id) + true)))) + +(sv/defmethod ::get-page + "Retrieves the page data from file and returns it. If no page-id is + specified, the first page will be returned. If object-id is + specified, only that object and its children will be returned in the + page objects data structure. + + If you specify the object-id, the page-id parameter becomes + mandatory. + + Mainly used for rendering purposes." + {::doc/added "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}] + (with-open [conn (db/open pool)] + (check-read-permissions! conn profile-id file-id) + (get-page conn params))) + + +;; --- COMMAND QUERY: get-team-shared-files + +(def ^:private sql:team-shared-files + "select f.id, + f.revn, + f.data, + f.project_id, + f.created_at, + f.modified_at, + f.name, + f.is_shared + from file as f + inner join project as p on (p.id = f.project_id) + where f.is_shared = true + and f.deleted_at is null + and p.deleted_at is null + and p.team_id = ? + order by f.modified_at desc") + +(defn get-team-shared-files + [conn {:keys [team-id] :as params}] + (let [assets-sample + (fn [assets limit] + (let [sorted-assets (->> (vals assets) + (sort-by #(str/lower (:name %))))] + + {:count (count sorted-assets) + :sample (into [] (take limit sorted-assets))})) + + library-summary + (fn [data] + {:components (assets-sample (:components data) 4) + :colors (assets-sample (:colors data) 3) + :typographies (assets-sample (:typographies data) 3)}) + + xform (comp + (map decode-row) + (map #(assoc % :library-summary (library-summary (:data %)))) + (map #(dissoc % :data)))] + + (into #{} xform (db/exec! conn [sql:team-shared-files team-id])))) + +(s/def ::get-team-shared-files + (s/keys :req-un [::profile-id ::team-id])) + +(sv/defmethod ::get-team-shared-files + "Get all file (libraries) for the specified team." + {::doc/added "1.17"} + [{:keys [pool] :as cfg} params] + (with-open [conn (db/open pool)] + (get-team-shared-files conn params))) + + +;; --- COMMAND QUERY: get-file-libraries + +(def ^:private sql:file-libraries + "WITH RECURSIVE libs AS ( + SELECT fl.*, flr.synced_at + FROM file AS fl + JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id) + WHERE flr.file_id = ?::uuid + UNION + SELECT fl.*, flr.synced_at + FROM file AS fl + JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id) + JOIN libs AS l ON (flr.file_id = l.id) + ) + SELECT l.id, + l.data, + l.project_id, + l.created_at, + l.modified_at, + l.deleted_at, + l.name, + l.revn, + l.synced_at + FROM libs AS l + WHERE l.deleted_at IS NULL OR l.deleted_at > now();") + +(defn get-file-libraries + [conn is-indirect file-id] + (let [xform (comp + (map #(assoc % :is-indirect is-indirect)) + (map decode-row))] + (into #{} xform (db/exec! conn [sql:file-libraries file-id])))) + +(s/def ::get-file-libraries + (s/keys :req-un [::profile-id ::file-id])) + +(sv/defmethod ::get-file-libraries + "Get libraries used by the specified file." + {::doc/added "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}] + (with-open [conn (db/open pool)] + (check-read-permissions! conn profile-id file-id) + (get-file-libraries conn false file-id))) + + +;; --- COMMAND QUERY: Files that use this File library + +(def ^:private sql:library-using-files + "SELECT f.id, + f.name + FROM file_library_rel AS flr + JOIN file AS f ON (f.id = flr.file_id) + WHERE flr.library_file_id = ? + AND (f.deleted_at IS NULL OR f.deleted_at > now())") + +(defn get-library-file-references + [conn file-id] + (db/exec! conn [sql:library-using-files file-id])) + +(s/def ::get-library-file-references + (s/keys :req-un [::profile-id ::file-id])) + +(sv/defmethod ::get-library-file-references + "Returns all the file references that use specified file (library) id." + {::doc/added "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}] + (with-open [conn (db/open pool)] + (check-read-permissions! conn profile-id file-id) + (get-library-file-references conn file-id))) + + +;; --- COMMAND QUERY: get-team-recent-files + +(def sql:team-recent-files + "with recent_files as ( + select f.id, + f.revn, + f.project_id, + f.created_at, + f.modified_at, + f.name, + f.is_shared, + row_number() over w as row_num + from file as f + join project as p on (p.id = f.project_id) + where p.team_id = ? + and p.deleted_at is null + and f.deleted_at is null + window w as (partition by f.project_id order by f.modified_at desc) + order by f.modified_at desc + ) + select * from recent_files where row_num <= 10;") + +(defn get-team-recent-files + [conn team-id] + (db/exec! conn [sql:team-recent-files team-id])) + +(s/def ::get-team-recent-files + (s/keys :req-un [::profile-id ::team-id])) + +(sv/defmethod ::get-team-recent-files + {::doc/added "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id team-id]}] + (with-open [conn (db/open pool)] + (teams/check-read-permissions! conn profile-id team-id) + (get-team-recent-files conn team-id))) + + +;; --- COMMAND QUERY: get-file-thumbnail + +(defn get-file-thumbnail + [conn file-id revn] + (let [sql (sql/select :file-thumbnail + (cond-> {:file-id file-id} + revn (assoc :revn revn)) + {:limit 1 + :order-by [[:revn :desc]]}) + row (db/exec-one! conn sql)] + (when-not row + (ex/raise :type :not-found + :code :file-thumbnail-not-found)) + + {:data (:data row) + :props (some-> (:props row) db/decode-transit-pgobject) + :revn (:revn row) + :file-id (:file-id row)})) + +(s/def ::revn ::us/integer) + +(s/def ::get-file-thumbnail + (s/keys :req-un [::profile-id ::file-id] + :opt-un [::revn])) + +(sv/defmethod ::get-file-thumbnail + {::doc/added "1.17"} + [{:keys [pool]} {:keys [profile-id file-id revn]}] + (with-open [conn (db/open pool)] + (check-read-permissions! conn profile-id file-id) + (-> (get-file-thumbnail conn file-id revn) + (with-meta {::rpc/transform-response (rpch/http-cache {:max-age (* 1000 60 60)})})))) + + + +;; --- COMMAND QUERY: get-file-data-for-thumbnail + +(defn get-file-data-for-thumbnail + [conn {:keys [data id] :as file}] + (letfn [;; function responsible on finding the frame marked to be + ;; used as thumbnail; the returned frame always have + ;; the :page-id set to the page that it belongs. + (get-thumbnail-frame [data] + (d/seek :use-for-thumbnail? + (for [page (-> data :pages-index vals) + frame (-> page :objects ctt/get-frames)] + (assoc frame :page-id (:id page))))) + + ;; function responsible to filter objects data structure of + ;; all unneeded shapes if a concrete frame is provided. If no + ;; frame, the objects is returned untouched. + (filter-objects [objects frame-id] + (d/index-by :id (cph/get-children-with-self objects frame-id))) + + ;; function responsible of assoc available thumbnails + ;; to frames and remove all children shapes from objects if + ;; thumbnails is available + (assoc-thumbnails [objects page-id thumbnails] + (loop [objects objects + frames (filter cph/frame-shape? (vals objects))] + + (if-let [frame (-> frames first)] + (let [frame-id (:id frame) + object-id (str page-id frame-id) + frame (if-let [thumb (get thumbnails object-id)] + (assoc frame :thumbnail thumb :shapes []) + (dissoc frame :thumbnail)) + + children-ids + (cph/get-children-ids objects frame-id) + + bounds + (when (:show-content frame) + (gsh/selection-rect (concat [frame] (->> children-ids (map (d/getf objects)))))) + + frame + (cond-> frame + (some? bounds) + (assoc :children-bounds bounds))] + + (if (:thumbnail frame) + (recur (-> objects + (assoc frame-id frame) + (d/without-keys children-ids)) + (rest frames)) + (recur (assoc objects frame-id frame) + (rest frames)))) + + objects)))] + + (let [frame (get-thumbnail-frame data) + frame-id (:id frame) + page-id (or (:page-id frame) + (-> data :pages first)) + + page (dm/get-in data [:pages-index page-id]) + frame-ids (if (some? frame) (list frame-id) (map :id (ctt/get-frames (:objects page)))) + + obj-ids (map #(str page-id %) frame-ids) + thumbs (retrieve-object-thumbnails conn id obj-ids)] + + (cond-> page + ;; If we have frame, we need to specify it on the page level + ;; and remove the all other unrelated objects. + (some? frame-id) + (-> (assoc :thumbnail-frame-id frame-id) + (update :objects filter-objects frame-id)) + + ;; Assoc the available thumbnails and prune not visible shapes + ;; for avoid transfer unnecessary data. + :always + (update :objects assoc-thumbnails page-id thumbs))))) + +(s/def ::get-file-data-for-thumbnail + (s/keys :req-un [::profile-id ::file-id] + :opt-un [::features])) + +(sv/defmethod ::get-file-data-for-thumbnail + "Retrieves the data for generate the thumbnail of the file. Used + mainly for render thumbnails on dashboard." + {::doc/added "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id file-id features] :as props}] + (with-open [conn (db/open pool)] + (check-read-permissions! conn profile-id file-id) + (let [file (retrieve-file conn file-id features)] + {:file-id file-id + :revn (:revn file) + :page (get-file-data-for-thumbnail conn file)}))) + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; MUTATION COMMANDS +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; --- MUTATION COMMAND: rename-file + +(defn rename-file + [conn {:keys [id name] :as params}] + (-> (db/update! conn :file + {:name name + :modified-at (dt/now)} + {:id id}) + (select-keys [:id :name :created-at :modified-at]))) + +(s/def ::rename-file + (s/keys :req-un [::profile-id ::name ::id])) + +(sv/defmethod ::rename-file + {::doc/added "1.17"} + [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}] + (db/with-atomic [conn pool] + (check-edition-permissions! conn profile-id id) + (rename-file conn params))) + + +;; --- MUTATION COMMAND: set-file-shared + +(defn unlink-files + [conn {:keys [id] :as params}] + (db/delete! conn :file-library-rel {:library-file-id id})) + +(defn set-file-shared + [conn {:keys [id is-shared] :as params}] + (-> (db/update! conn :file + {:is-shared is-shared} + {:id id}) + (select-keys [:id :name :is-shared]))) + +(defn absorb-library + "Find all files using a shared library, and absorb all library assets + into the file local libraries" + [conn {:keys [id] :as params}] + (let [library (db/get-by-id conn :file id)] + (when (:is-shared library) + (let [ldata (-> library decode-row pmg/migrate-file :data)] + (->> (db/query conn :file-library-rel {:library-file-id id}) + (map :file-id) + (keep #(db/get-by-id conn :file % {:check-deleted? false})) + (map decode-row) + (map pmg/migrate-file) + (run! (fn [{:keys [id data revn] :as file}] + (let [data (ctf/absorb-assets data ldata)] + (db/update! conn :file + {:revn (inc revn) + :data (blob/encode data) + :modified-at (dt/now)} + {:id id}))))))))) + +(s/def ::set-file-shared + (s/keys :req-un [::profile-id ::id ::is-shared])) + +(sv/defmethod ::set-file-shared + {::doc/added "1.17"} + [{:keys [pool] :as cfg} {:keys [id profile-id is-shared] :as params}] + (db/with-atomic [conn pool] + (check-edition-permissions! conn profile-id id) + (when-not is-shared + (absorb-library conn params) + (unlink-files conn params)) + (set-file-shared conn params))) + + +;; --- MUTATION COMMAND: delete-file + +(defn mark-file-deleted + [conn {:keys [id] :as params}] + (db/update! conn :file + {:deleted-at (dt/now)} + {:id id}) + nil) + +(s/def ::delete-file + (s/keys :req-un [::id ::profile-id])) + +(sv/defmethod ::delete-file + {::doc/added "1.17"} + [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}] + (db/with-atomic [conn pool] + (check-edition-permissions! conn profile-id id) + (absorb-library conn params) + (mark-file-deleted conn params))) + + +;; --- MUTATION COMMAND: link-file-to-library + +(def sql:link-file-to-library + "insert into file_library_rel (file_id, library_file_id) + values (?, ?) + on conflict do nothing;") + +(defn link-file-to-library + [conn {:keys [file-id library-id] :as params}] + (db/exec-one! conn [sql:link-file-to-library file-id library-id])) + +(s/def ::link-file-to-library + (s/keys :req-un [::profile-id ::file-id ::library-id])) + +(sv/defmethod ::link-file-to-library + {::doc/added "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id file-id library-id] :as params}] + (when (= file-id library-id) + (ex/raise :type :validation + :code :invalid-library + :hint "A file cannot be linked to itself")) + (db/with-atomic [conn pool] + (check-edition-permissions! conn profile-id file-id) + (check-edition-permissions! conn profile-id library-id) + (link-file-to-library conn params))) + + +;; --- MUTATION COMMAND: unlink-file-from-library + +(defn unlink-file-from-library + [conn {:keys [file-id library-id] :as params}] + (db/delete! conn :file-library-rel + {:file-id file-id + :library-file-id library-id})) + +(s/def ::unlink-file-from-library + (s/keys :req-un [::profile-id ::file-id ::library-id])) + +(sv/defmethod ::unlink-file-from-library + {::doc/added "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}] + (db/with-atomic [conn pool] + (check-edition-permissions! conn profile-id file-id) + (unlink-file-from-library conn params))) + + +;; --- MUTATION COMMAND: update-sync + +(defn update-sync + [conn {:keys [file-id library-id] :as params}] + (db/update! conn :file-library-rel + {:synced-at (dt/now)} + {:file-id file-id + :library-file-id library-id})) + +(s/def ::update-file-library-sync-status + (s/keys :req-un [::profile-id ::file-id ::library-id])) + +;; TODO: improve naming + +(sv/defmethod ::update-file-library-sync-status + "Update the synchronization statos of a file->library link" + {::doc/added "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}] + (db/with-atomic [conn pool] + (check-edition-permissions! conn profile-id file-id) + (update-sync conn params))) + + +;; --- MUTATION COMMAND: ignore-sync + +(defn ignore-sync + [conn {:keys [file-id date] :as params}] + (db/update! conn :file + {:ignore-sync-until date} + {:id file-id})) + +(s/def ::ignore-file-library-sync-status + (s/keys :req-un [::profile-id ::file-id ::date])) + +;; TODO: improve naming +(sv/defmethod ::ignore-file-library-sync-status + "Ignore updates in linked files" + {::doc/added "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}] + (db/with-atomic [conn pool] + (check-edition-permissions! conn profile-id file-id) + (ignore-sync conn params))) + + +;; --- MUTATION COMMAND: upsert-file-object-thumbnail + +(def sql:upsert-object-thumbnail + "insert into file_object_thumbnail(file_id, object_id, data) + values (?, ?, ?) + on conflict(file_id, object_id) do + update set data = ?;") + +(defn upsert-file-object-thumbnail! + [conn {:keys [file-id object-id data]}] + (if data + (db/exec-one! conn [sql:upsert-object-thumbnail file-id object-id data data]) + (db/delete! conn :file-object-thumbnail {:file-id file-id :object-id object-id}))) + +(s/def ::data (s/nilable ::us/string)) +(s/def ::thumbs/object-id ::us/string) +(s/def ::upsert-file-object-thumbnail + (s/keys :req-un [::profile-id ::file-id ::thumbs/object-id ::data])) + +(sv/defmethod ::upsert-file-object-thumbnail + {::doc/added "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}] + (db/with-atomic [conn pool] + (check-edition-permissions! conn profile-id file-id) + (upsert-file-object-thumbnail! conn params) + nil)) + +;; --- MUTATION COMMAND: upsert-file-thumbnail + +(def sql:upsert-file-thumbnail + "insert into file_thumbnail (file_id, revn, data, props) + values (?, ?, ?, ?::jsonb) + on conflict(file_id, revn) do + update set data = ?, props=?, updated_at=now();") + +(defn upsert-file-thumbnail + [conn {:keys [file-id revn data props]}] + (let [props (db/tjson (or props {}))] + (db/exec-one! conn [sql:upsert-file-thumbnail + file-id revn data props data props]))) + +(s/def ::revn ::us/integer) +(s/def ::props map?) +(s/def ::upsert-file-thumbnail + (s/keys :req-un [::profile-id ::file-id ::revn ::data ::props])) + +(sv/defmethod ::upsert-file-thumbnail + "Creates or updates the file thumbnail. Mainly used for paint the + grid thumbnails." + {::doc/added "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}] + (db/with-atomic [conn pool] + (check-edition-permissions! conn profile-id file-id) + (upsert-file-thumbnail conn params) + nil)) diff --git a/backend/src/app/rpc/commands/files/create.clj b/backend/src/app/rpc/commands/files/create.clj new file mode 100644 index 000000000..0baa57296 --- /dev/null +++ b/backend/src/app/rpc/commands/files/create.clj @@ -0,0 +1,83 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.rpc.commands.files.create + (:require + [app.common.data :as d] + [app.common.files.features :as ffeat] + [app.common.types.file :as ctf] + [app.common.uuid :as uuid] + [app.db :as db] + [app.loggers.audit :as audit] + [app.rpc.commands.files :as files] + [app.rpc.doc :as-alias doc] + [app.rpc.permissions :as perms] + [app.rpc.queries.projects :as proj] + [app.util.blob :as blob] + [app.util.objects-map :as omap] + [app.util.pointer-map :as pmap] + [app.util.services :as sv] + [clojure.spec.alpha :as s])) + +(defn create-file-role! + [conn {:keys [file-id profile-id role]}] + (let [params {:file-id file-id + :profile-id profile-id}] + (->> (perms/assign-role-flags params role) + (db/insert! conn :file-profile-rel)))) + +(defn create-file + [conn {:keys [id name project-id is-shared data revn + modified-at deleted-at + ignore-sync-until features] + :or {is-shared false revn 0} + :as params}] + (let [id (or id (:id data) (uuid/next)) + features (-> (into files/default-features features) + (files/check-features-compatibility!)) + + data (or data + (binding [ffeat/*current* features + ffeat/*wrap-with-objects-map-fn* (if (features "storate/objects-map") omap/wrap identity) + ffeat/*wrap-with-pointer-map-fn* (if (features "storage/pointer-map") pmap/wrap identity)] + (ctf/make-file-data id))) + + features (db/create-array conn "text" features) + file (db/insert! conn :file + (d/without-nils + {:id id + :project-id project-id + :name name + :revn revn + :is-shared is-shared + :data (blob/encode data) + :features features + :ignore-sync-until ignore-sync-until + :modified-at modified-at + :deleted-at deleted-at}))] + + (->> (assoc params :file-id id :role :owner) + (create-file-role! conn)) + + (files/decode-row file))) + +(s/def ::create-file + (s/keys :req-un [::files/profile-id + ::files/name + ::files/project-id] + :opt-un [::files/id + ::files/is-shared + ::files/features])) + +(sv/defmethod ::create-file + {::doc/added "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id project-id] :as params}] + (db/with-atomic [conn pool] + (proj/check-edition-permissions! conn profile-id project-id) + (let [team-id (files/retrieve-team-id conn project-id)] + (-> (create-file conn params) + (vary-meta assoc ::audit/props {:team-id team-id}))))) + diff --git a/backend/src/app/rpc/commands/files/temp.clj b/backend/src/app/rpc/commands/files/temp.clj new file mode 100644 index 000000000..350ddb23c --- /dev/null +++ b/backend/src/app/rpc/commands/files/temp.clj @@ -0,0 +1,96 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.rpc.commands.files.temp + (:require + [app.common.exceptions :as ex] + [app.common.pages :as cp] + [app.common.uuid :as uuid] + [app.db :as db] + [app.rpc.commands.files :as files] + [app.rpc.commands.files.create :as files.create] + [app.rpc.commands.files.update :as files.update] + [app.rpc.doc :as-alias doc] + [app.rpc.queries.projects :as proj] + [app.util.blob :as blob] + [app.util.services :as sv] + [app.util.time :as dt] + [clojure.spec.alpha :as s])) + +;; --- MUTATION COMMAND: create-temp-file + +(s/def ::create-temp-file ::files.create/create-file) + +(sv/defmethod ::create-temp-file + {::doc/added "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id project-id] :as params}] + (db/with-atomic [conn pool] + (proj/check-edition-permissions! conn profile-id project-id) + (files.create/create-file conn (assoc params :deleted-at (dt/in-future {:days 1}))))) + +;; --- MUTATION COMMAND: update-temp-file + +(defn update-temp-file + [conn {:keys [profile-id session-id id revn changes] :as params}] + (db/insert! conn :file-change + {:id (uuid/next) + :session-id session-id + :profile-id profile-id + :created-at (dt/now) + :file-id id + :revn revn + :data nil + :changes (blob/encode changes)})) + +(s/def ::update-temp-file + (s/keys :req-un [::files.update/changes + ::files.update/revn + ::files.update/session-id + ::files/id])) + +(sv/defmethod ::update-temp-file + {::doc/added "1.17"} + [{:keys [pool] :as cfg} params] + (db/with-atomic [conn pool] + (update-temp-file conn params) + nil)) + +;; --- MUTATION COMMAND: persist-temp-file + +(defn persist-temp-file + [conn {:keys [id] :as params}] + (let [file (db/get-by-id conn :file id) + revs (db/query conn :file-change + {:file-id id} + {:order-by [[:revn :asc]]}) + revn (count revs)] + + (when (nil? (:deleted-at file)) + (ex/raise :type :validation + :code :cant-persist-already-persisted-file)) + + (loop [revs (seq revs) + data (blob/decode (:data file))] + (if-let [rev (first revs)] + (recur (rest revs) + (->> rev :changes blob/decode (cp/process-changes data))) + (db/update! conn :file + {:deleted-at nil + :revn revn + :data (blob/encode data)} + {:id id}))) + nil)) + +(s/def ::persist-temp-file + (s/keys :req-un [::files/id + ::files/profile-id])) + +(sv/defmethod ::persist-temp-file + {::doc/added "1.17"} + [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}] + (db/with-atomic [conn pool] + (files/check-edition-permissions! conn profile-id id) + (persist-temp-file conn params))) diff --git a/backend/src/app/rpc/commands/files/update.clj b/backend/src/app/rpc/commands/files/update.clj new file mode 100644 index 000000000..9a6568663 --- /dev/null +++ b/backend/src/app/rpc/commands/files/update.clj @@ -0,0 +1,295 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.rpc.commands.files.update + (:require + [app.common.exceptions :as ex] + [app.common.files.features :as ffeat] + [app.common.logging :as l] + [app.common.pages :as cp] + [app.common.pages.migrations :as pmg] + [app.common.spec :as us] + [app.common.types.file :as ctf] + [app.common.uuid :as uuid] + [app.config :as cf] + [app.db :as db] + [app.loggers.audit :as audit] + [app.metrics :as mtx] + [app.msgbus :as mbus] + [app.rpc :as-alias rpc] + [app.rpc.climit :as-alias climit] + [app.rpc.commands.files :as files] + [app.rpc.doc :as-alias doc] + [app.util.blob :as blob] + [app.util.objects-map :as omap] + [app.util.pointer-map :as pmap] + [app.util.services :as sv] + [app.util.time :as dt] + [clojure.spec.alpha :as s])) + +;; --- SPECS + +(s/def ::changes + (s/coll-of map? :kind vector?)) + +(s/def ::hint-origin ::us/keyword) +(s/def ::hint-events + (s/every ::us/keyword :kind vector?)) + +(s/def ::change-with-metadata + (s/keys :req-un [::changes] + :opt-un [::hint-origin + ::hint-events])) + +(s/def ::changes-with-metadata + (s/every ::change-with-metadata :kind vector?)) + +(s/def ::session-id ::us/uuid) +(s/def ::revn ::us/integer) +(s/def ::update-file + (s/and + (s/keys :req-un [::files/id ::files/profile-id ::session-id ::revn] + :opt-un [::changes ::changes-with-metadata ::features]) + (fn [o] + (or (contains? o :changes) + (contains? o :changes-with-metadata))))) + +;; --- HELPERS + +;; File changes that affect to the library, and must be notified +;; to all clients using it. + +(def ^:private library-change-types + #{:add-color :mod-color :del-color + :add-media :mod-media :del-media + :add-component :mod-component :del-component + :add-typography :mod-typography :del-typography}) + +(def ^:private file-change-types + #{:add-obj :mod-obj :del-obj + :reg-objects :mov-objects}) + +(defn- library-change? + [{:keys [type] :as change}] + (or (contains? library-change-types type) + (and (contains? file-change-types type) + (some? (:component-id change))))) + +(def ^:private sql:get-file + "SELECT f.*, p.team_id + FROM file AS f + JOIN project AS p ON (p.id = f.project_id) + WHERE f.id = ? + AND (f.deleted_at IS NULL OR + f.deleted_at > now()) + FOR KEY SHARE") + +(defn get-file + [conn id] + (let [file (db/exec-one! conn [sql:get-file id])] + (when-not file + (ex/raise :type :not-found + :code :object-not-found + :hint (format "file with id '%s' does not exists" id))) + (update file :features db/decode-pgarray #{}))) + +(defn- wrap-with-pointer-map-context + [f] + (fn [{:keys [conn] :as cfg} {:keys [id] :as file}] + (binding [pmap/*tracked* (atom {}) + pmap/*load-fn* (partial files/load-pointer conn id) + ffeat/*wrap-with-pointer-map-fn* pmap/wrap] + (let [result (f cfg file)] + (files/persist-pointers! conn id) + result)))) + +(defn- wrap-with-objects-map-context + [f] + (fn [cfg file] + (binding [ffeat/*wrap-with-objects-map-fn* omap/wrap] + (f cfg file)))) + +(declare get-lagged-changes) +(declare send-notifications!) +(declare update-file) +(declare update-file*) +(declare take-snapshot?) + +;; If features are specified from params and the final feature +;; set is different than the persisted one, update it on the +;; database. + +(sv/defmethod ::update-file + {::climit/queue :update-file + ::climit/key-fn :id + ::doc/added "1.17"} + [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}] + + (db/with-atomic [conn pool] + (files/check-edition-permissions! conn profile-id id) + (db/xact-lock! conn id) + + (let [cfg (assoc cfg :conn conn) + tpoint (dt/tpoint)] + (-> (update-file cfg params) + (vary-meta assoc ::rpc/before-complete + (fn [] + (let [elapsed (tpoint)] + (l/trace :hint "update-file" :time (dt/format-duration elapsed))))))))) + +(defn update-file + [{:keys [conn metrics] :as cfg} {:keys [id profile-id changes changes-with-metadata] :as params}] + (let [file (get-file conn id) + features (->> (concat (:features file) + (:features params)) + (into files/default-features) + (files/check-features-compatibility!))] + + (files/check-edition-permissions! conn profile-id (:id file)) + + (binding [ffeat/*current* features + ffeat/*previous* (:features file)] + (let [update-fn (cond-> update-file* + (contains? features "storage/pointer-map") + (wrap-with-pointer-map-context) + + (contains? features "storage/objects-map") + (wrap-with-objects-map-context)) + + file (assoc file :features features) + changes (if changes-with-metadata + (->> changes-with-metadata (mapcat :changes) vec) + (vec changes)) + + params (assoc params :file file :changes changes)] + + (mtx/run! metrics {:id :update-file-changes :inc (count changes)}) + + (when (not= features (:features file)) + (let [features (db/create-array conn "text" features)] + (db/update! conn :file + {:features features} + {:id id}))) + + (-> (update-fn cfg params) + (vary-meta assoc ::audit/props {:project-id (:project-id file) + :team-id (:team-id file)})))))) + +(defn- update-file* + [{:keys [conn] :as cfg} {:keys [file changes session-id profile-id] :as params}] + (when (> (:revn params) + (:revn file)) + (ex/raise :type :validation + :code :revn-conflict + :hint "The incoming revision number is greater that stored version." + :context {:incoming-revn (:revn params) + :stored-revn (:revn file)})) + + (let [ts (dt/now) + file (-> file + (update :revn inc) + (update :data (fn [data] + (cond-> data + :always + (-> (blob/decode) + (assoc :id (:id file)) + (pmg/migrate-data)) + + (and (contains? ffeat/*current* "components/v2") + (not (contains? ffeat/*previous* "components/v2"))) + (ctf/migrate-to-components-v2) + + :always + (-> (cp/process-changes changes) + (blob/encode))))))] + (db/insert! conn :file-change + {:id (uuid/next) + :session-id session-id + :profile-id profile-id + :created-at ts + :file-id (:id file) + :revn (:revn file) + :features (db/create-array conn "text" (:features file)) + :data (when (take-snapshot? file) + (:data file)) + :changes (blob/encode changes)}) + + (db/update! conn :file + {:revn (:revn file) + :data (:data file) + :data-backend nil + :modified-at ts + :has-media-trimmed false} + {:id (:id file)}) + + (db/update! conn :project + {:modified-at ts} + {:id (:project-id file)}) + + (let [params (assoc params :file file)] + ;; Send asynchronous notifications + (send-notifications! cfg params) + + ;; Retrieve and return lagged data + (get-lagged-changes conn params)))) + +(defn- take-snapshot? + "Defines the rule when file `data` snapshot should be saved." + [{:keys [revn modified-at] :as file}] + (let [freq (or (cf/get :file-change-snapshot-every) 20) + timeout (or (cf/get :file-change-snapshot-timeout) + (dt/duration {:hours 1}))] + (or (= 1 freq) + (zero? (mod revn freq)) + (> (inst-ms (dt/diff modified-at (dt/now))) + (inst-ms timeout))))) + +(def ^:private + sql:lagged-changes + "select s.id, s.revn, s.file_id, + s.session_id, s.changes + from file_change as s + where s.file_id = ? + and s.revn > ? + order by s.created_at asc") + +(defn- get-lagged-changes + [conn params] + (->> (db/exec! conn [sql:lagged-changes (:id params) (:revn params)]) + (into [] (comp (map files/decode-row) + (map (fn [row] + (cond-> row + (= (:revn row) (:revn (:file params))) + (assoc :changes [])))))))) + +(defn- send-notifications! + [{:keys [conn] :as cfg} {:keys [file changes session-id] :as params}] + (let [lchanges (filter library-change? changes) + msgbus (:msgbus cfg)] + + ;; Asynchronously publish message to the msgbus + (mbus/pub! msgbus + :topic (:id file) + :message {:type :file-change + :profile-id (:profile-id params) + :file-id (:id file) + :session-id (:session-id params) + :revn (:revn file) + :changes changes}) + + (when (and (:is-shared file) (seq lchanges)) + (let [team-id (or (:team-id file) + (files/retrieve-team-id conn (:project-id file)))] + ;; Asynchronously publish message to the msgbus + (mbus/pub! msgbus + :topic team-id + :message {:type :library-change + :profile-id (:profile-id params) + :file-id (:id file) + :session-id session-id + :revn (:revn file) + :modified-at (dt/now) + :changes lchanges}))))) diff --git a/backend/src/app/rpc/commands/management.clj b/backend/src/app/rpc/commands/management.clj index c68ecaea3..7e56bec6f 100644 --- a/backend/src/app/rpc/commands/management.clj +++ b/backend/src/app/rpc/commands/management.clj @@ -14,11 +14,13 @@ [app.common.uuid :as uuid] [app.db :as db] [app.rpc.commands.binfile :as binfile] + [app.rpc.commands.files :as files] [app.rpc.doc :as-alias doc] [app.rpc.mutations.projects :refer [create-project-role create-project]] [app.rpc.queries.projects :as proj] [app.rpc.queries.teams :as teams] [app.util.blob :as blob] + [app.util.pointer-map :as pmap] [app.util.services :as sv] [app.util.time :as dt] [clojure.spec.alpha :as s] @@ -53,7 +55,7 @@ (assoc key (get index (get item key) (get item key))))) (defn- process-file - [file index] + [conn {:keys [id] :as file} index] (letfn [(process-form [form] (cond-> form ;; Relink library items @@ -97,18 +99,25 @@ res))) media media))] - - (update file :data - (fn [data] - (-> data - (blob/decode) - (assoc :id (:id file)) - (pmg/migrate-data) - (update :pages-index relink-shapes) - (update :components relink-shapes) - (update :media relink-media) - (d/without-nils) - (blob/encode)))))) + (-> file + (update :id #(get index %)) + (update :data + (fn [data] + (binding [pmap/*load-fn* (partial files/load-pointer conn id) + pmap/*tracked* (atom {})] + (let [file-id (get index id) + data (-> data + (blob/decode) + (assoc :id file-id) + (pmg/migrate-data) + (update :pages-index relink-shapes) + (update :components relink-shapes) + (update :media relink-media) + (d/without-nils) + (files/process-pointers pmap/clone) + (blob/encode))] + (files/persist-pointers! conn file-id) + data))))))) (def sql:retrieve-used-libraries "select flr.* @@ -166,9 +175,9 @@ file (-> file (assoc :created-at now) (assoc :modified-at now) - (assoc :ignore-sync-until ignore) - (update :id #(get index %)) - (process-file index))] + (assoc :ignore-sync-until ignore)) + + file (process-file conn file index)] (db/insert! conn :file file) (db/insert! conn :file-profile-rel diff --git a/backend/src/app/rpc/commands/viewer.clj b/backend/src/app/rpc/commands/viewer.clj new file mode 100644 index 000000000..1f757b86b --- /dev/null +++ b/backend/src/app/rpc/commands/viewer.clj @@ -0,0 +1,89 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.rpc.commands.viewer + (:require + [app.common.exceptions :as ex] + [app.db :as db] + [app.rpc.commands.comments :as comments] + [app.rpc.commands.files :as files] + [app.rpc.doc :as-alias doc] + [app.rpc.queries.share-link :as slnk] + [app.util.services :as sv] + [clojure.spec.alpha :as s])) + +;; --- Query: View Only Bundle + +(defn- get-project + [conn id] + (db/get-by-id conn :project id {:columns [:id :name :team-id]})) + +(defn- get-bundle + [conn file-id profile-id features] + (let [file (files/get-file conn file-id features) + project (get-project conn (:project-id file)) + libs (files/get-file-libraries conn false file-id) + users (comments/get-file-comments-users conn file-id profile-id) + + links (->> (db/query conn :share-link {:file-id file-id}) + (mapv slnk/decode-share-link-row)) + + fonts (db/query conn :team-font-variant + {:team-id (:team-id project) + :deleted-at nil})] + {:file file + :users users + :fonts fonts + :project project + :share-links links + :libraries libs})) + +(defn get-view-only-bundle + [conn {:keys [profile-id file-id share-id features] :as params}] + (let [slink (slnk/retrieve-share-link conn file-id share-id) + perms (files/get-permissions conn profile-id file-id share-id) + thumbs (files/retrieve-object-thumbnails conn file-id) + bundle (-> (get-bundle conn file-id profile-id features) + (assoc :permissions perms) + (assoc-in [:file :thumbnails] thumbs))] + + ;; When we have neither profile nor share, we just return a not + ;; found response to the user. + (when (and (not profile-id) + (not slink)) + (ex/raise :type :not-found + :code :object-not-found)) + + ;; When we have only profile, we need to check read permissions + ;; on file. + (when (and profile-id (not slink)) + (files/check-read-permissions! conn profile-id file-id)) + + (cond-> bundle + (some? slink) + (assoc :share slink) + + (and (some? slink) + (not (contains? (:flags slink) "view-all-pages"))) + (update-in [:file :data] (fn [data] + (let [allowed-pages (:pages slink)] + (-> data + (update :pages (fn [pages] (filterv #(contains? allowed-pages %) pages))) + (update :pages-index (fn [index] (select-keys index allowed-pages)))))))))) + +(s/def ::get-view-only-bundle + (s/keys :req-un [::files/file-id] + :opt-un [::files/profile-id + ::files/share-id + ::files/features])) + +(sv/defmethod ::get-view-only-bundle + {:auth false + ::doc/added "1.17"} + [{:keys [pool]} params] + (with-open [conn (db/open pool)] + (get-view-only-bundle conn params))) + diff --git a/backend/src/app/rpc/mutations/comments.clj b/backend/src/app/rpc/mutations/comments.clj index 771947e21..6b606ba35 100644 --- a/backend/src/app/rpc/mutations/comments.clj +++ b/backend/src/app/rpc/mutations/comments.clj @@ -10,8 +10,8 @@ [app.common.spec :as us] [app.db :as db] [app.rpc.commands.comments :as cmd.comments] + [app.rpc.commands.files :as cmd.files] [app.rpc.doc :as-alias doc] - [app.rpc.queries.files :as files] [app.rpc.retry :as retry] [app.util.services :as sv] [clojure.spec.alpha :as s])) @@ -27,7 +27,7 @@ ::doc/deprecated "1.15"} [{:keys [pool] :as cfg} {:keys [profile-id file-id share-id] :as params}] (db/with-atomic [conn pool] - (files/check-comment-permissions! conn profile-id file-id share-id) + (cmd.files/check-comment-permissions! conn profile-id file-id share-id) (cmd.comments/create-comment-thread conn params))) ;; --- Mutation: Update Comment Thread Status @@ -44,7 +44,7 @@ (db/with-atomic [conn pool] (let [cthr (db/get-by-id conn :comment-thread id {:for-update true})] (when-not cthr (ex/raise :type :not-found)) - (files/check-comment-permissions! conn profile-id (:file-id cthr) share-id) + (cmd.files/check-comment-permissions! conn profile-id (:file-id cthr) share-id) (cmd.comments/upsert-comment-thread-status! conn profile-id (:id cthr))))) @@ -61,7 +61,7 @@ (when-not thread (ex/raise :type :not-found)) - (files/check-comment-permissions! conn profile-id (:file-id thread) share-id) + (cmd.files/check-comment-permissions! conn profile-id (:file-id thread) share-id) (db/update! conn :comment-thread {:is-resolved is-resolved} {:id id}) diff --git a/backend/src/app/rpc/mutations/files.clj b/backend/src/app/rpc/mutations/files.clj index 8c036fbb1..20b3a5448 100644 --- a/backend/src/app/rpc/mutations/files.clj +++ b/backend/src/app/rpc/mutations/files.clj @@ -6,641 +6,233 @@ (ns app.rpc.mutations.files (:require - [app.common.data :as d] [app.common.exceptions :as ex] - [app.common.files.features :as ffeat] [app.common.logging :as l] - [app.common.pages :as cp] - [app.common.pages.migrations :as pmg] [app.common.spec :as us] - [app.common.types.file :as ctf] - [app.common.uuid :as uuid] - [app.config :as cf] [app.db :as db] [app.loggers.audit :as audit] - [app.metrics :as mtx] - [app.msgbus :as mbus] [app.rpc :as-alias rpc] [app.rpc.climit :as-alias climit] + [app.rpc.commands.files :as cmd.files] + [app.rpc.commands.files.create :as cmd.files.create] + [app.rpc.commands.files.temp :as cmd.files.temp] + [app.rpc.commands.files.update :as cmd.files.update] [app.rpc.doc :as-alias doc] - [app.rpc.permissions :as perms] - [app.rpc.queries.files :as files] [app.rpc.queries.projects :as proj] - [app.storage.impl :as simpl] - [app.util.blob :as blob] - [app.util.objects-map :as omap] [app.util.services :as sv] [app.util.time :as dt] - [clojure.spec.alpha :as s] - [promesa.core :as p])) - -(declare create-file) -(declare retrieve-team-id) - -;; --- Helpers & Specs - -(s/def ::frame-id ::us/uuid) -(s/def ::file-id ::us/uuid) -(s/def ::id ::us/uuid) -(s/def ::name ::us/string) -(s/def ::profile-id ::us/uuid) -(s/def ::project-id ::us/uuid) -(s/def ::url ::us/url) + [clojure.spec.alpha :as s])) ;; --- Mutation: Create File -(s/def ::features ::us/set-of-strings) -(s/def ::is-shared ::us/boolean) -(s/def ::create-file - (s/keys :req-un [::profile-id ::name ::project-id] - :opt-un [::id ::is-shared ::features ::components-v2])) +(s/def ::create-file ::cmd.files.create/create-file) (sv/defmethod ::create-file - {::doc/added "1.0"} - [{:keys [pool] :as cfg} {:keys [profile-id project-id] :as params}] + {::doc/added "1.0" + ::doc/deprecated "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id project-id features components-v2] :as params}] (db/with-atomic [conn pool] - (let [team-id (retrieve-team-id conn project-id)] - (proj/check-edition-permissions! conn profile-id project-id) - (with-meta - (create-file conn params) - {::audit/props {:team-id team-id}})))) + (proj/check-edition-permissions! conn profile-id project-id) + (let [team-id (cmd.files/retrieve-team-id conn project-id) + features (cond-> (or features #{}) + ;; BACKWARD COMPATIBILITY with the components-v2 param + components-v2 (conj "components/v2")) + params (assoc params :features features)] + (-> (cmd.files.create/create-file conn params) + (vary-meta assoc ::audit/props {:team-id team-id}))))) -(defn create-file-role - [conn {:keys [file-id profile-id role]}] - (let [params {:file-id file-id - :profile-id profile-id}] - (->> (perms/assign-role-flags params role) - (db/insert! conn :file-profile-rel)))) - -(defn create-file - [conn {:keys [id name project-id is-shared data revn - modified-at deleted-at ignore-sync-until - components-v2 features] - :or {is-shared false revn 0} - :as params}] - (let [id (or id (:id data) (uuid/next)) - - ;; BACKWARD COMPATIBILITY with the components-v2 param - features (cond-> (or features #{}) - components-v2 (conj "components/v2")) - - data (or data - (binding [ffeat/*current* features] - (ctf/make-file-data id))) - - features (db/create-array conn "text" features) - file (db/insert! conn :file - (d/without-nils - {:id id - :project-id project-id - :name name - :revn revn - :is-shared is-shared - :data (blob/encode data) - :features features - :ignore-sync-until ignore-sync-until - :modified-at modified-at - :deleted-at deleted-at}))] - - (->> (assoc params :file-id id :role :owner) - (create-file-role conn)) - - (-> file files/decode-row))) ;; --- Mutation: Rename File -(declare rename-file) - -(s/def ::rename-file - (s/keys :req-un [::profile-id ::name ::id])) +(s/def ::rename-file ::cmd.files/rename-file) (sv/defmethod ::rename-file - {::doc/added "1.0"} + {::doc/added "1.0" + ::doc/deprecated "1.17"} [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}] (db/with-atomic [conn pool] - (files/check-edition-permissions! conn profile-id id) - (rename-file conn params))) + (cmd.files/check-edition-permissions! conn profile-id id) + (cmd.files/rename-file conn params))) -(defn- rename-file - [conn {:keys [id name] :as params}] - (-> (db/update! conn :file - {:name name - :modified-at (dt/now)} - {:id id}) - (select-keys [:id :name :created-at :modified-at]))) ;; --- Mutation: Set File shared -(declare set-file-shared) -(declare unlink-files) -(declare absorb-library) - -(s/def ::set-file-shared - (s/keys :req-un [::profile-id ::id ::is-shared])) +(s/def ::set-file-shared ::cmd.files/set-file-shared) (sv/defmethod ::set-file-shared - {::doc/added "1.2"} + {::doc/added "1.2" + ::doc/deprecated "1.17"} [{:keys [pool] :as cfg} {:keys [id profile-id is-shared] :as params}] (db/with-atomic [conn pool] - (files/check-edition-permissions! conn profile-id id) + (cmd.files/check-edition-permissions! conn profile-id id) (when-not is-shared - (absorb-library conn params) - (unlink-files conn params)) - (-> (set-file-shared conn params) - (update :features db/decode-pgarray #{})))) - -(defn- unlink-files - [conn {:keys [id] :as params}] - (db/delete! conn :file-library-rel {:library-file-id id})) - -(defn- set-file-shared - [conn {:keys [id is-shared] :as params}] - (db/update! conn :file - {:is-shared is-shared} - {:id id})) + (cmd.files/absorb-library conn params) + (cmd.files/unlink-files conn params)) + (cmd.files/set-file-shared conn params))) ;; --- Mutation: Delete File -(declare mark-file-deleted) - -(s/def ::delete-file - (s/keys :req-un [::id ::profile-id])) +(s/def ::delete-file ::cmd.files/delete-file) (sv/defmethod ::delete-file - {::doc/added "1.0"} + {::doc/added "1.0" + ::doc/deprecated "1.17"} [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}] (db/with-atomic [conn pool] - (files/check-edition-permissions! conn profile-id id) - (absorb-library conn params) - (mark-file-deleted conn params))) - -(defn mark-file-deleted - [conn {:keys [id] :as params}] - (db/update! conn :file - {:deleted-at (dt/now)} - {:id id}) - nil) - -(defn absorb-library - "Find all files using a shared library, and absorb all library assets - into the file local libraries" - [conn {:keys [id] :as params}] - (let [library (db/get-by-id conn :file id)] - (when (:is-shared library) - (let [ldata (-> library files/decode-row pmg/migrate-file :data)] - (->> (db/query conn :file-library-rel {:library-file-id id}) - (keep (fn [{:keys [file-id]}] - (some->> (db/get-by-id conn :file file-id {:check-not-found false}) - (files/decode-row) - (pmg/migrate-file)))) - (run! (fn [{:keys [id data revn] :as file}] - (let [data (ctf/absorb-assets data ldata)] - (db/update! conn :file - {:revn (inc revn) - :data (blob/encode data) - :modified-at (dt/now)} - {:id id}))))))))) + (cmd.files/check-edition-permissions! conn profile-id id) + (cmd.files/absorb-library conn params) + (cmd.files/mark-file-deleted conn params))) ;; --- Mutation: Link file to library -(declare link-file-to-library) - -(s/def ::link-file-to-library - (s/keys :req-un [::profile-id ::file-id ::library-id])) +(s/def ::link-file-to-library ::cmd.files/link-file-to-library) (sv/defmethod ::link-file-to-library + {::doc/added "1.3" + ::doc/deprecated "1.17"} [{:keys [pool] :as cfg} {:keys [profile-id file-id library-id] :as params}] (when (= file-id library-id) (ex/raise :type :validation :code :invalid-library :hint "A file cannot be linked to itself")) (db/with-atomic [conn pool] - (files/check-edition-permissions! conn profile-id file-id) - (files/check-edition-permissions! conn profile-id library-id) - (link-file-to-library conn params))) - -(def sql:link-file-to-library - "insert into file_library_rel (file_id, library_file_id) - values (?, ?) - on conflict do nothing;") - -(defn- link-file-to-library - [conn {:keys [file-id library-id] :as params}] - (db/exec-one! conn [sql:link-file-to-library file-id library-id])) - + (cmd.files/check-edition-permissions! conn profile-id file-id) + (cmd.files/check-edition-permissions! conn profile-id library-id) + (cmd.files/link-file-to-library conn params))) ;; --- Mutation: Unlink file from library -(declare unlink-file-from-library) - -(s/def ::unlink-file-from-library - (s/keys :req-un [::profile-id ::file-id ::library-id])) +(s/def ::unlink-file-from-library ::cmd.files/unlink-file-from-library) (sv/defmethod ::unlink-file-from-library + {::doc/added "1.3" + ::doc/deprecated "1.17"} [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}] (db/with-atomic [conn pool] - (files/check-edition-permissions! conn profile-id file-id) - (unlink-file-from-library conn params))) - -(defn- unlink-file-from-library - [conn {:keys [file-id library-id] :as params}] - (db/delete! conn :file-library-rel - {:file-id file-id - :library-file-id library-id})) + (cmd.files/check-edition-permissions! conn profile-id file-id) + (cmd.files/unlink-file-from-library conn params))) ;; --- Mutation: Update synchronization status of a link -(declare update-sync) - -(s/def ::update-sync - (s/keys :req-un [::profile-id ::file-id ::library-id])) +(s/def ::update-sync ::cmd.files/update-file-library-sync-status) (sv/defmethod ::update-sync + {::doc/added "1.10" + ::doc/deprecated "1.17"} [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}] (db/with-atomic [conn pool] - (files/check-edition-permissions! conn profile-id file-id) - (update-sync conn params))) + (cmd.files/check-edition-permissions! conn profile-id file-id) + (cmd.files/update-sync conn params))) -(defn- update-sync - [conn {:keys [file-id library-id] :as params}] - (db/update! conn :file-library-rel - {:synced-at (dt/now)} - {:file-id file-id - :library-file-id library-id})) ;; --- Mutation: Ignore updates in linked files (declare ignore-sync) -(s/def ::ignore-sync - (s/keys :req-un [::profile-id ::file-id ::date])) +(s/def ::ignore-sync ::cmd.files/ignore-file-library-sync-status) (sv/defmethod ::ignore-sync + {::doc/added "1.10" + ::doc/deprecated "1.17"} [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}] (db/with-atomic [conn pool] - (files/check-edition-permissions! conn profile-id file-id) - (ignore-sync conn params))) - -(defn- ignore-sync - [conn {:keys [file-id date] :as params}] - (db/update! conn :file - {:ignore-sync-until date} - {:id file-id})) + (cmd.files/check-edition-permissions! conn profile-id file-id) + (cmd.files/ignore-sync conn params))) ;; --- MUTATION: update-file -;; A generic, Changes based (granular) file update method. - -;; File changes that affect to the library, and must be notified -;; to all clients using it. -(defn library-change? - [change] - (or (#{:add-color :mod-color :del-color - :add-media :mod-media :del-media - :add-component :mod-component :del-component - :add-typography :mod-typography :del-typography} (:type change)) - (and (#{:add-obj :mod-obj :del-obj - :reg-objects :mov-objects} (:type change)) - (some? (:component-id change))))) - -(declare insert-change) -(declare retrieve-lagged-changes) -(declare send-notifications) -(declare update-file) - -(s/def ::changes - (s/coll-of map? :kind vector?)) - -(s/def ::hint-origin ::us/keyword) -(s/def ::hint-events - (s/every ::us/keyword :kind vector?)) - -(s/def ::change-with-metadata - (s/keys :req-un [::changes] - :opt-un [::hint-origin - ::hint-events])) - -(s/def ::changes-with-metadata - (s/every ::change-with-metadata :kind vector?)) - -(s/def ::session-id ::us/uuid) -(s/def ::revn ::us/integer) (s/def ::components-v2 ::us/boolean) (s/def ::update-file - (s/and - (s/keys :req-un [::id ::session-id ::profile-id ::revn] - :opt-un [::changes ::changes-with-metadata ::components-v2 ::features]) - (fn [o] - (or (contains? o :changes) - (contains? o :changes-with-metadata))))) - -(def ^:private sql:retrieve-file - "SELECT f.*, p.team_id - FROM file AS f - JOIN project AS p ON (p.id = f.project_id) - WHERE f.id = ? - AND (f.deleted_at IS NULL OR - f.deleted_at > now()) - FOR KEY SHARE") + (s/and ::cmd.files.update/update-file + (s/keys :opt-un [::components-v2]))) (sv/defmethod ::update-file {::climit/queue :update-file - ::climit/key-fn :id} - [{:keys [pool] :as cfg} {:keys [id profile-id components-v2] :as params}] + ::climit/key-fn :id + ::doc/added "1.0" + ::doc/deprecated "1.17"} + [{:keys [pool] :as cfg} {:keys [id profile-id features components-v2] :as params}] (db/with-atomic [conn pool] (db/xact-lock! conn id) - (let [file (db/exec-one! conn [sql:retrieve-file id]) - features' (:features params #{}) - features (db/decode-pgarray (:features file) features') + (cmd.files/check-edition-permissions! conn profile-id id) - ;; BACKWARD COMPATIBILITY with the components-v2 parameter - features (cond-> features + (let [;; BACKWARD COMPATIBILITY with the components-v2 parameter + features (cond-> (or features #{}) components-v2 (conj "components/v2")) + tpoint (dt/tpoint) + params (assoc params :features features) + cfg (assoc cfg :conn conn)] - file (assoc file :features features) - tpoint (dt/tpoint)] - - (when-not file - (ex/raise :type :not-found - :code :object-not-found - :hint (format "file with id '%s' does not exists" id))) - - ;; If features are specified from params and the final feature - ;; set is different than the persisted one, update it on the - ;; database. - (when (not= features features') - (let [features (db/create-array conn "text" features)] - (db/update! conn :file - {:features features} - {:id id}))) - - (binding [ffeat/*current* features - ffeat/*wrap-objects-fn* (if (features "storate/objects-map") - omap/wrap - identity)] - (files/check-edition-permissions! conn profile-id (:id file)) - (with-meta - (update-file (assoc cfg :conn conn) - (assoc params :file file)) - {::audit/props - {:project-id (:project-id file) - :team-id (:team-id file)} - - ::rpc/before-complete - (fn [] - (let [elapsed (tpoint)] - (l/trace :hint "update-file" :time (dt/format-duration elapsed))))}))))) - - -(defn- take-snapshot? - "Defines the rule when file `data` snapshot should be saved." - [{:keys [revn modified-at] :as file}] - (let [freq (or (cf/get :file-change-snapshot-every) 20) - timeout (or (cf/get :file-change-snapshot-timeout) - (dt/duration {:hours 1}))] - (or (= 1 freq) - (zero? (mod revn freq)) - (> (inst-ms (dt/diff modified-at (dt/now))) - (inst-ms timeout))))) - -(defn- delete-from-storage - [{:keys [storage] :as cfg} file] - (p/do - (when-let [backend (simpl/resolve-backend storage (:data-backend file))] - (simpl/del-object backend file)))) - -(defn- update-file - [{:keys [conn metrics] :as cfg} - {:keys [file changes changes-with-metadata session-id profile-id] :as params}] - (when (> (:revn params) - (:revn file)) - - (ex/raise :type :validation - :code :revn-conflict - :hint "The incoming revision number is greater that stored version." - :context {:incoming-revn (:revn params) - :stored-revn (:revn file)})) - - (let [changes (if changes-with-metadata - (mapcat :changes changes-with-metadata) - changes) - - changes (vec changes) - - ;; Trace the number of changes processed - _ (mtx/run! metrics {:id :update-file-changes :inc (count changes)}) - - ts (dt/now) - file (-> file - (update :revn inc) - (update :data (fn [data] - ;; Trace the length of bytes of processed data - (mtx/run! metrics {:id :update-file-bytes-processed :inc (alength data)}) - (cond-> data - :always - (-> (blob/decode) - (assoc :id (:id file)) - (pmg/migrate-data)) - - - (contains? ffeat/*current* "components/v2") - (ctf/migrate-to-components-v2) - - :always - (-> (cp/process-changes changes) - (blob/encode))))))] - ;; Insert change to the xlog - (db/insert! conn :file-change - {:id (uuid/next) - :session-id session-id - :profile-id profile-id - :created-at ts - :file-id (:id file) - :revn (:revn file) - :data (when (take-snapshot? file) - (:data file)) - :changes (blob/encode changes)}) - - ;; Update file - (db/update! conn :file - {:revn (:revn file) - :data (:data file) - :data-backend nil - :modified-at ts - :has-media-trimmed false} - {:id (:id file)}) - - ;; We need to delete the data from external storage backend - (when-not (nil? (:data-backend file)) - @(delete-from-storage cfg file)) - - (db/update! conn :project - {:modified-at ts} - {:id (:project-id file)}) - - (let [params (assoc params :file file :changes changes)] - ;; Send asynchronous notifications - (send-notifications cfg params) - - ;; Retrieve and return lagged data - (retrieve-lagged-changes conn params)))) - -(def ^:private - sql:lagged-changes - "select s.id, s.revn, s.file_id, - s.session_id, s.changes - from file_change as s - where s.file_id = ? - and s.revn > ? - order by s.created_at asc") - -(defn- retrieve-lagged-changes - [conn params] - (->> (db/exec! conn [sql:lagged-changes (:id params) (:revn params)]) - (into [] (comp (map files/decode-row) - (map (fn [row] - (cond-> row - (= (:revn row) (:revn (:file params))) - (assoc :changes [])))))))) - -(defn- send-notifications - [{:keys [conn] :as cfg} {:keys [file changes session-id] :as params}] - (let [lchanges (filter library-change? changes) - msgbus (:msgbus cfg)] - - - ;; Asynchronously publish message to the msgbus - (mbus/pub! msgbus - :topic (:id file) - :message {:type :file-change - :profile-id (:profile-id params) - :file-id (:id file) - :session-id (:session-id params) - :revn (:revn file) - :changes changes}) - - (when (and (:is-shared file) (seq lchanges)) - (let [team-id (or (:team-id file) - (retrieve-team-id conn (:project-id file)))] - ;; Asynchronously publish message to the msgbus - (mbus/pub! msgbus - :topic team-id - :message {:type :library-change - :profile-id (:profile-id params) - :file-id (:id file) - :session-id session-id - :revn (:revn file) - :modified-at (dt/now) - :changes lchanges}))))) - -(defn- retrieve-team-id - [conn project-id] - (:team-id (db/get-by-id conn :project project-id {:columns [:team-id]}))) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; TEMPORARY FILES (behaves differently) -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -(s/def ::create-temp-file ::create-file) - -(sv/defmethod ::create-temp-file - [{:keys [pool] :as cfg} {:keys [profile-id project-id] :as params}] - (db/with-atomic [conn pool] - (proj/check-edition-permissions! conn profile-id project-id) - (create-file conn (assoc params :deleted-at (dt/in-future {:days 1}))))) - -(s/def ::update-temp-file - (s/keys :req-un [::changes ::revn ::session-id ::id])) - -(sv/defmethod ::update-temp-file - {::doc/added "1.7"} - [{:keys [pool] :as cfg} {:keys [profile-id session-id id revn changes] :as params}] - (db/with-atomic [conn pool] - (db/insert! conn :file-change - {:id (uuid/next) - :session-id session-id - :profile-id profile-id - :created-at (dt/now) - :file-id id - :revn revn - :data nil - :changes (blob/encode changes)}) - nil)) - -(s/def ::persist-temp-file - (s/keys :req-un [::id ::profile-id])) - -(sv/defmethod ::persist-temp-file - {::doc/added "1.7"} - [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}] - (db/with-atomic [conn pool] - (files/check-edition-permissions! conn profile-id id) - (let [file (db/get-by-id conn :file id) - revs (db/query conn :file-change - {:file-id id} - {:order-by [[:revn :asc]]}) - revn (count revs)] - - (when (nil? (:deleted-at file)) - (ex/raise :type :validation - :code :cant-persist-already-persisted-file)) - - (loop [revs (seq revs) - data (blob/decode (:data file))] - (if-let [rev (first revs)] - (recur (rest revs) - (->> rev :changes blob/decode (cp/process-changes data))) - (db/update! conn :file - {:deleted-at nil - :revn revn - :data (blob/encode data)} - {:id id}))) - nil))) + (-> (cmd.files.update/update-file cfg params) + (vary-meta assoc ::rpc/before-complete + (fn [] + (let [elapsed (tpoint)] + (l/trace :hint "update-file" :time (dt/format-duration elapsed))))))))) ;; --- Mutation: upsert object thumbnail -(def sql:upsert-object-thumbnail - "insert into file_object_thumbnail(file_id, object_id, data) - values (?, ?, ?) - on conflict(file_id, object_id) do - update set data = ?;") - -(s/def ::data (s/nilable ::us/string)) -(s/def ::object-id ::us/string) -(s/def ::upsert-file-object-thumbnail - (s/keys :req-un [::profile-id ::file-id ::object-id ::data])) +(s/def ::upsert-file-object-thumbnail ::cmd.files/upsert-file-object-thumbnail) (sv/defmethod ::upsert-file-object-thumbnail - [{:keys [pool] :as cfg} {:keys [profile-id file-id object-id data]}] + {::doc/added "1.13" + ::doc/deprecated "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}] (db/with-atomic [conn pool] - (files/check-edition-permissions! conn profile-id file-id) - (if data - (db/exec-one! conn [sql:upsert-object-thumbnail file-id object-id data data]) - (db/delete! conn :file-object-thumbnail {:file-id file-id :object-id object-id})) + (cmd.files/check-edition-permissions! conn profile-id file-id) + (cmd.files/upsert-file-object-thumbnail! conn params) nil)) + ;; --- Mutation: upsert file thumbnail -(def sql:upsert-file-thumbnail - "insert into file_thumbnail (file_id, revn, data, props) - values (?, ?, ?, ?::jsonb) - on conflict(file_id, revn) do - update set data = ?, props=?, updated_at=now();") - -(s/def ::revn ::us/integer) -(s/def ::props map?) -(s/def ::upsert-file-thumbnail - (s/keys :req-un [::profile-id ::file-id ::revn ::data ::props])) +(s/def ::upsert-file-thumbnail ::cmd.files/upsert-file-thumbnail) (sv/defmethod ::upsert-file-thumbnail "Creates or updates the file thumbnail. Mainly used for paint the grid thumbnails." - [{:keys [pool] :as cfg} {:keys [profile-id file-id revn data props]}] + {::doc/added "1.13" + ::doc/deprecated "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}] (db/with-atomic [conn pool] - (files/check-edition-permissions! conn profile-id file-id) - (let [props (db/tjson (or props {}))] - (db/exec-one! conn [sql:upsert-file-thumbnail - file-id revn data props data props]) - nil))) + (cmd.files/check-edition-permissions! conn profile-id file-id) + (cmd.files/upsert-file-thumbnail conn params) + nil)) + + +;; --- MUTATION COMMAND: create-temp-file + +(s/def ::create-temp-file ::cmd.files.temp/create-temp-file) + +(sv/defmethod ::create-temp-file + {::doc/added "1.7" + ::doc/deprecated "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id project-id] :as params}] + (db/with-atomic [conn pool] + (proj/check-edition-permissions! conn profile-id project-id) + (cmd.files.create/create-file conn (assoc params :deleted-at (dt/in-future {:days 1}))))) + +;; --- MUTATION COMMAND: update-temp-file + +(s/def ::update-temp-file ::cmd.files.temp/update-temp-file) + +(sv/defmethod ::update-temp-file + {::doc/added "1.7" + ::doc/deprecated "1.17"} + [{:keys [pool] :as cfg} params] + (db/with-atomic [conn pool] + (cmd.files.temp/update-temp-file conn params) + nil)) + +;; --- MUTATION COMMAND: persist-temp-file + +(s/def ::persist-temp-file ::cmd.files.temp/persist-temp-file) + +(sv/defmethod ::persist-temp-file + {::doc/added "1.7" + ::doc/deprecated "1.17"} + [{:keys [pool] :as cfg} {:keys [id profile-id] :as params}] + (db/with-atomic [conn pool] + (cmd.files/check-edition-permissions! conn profile-id id) + (cmd.files.temp/persist-temp-file conn params))) diff --git a/backend/src/app/rpc/mutations/share_link.clj b/backend/src/app/rpc/mutations/share_link.clj index ff4148a18..9e8ab45d6 100644 --- a/backend/src/app/rpc/mutations/share_link.clj +++ b/backend/src/app/rpc/mutations/share_link.clj @@ -10,7 +10,7 @@ [app.common.spec :as us] [app.common.uuid :as uuid] [app.db :as db] - [app.rpc.queries.files :as files] + [app.rpc.commands.files :as files] [app.util.services :as sv] [clojure.spec.alpha :as s])) diff --git a/backend/src/app/rpc/queries/comments.clj b/backend/src/app/rpc/queries/comments.clj index c83f4576a..e9db1a6c8 100644 --- a/backend/src/app/rpc/queries/comments.clj +++ b/backend/src/app/rpc/queries/comments.clj @@ -8,8 +8,8 @@ (:require [app.db :as db] [app.rpc.commands.comments :as cmd.comments] + [app.rpc.commands.files :as cmd.files] [app.rpc.doc :as-alias doc] - [app.rpc.queries.files :as files] [app.rpc.queries.teams :as teams] [app.util.services :as sv] [clojure.spec.alpha :as s])) @@ -52,7 +52,7 @@ ::doc/deprecated "1.15"} [{:keys [pool] :as cfg} {:keys [profile-id file-id share-id] :as params}] (with-open [conn (db/open pool)] - (files/check-comment-permissions! conn profile-id file-id share-id) + (cmd.files/check-comment-permissions! conn profile-id file-id share-id) (cmd.comments/get-comment-thread conn params))) ;; --- QUERY: Comments @@ -65,7 +65,7 @@ [{:keys [pool] :as cfg} {:keys [profile-id thread-id share-id] :as params}] (with-open [conn (db/open pool)] (let [thread (db/get-by-id conn :comment-thread thread-id)] - (files/check-comment-permissions! conn profile-id (:file-id thread) share-id)) + (cmd.files/check-comment-permissions! conn profile-id (:file-id thread) share-id)) (cmd.comments/get-comments conn thread-id))) @@ -78,5 +78,5 @@ ::doc/added "1.13"} [{:keys [pool] :as cfg} {:keys [profile-id file-id share-id]}] (with-open [conn (db/open pool)] - (files/check-comment-permissions! conn profile-id file-id share-id) + (cmd.files/check-comment-permissions! conn profile-id file-id share-id) (cmd.comments/get-file-comments-users conn file-id profile-id))) diff --git a/backend/src/app/rpc/queries/files.clj b/backend/src/app/rpc/queries/files.clj index 349c54bc2..8dedbc916 100644 --- a/backend/src/app/rpc/queries/files.clj +++ b/backend/src/app/rpc/queries/files.clj @@ -6,289 +6,56 @@ (ns app.rpc.queries.files (:require - [app.common.data :as d] - [app.common.data.macros :as dm] - [app.common.exceptions :as ex] - [app.common.geom.shapes :as gsh] - [app.common.pages.helpers :as cph] - [app.common.pages.migrations :as pmg] [app.common.spec :as us] - [app.common.types.file :as ctf] - [app.common.types.shape-tree :as ctt] [app.db :as db] - [app.db.sql :as sql] [app.rpc :as-alias rpc] + [app.rpc.commands.files :as cmd.files] + [app.rpc.doc :as-alias doc] [app.rpc.helpers :as rpch] - [app.rpc.permissions :as perms] [app.rpc.queries.projects :as projects] - [app.rpc.queries.share-link :refer [retrieve-share-link]] [app.rpc.queries.teams :as teams] - [app.util.blob :as blob] [app.util.services :as sv] - [clojure.spec.alpha :as s] - [cuerdas.core :as str])) - -(declare decode-row) - -;; --- Helpers & Specs - -(s/def ::frame-id ::us/uuid) -(s/def ::id ::us/uuid) -(s/def ::name ::us/string) -(s/def ::project-id ::us/uuid) -(s/def ::file-id ::us/uuid) -(s/def ::profile-id ::us/uuid) -(s/def ::team-id ::us/uuid) -(s/def ::search-term ::us/string) -(s/def ::components-v2 ::us/boolean) - -;; --- Query: File Permissions - -(def ^:private sql:file-permissions - "select fpr.is_owner, - fpr.is_admin, - fpr.can_edit - from file_profile_rel as fpr - where fpr.file_id = ? - and fpr.profile_id = ? - union all - select tpr.is_owner, - tpr.is_admin, - tpr.can_edit - from team_profile_rel as tpr - inner join project as p on (p.team_id = tpr.team_id) - inner join file as f on (p.id = f.project_id) - where f.id = ? - and tpr.profile_id = ? - union all - select ppr.is_owner, - ppr.is_admin, - ppr.can_edit - from project_profile_rel as ppr - inner join file as f on (f.project_id = ppr.project_id) - where f.id = ? - and ppr.profile_id = ?") - -(defn retrieve-file-permissions - [conn profile-id file-id] - (when (and profile-id file-id) - (db/exec! conn [sql:file-permissions - file-id profile-id - file-id profile-id - file-id profile-id]))) - -(defn get-permissions - ([conn profile-id file-id] - (let [rows (retrieve-file-permissions conn profile-id file-id) - is-owner (boolean (some :is-owner rows)) - is-admin (boolean (some :is-admin rows)) - can-edit (boolean (some :can-edit rows))] - (when (seq rows) - {:type :membership - :is-owner is-owner - :is-admin (or is-owner is-admin) - :can-edit (or is-owner is-admin can-edit) - :can-read true - :is-logged (some? profile-id)}))) - ([conn profile-id file-id share-id] - (let [perms (get-permissions conn profile-id file-id) - ldata (retrieve-share-link conn file-id share-id)] - - ;; NOTE: in a future when share-link becomes more powerful and - ;; will allow us specify which parts of the app is available, we - ;; will probably need to tweak this function in order to expose - ;; this flags to the frontend. - (cond - (some? perms) perms - (some? ldata) {:type :share-link - :can-read true - :is-logged (some? profile-id) - :who-comment (:who-comment ldata) - :who-inspect (:who-inspect ldata)})))) - -(def has-edit-permissions? - (perms/make-edition-predicate-fn get-permissions)) - -(def has-read-permissions? - (perms/make-read-predicate-fn get-permissions)) - -(def has-comment-permissions? - (perms/make-comment-predicate-fn get-permissions)) - -(def check-edition-permissions! - (perms/make-check-fn has-edit-permissions?)) - -(def check-read-permissions! - (perms/make-check-fn has-read-permissions?)) - -;; A user has comment permissions if she has read permissions, or comment permissions -(defn check-comment-permissions! - [conn profile-id file-id share-id] - (let [can-read (has-read-permissions? conn profile-id file-id) - can-comment (has-comment-permissions? conn profile-id file-id share-id)] - (when-not (or can-read can-comment) - (ex/raise :type :not-found - :code :object-not-found - :hint "not found")))) - -;; --- Query: Files search - -;; TODO: this query need to a good refactor - -(def ^:private sql:search-files - "with projects as ( - select p.* - from project as p - inner join team_profile_rel as tpr on (tpr.team_id = p.team_id) - where tpr.profile_id = ? - and p.team_id = ? - and p.deleted_at is null - and (tpr.is_admin = true or - tpr.is_owner = true or - tpr.can_edit = true) - union - select p.* - from project as p - inner join project_profile_rel as ppr on (ppr.project_id = p.id) - where ppr.profile_id = ? - and p.team_id = ? - and p.deleted_at is null - and (ppr.is_admin = true or - ppr.is_owner = true or - ppr.can_edit = true) - ) - select distinct - f.id, - f.project_id, - f.created_at, - f.modified_at, - f.name, - f.is_shared - from file as f - inner join projects as pr on (f.project_id = pr.id) - where f.name ilike ('%' || ? || '%') - and f.deleted_at is null - order by f.created_at asc") - -(s/def ::search-files - (s/keys :req-un [::profile-id ::team-id] - :opt-un [::search-term])) - -(sv/defmethod ::search-files - [{:keys [pool] :as cfg} {:keys [profile-id team-id search-term] :as params}] - (when search-term - (db/exec! pool [sql:search-files - profile-id team-id - profile-id team-id - search-term]))) + [clojure.spec.alpha :as s])) ;; --- Query: Project Files -(def ^:private sql:project-files - "select f.id, - f.project_id, - f.created_at, - f.modified_at, - f.name, - f.revn, - f.is_shared - from file as f - where f.project_id = ? - and f.deleted_at is null - order by f.modified_at desc") - -(s/def ::project-files - (s/keys :req-un [::profile-id ::project-id])) +(s/def ::project-files ::cmd.files/get-project-files) (sv/defmethod ::project-files + {::doc/added "1.1" + ::doc/deprecated "1.17"} [{:keys [pool] :as cfg} {:keys [profile-id project-id] :as params}] (with-open [conn (db/open pool)] (projects/check-read-permissions! conn profile-id project-id) - (db/exec! conn [sql:project-files project-id]))) + (cmd.files/get-project-files conn project-id))) ;; --- Query: File (By ID) -(defn retrieve-object-thumbnails - ([{:keys [pool]} file-id] - (let [sql (str/concat - "select object_id, data " - " from file_object_thumbnail" - " where file_id=?")] - (->> (db/exec! pool [sql file-id]) - (d/index-by :object-id :data)))) - - ([{:keys [pool]} file-id object-ids] - (with-open [conn (db/open pool)] - (let [sql (str/concat - "select object_id, data " - " from file_object_thumbnail" - " where file_id=? and object_id = ANY(?)") - ids (db/create-array conn "text" (seq object-ids))] - (->> (db/exec! conn [sql file-id ids]) - (d/index-by :object-id :data)))))) - -(defn retrieve-file - [{:keys [pool] :as cfg} id features] - (let [file (->> (db/get-by-id pool :file id) - (decode-row) - (pmg/migrate-file))] - - (if (contains? features "components/v2") - (update file :data ctf/migrate-to-components-v2) - (if (dm/get-in file [:data :options :components-v2]) - (ex/raise :type :restriction - :code :feature-disabled - :hint "tried to open a components/v2 file with feature disabled") - file)))) - -(s/def ::features ::us/set-of-strings) +(s/def ::components-v2 ::us/boolean) (s/def ::file - (s/keys :req-un [::profile-id ::id] - :opt-un [::features ::components-v2])) + (s/and ::cmd.files/get-file + (s/keys :opt-un [::components-v2]))) (sv/defmethod ::file "Retrieve a file by its ID. Only authenticated users." + {::doc/added "1.0" + ::doc/deprecated "1.17"} [{:keys [pool] :as cfg} {:keys [profile-id id features components-v2] :as params}] - (let [perms (get-permissions pool profile-id id) + (with-open [conn (db/open pool)] + (let [perms (cmd.files/get-permissions pool profile-id id) + ;; BACKWARD COMPATIBILTY with the components-v2 parameter + features (cond-> (or features #{}) + components-v2 (conj "components/v2"))] - ;; BACKWARD COMPATIBILTY with the components-v2 parameter - features (cond-> (or features #{}) - components-v2 (conj features "components/v2"))] - (check-read-permissions! perms) - (let [file (retrieve-file cfg id features) - thumbs (retrieve-object-thumbnails cfg id)] - (-> file - (assoc :thumbnails thumbs) + (cmd.files/check-read-permissions! perms) + (-> (cmd.files/get-file conn id features) (assoc :permissions perms))))) - ;; --- QUERY: page -(defn- prune-objects - "Given the page data and the object-id returns the page data with all - other not needed objects removed from the `:objects` data - structure." - [{:keys [objects] :as page} object-id] - (let [objects (cph/get-children-with-self objects object-id)] - (assoc page :objects (d/index-by :id objects)))) - -(defn- prune-thumbnails - "Given the page data, removes the `:thumbnail` prop from all - shapes." - [page] - (update page :objects d/update-vals #(dissoc % :thumbnail))) - -(s/def ::page-id ::us/uuid) -(s/def ::object-id ::us/uuid) - (s/def ::page - (s/and - (s/keys :req-un [::profile-id ::file-id] - :opt-un [::page-id ::object-id ::features ::components-v2]) - (fn [obj] - (if (contains? obj :object-id) - (contains? obj :page-id) - true)))) + (s/and ::cmd.files/get-page + (s/keys :opt-un [::components-v2]))) (sv/defmethod ::page "Retrieves the page data from file and returns it. If no page-id is @@ -300,288 +67,100 @@ mandatory. Mainly used for rendering purposes." - [{:keys [pool] :as cfg} {:keys [profile-id file-id page-id object-id features components-v2] :as props}] - (check-read-permissions! pool profile-id file-id) - (let [;; BACKWARD COMPATIBILTY with the components-v2 parameter - features (cond-> (or features #{}) - components-v2 (conj features "components/v2")) + {::doc/added "1.5" + ::doc/deprecated "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id file-id features components-v2] :as params}] + (with-open [conn (db/open pool)] + (cmd.files/check-read-permissions! conn profile-id file-id) + (let [;; BACKWARD COMPATIBILTY with the components-v2 parameter + features (cond-> (or features #{}) + components-v2 (conj "components/v2")) + params (assoc params :features features)] - file (retrieve-file cfg file-id features) - page-id (or page-id (-> file :data :pages first)) - page (dm/get-in file [:data :pages-index page-id])] - - (cond-> (prune-thumbnails page) - (uuid? object-id) - (prune-objects object-id)))) + (cmd.files/get-page conn params)))) ;; --- QUERY: file-data-for-thumbnail -(defn- get-file-thumbnail-data - [cfg {:keys [data id] :as file}] - (letfn [;; function responsible on finding the frame marked to be - ;; used as thumbnail; the returned frame always have - ;; the :page-id set to the page that it belongs. - (get-thumbnail-frame [data] - (d/seek :use-for-thumbnail? - (for [page (-> data :pages-index vals) - frame (-> page :objects ctt/get-frames)] - (assoc frame :page-id (:id page))))) - - ;; function responsible to filter objects data structure of - ;; all unneeded shapes if a concrete frame is provided. If no - ;; frame, the objects is returned untouched. - (filter-objects [objects frame-id] - (d/index-by :id (cph/get-children-with-self objects frame-id))) - - ;; function responsible of assoc available thumbnails - ;; to frames and remove all children shapes from objects if - ;; thumbnails is available - (assoc-thumbnails [objects page-id thumbnails] - (loop [objects objects - frames (filter cph/frame-shape? (vals objects))] - - (if-let [frame (-> frames first)] - (let [frame-id (:id frame) - object-id (str page-id frame-id) - frame (if-let [thumb (get thumbnails object-id)] - (assoc frame :thumbnail thumb :shapes []) - (dissoc frame :thumbnail)) - - children-ids - (cph/get-children-ids objects frame-id) - - bounds - (when (:show-content frame) - (gsh/selection-rect (concat [frame] (->> children-ids (map (d/getf objects)))))) - - frame - (cond-> frame - (some? bounds) - (assoc :children-bounds bounds))] - - (if (:thumbnail frame) - (recur (-> objects - (assoc frame-id frame) - (d/without-keys children-ids)) - (rest frames)) - (recur (assoc objects frame-id frame) - (rest frames)))) - - objects)))] - - (let [frame (get-thumbnail-frame data) - frame-id (:id frame) - page-id (or (:page-id frame) - (-> data :pages first)) - - page (dm/get-in data [:pages-index page-id]) - frame-ids (if (some? frame) (list frame-id) (map :id (ctt/get-frames (:objects page)))) - - obj-ids (map #(str page-id %) frame-ids) - thumbs (retrieve-object-thumbnails cfg id obj-ids)] - - (cond-> page - ;; If we have frame, we need to specify it on the page level - ;; and remove the all other unrelated objects. - (some? frame-id) - (-> (assoc :thumbnail-frame-id frame-id) - (update :objects filter-objects frame-id)) - - ;; Assoc the available thumbnails and prune not visible shapes - ;; for avoid transfer unnecessary data. - :always - (update :objects assoc-thumbnails page-id thumbs))))) - (s/def ::file-data-for-thumbnail - (s/keys :req-un [::profile-id ::file-id] - :opt-un [::components-v2 ::features])) + (s/and ::cmd.files/get-file-data-for-thumbnail + (s/keys :opt-un [::components-v2]))) (sv/defmethod ::file-data-for-thumbnail "Retrieves the data for generate the thumbnail of the file. Used mainly for render thumbnails on dashboard." + {::doc/added "1.11" + ::doc/deprecated "1.17"} [{:keys [pool] :as cfg} {:keys [profile-id file-id features components-v2] :as props}] - (check-read-permissions! pool profile-id file-id) - (let [;; BACKWARD COMPATIBILTY with the components-v2 parameter - features (cond-> (or features #{}) - components-v2 (conj features "components/v2")) - file (retrieve-file cfg file-id features)] - {:file-id file-id - :revn (:revn file) - :page (get-file-thumbnail-data cfg file)})) - + (with-open [conn (db/open pool)] + (cmd.files/check-read-permissions! conn profile-id file-id) + (let [;; BACKWARD COMPATIBILTY with the components-v2 parameter + features (cond-> (or features #{}) + components-v2 (conj "components/v2")) + file (cmd.files/retrieve-file conn file-id features)] + {:file-id file-id + :revn (:revn file) + :page (cmd.files/get-file-data-for-thumbnail conn file)}))) ;; --- Query: Shared Library Files -(def ^:private sql:team-shared-files - "select f.id, - f.revn, - f.data, - f.project_id, - f.created_at, - f.modified_at, - f.name, - f.is_shared - from file as f - inner join project as p on (p.id = f.project_id) - where f.is_shared = true - and f.deleted_at is null - and p.deleted_at is null - and p.team_id = ? - order by f.modified_at desc") - -(s/def ::team-shared-files - (s/keys :req-un [::profile-id ::team-id])) +(s/def ::team-shared-files ::cmd.files/get-team-shared-files) (sv/defmethod ::team-shared-files - [{:keys [pool] :as cfg} {:keys [team-id] :as params}] - (let [assets-sample - (fn [assets limit] - (let [sorted-assets (->> (vals assets) - (sort-by #(str/lower (:name %))))] - - {:count (count sorted-assets) - :sample (into [] (take limit sorted-assets))})) - - library-summary - (fn [data] - {:components (assets-sample (:components data) 4) - :colors (assets-sample (:colors data) 3) - :typographies (assets-sample (:typographies data) 3)}) - - xform (comp - (map decode-row) - (map #(assoc % :library-summary (library-summary (:data %)))) - (map #(dissoc % :data)))] - - (into #{} xform (db/exec! pool [sql:team-shared-files team-id])))) + {::doc/added "1.3" + ::doc/deprecated "1.17"} + [{:keys [pool] :as cfg} {:keys [profile-id team-id] :as params}] + (with-open [conn (db/open pool)] + (teams/check-read-permissions! conn profile-id team-id) + (cmd.files/get-team-shared-files conn params))) ;; --- Query: File Libraries used by a File -(def ^:private sql:file-libraries - "WITH RECURSIVE libs AS ( - SELECT fl.*, flr.synced_at - FROM file AS fl - JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id) - WHERE flr.file_id = ?::uuid - UNION - SELECT fl.*, flr.synced_at - FROM file AS fl - JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id) - JOIN libs AS l ON (flr.file_id = l.id) - ) - SELECT l.id, - l.data, - l.project_id, - l.created_at, - l.modified_at, - l.deleted_at, - l.name, - l.revn, - l.synced_at - FROM libs AS l - WHERE l.deleted_at IS NULL OR l.deleted_at > now();") - -(defn retrieve-file-libraries - [{:keys [pool] :as cfg} is-indirect file-id] - (let [xform (comp - (map #(assoc % :is-indirect is-indirect)) - (map decode-row))] - (into #{} xform (db/exec! pool [sql:file-libraries file-id])))) - -(s/def ::file-libraries - (s/keys :req-un [::profile-id ::file-id])) +(s/def ::file-libraries ::cmd.files/get-file-libraries) (sv/defmethod ::file-libraries + {::doc/added "1.3" + ::doc/deprecated "1.17"} [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}] - (check-read-permissions! pool profile-id file-id) - (retrieve-file-libraries cfg false file-id)) + (with-open [conn (db/open pool)] + (cmd.files/check-read-permissions! conn profile-id file-id) + (cmd.files/get-file-libraries conn false file-id))) ;; --- Query: Files that use this File library -(def ^:private sql:library-using-files - "SELECT f.id, - f.name - FROM file_library_rel AS flr - JOIN file AS f ON (f.id = flr.file_id) - WHERE flr.library_file_id = ? - AND (f.deleted_at IS NULL OR f.deleted_at > now())") - -(s/def ::library-using-files - (s/keys :req-un [::profile-id ::file-id])) +(s/def ::library-using-files ::cmd.files/get-library-file-references) (sv/defmethod ::library-using-files + {::doc/added "1.13" + ::doc/deprecated "1.17"} [{:keys [pool] :as cfg} {:keys [profile-id file-id] :as params}] - (check-read-permissions! pool profile-id file-id) - (db/exec! pool [sql:library-using-files file-id])) + (with-open [conn (db/open pool)] + (cmd.files/check-read-permissions! conn profile-id file-id) + (cmd.files/get-library-file-references conn file-id))) ;; --- QUERY: team-recent-files -(def sql:team-recent-files - "with recent_files as ( - select f.id, - f.revn, - f.project_id, - f.created_at, - f.modified_at, - f.name, - f.is_shared, - row_number() over w as row_num - from file as f - join project as p on (p.id = f.project_id) - where p.team_id = ? - and p.deleted_at is null - and f.deleted_at is null - window w as (partition by f.project_id order by f.modified_at desc) - order by f.modified_at desc - ) - select * from recent_files where row_num <= 10;") - - -(s/def ::team-recent-files - (s/keys :req-un [::profile-id ::team-id])) +(s/def ::team-recent-files ::cmd.files/get-team-recent-files) (sv/defmethod ::team-recent-files + {::doc/added "1.0" + ::doc/deprecated "1.17"} [{:keys [pool] :as cfg} {:keys [profile-id team-id]}] - (teams/check-read-permissions! pool profile-id team-id) - (db/exec! pool [sql:team-recent-files team-id])) + (with-open [conn (db/open pool)] + (teams/check-read-permissions! conn profile-id team-id) + (cmd.files/get-team-recent-files conn team-id))) + ;; --- QUERY: get file thumbnail -(s/def ::revn ::us/integer) - -(s/def ::file-thumbnail - (s/keys :req-un [::profile-id ::file-id] - :opt-un [::revn])) +(s/def ::file-thumbnail ::cmd.files/get-file-thumbnail) (sv/defmethod ::file-thumbnail + {::doc/added "1.13" + ::doc/deprecated "1.17"} [{:keys [pool]} {:keys [profile-id file-id revn]}] - (check-read-permissions! pool profile-id file-id) - (let [sql (sql/select :file-thumbnail - (cond-> {:file-id file-id} - revn (assoc :revn revn)) - {:limit 1 - :order-by [[:revn :desc]]}) - - row (db/exec-one! pool sql)] - - (when-not row - (ex/raise :type :not-found - :code :file-thumbnail-not-found)) - - (with-meta {:data (:data row) - :props (some-> (:props row) db/decode-transit-pgobject) - :revn (:revn row) - :file-id (:file-id row)} - {::rpc/transform-response (rpch/http-cache {:max-age (* 1000 60 60)})}))) - -;; --- Helpers - -(defn decode-row - [{:keys [data changes features] :as row}] - (when row - (cond-> row - features (assoc :features (db/decode-pgarray features)) - changes (assoc :changes (blob/decode changes)) - data (assoc :data (blob/decode data))))) + (with-open [conn (db/open pool)] + (cmd.files/check-read-permissions! conn profile-id file-id) + (-> (cmd.files/get-file-thumbnail conn file-id revn) + (with-meta {::rpc/transform-response (rpch/http-cache {:max-age (* 1000 60 60)})})))) diff --git a/backend/src/app/rpc/queries/fonts.clj b/backend/src/app/rpc/queries/fonts.clj index 70b9a7435..077766cd3 100644 --- a/backend/src/app/rpc/queries/fonts.clj +++ b/backend/src/app/rpc/queries/fonts.clj @@ -8,7 +8,7 @@ (:require [app.common.spec :as us] [app.db :as db] - [app.rpc.queries.files :as files] + [app.rpc.commands.files :as files] [app.rpc.queries.projects :as projects] [app.rpc.queries.teams :as teams] [app.util.services :as sv] diff --git a/backend/src/app/rpc/queries/viewer.clj b/backend/src/app/rpc/queries/viewer.clj index 2e64986f4..5c2531570 100644 --- a/backend/src/app/rpc/queries/viewer.clj +++ b/backend/src/app/rpc/queries/viewer.clj @@ -6,88 +6,26 @@ (ns app.rpc.queries.viewer (:require - [app.common.exceptions :as ex] [app.common.spec :as us] [app.db :as db] - [app.rpc.commands.comments :as comments] - [app.rpc.queries.files :as files] - [app.rpc.queries.share-link :as slnk] + [app.rpc.commands.viewer :as viewer] + [app.rpc.doc :as-alias doc] [app.util.services :as sv] - [clojure.spec.alpha :as s] - [promesa.core :as p])) + [clojure.spec.alpha :as s])) -;; --- Query: View Only Bundle - -(defn- retrieve-project - [pool id] - (db/get-by-id pool :project id {:columns [:id :name :team-id]})) - -(defn- retrieve-bundle - [{:keys [pool] :as cfg} file-id profile-id features] - (p/let [file (files/retrieve-file cfg file-id features) - project (retrieve-project pool (:project-id file)) - libs (files/retrieve-file-libraries cfg false file-id) - users (comments/get-file-comments-users pool file-id profile-id) - - links (->> (db/query pool :share-link {:file-id file-id}) - (mapv slnk/decode-share-link-row)) - - fonts (db/query pool :team-font-variant - {:team-id (:team-id project) - :deleted-at nil})] - {:file file - :users users - :fonts fonts - :project project - :share-links links - :libraries libs})) - -(s/def ::file-id ::us/uuid) -(s/def ::profile-id ::us/uuid) -(s/def ::share-id ::us/uuid) -(s/def ::features ::us/set-of-strings) - -;; TODO: deprecated, should be removed when version >= 1.18 (s/def ::components-v2 ::us/boolean) - (s/def ::view-only-bundle - (s/keys :req-un [::file-id] - :opt-un [::profile-id ::share-id ::features ::components-v2])) + (s/and ::viewer/get-view-only-bundle + (s/keys :opt-un [::components-v2]))) -(sv/defmethod ::view-only-bundle {:auth false} - [{:keys [pool] :as cfg} {:keys [profile-id file-id share-id features components-v2] :as params}] - (p/let [;; BACKWARD COMPATIBILTY with the components-v2 parameter +(sv/defmethod ::view-only-bundle + {:auth false + ::doc/added "1.3" + ::doc/deprecated "1.17"} + [{:keys [pool] :as cfg} {:keys [features components-v2] :as params}] + (with-open [conn (db/open pool)] + (let [;; BACKWARD COMPATIBILTY with the components-v2 parameter features (cond-> (or features #{}) - components-v2 (conj features "components/v2")) - - slink (slnk/retrieve-share-link pool file-id share-id) - perms (files/get-permissions pool profile-id file-id share-id) - thumbs (files/retrieve-object-thumbnails cfg file-id) - bundle (p/-> (retrieve-bundle cfg file-id profile-id features) - (assoc :permissions perms) - (assoc-in [:file :thumbnails] thumbs))] - ;; When we have neither profile nor share, we just return a not - ;; found response to the user. - (do - (when (and (not profile-id) - (not slink)) - (ex/raise :type :not-found - :code :object-not-found)) - - ;; When we have only profile, we need to check read permissions - ;; on file. - (when (and profile-id (not slink)) - (files/check-read-permissions! pool profile-id file-id)) - - (cond-> bundle - (some? slink) - (assoc :share slink) - - (and (some? slink) - (not (contains? (:flags slink) "view-all-pages"))) - (update-in [:file :data] (fn [data] - (let [allowed-pages (:pages slink)] - (-> data - (update :pages (fn [pages] (filterv #(contains? allowed-pages %) pages))) - (update :pages-index (fn [index] (select-keys index allowed-pages))))))))))) - + components-v2 (conj "components/v2")) + params (assoc params :features features)] + (viewer/get-view-only-bundle conn params)))) diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index 01730dbb6..707242f23 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -135,8 +135,7 @@ (update :features conj "storage/pointer-map")))) (migrate-to-omap [data file-id] - (binding [pmap/*tracked* (atom {}) - pmap/*metadata* {:file-id file-id}] + (binding [pmap/*tracked* (atom {})] (let [data (-> data (update :pages-index update-vals pmap/wrap) (update :components pmap/wrap))] @@ -144,7 +143,6 @@ (db/insert! h/*conn* :file-data-fragment {:id id :file-id file-id - :metadata (-> item meta db/tjson) :content (-> item deref blob/encode)})) data)))] diff --git a/backend/src/app/tasks/file_gc.clj b/backend/src/app/tasks/file_gc.clj index 530890317..fa1b68c18 100644 --- a/backend/src/app/tasks/file_gc.clj +++ b/backend/src/app/tasks/file_gc.clj @@ -17,11 +17,12 @@ [app.common.types.shape-tree :as ctt] [app.config :as cf] [app.db :as db] + [app.rpc.commands.files :as files] [app.util.blob :as blob] + [app.util.pointer-map :as pmap] [app.util.time :as dt] [clojure.set :as set] [clojure.spec.alpha :as s] - [cuerdas.core :as str] [integrant.core :as ig])) (declare ^:private retrieve-candidates) @@ -55,7 +56,7 @@ (recur (inc total) (rest files))) (do - (l/info :hint "task finished" :min-age (dt/format-duration min-age) :total total) + (l/info :hint "task finished" :min-age (dt/format-duration min-age) :processed total) ;; Allow optional rollback passed by params (when (:rollback? params) @@ -72,6 +73,7 @@ "select f.id, f.data, f.revn, + f.features, f.modified_at from file as f where f.has_media_trimmed is false @@ -86,17 +88,22 @@ (if id (do (l/warn :hint "explicit file id passed on params" :id id) - (db/query conn :file {:id id})) + (->> (db/query conn :file {:id id}) + (map #(update % :features db/decode-pgarray #{})))) (let [interval (db/interval min-age) get-chunk (fn [cursor] (let [rows (db/exec! conn [sql:retrieve-candidates-chunk interval cursor])] - [(some->> rows peek :modified-at) (seq rows)]))] + [(some->> rows peek :modified-at) + (map #(update % :features db/decode-pgarray #{}) rows)]))] + (d/iteration get-chunk :vf second :kf first :initk (dt/now))))) (defn collect-used-media + "Analyzes the file data and collects all references to external + assets. Returns a set of ids." [data] (let [xform (comp (map :objects) @@ -152,9 +159,8 @@ unused (set/difference stored using)] (when (seq unused) - (let [sql (str/concat - "delete from file_object_thumbnail " - " where file_id=? and object_id=ANY(?)") + (let [sql (str "delete from file_object_thumbnail " + " where file_id=? and object_id=ANY(?)") res (db/exec-one! conn [sql file-id (db/create-array conn "text" unused)])] (l/debug :hint "delete file object thumbnails" :file-id file-id :total (:next.jdbc/update-count res)))))) @@ -233,21 +239,41 @@ {:data new-data} {:id library-id}))))) +(def ^:private sql:get-unused-fragments + "SELECT id FROM file_data_fragment + WHERE file_id = ? AND id != ALL(?::uuid[])") + +(defn- clean-data-fragments! + [conn file-id data] + (let [used (->> (concat (vals data) + (vals (:pages-index data))) + (into #{} (comp (filter pmap/pointer-map?) + (map pmap/get-id))) + (db/create-array conn "uuid")) + rows (db/exec! conn [sql:get-unused-fragments file-id used])] + (doseq [fragment-id (map :id rows)] + (l/trace :hint "remove unused file data fragment" :id (str fragment-id)) + (db/delete! conn :file-data-fragment {:id fragment-id :file-id file-id})))) + (defn- process-file - [{:keys [conn] :as cfg} {:keys [id data revn modified-at] :as file}] + [{:keys [conn] :as cfg} {:keys [id data revn modified-at features] :as file}] (l/debug :hint "processing file" :id id :modified-at modified-at) - (let [data (-> (blob/decode data) - (assoc :id id) - (pmg/migrate-data))] + (binding [pmap/*load-fn* (partial files/load-pointer conn id)] + (let [data (-> (blob/decode data) + (assoc :id id) + (pmg/migrate-data))] - (clean-file-media! conn id data) - (clean-file-frame-thumbnails! conn id data) - (clean-file-thumbnails! conn id revn) - (clean-deleted-components! conn id data) + (clean-file-media! conn id data) + (clean-file-frame-thumbnails! conn id data) + (clean-file-thumbnails! conn id revn) + (clean-deleted-components! conn id data) - ;; Mark file as trimmed - (db/update! conn :file + (when (contains? features "storage/pointer-map") + (clean-data-fragments! conn id data)) + + ;; Mark file as trimmed + (db/update! conn :file {:has-media-trimmed true} {:id id}) - nil)) + nil))) diff --git a/backend/src/app/util/fressian.clj b/backend/src/app/util/fressian.clj index 7d54e3207..335f74b75 100644 --- a/backend/src/app/util/fressian.clj +++ b/backend/src/app/util/fressian.clj @@ -86,17 +86,25 @@ (write-tag! w tag 1) (write-list! w o)) +(defn begin-closed-list! + [^StreamingWriter w] + (.beginClosedList w)) + +(defn end-list! + [^StreamingWriter w] + (.endList w)) + (defn write-map-like "Writes a map as Fressian with the tag 'map' and all keys cached." [tag ^Writer w m] (write-tag! w tag 1) - (.beginClosedList ^StreamingWriter w) + (begin-closed-list! w) (loop [items (seq m)] (when-let [^clojure.lang.MapEntry item (first items)] (write-object! w (.key item) true) (write-object! w (.val item)) (recur (rest items)))) - (.endList ^StreamingWriter w)) + (end-list! w)) (defn read-map-like [^Reader rdr] diff --git a/backend/test/app/services_files_test.clj b/backend/test/app/services_files_test.clj index 004e29b2f..913ddba5e 100644 --- a/backend/test/app/services_files_test.clj +++ b/backend/test/app/services_files_test.clj @@ -496,7 +496,8 @@ (t/is (contains? (:objects result) shape1-id)) (t/is (contains? (:objects result) frame2-id)) (t/is (contains? (:objects result) shape2-id)) - (t/is (contains? (:objects result) uuid/zero))) + (t/is (contains? (:objects result) uuid/zero)) + ) ;; Query :page RPC method with page-id (let [data {::th/type :page @@ -523,6 +524,7 @@ :components-v2 true} {:keys [error result] :as out} (th/query! data)] ;; (th/print-result! out) + (t/is (nil? error)) (t/is (map? result)) (t/is (contains? result :objects)) (t/is (contains? (:objects result) frame1-id)) @@ -542,7 +544,9 @@ (t/is (not (th/success? out))) (let [{:keys [type code]} (-> out :error ex-data)] (t/is (= :validation type)) - (t/is (= :spec-validation code))))) + (t/is (= :spec-validation code)))) + + ) (t/testing "RPC :file-data-for-thumbnail" ;; Insert a thumbnail data for the frame-id diff --git a/backend/test/app/test_helpers.clj b/backend/test/app/test_helpers.clj index fea2b7075..0d9c3ddf5 100644 --- a/backend/test/app/test_helpers.clj +++ b/backend/test/app/test_helpers.clj @@ -18,7 +18,9 @@ [app.media] [app.migrations] [app.rpc.commands.auth :as cmd.auth] - [app.rpc.mutations.files :as files] + [app.rpc.commands.files :as files] + [app.rpc.commands.files.create :as files.create] + [app.rpc.commands.files.update :as files.update] [app.rpc.mutations.profile :as profile] [app.rpc.mutations.projects :as projects] [app.rpc.mutations.teams :as teams] @@ -178,11 +180,11 @@ (us/assert uuid? profile-id) (us/assert uuid? project-id) (with-open [conn (db/open pool)] - (#'files/create-file conn - (merge {:id (mk-uuid "file" i) - :name (str "file" i) - :components-v2 true} - params))))) + (files.create/create-file conn + (merge {:id (mk-uuid "file" i) + :name (str "file" i) + :components-v2 true} + params))))) (defn mark-file-deleted* ([params] (mark-file-deleted* *pool* params)) @@ -259,27 +261,27 @@ ([params] (create-file-role* *pool* params)) ([pool {:keys [file-id profile-id role] :or {role :owner}}] (with-open [conn (db/open pool)] - (#'files/create-file-role conn {:file-id file-id - :profile-id profile-id - :role role})))) + (files.create/create-file-role! conn {:file-id file-id + :profile-id profile-id + :role role})))) (defn update-file* ([params] (update-file* *pool* params)) ([pool {:keys [file-id changes session-id profile-id revn] :or {session-id (uuid/next) revn 0}}] (with-open [conn (db/open pool)] - (let [file (db/get-by-id conn :file file-id) - msgbus (:app.msgbus/msgbus *system*) - metrics (:app.metrics/metrics *system*)] - (#'files/update-file {:conn conn - :msgbus msgbus - :metrics metrics} - {:file file - :revn revn - :components-v2 true - :changes changes - :session-id session-id - :profile-id profile-id}))))) + (let [msgbus (:app.msgbus/msgbus *system*) + metrics (:app.metrics/metrics *system*) + features #{"components/v2"}] + (files.update/update-file {:conn conn + :msgbus msgbus + :metrics metrics} + {:id file-id + :revn revn + :features features + :changes changes + :session-id session-id + :profile-id profile-id}))))) ;; --- RPC HELPERS diff --git a/common/src/app/common/files/features.cljc b/common/src/app/common/files/features.cljc index a51c2344f..34d40800e 100644 --- a/common/src/app/common/files/features.cljc +++ b/common/src/app/common/files/features.cljc @@ -6,5 +6,12 @@ (ns app.common.files.features) +;; A set of enabled by default file features. Will be used in feature +;; negotiation on obtaining files from backend. + +(def enabled #{}) + +(def ^:dynamic *previous* #{}) (def ^:dynamic *current* #{}) -(def ^:dynamic *wrap-objects-fn* identity) +(def ^:dynamic *wrap-with-objects-map-fn* identity) +(def ^:dynamic *wrap-with-pointer-map-fn* identity) diff --git a/common/src/app/common/transit.cljc b/common/src/app/common/transit.cljc index 3f485ec7d..1c08c8594 100644 --- a/common/src/app/common/transit.cljc +++ b/common/src/app/common/transit.cljc @@ -33,6 +33,10 @@ (def write-handler-map (atom nil)) (def read-handler-map (atom nil)) +;; A generic pointer; mainly used for deserialize backend pointer-map +;; instances that serializes to pointer but may in other ways. +(defrecord Pointer [id]) + ;; --- HELPERS #?(:clj @@ -133,6 +137,11 @@ (.fromMillis ^js lxn/DateTime ms)))) :wfn (comp str inst-ms)} + {:id "penpot/pointer" + :class Pointer + :rfn (fn [[id meta]] + (Pointer. id meta {}))} + #?(:clj {:id "m" :class OffsetDateTime diff --git a/common/src/app/common/types/components_list.cljc b/common/src/app/common/types/components_list.cljc index 95abd7342..137990c8a 100644 --- a/common/src/app/common/types/components_list.cljc +++ b/common/src/app/common/types/components_list.cljc @@ -16,8 +16,8 @@ (defn add-component [file-data {:keys [id name path main-instance-id main-instance-page shapes]}] - (let [components-v2 (dm/get-in file-data [:options :components-v2]) - wrap-object-fn feat/*wrap-objects-fn*] + (let [components-v2 (dm/get-in file-data [:options :components-v2]) + wrap-object-fn feat/*wrap-with-objects-map-fn*] (cond-> file-data :always (assoc-in [:components id] @@ -35,7 +35,7 @@ (defn mod-component [file-data {:keys [id name path objects]}] - (let [wrap-objects-fn feat/*wrap-objects-fn*] + (let [wrap-objects-fn feat/*wrap-with-objects-map-fn*] (update-in file-data [:components id] (fn [component] (let [objects (some-> objects wrap-objects-fn)] diff --git a/common/src/app/common/types/page.cljc b/common/src/app/common/types/page.cljc index dd3485481..835df4b37 100644 --- a/common/src/app/common/types/page.cljc +++ b/common/src/app/common/types/page.cljc @@ -50,11 +50,13 @@ (defn make-empty-page [id name] - (let [wrap-fn ffeat/*wrap-objects-fn*] + (let [wrap-objects-fn ffeat/*wrap-with-objects-map-fn* + wrap-pointer-fn ffeat/*wrap-with-pointer-map-fn*] (-> empty-page-data (assoc :id id) (assoc :name name) - (update :objects wrap-fn)))) + (update :objects wrap-objects-fn) + (wrap-pointer-fn)))) ;; --- Helpers for flow diff --git a/frontend/src/app/main/data/dashboard.cljs b/frontend/src/app/main/data/dashboard.cljs index 8b3b682de..595a1029a 100644 --- a/frontend/src/app/main/data/dashboard.cljs +++ b/frontend/src/app/main/data/dashboard.cljs @@ -110,7 +110,7 @@ ptk/WatchEvent (watch [_ state _] (let [team-id (:current-team-id state)] - (->> (rp/query :team-members {:team-id team-id}) + (->> (rp/query! :team-members {:team-id team-id}) (rx/map team-members-fetched)))))) ;; --- EVENT: fetch-team-stats @@ -128,7 +128,7 @@ ptk/WatchEvent (watch [_ state _] (let [team-id (:current-team-id state)] - (->> (rp/query :team-stats {:team-id team-id}) + (->> (rp/query! :team-stats {:team-id team-id}) (rx/map team-stats-fetched)))))) ;; --- EVENT: fetch-team-invitations @@ -146,7 +146,7 @@ ptk/WatchEvent (watch [_ state _] (let [team-id (:current-team-id state)] - (->> (rp/query :team-invitations {:team-id team-id}) + (->> (rp/query! :team-invitations {:team-id team-id}) (rx/map team-invitations-fetched)))))) ;; --- EVENT: fetch-projects @@ -165,7 +165,7 @@ ptk/WatchEvent (watch [_ state _] (let [team-id (:current-team-id state)] - (->> (rp/query :projects {:team-id team-id}) + (->> (rp/query! :projects {:team-id team-id}) (rx/map projects-fetched)))))) ;; --- EVENT: search @@ -193,7 +193,7 @@ (watch [_ state _] (let [team-id (:current-team-id state) params (assoc params :team-id team-id)] - (->> (rp/query :search-files params) + (->> (rp/query! :search-files params) (rx/map search-result-fetched)))))) ;; --- EVENT: files @@ -222,7 +222,7 @@ (ptk/reify ::fetch-files ptk/WatchEvent (watch [_ _ _] - (->> (rp/query :project-files {:project-id project-id}) + (->> (rp/cmd! :get-project-files {:project-id project-id}) (rx/map #(files-fetched project-id %)))))) ;; --- EVENT: shared-files @@ -243,7 +243,7 @@ ptk/WatchEvent (watch [_ state _] (let [team-id (:current-team-id state)] - (->> (rp/query :team-shared-files {:team-id team-id}) + (->> (rp/cmd! :get-team-shared-files {:team-id team-id}) (rx/map shared-files-fetched)))))) ;; --- EVENT: Get files that use this shared-file @@ -269,7 +269,8 @@ ptk/WatchEvent (watch [_ _ _] (->> (rx/from files) - (rx/mapcat (fn [file] (rp/query :library-using-files {:file-id (:id file)}))) + (rx/map :id) + (rx/mapcat #(rp/cmd! :get-library-file-references {:file-id %})) (rx/reduce into []) (rx/map library-using-files-fetched))))) @@ -292,7 +293,7 @@ ptk/WatchEvent (watch [_ state _] (let [team-id (or team-id (:current-team-id state))] - (->> (rp/query :team-recent-files {:team-id team-id}) + (->> (rp/cmd! :get-team-recent-files {:team-id team-id}) (rx/map recent-files-fetched))))))) @@ -588,7 +589,7 @@ new-name (str name " " (tr "dashboard.copy-suffix"))] - (->> (rp/command! :duplicate-project {:project-id id :name new-name}) + (->> (rp/cmd! :duplicate-project {:project-id id :name new-name}) (rx/tap on-success) (rx/map project-duplicated) (rx/catch on-error)))))) @@ -608,7 +609,7 @@ :or {on-success identity on-error rx/throw}} (meta params)] - (->> (rp/command! :move-project {:project-id id :team-id team-id}) + (->> (rp/cmd! :move-project {:project-id id :team-id team-id}) (rx/tap on-success) (rx/catch on-error)))))) @@ -683,7 +684,7 @@ ptk/WatchEvent (watch [_ state _] (let [team-id (uuid/uuid (get-in state [:route :path-params :team-id]))] - (->> (rp/mutation :delete-file {:id id}) + (->> (rp/cmd! :delete-file {:id id}) (rx/map #(file-deleted team-id project-id))))))) ;; --- Rename File @@ -706,7 +707,7 @@ ptk/WatchEvent (watch [_ _ _] (let [params (select-keys params [:id :name])] - (->> (rp/mutation :rename-file params) + (->> (rp/cmd! :rename-file params) (rx/ignore)))))) ;; --- Set File shared @@ -731,7 +732,7 @@ ptk/WatchEvent (watch [_ _ _] (let [params {:id id :is-shared is-shared}] - (->> (rp/mutation :set-file-shared params) + (->> (rp/cmd! :set-file-shared params) (rx/ignore)))))) ;; --- EVENT: create-file @@ -774,7 +775,7 @@ (assoc :name name) (assoc :features features))] - (->> (rp/mutation! :create-file params) + (->> (rp/cmd! :create-file params) (rx/tap on-success) (rx/map #(with-meta (file-created %) (meta it))) (rx/catch on-error)))))) @@ -794,7 +795,7 @@ new-name (str name " " (tr "dashboard.copy-suffix"))] - (->> (rp/command! :duplicate-file {:file-id id :name new-name}) + (->> (rp/cmd! :duplicate-file {:file-id id :name new-name}) (rx/tap on-success) (rx/map file-created) (rx/catch on-error)))))) @@ -816,7 +817,7 @@ (let [{:keys [on-success on-error] :or {on-success identity on-error rx/throw}} (meta params)] - (->> (rp/command! :move-files {:ids ids :project-id project-id}) + (->> (rp/cmd! :move-files {:ids ids :project-id project-id}) (rx/tap on-success) (rx/catch on-error)))))) @@ -836,8 +837,7 @@ (let [{:keys [on-success on-error] :or {on-success identity on-error rx/throw}} (meta params)] - - (->> (rp/command! :clone-template {:project-id project-id :template-id template-id}) + (->> (rp/cmd! :clone-template {:project-id project-id :template-id template-id}) (rx/tap on-success) (rx/catch on-error)))))) diff --git a/frontend/src/app/main/data/workspace.cljs b/frontend/src/app/main/data/workspace.cljs index 6afd2e213..4b744acc3 100644 --- a/frontend/src/app/main/data/workspace.cljs +++ b/frontend/src/app/main/data/workspace.cljs @@ -357,7 +357,7 @@ ptk/WatchEvent (watch [_ _ _] (let [params {:id id :name name}] - (->> (rp/mutation :rename-file params) + (->> (rp/cmd! :rename-file params) (rx/ignore)))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/frontend/src/app/main/data/workspace/libraries.cljs b/frontend/src/app/main/data/workspace/libraries.cljs index 2572ab354..98a176d24 100644 --- a/frontend/src/app/main/data/workspace/libraries.cljs +++ b/frontend/src/app/main/data/workspace/libraries.cljs @@ -7,6 +7,7 @@ (ns app.main.data.workspace.libraries (:require [app.common.data :as d] + [app.common.files.features :as ffeat] [app.common.geom.point :as gpt] [app.common.logging :as log] [app.common.pages :as cp] @@ -19,7 +20,7 @@ [app.common.types.components-list :as ctkl] [app.common.types.container :as ctn] [app.common.types.file :as ctf] - [app.common.types.file.media-object :as ctfm] + [app.common.types.file.media-object :as ctfm] [app.common.types.pages-list :as ctpl] [app.common.types.shape-tree :as ctst] [app.common.types.typography :as ctt] @@ -359,7 +360,7 @@ (pcb/with-library-data data) (pcb/update-component id update-fn))] - (rx/of (dch/commit-changes changes)))))) + (rx/of (dch/commit-changes changes)))))) (defn duplicate-component "Create a new component copied from the one with the given id." @@ -428,7 +429,7 @@ component (ctf/get-deleted-component file-data component-id) page (ctpl/get-page file-data (:main-instance-page component)) - ; Make a new main instance, with the same id of the original + ; Make a new main instance, with the same id of the original [_main-instance shapes] (ctn/make-component-instance page component @@ -446,8 +447,8 @@ changes shapes) - ; restore-component change needs to be done after add main instance - ; because when undo changes, the orden is inverse + ; restore-component change needs to be done after add main instance + ; because when undo changes, the orden is inverse changes (pcb/restore-component changes component-id)] (rx/of (dch/commit-changes (assoc changes :file-id library-id))))))) @@ -508,12 +509,12 @@ (cph/clean-loops objects)) changes (reduce - (fn [changes id] - (dwlh/generate-detach-instance changes container id)) - (-> (pcb/empty-changes it) - (pcb/with-container container) - (pcb/with-objects objects)) - selected)] + (fn [changes id] + (dwlh/generate-detach-instance changes container id)) + (-> (pcb/empty-changes it) + (pcb/with-container container) + (pcb/with-objects objects)) + selected)] (rx/of (dch/commit-changes changes)))))) @@ -572,8 +573,8 @@ (log/debug :msg "RESET-COMPONENT finished" :js/rchanges (log-changes (:redo-changes changes) - file)) - (rx/of (dch/commit-changes changes)))))) + file)) + (rx/of (dch/commit-changes changes)))))) (defn update-component "Modify the component linked to the shape with the given id, in the @@ -624,11 +625,11 @@ (log/debug :msg "UPDATE-COMPONENT finished" :js/local-changes (log-changes - (:redo-changes local-changes) - file) + (:redo-changes local-changes) + file) :js/nonlocal-changes (log-changes - (:redo-changes nonlocal-changes) - file)) + (:redo-changes nonlocal-changes) + file)) (rx/of (when (seq (:redo-changes local-changes)) @@ -705,48 +706,48 @@ sync-typographies? (or (nil? asset-type) (= asset-type :typographies)) library-changes (reduce - pcb/concat-changes - (pcb/empty-changes it) - [(when sync-components? - (dwlh/generate-sync-library it file-id :components asset-id library-id state)) - (when sync-colors? - (dwlh/generate-sync-library it file-id :colors asset-id library-id state)) - (when sync-typographies? - (dwlh/generate-sync-library it file-id :typographies asset-id library-id state))]) + pcb/concat-changes + (pcb/empty-changes it) + [(when sync-components? + (dwlh/generate-sync-library it file-id :components asset-id library-id state)) + (when sync-colors? + (dwlh/generate-sync-library it file-id :colors asset-id library-id state)) + (when sync-typographies? + (dwlh/generate-sync-library it file-id :typographies asset-id library-id state))]) file-changes (reduce - pcb/concat-changes - (pcb/empty-changes it) - [(when sync-components? - (dwlh/generate-sync-file it file-id :components asset-id library-id state)) - (when sync-colors? - (dwlh/generate-sync-file it file-id :colors asset-id library-id state)) - (when sync-typographies? - (dwlh/generate-sync-file it file-id :typographies asset-id library-id state))]) + pcb/concat-changes + (pcb/empty-changes it) + [(when sync-components? + (dwlh/generate-sync-file it file-id :components asset-id library-id state)) + (when sync-colors? + (dwlh/generate-sync-file it file-id :colors asset-id library-id state)) + (when sync-typographies? + (dwlh/generate-sync-file it file-id :typographies asset-id library-id state))]) changes (pcb/concat-changes library-changes file-changes)] (log/debug :msg "SYNC-FILE finished" :js/rchanges (log-changes - (:redo-changes changes) - file)) + (:redo-changes changes) + file)) (rx/concat - (rx/of (dm/hide-tag :sync-dialog)) - (when (seq (:redo-changes changes)) - (rx/of (dch/commit-changes (assoc changes ;; TODO a ver qué pasa con esto - :file-id file-id)))) - (when (not= file-id library-id) - ;; When we have just updated the library file, give some time for the - ;; update to finish, before marking this file as synced. - ;; TODO: look for a more precise way of syncing this. - ;; Maybe by using the stream (second argument passed to watch) - ;; to wait for the corresponding changes-committed and then proceed - ;; with the :update-sync mutation. - (rx/concat (rx/timer 3000) - (rp/mutation :update-sync - {:file-id file-id - :library-id library-id}))) - (when (and (seq (:redo-changes library-changes)) - sync-components?) - (rx/of (sync-file-2nd-stage file-id library-id asset-id)))))))))) + (rx/of (dm/hide-tag :sync-dialog)) + (when (seq (:redo-changes changes)) + (rx/of (dch/commit-changes (assoc changes ;; TODO a ver qué pasa con esto + :file-id file-id)))) + (when (not= file-id library-id) + ;; When we have just updated the library file, give some time for the + ;; update to finish, before marking this file as synced. + ;; TODO: look for a more precise way of syncing this. + ;; Maybe by using the stream (second argument passed to watch) + ;; to wait for the corresponding changes-committed and then proceed + ;; with the :update-file-library-sync-status mutation. + (rx/concat (rx/timer 3000) + (rp/cmd! :update-file-library-sync-status + {:file-id file-id + :library-id library-id}))) + (when (and (seq (:redo-changes library-changes)) + sync-components?) + (rx/of (sync-file-2nd-stage file-id library-id asset-id)))))))))) (defn- sync-file-2nd-stage "If some components have been modified, we need to launch another synchronization @@ -775,8 +776,8 @@ (dwlh/generate-sync-library it file-id :components asset-id library-id state)])] (log/debug :msg "SYNC-FILE (2nd stage) finished" :js/rchanges (log-changes - (:redo-changes changes) - file)) + (:redo-changes changes) + file)) (when (seq (:redo-changes changes)) (rx/of (dch/commit-changes (assoc changes :file-id file-id)))))))) @@ -788,9 +789,9 @@ ptk/WatchEvent (watch [_ state _] - (rp/mutation :ignore-sync - {:file-id (get-in state [:workspace-file :id]) - :date (dt/now)})))) + (rp/cmd! :ignore-file-library-sync-status + {:file-id (get-in state [:workspace-file :id]) + :date (dt/now)})))) (defn notify-sync-file [file-id] @@ -852,8 +853,8 @@ (log/info :msg "DETECTED COMPONENTS CHANGED" :ids (map str components-changed)) (run! st/emit! - (map #(update-component-sync % (:id data)) - components-changed)))))] + (map #(update-component-sync % (:id data)) + components-changed)))))] (when components-v2 (->> change-s @@ -880,7 +881,7 @@ ptk/WatchEvent (watch [_ _ _] (let [params {:id id :is-shared is-shared}] - (->> (rp/mutation :set-file-shared params) + (->> (rp/cmd! :set-file-shared params) (rx/ignore)))))) (defn- shared-files-fetched @@ -898,7 +899,7 @@ (ptk/reify ::fetch-shared-files ptk/WatchEvent (watch [_ _ _] - (->> (rp/query :team-shared-files {:team-id team-id}) + (->> (rp/cmd! :get-team-shared-files {:team-id team-id}) (rx/map shared-files-fetched))))) ;; --- Link and unlink Files @@ -908,13 +909,16 @@ (ptk/reify ::attach-library ptk/WatchEvent (watch [_ state _] - (let [components-v2 (features/active-feature? state :components-v2) - fetched #(assoc-in %2 [:workspace-libraries (:id %1)] %1) - params {:file-id file-id - :library-id library-id}] - (->> (rp/mutation :link-file-to-library params) - (rx/mapcat #(rp/query :file {:id library-id :components-v2 components-v2})) - (rx/map #(partial fetched %))))))) + (let [features (cond-> ffeat/enabled + (features/active-feature? state :components-v2) + (conj "components/v2"))] + (rx/concat + (->> (rp/cmd! :link-file-to-library {:file-id file-id :library-id library-id}) + (rx/ignore)) + (->> (rp/cmd! :get-file {:id library-id :features features}) + (rx/map (fn [file] + (fn [state] + (assoc-in state [:workspace-libraries library-id] file)))))))))) (defn unlink-file-from-library [file-id library-id] @@ -927,5 +931,5 @@ (watch [_ _ _] (let [params {:file-id file-id :library-id library-id}] - (->> (rp/mutation :unlink-file-from-library params) + (->> (rp/cmd! :unlink-file-from-library params) (rx/ignore)))))) diff --git a/frontend/src/app/main/data/workspace/persistence.cljs b/frontend/src/app/main/data/workspace/persistence.cljs index 2d4ad1d25..cc148353c 100644 --- a/frontend/src/app/main/data/workspace/persistence.cljs +++ b/frontend/src/app/main/data/workspace/persistence.cljs @@ -8,6 +8,7 @@ (:require [app.common.data :as d] [app.common.data.macros :as dm] + [app.common.files.features :as ffeat] [app.common.logging :as log] [app.common.pages :as cp] [app.common.pages.changes-spec :as pcs] @@ -18,7 +19,6 @@ [app.config :as cf] [app.main.data.dashboard :as dd] [app.main.data.fonts :as df] - [app.main.data.modal :as modal] [app.main.data.workspace.changes :as dch] [app.main.data.workspace.state-helpers :as wsh] [app.main.data.workspace.thumbnails :as dwt] @@ -26,7 +26,6 @@ [app.main.repo :as rp] [app.main.store :as st] [app.util.http :as http] - [app.util.i18n :as i18n :refer [tr]] [app.util.router :as rt] [app.util.time :as dt] [beicon.core :as rx] @@ -138,7 +137,11 @@ (ptk/reify ::persist-changes ptk/WatchEvent (watch [_ state _] - (let [features (cond-> #{} + (let [;; this features set does not includes the ffeat/enabled + ;; because they are already available on the backend and + ;; this request provides a set of features to enable in + ;; this request. + features (cond-> #{} (features/active-feature? state :components-v2) (conj "components/v2")) sid (:session-id state) @@ -150,7 +153,7 @@ :features features}] (when (= file-id (:id params)) - (->> (rp/mutation :update-file params) + (->> (rp/cmd! :update-file params) (rx/mapcat (fn [lagged] (log/debug :hint "changes persisted" :lagged (count lagged)) (let [lagged (cond->> lagged @@ -285,14 +288,13 @@ ptk/WatchEvent (watch [_ state _] (let [share-id (-> state :viewer-local :share-id) - features (cond-> #{} + features (cond-> ffeat/enabled (features/active-feature? state :components-v2) (conj "components/v2"))] - - (->> (rx/zip (rp/query! :file-raw {:id file-id :features features}) + (->> (rx/zip (rp/cmd! :get-raw-file {:id file-id :features features}) (rp/query! :team-users {:file-id file-id}) (rp/query! :project {:id project-id}) - (rp/query! :file-libraries {:file-id file-id}) + (rp/cmd! :get-file-libraries {:file-id file-id}) (rp/cmd! :get-profiles-for-file-comments {:file-id file-id :share-id share-id})) (rx/take 1) (rx/map (fn [[file-raw users project libraries file-comments-users]] @@ -303,16 +305,7 @@ :file-comments-users file-comments-users})) (rx/mapcat (fn [{:keys [project] :as bundle}] (rx/of (ptk/data-event ::bundle-fetched bundle) - (df/load-team-fonts (:team-id project))))) - (rx/catch (fn [err] - (if (and (= (:type err) :restriction) - (= (:code err) :feature-disabled)) - (let [team-id (:current-team-id state)] - (rx/of (modal/show - {:type :alert - :message (tr "errors.components-v2") - :on-accept #(st/emit! (rt/nav :dashboard-projects {:team-id team-id}))}))) - (rx/throw err))))))))) + (df/load-team-fonts (:team-id project)))))))))) ;; --- Helpers diff --git a/frontend/src/app/main/data/workspace/thumbnails.cljs b/frontend/src/app/main/data/workspace/thumbnails.cljs index 5f894f081..ce478da9b 100644 --- a/frontend/src/app/main/data/workspace/thumbnails.cljs +++ b/frontend/src/app/main/data/workspace/thumbnails.cljs @@ -81,7 +81,7 @@ (rx/merge ;; Update the local copy of the thumbnails so we don't need to request it again (rx/of #(assoc-in % [:workspace-file :thumbnails object-id] data)) - (->> (rp/mutation! :upsert-file-object-thumbnail params) + (->> (rp/cmd! :upsert-file-object-thumbnail params) (rx/ignore)))) (rx/empty)))))))))) diff --git a/frontend/src/app/main/errors.cljs b/frontend/src/app/main/errors.cljs index f0d724c0e..fe9eb8744 100644 --- a/frontend/src/app/main/errors.cljs +++ b/frontend/src/app/main/errors.cljs @@ -14,6 +14,7 @@ [app.common.spec :as us] [app.config :as cf] [app.main.data.messages :as msg] + [app.main.data.modal :as modal] [app.main.data.users :as du] [app.main.store :as st] [app.util.globals :as glob] @@ -167,6 +168,26 @@ (ts/schedule #(st/emit! (rt/assign-exception error)))) +(defmethod ptk/handle-error :restriction + [{:keys [code] :as error}] + (cond + (= :feature-mismatch code) + (let [message (tr "errors.feature-mismatch" (:feature error))] + (st/emit! (modal/show + {:type :alert + :message message + :on-accept #(prn "kaka")}))) + + (= :features-not-supported code) + (let [message (tr "errors.feature-not-supported" (:feature error))] + (st/emit! (modal/show + {:type :alert + :message message + :on-accept #(prn "kaka")}))) + + :else + (ptk/handle-error (assoc error :type :server-error)))) + ;; This happens when the backed server fails to process the ;; request. This can be caused by an internal assertion or any other ;; uncontrolled error. diff --git a/frontend/src/app/main/features.cljs b/frontend/src/app/main/features.cljs index c8126335e..04d77115b 100644 --- a/frontend/src/app/main/features.cljs +++ b/frontend/src/app/main/features.cljs @@ -16,7 +16,7 @@ (log/set-level! :debug) -(def features-list #{:auto-layout :components-v2}) +(def available-features #{:auto-layout :components-v2}) (defn- toggle-feature [feature] @@ -38,14 +38,14 @@ (defn toggle-feature! [feature] - (assert (contains? features-list feature) "Not supported feature") + (assert (contains? available-features feature) "Not supported feature") (st/emit! (toggle-feature feature))) (defn active-feature? ([feature] (active-feature? @st/state feature)) ([state feature] - (assert (contains? features-list feature) "Not supported feature") + (assert (contains? available-features feature) "Not supported feature") (contains? (get state :features) feature))) (def features @@ -57,7 +57,7 @@ (defn use-feature [feature] - (assert (contains? features-list feature) "Not supported feature") + (assert (contains? available-features feature) "Not supported feature") (let [active-feature-ref (mf/use-memo (mf/deps feature) #(active-feature feature)) active-feature? (mf/deref active-feature-ref)] active-feature?)) @@ -69,6 +69,6 @@ (when *assert* ;; By default, all features disabled, except in development ;; environment, that are enabled except components-v2 - (doseq [f features-list] + (doseq [f available-features] (when (not= f :components-v2) (toggle-feature! f))))) diff --git a/frontend/src/app/main/repo.cljs b/frontend/src/app/main/repo.cljs index d7dc4727a..dcac66f92 100644 --- a/frontend/src/app/main/repo.cljs +++ b/frontend/src/app/main/repo.cljs @@ -12,6 +12,8 @@ [app.util.http :as http] [beicon.core :as rx])) +(derive :get-file ::query) + (defn handle-response [{:keys [status body] :as response}] (cond @@ -48,7 +50,6 @@ query api." ([id params] (send-query! id params nil)) - ([id params {:keys [raw-transit?]}] (let [decode-transit (if raw-transit? http/conditional-error-decode-transit @@ -74,14 +75,24 @@ (defn- send-command! "A simple helper for a common case of sending and receiving transit data to the penpot mutation api." - [id params {:keys [response-type form-data?]}] - (->> (http/send! {:method :post - :uri (u/join @cf/public-uri "api/rpc/command/" (name id)) - :credentials "include" - :body (if form-data? (http/form-data params) (http/transit-data params)) - :response-type (or response-type :text)}) - (rx/map http/conditional-decode-transit) - (rx/mapcat handle-response))) + [id params {:keys [response-type form-data? raw-transit?]}] + (let [decode-fn (if raw-transit? + http/conditional-error-decode-transit + http/conditional-decode-transit) + method (if (isa? id ::query) :get :post)] + + (->> (http/send! {:method method + :uri (u/join @cf/public-uri "api/rpc/command/" (name id)) + :credentials "include" + :body (when (= method :post) + (if form-data? + (http/form-data params) + (http/transit-data params))) + :query (when (= method :get) + params) + :response-type (or response-type :text)}) + (rx/map decode-fn) + (rx/mapcat handle-response)))) (defn- dispatch [& args] (first args)) @@ -93,9 +104,9 @@ [id params] (send-query! id params)) -(defmethod query :file-raw +(defmethod command :get-raw-file [_id params] - (send-query! :file params {:raw-transit? true})) + (send-command! :get-file params {:raw-transit? true})) (defmethod mutation :default [id params] diff --git a/frontend/src/app/main/ui/dashboard/grid.cljs b/frontend/src/app/main/ui/dashboard/grid.cljs index 0a5c23994..79e9193f3 100644 --- a/frontend/src/app/main/ui/dashboard/grid.cljs +++ b/frontend/src/app/main/ui/dashboard/grid.cljs @@ -7,6 +7,7 @@ (ns app.main.ui.dashboard.grid (:require [app.common.data.macros :as dm] + [app.common.files.features :as ffeat] [app.common.logging :as log] [app.main.data.dashboard :as dd] [app.main.data.messages :as msg] @@ -34,18 +35,21 @@ [cuerdas.core :as str] [rumext.v2 :as mf])) -(log/set-level! :info) +(log/set-level! :debug) ;; --- Grid Item Thumbnail (defn ask-for-thumbnail "Creates some hooks to handle the files thumbnails cache" [file] - (wrk/ask! {:cmd :thumbnails/generate - :revn (:revn file) - :file-id (:id file) - :file-name (:name file) - :components-v2 (features/active-feature? :components-v2)})) + (let [features (cond-> ffeat/enabled + (features/active-feature? :components-v2) + (conj "components/v2"))] + (wrk/ask! {:cmd :thumbnails/generate + :revn (:revn file) + :file-id (:id file) + :file-name (:name file) + :features features}))) (mf/defc grid-item-thumbnail {::mf/wrap [mf/memo]} @@ -61,10 +65,10 @@ (rx/subscribe-on :af) (rx/subs (fn [{:keys [data fonts] :as params}] (run! fonts/ensure-loaded! fonts) - (log/info :hint "loaded thumbnail" - :file-id (dm/str (:id file)) - :file-name (:name file) - :elapsed (str/ffmt "%ms" (tp))) + (log/debug :hint "loaded thumbnail" + :file-id (dm/str (:id file)) + :file-name (:name file) + :elapsed (str/ffmt "%ms" (tp))) (when-let [node (mf/ref-val container)] (dom/set-html! node data)))))))) diff --git a/frontend/src/app/render.cljs b/frontend/src/app/render.cljs index a5660ff1d..1276b4775 100644 --- a/frontend/src/app/render.cljs +++ b/frontend/src/app/render.cljs @@ -98,22 +98,23 @@ [{:keys [page-id file-id object-id render-embed?]}] (let [components-v2 (features/use-feature :components-v2) fetch-state (mf/use-fn - (mf/deps file-id page-id object-id) + (mf/deps file-id page-id object-id components-v2) (fn [] - (->> (rx/zip - (repo/query! :font-variants {:file-id file-id}) - (repo/query! :page {:file-id file-id - :page-id page-id - :object-id object-id - :components-v2 components-v2})) - (rx/tap (fn [[fonts]] - (when (seq fonts) - (st/emit! (df/fonts-fetched fonts))))) - (rx/map (comp :objects second)) - (rx/map (fn [objects] - (let [objects (render/adapt-objects-for-shape objects object-id)] - {:objects objects - :object (get objects object-id)})))))) + (let [features (cond-> #{} components-v2 (conj "components/v2"))] + (->> (rx/zip + (repo/query! :font-variants {:file-id file-id}) + (repo/cmd! :page {:file-id file-id + :page-id page-id + :object-id object-id + :features features})) + (rx/tap (fn [[fonts]] + (when (seq fonts) + (st/emit! (df/fonts-fetched fonts))))) + (rx/map (comp :objects second)) + (rx/map (fn [objects] + (let [objects (render/adapt-objects-for-shape objects object-id)] + {:objects objects + :object (get objects object-id)}))))))) {:keys [objects object]} (use-resource fetch-state)] @@ -137,17 +138,18 @@ [{:keys [page-id file-id object-ids render-embed?]}] (let [components-v2 (features/use-feature :components-v2) fetch-state (mf/use-fn - (mf/deps file-id page-id) + (mf/deps file-id page-id components-v2) (fn [] - (->> (rx/zip - (repo/query! :font-variants {:file-id file-id}) - (repo/query! :page {:file-id file-id - :page-id page-id - :components-v2 components-v2})) - (rx/tap (fn [[fonts]] - (when (seq fonts) - (st/emit! (df/fonts-fetched fonts))))) - (rx/map (comp :objects second))))) + (let [features (cond-> #{} components-v2 (conj "components/v2"))] + (->> (rx/zip + (repo/query! :font-variants {:file-id file-id}) + (repo/cmd! :get-page {:file-id file-id + :page-id page-id + :features features})) + (rx/tap (fn [[fonts]] + (when (seq fonts) + (st/emit! (df/fonts-fetched fonts))))) + (rx/map (comp :objects second)))))) objects (use-resource fetch-state)] @@ -204,7 +206,7 @@ [{:keys [file-id embed] :as props}] (let [fetch (mf/use-fn (mf/deps file-id) - (fn [] (repo/query! :file {:id file-id}))) + (fn [] (repo/cmd! :get-file {:id file-id}))) file (use-resource fetch) state (mf/use-state nil)] diff --git a/frontend/src/app/util/http.cljs b/frontend/src/app/util/http.cljs index aed2f35c3..fcf9f55c3 100644 --- a/frontend/src/app/util/http.cljs +++ b/frontend/src/app/util/http.cljs @@ -139,6 +139,7 @@ [{:keys [body headers] :as response}] (let [contenttype (get headers "content-type")] (if (and (str/starts-with? contenttype "application/transit+json") + (string? body) (pos? (count body))) (assoc response :body (t/decode-str body)) response))) diff --git a/frontend/src/app/worker/export.cljs b/frontend/src/app/worker/export.cljs index ac3ecfac4..c40a172b9 100644 --- a/frontend/src/app/worker/export.cljs +++ b/frontend/src/app/worker/export.cljs @@ -155,14 +155,15 @@ (->> (r/render-components (:data file) :deleted-components) (rx/map #(vector (str (:id file) "/deleted-components.svg") %)))) -(defn fetch-file-with-libraries [file-id components-v2] - (->> (rx/zip (rp/query :file {:id file-id :components-v2 components-v2}) - (rp/query :file-libraries {:file-id file-id})) - (rx/map - (fn [[file file-libraries]] - (let [libraries-ids (->> file-libraries (map :id) (filterv #(not= (:id file) %)))] - (-> file - (assoc :libraries libraries-ids))))))) +(defn fetch-file-with-libraries + [file-id components-v2] + (let [features (cond-> #{} components-v2 (conj "components/v2"))] + (->> (rx/zip (rp/cmd! :get-file {:id file-id :features features}) + (rp/cmd! :get-file-libraries {:file-id file-id})) + (rx/map + (fn [[file file-libraries]] + (let [libraries-ids (->> file-libraries (map :id) (filterv #(not= (:id file) %)))] + (assoc file :libraries libraries-ids))))))) (defn get-component-ref-file [objects shape] diff --git a/frontend/src/app/worker/impl.cljs b/frontend/src/app/worker/impl.cljs index 4cd27ac2c..4e19add0e 100644 --- a/frontend/src/app/worker/impl.cljs +++ b/frontend/src/app/worker/impl.cljs @@ -6,6 +6,7 @@ (ns app.worker.impl (:require + [app.common.data.macros :as dm] [app.common.logging :as log] [app.common.pages.changes :as ch] [app.common.transit :as t] @@ -36,18 +37,16 @@ (let [data (-> (t/decode-str file-raw) :data) message (assoc message :data data)] (reset! state data) - (handler (-> message - (assoc :cmd :selection/initialize-index))) - (handler (-> message - (assoc :cmd :snaps/initialize-index))))) + (handler (assoc message :cmd :selection/initialize-index)) + (handler (assoc message :cmd :snaps/initialize-index)))) (defmethod handler :update-page-indices [{:keys [page-id changes] :as message}] - (let [old-page (get-in @state [:pages-index page-id])] + (let [old-page (dm/get-in @state [:pages-index page-id])] (swap! state ch/process-changes changes false) - (let [new-page (get-in @state [:pages-index page-id]) + (let [new-page (dm/get-in @state [:pages-index page-id]) message (assoc message :old-page old-page :new-page new-page)] diff --git a/frontend/src/app/worker/import.cljs b/frontend/src/app/worker/import.cljs index f7610e0cd..a58f4e054 100644 --- a/frontend/src/app/worker/import.cljs +++ b/frontend/src/app/worker/import.cljs @@ -15,7 +15,6 @@ [app.common.logging :as log] [app.common.media :as cm] [app.common.text :as ct] - [app.common.types.file :as ctf] [app.common.uuid :as uuid] [app.main.repo :as rp] [app.util.http :as http] @@ -128,16 +127,15 @@ (defn create-file "Create a new file on the back-end" [context components-v2] - (let [resolve (:resolve context) - file-id (resolve (:file-id context))] - (rp/mutation :create-temp-file - {:id file-id - :name (:name context) - :is-shared (:shared context) - :project-id (:project-id context) - :data (-> ctf/empty-file-data - (assoc :id file-id) - (assoc-in [:options :components-v2] components-v2))}))) + (let [resolve-fn (:resolve context) + file-id (resolve-fn (:file-id context)) + features (cond-> #{} components-v2 (conj "components/v2"))] + (rp/cmd! :create-temp-file + {:id file-id + :name (:name context) + :is-shared (:shared context) + :project-id (:project-id context) + :features features}))) (defn link-file-libraries "Create a new file on the back-end" @@ -147,7 +145,7 @@ libraries (->> context :libraries (mapv resolve))] (->> (rx/from libraries) (rx/map #(hash-map :file-id file-id :library-id %)) - (rx/flat-map (partial rp/mutation :link-file-to-library))))) + (rx/flat-map (partial rp/cmd! :link-file-to-library))))) (defn send-changes "Creates batches of changes to be sent to the backend" @@ -165,17 +163,17 @@ (->> (rx/from (d/enumerate batches)) (rx/merge-map (fn [[i change-batch]] - (->> (rp/mutation :update-temp-file - {:id file-id - :session-id session-id - :revn i - :changes change-batch}) + (->> (rp/cmd! :update-temp-file + {:id file-id + :session-id session-id + :revn i + :changes change-batch}) (rx/tap #(do (swap! processed inc) (progress! context :upload-data @processed total)))))) (rx/map first) (rx/ignore)) - (->> (rp/mutation :persist-temp-file {:id file-id}) + (->> (rp/cmd! :persist-temp-file {:id file-id}) ;; We use merge to keep some information not stored in back-end (rx/map #(merge file %)))))) diff --git a/frontend/src/app/worker/thumbnails.cljs b/frontend/src/app/worker/thumbnails.cljs index 6b3c757c3..4bb06f6c1 100644 --- a/frontend/src/app/worker/thumbnails.cljs +++ b/frontend/src/app/worker/thumbnails.cljs @@ -7,6 +7,7 @@ (ns app.worker.thumbnails (:require ["react-dom/server" :as rds] + [app.common.logging :as log] [app.common.uri :as u] [app.config :as cf] [app.main.fonts :as fonts] @@ -17,6 +18,9 @@ [debug :refer [debug?]] [rumext.v2 :as mf])) +(log/set-level! :trace) + + (defn- handle-response [{:keys [body status] :as response}] (cond @@ -48,30 +52,29 @@ (= :request-body-too-large code))) (defn- request-data-for-thumbnail - [file-id revn components-v2] - (let [path "api/rpc/query/file-data-for-thumbnail" - params {:file-id file-id - :revn revn - :strip-frames-with-thumbnails true - :components-v2 components-v2} - request {:method :get - :uri (u/join @cf/public-uri path) - :credentials "include" - :query params}] + [file-id revn features] + (let [path "api/rpc/command/get-file-data-for-thumbnail" + params {:file-id file-id + :revn revn + :strip-frames-with-thumbnails true + :features features} + request {:method :get + :uri (u/join @cf/public-uri path) + :credentials "include" + :query params}] (->> (http/send! request) (rx/map http/conditional-decode-transit) (rx/mapcat handle-response)))) (defn- request-thumbnail [file-id revn] - (let [path "api/rpc/query/file-thumbnail" + (let [path "api/rpc/command/get-file-thumbnail" params {:file-id file-id :revn revn} request {:method :get :uri (u/join @cf/public-uri path) :credentials "include" :query params}] - (->> (http/send! request) (rx/map http/conditional-decode-transit) (rx/mapcat handle-response)))) @@ -91,7 +94,7 @@ (defn- persist-thumbnail [{:keys [file-id data revn fonts]}] - (let [path "api/rpc/mutation/upsert-file-thumbnail" + (let [path "api/rpc/command/upsert-file-thumbnail" params {:file-id file-id :revn revn :props {:fonts fonts} @@ -108,19 +111,22 @@ (rx/map (constantly params))))) (defmethod impl/handler :thumbnails/generate - [{:keys [file-id revn components-v2] :as message}] + [{:keys [file-id revn features] :as message}] (letfn [(on-result [{:keys [data props]}] {:data data :fonts (:fonts props)}) (on-cache-miss [_] - (->> (request-data-for-thumbnail file-id revn components-v2) + (log/debug :hint "request-thumbnail" :file-id file-id :revn revn :cache "miss") + (->> (request-data-for-thumbnail file-id revn features) (rx/map render-thumbnail) (rx/mapcat persist-thumbnail)))] (if (debug? :disable-thumbnail-cache) - (->> (request-data-for-thumbnail file-id revn components-v2) + (->> (request-data-for-thumbnail file-id revn features) (rx/map render-thumbnail)) (->> (request-thumbnail file-id revn) + (rx/tap (fn [_] + (log/debug :hint "request-thumbnail" :file-id file-id :revn revn :cache "hit"))) (rx/catch not-found? on-cache-miss) (rx/map on-result))))) diff --git a/frontend/translations/en.po b/frontend/translations/en.po index 3da66fe2a..058be9b70 100644 --- a/frontend/translations/en.po +++ b/frontend/translations/en.po @@ -710,9 +710,13 @@ msgstr "The fonts %s could not be loaded" msgid "errors.clipboard-not-implemented" msgstr "Your browser cannot do this operation" -#: src/app/main/data/workspace/persistence.cljs -msgid "errors.components-v2" -msgstr "This file has already used with Components V2 enabled." +#: src/app/main/errors.cljs +msgid "errors.feature-not-supported" +msgstr "Feature '%s' is not supported." + +#: src/app/main/errors.cljs +msgid "errors.feature-mismatch" +msgstr "Looks like you are opening a file that has the feature '%s' enabled bug your penpot frontend does not supports it or has it disabled." #: src/app/main/ui/auth/verify_token.cljs, src/app/main/ui/settings/change_email.cljs msgid "errors.email-already-exists"