From 6f7f74f7c64b8b2810dfd4e7743c7683816bd880 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 29 Mar 2022 17:59:38 +0200 Subject: [PATCH] :bug: Add migrations to fix wrongly migrated data Also port the migration introduced in main branch for the recent hotfix --- common/src/app/common/data.cljc | 20 +- common/src/app/common/pages/common.cljc | 2 +- common/src/app/common/pages/helpers.cljc | 4 + common/src/app/common/pages/migrations.cljc | 301 +++++++++++--------- 4 files changed, 191 insertions(+), 136 deletions(-) diff --git a/common/src/app/common/data.cljc b/common/src/app/common/data.cljc index e42abf288..a67869186 100644 --- a/common/src/app/common/data.cljc +++ b/common/src/app/common/data.cljc @@ -6,7 +6,8 @@ (ns app.common.data "Data manipulation and query helper functions." - (:refer-clojure :exclude [read-string hash-map merge name parse-double group-by iteration]) + (:refer-clojure :exclude [read-string hash-map merge name update-vals + parse-double group-by iteration]) #?(:cljs (:require-macros [app.common.data])) (:require @@ -198,6 +199,23 @@ ([mfn coll] (into {} (mapm mfn) coll))) +;; TEMPORARY COPY of clojure.core/update-vals until we migrate to clojure 1.11 + +(defn update-vals + "m f => {k (f v) ...} + Given a map m and a function f of 1-argument, returns a new map where the keys of m + are mapped to result of applying f to the corresponding values of m." + [m f] + (with-meta + (persistent! + (reduce-kv (fn [acc k v] (assoc! acc k (f v))) + (if #?(:clj (instance? clojure.lang.IEditableCollection m) + :cljs (implements? core/IEditableCollection m)) + (transient m) + (transient {})) + m)) + (meta m))) + (defn removev "Returns a vector of the items in coll for which (fn item) returns logical false" [fn coll] diff --git a/common/src/app/common/pages/common.cljc b/common/src/app/common/pages/common.cljc index 8cd547d62..a8221f7f0 100644 --- a/common/src/app/common/pages/common.cljc +++ b/common/src/app/common/pages/common.cljc @@ -9,7 +9,7 @@ [app.common.colors :as clr] [app.common.uuid :as uuid])) -(def file-version 16) +(def file-version 17) (def default-color clr/gray-20) (def root uuid/zero) diff --git a/common/src/app/common/pages/helpers.cljc b/common/src/app/common/pages/helpers.cljc index ce5dd18a4..39d881c53 100644 --- a/common/src/app/common/pages/helpers.cljc +++ b/common/src/app/common/pages/helpers.cljc @@ -34,6 +34,10 @@ [{:keys [type]}] (= type :text)) +(defn ^boolean image-shape? + [{:keys [type]}] + (= type :image)) + (defn ^boolean unframed-shape? "Checks if it's a non-frame shape in the top level." [shape] diff --git a/common/src/app/common/pages/migrations.cljc b/common/src/app/common/pages/migrations.cljc index d70d0e345..9defdcdb0 100644 --- a/common/src/app/common/pages/migrations.cljc +++ b/common/src/app/common/pages/migrations.cljc @@ -12,7 +12,9 @@ [app.common.geom.shapes.path :as gsp] [app.common.math :as mth] [app.common.pages :as cp] - [app.common.uuid :as uuid])) + [app.common.pages.helpers :as cph] + [app.common.uuid :as uuid] + [cuerdas.core :as str])) ;; TODO: revisit this and rename to file-migrations @@ -45,17 +47,16 @@ ;; Ensure that all :shape attributes on shapes are vectors. (defmethod migrate 2 [data] - (letfn [(update-object [_ object] + (letfn [(update-object [object] (d/update-when object :shapes (fn [shapes] (if (seq? shapes) (into [] shapes) shapes)))) + (update-page [page] + (update page :objects d/update-vals update-object))] - (update-page [_ page] - (update page :objects #(d/mapm update-object %)))] - - (update data :pages-index #(d/mapm update-page %)))) + (update data :pages-index d/update-vals update-page))) ;; Changes paths formats (defmethod migrate 3 @@ -89,7 +90,7 @@ (empty? (:points shape)) (assoc :points (gsh/rect->points (:selrect shape)))))) - (update-object [_ object] + (update-object [object] (cond-> object (= :curve (:type object)) (assoc :type :path) @@ -97,25 +98,22 @@ (#{:curve :path} (:type object)) (migrate-path) - (= :frame (:type object)) + (cph/frame-shape? object) (fix-frames-selrects) (and (empty? (:points object)) (not= (:id object) uuid/zero)) (fix-empty-points) + ;; Setup an empty transformation to re-calculate selrects + ;; and points data :always - (-> - ;; Setup an empty transformation to re-calculate selrects - ;; and points data - (assoc :modifiers {:displacement (gmt/matrix)}) - (gsh/transform-shape)) + (-> (assoc :modifiers {:displacement (gmt/matrix)}) + (gsh/transform-shape)))) - )) + (update-page [page] + (update page :objects d/update-vals update-object))] - (update-page [_ page] - (update page :objects #(d/mapm update-object %)))] - - (update data :pages-index #(d/mapm update-page %)))) + (update data :pages-index d/update-vals update-page))) ;; We did rollback version 4 migration. ;; Keep this in order to remember the next version to be 5 @@ -124,61 +122,55 @@ ;; Put the id of the local file in :component-file in instances of local components (defmethod migrate 5 [data] - (letfn [(update-object [_ object] + (letfn [(update-object [object] (if (and (some? (:component-id object)) (nil? (:component-file object))) (assoc object :component-file (:id data)) object)) - (update-page [_ page] - (update page :objects #(d/mapm update-object %)))] - - (update data :pages-index #(d/mapm update-page %)))) - -(defn fix-line-paths - "Fixes issues with selrect/points for shapes with width/height = 0 (line-like paths)" - [_ shape] - (if (= (:type shape) :path) - (let [{:keys [width height]} (gsh/points->rect (:points shape))] - (if (or (mth/almost-zero? width) (mth/almost-zero? height)) - (let [selrect (gsh/content->selrect (:content shape)) - points (gsh/rect->points selrect) - transform (gmt/matrix) - transform-inv (gmt/matrix)] - (assoc shape - :selrect selrect - :points points - :transform transform - :transform-inverse transform-inv)) - shape)) - shape)) + (update-page [page] + (update page :objects d/update-vals update-object))] + (update data :pages-index d/update-vals update-page))) (defmethod migrate 6 [data] - (letfn [(update-container [_ container] - (-> container - (update :objects #(d/mapm fix-line-paths %))))] + ;; Fixes issues with selrect/points for shapes with width/height = 0 (line-like paths)" + (letfn [(fix-line-paths [shape] + (if (= (:type shape) :path) + (let [{:keys [width height]} (gsh/points->rect (:points shape))] + (if (or (mth/almost-zero? width) (mth/almost-zero? height)) + (let [selrect (gsh/content->selrect (:content shape)) + points (gsh/rect->points selrect) + transform (gmt/matrix) + transform-inv (gmt/matrix)] + (assoc shape + :selrect selrect + :points points + :transform transform + :transform-inverse transform-inv)) + shape)) + shape)) + + (update-container [container] + (update container :objects d/update-vals fix-line-paths))] (-> data - (update :components #(d/mapm update-container %)) - (update :pages-index #(d/mapm update-container %))))) - + (update :pages-index d/update-vals update-container) + (update :components d/update-vals update-container)))) ;; Remove interactions pointing to deleted frames (defmethod migrate 7 [data] - (letfn [(update-object [page _ object] + (letfn [(update-object [page object] (d/update-when object :interactions - (fn [interactions] - (filterv #(get-in page [:objects (:destination %)]) - interactions)))) + (fn [interactions] + (filterv #(get-in page [:objects (:destination %)]) interactions)))) - (update-page [_ page] - (update page :objects #(d/mapm (partial update-object page) %)))] - - (update data :pages-index #(d/mapm update-page %)))) + (update-page [page] + (update page :objects d/update-vals (partial update-object page)))] + (update data :pages-index d/update-vals update-page))) ;; Remove groups without any shape, both in pages and components @@ -210,7 +202,7 @@ [(count deleted) (d/mapm #(clean-parents %2 deleted) result)])))) - (clean-container [_ container] + (clean-container [container] (loop [n 0 objects (:objects container)] (let [[deleted objects] (clean-objects objects)] @@ -219,8 +211,8 @@ (assoc container :objects objects)))))] (-> data - (update :pages-index #(d/mapm clean-container %)) - (d/update-when :components #(d/mapm clean-container %))))) + (update :pages-index d/update-vals clean-container) + (update :components d/update-vals clean-container)))) (defmethod migrate 9 [data] @@ -252,35 +244,35 @@ (defmethod migrate 10 [data] - (letfn [(update-page [_ page] + (letfn [(update-page [page] (d/update-in-when page [:objects uuid/zero] dissoc :points :selrect))] - (update data :pages-index #(d/mapm update-page %)))) + (update data :pages-index d/update-vals update-page))) (defmethod migrate 11 [data] - (letfn [(update-object [objects _id shape] - (if (= :frame (:type shape)) + (letfn [(update-object [objects shape] + (if (cph/frame-shape? shape) (d/update-when shape :shapes (fn [shapes] (filterv (fn [id] (contains? objects id)) shapes))) shape)) - (update-page [_ page] - (update page :objects #(d/mapm (partial update-object %) %)))] - - (update data :pages-index #(d/mapm update-page %)))) + (update-page [page] + (update page :objects (fn [objects] + (d/update-vals objects (partial update-object objects)))))] + (update data :pages-index d/update-vals update-page))) (defmethod migrate 12 [data] - (letfn [(update-grid [_key grid] + (letfn [(update-grid [grid] (cond-> grid (= :auto (:size grid)) (assoc :size nil))) - (update-page [_id page] - (d/update-in-when page [:options :saved-grids] #(d/mapm update-grid %)))] + (update-page [page] + (d/update-in-when page [:options :saved-grids] d/update-vals update-grid))] - (update data :pages-index #(d/mapm update-page %)))) + (update data :pages-index d/update-vals update-page))) ;; Add rx and ry to images (defmethod migrate 13 @@ -291,83 +283,124 @@ (assoc :rx 0) (assoc :ry 0)) shape)) - (update-object [_ object] + + (update-object [object] (cond-> object - (= :image (:type object)) + (cph/image-shape? object) (fix-radius))) - (update-page [_ page] - (update page :objects #(d/mapm update-object %)))] + (update-page [page] + (update page :objects d/update-vals update-object))] - (update data :pages-index #(d/mapm update-page %)))) + (update data :pages-index d/update-vals update-page))) -(defn set-fills - [shape] - (let [attrs {:fill-color (:fill-color shape) - :fill-color-gradient (:fill-color-gradient shape) - :fill-color-ref-file (:fill-color-ref-file shape) - :fill-color-ref-id (:fill-color-ref-id shape) - :fill-opacity (:fill-opacity shape)} - - clean-attrs (d/without-nils attrs)] - (cond-> shape - (d/not-empty? clean-attrs) - (assoc :fills [clean-attrs])))) - -;; Add fills to shapes (defmethod migrate 14 [data] - (letfn [(update-object [_ object] - (cond-> object - (and (not (= :text (:type object))) (nil? (:fills object))) - (set-fills))) + (letfn [(process-shape [shape] + (let [fill-color (str/upper (:fill-color shape)) + fill-opacity (:fill-opacity shape)] + (cond-> shape + (and (= 1 fill-opacity) + (or (= "#B1B2B5" fill-color) + (= "#7B7D85" fill-color))) + (dissoc :fill-color :fill-opacity)))) - (update-page [_ page] - (update page :objects #(d/mapm update-object %)))] - (update data :pages-index #(d/mapm update-page %)))) + (update-container [{:keys [objects] :as container}] + (loop [objects objects + shapes (->> (vals objects) + (filter cph/image-shape?))] + (if-let [shape (first shapes)] + (let [{:keys [id frame-id] :as shape'} (process-shape shape)] + (if (identical? shape shape') + (recur objects (rest shapes)) + (recur (-> objects + (assoc id shape') + (d/update-when frame-id dissoc :thumbnail)) + (rest shapes)))) + (assoc container :objects objects))))] -(defn set-strokes - [shape] - (let [attrs {:stroke-style (:stroke-style shape) - :stroke-alignment (:stroke-alignment shape) - :stroke-width (:stroke-width shape) - :stroke-color (:stroke-color shape) - :stroke-color-ref-id (:stroke-color-ref-id shape) - :stroke-color-ref-file (:stroke-color-ref-file shape) - :stroke-opacity (:stroke-opacity shape) - :stroke-color-gradient (:stroke-color-gradient shape) - :stroke-cap-start (:stroke-cap-start shape) - :stroke-cap-end (:stroke-cap-end shape)} + (-> data + (update :pages-index d/update-vals update-container) + (update :components d/update-vals update-container)))) - clean-attrs (d/without-nils attrs)] - (cond-> shape - (d/not-empty? clean-attrs) - (assoc :strokes [clean-attrs])))) -;; Add strokes to shapes -(defmethod migrate 15 - [data] - (letfn [(update-object [_ object] - (cond-> object - (and (not (= :text (:type object))) (nil? (:strokes object))) - (set-strokes))) - - (update-page [_ page] - (update page :objects #(d/mapm update-object %)))] - (update data :pages-index #(d/mapm update-page %)))) - -;; Add fills and strokes to components +(defmethod migrate 15 [data] data) +;; Add fills and strokes (defmethod migrate 16 [data] - (letfn [(update-object [_ object] - (cond-> object - (and (not (= :text (:type object))) (nil? (:strokes object))) - (set-strokes) + (letfn [(assign-fills [shape] + (let [attrs {:fill-color (:fill-color shape) + :fill-color-gradient (:fill-color-gradient shape) + :fill-color-ref-file (:fill-color-ref-file shape) + :fill-color-ref-id (:fill-color-ref-id shape) + :fill-opacity (:fill-opacity shape)} + clean-attrs (d/without-nils attrs)] + (cond-> shape + (d/not-empty? clean-attrs) + (assoc :fills [clean-attrs])))) + + (assign-strokes [shape] + (let [attrs {:stroke-style (:stroke-style shape) + :stroke-alignment (:stroke-alignment shape) + :stroke-width (:stroke-width shape) + :stroke-color (:stroke-color shape) + :stroke-color-ref-id (:stroke-color-ref-id shape) + :stroke-color-ref-file (:stroke-color-ref-file shape) + :stroke-opacity (:stroke-opacity shape) + :stroke-color-gradient (:stroke-color-gradient shape) + :stroke-cap-start (:stroke-cap-start shape) + :stroke-cap-end (:stroke-cap-end shape)} + clean-attrs (d/without-nils attrs)] + (cond-> shape + (d/not-empty? clean-attrs) + (assoc :strokes [clean-attrs])))) + + (update-object [object] + (cond-> object + (and (not (cph/text-shape? object)) + (not (contains? object :strokes))) + (assign-strokes) + + (and (not (cph/text-shape? object)) + (not (contains? object :fills))) + (assign-fills))) + + (update-container [container] + (update container :objects d/update-vals update-object))] - (and (not (= :text (:type object))) (nil? (:fills object))) - (set-fills))) - (update-container [_ container] - (update container :objects #(d/mapm update-object %)))] (-> data - (update :components #(d/mapm update-container %))))) + (update :pages-index d/update-vals update-container) + (update :components d/update-vals update-container)))) + +(defmethod migrate 17 + [data] + (letfn [(affected-object? [object] + (and (cph/image-shape? object) + (some? (:fills object)) + (= 1 (count (:fills object))) + (some? (:fill-color object)) + (some? (:fill-opacity object)) + (let [color-old (str/upper (:fill-color object)) + color-new (str/upper (get-in object [:fills 0 :fill-color])) + opacity-old (:fill-opacity object) + opacity-new (get-in object [:fills 0 :fill-opacity])] + (and (= color-old color-new) + (or (= "#B1B2B5" color-old) + (= "#7B7D85" color-old)) + (= 1 opacity-old opacity-new))))) + + (update-object [object] + (cond-> object + (affected-object? object) + (assoc :fills []))) + + (update-container [container] + (update container :objects d/update-vals update-object))] + + (-> data + (update :pages-index d/update-vals update-container) + (update :components d/update-vals update-container)))) + +;; TODO: pending to do a migration for delete already not used fill +;; and stroke props. This should be done for >1.14.x version.