From 451d6c1d7b98fec7f5de547b970496663bf585c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A9s=20Moya?= <andres.moya@kaleidos.net>
Date: Mon, 13 Nov 2023 17:24:43 +0100
Subject: [PATCH] :sparkles: Add optional validation when migrating files to
 components-v2

---
 backend/src/app/features/components_v2.clj | 15 +++++++++------
 backend/src/app/srepl/components_v2.clj    | 20 +++++++++++---------
 2 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj
index cf22b7298..ed244b260 100644
--- a/backend/src/app/features/components_v2.clj
+++ b/backend/src/app/features/components_v2.clj
@@ -13,6 +13,7 @@
    [app.common.files.libraries-helpers :as cflh]
    [app.common.files.migrations :as pmg]
    [app.common.files.shapes-helpers :as cfsh]
+   [app.common.files.validate :as cfv]
    [app.common.geom.point :as gpt]
    [app.common.geom.rect :as grc]
    [app.common.geom.shapes :as gsh]
@@ -623,7 +624,7 @@
         (update fdata :options assoc :components-v2 true)))))
 
 (defn- process-file
-  [{:keys [id] :as file}]
+  [{:keys [id] :as file} & {:keys [validate?]}]
   (let [conn (::db/conn *system*)]
     (binding [pmap/*tracked* (atom {})
               pmap/*load-fn* (partial files/load-pointer conn id)
@@ -659,10 +660,13 @@
                      :revn (:revn file)}
                     {:id (:id file)})
 
+        (when validate?
+          (cfv/validate-file file libs :throw? true))
+
         (dissoc file :data)))))
 
 (defn migrate-file!
-  [system file-id]
+  [system file-id & {:keys [validate?]}]
   (let [tpoint  (dt/tpoint)
         file-id (if (string? file-id)
                   (parse-uuid file-id)
@@ -678,7 +682,7 @@
                       (binding [*system* system]
                         (-> (db/get conn :file {:id file-id})
                             (update :features db/decode-pgarray #{})
-                            (process-file))))))
+                            (process-file :validate? validate?))))))
 
       (finally
         (let [elapsed (tpoint)
@@ -701,9 +705,8 @@
                                    (assoc :elapsed/total-by-file total)
                                    (assoc :processed/files completed)))))))))))
 
-
 (defn migrate-team!
-  [system team-id]
+  [system team-id & {:keys [validate?]}]
   (let [tpoint  (dt/tpoint)
         team-id (if (string? team-id)
                   (parse-uuid team-id)
@@ -737,7 +740,7 @@
                               rows (->> (db/exec! conn [sql team-id])
                                         (map :id))]
 
-                          (run! (partial migrate-file! system) rows)
+                          (run! #(migrate-file! system % :validate? validate?) rows)
                           (some-> *stats* (swap! assoc :current/files (count rows)))
 
                           (let [features (-> features
diff --git a/backend/src/app/srepl/components_v2.clj b/backend/src/app/srepl/components_v2.clj
index af69bb79b..e8ffb1a7f 100644
--- a/backend/src/app/srepl/components_v2.clj
+++ b/backend/src/app/srepl/components_v2.clj
@@ -110,13 +110,14 @@
           (l/dbg :hint "migrate:end" :elapsed elapsed))))))
 
 (defn migrate-files!
-  [{:keys [::db/pool] :as system} & {:keys [chunk-size max-jobs max-items start-at preset rollback skip-on-error]
+  [{:keys [::db/pool] :as system} & {:keys [chunk-size max-jobs max-items start-at preset rollback skip-on-error validate]
                                      :or {chunk-size 10
                                           skip-on-error true
                                           max-jobs 10
                                           max-items Long/MAX_VALUE
                                           preset :shutdown-on-failure
-                                          rollback true}}]
+                                          rollback true
+                                          validate false}}]
   (letfn [(get-chunk [cursor]
             (let [sql  (str/concat
                         "SELECT id, created_at FROM file "
@@ -151,7 +152,7 @@
                     (ps/acquire! feat/*semaphore*)
                     (px/submit! scope (fn []
                                         (-> (assoc system ::db/rollback rollback)
-                                            (feat/migrate-file! file-id)))))
+                                            (feat/migrate-file! file-id :validate? validate)))))
                   (get-candidates))
 
             (p/await! scope))
@@ -171,8 +172,8 @@
 
 (defn migrate-team!
   [{:keys [::db/pool] :as system} team-id
-   & {:keys [rollback skip-on-error]
-      :or {rollback true skip-on-error true}}]
+   & {:keys [rollback skip-on-error validate]
+      :or {rollback true skip-on-error true validate false}}]
   (l/dbg :hint "migrate:start")
 
   (let [total  (get-total-files pool :team-id team-id)
@@ -185,7 +186,7 @@
       (binding [feat/*stats* stats
                 feat/*skip-on-error* skip-on-error]
         (-> (assoc system ::db/rollback rollback)
-            (feat/migrate-team! team-id))
+            (feat/migrate-team! team-id :validate? validate))
 
         (print-stats!
          (-> (deref feat/*stats*)
@@ -203,13 +204,14 @@
 
 (defn migrate-teams!
   [{:keys [::db/pool] :as system}
-   & {:keys [chunk-size max-jobs max-items start-at rollback preset skip-on-error max-time]
+   & {:keys [chunk-size max-jobs max-items start-at rollback preset skip-on-error max-time validate]
       :or {chunk-size 10000
            rollback true
            skip-on-error true
            preset :shutdown-on-failure
            max-jobs Integer/MAX_VALUE
-           max-items Long/MAX_VALUE}}]
+           max-items Long/MAX_VALUE
+           validate false}}]
 
   (letfn [(get-chunk [cursor]
             (let [sql  (str/concat
@@ -232,7 +234,7 @@
           (migrate-team [team-id]
             (try
               (-> (assoc system ::db/rollback rollback)
-                  (feat/migrate-team! team-id))
+                  (feat/migrate-team! team-id :validate? validate))
               (catch Throwable cause
                 (l/err :hint "unexpected error on processing team" :team-id (dm/str team-id) :cause cause))))