✨ Improve schema validation handling

And properly honor the file-validation flag

backend/src/app/rpc/commands/binfile.clj

@@ -630,10 +630,12 @@
   (pu/with-open [input (zstd-input-stream input)
                  input (io/data-input-stream input)]
     (binding [*state* (volatile! {:media [] :index {}})]
-      (let [team     (teams/get-team options
+      (let [team      (teams/get-team options
                                      :profile-id profile-id
                                      :project-id project-id)
-            features (cfeat/get-team-enabled-features cf/flags team)]
+
+            validate? (contains? cf/flags :file-validation)
+            features  (cfeat/get-team-enabled-features cf/flags team)]
 
         ;; Process all sections
         (run! (fn [section]
@@ -651,13 +653,13 @@
         (doseq [[feature file-id] (-> *state* deref :pending-to-migrate)]
           (case feature
             "components/v2"
-            (features.components-v2/migrate-file! options file-id)
+            (features.components-v2/migrate-file! options file-id
+                                                  :validate? validate?
+                                                  :throw-on-validate? true)
 
             "fdata/shape-data-type"
             nil
 
-            ;; "fdata/shape-data-type"
-            ;; (features.fdata/enable-objects-map
             (ex/raise :type :internal
                       :code :no-migration-defined
                       :hint (str/ffmt "no migation for feature '%' on file importation" feature)
@@ -774,17 +776,16 @@
                                              ;; environments messed up with the version
                                              ;; numbers When this problem is fixed delete
                                              ;; the following line
-                                             (assoc :version 0)
+                                             (assoc :version 22)
                                              (update :pages-index relink-shapes)
                                              (update :components relink-shapes)
                                              (update :media relink-media)
                                              (pmg/migrate-data)
                                              (d/without-nils))))
 
-                         ;; Without providing all libs, here we just
+                         (cond-> (contains? cf/flags :file-validation)
-                         ;; peform a structural file data validation,
+                           (fval/validate-file-schema!))
-                         ;; full referential check is omited.
+
-                         (fval/validate-file!)
                          (postprocess-file)
                          (update :features #(db/create-array conn "text" %))
                          (update :data blob/encode))]

backend/src/app/rpc/commands/files_update.clj

@@ -266,6 +266,7 @@
       ;; Retrieve and return lagged data
       (get-lagged-changes conn params))))
 
+
 (defn- update-file-data
   [conn file changes skip-validate]
   (let [file (update file :data (fn [data]
@@ -278,7 +279,6 @@
         ;; some other way to do general validation
         libs (when (and (contains? cf/flags :file-validation)
                         (not skip-validate))
-               ;; FIXME: we need properly handle pointer-map here ????
                (->> (files/get-file-libraries conn (:id file))
                     (into [file] (map (fn [{:keys [id]}]
                                         (binding [pmap/*load-fn* (partial files/load-pointer conn id)]
@@ -293,7 +293,10 @@
         (update :data cpc/process-changes changes)
 
         ;; If `libs` is defined, then full validation is performed
-        (val/validate-file! libs)
+        (cond-> (and (contains? cf/flags :file-validation)
+                     (not skip-validate))
+          (-> (val/validate-file! libs)
+              (val/validate-file-schema!)))
 
         (cond-> (and (contains? cfeat/*current* "fdata/objects-map")
                      (not (contains? cfeat/*previous* "fdata/objects-map")))

backend/src/app/srepl/components_v2.clj

@@ -238,7 +238,7 @@
           (migrate-team [team-id]
             (try
               (-> (assoc system ::db/rollback rollback?)
-                  (feat/migrate-team! team-id :validate? validate?))
+                  (feat/migrate-team! team-id :validate? validate? :throw-on-validate? (not skip-on-error)))
               (catch Throwable cause
                 (l/err :hint "unexpected error on processing team" :team-id (dm/str team-id) :cause cause))))
 

common/src/app/common/files/defaults.cljc

@@ -6,4 +6,4 @@
 
 (ns app.common.files.defaults)
 
-(def version 35)
+(def version 36)

common/src/app/common/files/migrations.cljc

@@ -651,3 +651,15 @@
       (migrate)
       (assoc :version 35)))
 
+
+(defmethod migrate 36
+  [data]
+  (letfn [(update-container [container]
+            (d/update-when container :objects (fn [objects]
+                                                (if (contains? objects nil)
+                                                  (dissoc objects nil)
+                                                  objects))))]
+    (-> data
+        (update :pages-index update-vals update-container)
+        (update :components update-vals update-container))))
+

common/src/app/common/files/validate.cljc

@@ -442,44 +442,40 @@
   "Get schema explain data for file data"
   (sm/lazy-explainer ::ctf/data))
 
-(defn validate-file!
+(defn validate-file-schema!
-  "Validate file data structure.
+  [{:keys [id data] :as file}]
+  (when-not (valid-fdata? data)
+    (if (some? *errors*)
+      (vswap! *errors* conj
+              {:code :invalid-file-data-structure
+               :hint (str/ffmt "invalid file data structure found on file '%'" id)
+               :file-id id})
+      (ex/raise :type :validation
+                :code :data-validation
+                :hint (str/ffmt "invalid file data structure found on file '%'" id)
+                :file-id id
+                ::sm/explain (get-fdata-explain data))))
+  file)
 
-  If libraries are provided, then a full referential integrity and
+(defn validate-file!
-  semantic coherence check will be performed on all content of the
+  "Validate full referential integrity and semantic coherence on file data.
-  file.
 
   Raises a validation exception on first error found."
+  [{:keys [data] :as file} libraries]
 
-  ([file] (validate-file! file nil))
+  (doseq [page (filter :id (ctpl/pages-seq data))]
-  ([{:keys [data] :as file} libraries]
+    (validate-shape! uuid/zero file page libraries))
-   ;; (when-not (valid-fdata? data)
-   ;;   (if (some? *errors*)
-   ;;     (vswap! *errors* conj
-   ;;             {:code :invalid-file-data-structure
-   ;;              :hint (str/ffmt "invalid file data structure found on file '%'" id)
-   ;;              :file-id id})
-   ;;     (ex/raise :type :validation
-   ;;               :code :data-validation
-   ;;               :hint (str/ffmt "invalid file data found on file '%'" id)
-   ;;               :file-id id
-   ;;               ::sm/explain (get-fdata-explain data))))
 
-   ;; If `libraries` is provided, this means the full file
+  (doseq [component (vals (:components data))]
-   ;; validation is activated so we proceed to execute the
+    (validate-component! component file))
-   ;; validation
-   (when (some? libraries)
-     (doseq [page (filter :id (ctpl/pages-seq data))]
-       (validate-shape! uuid/zero file page libraries))
-     (doseq [component (vals (:components data))]
-       (validate-component! component file)))
 
-   file))
+   file)
 
 (defn validate-file
-  "Validate referencial integrity and semantic coherence of
+  "Validate structure, referencial integrity and semantic coherence of
   all contents of a file. Returns a list of errors."
   [file libraries]
   (binding [*errors* (volatile! [])]
+    (validate-file-schema! file)
     (validate-file! file libraries)
     (deref *errors*)))