Merge pull request #3490 from penpot/niwinz-enhancements-srepl

✨ Add file snapshot related internal functions

backend/resources/app/templates/api-doc.css

@@ -156,7 +156,7 @@ h4 {
 }
 
 .rpc-row-info > .module {
-  width: 120px;
+  width: 150px;
   font-weight: bold;
   border-right: 1px dotted #777;
   text-align: right;

backend/src/app/db.clj

@@ -5,7 +5,7 @@
 ;; Copyright (c) KALEIDOS INC
 
 (ns app.db
-  (:refer-clojure :exclude [get])
+  (:refer-clojure :exclude [get run!])
   (:require
    [app.common.data :as d]
    [app.common.exceptions :as ex]
@@ -391,6 +391,52 @@
   ([^Connection conn ^Savepoint sp]
    (.rollback conn sp)))
 
+(defn tx-run!
+  [cfg f]
+  (cond
+    (connection? cfg)
+    (tx-run! {::conn cfg} f)
+
+    (pool? cfg)
+    (tx-run! {::pool cfg} f)
+
+    (::conn cfg)
+    (let [conn (::conn cfg)
+          sp   (savepoint conn)]
+      (try
+        (let [result (f cfg)]
+          (release! conn sp)
+          result)
+        (catch Throwable cause
+          (rollback! sp)
+          (throw cause))))
+
+    (::pool cfg)
+    (with-atomic [conn (::pool cfg)]
+      (f (assoc cfg ::conn conn)))
+
+    :else
+    (throw (IllegalArgumentException. "invalid arguments"))))
+
+(defn run!
+  [cfg f]
+  (cond
+    (connection? cfg)
+    (run! {::conn cfg} f)
+
+    (pool? cfg)
+    (run! {::pool cfg} f)
+
+    (::conn cfg)
+    (f cfg)
+
+    (::pool cfg)
+    (with-open [^Connection conn (open (::pool cfg))]
+      (f (assoc cfg ::conn conn)))
+
+    :else
+    (throw (IllegalArgumentException. "invalid arguments"))))
+
 (defn interval
   [o]
   (cond

backend/src/app/migrations.clj

@@ -324,6 +324,9 @@
    {:name "0104-mod-file-thumbnail-table"
     :fn (mg/resource "app/migrations/sql/0104-mod-file-thumbnail-table.sql")}
 
+   {:name "0105-mod-file-change-table"
+    :fn (mg/resource "app/migrations/sql/0105-mod-file-change-table.sql")}
+
    ])
 
 (defn apply-migrations!

backend/src/app/migrations/sql/0105-mod-file-change-table.sql

@@ -0,0 +1,9 @@
+ALTER TABLE file_change
+  ADD COLUMN label text NULL;
+
+ALTER TABLE file_change
+  ALTER COLUMN label SET STORAGE external;
+
+CREATE INDEX file_change__label__idx
+    ON file_change (file_id, label)
+ WHERE label is not null;

backend/src/app/rpc.clj

@@ -214,6 +214,7 @@
           'app.rpc.commands.files-share
           'app.rpc.commands.files-temp
           'app.rpc.commands.files-update
+          'app.rpc.commands.files-snapshot
           'app.rpc.commands.files-thumbnails
           'app.rpc.commands.ldap
           'app.rpc.commands.management

backend/src/app/rpc/commands/audit.clj

@@ -68,6 +68,7 @@
    ::climit/key-fn ::rpc/profile-id
    ::sm/params schema:push-audit-events
    ::audit/skip true
+   ::doc/skip true
    ::doc/added "1.17"}
   [{:keys [::db/pool] :as cfg} params]
   (if (or (db/read-only? pool)

backend/src/app/rpc/commands/files_snapshot.clj

@@ -0,0 +1,136 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.rpc.commands.files-snapshot
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.logging :as l]
+   [app.common.schema :as sm]
+   [app.common.uuid :as uuid]
+   [app.config :as cf]
+   [app.db :as db]
+   [app.main :as-alias main]
+   [app.media :as media]
+   [app.rpc :as-alias rpc]
+   [app.rpc.commands.profile :as profile]
+   [app.rpc.doc :as-alias doc]
+   [app.storage :as sto]
+   [app.util.services :as sv]
+   [app.util.time :as dt]))
+
+(defn check-authorized!
+  [{:keys [::db/pool]} profile-id]
+  (when-not (or (= "devenv" (cf/get :host))
+                (let [profile (ex/ignoring (profile/get-profile pool profile-id))
+                      admins  (or (cf/get :admins) #{})]
+                  (contains? admins (:email profile))))
+    (ex/raise :type :authentication
+              :code :authentication-required
+              :hint "only admins allowed")))
+
+(defn get-file-snapshots
+  [{:keys [::db/conn]} {:keys [file-id limit start-at]
+                        :or {limit Long/MAX_VALUE}}]
+  (let [query    (str "select id, label, revn, created_at "
+                      "  from file_change "
+                      " where file_id = ? "
+                      "   and created_at < ? "
+                      "   and label is not null "
+                      "   and data is not null "
+                      " order by created_at desc "
+                      " limit ?")
+        start-at (or start-at (dt/now))
+        limit    (min limit 20)]
+
+    (->> (db/exec! conn [query file-id start-at limit])
+         (mapv (fn [row]
+                 (update row :created-at dt/format-instant :rfc1123))))))
+
+(def ^:private schema:get-file-snapshots
+  [:map [:file-id ::sm/uuid]])
+
+(sv/defmethod ::get-file-snapshots
+  {::doc/added "1.20"
+   ::doc/skip true
+   ::sm/params schema:get-file-snapshots}
+  [cfg {:keys [::rpc/profile-id] :as params}]
+  (check-authorized! cfg profile-id)
+  (db/run! cfg #(get-file-snapshots % params)))
+
+(defn restore-file-snapshot!
+  [{:keys [::db/conn ::sto/storage] :as cfg} {:keys [file-id id]}]
+  (let [storage  (media/configure-assets-storage storage conn)
+        params   {:id id :file-id file-id}
+        options  {:columns [:id :data :revn]}
+        snapshot (db/get* conn :file-change params options)]
+
+    (when (and (some? snapshot)
+               (some? (:data snapshot)))
+
+      (l/debug :hint "snapshot found"
+               :snapshot-id (:id snapshot)
+               :file-id file-id)
+
+      (db/update! conn :file
+                    {:data (:data snapshot)}
+                    {:id file-id})
+
+      ;; clean object thumbnails
+      (let [sql (str "delete from file_object_thumbnail "
+                     " where file_id=? returning media_id")
+            res (db/exec! conn [sql file-id])]
+
+        (doseq [media-id (into #{} (keep :media-id) res)]
+          (sto/del-object! storage media-id)))
+
+      ;; clean object thumbnails
+      (let [sql (str "delete from file_thumbnail "
+                     " where file_id=? returning media_id")
+            res (db/exec! conn [sql file-id])]
+        (doseq [media-id (into #{} (keep :media-id) res)]
+          (sto/del-object! storage media-id)))
+
+      {:id (:id snapshot)})))
+
+(def ^:private schema:restore-file-snapshot
+  [:map
+   [:file-id ::sm/uuid]
+   [:id ::sm/uuid]])
+
+(sv/defmethod ::restore-file-snapshot
+  {::doc/added "1.20"
+   ::doc/skip true
+   ::sm/params schema:restore-file-snapshot}
+  [cfg {:keys [::rpc/profile-id] :as params}]
+  (check-authorized! cfg profile-id)
+  (db/tx-run! cfg #(restore-file-snapshot! % params)))
+
+(defn take-file-snapshot!
+  [{:keys [::db/conn]} {:keys [file-id label]}]
+  (when-let [file (db/get* conn :file {:id file-id})]
+    (let [id    (uuid/next)
+          label (or label (str "Snapshot at " (dt/format-instant (dt/now) :rfc1123)))]
+      (l/debug :hint "persisting file snapshot" :file-id file-id :label label)
+      (db/insert! conn :file-change
+                  {:id id
+                   :revn (:revn file)
+                   :data (:data file)
+                   :features (:features file)
+                   :file-id (:id file)
+                   :label label})
+      {:id id})))
+
+(def ^:private schema:take-file-snapshot
+  [:map [:file-id ::sm/uuid]])
+
+(sv/defmethod ::take-file-snapshot
+  {::doc/added "1.20"
+   ::doc/skip true
+   ::sm/params schema:take-file-snapshot}
+  [cfg {:keys [::rpc/profile-id] :as params}]
+  (check-authorized! cfg profile-id)
+  (db/tx-run! cfg #(take-file-snapshot! % params)))
+

backend/src/app/rpc/doc.clj

@@ -75,6 +75,7 @@
      (->> methods
           (map val)
           (map first)
+          (remove ::skip)
           (map get-context)
           (sort-by (juxt :module :name)))}))
 

backend/src/app/srepl/helpers.clj

@@ -6,6 +6,7 @@
 
 (ns app.srepl.helpers
   "A  main namespace for server repl."
+  (:refer-clojure :exclude [parse-uuid])
   #_:clj-kondo/ignore
   (:require
    [app.auth :refer [derive-password]]
@@ -39,6 +40,26 @@
 (def ^:dynamic *conn*)
 (def ^:dynamic *pool*)
 
+(defn println!
+  [& params]
+  (locking println
+    (apply println params)))
+
+(defn parse-uuid
+  [v]
+  (if (uuid? v)
+    v
+    (d/parse-uuid v)))
+
+(defn resolve-connectable
+  [o]
+  (if (db/connection? o)
+    o
+    (if (db/pool? o)
+      o
+      (or (::db/conn o)
+          (::db/pool o)))))
+
 (defn reset-password!
   "Reset a password to a specific one for a concrete user or all users
   if email is `:all` keyword."
@@ -104,7 +125,7 @@
           (dissoc file :data))))))
 
 (def ^:private sql:retrieve-files-chunk
-  "SELECT id, name, created_at, data FROM file
+  "SELECT id, name, created_at, revn, data FROM file
     WHERE created_at < ? AND deleted_at is NULL
     ORDER BY created_at desc LIMIT ?")
 
@@ -150,11 +171,6 @@
 
     (when (fn? on-end) (on-end))))
 
-(defn- println!
-  [& params]
-  (locking println
-    (apply println params)))
-
 (defn process-files!
   "Apply a function to all files in the database, reading them in
   batches."

backend/src/app/srepl/main.clj

@@ -8,20 +8,25 @@
   "A collection of adhoc fixes scripts."
   #_:clj-kondo/ignore
   (:require
+   [app.common.data :as d]
    [app.common.logging :as l]
    [app.common.pprint :as p]
    [app.common.spec :as us]
+   [app.common.uuid :as uuid]
    [app.db :as db]
+   [app.media :as media]
    [app.rpc.commands.auth :as auth]
    [app.rpc.commands.profile :as profile]
+   [app.rpc.commands.files-snapshot :as fsnap]
    [app.srepl.fixes :as f]
    [app.srepl.helpers :as h]
+   [app.storage :as sto]
    [app.util.blob :as blob]
    [app.util.objects-map :as omap]
    [app.util.pointer-map :as pmap]
    [app.util.time :as dt]
    [app.worker :as wrk]
-   [clojure.pprint :refer [pprint]]
+   [clojure.pprint :refer [pprint print-table]]
    [cuerdas.core :as str]))
 
 (defn print-available-tasks
@@ -101,7 +106,6 @@
         (db/delete! conn :http-session {:profile-id (:id profile)})
         :blocked))))
 
-
 (defn enable-objects-map-feature-on-file!
   [system & {:keys [save? id]}]
   (letfn [(update-file [{:keys [features] :as file}]
@@ -164,3 +168,32 @@
   (alter-var-root var (fn [f]
                         (or (::original (meta f)) f))))
 
+(defn take-file-snapshot!
+  "An internal helper that persist the file snapshot using non-gc
+  collectable file-changes entry."
+  [system & {:keys [file-id label]}]
+  (let [file-id (h/parse-uuid file-id)]
+    (db/tx-run! system
+                (fn [cfg]
+                  (fsnap/take-file-snapshot! cfg {:file-id file-id :label label})))))
+
+(defn restore-file-snapshot!
+  [system & {:keys [file-id id]}]
+  (db/tx-run! system
+              (fn [cfg]
+                (let [file-id (h/parse-uuid file-id)
+                      id      (h/parse-uuid id)]
+
+                  (if (and (uuid? id) (uuid? file-id))
+                    (fsnap/restore-file-snapshot! cfg {:id id :file-id file-id})
+                    (println "=> invalid parameters"))))))
+
+
+(defn list-file-snapshots!
+  [system & {:keys [file-id limit]}]
+  (db/tx-run! system (fn [system]
+                       (let [params {:file-id (h/parse-uuid file-id)
+                                     :limit limit}]
+                         (->> (fsnap/get-file-snapshots system (d/without-nils params))
+                              (print-table [:id :revn :created-at :label]))))))
+

backend/src/app/tasks/file_xlog_gc.clj

@@ -17,7 +17,8 @@
 (def ^:private
   sql:delete-files-xlog
   "delete from file_change
-    where created_at < now() - ?::interval")
+    where created_at < now() - ?::interval
+      and label is NULL")
 
 (defmethod ig/pre-init-spec ::handler [_]
   (s/keys :req [::db/pool]))

frontend/src/debug.cljs

@@ -11,7 +11,9 @@
    [app.common.math :as mth]
    [app.common.transit :as t]
    [app.common.types.file :as ctf]
+   [app.common.uri :as u]
    [app.common.uuid :as uuid]
+   [app.config :as cf]
    [app.main.data.dashboard.shortcuts]
    [app.main.data.viewer.shortcuts]
    [app.main.data.workspace :as dw]
@@ -20,6 +22,7 @@
    [app.main.data.workspace.shortcuts]
    [app.main.store :as st]
    [app.util.dom :as dom]
+   [app.util.http :as http]
    [app.util.object :as obj]
    [app.util.timers :as timers]
    [beicon.core :as rx]
@@ -410,3 +413,47 @@
   [id shape-ref]
   (st/emit! (dw/set-shape-ref id shape-ref)))
 
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; SNAPSHOTS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+
+(defn ^:export list-available-snapshots
+  [file-id]
+  (let [file-id (d/parse-uuid file-id)]
+    (->> (http/send! {:method :post
+                      :uri (u/join cf/public-uri "api/rpc/command/get-file-snapshots")
+                      :body (http/transit-data {:file-id file-id})})
+         (rx/map http/conditional-decode-transit)
+         (rx/map :body)
+         (rx/subs (fn [result]
+                    (let [result (->> result
+                                      (map (fn [row]
+                                             (update row :id str))))]
+                      (js/console.table (clj->js result))))))))
+
+
+(defn ^:export take-snapshot
+  [file-id label]
+  (let [file-id (d/parse-uuid file-id)]
+    (->> (http/send! {:method :post
+                      :uri (u/join cf/public-uri "api/rpc/command/take-file-snapshot")
+                      :body (http/transit-data {:file-id file-id :label label})})
+         (rx/map http/conditional-decode-transit)
+         (rx/map :body)
+         (rx/subs (fn [{:keys [id]}]
+                    (println "Snapshot saved:" (str id)))))))
+
+(defn ^:export restore-snapshot
+  [file-id id]
+  (let [file-id (d/parse-uuid file-id)
+        id      (d/parse-uuid id)]
+    (->> (http/send! {:method :post
+                      :uri (u/join cf/public-uri "api/rpc/command/restore-file-snapshot")
+                      :body (http/transit-data {:file-id file-id :id id})})
+         (rx/map http/conditional-decode-transit)
+         (rx/map :body)
+         (rx/subs (fn [_]
+                    (println "Snapshot restored " id)
+                    #_(.reload js/location))))))