From 494c585e2fc50b0b186d13d7721695609a0b70d9 Mon Sep 17 00:00:00 2001
From: Andrey Antukh <niwi@niwi.nz>
Date: Mon, 10 Jul 2023 13:32:20 +0200
Subject: [PATCH] :sparkles: Make builtin templates download ondemand if cache
is not present
---
backend/src/app/main.clj | 7 +-
backend/src/app/rpc/commands/binfile.clj | 8 +--
backend/src/app/rpc/commands/management.clj | 53 ++++++++-------
backend/src/app/setup.clj | 2 +-
backend/src/app/setup/builtin_templates.clj | 72 ---------------------
backend/src/app/setup/templates.clj | 65 +++++++++++++++++++
backend/test/backend_tests/helpers.clj | 4 +-
7 files changed, 103 insertions(+), 108 deletions(-)
delete mode 100644 backend/src/app/setup/builtin_templates.clj
create mode 100644 backend/src/app/setup/templates.clj
diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj
index 249adc019..6670da450 100644
--- a/backend/src/app/main.clj
+++ b/backend/src/app/main.clj
@@ -29,6 +29,7 @@
[app.redis :as-alias rds]
[app.rpc :as-alias rpc]
[app.rpc.doc :as-alias rpc.doc]
+ [app.setup :as-alias setup]
[app.srepl :as-alias srepl]
[app.storage :as-alias sto]
[app.storage.fs :as-alias sto.fs]
@@ -322,11 +323,10 @@
::rpc/climit (ig/ref ::rpc/climit)
::rpc/rlimit (ig/ref ::rpc/rlimit)
-
+ ::setup/templates (ig/ref ::setup/templates)
::props (ig/ref :app.setup/props)
:pool (ig/ref ::db/pool)
- :templates (ig/ref :app.setup/builtin-templates)
}
:app.rpc.doc/routes
@@ -400,8 +400,7 @@
{::srepl/port (cf/get :prepl-port 6063)
::srepl/host (cf/get :prepl-host "localhost")}
- :app.setup/builtin-templates
- {::http.client/client (ig/ref ::http.client/client)}
+ ::setup/templates {}
:app.setup/props
{::db/pool (ig/ref ::db/pool)
diff --git a/backend/src/app/rpc/commands/binfile.clj b/backend/src/app/rpc/commands/binfile.clj
index e485fa44b..b039b60a7 100644
--- a/backend/src/app/rpc/commands/binfile.clj
+++ b/backend/src/app/rpc/commands/binfile.clj
@@ -592,7 +592,7 @@
(let [options (-> options
(assoc ::section section)
(assoc ::input input)
- (assoc :conn conn))]
+ (assoc ::db/conn conn))]
(binding [*options* options]
(read-section options))))
[:v1/metadata :v1/files :v1/rels :v1/sobjects])
@@ -620,7 +620,7 @@
(update :components pmap-wrap))))
(defmethod read-section :v1/files
- [{:keys [conn ::input ::migrate? ::project-id ::timestamp ::overwrite?]}]
+ [{:keys [::db/conn ::input ::migrate? ::project-id ::timestamp ::overwrite?]}]
(doseq [expected-file-id (-> *state* deref :files)]
(let [file (read-obj! input)
media' (read-obj! input)
@@ -678,7 +678,7 @@
(db/delete! conn :file-thumbnail {:file-id file-id'})))))))
(defmethod read-section :v1/rels
- [{:keys [conn ::input ::timestamp]}]
+ [{:keys [::db/conn ::input ::timestamp]}]
(let [rels (read-obj! input)]
;; Insert all file relations
(doseq [rel rels]
@@ -693,7 +693,7 @@
(db/insert! conn :file-library-rel rel)))))
(defmethod read-section :v1/sobjects
- [{:keys [::sto/storage conn ::input ::overwrite?]}]
+ [{:keys [::sto/storage ::db/conn ::input ::overwrite?]}]
(let [storage (media/configure-assets-storage storage)
ids (read-obj! input)]
diff --git a/backend/src/app/rpc/commands/management.clj b/backend/src/app/rpc/commands/management.clj
index 871d89ffe..23dbd1358 100644
--- a/backend/src/app/rpc/commands/management.clj
+++ b/backend/src/app/rpc/commands/management.clj
@@ -10,6 +10,7 @@
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.pages.migrations :as pmg]
+ [app.common.schema :as sm]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.db :as db]
@@ -20,6 +21,8 @@
[app.rpc.commands.projects :as proj]
[app.rpc.commands.teams :as teams :refer [create-project-role create-project]]
[app.rpc.doc :as-alias doc]
+ [app.setup :as-alias setup]
+ [app.setup.templates :as tmpl]
[app.util.blob :as blob]
[app.util.pointer-map :as pmap]
[app.util.services :as sv]
@@ -361,7 +364,6 @@
nil))
-
(s/def ::move-project
(s/keys :req [::rpc/profile-id]
:req-un [::team-id ::project-id]))
@@ -376,41 +378,42 @@
;; --- COMMAND: Clone Template
-(declare clone-template)
-
-(s/def ::template-id ::us/not-empty-string)
-(s/def ::clone-template
- (s/keys :req [::rpc/profile-id]
- :req-un [::project-id ::template-id]))
-
-(sv/defmethod ::clone-template
- "Clone into the specified project the template by its id."
- {::doc/added "1.16"
- ::webhooks/event? true}
- [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
- (db/with-atomic [conn pool]
- (-> (assoc cfg :conn conn)
- (clone-template (assoc params :profile-id profile-id)))))
-
-(defn- clone-template
- [{:keys [conn templates] :as cfg} {:keys [profile-id template-id project-id]}]
- (let [template (d/seek #(= (:id %) template-id) templates)
+(defn- clone-template!
+ [{:keys [::db/conn] :as cfg} {:keys [profile-id template-id project-id]}]
+ (let [template (tmpl/get-template-stream cfg template-id)
project (db/get-by-id conn :project project-id {:columns [:id :team-id]})]
- (teams/check-edition-permissions! conn profile-id (:team-id project))
-
(when-not template
(ex/raise :type :not-found
:code :template-not-found
:hint "template not found"))
+ (teams/check-edition-permissions! conn profile-id (:team-id project))
+
(-> cfg
- (assoc ::binfile/input (:path template))
+ ;; FIXME: maybe reuse the conn instead of creating more
+ ;; connections in the import process?
+ (dissoc ::db/conn)
+ (assoc ::binfile/input template)
(assoc ::binfile/project-id (:id project))
(assoc ::binfile/ignore-index-errors? true)
(assoc ::binfile/migrate? true)
(binfile/import!))))
+(def schema:clone-template
+ [:map {:title "clone-template"}
+ [:project-id ::sm/uuid]
+ [:template-id ::sm/word-string]])
+
+(sv/defmethod ::clone-template
+ "Clone into the specified project the template by its id."
+ {::doc/added "1.16"
+ ::webhooks/event? true
+ ::sm/params schema:clone-template}
+ [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
+ (db/with-atomic [conn pool]
+ (-> (assoc cfg ::db/conn conn)
+ (clone-template! (assoc params :profile-id profile-id)))))
;; --- COMMAND: Get list of builtin templates
@@ -420,9 +423,9 @@
{::doc/added "1.10"
::doc/deprecated "1.19"}
[cfg _params]
- (mapv #(select-keys % [:id :name :thumbnail-uri]) (:templates cfg)))
+ (mapv #(select-keys % [:id :name :thumbnail-uri]) (::setup/templates cfg)))
(sv/defmethod ::get-builtin-templates
{::doc/added "1.19"}
[cfg _params]
- (mapv #(select-keys % [:id :name :thumbnail-uri]) (:templates cfg)))
+ (mapv #(select-keys % [:id :name :thumbnail-uri]) (::setup/templates cfg)))
diff --git a/backend/src/app/setup.clj b/backend/src/app/setup.clj
index 856853fc8..8e889e2b4 100644
--- a/backend/src/app/setup.clj
+++ b/backend/src/app/setup.clj
@@ -12,8 +12,8 @@
[app.common.uuid :as uuid]
[app.db :as db]
[app.main :as-alias main]
- [app.setup.builtin-templates]
[app.setup.keys :as keys]
+ [app.setup.templates]
[buddy.core.codecs :as bc]
[buddy.core.nonce :as bn]
[clojure.spec.alpha :as s]
diff --git a/backend/src/app/setup/builtin_templates.clj b/backend/src/app/setup/builtin_templates.clj
deleted file mode 100644
index 23b6875aa..000000000
--- a/backend/src/app/setup/builtin_templates.clj
+++ /dev/null
@@ -1,72 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS INC
-
-(ns app.setup.builtin-templates
- "A service/module that is responsible for download, load & internally
- expose a set of builtin penpot file templates."
- (:require
- [app.common.logging :as l]
- [app.common.spec :as us]
- [app.http.client :as http]
- [clojure.edn :as edn]
- [clojure.java.io :as io]
- [clojure.spec.alpha :as s]
- [datoteka.fs :as fs]
- [integrant.core :as ig]))
-
-(declare download-all!)
-
-(s/def ::id ::us/not-empty-string)
-(s/def ::name ::us/not-empty-string)
-(s/def ::thumbnail-uri ::us/not-empty-string)
-(s/def ::file-uri ::us/not-empty-string)
-(s/def ::path fs/path?)
-
-(s/def ::template
- (s/keys :req-un [::id ::name ::thumbnail-uri ::file-uri]
- :opt-un [::path]))
-
-(defmethod ig/pre-init-spec :app.setup/builtin-templates [_]
- (s/keys :req [::http/client]))
-
-(defmethod ig/init-key :app.setup/builtin-templates
- [_ cfg]
- (let [presets (-> "app/onboarding.edn" io/resource slurp edn/read-string)]
- (l/info :hint "loading template files" :total (count presets))
- (let [result (download-all! cfg presets)]
- (us/conform (s/coll-of ::template) result))))
-
-(defn- download-preset!
- [cfg {:keys [path file-uri] :as preset}]
- (let [response (http/req! cfg
- {:method :get
- :uri file-uri}
- {:response-type :input-stream
- :sync? true})]
- (us/verify! (= 200 (:status response)) "unexpected response found on fetching preset")
- (with-open [output (io/output-stream path)]
- (with-open [input (io/input-stream (:body response))]
- (io/copy input output)))))
-
-(defn- download-all!
- "Download presets to the default directory, if preset is already
- downloaded, no action will be performed."
- [cfg presets]
- (let [dest (fs/join fs/*cwd* "builtin-templates")]
- (when-not (fs/exists? dest)
- (fs/create-dir dest))
-
- (doall
- (map (fn [item]
- (let [path (fs/join dest (:id item))
- item (assoc item :path path)]
- (if (fs/exists? path)
- (l/trace :hint "template file already present" :id (:id item))
- (do
- (l/trace :hint "downloading template file" :id (:id item) :dest (str path))
- (download-preset! cfg item)))
- item))
- presets))))
diff --git a/backend/src/app/setup/templates.clj b/backend/src/app/setup/templates.clj
new file mode 100644
index 000000000..98afd340c
--- /dev/null
+++ b/backend/src/app/setup/templates.clj
@@ -0,0 +1,65 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.setup.templates
+ "A service/module that is responsible for download, load & internally
+ expose a set of builtin penpot file templates."
+ (:require
+ [app.common.data :as d]
+ [app.common.data.macros :as dm]
+ [app.common.logging :as l]
+ [app.common.schema :as sm]
+ [app.http.client :as http]
+ [app.setup :as-alias setup]
+ [clojure.edn :as edn]
+ [clojure.java.io :as io]
+ [datoteka.fs :as fs]
+ [integrant.core :as ig]))
+
+(def ^:private schema:template
+ [:map {:title "Template"}
+ [:id ::sm/word-string]
+ [:name ::sm/word-string]
+ [:thumbnail-uri ::sm/word-string]
+ [:file-uri ::sm/word-string]])
+
+(def ^:private schema:templates
+ [:vector schema:template])
+
+(defmethod ig/init-key ::setup/templates
+ [_ _]
+ (let [templates (-> "app/onboarding.edn" io/resource slurp edn/read-string)
+ dest (fs/join fs/*cwd* "builtin-templates")]
+
+ (dm/verify!
+ "expected a valid templates file"
+ (sm/valid? schema:templates templates))
+
+ (doseq [{:keys [id path] :as template} templates]
+ (let [path (or path (fs/join dest id))]
+ (if (fs/exists? path)
+ (l/debug :hint "template file" :id id :state "present" :path (dm/str path))
+ (l/debug :hint "template file" :id id :state "absent"))))
+
+ templates))
+
+(defn get-template-stream
+ [cfg template-id]
+ (when-let [template (d/seek #(= (:id %) template-id)
+ (::setup/templates cfg))]
+ (let [dest (fs/join fs/*cwd* "builtin-templates")
+ path (or (:path template) (fs/join dest template-id))]
+ (if (fs/exists? path)
+ (io/input-stream path)
+ (let [resp (http/req! cfg
+ {:method :get :uri (:file-uri template)}
+ {:response-type :input-stream :sync? true})]
+
+ (dm/verify!
+ "unexpected response found on fetching template"
+ (= 200 (:status resp)))
+
+ (io/input-stream (:body resp)))))))
diff --git a/backend/test/backend_tests/helpers.clj b/backend/test/backend_tests/helpers.clj
index d2edaeec9..dbd7f464d 100644
--- a/backend/test/backend_tests/helpers.clj
+++ b/backend/test/backend_tests/helpers.clj
@@ -128,7 +128,7 @@
(assoc-in [::db/pool ::db/uri] (:database-uri config))
(assoc-in [::db/pool ::db/username] (:database-username config))
(assoc-in [::db/pool ::db/password] (:database-password config))
- (assoc-in [:app.rpc/methods :templates] templates)
+ (assoc-in [:app.rpc/methods :app.setup/templates] templates)
(dissoc :app.srepl/server
:app.http/server
:app.http/router
@@ -136,7 +136,7 @@
:app.auth.oidc/gitlab-provider
:app.auth.oidc/github-provider
:app.auth.oidc/generic-provider
- :app.setup/builtin-templates
+ :app.setup/templates
:app.auth.oidc/routes
:app.worker/monitor
:app.http.oauth/handler