0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-03-19 19:21:23 -05:00

Make builtin templates download ondemand if cache is not present

This commit is contained in:
Andrey Antukh 2023-07-10 13:32:20 +02:00
parent 02b41abaf8
commit 494c585e2f
7 changed files with 103 additions and 108 deletions

View file

@ -29,6 +29,7 @@
[app.redis :as-alias rds]
[app.rpc :as-alias rpc]
[app.rpc.doc :as-alias rpc.doc]
[app.setup :as-alias setup]
[app.srepl :as-alias srepl]
[app.storage :as-alias sto]
[app.storage.fs :as-alias sto.fs]
@ -322,11 +323,10 @@
::rpc/climit (ig/ref ::rpc/climit)
::rpc/rlimit (ig/ref ::rpc/rlimit)
::setup/templates (ig/ref ::setup/templates)
::props (ig/ref :app.setup/props)
:pool (ig/ref ::db/pool)
:templates (ig/ref :app.setup/builtin-templates)
}
:app.rpc.doc/routes
@ -400,8 +400,7 @@
{::srepl/port (cf/get :prepl-port 6063)
::srepl/host (cf/get :prepl-host "localhost")}
:app.setup/builtin-templates
{::http.client/client (ig/ref ::http.client/client)}
::setup/templates {}
:app.setup/props
{::db/pool (ig/ref ::db/pool)

View file

@ -592,7 +592,7 @@
(let [options (-> options
(assoc ::section section)
(assoc ::input input)
(assoc :conn conn))]
(assoc ::db/conn conn))]
(binding [*options* options]
(read-section options))))
[:v1/metadata :v1/files :v1/rels :v1/sobjects])
@ -620,7 +620,7 @@
(update :components pmap-wrap))))
(defmethod read-section :v1/files
[{:keys [conn ::input ::migrate? ::project-id ::timestamp ::overwrite?]}]
[{:keys [::db/conn ::input ::migrate? ::project-id ::timestamp ::overwrite?]}]
(doseq [expected-file-id (-> *state* deref :files)]
(let [file (read-obj! input)
media' (read-obj! input)
@ -678,7 +678,7 @@
(db/delete! conn :file-thumbnail {:file-id file-id'})))))))
(defmethod read-section :v1/rels
[{:keys [conn ::input ::timestamp]}]
[{:keys [::db/conn ::input ::timestamp]}]
(let [rels (read-obj! input)]
;; Insert all file relations
(doseq [rel rels]
@ -693,7 +693,7 @@
(db/insert! conn :file-library-rel rel)))))
(defmethod read-section :v1/sobjects
[{:keys [::sto/storage conn ::input ::overwrite?]}]
[{:keys [::sto/storage ::db/conn ::input ::overwrite?]}]
(let [storage (media/configure-assets-storage storage)
ids (read-obj! input)]

View file

@ -10,6 +10,7 @@
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.pages.migrations :as pmg]
[app.common.schema :as sm]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.db :as db]
@ -20,6 +21,8 @@
[app.rpc.commands.projects :as proj]
[app.rpc.commands.teams :as teams :refer [create-project-role create-project]]
[app.rpc.doc :as-alias doc]
[app.setup :as-alias setup]
[app.setup.templates :as tmpl]
[app.util.blob :as blob]
[app.util.pointer-map :as pmap]
[app.util.services :as sv]
@ -361,7 +364,6 @@
nil))
(s/def ::move-project
(s/keys :req [::rpc/profile-id]
:req-un [::team-id ::project-id]))
@ -376,41 +378,42 @@
;; --- COMMAND: Clone Template
(declare clone-template)
(s/def ::template-id ::us/not-empty-string)
(s/def ::clone-template
(s/keys :req [::rpc/profile-id]
:req-un [::project-id ::template-id]))
(sv/defmethod ::clone-template
"Clone into the specified project the template by its id."
{::doc/added "1.16"
::webhooks/event? true}
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
(db/with-atomic [conn pool]
(-> (assoc cfg :conn conn)
(clone-template (assoc params :profile-id profile-id)))))
(defn- clone-template
[{:keys [conn templates] :as cfg} {:keys [profile-id template-id project-id]}]
(let [template (d/seek #(= (:id %) template-id) templates)
(defn- clone-template!
[{:keys [::db/conn] :as cfg} {:keys [profile-id template-id project-id]}]
(let [template (tmpl/get-template-stream cfg template-id)
project (db/get-by-id conn :project project-id {:columns [:id :team-id]})]
(teams/check-edition-permissions! conn profile-id (:team-id project))
(when-not template
(ex/raise :type :not-found
:code :template-not-found
:hint "template not found"))
(teams/check-edition-permissions! conn profile-id (:team-id project))
(-> cfg
(assoc ::binfile/input (:path template))
;; FIXME: maybe reuse the conn instead of creating more
;; connections in the import process?
(dissoc ::db/conn)
(assoc ::binfile/input template)
(assoc ::binfile/project-id (:id project))
(assoc ::binfile/ignore-index-errors? true)
(assoc ::binfile/migrate? true)
(binfile/import!))))
(def schema:clone-template
[:map {:title "clone-template"}
[:project-id ::sm/uuid]
[:template-id ::sm/word-string]])
(sv/defmethod ::clone-template
"Clone into the specified project the template by its id."
{::doc/added "1.16"
::webhooks/event? true
::sm/params schema:clone-template}
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id] :as params}]
(db/with-atomic [conn pool]
(-> (assoc cfg ::db/conn conn)
(clone-template! (assoc params :profile-id profile-id)))))
;; --- COMMAND: Get list of builtin templates
@ -420,9 +423,9 @@
{::doc/added "1.10"
::doc/deprecated "1.19"}
[cfg _params]
(mapv #(select-keys % [:id :name :thumbnail-uri]) (:templates cfg)))
(mapv #(select-keys % [:id :name :thumbnail-uri]) (::setup/templates cfg)))
(sv/defmethod ::get-builtin-templates
{::doc/added "1.19"}
[cfg _params]
(mapv #(select-keys % [:id :name :thumbnail-uri]) (:templates cfg)))
(mapv #(select-keys % [:id :name :thumbnail-uri]) (::setup/templates cfg)))

View file

@ -12,8 +12,8 @@
[app.common.uuid :as uuid]
[app.db :as db]
[app.main :as-alias main]
[app.setup.builtin-templates]
[app.setup.keys :as keys]
[app.setup.templates]
[buddy.core.codecs :as bc]
[buddy.core.nonce :as bn]
[clojure.spec.alpha :as s]

View file

@ -1,72 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC
(ns app.setup.builtin-templates
"A service/module that is responsible for download, load & internally
expose a set of builtin penpot file templates."
(:require
[app.common.logging :as l]
[app.common.spec :as us]
[app.http.client :as http]
[clojure.edn :as edn]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[datoteka.fs :as fs]
[integrant.core :as ig]))
(declare download-all!)
(s/def ::id ::us/not-empty-string)
(s/def ::name ::us/not-empty-string)
(s/def ::thumbnail-uri ::us/not-empty-string)
(s/def ::file-uri ::us/not-empty-string)
(s/def ::path fs/path?)
(s/def ::template
(s/keys :req-un [::id ::name ::thumbnail-uri ::file-uri]
:opt-un [::path]))
(defmethod ig/pre-init-spec :app.setup/builtin-templates [_]
(s/keys :req [::http/client]))
(defmethod ig/init-key :app.setup/builtin-templates
[_ cfg]
(let [presets (-> "app/onboarding.edn" io/resource slurp edn/read-string)]
(l/info :hint "loading template files" :total (count presets))
(let [result (download-all! cfg presets)]
(us/conform (s/coll-of ::template) result))))
(defn- download-preset!
[cfg {:keys [path file-uri] :as preset}]
(let [response (http/req! cfg
{:method :get
:uri file-uri}
{:response-type :input-stream
:sync? true})]
(us/verify! (= 200 (:status response)) "unexpected response found on fetching preset")
(with-open [output (io/output-stream path)]
(with-open [input (io/input-stream (:body response))]
(io/copy input output)))))
(defn- download-all!
"Download presets to the default directory, if preset is already
downloaded, no action will be performed."
[cfg presets]
(let [dest (fs/join fs/*cwd* "builtin-templates")]
(when-not (fs/exists? dest)
(fs/create-dir dest))
(doall
(map (fn [item]
(let [path (fs/join dest (:id item))
item (assoc item :path path)]
(if (fs/exists? path)
(l/trace :hint "template file already present" :id (:id item))
(do
(l/trace :hint "downloading template file" :id (:id item) :dest (str path))
(download-preset! cfg item)))
item))
presets))))

View file

@ -0,0 +1,65 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC
(ns app.setup.templates
"A service/module that is responsible for download, load & internally
expose a set of builtin penpot file templates."
(:require
[app.common.data :as d]
[app.common.data.macros :as dm]
[app.common.logging :as l]
[app.common.schema :as sm]
[app.http.client :as http]
[app.setup :as-alias setup]
[clojure.edn :as edn]
[clojure.java.io :as io]
[datoteka.fs :as fs]
[integrant.core :as ig]))
(def ^:private schema:template
[:map {:title "Template"}
[:id ::sm/word-string]
[:name ::sm/word-string]
[:thumbnail-uri ::sm/word-string]
[:file-uri ::sm/word-string]])
(def ^:private schema:templates
[:vector schema:template])
(defmethod ig/init-key ::setup/templates
[_ _]
(let [templates (-> "app/onboarding.edn" io/resource slurp edn/read-string)
dest (fs/join fs/*cwd* "builtin-templates")]
(dm/verify!
"expected a valid templates file"
(sm/valid? schema:templates templates))
(doseq [{:keys [id path] :as template} templates]
(let [path (or path (fs/join dest id))]
(if (fs/exists? path)
(l/debug :hint "template file" :id id :state "present" :path (dm/str path))
(l/debug :hint "template file" :id id :state "absent"))))
templates))
(defn get-template-stream
[cfg template-id]
(when-let [template (d/seek #(= (:id %) template-id)
(::setup/templates cfg))]
(let [dest (fs/join fs/*cwd* "builtin-templates")
path (or (:path template) (fs/join dest template-id))]
(if (fs/exists? path)
(io/input-stream path)
(let [resp (http/req! cfg
{:method :get :uri (:file-uri template)}
{:response-type :input-stream :sync? true})]
(dm/verify!
"unexpected response found on fetching template"
(= 200 (:status resp)))
(io/input-stream (:body resp)))))))

View file

@ -128,7 +128,7 @@
(assoc-in [::db/pool ::db/uri] (:database-uri config))
(assoc-in [::db/pool ::db/username] (:database-username config))
(assoc-in [::db/pool ::db/password] (:database-password config))
(assoc-in [:app.rpc/methods :templates] templates)
(assoc-in [:app.rpc/methods :app.setup/templates] templates)
(dissoc :app.srepl/server
:app.http/server
:app.http/router
@ -136,7 +136,7 @@
:app.auth.oidc/gitlab-provider
:app.auth.oidc/github-provider
:app.auth.oidc/generic-provider
:app.setup/builtin-templates
:app.setup/templates
:app.auth.oidc/routes
:app.worker/monitor
:app.http.oauth/handler