Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-01-25 07:58:49 -05:00
Code Issues Activity

🎉 Add onboarding data to the database.

Browse source
This commit is contained in:
Andrey Antukh 2021-02-24 16:22:54 +01:00 • committed by Alonso Torres
parent 85781c5b7f
commit e3727aaefe
10 changed files with 183 additions and 179 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
backend/src/app/config.clj
View file

@ -76,8 +76,8 @@
:ldap-attrs-fullname "cn"
:ldap-attrs-photo "jpegPhoto"
;; :initial-data-file "resources/initial-data.json"
;; :initial-data-project-name "Penpot Oboarding"
;; a server prop key where initial project is stored.
:initial-project-skey "initial-project"
})
(s/def ::allow-demo-users ::us/boolean)
@ -103,8 +103,7 @@
(s/def ::http-session-idle-max-age ::dt/duration)
(s/def ::http-session-updater-batch-max-age ::dt/duration)
(s/def ::http-session-updater-batch-max-size ::us/integer)
(s/def ::initial-data-file ::us/string)
(s/def ::initial-data-project-name ::us/string)
(s/def ::initial-project-skey ::us/string)
(s/def ::ldap-attrs-email ::us/string)
(s/def ::ldap-attrs-fullname ::us/string)
(s/def ::ldap-attrs-photo ::us/string)
@ -161,8 +160,8 @@
::database-username
::default-blob-version
::error-report-webhook
::feedback-enabled
::feedback-destination
::feedback-enabled
::github-client-id
::github-client-secret
::gitlab-base-uri
@ -170,33 +169,37 @@
::gitlab-client-secret
::google-client-id
::google-client-secret
::host
::http-server-port
::http-session-idle-max-age
::http-session-updater-batch-max-age
::http-session-updater-batch-max-size
::http-session-idle-max-age
::host
::ldap-attrs-username
::initial-project-skey
::ldap-attrs-email
::ldap-attrs-fullname
::ldap-attrs-photo
::ldap-attrs-username
::ldap-base-dn
::ldap-bind-dn
::ldap-bind-password
::ldap-base-dn
::ldap-host
::ldap-port
::ldap-ssl
::ldap-starttls
::ldap-user-query
::public-uri
::profile-complaint-threshold
::local-assets-uri
::loggers-loki-uri
::loggers-zmq-uri
::profile-bounce-max-age
::profile-bounce-threshold
::profile-complaint-max-age
::profile-bounce-max-age
::profile-complaint-threshold
::public-uri
::redis-uri
::registration-domain-whitelist
::registration-enabled
::rlimits-password
::rlimits-image
::rlimits-password
::smtp-default-from
::smtp-default-reply-to
::smtp-enabled
@ -206,23 +209,18 @@
::smtp-ssl
::smtp-tls
::smtp-username
::storage-backend
::storage-fs-directory
::srepl-host
::srepl-port
::local-assets-uri
::loggers-loki-uri
::loggers-zmq-uri
::storage-backend
::storage-fs-directory
::storage-s3-bucket
::storage-s3-region
::telemetry-enabled
::telemetry-with-taiga
::telemetry-server-enabled
::telemetry-server-port
::telemetry-uri
::tenant
::initial-data-file
::initial-data-project-name]))
::telemetry-with-taiga
::tenant]))
(defn- env->config
[env]

5
backend/src/app/db.clj
View file

@ -200,11 +200,6 @@
(sql/insert table params opts)
(assoc opts :return-keys true))))
(defn insert-multi!
[ds table param-list]
(doseq [params param-list]
(insert! ds table params)))
(defn update!
([ds table params where] (update! ds table params where nil))
([ds table params where opts]

117
backend/src/app/db/profile_initial_data.clj
View file

@ -1,117 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
(ns app.db.profile-initial-data
(:require
[app.common.uuid :as uuid]
[app.config :as cfg]
[app.db :as db]
[app.rpc.mutations.projects :as projects]
[app.util.transit :as tr]
[clojure.java.io :as io]
[datoteka.core :as fs]))
(def sql:file
"select * from file where project_id = ?")
(def sql:file-library-rel
"with file_ids as (select id from file where project_id = ?)
select *
from file_library_rel
where file_id in (select id from file_ids)")
(def sql:file-media-object
"with file_ids as (select id from file where project_id = ?)
select *
from file_media_object
where file_id in (select id from file_ids)")
(defn change-ids
"Given a collection and a map from ID to ID. Changes all the `keys` properties
so they point to the new ID existing in `map-ids`"
[map-ids coll keys]
(let [generate-id
(fn [map-ids {:keys [id]}]
(assoc map-ids id (uuid/next)))
remap-key
(fn [obj map-ids key]
(cond-> obj
(contains? obj key)
(assoc key (get map-ids (get obj key) (get obj key)))))
change-id
(fn [map-ids obj]
(reduce #(remap-key %1 map-ids %2) obj keys))
new-map-ids (reduce generate-id map-ids coll)]
[new-map-ids (map (partial change-id new-map-ids) coll)]))
(defn create-initial-data-dump
[conn project-id output-path]
(let [ ;; Retrieve data from templates
opath (fs/path output-path)
file (db/exec! conn [sql:file, project-id])
file-library-rel (db/exec! conn [sql:file-library-rel, project-id])
file-media-object (db/exec! conn [sql:file-media-object, project-id])
data {:file file
:file-library-rel file-library-rel
:file-media-object file-media-object}]
(with-open [output (io/output-stream opath)]
(tr/encode-stream data output)
nil)))
(defn read-initial-data
[path]
(when (fs/exists? path)
(with-open [input (io/input-stream (fs/path path))]
(tr/decode-stream input))))
(defn create-profile-initial-data
([conn profile]
(when-let [initial-data-path (:initial-data-file cfg/config)]
(create-profile-initial-data conn initial-data-path profile)))
([conn file profile]
(when-let [{:keys [file file-library-rel file-media-object]} (read-initial-data file)]
(let [sample-project-name (:initial-data-project-name cfg/config "Penpot Onboarding")
proj (projects/create-project conn {:profile-id (:id profile)
:team-id (:default-team-id profile)
:name sample-project-name})
map-ids {}
;; Create new ID's and change the references
[map-ids file] (change-ids map-ids file #{:id})
[map-ids file-library-rel] (change-ids map-ids file-library-rel #{:file-id :library-file-id})
[_ file-media-object] (change-ids map-ids file-media-object #{:id :file-id :media-id :thumbnail-id})
file (map #(assoc % :project-id (:id proj)) file)
file-profile-rel (map #(array-map :file-id (:id %)
:profile-id (:id profile)
:is-owner true
:is-admin true
:can-edit true)
file)]
(projects/create-project-profile conn {:project-id (:id proj)
:profile-id (:id profile)})
(projects/create-team-project-profile conn {:team-id (:default-team-id profile)
:project-id (:id proj)
:profile-id (:id profile)})
;; Re-insert into the database
(db/insert-multi! conn :file file)
(db/insert-multi! conn :file-profile-rel file-profile-rel)
(db/insert-multi! conn :file-library-rel file-library-rel)
(db/insert-multi! conn :file-media-object file-media-object)))))

3
backend/src/app/migrations.clj
View file

@ -160,6 +160,9 @@
{:name "0049-mod-http-session-table"
:fn (mg/resource "app/migrations/sql/0049-mod-http-session-table.sql")}
{:name "0050-mod-server-prop-table"
:fn (mg/resource "app/migrations/sql/0050-mod-server-prop-table.sql")}
])

4
backend/src/app/migrations/sql/0050-mod-server-prop-table.sql Normal file
View file

@ -0,0 +1,4 @@
ALTER TABLE server_prop
ADD COLUMN preload boolean DEFAULT false;
UPDATE server_prop SET preload = true;

6
backend/src/app/rpc/mutations/demo.clj
View file

@ -5,7 +5,7 @@
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) 2020-2021 UXBOX Labs SL
(ns app.rpc.mutations.demo
"A demo specific mutations."
@ -14,8 +14,8 @@
[app.common.uuid :as uuid]
[app.config :as cfg]
[app.db :as db]
[app.db.profile-initial-data :refer [create-profile-initial-data]]
[app.rpc.mutations.profile :as profile]
[app.setup.initial-data :as sid]
[app.tasks :as tasks]
[app.util.services :as sv]
[buddy.core.codecs :as bc]
@ -48,7 +48,7 @@
(db/with-atomic [conn pool]
(->> (#'profile/create-profile conn params)
(#'profile/create-profile-relations conn)
(create-profile-initial-data conn))
(sid/load-initial-project! conn))
;; Schedule deletion of the demo profile
(tasks/submit! conn {:name "delete-profile"

7
backend/src/app/rpc/mutations/profile.clj
View file

@ -14,12 +14,12 @@
[app.common.uuid :as uuid]
[app.config :as cfg]
[app.db :as db]
[app.db.profile-initial-data :refer [create-profile-initial-data]]
[app.emails :as emails]
[app.media :as media]
[app.rpc.mutations.projects :as projects]
[app.rpc.mutations.teams :as teams]
[app.rpc.queries.profile :as profile]
[app.setup.initial-data :as sid]
[app.storage :as sto]
[app.tasks :as tasks]
[app.util.services :as sv]
@ -81,7 +81,8 @@
(let [profile (->> (create-profile conn params)
(create-profile-relations conn))
profile (assoc profile ::created true)]
(create-profile-initial-data conn profile)
(sid/load-initial-project! conn profile)
(if-let [token (:invitation-token params)]
;; If invitation token comes in params, this is because the
@ -309,7 +310,7 @@
(register-profile [conn params]
(let [profile (->> (create-profile conn params)
(create-profile-relations conn))]
(create-profile-initial-data conn profile)
(sid/load-initial-project! conn profile)
(assoc profile ::created true)))]
(let [profile (profile/retrieve-profile-data-by-email conn email)

19
backend/src/app/setup.clj
View file

@ -37,20 +37,25 @@
(let [key (-> (bn/random-bytes 64)
(bc/bytes->b64u)
(bc/bytes->str))]
(db/exec-one! conn ["insert into server_prop (id, content)
values ('secret-key', ?) on conflict do nothing"
(db/tjson key)])))
(db/insert! conn :server-prop
{:id "secret-key"
:preload true
:content (db/tjson key)}
{:on-conflict-do-nothing true})))
(defn- initialize-instance-id!
[{:keys [conn] :as cfg}]
(let [iid (uuid/random)]
(db/exec-one! conn ["insert into server_prop (id, content)
values ('instance-id', ?::jsonb) on conflict do nothing"
(db/tjson iid)])))
(db/insert! conn :server-prop
{:id "instance-id"
:preload true
:content (db/tjson iid)}
{:on-conflict-do-nothing true})))
(defn- retrieve-all
[{:keys [conn] :as cfg}]
(reduce (fn [acc row]
(assoc acc (keyword (:id row)) (db/decode-transit-pgobject (:content row))))
{}
(db/exec! conn ["select * from server_prop;"])))
(db/query conn :server-prop {:preload true})))

137
backend/src/app/setup/initial_data.clj Normal file
View file

@ -0,0 +1,137 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2021 UXBOX Labs SL
(ns app.setup.initial-data
(:refer-clojure :exclude [load])
(:require
[app.common.uuid :as uuid]
[app.config :as cfg]
[app.db :as db]
[app.rpc.mutations.projects :as projects]
[app.rpc.queries.profile :as profile]))
;; --- DUMP GENERATION
(def sql:file
"select * from file where project_id = ?")
(def sql:file-library-rel
"with file_ids as (select id from file where project_id = ?)
select *
from file_library_rel
where file_id in (select id from file_ids)")
(def sql:file-media-object
"with file_ids as (select id from file where project_id = ?)
select *
from file_media_object
where file_id in (select id from file_ids)")
(defn dump
([system project-id] (dump system project-id nil))
([system project-id {:keys [skey project-name]
:or {project-name "Penpot Onboarding"}}]
(db/with-atomic [conn (:app.db/pool system)]
(let [skey (or skey (cfg/get :initial-project-skey))
file (db/exec! conn [sql:file project-id])
file-library-rel (db/exec! conn [sql:file-library-rel project-id])
file-media-object (db/exec! conn [sql:file-media-object project-id])
data {:project-name project-name
:file file
:file-library-rel file-library-rel
:file-media-object file-media-object}]
(db/delete! conn :server-prop
{:id skey})
(db/insert! conn :server-prop
{:id skey
:preload false
:content (db/tjson data)})
nil))))
;; --- DUMP LOADING
(defn- remap-ids
"Given a collection and a map from ID to ID. Changes all the `keys`
properties so they point to the new ID existing in `map-ids`"
[map-ids coll keys]
(let [generate-id
(fn [map-ids {:keys [id]}]
(assoc map-ids id (uuid/next)))
remap-key
(fn [obj map-ids key]
(cond-> obj
(contains? obj key)
(assoc key (get map-ids (get obj key) (get obj key)))))
change-id
(fn [map-ids obj]
(reduce #(remap-key %1 map-ids %2) obj keys))
new-map-ids (reduce generate-id map-ids coll)]
[new-map-ids (map (partial change-id new-map-ids) coll)]))
(defn- retrieve-data
[conn skey]
(when-let [row (db/exec-one! conn ["select content from server_prop where id = ?" skey])]
(when-let [content (:content row)]
(when (db/pgobject? content)
(db/decode-transit-pgobject content)))))
(defn load-initial-project!
([conn profile] (load-initial-project! conn profile nil))
([conn profile opts]
(let [skey (or (:skey opts) (cfg/get :initial-project-skey))
data (retrieve-data conn skey)]
(when data
(let [project (projects/create-project conn {:profile-id (:id profile)
:team-id (:default-team-id profile)
:name (:project-name data)})
map-ids {}
[map-ids file] (remap-ids map-ids (:file data) #{:id})
[map-ids file-library-rel] (remap-ids map-ids (:file-library-rel data) #{:file-id :library-file-id})
[_ file-media-object] (remap-ids map-ids (:file-media-object data) #{:id :file-id :media-id :thumbnail-id})
file (map #(assoc % :project-id (:id project)) file)
file-profile-rel (map #(array-map :file-id (:id %)
:profile-id (:id profile)
:is-owner true
:is-admin true
:can-edit true)
file)]
(projects/create-project-profile conn {:project-id (:id project)
:profile-id (:id profile)})
(projects/create-team-project-profile conn {:team-id (:default-team-id profile)
:project-id (:id project)
:profile-id (:id profile)})
;; Re-insert into the database
(doseq [params file]
(db/insert! conn :file params))
(doseq [params file-profile-rel]
(db/insert! conn :file-profile-rel params))
(doseq [params file-library-rel]
(db/insert! conn :file-library-rel params))
(doseq [params file-media-object]
(db/insert! conn :file-media-object params)))))))
(defn load
[system {:keys [email] :as opts}]
(db/with-atomic [conn (:app.db/pool system)]
(when-let [profile (profile/retrieve-profile-data-by-email conn email)]
(load-initial-project! conn profile opts)
true)))

22
backend/src/app/srepl/main.clj
View file

@ -6,7 +6,6 @@
[app.common.pages.migrations :as pmg]
[app.config :as cfg]
[app.db :as db]
[app.db.profile-initial-data :as pid]
[app.main :refer [system]]
[app.rpc.queries.profile :as prof]
[app.srepl.dev :as dev]
@ -53,27 +52,6 @@
;; (fn [{:keys [data] :as file}]
;; (update-in data [:pages-index #uuid "878278c0-3ef0-11eb-9d67-8551e7624f43" :objects] dissoc nil))))
(def default-project-id #uuid "5761a890-3b81-11eb-9e7d-556a2f641513")
(defn initial-data-dump
([system file] (initial-data-dump system default-project-id file))
([system project-id path]
(db/with-atomic [conn (:app.db/pool system)]
(pid/create-initial-data-dump conn project-id path))))
(defn load-data-into-user
([system user-email]
(if-let [file (:initial-data-file cfg/config)]
(load-data-into-user system file user-email)
(prn "Data file not found in configuration")))
([system file user-email]
(db/with-atomic [conn (:app.db/pool system)]
(let [profile (prof/retrieve-profile-data-by-email conn user-email)
profile (merge profile (prof/retrieve-additional-data conn (:id profile)))]
(pid/create-profile-initial-data conn file profile)))))
;; Migrate
(defn update-file-data-blob-format