From e3727aaefe5396cd4799ce0a3720d4f21c291b1a Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 24 Feb 2021 16:22:54 +0100 Subject: [PATCH] :tada: Add onboarding data to the database. --- backend/src/app/config.clj | 42 +++--- backend/src/app/db.clj | 5 - backend/src/app/db/profile_initial_data.clj | 117 --------------- backend/src/app/migrations.clj | 3 + .../sql/0050-mod-server-prop-table.sql | 4 + backend/src/app/rpc/mutations/demo.clj | 6 +- backend/src/app/rpc/mutations/profile.clj | 7 +- backend/src/app/setup.clj | 19 ++- backend/src/app/setup/initial_data.clj | 137 ++++++++++++++++++ backend/src/app/srepl/main.clj | 22 --- 10 files changed, 183 insertions(+), 179 deletions(-) delete mode 100644 backend/src/app/db/profile_initial_data.clj create mode 100644 backend/src/app/migrations/sql/0050-mod-server-prop-table.sql create mode 100644 backend/src/app/setup/initial_data.clj diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index 742b20727..b78e49fbf 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -76,8 +76,8 @@ :ldap-attrs-fullname "cn" :ldap-attrs-photo "jpegPhoto" - ;; :initial-data-file "resources/initial-data.json" - ;; :initial-data-project-name "Penpot Oboarding" + ;; a server prop key where initial project is stored. + :initial-project-skey "initial-project" }) (s/def ::allow-demo-users ::us/boolean) @@ -103,8 +103,7 @@ (s/def ::http-session-idle-max-age ::dt/duration) (s/def ::http-session-updater-batch-max-age ::dt/duration) (s/def ::http-session-updater-batch-max-size ::us/integer) -(s/def ::initial-data-file ::us/string) -(s/def ::initial-data-project-name ::us/string) +(s/def ::initial-project-skey ::us/string) (s/def ::ldap-attrs-email ::us/string) (s/def ::ldap-attrs-fullname ::us/string) (s/def ::ldap-attrs-photo ::us/string) @@ -161,8 +160,8 @@ ::database-username ::default-blob-version ::error-report-webhook - ::feedback-enabled ::feedback-destination + ::feedback-enabled ::github-client-id ::github-client-secret ::gitlab-base-uri @@ -170,33 +169,37 @@ ::gitlab-client-secret ::google-client-id ::google-client-secret + ::host ::http-server-port + ::http-session-idle-max-age ::http-session-updater-batch-max-age ::http-session-updater-batch-max-size - ::http-session-idle-max-age - ::host - ::ldap-attrs-username + ::initial-project-skey ::ldap-attrs-email ::ldap-attrs-fullname ::ldap-attrs-photo + ::ldap-attrs-username + ::ldap-base-dn ::ldap-bind-dn ::ldap-bind-password - ::ldap-base-dn ::ldap-host ::ldap-port ::ldap-ssl ::ldap-starttls ::ldap-user-query - ::public-uri - ::profile-complaint-threshold + ::local-assets-uri + ::loggers-loki-uri + ::loggers-zmq-uri + ::profile-bounce-max-age ::profile-bounce-threshold ::profile-complaint-max-age - ::profile-bounce-max-age + ::profile-complaint-threshold + ::public-uri ::redis-uri ::registration-domain-whitelist ::registration-enabled - ::rlimits-password ::rlimits-image + ::rlimits-password ::smtp-default-from ::smtp-default-reply-to ::smtp-enabled @@ -206,23 +209,18 @@ ::smtp-ssl ::smtp-tls ::smtp-username - ::storage-backend - ::storage-fs-directory ::srepl-host ::srepl-port - ::local-assets-uri - ::loggers-loki-uri - ::loggers-zmq-uri + ::storage-backend + ::storage-fs-directory ::storage-s3-bucket ::storage-s3-region ::telemetry-enabled - ::telemetry-with-taiga ::telemetry-server-enabled ::telemetry-server-port ::telemetry-uri - ::tenant - ::initial-data-file - ::initial-data-project-name])) + ::telemetry-with-taiga + ::tenant])) (defn- env->config [env] diff --git a/backend/src/app/db.clj b/backend/src/app/db.clj index 6b25433af..73026bae7 100644 --- a/backend/src/app/db.clj +++ b/backend/src/app/db.clj @@ -200,11 +200,6 @@ (sql/insert table params opts) (assoc opts :return-keys true)))) -(defn insert-multi! - [ds table param-list] - (doseq [params param-list] - (insert! ds table params))) - (defn update! ([ds table params where] (update! ds table params where nil)) ([ds table params where opts] diff --git a/backend/src/app/db/profile_initial_data.clj b/backend/src/app/db/profile_initial_data.clj deleted file mode 100644 index 0d810ee25..000000000 --- a/backend/src/app/db/profile_initial_data.clj +++ /dev/null @@ -1,117 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; This Source Code Form is "Incompatible With Secondary Licenses", as -;; defined by the Mozilla Public License, v. 2.0. -;; -;; Copyright (c) 2020-2021 UXBOX Labs SL - -(ns app.db.profile-initial-data - (:require - [app.common.uuid :as uuid] - [app.config :as cfg] - [app.db :as db] - [app.rpc.mutations.projects :as projects] - [app.util.transit :as tr] - [clojure.java.io :as io] - [datoteka.core :as fs])) - -(def sql:file - "select * from file where project_id = ?") - -(def sql:file-library-rel - "with file_ids as (select id from file where project_id = ?) - select * - from file_library_rel - where file_id in (select id from file_ids)") - -(def sql:file-media-object - "with file_ids as (select id from file where project_id = ?) - select * - from file_media_object - where file_id in (select id from file_ids)") - -(defn change-ids - "Given a collection and a map from ID to ID. Changes all the `keys` properties - so they point to the new ID existing in `map-ids`" - [map-ids coll keys] - (let [generate-id - (fn [map-ids {:keys [id]}] - (assoc map-ids id (uuid/next))) - - remap-key - (fn [obj map-ids key] - (cond-> obj - (contains? obj key) - (assoc key (get map-ids (get obj key) (get obj key))))) - - change-id - (fn [map-ids obj] - (reduce #(remap-key %1 map-ids %2) obj keys)) - - new-map-ids (reduce generate-id map-ids coll)] - - [new-map-ids (map (partial change-id new-map-ids) coll)])) - -(defn create-initial-data-dump - [conn project-id output-path] - (let [ ;; Retrieve data from templates - opath (fs/path output-path) - file (db/exec! conn [sql:file, project-id]) - file-library-rel (db/exec! conn [sql:file-library-rel, project-id]) - file-media-object (db/exec! conn [sql:file-media-object, project-id]) - - data {:file file - :file-library-rel file-library-rel - :file-media-object file-media-object}] - (with-open [output (io/output-stream opath)] - (tr/encode-stream data output) - nil))) - -(defn read-initial-data - [path] - (when (fs/exists? path) - (with-open [input (io/input-stream (fs/path path))] - (tr/decode-stream input)))) - -(defn create-profile-initial-data - ([conn profile] - (when-let [initial-data-path (:initial-data-file cfg/config)] - (create-profile-initial-data conn initial-data-path profile))) - - ([conn file profile] - (when-let [{:keys [file file-library-rel file-media-object]} (read-initial-data file)] - (let [sample-project-name (:initial-data-project-name cfg/config "Penpot Onboarding") - - proj (projects/create-project conn {:profile-id (:id profile) - :team-id (:default-team-id profile) - :name sample-project-name}) - - map-ids {} - - ;; Create new ID's and change the references - [map-ids file] (change-ids map-ids file #{:id}) - [map-ids file-library-rel] (change-ids map-ids file-library-rel #{:file-id :library-file-id}) - [_ file-media-object] (change-ids map-ids file-media-object #{:id :file-id :media-id :thumbnail-id}) - - file (map #(assoc % :project-id (:id proj)) file) - file-profile-rel (map #(array-map :file-id (:id %) - :profile-id (:id profile) - :is-owner true - :is-admin true - :can-edit true) - file)] - - (projects/create-project-profile conn {:project-id (:id proj) - :profile-id (:id profile)}) - - (projects/create-team-project-profile conn {:team-id (:default-team-id profile) - :project-id (:id proj) - :profile-id (:id profile)}) - - ;; Re-insert into the database - (db/insert-multi! conn :file file) - (db/insert-multi! conn :file-profile-rel file-profile-rel) - (db/insert-multi! conn :file-library-rel file-library-rel) - (db/insert-multi! conn :file-media-object file-media-object))))) diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index 9afa129bf..4b329d2eb 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -160,6 +160,9 @@ {:name "0049-mod-http-session-table" :fn (mg/resource "app/migrations/sql/0049-mod-http-session-table.sql")} + + {:name "0050-mod-server-prop-table" + :fn (mg/resource "app/migrations/sql/0050-mod-server-prop-table.sql")} ]) diff --git a/backend/src/app/migrations/sql/0050-mod-server-prop-table.sql b/backend/src/app/migrations/sql/0050-mod-server-prop-table.sql new file mode 100644 index 000000000..11f8d140e --- /dev/null +++ b/backend/src/app/migrations/sql/0050-mod-server-prop-table.sql @@ -0,0 +1,4 @@ +ALTER TABLE server_prop + ADD COLUMN preload boolean DEFAULT false; + +UPDATE server_prop SET preload = true; diff --git a/backend/src/app/rpc/mutations/demo.clj b/backend/src/app/rpc/mutations/demo.clj index b7ce34695..80658a5da 100644 --- a/backend/src/app/rpc/mutations/demo.clj +++ b/backend/src/app/rpc/mutations/demo.clj @@ -5,7 +5,7 @@ ;; This Source Code Form is "Incompatible With Secondary Licenses", as ;; defined by the Mozilla Public License, v. 2.0. ;; -;; Copyright (c) 2020 UXBOX Labs SL +;; Copyright (c) 2020-2021 UXBOX Labs SL (ns app.rpc.mutations.demo "A demo specific mutations." @@ -14,8 +14,8 @@ [app.common.uuid :as uuid] [app.config :as cfg] [app.db :as db] - [app.db.profile-initial-data :refer [create-profile-initial-data]] [app.rpc.mutations.profile :as profile] + [app.setup.initial-data :as sid] [app.tasks :as tasks] [app.util.services :as sv] [buddy.core.codecs :as bc] @@ -48,7 +48,7 @@ (db/with-atomic [conn pool] (->> (#'profile/create-profile conn params) (#'profile/create-profile-relations conn) - (create-profile-initial-data conn)) + (sid/load-initial-project! conn)) ;; Schedule deletion of the demo profile (tasks/submit! conn {:name "delete-profile" diff --git a/backend/src/app/rpc/mutations/profile.clj b/backend/src/app/rpc/mutations/profile.clj index 28f4823dc..40914de57 100644 --- a/backend/src/app/rpc/mutations/profile.clj +++ b/backend/src/app/rpc/mutations/profile.clj @@ -14,12 +14,12 @@ [app.common.uuid :as uuid] [app.config :as cfg] [app.db :as db] - [app.db.profile-initial-data :refer [create-profile-initial-data]] [app.emails :as emails] [app.media :as media] [app.rpc.mutations.projects :as projects] [app.rpc.mutations.teams :as teams] [app.rpc.queries.profile :as profile] + [app.setup.initial-data :as sid] [app.storage :as sto] [app.tasks :as tasks] [app.util.services :as sv] @@ -81,7 +81,8 @@ (let [profile (->> (create-profile conn params) (create-profile-relations conn)) profile (assoc profile ::created true)] - (create-profile-initial-data conn profile) + + (sid/load-initial-project! conn profile) (if-let [token (:invitation-token params)] ;; If invitation token comes in params, this is because the @@ -309,7 +310,7 @@ (register-profile [conn params] (let [profile (->> (create-profile conn params) (create-profile-relations conn))] - (create-profile-initial-data conn profile) + (sid/load-initial-project! conn profile) (assoc profile ::created true)))] (let [profile (profile/retrieve-profile-data-by-email conn email) diff --git a/backend/src/app/setup.clj b/backend/src/app/setup.clj index 3dbbab6d7..21853d79b 100644 --- a/backend/src/app/setup.clj +++ b/backend/src/app/setup.clj @@ -37,20 +37,25 @@ (let [key (-> (bn/random-bytes 64) (bc/bytes->b64u) (bc/bytes->str))] - (db/exec-one! conn ["insert into server_prop (id, content) - values ('secret-key', ?) on conflict do nothing" - (db/tjson key)]))) + (db/insert! conn :server-prop + {:id "secret-key" + :preload true + :content (db/tjson key)} + {:on-conflict-do-nothing true}))) (defn- initialize-instance-id! [{:keys [conn] :as cfg}] (let [iid (uuid/random)] - (db/exec-one! conn ["insert into server_prop (id, content) - values ('instance-id', ?::jsonb) on conflict do nothing" - (db/tjson iid)]))) + + (db/insert! conn :server-prop + {:id "instance-id" + :preload true + :content (db/tjson iid)} + {:on-conflict-do-nothing true}))) (defn- retrieve-all [{:keys [conn] :as cfg}] (reduce (fn [acc row] (assoc acc (keyword (:id row)) (db/decode-transit-pgobject (:content row)))) {} - (db/exec! conn ["select * from server_prop;"]))) + (db/query conn :server-prop {:preload true}))) diff --git a/backend/src/app/setup/initial_data.clj b/backend/src/app/setup/initial_data.clj new file mode 100644 index 000000000..6fd93ee39 --- /dev/null +++ b/backend/src/app/setup/initial_data.clj @@ -0,0 +1,137 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; This Source Code Form is "Incompatible With Secondary Licenses", as +;; defined by the Mozilla Public License, v. 2.0. +;; +;; Copyright (c) 2021 UXBOX Labs SL + +(ns app.setup.initial-data + (:refer-clojure :exclude [load]) + (:require + [app.common.uuid :as uuid] + [app.config :as cfg] + [app.db :as db] + [app.rpc.mutations.projects :as projects] + [app.rpc.queries.profile :as profile])) + +;; --- DUMP GENERATION + +(def sql:file + "select * from file where project_id = ?") + +(def sql:file-library-rel + "with file_ids as (select id from file where project_id = ?) + select * + from file_library_rel + where file_id in (select id from file_ids)") + +(def sql:file-media-object + "with file_ids as (select id from file where project_id = ?) + select * + from file_media_object + where file_id in (select id from file_ids)") + +(defn dump + ([system project-id] (dump system project-id nil)) + ([system project-id {:keys [skey project-name] + :or {project-name "Penpot Onboarding"}}] + (db/with-atomic [conn (:app.db/pool system)] + (let [skey (or skey (cfg/get :initial-project-skey)) + file (db/exec! conn [sql:file project-id]) + file-library-rel (db/exec! conn [sql:file-library-rel project-id]) + file-media-object (db/exec! conn [sql:file-media-object project-id]) + data {:project-name project-name + :file file + :file-library-rel file-library-rel + :file-media-object file-media-object}] + + (db/delete! conn :server-prop + {:id skey}) + (db/insert! conn :server-prop + {:id skey + :preload false + :content (db/tjson data)}) + nil)))) + + +;; --- DUMP LOADING + +(defn- remap-ids + "Given a collection and a map from ID to ID. Changes all the `keys` + properties so they point to the new ID existing in `map-ids`" + [map-ids coll keys] + (let [generate-id + (fn [map-ids {:keys [id]}] + (assoc map-ids id (uuid/next))) + + remap-key + (fn [obj map-ids key] + (cond-> obj + (contains? obj key) + (assoc key (get map-ids (get obj key) (get obj key))))) + + change-id + (fn [map-ids obj] + (reduce #(remap-key %1 map-ids %2) obj keys)) + + new-map-ids (reduce generate-id map-ids coll)] + + [new-map-ids (map (partial change-id new-map-ids) coll)])) + +(defn- retrieve-data + [conn skey] + (when-let [row (db/exec-one! conn ["select content from server_prop where id = ?" skey])] + (when-let [content (:content row)] + (when (db/pgobject? content) + (db/decode-transit-pgobject content))))) + +(defn load-initial-project! + ([conn profile] (load-initial-project! conn profile nil)) + ([conn profile opts] + (let [skey (or (:skey opts) (cfg/get :initial-project-skey)) + data (retrieve-data conn skey)] + (when data + (let [project (projects/create-project conn {:profile-id (:id profile) + :team-id (:default-team-id profile) + :name (:project-name data)}) + + map-ids {} + + [map-ids file] (remap-ids map-ids (:file data) #{:id}) + [map-ids file-library-rel] (remap-ids map-ids (:file-library-rel data) #{:file-id :library-file-id}) + [_ file-media-object] (remap-ids map-ids (:file-media-object data) #{:id :file-id :media-id :thumbnail-id}) + + file (map #(assoc % :project-id (:id project)) file) + file-profile-rel (map #(array-map :file-id (:id %) + :profile-id (:id profile) + :is-owner true + :is-admin true + :can-edit true) + file)] + + (projects/create-project-profile conn {:project-id (:id project) + :profile-id (:id profile)}) + + (projects/create-team-project-profile conn {:team-id (:default-team-id profile) + :project-id (:id project) + :profile-id (:id profile)}) + + ;; Re-insert into the database + (doseq [params file] + (db/insert! conn :file params)) + (doseq [params file-profile-rel] + (db/insert! conn :file-profile-rel params)) + (doseq [params file-library-rel] + (db/insert! conn :file-library-rel params)) + (doseq [params file-media-object] + (db/insert! conn :file-media-object params))))))) + +(defn load + [system {:keys [email] :as opts}] + (db/with-atomic [conn (:app.db/pool system)] + (when-let [profile (profile/retrieve-profile-data-by-email conn email)] + (load-initial-project! conn profile opts) + true))) + diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index c626a6def..8dd2cf197 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -6,7 +6,6 @@ [app.common.pages.migrations :as pmg] [app.config :as cfg] [app.db :as db] - [app.db.profile-initial-data :as pid] [app.main :refer [system]] [app.rpc.queries.profile :as prof] [app.srepl.dev :as dev] @@ -53,27 +52,6 @@ ;; (fn [{:keys [data] :as file}] ;; (update-in data [:pages-index #uuid "878278c0-3ef0-11eb-9d67-8551e7624f43" :objects] dissoc nil)))) -(def default-project-id #uuid "5761a890-3b81-11eb-9e7d-556a2f641513") - -(defn initial-data-dump - ([system file] (initial-data-dump system default-project-id file)) - ([system project-id path] - (db/with-atomic [conn (:app.db/pool system)] - (pid/create-initial-data-dump conn project-id path)))) - -(defn load-data-into-user - ([system user-email] - (if-let [file (:initial-data-file cfg/config)] - (load-data-into-user system file user-email) - (prn "Data file not found in configuration"))) - - ([system file user-email] - (db/with-atomic [conn (:app.db/pool system)] - (let [profile (prof/retrieve-profile-data-by-email conn user-email) - profile (merge profile (prof/retrieve-additional-data conn (:id profile)))] - (pid/create-profile-initial-data conn file profile))))) - - ;; Migrate (defn update-file-data-blob-format