♻️ Refactor the CLI media loader.

backend/deps.edn

@@ -11,11 +11,12 @@
         funcool/datoteka {:mvn/version "1.1.0"}
         expound/expound {:mvn/version "0.7.2"}
         instaparse/instaparse {:mvn/version "1.4.10"}
+        com.cognitect/transit-clj {:mvn/version "0.8.319"}
 
         ;; vertx deps
+        ;; vertx-clojure/vertx {:mvn/version "0.0.0-SNAPSHOT"}
         metosin/reitit-core {:mvn/version "0.3.10"}
         metosin/sieppari {:mvn/version "0.0.0-alpha8"}
-        com.cognitect/transit-clj {:mvn/version "0.8.319"}
         io.vertx/vertx-core {:mvn/version "3.8.1"}
         io.vertx/vertx-web {:mvn/version "3.8.1"}
         io.vertx/vertx-pg-client {:mvn/version "3.8.1"}

backend/src/uxbox/core.clj

@@ -6,11 +6,10 @@
 
 (ns uxbox.core
   (:require
-   [vertx.core :as vx]
+   [vertx.core :as vc]
+   [vertx.timers :as vt]
    [mount.core :as mount :refer [defstate]]))
 
 (defstate system
-  :start (vx/system)
+  :start (vc/system)
   :stop (.close system))
-
-

backend/src/uxbox/http.clj

@@ -38,7 +38,6 @@
                    :allow-headers #{:x-requested-with :content-type :cookie}}
 
         interceptors [(vxi/cookies)
-                      (vxi/headers)
                       (vxi/params)
                       (vxi/cors cors-opts)
                       interceptors/parse-request-body
@@ -66,6 +65,6 @@
     (vh/server ctx {:handler handler
                     :port (:http-server-port cfg/config)})))
 
-(defstate http-verticle
+(defstate server
   :start (let [factory (vc/verticle {:on-start on-start})]
            @(vc/deploy! system factory {:instances 4})))

backend/src/uxbox/http/handlers.clj

@@ -83,4 +83,3 @@
    :body {:params (:params req)
           :cookies (:cookies req)
           :headers (:headers req)}})
-

backend/src/uxbox/media_loader.clj

@@ -0,0 +1,252 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) 2016-2019 Andrey Antukh <niwi@niwi.nz>
+
+(ns uxbox.media-loader
+  "Media collections importer (command line helper)."
+  (:require
+   [clojure.tools.logging :as log]
+   [clojure.spec.alpha :as s]
+   [clojure.pprint :refer [pprint]]
+   [clojure.java.io :as io]
+   [clojure.edn :as edn]
+   [promesa.core :as p]
+   [mount.core :as mount]
+   [cuerdas.core :as str]
+   [datoteka.storages :as st]
+   [datoteka.core :as fs]
+   [uxbox.config]
+   [uxbox.db :as db]
+   [uxbox.http]
+   [uxbox.migrations]
+   [uxbox.media :as media]
+   [uxbox.util.svg :as svg]
+   [uxbox.util.transit :as t]
+   [uxbox.util.spec :as us]
+   [uxbox.util.blob :as blob]
+   [uxbox.util.uuid :as uuid]
+   [uxbox.util.data :as data])
+  (:import
+   java.io.Reader
+   java.io.PushbackReader
+   org.im4java.core.Info))
+
+;; --- Constants & Helpers
+
+(def ^:const +images-uuid-ns+ #uuid "3642a582-565f-4070-beba-af797ab27a6e")
+(def ^:const +icons-uuid-ns+ #uuid "3642a582-565f-4070-beba-af797ab27a6f")
+
+(s/def ::name ::us/string)
+(s/def ::path ::us/string)
+(s/def ::regex us/regex?)
+(s/def ::import-item
+  (s/keys :req-un [::name ::path ::regex]))
+
+(defn exit!
+  ([] (exit! 0))
+  ([code]
+   (System/exit code)))
+
+;; --- Icons Collections Importer
+
+(defn- create-icons-collection
+  "Create or replace icons collection by its name."
+  [conn {:keys [name] :as item}]
+  (log/info "Creating or updating icons collection:" name)
+  (let [id (uuid/namespaced +icons-uuid-ns+ name)
+        sql "insert into icons_collections (id, user_id, name)
+             values ($1, '00000000-0000-0000-0000-000000000000'::uuid, $2)
+                 on conflict (id)
+                 do update set name = $2
+             returning *;"
+        sqlv [sql id name]]
+    (-> (db/query-one conn [sql id name])
+        (p/then' (constantly id)))))
+
+(def create-icon-sql
+  "insert into icons (user_id, id, collection_id, name, metadata, content)
+   values ('00000000-0000-0000-0000-000000000000'::uuid, $1, $2, $3, $4, $5)
+       on conflict (id)
+       do update set name = $3,
+                     metadata = $4,
+                     content = $5,
+                     collection_id = $2,
+                     user_id = '00000000-0000-0000-0000-000000000000'::uuid
+   returning *;")
+
+(defn- create-or-update-icon
+  [conn id icon-id localpath]
+  (s/assert fs/path? localpath)
+  (s/assert ::us/uuid id)
+  (s/assert ::us/uuid icon-id)
+  (let [filename (fs/name localpath)
+        extension (second (fs/split-ext filename))
+        data (svg/parse localpath)
+        mdata (select-keys data [:width :height :view-box])]
+    (db/query-one conn [create-icon-sql icon-id id
+                        (:name data filename)
+                        (blob/encode mdata)
+                        (:content data)])))
+
+(defn- import-icon
+  [conn id fpath]
+  (s/assert ::us/uuid id)
+  (s/assert fs/path? fpath)
+  (let [filename (fs/name fpath)
+        icon-id (uuid/namespaced +icons-uuid-ns+ (str id filename))]
+    (log/info "Creating or updating icon" filename icon-id)
+    (-> (create-or-update-icon conn id icon-id fpath)
+        (p/then (constantly nil)))))
+
+(defn- import-icons
+  [conn coll-id {:keys [path regex] :as item}]
+  (p/run! (fn [fpath]
+            (when (re-matches regex (str fpath))
+              (import-icon conn coll-id fpath)))
+          (->> (fs/list-dir path)
+               (filter fs/regular-file?))))
+
+(defn- process-icons-collection
+  [conn basedir {:keys [path regex] :as item}]
+  (s/assert ::import-item item)
+  (-> (create-icons-collection conn item)
+      (p/then (fn [coll-id]
+                (->> (assoc item :path (fs/join basedir path))
+                     (import-icons conn coll-id))))))
+
+;; --- Images Collections Importer
+
+(defn- create-images-collection
+  "Create or replace image collection by its name."
+  [conn {:keys [name] :as item}]
+  (log/info "Creating or updating image collection:" name)
+  (let [id (uuid/namespaced +icons-uuid-ns+ name)
+        sql "insert into images_collections (id, user_id, name)
+             values ($1, '00000000-0000-0000-0000-000000000000'::uuid, $2)
+                 on conflict (id)
+                 do update set name = $2
+             returning *;"
+        sqlv [sql id name]]
+    (-> (db/query-one conn [sql id name])
+        (p/then' (constantly id)))))
+
+(defn- retrieve-image-size
+  [path]
+  (let [info (Info. (str path) true)]
+    [(.getImageWidth info) (.getImageHeight info)]))
+
+(defn- image-exists?
+  [conn id]
+  (s/assert ::us/uuid id)
+  (let [sql "select id
+               from images as i
+              where i.id = $1
+                and i.user_id = '00000000-0000-0000-0000-000000000000'::uuid"]
+    (-> (db/query-one conn [sql id])
+        (p/then (fn [row] (if row true false))))))
+
+(def create-image-sql
+ "insert into images (user_id, id, collection_id, name, path, width, height, mimetype)
+  values ('00000000-0000-0000-0000-000000000000'::uuid, $1, $2, $3, $4, $5, $6, $7)
+  returning *;")
+
+(defn- create-image
+  [conn id image-id localpath]
+  (s/assert fs/path? localpath)
+  (s/assert ::us/uuid id)
+  (s/assert ::us/uuid image-id)
+  (let [storage media/images-storage
+        filename (fs/name localpath)
+        [width height] (retrieve-image-size localpath)
+        extension (second (fs/split-ext filename))
+        mimetype (case extension
+                   ".jpg" "image/jpeg"
+                   ".png" "image/png")]
+    (-> (st/save storage filename localpath)
+        (p/then (fn [path]
+                  (db/query-one conn [create-image-sql image-id id
+                                      filename
+                                      (str path)
+                                      width
+                                      height
+                                      mimetype])))
+        (p/then (constantly nil)))))
+
+(defn- import-image
+  [conn id fpath]
+  (s/assert ::us/uuid id)
+  (s/assert fs/path? fpath)
+  (let [filename (fs/name fpath)
+        image-id (uuid/namespaced +images-uuid-ns+ (str id filename))]
+    (-> (image-exists? conn image-id)
+        (p/then (fn [exists?]
+                  (when-not exists?
+                    (log/info "Creating image" filename image-id)
+                    (create-image conn id image-id fpath))))
+        (p/then (constantly nil)))))
+
+(defn- import-images
+  [conn coll-id {:keys [path regex] :as item}]
+  (p/run! (fn [fpath]
+            (when (re-matches regex (str fpath))
+              (import-image conn coll-id fpath)))
+          (->> (fs/list-dir path)
+               (filter fs/regular-file?))))
+
+(defn- process-images-collection
+  [conn basedir {:keys [path regex] :as item}]
+  (s/assert ::import-item item)
+  (-> (create-images-collection conn item)
+      (p/then (fn [coll-id]
+                (->> (assoc item :path (fs/join basedir path))
+                     (import-images conn coll-id))))))
+
+;; --- Entry Point
+
+(defn- validate-path
+  [path]
+  (when-not path
+    (log/error "No path is provided")
+    (exit! -1))
+  (when-not (fs/exists? path)
+    (log/error "Path does not exists.")
+    (exit! -1))
+  (when (fs/directory? path)
+    (log/error "The provided path is a directory.")
+    (exit! -1))
+  (fs/path path))
+
+(defn- read-import-file
+  [path]
+  (let [path (validate-path path)
+        reader (java.io.PushbackReader. (io/reader path))]
+    [(fs/parent path)
+     (read reader)]))
+
+(defn- start-system
+  []
+  (-> (mount/except #{#'uxbox.http/server})
+      (mount/start)))
+
+(defn- stop-system
+  []
+  (mount/stop))
+
+(defn- importer
+  [conn basedir data]
+  (let [images (:images data)
+        icons (:icons data)]
+    (p/do!
+     (p/run! #(process-images-collection conn basedir %) images)
+     (p/run! #(process-icons-collection conn basedir %) icons))))
+
+(defn -main
+  [& [path]]
+  (let [[basedir data] (read-import-file path)]
+    (start-system)
+    (-> (db/with-atomic [conn db/pool]
+          (importer conn basedir data))
+        (p/finally (fn [_ _]
+                     (stop-system))))))

backend/src/uxbox/services/svgparse.clj

@@ -1,99 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) 2016 Andrey Antukh <niwi@niwi.nz>
-
-(ns uxbox.services.svgparse
-  (:require [clojure.spec.alpha :as s]
-            [cuerdas.core :as str]
-            [uxbox.util.spec :as us]
-            [uxbox.services.core :as core]
-            [uxbox.util.exceptions :as ex])
-  (:import org.jsoup.Jsoup
-           java.io.InputStream))
-
-;; (s/def ::content string?)
-;; (s/def ::width number?)
-;; (s/def ::height number?)
-;; (s/def ::name string?)
-;; (s/def ::view-box (s/coll-of number? :min-count 4 :max-count 4))
-;; (s/def ::svg-entity (s/keys :req-un [::content ::width ::height ::view-box]
-;;                             :opt-un [::name]))
-
-;; ;; --- Implementation
-
-;; (defn- parse-double
-;;   [data]
-;;   {:pre [(string? data)]}
-;;   (Double/parseDouble data))
-
-;; (defn- parse-viewbox
-;;   [data]
-;;   {:pre [(string? data)]}
-;;   (mapv parse-double (str/split data #"\s+")))
-
-;; (defn- assoc-attr
-;;   [acc attr]
-;;   (let [key (.getKey attr)
-;;         val (.getValue attr)]
-;;     (case key
-;;       "width" (assoc acc :width (parse-double val))
-;;       "height" (assoc acc :height (parse-double val))
-;;       "viewbox" (assoc acc :view-box (parse-viewbox val))
-;;       "sodipodi:docname" (assoc acc :name val)
-;;       acc)))
-
-;; (defn- parse-attrs
-;;   [element]
-;;   (let [attrs (.attributes element)]
-;;     (reduce assoc-attr {} attrs)))
-
-;; (defn- parse-svg
-;;   [data]
-;;   (try
-;;     (let [document (Jsoup/parse data)
-;;           svgelement (some-> (.body document)
-;;                              (.getElementsByTag "svg")
-;;                              (first))
-;;           innerxml (.html svgelement)
-;;           attrs (parse-attrs svgelement)]
-;;       (merge {:content innerxml} attrs))
-;;     (catch java.lang.IllegalArgumentException e
-;;       (ex/raise :type :validation
-;;                 :code ::invalid-input
-;;                 :message "Input does not seems to be a valid svg."))
-;;     (catch java.lang.NullPointerException e
-;;       (ex/raise :type :validation
-;;                 :code ::invalid-input
-;;                 :message "Input does not seems to be a valid svg."))
-
-;;     (catch org.jsoup.UncheckedIOException e
-;;       (ex/raise :type :validation
-;;                 :code ::invalid-input
-;;                 :message "Input does not seems to be a valid svg."))
-
-;;     (catch Exception e
-;;       (ex/raise :code ::unexpected))))
-
-;; ;; --- Public Api
-
-;; (defn parse-string
-;;   "Parse SVG from a string."
-;;   [data]
-;;   {:pre [(string? data)]}
-;;   (let [result (parse-svg data)]
-;;     (if (s/valid? ::svg-entity result)
-;;       result
-;;       (ex/raise :type :validation
-;;                 :code ::invalid-result
-;;                 :message "The result does not conform valid svg entity."))))
-
-;; (defn parse
-;;   [data]
-;;   (parse-string (slurp data)))
-
-;; (defmethod core/query :parse-svg
-;;   [{:keys [data] :as params}]
-;;   {:pre [(string? data)]}
-;;   (parse-string data))

backend/src/uxbox/util/spec.clj

@@ -2,7 +2,7 @@
 ;; License, v. 2.0. If a copy of the MPL was not distributed with this
 ;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ;;
-;; Copyright (c) 2016 Andrey Antukh <niwi@niwi.nz>
+;; Copyright (c) 2016-2019 Andrey Antukh <niwi@niwi.nz>
 
 (ns uxbox.util.spec
   (:refer-clojure :exclude [keyword uuid vector boolean map set])
@@ -21,6 +21,9 @@
 (def uuid-rx
   #"^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$")
 
+(def number-rx
+  #"^[+-]?([0-9]*\.?[0-9]+|[0-9]+\.?[0-9]*)([eE][+-]?[0-9]+)?$")
+
 ;; --- Public Api
 
 (defn conform
@@ -104,12 +107,23 @@
    (fs/path? v) v
    :else ::s/invalid))
 
+(defn- number-conformer
+  [v]
+  (cond
+    (number? v) v
+    (re-matches number-rx v) (Double/parseDouble v)
+    :else ::s/invalid))
+
+
 ;; --- Default Specs
 
 (s/def ::string string?)
 (s/def ::integer (s/conformer integer-conformer str))
 (s/def ::uuid (s/conformer uuid-conformer str))
 (s/def ::boolean (s/conformer boolean-conformer boolean-unformer))
+(s/def ::number (s/conformer number-conformer str))
+(s/def ::path (s/conformer path-conformer str))
+
 (s/def ::positive pos?)
 (s/def ::negative neg?)
 (s/def ::uploaded-file any?)
@@ -118,7 +132,6 @@
 (s/def ::file any?)
 
 (s/def ::name ::string)
-(s/def ::path (s/conformer path-conformer str))
 (s/def ::size ::integer)
 (s/def ::mtype ::string)
 (s/def ::upload

backend/src/uxbox/util/svg.clj

@@ -0,0 +1,99 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) 2016-2019 Andrey Antukh <niwi@niwi.nz>
+
+(ns uxbox.util.svg
+  "Icons SVG parsing helpers."
+  (:require
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]
+   [uxbox.util.spec :as us]
+   [uxbox.util.exceptions :as ex])
+  (:import
+   org.jsoup.Jsoup
+   org.jsoup.nodes.Attribute
+   org.jsoup.nodes.Element
+   org.jsoup.nodes.Document
+   java.io.InputStream))
+
+(s/def ::content ::us/string)
+(s/def ::width ::us/number)
+(s/def ::height ::us/number)
+(s/def ::name ::us/string)
+(s/def ::view-box (s/coll-of ::us/number :min-count 4 :max-count 4))
+
+(s/def ::svg-entity
+  (s/keys :req-un [::content ::width ::height ::view-box]
+          :opt-un [::name]))
+
+;; --- Implementation
+
+(defn- parse-double
+  [data]
+  (s/assert ::us/string data)
+  (Double/parseDouble data))
+
+(defn- parse-viewbox
+  [data]
+  (s/assert ::us/string data)
+  (mapv parse-double (str/split data #"\s+")))
+
+(defn- parse-attrs
+  [^Element element]
+  (persistent!
+   (reduce (fn [acc ^Attribute attr]
+             (let [key (.getKey attr)
+                   val (.getValue attr)]
+               (case key
+                 "width" (assoc! acc :width (parse-double val))
+                 "height" (assoc! acc :height (parse-double val))
+                 "viewbox" (assoc! acc :view-box (parse-viewbox val))
+                 "sodipodi:docname" (assoc! acc :name val)
+                 acc)))
+           (transient {})
+           (.attributes element))))
+
+(defn- impl-parse
+  [data]
+  (try
+    (let [document (Jsoup/parse ^String data)
+          element (some-> (.body ^Document document)
+                          (.getElementsByTag "svg")
+                          (first))
+          content (.html element)
+          attrs (parse-attrs element)]
+      (assoc attrs :content content))
+    (catch java.lang.IllegalArgumentException e
+      (ex/raise :type :validation
+                :code ::invalid-input
+                :message "Input does not seems to be a valid svg."))
+    (catch java.lang.NullPointerException e
+      (ex/raise :type :validation
+                :code ::invalid-input
+                :message "Input does not seems to be a valid svg."))
+    (catch org.jsoup.UncheckedIOException e
+      (ex/raise :type :validation
+                :code ::invalid-input
+                :message "Input does not seems to be a valid svg."))
+    (catch Exception e
+      (ex/raise :type :internal
+                :code ::unexpected))))
+
+;; --- Public Api
+
+(defn parse-string
+  "Parse SVG from a string."
+  [data]
+  (s/assert ::us/string data)
+  (let [result (impl-parse data)]
+    (if (s/valid? ::svg-entity result)
+      result
+      (ex/raise :type :validation
+                :code ::invalid-result
+                :message "The result does not conform valid svg entity."))))
+
+(defn parse
+  [data]
+  (parse-string (slurp data)))

backend/src/uxbox/util/template.clj

@@ -8,6 +8,7 @@
   "A lightweight abstraction over mustache.java template engine.
   The documentation can be found: http://mustache.github.io/mustache.5.html"
   (:require
+   [clojure.tools.logging :as log]
    [clojure.walk :as walk]
    [clojure.java.io :as io]
    [uxbox.util.exceptions :as ex])
@@ -33,7 +34,11 @@
                      (fn? x)
                      (reify Function
                        (apply [this content]
-                         (x content)))
+                         (try
+                           (x content)
+                           (catch Exception e
+                             (log/error e "Error on executing" x)
+                             ""))))
 
                      (or (vector? x) (list? x))
                      (java.util.ArrayList. x)