diff --git a/backend/src/app/media_storage.clj b/backend/src/app/media_storage.clj deleted file mode 100644 index 7def22f9c..000000000 --- a/backend/src/app/media_storage.clj +++ /dev/null @@ -1,31 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; This Source Code Form is "Incompatible With Secondary Licenses", as -;; defined by the Mozilla Public License, v. 2.0. -;; -;; Copyright (c) 2020 UXBOX Labs SL - -(ns app.media-storage - "A media storage impl for app." - (:require - [integrant.core :as ig] - [app.common.spec :as us] - [clojure.spec.alpha :as s] - [app.config :refer [config]] - [app.util.storage :as ust])) - -(s/def ::media-directory ::us/not-empty-string) -(s/def ::media-uri ::us/not-empty-string) - -(defmethod ig/pre-init-spec ::storage [_] - (s/keys :req-un [::media-directory - ::media-uri])) - -(defmethod ig/init-key ::storage - [_ cfg] - (ust/create {:base-path (:media-directory cfg) - :base-uri (:media-uri cfg) - :xf (comp ust/random-path - ust/slugify-filename)})) diff --git a/backend/src/app/util/storage.clj b/backend/src/app/util/storage.clj deleted file mode 100644 index e9b28d062..000000000 --- a/backend/src/app/util/storage.clj +++ /dev/null @@ -1,192 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; This Source Code Form is "Incompatible With Secondary Licenses", as -;; defined by the Mozilla Public License, v. 2.0. -;; -;; Copyright (c) 2020 UXBOX Labs SL - -(ns app.util.storage - "A local filesystem storage implementation." - (:require - [app.common.exceptions :as ex] - [buddy.core.codecs :as bc] - [buddy.core.nonce :as bn] - [clojure.java.io :as io] - [clojure.spec.alpha :as s] - [cuerdas.core :as str] - [datoteka.core :as fs]) - (:import - java.io.ByteArrayInputStream - java.io.InputStream - java.io.OutputStream - java.net.URI - java.nio.file.NoSuchFileException - java.nio.file.Path)) - -(defn uri - [v] - (cond - (instance? URI v) v - (string? v) (URI. v) - :else (throw (IllegalArgumentException. "unexpected input")))) - -(defn- normalize-path - [^Path base ^Path path] - (when (fs/absolute? path) - (ex/raise :type :filesystem-error - :code :suspicious-operation - :hint "Suspicios operation: absolute path not allowed." - :contex {:path path :base base})) - (let [^Path fullpath (.resolve base path) - ^Path fullpath (.normalize fullpath)] - (when-not (.startsWith fullpath base) - (ex/raise :type :filesystem-error - :code :suspicious-operation - :hint "Suspicios operation: go to parent dir is not allowed." - :contex {:path path :base base})) - fullpath)) - -(defn- transform-path - [storage ^Path path] - (if-let [xf (::xf storage)] - ((xf (fn [_ b] b)) nil path) - path)) - -(defn blob - [^String v] - (let [data (.getBytes v "UTF-8")] - (ByteArrayInputStream. ^bytes data))) - -(defn save! - [storage path content] - (s/assert ::storage storage) - (let [^Path base (::base-path storage) - ^Path path (->> (fs/path path) - (transform-path storage)) - ^Path fullpath (normalize-path base path)] - (when-not (fs/exists? (.getParent fullpath)) - (fs/create-dir (.getParent fullpath))) - (loop [iteration nil] - (let [[basepath ext] (fs/split-ext fullpath) - candidate (fs/path (str basepath iteration ext))] - (if (fs/exists? candidate) - (recur (if (nil? iteration) 1 (inc iteration))) - (with-open [^InputStream src (io/input-stream content) - ^OutputStream dst (io/output-stream candidate)] - (io/copy src dst) - (fs/relativize candidate base))))))) - -(defn delete! - [storage path] - (s/assert ::storage storage) - (try - (->> (fs/path path) - (normalize-path (::base-path storage)) - (fs/delete)) - true - (catch NoSuchFileException _e - false))) - -(defn clear! - [storage] - (s/assert ::storage storage) - (fs/delete (::base-path storage)) - (fs/create-dir (::base-path storage)) - nil) - -(defn exists? - [storage path] - (s/assert ::storage storage) - (->> (fs/path path) - (normalize-path (::base-path storage)) - (fs/exists?))) - -(defn lookup - [storage path] - (s/assert ::storage storage) - (->> (fs/path path) - (normalize-path (::base-path storage)))) - -(defn public-uri - [storage path] - (s/assert ::storage storage) - (let [^URI base (::base-uri storage) - ^String path (str path)] - (.resolve base path))) - -(s/def ::base-path (s/or :path fs/path? :str string?)) -(s/def ::base-uri (s/or :uri #(instance? URI %) :str string?)) -(s/def ::xf fn?) - -(s/def ::storage - (s/keys :req [::base-path] :opt [::xf ::base-uri])) - -(s/def ::create-options - (s/keys :req-un [::base-path] :opt-un [::xf ::base-uri])) - -(defn create - "Create an instance of local FileSystem storage providing an - absolute base path. - - If that path does not exists it will be automatically created, - if it exists but is not a directory, an exception will be - raised. - - This function expects a map with the following options: - - `:base-path`: a fisical directory on your local machine - - `:base-uri`: a base uri used for resolve the files - " - [{:keys [base-path base-uri xf] :as options}] - (s/assert ::create-options options) - (let [^Path base-path (fs/path base-path)] - (when (and (fs/exists? base-path) - (not (fs/directory? base-path))) - (ex/raise :type :filesystem-error - :code :file-already-exists - :hint "File already exists, expects directory.")) - (when-not (fs/exists? base-path) - (fs/create-dir base-path)) - (cond-> {::base-path base-path} - base-uri (assoc ::base-uri (uri base-uri)) - xf (assoc ::xf xf)))) - -;; This is don't need to be secure and we dont need to reseed it; the -;; security guarranties of this prng instance are very low (we only -;; use it for generate a random path where store the file). - -(def ^:private prng - (delay - (doto (java.security.SecureRandom/getInstance "SHA1PRNG") - (.setSeed ^bytes (bn/random-bytes 64))))) - -(defn with-xf - [storage xfm] - (let [xf (::xf storage)] - (if (nil? xf) - (assoc storage ::xf xfm) - (assoc storage ::xf (comp xf xfm))))) - -(def random-path - (map (fn [^Path path] - (let [name (str (.getFileName path)) - hash (-> (bn/random-bytes 10 @prng) - (bc/bytes->b64u) - (bc/bytes->str)) - tokens (re-seq #"[\w\d\-\_]{2}" hash) - path-tokens (take 3 tokens) - rest-tokens (drop 3 tokens) - path (fs/path path-tokens) - frest (apply str rest-tokens)] - (fs/path (list path frest name)))))) - -(def slugify-filename - (map (fn [path] - (let [parent (or (fs/parent path) "") - [name ext] (fs/split-ext (fs/name path))] - (fs/path parent (str (str/uslug name) ext)))))) - -(defn prefix-path - [prefix] - (map (fn [^Path path] (fs/join (fs/path prefix) path)))) diff --git a/backend/tests/app/tests/helpers.clj b/backend/tests/app/tests/helpers.clj index 07635e55e..339646a87 100644 --- a/backend/tests/app/tests/helpers.clj +++ b/backend/tests/app/tests/helpers.clj @@ -15,7 +15,6 @@ [app.config :as cfg] [app.db :as db] [app.main :as main] - [app.media-storage] [app.media] [app.migrations] [app.rpc.mutations.files :as files] @@ -23,7 +22,6 @@ [app.rpc.mutations.projects :as projects] [app.rpc.mutations.teams :as teams] [app.util.blob :as blob] - [app.util.storage :as ust] [clojure.java.io :as io] [clojure.spec.alpha :as s] [cuerdas.core :as str] @@ -46,7 +44,6 @@ :app.http.auth/google :app.http.auth/gitlab :app.worker/scheduler - :app.worker/executor :app.worker/worker)) _ (ig/load-namespaces config) system (-> (ig/prep config) @@ -70,10 +67,7 @@ (db/exec! conn [(str "TRUNCATE " (apply str (interpose ", " result)) " CASCADE;")])))) - (try - (next) - (finally - (ust/clear! (:app.media-storage/storage *system*))))) + (next)) (defn mk-uuid [prefix & args] diff --git a/backend/tests/app/tests/test_services_files.clj b/backend/tests/app/tests/test_services_files.clj index 9e35f82f8..2fa54bd67 100644 --- a/backend/tests/app/tests/test_services_files.clj +++ b/backend/tests/app/tests/test_services_files.clj @@ -14,8 +14,7 @@ [app.common.uuid :as uuid] [app.db :as db] [app.http :as http] - [app.tests.helpers :as th] - [app.util.storage :as ust])) + [app.tests.helpers :as th])) (t/use-fixtures :once th/state-init) (t/use-fixtures :each th/database-reset) diff --git a/backend/tests/app/tests/test_services_media.clj b/backend/tests/app/tests/test_services_media.clj index 279472802..a2681b478 100644 --- a/backend/tests/app/tests/test_services_media.clj +++ b/backend/tests/app/tests/test_services_media.clj @@ -13,8 +13,7 @@ [datoteka.core :as fs] [app.common.uuid :as uuid] [app.db :as db] - [app.tests.helpers :as th] - [app.util.storage :as ust])) + [app.tests.helpers :as th])) (t/use-fixtures :once th/state-init) (t/use-fixtures :each th/database-reset) diff --git a/backend/tests/app/tests/test_services_profile.clj b/backend/tests/app/tests/test_services_profile.clj index 4d2e6765a..53a8af201 100644 --- a/backend/tests/app/tests/test_services_profile.clj +++ b/backend/tests/app/tests/test_services_profile.clj @@ -15,7 +15,6 @@ [cuerdas.core :as str] [datoteka.core :as fs] [app.db :as db] - ;; [app.services.mutations.profile :as profile] [app.tests.helpers :as th])) (t/use-fixtures :once th/state-init) @@ -64,8 +63,8 @@ (t/testing "update profile" (let [data (assoc profile - :profile-id (:id profile) ::th/type :update-profile + :profile-id (:id profile) :fullname "Full Name" :lang "en" :theme "dark") diff --git a/backend/tests/app/tests/test_services_viewer.clj b/backend/tests/app/tests/test_services_viewer.clj index d7ba55e5f..80cb4770b 100644 --- a/backend/tests/app/tests/test_services_viewer.clj +++ b/backend/tests/app/tests/test_services_viewer.clj @@ -13,8 +13,7 @@ [datoteka.core :as fs] [app.common.uuid :as uuid] [app.db :as db] - [app.tests.helpers :as th] - [app.util.storage :as ust])) + [app.tests.helpers :as th])) (t/use-fixtures :once th/state-init) (t/use-fixtures :each th/database-reset)