0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-03-13 08:11:30 -05:00

Fix all linter issues on backend code.

This commit is contained in:
Andrey Antukh 2021-01-31 19:25:26 +01:00 committed by Alonso Torres
parent 6dafc087e9
commit b80295a21c
48 changed files with 134 additions and 256 deletions

View file

@ -21,9 +21,6 @@
}
:unresolved-symbol
{:exclude ['(app.services.mutations/defmutation)
'(app.services.queries/defquery)
'(app.util.dispatcher/defservice)
'(mount.core/defstate)
{:exclude ['(app.util.services/defmethod)
]}}}

View file

@ -9,17 +9,14 @@
(ns app.cli.migrate-media
(:require
[app.common.pages :as cp]
[app.common.uuid :as uuid]
[app.common.media :as cm]
[app.config :as cfg]
[app.db :as db]
[datoteka.core :as fs]
[app.main :as main]
[app.util.blob :as blob]
[app.storage :as sto]
[cuerdas.core :as str]
[clojure.tools.logging :as log]
[cuerdas.core :as str]
[datoteka.core :as fs]
[integrant.core :as ig]))
(declare migrate-profiles)

View file

@ -12,7 +12,6 @@
[app.common.exceptions :as ex]
[app.common.geom.point :as gpt]
[app.common.spec :as us]
[app.config :as cfg]
[app.db.sql :as sql]
[app.util.json :as json]
[app.util.migrations :as mg]
@ -20,7 +19,6 @@
[app.util.transit :as t]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[clojure.string :as str]
[integrant.core :as ig]
[next.jdbc :as jdbc]
[next.jdbc.date-time :as jdbc-dt])
@ -186,17 +184,17 @@
(sql/insert table params opts)
(assoc opts :return-keys true))))
(defn- select-values [map ks]
(reduce #(conj %1 (map %2)) [] ks))
;; (defn- select-values [map ks]
;; (reduce #(conj %1 (map %2)) [] ks))
(defn insert-multi!
[ds table param-list]
(doseq [params param-list]
(insert! ds table params))
(insert! ds table params))
;; FIXME: Won't work
#_(let [keys (->> param-list first keys (into []))
params (->> param-list (mapv #(->> keys (select-values %) (into []))) )]
(jdbc-sql/insert-multi! ds table keys params default-options)))
params (->> param-list (mapv #(->> keys (select-values %) (into []))) )]
(jdbc-sql/insert-multi! ds table keys params default-options)))
(defn update!
([ds table params where] (update! ds table params where nil))

View file

@ -15,15 +15,14 @@
[app.common.uuid :as uuid]
[app.config :as cfg]
[app.db :as db]
[app.tasks :as tasks]
[app.worker :as wrk]
[app.util.json :as json]
[app.util.http :as http]
[app.util.json :as json]
[app.util.template :as tmpl]
[app.worker :as wrk]
[clojure.core.async :as a]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[clojure.java.io :as io]
[cuerdas.core :as str]
[integrant.core :as ig]
[promesa.exec :as px])
@ -47,7 +46,7 @@
:opt-un [::uri]))
(defmethod ig/init-key ::reporter
[_ {:keys [executor uri] :as cfg}]
[_ {:keys [executor] :as cfg}]
(log/info "Intializing error reporter.")
(let [close-ch (a/chan 1)]
(a/go-loop []
@ -91,10 +90,10 @@
[cfg {:keys [message host version id] :as cdata}]
(try
(let [uri (:uri cfg)
prefix (str/<< "Unhandled exception (@channel):\n"
"- detail: ~(:public-uri cfg/config)/dbg/error-by-id/~{id}\n"
"- host: `~{host}`\n"
"- version: `~{version}`\n")
prefix (str "Unhandled exception (@channel):\n"
"- detail: " (:public-uri cfg/config) "/dbg/error-by-id/" id "\n"
"- host: `" host "`\n"
"- version: `" version "`\n")
text (str prefix "```\n" message "\n```")
rsp (http/send! {:uri uri
:method :post

View file

@ -10,11 +10,8 @@
(ns app.http
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.config :as cfg]
[app.http.assets :as assets]
[app.http.auth :as auth]
[app.http.errors :as errors]
[app.http.middleware :as middleware]
@ -26,7 +23,6 @@
[ring.adapter.jetty9 :as jetty])
(:import
org.eclipse.jetty.server.Server
org.eclipse.jetty.server.Handler
org.eclipse.jetty.server.handler.ErrorHandler
org.eclipse.jetty.server.handler.StatisticsHandler))
@ -117,7 +113,7 @@
:body "internal server error"})))))))
(defn- create-router
[{:keys [session rpc google-auth gitlab-auth github-auth metrics ldap-auth storage svgparse assets] :as cfg}]
[{:keys [session rpc google-auth gitlab-auth github-auth metrics ldap-auth svgparse assets] :as cfg}]
(rr/router
[["/metrics" {:get (:handler metrics)}]

View file

@ -13,13 +13,12 @@
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.db :as db]
[app.metrics :as mtx]
[app.storage :as sto]
[app.util.time :as dt]
[app.metrics :as mtx]
[cuerdas.core :as str]
[clojure.spec.alpha :as s]
[lambdaisland.uri :as u]
[integrant.core :as ig]))
[integrant.core :as ig]
[lambdaisland.uri :as u]))
(def ^:private cache-max-age
(dt/duration {:hours 24}))
@ -73,14 +72,14 @@
:body ""}))))
(defn- generic-handler
[{:keys [storage] :as cfg} request id]
[{:keys [storage] :as cfg} _request id]
(let [obj (sto/get-object storage id)]
(if obj
(serve-object cfg obj)
{:status 404 :body ""})))
(defn objects-handler
[{:keys [storage] :as cfg} request]
[cfg request]
(let [id (get-in request [:path-params :id])]
(generic-handler cfg request (coerce-id id))))

View file

@ -12,7 +12,6 @@
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.config :as cfg]
[app.db :as db]
[app.http.session :as session]
[app.util.http :as http]
[app.util.time :as dt]
@ -46,7 +45,7 @@
(str (assoc public :path "/api/oauth/github/callback"))))
(defn- get-access-token
[cfg code state]
[cfg state code]
(let [params {:client_id (:client-id cfg)
:client_secret (:client-secret cfg)
:code code
@ -93,13 +92,13 @@
nil))))
(defn auth
[{:keys [tokens] :as cfg} request]
[{:keys [tokens] :as cfg} _request]
(let [state (tokens :generate
{:iss :github-oauth
:exp (dt/in-future "15m")})
params {:client_id (:client-id cfg/config)
:redirect_uri (build-redirect-url)
:redirect_uri (build-redirect-url cfg)
:state state
:scope scope}
query (u/map->query-string params)
@ -112,9 +111,9 @@
[{:keys [tokens rpc session] :as cfg} request]
(let [state (get-in request [:params :state])
_ (tokens :verify {:token state :iss :github-oauth})
info (some-> (get-in request [:params :code])
(get-access-token state)
(get-user-info))]
info (some->> (get-in request [:params :code])
(get-access-token cfg state)
(get-user-info))]
(when-not info
(ex/raise :type :authentication
@ -158,7 +157,7 @@
::client-secret]))
(defn- default-handler
[req]
[_]
(ex/raise :type :not-found))
(defmethod ig/init-key :app.http.auth/github

View file

@ -9,10 +9,9 @@
(ns app.http.auth.gitlab
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.common.data :as d]
[app.config :as cfg]
[app.http.session :as session]
[app.util.http :as http]
[app.util.time :as dt]
@ -75,8 +74,8 @@
(defn- get-user-info
[token]
(let [req {:uri (build-user-info-url)
[cfg token]
(let [req {:uri (build-user-info-url cfg)
:headers {"Authorization" (str "Bearer " token)}
:method :get}
res (http/send! req)]
@ -102,12 +101,12 @@
:exp (dt/in-future "15m")})
params {:client_id (:client-id cfg)
:redirect_uri (build-redirect-url)
:redirect_uri (build-redirect-url cfg)
:response_type "code"
:state token
:scope scope}
query (uri/map->query-string params)
uri (-> (build-oauth-uri)
uri (-> (build-oauth-uri cfg)
(assoc :query query))]
{:status 200
:body {:redirect-uri (str uri)}}))
@ -118,7 +117,7 @@
_ (tokens :verify {:token token :iss :gitlab-oauth})
info (some->> (get-in request [:params :code])
(get-access-token cfg)
(get-user-info))]
(get-user-info cfg))]
(when-not info
(ex/raise :type :authentication
@ -167,7 +166,7 @@
(d/without-nils cfg)))
(defn- default-handler
[req]
[_]
(ex/raise :type :not-found))
(defmethod ig/init-key :app.http.auth/gitlab

View file

@ -11,7 +11,6 @@
(:require
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.config :as cfg]
[app.http.session :as session]
[app.util.http :as http]
[app.util.time :as dt]
@ -142,7 +141,7 @@
::client-secret]))
(defn- default-handler
[req]
[_]
(ex/raise :type :not-found))
(defmethod ig/init-key :app.http.auth/google

View file

@ -10,7 +10,6 @@
(ns app.http.errors
"A errors handling for the http server."
(:require
[app.common.exceptions :as ex]
[app.common.uuid :as uuid]
[app.config :as cfg]
[clojure.pprint :refer [pprint]]

View file

@ -9,16 +9,14 @@
(ns app.http.middleware
(:require
[app.common.exceptions :as ex]
[app.metrics :as mtx]
[app.util.transit :as t]
[app.util.json :as json]
[app.util.transit :as t]
[clojure.java.io :as io]
[ring.middleware.cookies :refer [wrap-cookies]]
[ring.middleware.keyword-params :refer [wrap-keyword-params]]
[ring.middleware.multipart-params :refer [wrap-multipart-params]]
[ring.middleware.params :refer [wrap-params]]
[ring.middleware.resource :refer [wrap-resource]]))
[ring.middleware.params :refer [wrap-params]]))
(defn wrap-server-timing
[handler]
@ -46,15 +44,14 @@
:json (parse-json body)
:transit (parse-transit body))
(catch Exception e
(let [type :json-verbose
data {:type :parse
(let [data {:type :parse
:hint "unable to parse request body"
:message (ex-message e)}]
{:status 400
:headers {"content-type" "application/transit+json"}
:body (t/encode-str data {:type type})}))))]
:body (t/encode-str data {:type :json-verbose})}))))]
(fn [{:keys [headers body request-method] :as request}]
(fn [{:keys [headers body] :as request}]
(let [ctype (get headers "content-type")]
(handler
(case ctype

View file

@ -9,12 +9,12 @@
(ns app.http.session
(:require
[clojure.spec.alpha :as s]
[integrant.core :as ig]
[app.db :as db]
[app.http.errors :refer [update-thread-context!]]
[buddy.core.codecs :as bc]
[buddy.core.nonce :as bn]))
[buddy.core.nonce :as bn]
[clojure.spec.alpha :as s]
[integrant.core :as ig]))
(defn next-session-id
([] (next-session-id 96))

View file

@ -9,8 +9,8 @@
(ns app.main
(:require
[app.config :as cfg]
[app.common.data :as d]
[app.config :as cfg]
[app.util.time :as dt]
[clojure.pprint :as pprint]
[clojure.tools.logging :as log]

View file

@ -14,11 +14,8 @@
[app.common.exceptions :as ex]
[app.common.media :as cm]
[app.common.spec :as us]
[app.config :as cfg]
[app.rlimits :as rlm]
[app.svgparse :as svg]
[app.util.http :as http]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[cuerdas.core :as str]
[datoteka.core :as fs])

View file

@ -12,9 +12,8 @@
[app.common.exceptions :as ex]
[app.util.time :as dt]
[app.worker]
[clojure.tools.logging :as log]
[clojure.spec.alpha :as s]
[cuerdas.core :as str]
[clojure.tools.logging :as log]
[integrant.core :as ig]
[next.jdbc :as jdbc])
(:import
@ -67,7 +66,7 @@
{::original origf})))}))
(defn- handler
[registry request]
[registry _request]
(let [samples (.metricFamilySamples ^CollectorRegistry registry)
writer (StringWriter.)]
(TextFormat/write004 writer samples)
@ -75,7 +74,7 @@
:body (.toString writer)}))
(defmethod ig/init-key ::metrics
[_ opts]
[_ _cfg]
(log/infof "Initializing prometheus registry and instrumentation.")
(let [registry (create-registry)]
(instrument-workers! registry)
@ -180,7 +179,7 @@
(observe val))))))
(defn create
[{:keys [type name] :as props}]
[{:keys [type] :as props}]
(case type
:counter (make-counter props)
:gauge (make-gauge props)
@ -209,13 +208,13 @@
(with-meta
(fn
([a]
(mobj :inc)
(mobj :inc labels)
(origf a))
([a b]
(mobj :inc)
(mobj :inc labels)
(origf a b))
([a b & more]
(mobj :inc)
(mobj :inc labels)
(apply origf a b more)))
(assoc mdata ::original origf)))))

View file

@ -9,10 +9,9 @@
(ns app.migrations
(:require
[integrant.core :as ig]
[app.db :as db]
[app.migrations.migration-0023 :as mg0023]
[app.util.migrations :as mg]
[app.migrations.migration-0023 :as mg0023]))
[integrant.core :as ig]))
(def migrations
[{:name "0001-add-extensions"

View file

@ -123,7 +123,7 @@
(when (jetty/connected? conn)
(jetty/send! conn data)
true)
(catch java.lang.NullPointerException e
(catch java.lang.NullPointerException _e
false)))
(defn websocket

View file

@ -8,7 +8,6 @@
(:refer-clojure :exclude [run!])
(:require
[app.common.spec :as us]
[app.config :as cfg]
[app.util.redis :as redis]
[clojure.spec.alpha :as s]
[integrant.core :as ig])

View file

@ -10,7 +10,6 @@
(ns app.rlimits
"Resource usage limits (in other words: semaphores)."
(:require
[app.common.exceptions :as ex]
[app.common.spec :as us]
[clojure.spec.alpha :as s]
[integrant.core :as ig])

View file

@ -22,7 +22,7 @@
[integrant.core :as ig]))
(defn- default-handler
[req]
[_]
(ex/raise :type :not-found))
(defn- rpc-query-handler

View file

@ -15,8 +15,8 @@
[app.rpc.queries.comments :as comments]
[app.rpc.queries.files :as files]
[app.util.blob :as blob]
[app.util.time :as dt]
[app.util.services :as sv]
[app.util.time :as dt]
[clojure.spec.alpha :as s]))
;; --- Mutation: Create Comment Thread

View file

@ -10,8 +10,8 @@
(ns app.rpc.mutations.demo
"A demo specific mutations."
(:require
[app.common.uuid :as uuid]
[app.common.exceptions :as ex]
[app.common.uuid :as uuid]
[app.config :as cfg]
[app.db :as db]
[app.db.profile-initial-data :refer [create-profile-initial-data]]

View file

@ -20,7 +20,6 @@
[app.util.http :as http]
[app.util.services :as sv]
[app.util.time :as dt]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[datoteka.core :as fs]))
@ -84,7 +83,7 @@
(defn create-file-media-object
[{:keys [conn storage svgc] :as cfg} {:keys [id file-id is-local name content] :as params}]
[{:keys [conn storage svgc] :as cfg} {:keys [file-id is-local name content] :as params}]
(media/validate-media-type (:content-type content))
(let [storage (assoc storage :conn conn)
source-path (fs/path (:tempfile content))

View file

@ -11,7 +11,6 @@
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.media :as cm]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.db :as db]
@ -21,13 +20,9 @@
[app.rpc.queries.profile :as profile]
[app.rpc.queries.teams :as teams]
[app.storage :as sto]
[app.tasks :as tasks]
[app.util.services :as sv]
[app.util.time :as dt]
[buddy.core.codecs :as bc]
[buddy.core.nonce :as bn]
[clojure.spec.alpha :as s]
[cuerdas.core :as str]
[datoteka.core :as fs]))
;; --- Helpers & Specs
@ -266,10 +261,7 @@
(defn upload-photo
[{:keys [storage] :as cfg} {:keys [file]}]
(let [prefix (-> (bn/random-bytes 8)
(bc/bytes->b64u)
(bc/bytes->str))
thumb (media/run cfg
(let [thumb (media/run cfg
{:cmd :profile-thumbnail
:format :jpeg
:quality 85

View file

@ -10,9 +10,8 @@
(ns app.rpc.permissions
"A permission checking helper factories."
(:require
[app.common.spec :as us]
[app.common.exceptions :as ex]
[clojure.spec.alpha :as s]))
[app.common.spec :as us]))
(defn make-edition-check-fn
"A simple factory for edition permission check functions."

View file

@ -15,8 +15,8 @@
[app.db :as db]
[app.rpc.permissions :as perms]
[app.rpc.queries.projects :as projects]
[app.util.services :as sv]
[app.util.blob :as blob]
[app.util.services :as sv]
[clojure.spec.alpha :as s]))
(declare decode-row)

View file

@ -9,7 +9,6 @@
(ns app.rpc.queries.projects
(:require
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.db :as db]
[app.rpc.permissions :as perms]

View file

@ -10,16 +10,11 @@
(ns app.sprops
"Server props module."
(:require
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.config :as cfg]
[app.db :as db]
[app.util.time :as dt]
[buddy.core.codecs :as bc]
[buddy.core.nonce :as bn]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]))
(declare initialize)

View file

@ -10,12 +10,12 @@
(ns app.srepl
"Server Repl."
(:require
[integrant.core :as ig]
[app.srepl.main]
[app.common.spec :as us]
[app.srepl.main]
[clojure.core.server :as ccs]
[clojure.main :as cm]
[clojure.spec.alpha :as s]
[clojure.main :as cm]))
[integrant.core :as ig]))
(defn- repl-init
[]

View file

@ -14,7 +14,6 @@
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.config :as cfg]
[app.db :as db]
[app.storage.db :as sdb]
[app.storage.fs :as sfs]
@ -27,7 +26,6 @@
[cuerdas.core :as str]
[datoteka.core :as fs]
[integrant.core :as ig]
[lambdaisland.uri :as u]
[promesa.exec :as px])
(:import
java.io.InputStream))
@ -198,7 +196,7 @@
(defn clone-object
"Creates a clone of the provided object using backend basded efficient
method. Always clones objects to the configured default."
[{:keys [pool conn executor] :as storage} object]
[{:keys [pool conn] :as storage} object]
(us/assert ::storage storage)
(let [storage (assoc storage :conn (or conn pool))
object* (create-database-object storage object)]
@ -242,7 +240,7 @@
(defn get-object-path
"Get the Path to the object. Only works with `:fs` type of
storages."
[{:keys [backend conn path] :as storage} object]
[storage object]
(let [backend (resolve-backend storage (:backend object))]
(when (not= :fs (:type backend))
(ex/raise :type :internal
@ -304,7 +302,7 @@
backend (assoc backend :conn conn)]
(impl/del-objects-in-bulk backend ids)))]
(fn [task]
(fn [_]
(db/with-atomic [conn pool]
(loop [n 0]
(if-let [[groups total] (retrieve-deleted-objects conn)]
@ -373,7 +371,7 @@
(db/exec-one! conn ["update storage_object set touched_at=null where id = ANY(?)"
(db/create-array conn "uuid" (into-array java.util.UUID ids))]))]
(fn [task]
(fn [_]
(db/with-atomic [conn pool]
(loop [cntf 0
cntd 0]
@ -424,7 +422,7 @@
[_ {:keys [pool storage] :as cfg}]
(letfn [(group-results [rows]
(let [conj (fnil conj [])]
(reduce (fn [acc {:keys [id backend exist] :as row}]
(reduce (fn [acc {:keys [id exist] :as row}]
(cond-> (update acc :all conj id)
(false? exist)
(update :to-delete conj (dissoc row :exist))))
@ -451,7 +449,7 @@
(let [ids (db/create-array conn "uuid" (into-array java.util.UUID ids))]
(db/exec-one! conn ["delete from storage_pending where id = ANY(?)" ids])))]
(fn [task]
(fn [_]
(db/with-atomic [conn pool]
(loop [n 0 d 0]
(if-let [{:keys [all to-delete]} (retrieve-pending conn)]

View file

@ -9,21 +9,13 @@
(ns app.storage.db
(:require
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.db :as db]
[app.storage.impl :as impl]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[datoteka.core :as fs]
[lambdaisland.uri :as u]
[integrant.core :as ig])
(:import
org.postgresql.largeobject.LargeObject
java.io.ByteArrayInputStream
java.io.ByteArrayOutputStream
java.io.InputStream
java.io.OutputStream))
java.io.ByteArrayInputStream))
;; --- BACKEND INIT
@ -58,11 +50,11 @@
(ByteArrayInputStream. (:data result))))
(defmethod impl/get-object-url :db
[backend {:keys [id] :as object}]
[_ _]
(throw (UnsupportedOperationException. "not supported")))
(defmethod impl/del-objects-in-bulk :db
[backend ids]
;; NOOP: because delting the row already deletes the file data from
[_ _]
;; NOOP: because deleting the row already deletes the file data from
;; the database.
nil)

View file

@ -9,17 +9,15 @@
(ns app.storage.fs
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.db :as db]
[app.storage.impl :as impl]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[datoteka.core :as fs]
[cuerdas.core :as str]
[lambdaisland.uri :as u]
[integrant.core :as ig])
[datoteka.core :as fs]
[integrant.core :as ig]
[lambdaisland.uri :as u])
(:import
java.io.InputStream
java.io.OutputStream
@ -34,7 +32,7 @@
(s/keys :opt-un [::directory]))
(defmethod ig/init-key ::backend
[key cfg]
[_ cfg]
;; Return a valid backend data structure only if all optional
;; parameters are provided.
(when (string? (:directory cfg))

View file

@ -13,14 +13,13 @@
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[clojure.java.io :as io]
[buddy.core.codecs :as bc])
[buddy.core.codecs :as bc]
[clojure.java.io :as io])
(:import
java.nio.ByteBuffer
java.util.UUID
java.io.ByteArrayInputStream
java.io.InputStream
java.nio.file.Path
java.nio.file.Files))
;; --- API Definition

View file

@ -13,23 +13,18 @@
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.db :as db]
[app.storage.impl :as impl]
[app.util.time :as dt]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[lambdaisland.uri :as u]
[integrant.core :as ig])
[integrant.core :as ig]
[lambdaisland.uri :as u])
(:import
java.io.InputStream
java.io.OutputStream
java.nio.file.Path
java.time.Duration
java.util.Collection
software.amazon.awssdk.core.sync.RequestBody
software.amazon.awssdk.regions.Region
software.amazon.awssdk.services.s3.S3Client
software.amazon.awssdk.services.s3.S3ClientBuilder
software.amazon.awssdk.services.s3.model.Delete
software.amazon.awssdk.services.s3.model.CopyObjectRequest
software.amazon.awssdk.services.s3.model.DeleteObjectsRequest
@ -113,7 +108,7 @@
:eu-central-1 Region/EU_CENTRAL_1))
(defn- build-s3-client
[{:keys [region bucket]}]
[{:keys [region]}]
(.. (S3Client/builder)
(region (lookup-region region))
(build)))

View file

@ -10,18 +10,14 @@
(ns app.svgparse
(:require
[app.common.exceptions :as ex]
[app.common.spec :as us]
[cuerdas.core :as str]
[app.metrics :as mtx]
[clojure.java.io :as io]
[clojure.java.shell :as shell]
[clojure.spec.alpha :as s]
[clojure.xml :as xml]
[app.util.graal :as graal]
[app.util.pool :as pool]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[clojure.xml :as xml]
[integrant.core :as ig])
(:import
java.io.InputStream
java.util.function.Consumer
org.apache.commons.io.IOUtils))
@ -36,7 +32,7 @@
(s/keys :req-un [::mtx/metrics]))
(defmethod ig/init-key ::svgc
[_ {:keys [metrics] :as cfg}]
[_ _]
(let [ctx-pool (prepare-context-pool)]
(with-meta
(fn [data]
@ -69,7 +65,6 @@
(defn- do-svg-clean
[ctx data]
(let [res (promise)
bindings (graal/get-bindings ctx "js")
optimize (-> (graal/get-bindings ctx "js")
(graal/get-member "svgc")
(graal/get-member "optimize"))

View file

@ -11,11 +11,11 @@
"Generic task for permanent deletion of objects."
(:require
[app.common.spec :as us]
[integrant.core :as ig]
[app.db :as db]
[app.metrics :as mtx]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]))
[clojure.tools.logging :as log]
[integrant.core :as ig]))
(declare handler)
(declare handle-deletion)

View file

@ -13,11 +13,8 @@
after some period of inactivity (the default threshold is 72h)."
(:require
[app.common.pages.migrations :as pmg]
[app.common.spec :as us]
[app.db :as db]
[app.metrics :as mtx]
[app.storage :as sto]
[app.tasks :as tasks]
[app.util.blob :as blob]
[app.util.time :as dt]
[clojure.spec.alpha :as s]
@ -25,8 +22,8 @@
[integrant.core :as ig]))
(declare handler)
(declare retrieve-candidates)
(declare process-file)
(declare retrieve-candidates)
(s/def ::storage some?)
@ -52,11 +49,6 @@
(run! (partial process-file cfg) files)
(recur)))))))
(defn- decode-row
[{:keys [data] :as row}]
(cond-> row
(bytes? data) (assoc :data (blob/decode data))))
(def ^:private
sql:retrieve-candidates-chunk
"select f.id,
@ -94,14 +86,14 @@
(into (keys (:media data)))))
(defn- process-file
[{:keys [conn storage] :as cfg} {:keys [id data age] :as file}]
(let [data (-> (blob/decode data)
(assoc :id id)
(pmg/migrate-data))
[{:keys [conn] :as cfg} {:keys [id data age] :as file}]
(let [data (-> (blob/decode data)
(assoc :id id)
(pmg/migrate-data))
used (collect-used-media data)
unused (->> (db/query conn :file-media-object {:file-id id})
(remove #(contains? used (:id %))))]
used (collect-used-media data)
unused (->> (db/query conn :file-media-object {:file-id id})
(remove #(contains? used (:id %))))]
(log/infof "processing file: id='%s' age='%s' to-delete=%s" id age (count unused))
@ -114,9 +106,7 @@
(log/debugf "deleting media object: id='%s' media-id='%s' thumb-id='%s'"
(:id mobj) (:media-id mobj) (:thumbnail-id mobj))
;; NOTE: deleting the file-media-object in the database
;; automatically marks to be deleted the associated storage
;; objects with the specialized trigger attached
;; to :file-media-object table.
;; automatically marks as toched the referenced storage objects.
(db/delete! conn :file-media-object {:id (:id mobj)}))
nil))

View file

@ -11,17 +11,17 @@
"A maintenance task that performs a garbage collection of the file
change (transaction) log."
(:require
[app.common.spec :as us]
[integrant.core :as ig]
[app.db :as db]
[app.metrics :as mtx]
[app.util.time :as dt]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]))
[clojure.tools.logging :as log]
[integrant.core :as ig]))
(declare handler)
(s/def ::max-age ::dt/duration)
(defmethod ig/pre-init-spec ::handler [_]
(s/keys :req-un [::db/pool ::mtx/metrics ::max-age]))

View file

@ -9,12 +9,11 @@
(ns app.tasks.sendmail
(:require
[app.common.spec :as us]
[app.config :as cfg]
[app.metrics :as mtx]
[app.util.emails :as emails]
[clojure.tools.logging :as log]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]))
(declare handler)

View file

@ -11,7 +11,6 @@
"A maintenance task that performs a cleanup of already executed tasks
from the database table."
(:require
[app.common.spec :as us]
[app.db :as db]
[app.metrics :as mtx]
[app.util.time :as dt]

View file

@ -12,15 +12,13 @@
information about the current instance and send it to the telemetry
server."
(:require
[app.config :as cfg]
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.config :as cfg]
[app.db :as db]
[app.util.http :as http]
[app.util.json :as json]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]))
(declare handler)
@ -60,16 +58,16 @@
[{:keys [sprops] :as cfg}]
(let [instance-id (:instance-id sprops)
data (retrieve-stats cfg)
data (assoc data :instance-id instance-id)]
(let [response (http/send! {:method :post
:uri (:uri cfg)
:headers {"content-type" "application/json"}
:body (json/encode-str data)})]
(when (not= 200 (:status response))
(ex/raise :type :internal
:code :invalid-response-from-google
:context {:status (:status response)
:body (:body response)})))))
data (assoc data :instance-id instance-id)
response (http/send! {:method :post
:uri (:uri cfg)
:headers {"content-type" "application/json"}
:body (json/encode-str data)})]
(when (not= 200 (:status response))
(ex/raise :type :internal
:code :invalid-response-from-google
:context {:status (:status response)
:body (:body response)}))))
(defn retrieve-num-teams
[conn]

View file

@ -11,7 +11,7 @@
(:require
[app.common.spec :as us]
[app.db :as db]
[app.http.middleware :refer [wrap-parse-request-body wrap-errors]]
[app.http.middleware :refer [wrap-parse-request-body]]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]

View file

@ -12,14 +12,12 @@
(:require
[app.common.exceptions :as ex]
[app.common.spec :as us]
[integrant.core :as ig]
[app.config :as cfg]
[app.util.time :as dt]
[app.util.transit :as t]
[buddy.core.kdf :as bk]
[buddy.sign.jwe :as jwe]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]))
[integrant.core :as ig]))
(defn- derive-tokens-secret
[key]

View file

@ -6,49 +6,16 @@
(ns app.util.async
(:require
[clojure.spec.alpha :as s]
[clojure.core.async :as a]
[cuerdas.core :as str])
[clojure.spec.alpha :as s])
(:import
java.util.concurrent.Executor
java.util.concurrent.ThreadFactory
java.util.concurrent.ForkJoinPool
java.util.concurrent.ForkJoinPool$ForkJoinWorkerThreadFactory
java.util.concurrent.ExecutorService
java.util.concurrent.atomic.AtomicLong))
java.util.concurrent.Executor))
(s/def ::executor #(instance? Executor %))
(defonce processors
(delay (.availableProcessors (Runtime/getRuntime))))
;; (defn forkjoin-thread-factory
;; [f]
;; (reify ForkJoinPool$ForkJoinWorkerThreadFactory
;; (newThread [this pool]
;; (let [wth (.newThread ForkJoinPool/defaultForkJoinWorkerThreadFactory pool)]
;; (f wth)))))
;; (defn forkjoin-named-thread-factory
;; [name]
;; (reify ForkJoinPool$ForkJoinWorkerThreadFactory
;; (newThread [this pool]
;; (let [wth (.newThread ForkJoinPool/defaultForkJoinWorkerThreadFactory pool)]
;; (.setName wth (str name ":" (.getPoolIndex wth)))
;; wth))))
;; (defn forkjoin-pool
;; [{:keys [factory async? parallelism]
;; :or {async? true}
;; :as opts}]
;; (let [parallelism (or parallelism @processors)
;; factory (cond
;; (fn? factory) (forkjoin-thread-factory factory)
;; (instance? ForkJoinPool$ForkJoinWorkerThreadFactory factory) factory
;; (nil? factory) ForkJoinPool/defaultForkJoinWorkerThreadFactory
;; :else (throw (ex-info "Unexpected thread factory" {:factory factory})))]
;; (ForkJoinPool. (or parallelism @processors) factory nil async?)))
(defmacro go-try
[& body]
`(a/go
@ -84,7 +51,7 @@
(finally
(a/close! c)))))
c
(catch java.util.concurrent.RejectedExecutionException e
(catch java.util.concurrent.RejectedExecutionException _e
(a/close! c)
c))))

View file

@ -10,7 +10,6 @@
(ns app.util.graal
"Graal Polyglot integration layer."
(:import
java.util.function.Consumer
org.graalvm.polyglot.Context
org.graalvm.polyglot.Source
org.graalvm.polyglot.Value))

View file

@ -39,13 +39,10 @@
(defn- impl-migrate-single
[pool modname {:keys [name] :as migration}]
(letfn [(execute []
(register! pool modname name)
((:fn migration) pool))]
(when-not (registered? pool modname (:name migration))
(log/info (str/format "applying migration %s/%s" modname name))
(register! pool modname name)
((:fn migration) pool))))
(when-not (registered? pool modname (:name migration))
(log/info (str/format "applying migration %s/%s" modname name))
(register! pool modname name)
((:fn migration) pool)))
(defn- impl-migrate
[conn migrations _opts]

View file

@ -128,9 +128,12 @@
(decode)))
(defn encode-str
[message]
(->> (encode message)
(bytes->str)))
([message]
(->> (encode message)
(bytes->str)))
([message opts]
(->> (encode message opts)
(bytes->str))))
(defn encode-verbose-str
[message]

View file

@ -12,7 +12,6 @@
(:require
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.config :as cfg]
[app.db :as db]
[app.util.async :as aa]
[app.util.time :as dt]
@ -20,17 +19,13 @@
[clojure.pprint :refer [pprint]]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[cuerdas.core :as str]
[integrant.core :as ig]
[promesa.exec :as px])
(:import
org.eclipse.jetty.util.thread.QueuedThreadPool
java.util.concurrent.ExecutorService
java.util.concurrent.Executors
java.util.concurrent.Executor
java.time.Duration
java.time.Instant
java.util.Date))
java.util.concurrent.Executor))
(s/def ::executor #(instance? Executor %))
@ -236,7 +231,7 @@
{:status :retry :task item :error error})))))
(defn- run-task
[{:keys [tasks conn]} item]
[{:keys [tasks]} item]
(try
(log/debugf "Started task '%s/%s/%s'." (:name item) (:id item) (:retry-num item))
(handle-task tasks item)
@ -255,7 +250,7 @@
for update skip locked")
(defn- event-loop-fn*
[{:keys [tasks pool executor batch-size] :as cfg}]
[{:keys [pool executor batch-size] :as cfg}]
(db/with-atomic [conn pool]
(let [queue (:queue cfg)
items (->> (db/exec! conn [sql:select-next-tasks queue batch-size])
@ -304,7 +299,7 @@
(s/keys :req-un [::executor ::db/pool ::schedule]))
(defmethod ig/init-key ::scheduler
[_ {:keys [executor schedule] :as cfg}]
[_ {:keys [schedule] :as cfg}]
(let [scheduler (Executors/newScheduledThreadPool (int 1))
schedule (filter some? schedule)
cfg (assoc cfg