Penpot/penpot
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2025-04-16 00:41:25 -05:00
Code Issues Activity

Merge branch 'staging' into main

Browse source
This commit is contained in:
Andrey Antukh 2021-05-06 12:55:34 +02:00
commit fa4410bea3
563 changed files with 26306 additions and 14719 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
.circleci/config.yml
View file

@ -13,7 +13,7 @@ jobs:
environment:
POSTGRES_USER: penpot_test
POSTGRES_PASSWORD: penpot_test
POSTGRES_DB: penpot
POSTGRES_DB: penpot_test
- image: circleci/redis:6.0.8
@ -45,10 +45,10 @@ jobs:
name: backend test
command: "clojure -M:dev:tests"
environment:
PENPOT_DATABASE_URI: "postgresql://localhost/penpot"
PENPOT_DATABASE_USERNAME: penpot_test
PENPOT_DATABASE_PASSWORD: penpot_test
PENPOT_REDIS_URI: "redis://localhost/1"
PENPOT_TEST_DATABASE_URI: "postgresql://localhost/penpot_test"
PENPOT_TEST_DATABASE_USERNAME: penpot_test
PENPOT_TEST_DATABASE_PASSWORD: penpot_test
PENPOT_TEST_REDIS_URI: "redis://localhost/1"
- run:
working_directory: "./frontend"
@ -57,7 +57,8 @@ jobs:
yarn install
npx shadow-cljs compile tests
environment:
PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin
JAVA_HOME: /usr/lib/jvm/openjdk16
PATH: /usr/local/nodejs/bin/:/usr/local/bin:/bin:/usr/bin:/usr/lib/jvm/openjdk16/bin
- save_cache:
paths:

1
.gitignore vendored
View file

@ -19,6 +19,7 @@ node_modules
/backend/dist/
/backend/logs/
/backend/-
/telemetry/
/frontend/npm-debug.log
/frontend/target/
/frontend/dist/

105
.gitpod.yml Normal file
View file

@ -0,0 +1,105 @@
image:
file: docker/gitpod/Dockerfile
ports:
# nginx
- port: 3449
onOpen: open-preview
# frontend nREPL
- port: 3447
onOpen: ignore
visibility: private
# frontend shadow server
- port: 3448
onOpen: ignore
visibility: private
# backend
- port: 6060
onOpen: ignore
# exporter shadow server
- port: 9630
onOpen: ignore
visibility: private
# exporter http server
- port: 6061
onOpen: ignore
# mailhog web interface
- port: 8025
onOpen: ignore
# mailhog postfix
- port: 1025
onOpen: ignore
# postgres
- port: 5432
onOpen: ignore
# redis
- port: 6379
onOpen: ignore
# openldap
- port: 389
onOpen: ignore
tasks:
# https://github.com/gitpod-io/gitpod/issues/666#issuecomment-534347856
- name: gulp
command: >
cd $GITPOD_REPO_ROOT/frontend/;
yarn && gp sync-done 'frontend-yarn';
npx gulp --theme=${PENPOT_THEME} watch
- name: frontend shadow watch
command: >
cd $GITPOD_REPO_ROOT/frontend/;
gp sync-await 'frontend-yarn';
npx shadow-cljs watch main
- init: gp await-port 5432 && psql -f $GITPOD_REPO_ROOT/docker/gitpod/files/postgresql_init.sql
name: backend
command: >
cd $GITPOD_REPO_ROOT/backend/;
./scripts/start-dev
- name: exporter shadow watch
command:
cd $GITPOD_REPO_ROOT/exporter/;
gp sync-await 'frontend-yarn';
yarn && npx shadow-cljs watch main
- name: exporter web server
command: >
cd $GITPOD_REPO_ROOT/exporter/;
./scripts/wait-and-start.sh
- name: signed terminal
before: >
[[ ! -z ${GNUGPG} ]] &&
cd ~ &&
rm -rf .gnupg &&
echo ${GNUGPG} | base64 -d | tar --no-same-owner -xzvf -
init: >
[[ ! -z ${GNUGPG_KEY} ]] &&
git config --global commit.gpgsign true &&
git config --global user.signingkey ${GNUGPG_KEY}
command: cd $GITPOD_REPO_ROOT
- name: redis
command: redis-server
- before: go get github.com/mailhog/MailHog
name: mailhog
command: MailHog
- name: Nginx
command: >
nginx &&
multitail /var/log/nginx/access.log -I /var/log/nginx/error.log

44
CHANGES.md
View file

@ -3,13 +3,57 @@
## :rocket: Next
### :sparkles: New features
### :bug: Bugs fixed
### :arrow_up: Deps updates
### :boom: Breaking changes
### :heart: Community contributions by (Thank you!)
## 1.5.0-alpha
### :sparkles: New features
- Add integration with gitpod.io (an online IDE) [#807](https://github.com/penpot/penpot/pull/807)
- Allow basic math operations in inputs [Taiga 1383](https://tree.taiga.io/project/penpot/us/1383)
- Autocomplete color names in hex inputs [Taiga 1596](https://tree.taiga.io/project/penpot/us/1596)
- Allow to group assets (components and graphics) [Taiga #1289](https://tree.taiga.io/project/penpot/us/1289)
- Change icon of pinned projects [Taiga 1298](https://tree.taiga.io/project/penpot/us/1298)
- Internal: refactor of http client, replace internal xhr usage with more modern Fetch API.
- New features for paths: snap points on edition, add/remove nodes, merge/join/split nodes. [Taiga #907](https://tree.taiga.io/project/penpot/us/907)
- Add OpenID-Connect support.
- Reimplement social auth providers on top of the generic openid impl.
### :bug: Bugs fixed
- Fix problem with pan and space [#811](https://github.com/penpot/penpot/issues/811)
- Fix issue when parsing exponential numbers in paths
- Remove legacy system user and team [#843](https://github.com/penpot/penpot/issues/843)
- Fix ordering of copy pasted objects [Taiga #1618](https://tree.taiga.io/project/penpot/issue/1617)
- Fix problems with blending modes [#837](https://github.com/penpot/penpot/issues/837)
- Fix problem with zoom an selection rect [#845](https://github.com/penpot/penpot/issues/845)
- Fix problem displaying team statistics [#859](https://github.com/penpot/penpot/issues/859)
- Fix problems with text editor selection [Taiga #1546](https://tree.taiga.io/project/penpot/issue/1546)
- Fix problem when opening the context menu in dashboard at the bottom [#856](https://github.com/penpot/penpot/issues/856)
- Fix problem when clicking an interactive group in view mode [#863](https://github.com/penpot/penpot/issues/863)
- Fix visibility of pages in sitemap when changing page [Taiga #1618](https://tree.taiga.io/project/penpot/issue/1618)
- Fix visual problem with group invite [Taiga #1290](https://tree.taiga.io/project/penpot/issue/1290)
- Fix issues with promote owner panel [Taiga #763](https://tree.taiga.io/project/penpot/issue/763)
- Allow use library colors when defining gradients [Taiga #1614](https://tree.taiga.io/project/penpot/issue/1614)
- Fix group selrect not updating after alignment [#895](https://github.com/penpot/penpot/issues/895)
### :arrow_up: Deps updates
### :boom: Breaking changes
- Translations refactor: now penpot uses gettext instead of a custom
JSON, and each locale has its own separated file. All translations
should be contributed via the weblate.org service.
### :heart: Community contributions by (Thank you!)
- madmath03 (by [Monogramm](https://github.com/Monogramm)) [#807](https://github.com/penpot/penpot/pull/807)
- zzkt [#814](https://github.com/penpot/penpot/pull/814)
## 1.4.1-alpha

3
README.md
View file

@ -5,6 +5,7 @@
[![License: MPL-2.0][uri_license_image]][uri_license]
[![Gitter](https://badges.gitter.im/sereno-xyz/community.svg)](https://gitter.im/penpot/community)
[![Managed with Taiga.io](https://img.shields.io/badge/managed%20with-TAIGA.io-709f14.svg)](https://tree.taiga.io/project/penpot/ "Managed with Taiga.io")
[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/penpot/penpot)
# PENPOT #
@ -39,4 +40,6 @@ Please refer to the [help center](https://help.penpot.app).
This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at http://mozilla.org/MPL/2.0/.
Copyright (c) UXBOX Labs SL
```

20
backend/deps.edn
View file

@ -4,10 +4,10 @@
"jcenter" {:url "https://jcenter.bintray.com/"}}
:deps
{org.clojure/clojure {:mvn/version "1.10.3"}
org.clojure/clojurescript {:mvn/version "1.10.773"}
org.clojure/data.json {:mvn/version "1.1.0"}
org.clojure/data.json {:mvn/version "2.2.1"}
org.clojure/core.async {:mvn/version "1.3.610"}
org.clojure/tools.cli {:mvn/version "1.0.206"}
org.clojure/clojurescript {:mvn/version "1.10.844"}
;; Logging
org.clojure/tools.logging {:mvn/version "1.1.0"}
@ -15,12 +15,12 @@
org.apache.logging.log4j/log4j-core {:mvn/version "2.14.1"}
org.apache.logging.log4j/log4j-web {:mvn/version "2.14.1"}
org.apache.logging.log4j/log4j-jul {:mvn/version "2.14.1"}
org.apache.logging.log4j/log4j-slf4j-impl {:mvn/version "2.14.1"}
org.slf4j/slf4j-api {:mvn/version "1.7.30"}
org.apache.logging.log4j/log4j-slf4j18-impl {:mvn/version "2.14.1"}
org.slf4j/slf4j-api {:mvn/version "2.0.0-alpha1"}
org.zeromq/jeromq {:mvn/version "0.5.2"}
com.taoensso/nippy {:mvn/version "3.1.1"}
com.github.luben/zstd-jni {:mvn/version "1.4.9-1"}
com.github.luben/zstd-jni {:mvn/version "1.4.9-5"}
;; NOTE: don't upgrade to latest version, breaking change is
;; introduced on 0.10.0 that suffixes counters with _total if they
@ -36,10 +36,10 @@
expound/expound {:mvn/version "0.8.9"}
com.cognitect/transit-clj {:mvn/version "1.0.324"}
io.lettuce/lettuce-core {:mvn/version "6.0.2.RELEASE"}
io.lettuce/lettuce-core {:mvn/version "6.1.1.RELEASE"}
java-http-clj/java-http-clj {:mvn/version "0.4.2"}
info.sunng/ring-jetty9-adapter {:mvn/version "0.15.0"}
info.sunng/ring-jetty9-adapter {:mvn/version "0.15.1"}
com.github.seancorfield/next.jdbc {:mvn/version "1.1.646"}
metosin/reitit-ring {:mvn/version "0.5.12"}
metosin/jsonista {:mvn/version "0.3.1"}
@ -64,12 +64,12 @@
org.im4java/im4java {:mvn/version "1.4.0"}
org.lz4/lz4-java {:mvn/version "1.7.1"}
commons-io/commons-io {:mvn/version "2.8.0"}
com.sun.mail/jakarta.mail {:mvn/version "2.0.0"}
com.sun.mail/jakarta.mail {:mvn/version "2.0.1"}
org.clojars.pntblnk/clj-ldap {:mvn/version "0.0.17"}
integrant/integrant {:mvn/version "0.8.0"}
software.amazon.awssdk/s3 {:mvn/version "2.16.19"}
software.amazon.awssdk/s3 {:mvn/version "2.16.44"}
;; exception printing
io.aviso/pretty {:mvn/version "0.1.37"}
@ -96,7 +96,7 @@
:main-opts ["-m" "kaocha.runner"]}
:outdated
{:extra-deps {com.github.liquidz/antq {:mvn/version "0.12.0"}}
{:extra-deps {com.github.liquidz/antq {:mvn/version "RELEASE"}}
:main-opts ["-m" "antq.core"]}
:jmx-remote

7
backend/dev/user.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2016-2020 Andrey Antukh <niwi@niwi.nz>
;; Copyright (c) UXBOX Labs SL
(ns user
(:require
@ -70,7 +67,7 @@
[]
(alter-var-root #'system (fn [sys]
(when sys (ig/halt! sys))
(-> (main/build-system-config cfg/config)
(-> main/system-config
(ig/prep)
(ig/init))))
:started)

2
backend/resources/emails/invite-to-team/en.subj
View file

@ -1 +1 @@
Inviation to join {{team}}
Invitation to join {{team}}

6
backend/resources/log4j2-devenv.xml
View file

@ -6,7 +6,7 @@
</Console>
<RollingFile name="main" fileName="logs/main.log" filePattern="logs/main-%i.log">
<PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] [%t] %level{length=1} %logger{36} - %msg%n"/>
<PatternLayout pattern="[%d{YYYY-MM-dd HH:mm:ss.SSS}] %level{length=1} %logger{36} - %msg%n"/>
<Policies>
<SizeBasedTriggeringPolicy size="50M"/>
</Policies>
@ -32,6 +32,10 @@
<AppenderRef ref="main" level="trace" />
</Logger>
<Logger name="penpot" level="fatal" additivity="false">
<AppenderRef ref="main" level="fatal" />
</Logger>
<Logger name="user" level="trace" additivity="false">
<AppenderRef ref="main" level="trace" />
</Logger>

4
backend/resources/log4j2.xml
View file

@ -14,6 +14,10 @@
<AppenderRef ref="console" />
</Logger>
<Logger name="penpot" level="fatal" additivity="false">
<AppenderRef ref="console" />
</Logger>
<Root level="info">
<AppenderRef ref="console" />
</Root>

3
backend/scripts/build
View file

@ -4,9 +4,6 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) UXBOX Labs SL
(ns build

2
backend/scripts/repl
View file

@ -2,7 +2,7 @@
export PENPOT_ASSERTS_ENABLED=true
export OPTIONS="-A:jmx-remote:dev -J-Dclojure.tools.logging.factory=clojure.tools.logging.impl/log4j2-factory -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -J-Xms512m -J-Xmx512m -J-Dlog4j2.configurationFile=log4j2-devenv.xml"
export OPTIONS="-A:jmx-remote:dev -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -J-Xms512m -J-Xmx512m -J-Dlog4j2.configurationFile=log4j2-devenv.xml"
export OPTIONS_EVAL="nil"
# export OPTIONS_EVAL="(set! *warn-on-reflection* true)"

46
backend/src/app/cli/fixtures.clj
View file

@ -2,23 +2,19 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.cli.fixtures
"A initial fixtures."
(:require
[app.common.pages :as cp]
[app.common.uuid :as uuid]
[app.config :as cfg]
[app.db :as db]
[app.main :as main]
[app.rpc.mutations.profile :as profile]
[app.util.blob :as blob]
[app.util.logging :as l]
[buddy.hashers :as hashers]
[clojure.tools.logging :as log]
[integrant.core :as ig]))
(defn- mk-uuid
@ -75,7 +71,9 @@
(let [rng (java.util.Random. 1)]
(letfn [(create-profile [conn index]
(let [id (mk-uuid "profile" index)
_ (log/info "create profile" index id)
_ (l/info :action "create profile"
:index index
:id id)
prof (register-profile conn
{:id id
@ -91,20 +89,22 @@
prof))
(create-profiles [conn]
(log/info "create profiles")
(l/info :action "create profiles")
(collect (partial create-profile conn)
(range (:num-profiles opts))))
(create-team [conn index]
(let [id (mk-uuid "team" index)
name (str "Team" index)]
(log/info "create team" index id)
(l/info :action "create team"
:index index
:id id)
(db/insert! conn :team {:id id
:name name})
id))
(create-teams [conn]
(log/info "create teams")
(l/info :action "create teams")
(collect (partial create-team conn)
(range (:num-teams opts))))
@ -112,7 +112,9 @@
(let [id (mk-uuid "file" project-id index)
name (str "file" index)
data (cp/make-file-data id)]
(log/info "create file" index id)
(l/info :action "create file"
:index index
:id id)
(db/insert! conn :file
{:id id
:data (blob/encode data)
@ -127,7 +129,7 @@
id))
(create-files [conn owner-id project-id]
(log/info "create files")
(l/info :action "create files")
(run! (partial create-file conn owner-id project-id)
(range (:num-files-per-project opts))))
@ -139,7 +141,9 @@
(str "project " index)
"Drafts")
is-default (nil? index)]
(log/info "create project" index id)
(l/info :action "create project"
:index index
:id id)
(db/insert! conn :project
{:id id
:team-id team-id
@ -154,7 +158,7 @@
id))
(create-projects [conn team-id profile-ids]
(log/info "create projects")
(l/info :action "create projects")
(let [owner-id (rng-nth rng profile-ids)
project-ids (conj
(collect (partial create-project conn team-id owner-id)
@ -171,14 +175,16 @@
:can-edit true}))
(setup-team [conn team-id profile-ids]
(log/info "setup team" team-id profile-ids)
(l/info :action "setup team"
:team-id team-id
:profile-ids (pr-str profile-ids))
(assign-profile-to-team conn team-id true (first profile-ids))
(run! (partial assign-profile-to-team conn team-id false)
(rest profile-ids))
(create-projects conn team-id profile-ids))
(assign-teams-and-profiles [conn teams profiles]
(log/info "assign teams and profiles")
(l/info :action "assign teams and profiles")
(loop [team-id (first teams)
teams (rest teams)]
(when-not (nil? team-id)
@ -195,7 +201,9 @@
project-id (:default-project-id owner)
data (cp/make-file-data id)]
(log/info "create draft file" index id)
(l/info :action "create draft file"
:index index
:id id)
(db/insert! conn :file
{:id id
:data (blob/encode data)
@ -233,7 +241,7 @@
(defn run
[{:keys [preset] :or {preset :small}}]
(let [config (select-keys (main/build-system-config cfg/config)
(let [config (select-keys main/system-config
[:app.db/pool
:app.telemetry/migrations
:app.migrations/migrations
@ -245,6 +253,6 @@
(try
(run-in-system system preset)
(catch Exception e
(log/errorf e "unhandled exception"))
(l/error :hint "unhandled exception" :cause e))
(finally
(ig/halt! system)))))

12
backend/src/app/cli/manage.clj
View file

@ -2,22 +2,18 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.cli.manage
"A manage cli api."
(:require
[app.config :as cfg]
[app.db :as db]
[app.main :as main]
[app.rpc.mutations.profile :as profile]
[app.rpc.queries.profile :refer [retrieve-profile-data-by-email]]
[app.util.logging :as l]
[clojure.string :as str]
[clojure.tools.cli :refer [parse-opts]]
[clojure.tools.logging :as log]
[integrant.core :as ig])
(:import
java.io.Console))
@ -26,7 +22,7 @@
(defn init-system
[]
(let [data (-> (main/build-system-config cfg/config)
(let [data (-> main/system-config
(select-keys [:app.db/pool :app.metrics/metrics])
(assoc :app.migrations/all {}))]
(-> data ig/prep ig/init)))
@ -35,7 +31,7 @@
[{:keys [label type] :or {type :text}}]
(let [^Console console (System/console)]
(when-not console
(log/error "no console found, can proceed")
(l/error :hint "no console found, can proceed")
(System/exit 1))
(binding [*out* (.writer console)]

19
backend/src/app/cli/migrate_media.clj
View file

@ -2,19 +2,16 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.cli.migrate-media
(:require
[app.common.media :as cm]
[app.config :as cfg]
[app.config :as cf]
[app.db :as db]
[app.main :as main]
[app.storage :as sto]
[clojure.tools.logging :as log]
[app.util.logging :as l]
[cuerdas.core :as str]
[datoteka.core :as fs]
[integrant.core :as ig]))
@ -34,7 +31,7 @@
(defn run
[]
(let [config (select-keys (main/build-system-config cfg/config)
(let [config (select-keys main/system-config
[:app.db/pool
:app.migrations/migrations
:app.metrics/metrics
@ -49,7 +46,7 @@
(run-in-system)
(ig/halt!))
(catch Exception e
(log/errorf e "Unhandled exception.")))))
(l/error :hint "unhandled exception" :cause e)))))
;; --- IMPL
@ -60,7 +57,7 @@
(->> (db/exec! conn ["select * from profile"])
(filter #(not (str/empty? (:photo %))))
(seq)))]
(let [base (fs/path (:storage-fs-old-directory cfg/config))
(let [base (fs/path (cf/get :storage-fs-old-directory))
storage (-> (:app.storage/storage system)
(assoc :conn conn))]
(doseq [profile (retrieve-profiles conn)]
@ -81,7 +78,7 @@
(->> (db/exec! conn ["select * from team"])
(filter #(not (str/empty? (:photo %))))
(seq)))]
(let [base (fs/path (:storage-fs-old-directory cfg/config))
(let [base (fs/path (cf/get :storage-fs-old-directory))
storage (-> (:app.storage/storage system)
(assoc :conn conn))]
(doseq [team (retrieve-teams conn)]
@ -105,7 +102,7 @@
from file_media_object as fmo
join file_media_thumbnail as fth on (fth.media_object_id = fmo.id)"])
(seq)))]
(let [base (fs/path (:storage-fs-old-directory cfg/config))
(let [base (fs/path (cf/get :storage-fs-old-directory))
storage (-> (:app.storage/storage system)
(assoc :conn conn))]
(doseq [mobj (retrieve-media-objects conn)]

77
backend/src/app/config.clj
View file

@ -2,9 +2,6 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) UXBOX Labs SL
(ns app.config
@ -16,10 +13,19 @@
[app.util.time :as dt]
[clojure.core :as c]
[clojure.java.io :as io]
[clojure.pprint :as pprint]
[clojure.spec.alpha :as s]
[cuerdas.core :as str]
[environ.core :refer [env]]))
(prefer-method print-method
clojure.lang.IRecord
clojure.lang.IDeref)
(prefer-method pprint/simple-dispatch
clojure.lang.IPersistentMap
clojure.lang.IDeref)
(def defaults
{:http-server-port 6060
:host "devenv"
@ -99,9 +105,17 @@
(s/def ::gitlab-client-secret ::us/string)
(s/def ::google-client-id ::us/string)
(s/def ::google-client-secret ::us/string)
(s/def ::oidc-client-id ::us/string)
(s/def ::oidc-client-secret ::us/string)
(s/def ::oidc-base-uri ::us/string)
(s/def ::oidc-token-uri ::us/string)
(s/def ::oidc-auth-uri ::us/string)
(s/def ::oidc-user-uri ::us/string)
(s/def ::oidc-scopes ::us/set-of-str)
(s/def ::oidc-roles ::us/set-of-str)
(s/def ::oidc-roles-attr ::us/keyword)
(s/def ::host ::us/string)
(s/def ::http-server-port ::us/integer)
(s/def ::http-session-cookie-name ::us/string)
(s/def ::http-session-idle-max-age ::dt/duration)
(s/def ::http-session-updater-batch-max-age ::dt/duration)
(s/def ::http-session-updater-batch-max-size ::us/integer)
@ -172,6 +186,15 @@
::gitlab-client-secret
::google-client-id
::google-client-secret
::oidc-client-id
::oidc-client-secret
::oidc-base-uri
::oidc-token-uri
::oidc-auth-uri
::oidc-user-uri
::oidc-scopes
::oidc-roles-attr
::oidc-roles
::host
::http-server-port
::http-session-idle-max-age
@ -222,42 +245,33 @@
::telemetry-server-enabled
::telemetry-server-port
::telemetry-uri
::telemetry-referer
::telemetry-with-taiga
::tenant]))
(defn- env->config
[env]
(reduce-kv
(fn [acc k v]
(cond-> acc
(str/starts-with? (name k) "penpot-")
(assoc (keyword (subs (name k) 7)) v)
(str/starts-with? (name k) "app-")
(assoc (keyword (subs (name k) 4)) v)))
{}
env))
(defn read-env
[prefix]
(let [prefix (str prefix "-")
len (count prefix)]
(reduce-kv
(fn [acc k v]
(cond-> acc
(str/starts-with? (name k) prefix)
(assoc (keyword (subs (name k) len)) v)))
{}
env)))
(defn- read-config
[env]
(->> (env->config env)
[]
(->> (read-env "penpot")
(merge defaults)
(us/conform ::config)))
(defn- read-test-config
[env]
(merge {:redis-uri "redis://redis/1"
:database-uri "postgresql://postgres/penpot_test"
:storage-fs-directory "/tmp/app/storage"
:migrations-verbose false}
(read-config env)))
(def version (v/parse (or (some-> (io/resource "version.txt")
(slurp)
(str/trim))
"%version%")))
(def config (read-config env))
(def test-config (read-test-config env))
(def config (atom (read-config)))
(def deletion-delay
(dt/duration {:days 7}))
@ -265,6 +279,9 @@
(defn get
"A configuration getter. Helps code be more testable."
([key]
(c/get config key))
(c/get @config key))
([key default]
(c/get config key default)))
(c/get @config key default)))
;; Set value for all new threads bindings.
(alter-var-root #'*assert* (constantly (get :asserts-enabled)))

20
backend/src/app/db.clj
View file

@ -2,25 +2,23 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.db
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.geom.point :as gpt]
[app.common.spec :as us]
[app.db.sql :as sql]
[app.metrics :as mtx]
[app.util.json :as json]
[app.util.logging :as l]
[app.util.migrations :as mg]
[app.util.time :as dt]
[app.util.transit :as t]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]
[next.jdbc :as jdbc]
[next.jdbc.date-time :as jdbc-dt])
@ -48,8 +46,8 @@
(declare instrument-jdbc!)
(s/def ::name keyword?)
(s/def ::uri ::us/not-empty-string)
(s/def ::name ::us/not-empty-string)
(s/def ::min-pool-size ::us/integer)
(s/def ::max-pool-size ::us/integer)
(s/def ::migrations map?)
@ -59,14 +57,16 @@
(defmethod ig/init-key ::pool
[_ {:keys [migrations metrics] :as cfg}]
(log/infof "initialize connection pool '%s' with uri '%s'" (:name cfg) (:uri cfg))
(l/info :action "initialize connection pool"
:name (d/name (:name cfg))
:uri (:uri cfg))
(instrument-jdbc! (:registry metrics))
(let [pool (create-pool cfg)]
(when (seq migrations)
(with-open [conn ^AutoCloseable (open pool)]
(mg/setup! conn)
(doseq [[mname steps] migrations]
(mg/migrate! conn {:name (name mname) :steps steps}))))
(doseq [[name steps] migrations]
(mg/migrate! conn {:name (d/name name) :steps steps}))))
pool))
(defmethod ig/halt-key! ::pool
@ -100,7 +100,7 @@
mtf (PrometheusMetricsTrackerFactory. (:registry metrics))]
(doto config
(.setJdbcUrl (str "jdbc:" dburi))
(.setPoolName (:name cfg "default"))
(.setPoolName (d/name (:name cfg)))
(.setAutoCommit true)
(.setReadOnly false)
(.setConnectionTimeout 8000) ;; 8seg

5
backend/src/app/db/sql.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.db.sql
(:refer-clojure :exclude [update])

99
backend/src/app/emails.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.emails
"Main api for send emails."
@ -14,18 +11,13 @@
[app.config :as cfg]
[app.db :as db]
[app.db.sql :as sql]
[app.tasks :as tasks]
[app.util.emails :as emails]
[clojure.spec.alpha :as s]))
[app.util.logging :as l]
[app.worker :as wrk]
[clojure.spec.alpha :as s]
[integrant.core :as ig]))
;; --- Defaults
(defn default-context
[]
{:assets-uri (:assets-uri cfg/config)
:public-uri (:public-uri cfg/config)})
;; --- Public API
;; --- PUBLIC API
(defn render
[email-factory context]
@ -33,17 +25,20 @@
(defn send!
"Schedule the email for sending."
[conn email-factory context]
(us/verify fn? email-factory)
(us/verify map? context)
(let [email (email-factory context)]
(tasks/submit! conn {:name "sendmail"
:delay 0
:max-retries 1
:priority 200
:props email})))
[{:keys [::conn ::factory] :as context}]
(us/verify fn? factory)
(us/verify some? conn)
(let [email (factory context)]
(wrk/submit! (assoc email
::wrk/task :sendmail
::wrk/delay 0
::wrk/max-retries 1
::wrk/priority 200
::wrk/conn conn))))
;; --- BOUNCE/COMPLAINS HANDLING
(def sql:profile-complaint-report
"select (select count(*)
from profile_complaint_report
@ -91,7 +86,7 @@
(>= (count reports) threshold))))
;; --- Emails
;; --- EMAIL FACTORIES
(s/def ::subject ::us/string)
(s/def ::content ::us/string)
@ -101,7 +96,7 @@
(def feedback
"A profile feedback email."
(emails/template-factory ::feedback default-context))
(emails/template-factory ::feedback))
(s/def ::name ::us/string)
(s/def ::register
@ -109,7 +104,7 @@
(def register
"A new profile registration welcome email."
(emails/template-factory ::register default-context))
(emails/template-factory ::register))
(s/def ::token ::us/string)
(s/def ::password-recovery
@ -117,7 +112,7 @@
(def password-recovery
"A password recovery notification email."
(emails/template-factory ::password-recovery default-context))
(emails/template-factory ::password-recovery))
(s/def ::pending-email ::us/email)
(s/def ::change-email
@ -125,7 +120,7 @@
(def change-email
"Password change confirmation email"
(emails/template-factory ::change-email default-context))
(emails/template-factory ::change-email))
(s/def :internal.emails.invite-to-team/invited-by ::us/string)
(s/def :internal.emails.invite-to-team/team ::us/string)
@ -138,4 +133,50 @@
(def invite-to-team
"Teams member invitation email."
(emails/template-factory ::invite-to-team default-context))
(emails/template-factory ::invite-to-team))
;; --- SENDMAIL TASK
(declare send-console!)
(s/def ::username ::cfg/smtp-username)
(s/def ::password ::cfg/smtp-password)
(s/def ::tls ::cfg/smtp-tls)
(s/def ::ssl ::cfg/smtp-ssl)
(s/def ::host ::cfg/smtp-host)
(s/def ::port ::cfg/smtp-port)
(s/def ::default-reply-to ::cfg/smtp-default-reply-to)
(s/def ::default-from ::cfg/smtp-default-from)
(s/def ::enabled ::cfg/smtp-enabled)
(defmethod ig/pre-init-spec ::sendmail-handler [_]
(s/keys :req-un [::enabled]
:opt-un [::username
::password
::tls
::ssl
::host
::port
::default-from
::default-reply-to]))
(defmethod ig/init-key ::sendmail-handler
[_ cfg]
(fn [{:keys [props] :as task}]
(if (:enabled cfg)
(emails/send! cfg props)
(send-console! cfg props))))
(defn- send-console!
[cfg email]
(let [baos (java.io.ByteArrayOutputStream.)
mesg (emails/smtp-message cfg email)]
(.writeTo mesg baos)
(let [out (with-out-str
(println "email console dump:")
(println "******** start email" (:id email) "**********")
(println (.toString baos))
(println "******** end email "(:id email) "**********"))]
(l/info :email out))))

127
backend/src/app/http.clj
View file

@ -2,22 +2,18 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.http
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.config :as cfg]
[app.http.errors :as errors]
[app.http.middleware :as middleware]
[app.metrics :as mtx]
[app.util.log4j :refer [update-thread-context!]]
[app.util.logging :as l]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]
[reitit.ring :as rr]
[ring.adapter.jetty9 :as jetty])
@ -26,30 +22,32 @@
org.eclipse.jetty.server.handler.ErrorHandler
org.eclipse.jetty.server.handler.StatisticsHandler))
(declare router-handler)
(s/def ::handler fn?)
(s/def ::router some?)
(s/def ::ws (s/map-of ::us/string fn?))
(s/def ::port ::cfg/http-server-port)
(s/def ::port ::us/integer)
(s/def ::name ::us/string)
(defmethod ig/pre-init-spec ::server [_]
(s/keys :req-un [::handler ::port]
:opt-un [::ws ::name ::mtx/metrics]))
(s/keys :req-un [::port]
:opt-un [::ws ::name ::mtx/metrics ::router ::handler]))
(defmethod ig/prep-key ::server
[_ cfg]
(merge {:name "http"}
(d/without-nils cfg)))
(merge {:name "http"} (d/without-nils cfg)))
(defmethod ig/init-key ::server
[_ {:keys [handler ws port name metrics] :as opts}]
(log/infof "starting '%s' server on port %s." name port)
[_ {:keys [handler router ws port name metrics] :as opts}]
(l/info :msg "starting http server" :port port :name name)
(let [pre-start (fn [^Server server]
(let [handler (doto (ErrorHandler.)
(.setShowStacks true)
(.setServer server))]
(.setErrorHandler server ^ErrorHandler handler)
(when metrics
(let [stats (new StatisticsHandler)]
(let [stats (StatisticsHandler.)]
(.setHandler ^StatisticsHandler stats (.getHandler server))
(.setHandler server stats)
(mtx/instrument-jetty! (:registry metrics) stats)))))
@ -63,61 +61,71 @@
(when (seq ws)
{:websockets ws}))
handler (cond
(fn? handler) handler
(some? router) (router-handler router)
:else (ex/raise :type :internal
:code :invalid-argument
:hint "Missing `handler` or `router` option."))
server (jetty/run-jetty handler options)]
(assoc opts :server server)))
(defmethod ig/halt-key! ::server
[_ {:keys [server name port] :as opts}]
(log/infof "stoping '%s' server on port %s." name port)
(l/info :msg "stoping http server"
:name name
:port port)
(jetty/stop-server server))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Http Main Handler (Router)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(declare create-router)
(s/def ::rpc map?)
(s/def ::session map?)
(s/def ::metrics map?)
(s/def ::oauth map?)
(s/def ::storage map?)
(s/def ::assets map?)
(s/def ::feedback fn?)
(defmethod ig/pre-init-spec ::router [_]
(s/keys :req-un [::rpc ::session ::metrics ::oauth ::storage ::assets ::feedback]))
(defmethod ig/init-key ::router
[_ cfg]
(let [handler (rr/ring-handler
(create-router cfg)
(rr/routes
(rr/create-resource-handler {:path "/"})
(rr/create-default-handler))
{:middleware [middleware/server-timing]})]
(defn- router-handler
[router]
(let [handler (rr/ring-handler router
(rr/routes
(rr/create-resource-handler {:path "/"})
(rr/create-default-handler))
{:middleware [middleware/server-timing]})]
(fn [request]
(try
(handler request)
(catch Throwable e
(try
(let [cdata (errors/get-error-context request e)]
(update-thread-context! cdata)
(log/errorf e "unhandled exception: %s (id: %s)" (ex-message e) (str (:id cdata)))
{:status 500
:body "internal server error"})
(l/update-thread-context! cdata)
(l/error :hint "unhandled exception"
:message (ex-message e)
:error-id (str (:id cdata))
:cause e))
{:status 500 :body "internal server error"}
(catch Throwable e
(log/errorf e "unhandled exception: %s" (ex-message e))
{:status 500
:body "internal server error"})))))))
(l/error :hint "unhandled exception"
:message (ex-message e)
:cause e)
{:status 500 :body "internal server error"})))))))
(defn- create-router
[{:keys [session rpc oauth metrics svgparse assets feedback] :as cfg}]
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Http Main Handler (Router)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(s/def ::rpc map?)
(s/def ::session map?)
(s/def ::oauth map?)
(s/def ::storage map?)
(s/def ::assets map?)
(s/def ::feedback fn?)
(defmethod ig/pre-init-spec ::router [_]
(s/keys :req-un [::rpc ::session ::mtx/metrics ::oauth ::storage ::assets ::feedback]))
(defmethod ig/init-key ::router
[_ {:keys [session rpc oauth metrics assets feedback] :as cfg}]
(rr/router
[["/metrics" {:get (:handler metrics)}]
["/assets" {:middleware [[middleware/format-response-body]
[middleware/errors errors/handle]]}
[middleware/errors errors/handle]
[middleware/cookies]
(:middleware session)]}
["/by-id/:id" {:get (:objects-handler assets)}]
["/by-file-media-id/:id" {:get (:file-objects-handler assets)}]
["/by-file-media-id/:id/thumbnail" {:get (:file-thumbnails-handler assets)}]]
@ -137,20 +145,13 @@
[middleware/errors errors/handle]
[middleware/cookies]]}
["/svg/parse" {:post svgparse}]
["/feedback" {:middleware [(:middleware session)]
:post feedback}]
["/oauth"
["/google" {:post (get-in oauth [:google :handler])}]
["/google/callback" {:get (get-in oauth [:google :callback-handler])}]
["/gitlab" {:post (get-in oauth [:gitlab :handler])}]
["/gitlab/callback" {:get (get-in oauth [:gitlab :callback-handler])}]
["/github" {:post (get-in oauth [:github :handler])}]
["/github/callback" {:get (get-in oauth [:github :callback-handler])}]]
["/auth/oauth/:provider" {:post (:handler oauth)}]
["/auth/oauth/:provider/callback" {:get (:callback-handler oauth)}]
["/rpc" {:middleware [(:middleware session)]}
["/query/:type" {:get (:query-handler rpc)}]
["/query/:type" {:get (:query-handler rpc)
:post (:query-handler rpc)}]
["/mutation/:type" {:post (:mutation-handler rpc)}]]]]))

9
backend/src/app/http/assets.clj
View file

@ -2,23 +2,20 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.http.assets
"Assets related handlers."
(:require
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.common.uri :as u]
[app.db :as db]
[app.metrics :as mtx]
[app.storage :as sto]
[app.util.time :as dt]
[clojure.spec.alpha :as s]
[integrant.core :as ig]
[lambdaisland.uri :as u]))
[integrant.core :as ig]))
(def ^:private cache-max-age
(dt/duration {:hours 24}))

31
backend/src/app/http/awsns.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 Andrey Antukh <niwi@niwi.nz>
;; Copyright (c) UXBOX Labs SL
(ns app.http.awsns
"AWS SNS webhook handler for bounces."
@ -14,9 +11,8 @@
[app.db :as db]
[app.db.sql :as sql]
[app.util.http :as http]
[clojure.pprint :refer [pprint]]
[app.util.logging :as l]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[cuerdas.core :as str]
[integrant.core :as ig]
[jsonista.core :as j]))
@ -25,11 +21,6 @@
(declare parse-notification)
(declare process-report)
(defn- pprint-report
[message]
(binding [clojure.pprint/*print-right-margin* 120]
(with-out-str (pprint message))))
(defmethod ig/pre-init-spec ::handler [_]
(s/keys :req-un [::db/pool]))
@ -42,19 +33,17 @@
(= mtype "SubscriptionConfirmation")
(let [surl (get body "SubscribeURL")
stopic (get body "TopicArn")]
(log/infof "subscription received (topic=%s, url=%s)" stopic surl)
(l/info :action "subscription received" :topic stopic :url surl)
(http/send! {:uri surl :method :post :timeout 10000}))
(= mtype "Notification")
(when-let [message (parse-json (get body "Message"))]
;; (log/infof "Received: %s" (pr-str message))
(let [notification (parse-notification cfg message)]
(process-report cfg notification)))
:else
(log/warn (str "unexpected data received\n"
(pprint-report body))))
(l/warn :hint "unexpected data received"
:report (pr-str body)))
{:status 200 :body ""})))
(defn- parse-bounce
@ -184,15 +173,15 @@
(defn- process-report
[cfg {:keys [type profile-id] :as report}]
(log/trace (str "procesing report:\n" (pprint-report report)))
(l/trace :action "procesing report" :report (pr-str report))
(cond
;; In this case we receive a bounce/complaint notification without
;; confirmed identity, we just emit a warning but do nothing about
;; it because this is not a normal case. All notifications should
;; come with profile identity.
(nil? profile-id)
(log/warn (str "a notification without identity recevied from AWS\n"
(pprint-report report)))
(l/warn :msg "a notification without identity recevied from AWS"
:report (pr-str report))
(= "bounce" type)
(register-bounce-for-profile cfg report)
@ -201,7 +190,7 @@
(register-complaint-for-profile cfg report)
:else
(log/warn (str "unrecognized report received from AWS\n"
(pprint-report report)))))
(l/warn :msg "unrecognized report received from AWS"
:report (pr-str report))))

34
backend/src/app/http/errors.clj
View file

@ -2,18 +2,14 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.http.errors
"A errors handling for the http server."
(:require
[app.common.exceptions :as ex]
[app.common.uuid :as uuid]
[app.util.log4j :refer [update-thread-context!]]
[clojure.tools.logging :as log]
[app.util.logging :as l]
[cuerdas.core :as str]
[expound.alpha :as expound]))
@ -73,8 +69,11 @@
[error request]
(let [edata (ex-data error)
cdata (get-error-context request error)]
(update-thread-context! cdata)
(log/errorf error "internal error: assertion (id: %s)" (str (:id cdata)))
(l/update-thread-context! cdata)
(l/error :hint "internal error: assertion"
:error-id (str (:id cdata))
:cause error)
{:status 500
:body {:type :server-error
:data (-> edata
@ -97,10 +96,11 @@
(ex/exception? (:handling edata)))
(handle-exception (:handling edata) request)
(let [cdata (get-error-context request error)]
(update-thread-context! cdata)
(log/errorf error "internal error: %s (id: %s)"
(ex-message error)
(str (:id cdata)))
(l/update-thread-context! cdata)
(l/error :hint "internal error"
:error-message (ex-message error)
:error-id (str (:id cdata))
:cause error)
{:status 500
:body {:type :server-error
:hint (ex-message error)
@ -111,11 +111,11 @@
(let [cdata (get-error-context request error)
state (.getSQLState ^java.sql.SQLException error)]
(update-thread-context! cdata)
(log/errorf error "PSQL Exception: %s (id: %s, state: %s)"
(ex-message error)
(str (:id cdata))
state)
(l/update-thread-context! cdata)
(l/error :hint "psql exception"
:error-message (ex-message error)
:error-id (str (:id cdata))
:sql-state state)
(cond
(= state "57014")

22
backend/src/app/http/feedback.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.http.feedback
"A general purpose feedback module."
@ -15,7 +12,7 @@
[app.common.spec :as us]
[app.config :as cfg]
[app.db :as db]
[app.emails :as emails]
[app.emails :as eml]
[app.rpc.queries.profile :as profile]
[clojure.spec.alpha :as s]
[integrant.core :as ig]))
@ -62,11 +59,12 @@
[pool profile params]
(let [params (us/conform ::feedback params)
destination (cfg/get :feedback-destination)]
(emails/send! pool emails/feedback
{:to destination
:profile profile
:reply-to (:from params)
:email (:from params)
:subject (:subject params)
:content (:content params)})
(eml/send! {::eml/conn pool
::eml/factory eml/feedback
:to destination
:profile profile
:reply-to (:from params)
:email (:from params)
:subject (:subject params)
:content (:content params)})
nil))

21
backend/src/app/http/middleware.clj
View file

@ -2,15 +2,13 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.http.middleware
(:require
[app.metrics :as mtx]
[app.util.json :as json]
[app.util.logging :as l]
[app.util.transit :as t]
[buddy.core.codecs :as bc]
[buddy.core.hash :as bh]
@ -165,3 +163,18 @@
(def etag
{:name ::etag
:compile (constantly wrap-etag)})
(defn activity-logger
[handler]
(let [logger "penpot.profile-activity"]
(fn [{:keys [headers] :as request}]
(let [ip-addr (get headers "x-forwarded-for")
profile-id (:profile-id request)
qstring (:query-string request)]
(l/info ::l/async true
::l/logger logger
:ip-addr ip-addr
:profile-id profile-id
:uri (str (:uri request) (when qstring (str "?" qstring)))
:method (name (:request-method request)))
(handler request)))))

298
backend/src/app/http/oauth.clj Normal file
View file

@ -0,0 +1,298 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) UXBOX Labs SL
(ns app.http.oauth
(:require
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.common.uri :as u]
[app.config :as cf]
[app.util.http :as http]
[app.util.logging :as l]
[app.util.time :as dt]
[clojure.data.json :as json]
[clojure.set :as set]
[clojure.spec.alpha :as s]
[cuerdas.core :as str]
[integrant.core :as ig]))
(defn redirect-response
[uri]
{:status 302
:headers {"location" (str uri)}
:body ""})
(defn generate-error-redirect-uri
[cfg]
(-> (u/uri (:public-uri cfg))
(assoc :path "/#/auth/login")
(assoc :query (u/map->query-string {:error "unable-to-auth"}))))
(defn register-profile
[{:keys [rpc] :as cfg} info]
(let [method-fn (get-in rpc [:methods :mutation :login-or-register])
profile (method-fn info)]
(cond-> profile
(some? (:invitation-token info))
(assoc :invitation-token (:invitation-token info)))))
(defn generate-redirect-uri
[{:keys [tokens] :as cfg} profile]
(let [token (or (:invitation-token profile)
(tokens :generate {:iss :auth
:exp (dt/in-future "15m")
:profile-id (:id profile)}))]
(-> (u/uri (:public-uri cfg))
(assoc :path "/#/auth/verify-token")
(assoc :query (u/map->query-string {:token token})))))
(defn- build-redirect-uri
[{:keys [provider] :as cfg}]
(let [public (u/uri (:public-uri cfg))]
(str (assoc public :path (str "/api/auth/oauth/" (:name provider) "/callback")))))
(defn- build-auth-uri
[{:keys [provider] :as cfg} state]
(let [params {:client_id (:client-id provider)
:redirect_uri (build-redirect-uri cfg)
:response_type "code"
:state state
:scope (str/join " " (:scopes provider []))}
query (u/map->query-string params)]
(-> (u/uri (:auth-uri provider))
(assoc :query query)
(str))))
(defn retrieve-access-token
[{:keys [provider] :as cfg} code]
(try
(let [params {:client_id (:client-id provider)
:client_secret (:client-secret provider)
:code code
:grant_type "authorization_code"
:redirect_uri (build-redirect-uri cfg)}
req {:method :post
:headers {"content-type" "application/x-www-form-urlencoded"}
:uri (:token-uri provider)
:body (u/map->query-string params)}
res (http/send! req)]
(when (= 200 (:status res))
(let [data (json/read-str (:body res))]
{:token (get data "access_token")
:type (get data "token_type")})))
(catch Exception e
(l/error :hint "unexpected error on retrieve-access-token"
:cause e)
nil)))
(defn- retrieve-user-info
[{:keys [provider] :as cfg} tdata]
(try
(let [req {:uri (:user-uri provider)
:headers {"Authorization" (str (:type tdata) " " (:token tdata))}
:timeout 6000
:method :get}
res (http/send! req)]
(when (= 200 (:status res))
(let [{:keys [name] :as data} (json/read-str (:body res) :key-fn keyword)]
(-> data
(assoc :backend (:name provider))
(assoc :fullname name)))))
(catch Exception e
(l/error :hint "unexpected exception on retrieve-user-info"
:cause e)
nil)))
(defn retrieve-info
[{:keys [tokens provider] :as cfg} request]
(let [state (get-in request [:params :state])
state (tokens :verify {:token state :iss :oauth})
info (some->> (get-in request [:params :code])
(retrieve-access-token cfg)
(retrieve-user-info cfg))]
(when-not info
(ex/raise :type :internal
:code :unable-to-auth))
;; If the provider is OIDC, we can proceed to check
;; roles if they are defined.
(when (and (= "oidc" (:name provider))
(seq (:roles provider)))
(let [provider-roles (into #{} (:roles provider))
profile-roles (let [attr (cf/get :oidc-roles-attr :roles)
roles (get info attr)]
(cond
(string? roles) (into #{} (str/words roles))
(vector? roles) (into #{} roles)
:else #{}))]
;; check if profile has a configured set of roles
(when-not (set/subset? provider-roles profile-roles)
(ex/raise :type :internal
:code :unable-to-auth
:hint "not enought permissions"))))
(cond-> info
(some? (:invitation-token state))
(assoc :invitation-token (:invitation-token state)))))
;; --- HTTP HANDLERS
(defn- auth-handler
[{:keys [tokens] :as cfg} request]
(let [invitation (get-in request [:params :invitation-token])
state (tokens :generate
{:iss :oauth
:invitation-token invitation
:exp (dt/in-future "15m")})
uri (build-auth-uri cfg state)]
{:status 200
:body {:redirect-uri uri}}))
(defn- callback-handler
[{:keys [session] :as cfg} request]
(try
(let [info (retrieve-info cfg request)
profile (register-profile cfg info)
uri (generate-redirect-uri cfg profile)
sxf ((:create session) (:id profile))]
(->> (redirect-response uri)
(sxf request)))
(catch Exception _e
(-> (generate-error-redirect-uri cfg)
(redirect-response)))))
;; --- INIT
(declare initialize)
(s/def ::public-uri ::us/not-empty-string)
(s/def ::session map?)
(s/def ::tokens fn?)
(s/def ::rpc map?)
(defmethod ig/pre-init-spec :app.http.oauth/handlers [_]
(s/keys :req-un [::public-uri ::session ::tokens ::rpc]))
(defn wrap-handler
[cfg handler]
(fn [request]
(let [provider (get-in request [:path-params :provider])
provider (get-in @cfg [:providers provider])]
(when-not provider
(ex/raise :type :not-found
:context {:provider provider}
:hint "provider not configured"))
(-> (assoc @cfg :provider provider)
(handler request)))))
(defmethod ig/init-key :app.http.oauth/handlers
[_ cfg]
(let [cfg (initialize cfg)]
{:handler (wrap-handler cfg auth-handler)
:callback-handler (wrap-handler cfg callback-handler)}))
(defn- discover-oidc-config
[{:keys [base-uri] :as opts}]
(let [discovery-uri (u/join base-uri ".well-known/openid-configuration")
response (http/send! {:method :get :uri (str discovery-uri)})]
(when (= 200 (:status response))
(let [data (json/read-str (:body response))]
(assoc opts
:token-uri (get data "token_endpoint")
:auth-uri (get data "authorization_endpoint")
:user-uri (get data "userinfo_endpoint"))))))
(defn- initialize-oidc-provider
[cfg]
(let [opts {:base-uri (cf/get :oidc-base-uri)
:client-id (cf/get :oidc-client-id)
:client-secret (cf/get :oidc-client-secret)
:token-uri (cf/get :oidc-token-uri)
:auth-uri (cf/get :oidc-auth-uri)
:user-uri (cf/get :oidc-user-uri)
:scopes (into #{"openid" "profile" "email" "name"}
(cf/get :oidc-scopes #{}))
:roles-attr (cf/get :oidc-roles-attr)
:roles (cf/get :oidc-roles)
:name "oidc"}]
(if (and (string? (:base-uri opts))
(string? (:client-id opts))
(string? (:client-secret opts)))
(if (and (string? (:token-uri opts))
(string? (:user-uri opts))
(string? (:auth-uri opts)))
(do
(l/info :action "initialize" :provider "oid" :method "static")
(assoc-in cfg [:providers "oidc"] opts))
(let [opts (discover-oidc-config opts)]
(l/info :action "initialize" :provider "oid" :method "discover")
(assoc-in cfg [:providers "oidc"] opts)))
cfg)))
(defn- initialize-google-provider
[cfg]
(let [opts {:client-id (cf/get :google-client-id)
:client-secret (cf/get :google-client-secret)
:scopes #{"email" "profile" "openid"
"https://www.googleapis.com/auth/userinfo.email"
"https://www.googleapis.com/auth/userinfo.profile"}
:auth-uri "https://accounts.google.com/o/oauth2/v2/auth"
:token-uri "https://oauth2.googleapis.com/token"
:user-uri "https://openidconnect.googleapis.com/v1/userinfo"
:name "google"}]
(if (and (string? (:client-id opts))
(string? (:client-secret opts)))
(do
(l/info :action "initialize" :provider "google")
(assoc-in cfg [:providers "google"] opts))
cfg)))
(defn- initialize-github-provider
[cfg]
(let [opts {:client-id (cf/get :github-client-id)
:client-secret (cf/get :github-client-secret)
:scopes #{"read:user"
"user:email"}
:auth-uri "https://github.com/login/oauth/authorize"
:token-uri "https://github.com/login/oauth/access_token"
:user-uri "https://api.github.com/user"
:name "github"}]
(if (and (string? (:client-id opts))
(string? (:client-secret opts)))
(do
(l/info :action "initialize" :provider "github")
(assoc-in cfg [:providers "github"] opts))
cfg)))
(defn- initialize-gitlab-provider
[cfg]
(let [base (cf/get :gitlab-base-uri "https://gitlab.com")
opts {:base-uri base
:client-id (cf/get :gitlab-client-id)
:client-secret (cf/get :gitlab-client-secret)
:scopes #{"read_user"}
:auth-uri (str base "/oauth/authorize")
:token-uri (str base "/oauth/token")
:user-uri (str base "/api/v4/user")
:name "gitlab"}]
(if (and (string? (:client-id opts))
(string? (:client-secret opts)))
(do
(l/info :action "initialize" :provider "gitlab")
(assoc-in cfg [:providers "gitlab"] opts))
cfg)))
(defn- initialize
[cfg]
(let [cfg (agent cfg :error-mode :continue)]
(send-off cfg initialize-google-provider)
(send-off cfg initialize-gitlab-provider)
(send-off cfg initialize-github-provider)
(send-off cfg initialize-oidc-provider)
cfg))

159
backend/src/app/http/oauth/github.clj
View file

@ -1,159 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
(ns app.http.oauth.github
(:require
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.config :as cfg]
[app.http.oauth.google :as gg]
[app.util.http :as http]
[app.util.time :as dt]
[clojure.data.json :as json]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]
[lambdaisland.uri :as u]))
(def base-github-uri
(u/uri "https://github.com"))
(def base-api-github-uri
(u/uri "https://api.github.com"))
(def authorize-uri
(assoc base-github-uri :path "/login/oauth/authorize"))
(def token-url
(assoc base-github-uri :path "/login/oauth/access_token"))
(def user-info-url
(assoc base-api-github-uri :path "/user"))
(def scope "user:email")
(defn- build-redirect-url
[cfg]
(let [public (u/uri (:public-uri cfg))]
(str (assoc public :path "/api/oauth/github/callback"))))
(defn- get-access-token
[cfg state code]
(try
(let [params {:client_id (:client-id cfg)
:client_secret (:client-secret cfg)
:code code
:state state
:redirect_uri (build-redirect-url cfg)}
req {:method :post
:headers {"content-type" "application/x-www-form-urlencoded"
"accept" "application/json"}
:uri (str token-url)
:timeout 6000
:body (u/map->query-string params)}
res (http/send! req)]
(when (= 200 (:status res))
(-> (json/read-str (:body res))
(get "access_token"))))
(catch Exception e
(log/error e "unexpected error on get-access-token")
nil)))
(defn- get-user-info
[_ token]
(try
(let [req {:uri (str user-info-url)
:headers {"authorization" (str "token " token)}
:timeout 6000
:method :get}
res (http/send! req)]
(when (= 200 (:status res))
(let [data (json/read-str (:body res))]
{:email (get data "email")
:backend "github"
:fullname (get data "name")})))
(catch Exception e
(log/error e "unexpected exception on get-user-info")
nil)))
(defn- retrieve-info
[{:keys [tokens] :as cfg} request]
(let [token (get-in request [:params :state])
state (tokens :verify {:token token :iss :github-oauth})
info (some->> (get-in request [:params :code])
(get-access-token cfg state)
(get-user-info cfg))]
(when-not info
(ex/raise :type :internal
:code :unable-to-auth))
(cond-> info
(some? (:invitation-token state))
(assoc :invitation-token (:invitation-token state)))))
(defn auth-handler
[{:keys [tokens] :as cfg} request]
(let [invitation (get-in request [:params :invitation-token])
state (tokens :generate {:iss :github-oauth
:invitation-token invitation
:exp (dt/in-future "15m")})
params {:client_id (:client-id cfg/config)
:redirect_uri (build-redirect-url cfg)
:state state
:scope scope}
query (u/map->query-string params)
uri (-> authorize-uri
(assoc :query query))]
{:status 200
:body {:redirect-uri (str uri)}}))
(defn- callback-handler
[{:keys [session] :as cfg} request]
(try
(let [info (retrieve-info cfg request)
profile (gg/register-profile cfg info)
uri (gg/generate-redirect-uri cfg profile)
sxf ((:create session) (:id profile))]
(->> (gg/redirect-response uri)
(sxf request)))
(catch Exception _e
(-> (gg/generate-error-redirect-uri cfg)
(gg/redirect-response)))))
;; --- ENTRY POINT
(s/def ::client-id ::us/not-empty-string)
(s/def ::client-secret ::us/not-empty-string)
(s/def ::public-uri ::us/not-empty-string)
(s/def ::session map?)
(s/def ::tokens fn?)
(defmethod ig/pre-init-spec :app.http.oauth/github [_]
(s/keys :req-un [::public-uri
::session
::tokens]
:opt-un [::client-id
::client-secret]))
(defn- default-handler
[_]
(ex/raise :type :not-found))
(defmethod ig/init-key :app.http.oauth/github
[_ cfg]
(if (and (:client-id cfg)
(:client-secret cfg))
{:handler #(auth-handler cfg %)
:callback-handler #(callback-handler cfg %)}
{:handler default-handler
:callback-handler default-handler}))

167
backend/src/app/http/oauth/gitlab.clj
View file

@ -1,167 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
(ns app.http.oauth.gitlab
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.http.oauth.google :as gg]
[app.util.http :as http]
[app.util.time :as dt]
[clojure.data.json :as json]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]
[lambdaisland.uri :as u]))
(def scope "read_user")
(defn- build-redirect-url
[cfg]
(let [public (u/uri (:public-uri cfg))]
(str (assoc public :path "/api/oauth/gitlab/callback"))))
(defn- build-oauth-uri
[cfg]
(let [base-uri (u/uri (:base-uri cfg))]
(assoc base-uri :path "/oauth/authorize")))
(defn- build-token-url
[cfg]
(let [base-uri (u/uri (:base-uri cfg))]
(str (assoc base-uri :path "/oauth/token"))))
(defn- build-user-info-url
[cfg]
(let [base-uri (u/uri (:base-uri cfg))]
(str (assoc base-uri :path "/api/v4/user"))))
(defn- get-access-token
[cfg code]
(try
(let [params {:client_id (:client-id cfg)
:client_secret (:client-secret cfg)
:code code
:grant_type "authorization_code"
:redirect_uri (build-redirect-url cfg)}
req {:method :post
:headers {"content-type" "application/x-www-form-urlencoded"}
:uri (build-token-url cfg)
:body (u/map->query-string params)}
res (http/send! req)]
(when (= 200 (:status res))
(-> (json/read-str (:body res))
(get "access_token"))))
(catch Exception e
(log/error e "unexpected error on get-access-token")
nil)))
(defn- get-user-info
[cfg token]
(try
(let [req {:uri (build-user-info-url cfg)
:headers {"Authorization" (str "Bearer " token)}
:timeout 6000
:method :get}
res (http/send! req)]
(when (= 200 (:status res))
(let [data (json/read-str (:body res))]
{:email (get data "email")
:backend "gitlab"
:fullname (get data "name")})))
(catch Exception e
(log/error e "unexpected exception on get-user-info")
nil)))
(defn- retrieve-info
[{:keys [tokens] :as cfg} request]
(let [token (get-in request [:params :state])
state (tokens :verify {:token token :iss :gitlab-oauth})
info (some->> (get-in request [:params :code])
(get-access-token cfg)
(get-user-info cfg))]
(when-not info
(ex/raise :type :internal
:code :unable-to-auth))
(cond-> info
(some? (:invitation-token state))
(assoc :invitation-token (:invitation-token state)))))
(defn- auth-handler
[{:keys [tokens] :as cfg} request]
(let [invitation (get-in request [:params :invitation-token])
state (tokens :generate
{:iss :gitlab-oauth
:invitation-token invitation
:exp (dt/in-future "15m")})
params {:client_id (:client-id cfg)
:redirect_uri (build-redirect-url cfg)
:response_type "code"
:state state
:scope scope}
query (u/map->query-string params)
uri (-> (build-oauth-uri cfg)
(assoc :query query))]
{:status 200
:body {:redirect-uri (str uri)}}))
(defn- callback-handler
[{:keys [session] :as cfg} request]
(try
(let [info (retrieve-info cfg request)
profile (gg/register-profile cfg info)
uri (gg/generate-redirect-uri cfg profile)
sxf ((:create session) (:id profile))]
(->> (gg/redirect-response uri)
(sxf request)))
(catch Exception _e
(-> (gg/generate-error-redirect-uri cfg)
(gg/redirect-response)))))
(s/def ::client-id ::us/not-empty-string)
(s/def ::client-secret ::us/not-empty-string)
(s/def ::base-uri ::us/not-empty-string)
(s/def ::public-uri ::us/not-empty-string)
(s/def ::session map?)
(s/def ::tokens fn?)
(defmethod ig/pre-init-spec :app.http.oauth/gitlab [_]
(s/keys :req-un [::public-uri
::session
::tokens]
:opt-un [::base-uri
::client-id
::client-secret]))
(defmethod ig/prep-key :app.http.oauth/gitlab
[_ cfg]
(d/merge {:base-uri "https://gitlab.com"}
(d/without-nils cfg)))
(defn- default-handler
[_]
(ex/raise :type :not-found))
(defmethod ig/init-key :app.http.oauth/gitlab
[_ cfg]
(if (and (:client-id cfg)
(:client-secret cfg))
{:handler #(auth-handler cfg %)
:callback-handler #(callback-handler cfg %)}
{:handler default-handler
:callback-handler default-handler}))

182
backend/src/app/http/oauth/google.clj
View file

@ -1,182 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
(ns app.http.oauth.google
(:require
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.util.http :as http]
[app.util.time :as dt]
[clojure.data.json :as json]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]
[lambdaisland.uri :as u]))
(def base-goauth-uri "https://accounts.google.com/o/oauth2/v2/auth")
(def scope
(str "email profile "
"https://www.googleapis.com/auth/userinfo.email "
"https://www.googleapis.com/auth/userinfo.profile "
"openid"))
(defn- build-redirect-url
[cfg]
(let [public (u/uri (:public-uri cfg))]
(str (assoc public :path "/api/oauth/google/callback"))))
(defn- get-access-token
[cfg code]
(try
(let [params {:code code
:client_id (:client-id cfg)
:client_secret (:client-secret cfg)
:redirect_uri (build-redirect-url cfg)
:grant_type "authorization_code"}
req {:method :post
:headers {"content-type" "application/x-www-form-urlencoded"}
:uri "https://oauth2.googleapis.com/token"
:timeout 6000
:body (u/map->query-string params)}
res (http/send! req)]
(when (= 200 (:status res))
(-> (json/read-str (:body res))
(get "access_token"))))
(catch Exception e
(log/error e "unexpected error on get-access-token")
nil)))
(defn- get-user-info
[_ token]
(try
(let [req {:uri "https://openidconnect.googleapis.com/v1/userinfo"
:headers {"Authorization" (str "Bearer " token)}
:timeout 6000
:method :get}
res (http/send! req)]
(when (= 200 (:status res))
(let [data (json/read-str (:body res))]
{:email (get data "email")
:backend "google"
:fullname (get data "name")})))
(catch Exception e
(log/error e "unexpected exception on get-user-info")
nil)))
(defn- retrieve-info
[{:keys [tokens] :as cfg} request]
(let [token (get-in request [:params :state])
state (tokens :verify {:token token :iss :google-oauth})
info (some->> (get-in request [:params :code])
(get-access-token cfg)
(get-user-info cfg))]
(when-not info
(ex/raise :type :internal
:code :unable-to-auth))
(cond-> info
(some? (:invitation-token state))
(assoc :invitation-token (:invitation-token state)))))
(defn register-profile
[{:keys [rpc] :as cfg} info]
(let [method-fn (get-in rpc [:methods :mutation :login-or-register])
profile (method-fn {:email (:email info)
:backend (:backend info)
:fullname (:fullname info)})]
(cond-> profile
(some? (:invitation-token info))
(assoc :invitation-token (:invitation-token info)))))
(defn generate-redirect-uri
[{:keys [tokens] :as cfg} profile]
(let [token (or (:invitation-token profile)
(tokens :generate {:iss :auth
:exp (dt/in-future "15m")
:profile-id (:id profile)}))]
(-> (u/uri (:public-uri cfg))
(assoc :path "/#/auth/verify-token")
(assoc :query (u/map->query-string {:token token})))))
(defn generate-error-redirect-uri
[cfg]
(-> (u/uri (:public-uri cfg))
(assoc :path "/#/auth/login")
(assoc :query (u/map->query-string {:error "unable-to-auth"}))))
(defn redirect-response
[uri]
{:status 302
:headers {"location" (str uri)}
:body ""})
(defn- auth-handler
[{:keys [tokens] :as cfg} request]
(let [invitation (get-in request [:params :invitation-token])
state (tokens :generate
{:iss :google-oauth
:invitation-token invitation
:exp (dt/in-future "15m")})
params {:scope scope
:access_type "offline"
:include_granted_scopes true
:state state
:response_type "code"
:redirect_uri (build-redirect-url cfg)
:client_id (:client-id cfg)}
query (u/map->query-string params)
uri (-> (u/uri base-goauth-uri)
(assoc :query query))]
{:status 200
:body {:redirect-uri (str uri)}}))
(defn- callback-handler
[{:keys [session] :as cfg} request]
(try
(let [info (retrieve-info cfg request)
profile (register-profile cfg info)
uri (generate-redirect-uri cfg profile)
sxf ((:create session) (:id profile))]
(->> (redirect-response uri)
(sxf request)))
(catch Exception _e
(-> (generate-error-redirect-uri cfg)
(redirect-response)))))
(s/def ::client-id ::us/not-empty-string)
(s/def ::client-secret ::us/not-empty-string)
(s/def ::public-uri ::us/not-empty-string)
(s/def ::session map?)
(s/def ::tokens fn?)
(defmethod ig/pre-init-spec :app.http.oauth/google [_]
(s/keys :req-un [::public-uri
::session
::tokens]
:opt-un [::client-id
::client-secret]))
(defn- default-handler
[_]
(ex/raise :type :not-found))
(defmethod ig/init-key :app.http.oauth/google
[_ cfg]
(if (and (:client-id cfg)
(:client-secret cfg))
{:handler #(auth-handler cfg %)
:callback-handler #(callback-handler cfg %)}
{:handler default-handler
:callback-handler default-handler}))

121
backend/src/app/http/session.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.http.session
(:require
@ -15,101 +12,98 @@
[app.db :as db]
[app.metrics :as mtx]
[app.util.async :as aa]
[app.util.log4j :refer [update-thread-context!]]
[app.util.logging :as l]
[app.util.time :as dt]
[app.worker :as wrk]
[buddy.core.codecs :as bc]
[buddy.core.nonce :as bn]
[clojure.core.async :as a]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]))
;; A default cookie name for storing the session. We don't allow
;; configure it.
(def cookie-name "auth-token")
;; --- IMPL
(defn- next-session-id
([] (next-session-id 96))
([n]
(-> (bn/random-nonce n)
(bc/bytes->b64u)
(bc/bytes->str))))
(defn- create-session
[{:keys [conn tokens] :as cfg} {:keys [profile-id headers] :as request}]
(let [token (tokens :generate {:iss "authentication"
:iat (dt/now)
:uid profile-id})
params {:user-agent (get headers "user-agent")
:profile-id profile-id
:id token}]
(db/insert! conn :http-session params)))
(defn- create
[{:keys [conn] :as cfg} {:keys [profile-id user-agent]}]
(let [id (next-session-id)]
(db/insert! conn :http-session {:id id
:profile-id profile-id
:user-agent user-agent})
id))
(defn- delete
[{:keys [conn cookie-name] :as cfg} {:keys [cookies] :as request}]
(defn- delete-session
[{:keys [conn] :as cfg} {:keys [cookies] :as request}]
(when-let [token (get-in cookies [cookie-name :value])]
(db/delete! conn :http-session {:id token}))
nil)
(defn- retrieve
[{:keys [conn] :as cfg} token]
(when token
(db/exec-one! conn ["select id, profile_id from http_session where id = ?" token])))
(defn- retrieve-session
[{:keys [conn] :as cfg} id]
(when id
(db/exec-one! conn ["select id, profile_id from http_session where id = ?" id])))
(defn- retrieve-from-request
[{:keys [cookie-name] :as cfg} {:keys [cookies] :as request}]
[cfg {:keys [cookies] :as request}]
(->> (get-in cookies [cookie-name :value])
(retrieve cfg)))
(retrieve-session cfg)))
(defn- cookies
[{:keys [cookie-name] :as cfg} vals]
{cookie-name (merge vals {:path "/" :http-only true})})
(defn- add-cookies
[response {:keys [id] :as session}]
(assoc response :cookies {cookie-name {:path "/" :http-only true :value id}}))
(defn- clear-cookies
[response]
(assoc response :cookies {cookie-name {:value "" :max-age -1}}))
(defn- middleware
[cfg handler]
(fn [request]
(if-let [{:keys [id profile-id] :as session} (retrieve-from-request cfg request)]
(let [ech (::events-ch cfg)]
(a/>!! ech id)
(update-thread-context! {:profile-id profile-id})
(do
(a/>!! (::events-ch cfg) id)
(l/update-thread-context! {:profile-id profile-id})
(handler (assoc request :profile-id profile-id)))
(handler request))))
;; --- STATE INIT: SESSION
(s/def ::cookie-name ::cfg/http-session-cookie-name)
(defmethod ig/pre-init-spec ::session [_]
(s/keys :req-un [::db/pool]
:opt-un [::cookie-name]))
(s/keys :req-un [::db/pool]))
(defmethod ig/prep-key ::session
[_ cfg]
(merge {:cookie-name "auth-token"
:buffer-size 64}
(d/without-nils cfg)))
(d/merge {:buffer-size 64}
(d/without-nils cfg)))
(defmethod ig/init-key ::session
[_ {:keys [pool] :as cfg}]
(let [events (a/chan (a/dropping-buffer (:buffer-size cfg)))
cfg (assoc cfg
:conn pool
::events-ch events)]
cfg (-> cfg
(assoc :conn pool)
(assoc ::events-ch events))]
(-> cfg
(assoc :middleware #(middleware cfg %))
(assoc :create (fn [profile-id]
(fn [request response]
(let [uagent (get-in request [:headers "user-agent"])
value (create cfg {:profile-id profile-id :user-agent uagent})]
(assoc response :cookies (cookies cfg {:value value}))))))
(let [request (assoc request :profile-id profile-id)
session (create-session cfg request)]
(add-cookies response session)))))
(assoc :delete (fn [request response]
(delete cfg request)
(assoc response
:status 204
:body ""
:cookies (cookies cfg {:value "" :max-age -1})))))))
(delete-session cfg request)
(-> response
(assoc :status 204)
(assoc :body "")
(clear-cookies)))))))
(defmethod ig/halt-key! ::session
[_ data]
(a/close! (::events-ch data)))
;; --- STATE INIT: SESSION UPDATER
(declare batch-events)
@ -132,9 +126,9 @@
(defmethod ig/init-key ::updater
[_ {:keys [session metrics] :as cfg}]
(log/infof "initialize session updater (max-batch-age=%s, max-batch-size=%s)"
(str (:max-batch-age cfg))
(str (:max-batch-size cfg)))
(l/info :action "initialize session updater"
:max-batch-age (str (:max-batch-age cfg))
:max-batch-size (str (:max-batch-size cfg)))
(let [input (batch-events cfg (::events-ch session))
mcnt (mtx/create
{:name "http_session_update_total"
@ -146,8 +140,13 @@
(let [result (a/<! (update-sessions cfg batch))]
(mcnt :inc)
(if (ex/exception? result)
(log/error result "updater: unexpected error on update sessions")
(log/debugf "updater: updated %s sessions (reason: %s)." result (name reason)))
(l/error :task "updater"
:hint "unexpected error on update sessions"
:cause result)
(l/debug :task "updater"
:action "update sessions"
:reason (name reason)
:count result))
(recur))))))
(defn- timeout-chan
@ -209,7 +208,9 @@
(let [interval (db/interval max-age)
result (db/exec-one! conn [sql:delete-expired interval])
result (:next.jdbc/update-count result)]
(log/debugf "gc-task: removed %s rows from http-session table" result)
(l/debug :task "gc"
:action "clean http sessions"
:count result)
result))))
(def ^:private

19
backend/src/app/loggers/loki.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.loggers.loki
"A Loki integration."
@ -15,10 +12,10 @@
[app.util.async :as aa]
[app.util.http :as http]
[app.util.json :as json]
[app.util.logging :as l]
[app.worker :as wrk]
[clojure.core.async :as a]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]))
(declare handle-event)
@ -33,13 +30,13 @@
(defmethod ig/init-key ::reporter
[_ {:keys [receiver uri] :as cfg}]
(when uri
(log/info "intializing loki reporter")
(l/info :msg "intializing loki reporter" :uri uri)
(let [output (a/chan (a/sliding-buffer 1024))]
(receiver :sub output)
(a/go-loop []
(let [msg (a/<! output)]
(if (nil? msg)
(log/info "stoping error reporting loop")
(l/info :msg "stoping error reporting loop")
(do
(a/<! (handle-event cfg msg))
(recur)))))
@ -75,10 +72,14 @@
(if (= (:status response) 204)
true
(do
(log/errorf "error on sending log to loki (try %s)\n%s" i (pr-str response))
(l/error :hint "error on sending log to loki"
:try i
:rsp (pr-str response))
false)))
(catch Exception e
(log/errorf e "error on sending message to loki (try %s)" i)
(l/error :hint "error on sending message to loki"
:cause e
:try i)
false)))
(defn- handle-event

22
backend/src/app/loggers/mattermost.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.loggers.mattermost
"A mattermost integration for error reporting."
@ -18,12 +15,12 @@
[app.util.async :as aa]
[app.util.http :as http]
[app.util.json :as json]
[app.util.logging :as l]
[app.util.template :as tmpl]
[app.worker :as wrk]
[clojure.core.async :as a]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[cuerdas.core :as str]
[integrant.core :as ig]))
@ -43,14 +40,14 @@
(defmethod ig/init-key ::reporter
[_ {:keys [receiver] :as cfg}]
(log/info "intializing mattermost error reporter")
(l/info :msg "intializing mattermost error reporter")
(let [output (a/chan (a/sliding-buffer 128)
(filter #(= (:level %) "error")))]
(receiver :sub output)
(a/go-loop []
(let [msg (a/<! output)]
(if (nil? msg)
(log/info "stoping error reporting loop")
(l/info :msg "stoping error reporting loop")
(do
(a/<! (handle-event cfg msg))
(recur)))))
@ -65,7 +62,7 @@
(try
(let [uri (:uri cfg)
text (str "Unhandled exception (@channel):\n"
"- detail: " (:public-uri cfg/config) "/dbg/error-by-id/" id "\n"
"- detail: " (cfg/get :public-uri) "/dbg/error-by-id/" id "\n"
"- host: `" host "`\n"
"- version: `" version "`\n"
(when error
@ -75,10 +72,12 @@
:headers {"content-type" "application/json"}
:body (json/encode-str {:text text})})]
(when (not= (:status rsp) 200)
(log/errorf "error on sending data to mattermost\n%s" (pr-str rsp))))
(l/error :hint "error on sending data to mattermost"
:response (pr-str rsp))))
(catch Exception e
(log/error e "unexpected exception on error reporter"))))
(l/error :hint "unexpected exception on error reporter"
:cause e))))
(defn- persist-on-database!
[{:keys [pool] :as cfg} {:keys [id] :as cdata}]
@ -116,7 +115,8 @@
(send-mattermost-notification! cfg cdata))
(persist-on-database! cfg cdata))
(catch Exception e
(log/error e "unexpected exception on error reporter")))))
(l/error :hint "unexpected exception on error reporter"
:cause e)))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Http Handler

9
backend/src/app/loggers/zmq.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.loggers.zmq
"A generic ZMQ listener."
@ -13,10 +10,10 @@
[app.common.data :as d]
[app.common.spec :as us]
[app.util.json :as json]
[app.util.logging :as l]
[app.util.time :as dt]
[clojure.core.async :as a]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[cuerdas.core :as str]
[integrant.core :as ig])
(:import
@ -34,7 +31,7 @@
(defmethod ig/init-key ::receiver
[_ {:keys [endpoint] :as cfg}]
(log/infof "intializing ZMQ receiver on '%s'" endpoint)
(l/info :msg "intializing ZMQ receiver" :bind endpoint)
(let [buffer (a/chan 1)
output (a/chan 1 (comp (filter map?)
(map prepare)))

547
backend/src/app/main.clj
View file

@ -2,356 +2,296 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.main
(:require
[app.common.data :as d]
[app.config :as cfg]
[app.config :as cf]
[app.util.logging :as l]
[app.util.time :as dt]
[clojure.pprint :as pprint]
[clojure.tools.logging :as log]
[integrant.core :as ig]))
;; Set value for all new threads bindings.
(alter-var-root #'*assert* (constantly (:asserts-enabled cfg/config)))
(def system-config
{:app.db/pool
{:uri (cf/get :database-uri)
:username (cf/get :database-username)
:password (cf/get :database-password)
:metrics (ig/ref :app.metrics/metrics)
:migrations (ig/ref :app.migrations/all)
:name :main
:min-pool-size 0
:max-pool-size 20}
(derive :app.telemetry/server :app.http/server)
:app.metrics/metrics
{:definitions
{:profile-register
{:name "actions_profile_register_count"
:help "A global counter of user registrations."
:type :counter}
:profile-activation
{:name "actions_profile_activation_count"
:help "A global counter of profile activations"
:type :counter}}}
;; --- Entry point
:app.migrations/all
{:main (ig/ref :app.migrations/migrations)}
(defn build-system-config
[config]
(d/deep-merge
{:app.db/pool
{:uri (:database-uri config)
:username (:database-username config)
:password (:database-password config)
:metrics (ig/ref :app.metrics/metrics)
:migrations (ig/ref :app.migrations/all)
:name "main"
:min-pool-size 0
:max-pool-size 20}
:app.migrations/migrations
{}
:app.metrics/metrics
{:definitions
{:profile-register
{:name "actions_profile_register_count"
:help "A global counter of user registrations."
:type :counter}
:profile-activation
{:name "actions_profile_activation_count"
:help "A global counter of profile activations"
:type :counter}}}
:app.msgbus/msgbus
{:backend (cf/get :msgbus-backend :redis)
:redis-uri (cf/get :redis-uri)}
:app.migrations/all
{:main (ig/ref :app.migrations/migrations)
:telemetry (ig/ref :app.telemetry/migrations)}
:app.tokens/tokens
{:sprops (ig/ref :app.setup/props)}
:app.migrations/migrations
{}
:app.storage/gc-deleted-task
{:pool (ig/ref :app.db/pool)
:storage (ig/ref :app.storage/storage)
:min-age (dt/duration {:hours 2})}
:app.telemetry/migrations
{}
:app.storage/gc-touched-task
{:pool (ig/ref :app.db/pool)}
:app.msgbus/msgbus
{:backend (:msgbus-backend config :redis)
:redis-uri (:redis-uri config)}
:app.storage/recheck-task
{:pool (ig/ref :app.db/pool)
:storage (ig/ref :app.storage/storage)}
:app.tokens/tokens
{:sprops (ig/ref :app.setup/props)}
:app.http.session/session
{:pool (ig/ref :app.db/pool)
:tokens (ig/ref :app.tokens/tokens)}
:app.storage/gc-deleted-task
{:pool (ig/ref :app.db/pool)
:storage (ig/ref :app.storage/storage)
:min-age (dt/duration {:hours 2})}
:app.http.session/gc-task
{:pool (ig/ref :app.db/pool)
:max-age (cf/get :http-session-idle-max-age)}
:app.storage/gc-touched-task
{:pool (ig/ref :app.db/pool)}
:app.http.session/updater
{:pool (ig/ref :app.db/pool)
:metrics (ig/ref :app.metrics/metrics)
:executor (ig/ref :app.worker/executor)
:session (ig/ref :app.http.session/session)
:max-batch-age (cf/get :http-session-updater-batch-max-age)
:max-batch-size (cf/get :http-session-updater-batch-max-size)}
:app.storage/recheck-task
{:pool (ig/ref :app.db/pool)
:storage (ig/ref :app.storage/storage)}
:app.http.awsns/handler
{:tokens (ig/ref :app.tokens/tokens)
:pool (ig/ref :app.db/pool)}
:app.http.session/session
{:pool (ig/ref :app.db/pool)
:cookie-name (:http-session-cookie-name config)}
:app.http/server
{:port (cf/get :http-server-port)
:router (ig/ref :app.http/router)
:metrics (ig/ref :app.metrics/metrics)
:ws {"/ws/notifications" (ig/ref :app.notifications/handler)}}
:app.http.session/gc-task
{:pool (ig/ref :app.db/pool)
:max-age (:http-session-idle-max-age config)}
:app.http/router
{:rpc (ig/ref :app.rpc/rpc)
:session (ig/ref :app.http.session/session)
:tokens (ig/ref :app.tokens/tokens)
:public-uri (cf/get :public-uri)
:metrics (ig/ref :app.metrics/metrics)
:oauth (ig/ref :app.http.oauth/handlers)
:assets (ig/ref :app.http.assets/handlers)
:storage (ig/ref :app.storage/storage)
:sns-webhook (ig/ref :app.http.awsns/handler)
:feedback (ig/ref :app.http.feedback/handler)
:error-report-handler (ig/ref :app.loggers.mattermost/handler)}
:app.http.session/updater
{:pool (ig/ref :app.db/pool)
:metrics (ig/ref :app.metrics/metrics)
:executor (ig/ref :app.worker/executor)
:session (ig/ref :app.http.session/session)
:max-batch-age (:http-session-updater-batch-max-age config)
:max-batch-size (:http-session-updater-batch-max-size config)}
:app.http.assets/handlers
{:metrics (ig/ref :app.metrics/metrics)
:assets-path (cf/get :assets-path)
:storage (ig/ref :app.storage/storage)
:cache-max-age (dt/duration {:hours 24})
:signature-max-age (dt/duration {:hours 24 :minutes 5})}
:app.http.awsns/handler
{:tokens (ig/ref :app.tokens/tokens)
:pool (ig/ref :app.db/pool)}
:app.http.feedback/handler
{:pool (ig/ref :app.db/pool)}
:app.http/server
{:port (:http-server-port config)
:handler (ig/ref :app.http/router)
:metrics (ig/ref :app.metrics/metrics)
:ws {"/ws/notifications" (ig/ref :app.notifications/handler)}}
:app.http.oauth/handlers
{:rpc (ig/ref :app.rpc/rpc)
:session (ig/ref :app.http.session/session)
:tokens (ig/ref :app.tokens/tokens)
:public-uri (cf/get :public-uri)}
:app.http/router
{:rpc (ig/ref :app.rpc/rpc)
:session (ig/ref :app.http.session/session)
:tokens (ig/ref :app.tokens/tokens)
:public-uri (:public-uri config)
:metrics (ig/ref :app.metrics/metrics)
:oauth (ig/ref :app.http.oauth/all)
:assets (ig/ref :app.http.assets/handlers)
:svgparse (ig/ref :app.svgparse/handler)
:storage (ig/ref :app.storage/storage)
:sns-webhook (ig/ref :app.http.awsns/handler)
:feedback (ig/ref :app.http.feedback/handler)
:error-report-handler (ig/ref :app.loggers.mattermost/handler)}
;; RLimit definition for password hashing
:app.rlimits/password
(cf/get :rlimits-password)
:app.http.assets/handlers
{:metrics (ig/ref :app.metrics/metrics)
:assets-path (:assets-path config)
:storage (ig/ref :app.storage/storage)
:cache-max-age (dt/duration {:hours 24})
:signature-max-age (dt/duration {:hours 24 :minutes 5})}
;; RLimit definition for image processing
:app.rlimits/image
(cf/get :rlimits-image)
:app.http.feedback/handler
{:pool (ig/ref :app.db/pool)}
;; A collection of rlimits as hash-map.
:app.rlimits/all
{:password (ig/ref :app.rlimits/password)
:image (ig/ref :app.rlimits/image)}
:app.http.oauth/all
{:google (ig/ref :app.http.oauth/google)
:gitlab (ig/ref :app.http.oauth/gitlab)
:github (ig/ref :app.http.oauth/github)}
:app.rpc/rpc
{:pool (ig/ref :app.db/pool)
:session (ig/ref :app.http.session/session)
:tokens (ig/ref :app.tokens/tokens)
:metrics (ig/ref :app.metrics/metrics)
:storage (ig/ref :app.storage/storage)
:msgbus (ig/ref :app.msgbus/msgbus)
:rlimits (ig/ref :app.rlimits/all)
:public-uri (cf/get :public-uri)}
:app.http.oauth/google
{:rpc (ig/ref :app.rpc/rpc)
:session (ig/ref :app.http.session/session)
:tokens (ig/ref :app.tokens/tokens)
:public-uri (:public-uri config)
:client-id (:google-client-id config)
:client-secret (:google-client-secret config)}
:app.notifications/handler
{:msgbus (ig/ref :app.msgbus/msgbus)
:pool (ig/ref :app.db/pool)
:session (ig/ref :app.http.session/session)
:metrics (ig/ref :app.metrics/metrics)
:executor (ig/ref :app.worker/executor)}
:app.http.oauth/github
{:rpc (ig/ref :app.rpc/rpc)
:session (ig/ref :app.http.session/session)
:tokens (ig/ref :app.tokens/tokens)
:public-uri (:public-uri config)
:client-id (:github-client-id config)
:client-secret (:github-client-secret config)}
:app.worker/executor
{:min-threads 0
:max-threads 256
:idle-timeout 60000
:name :worker}
:app.http.oauth/gitlab
{:rpc (ig/ref :app.rpc/rpc)
:session (ig/ref :app.http.session/session)
:tokens (ig/ref :app.tokens/tokens)
:public-uri (:public-uri config)
:base-uri (:gitlab-base-uri config)
:client-id (:gitlab-client-id config)
:client-secret (:gitlab-client-secret config)}
:app.worker/worker
{:executor (ig/ref :app.worker/executor)
:tasks (ig/ref :app.worker/registry)
:metrics (ig/ref :app.metrics/metrics)
:pool (ig/ref :app.db/pool)}
;; HTTP Handler for SVG parsing
:app.svgparse/handler
{:metrics (ig/ref :app.metrics/metrics)}
:app.worker/scheduler
{:executor (ig/ref :app.worker/executor)
:tasks (ig/ref :app.worker/registry)
:pool (ig/ref :app.db/pool)
:schedule
[{:cron #app/cron "0 0 0 */1 * ? *" ;; daily
:task :file-media-gc}
;; RLimit definition for password hashing
:app.rlimits/password
(:rlimits-password config)
{:cron #app/cron "0 0 */1 * * ?" ;; hourly
:task :file-xlog-gc}
;; RLimit definition for image processing
:app.rlimits/image
(:rlimits-image config)
{:cron #app/cron "0 0 1 */1 * ?" ;; daily (1 hour shift)
:task :storage-deleted-gc}
;; A collection of rlimits as hash-map.
:app.rlimits/all
{:password (ig/ref :app.rlimits/password)
:image (ig/ref :app.rlimits/image)}
{:cron #app/cron "0 0 2 */1 * ?" ;; daily (2 hour shift)
:task :storage-touched-gc}
:app.rpc/rpc
{:pool (ig/ref :app.db/pool)
:session (ig/ref :app.http.session/session)
:tokens (ig/ref :app.tokens/tokens)
:metrics (ig/ref :app.metrics/metrics)
:storage (ig/ref :app.storage/storage)
:msgbus (ig/ref :app.msgbus/msgbus)
:rlimits (ig/ref :app.rlimits/all)}
{:cron #app/cron "0 0 3 */1 * ?" ;; daily (3 hour shift)
:task :session-gc}
:app.notifications/handler
{:msgbus (ig/ref :app.msgbus/msgbus)
:pool (ig/ref :app.db/pool)
:session (ig/ref :app.http.session/session)
:metrics (ig/ref :app.metrics/metrics)
:executor (ig/ref :app.worker/executor)}
{:cron #app/cron "0 0 */1 * * ?" ;; hourly
:task :storage-recheck}
:app.worker/executor
{:name "worker"}
{:cron #app/cron "0 0 0 */1 * ?" ;; daily
:task :tasks-gc}
:app.worker/worker
{:executor (ig/ref :app.worker/executor)
:pool (ig/ref :app.db/pool)
:tasks (ig/ref :app.tasks/registry)}
(when (cf/get :telemetry-enabled)
{:cron #app/cron "0 0 */6 * * ?" ;; every 6h
:task :telemetry})]}
:app.worker/scheduler
{:executor (ig/ref :app.worker/executor)
:pool (ig/ref :app.db/pool)
:tasks (ig/ref :app.tasks/registry)
:schedule
[{:id "file-media-gc"
:cron #app/cron "0 0 0 */1 * ? *" ;; daily
:task :file-media-gc}
:app.worker/registry
{:metrics (ig/ref :app.metrics/metrics)
:tasks
{:sendmail (ig/ref :app.emails/sendmail-handler)
:delete-object (ig/ref :app.tasks.delete-object/handler)
:delete-profile (ig/ref :app.tasks.delete-profile/handler)
:file-media-gc (ig/ref :app.tasks.file-media-gc/handler)
:file-xlog-gc (ig/ref :app.tasks.file-xlog-gc/handler)
:storage-deleted-gc (ig/ref :app.storage/gc-deleted-task)
:storage-touched-gc (ig/ref :app.storage/gc-touched-task)
:storage-recheck (ig/ref :app.storage/recheck-task)
:tasks-gc (ig/ref :app.tasks.tasks-gc/handler)
:telemetry (ig/ref :app.tasks.telemetry/handler)
:session-gc (ig/ref :app.http.session/gc-task)}}
{:id "file-xlog-gc"
:cron #app/cron "0 0 */1 * * ?" ;; hourly
:task :file-xlog-gc}
:app.emails/sendmail-handler
{:host (cf/get :smtp-host)
:port (cf/get :smtp-port)
:ssl (cf/get :smtp-ssl)
:tls (cf/get :smtp-tls)
:enabled (cf/get :smtp-enabled)
:username (cf/get :smtp-username)
:password (cf/get :smtp-password)
:metrics (ig/ref :app.metrics/metrics)
:default-reply-to (cf/get :smtp-default-reply-to)
:default-from (cf/get :smtp-default-from)}
{:id "storage-deleted-gc"
:cron #app/cron "0 0 1 */1 * ?" ;; daily (1 hour shift)
:task :storage-deleted-gc}
:app.tasks.tasks-gc/handler
{:pool (ig/ref :app.db/pool)
:max-age (dt/duration {:hours 24})
:metrics (ig/ref :app.metrics/metrics)}
{:id "storage-touched-gc"
:cron #app/cron "0 0 2 */1 * ?" ;; daily (2 hour shift)
:task :storage-touched-gc}
:app.tasks.delete-object/handler
{:pool (ig/ref :app.db/pool)
:metrics (ig/ref :app.metrics/metrics)}
{:id "session-gc"
:cron #app/cron "0 0 3 */1 * ?" ;; daily (3 hour shift)
:task :session-gc}
:app.tasks.delete-storage-object/handler
{:pool (ig/ref :app.db/pool)
:storage (ig/ref :app.storage/storage)
:metrics (ig/ref :app.metrics/metrics)}
{:id "storage-recheck"
:cron #app/cron "0 0 */1 * * ?" ;; hourly
:task :storage-recheck}
:app.tasks.delete-profile/handler
{:pool (ig/ref :app.db/pool)
:metrics (ig/ref :app.metrics/metrics)}
{:id "tasks-gc"
:cron #app/cron "0 0 0 */1 * ?" ;; daily
:task :tasks-gc}
:app.tasks.file-media-gc/handler
{:pool (ig/ref :app.db/pool)
:metrics (ig/ref :app.metrics/metrics)
:max-age (dt/duration {:hours 48})}
(when (:telemetry-enabled config)
{:id "telemetry"
:cron #app/cron "0 0 */6 * * ?" ;; every 6h
:uri (:telemetry-uri config)
:task :telemetry})]}
:app.tasks.file-xlog-gc/handler
{:pool (ig/ref :app.db/pool)
:metrics (ig/ref :app.metrics/metrics)
:max-age (dt/duration {:hours 48})}
:app.tasks/registry
{:metrics (ig/ref :app.metrics/metrics)
:tasks
{:sendmail (ig/ref :app.tasks.sendmail/handler)
:delete-object (ig/ref :app.tasks.delete-object/handler)
:delete-profile (ig/ref :app.tasks.delete-profile/handler)
:file-media-gc (ig/ref :app.tasks.file-media-gc/handler)
:file-xlog-gc (ig/ref :app.tasks.file-xlog-gc/handler)
:storage-deleted-gc (ig/ref :app.storage/gc-deleted-task)
:storage-touched-gc (ig/ref :app.storage/gc-touched-task)
:storage-recheck (ig/ref :app.storage/recheck-task)
:tasks-gc (ig/ref :app.tasks.tasks-gc/handler)
:telemetry (ig/ref :app.tasks.telemetry/handler)
:session-gc (ig/ref :app.http.session/gc-task)}}
:app.tasks.telemetry/handler
{:pool (ig/ref :app.db/pool)
:version (:full cf/version)
:uri (cf/get :telemetry-uri)
:sprops (ig/ref :app.setup/props)}
:app.tasks.sendmail/handler
{:host (:smtp-host config)
:port (:smtp-port config)
:ssl (:smtp-ssl config)
:tls (:smtp-tls config)
:enabled (:smtp-enabled config)
:username (:smtp-username config)
:password (:smtp-password config)
:metrics (ig/ref :app.metrics/metrics)
:default-reply-to (:smtp-default-reply-to config)
:default-from (:smtp-default-from config)}
:app.srepl/server
{:port (cf/get :srepl-port)
:host (cf/get :srepl-host)}
:app.tasks.tasks-gc/handler
{:pool (ig/ref :app.db/pool)
:max-age (dt/duration {:hours 24})
:metrics (ig/ref :app.metrics/metrics)}
:app.setup/props
{:pool (ig/ref :app.db/pool)}
:app.tasks.delete-object/handler
{:pool (ig/ref :app.db/pool)
:metrics (ig/ref :app.metrics/metrics)}
:app.loggers.zmq/receiver
{:endpoint (cf/get :loggers-zmq-uri)}
:app.tasks.delete-storage-object/handler
{:pool (ig/ref :app.db/pool)
:storage (ig/ref :app.storage/storage)
:metrics (ig/ref :app.metrics/metrics)}
:app.loggers.loki/reporter
{:uri (cf/get :loggers-loki-uri)
:receiver (ig/ref :app.loggers.zmq/receiver)
:executor (ig/ref :app.worker/executor)}
:app.tasks.delete-profile/handler
{:pool (ig/ref :app.db/pool)
:metrics (ig/ref :app.metrics/metrics)}
:app.loggers.mattermost/reporter
{:uri (cf/get :error-report-webhook)
:receiver (ig/ref :app.loggers.zmq/receiver)
:pool (ig/ref :app.db/pool)
:executor (ig/ref :app.worker/executor)}
:app.tasks.file-media-gc/handler
{:pool (ig/ref :app.db/pool)
:metrics (ig/ref :app.metrics/metrics)
:max-age (dt/duration {:hours 48})}
:app.loggers.mattermost/handler
{:pool (ig/ref :app.db/pool)}
:app.tasks.file-xlog-gc/handler
{:pool (ig/ref :app.db/pool)
:metrics (ig/ref :app.metrics/metrics)
:max-age (dt/duration {:hours 48})}
:app.storage/storage
{:pool (ig/ref :app.db/pool)
:executor (ig/ref :app.worker/executor)
:backend (cf/get :storage-backend :fs)
:backends {:s3 (ig/ref [::main :app.storage.s3/backend])
:db (ig/ref [::main :app.storage.db/backend])
:fs (ig/ref [::main :app.storage.fs/backend])
:tmp (ig/ref [::tmp :app.storage.fs/backend])}}
:app.tasks.telemetry/handler
{:pool (ig/ref :app.db/pool)
:version (:full cfg/version)
:uri (:telemetry-uri config)
:sprops (ig/ref :app.setup/props)}
[::main :app.storage.s3/backend]
{:region (cf/get :storage-s3-region)
:bucket (cf/get :storage-s3-bucket)}
:app.srepl/server
{:port (:srepl-port config)
:host (:srepl-host config)}
[::main :app.storage.fs/backend]
{:directory (cf/get :storage-fs-directory)}
:app.setup/props
{:pool (ig/ref :app.db/pool)}
[::tmp :app.storage.fs/backend]
{:directory "/tmp/penpot"}
:app.loggers.zmq/receiver
{:endpoint (:loggers-zmq-uri config)}
:app.loggers.loki/reporter
{:uri (:loggers-loki-uri config)
:receiver (ig/ref :app.loggers.zmq/receiver)
:executor (ig/ref :app.worker/executor)}
:app.loggers.mattermost/reporter
{:uri (:error-report-webhook config)
:receiver (ig/ref :app.loggers.zmq/receiver)
:pool (ig/ref :app.db/pool)
:executor (ig/ref :app.worker/executor)}
:app.loggers.mattermost/handler
{:pool (ig/ref :app.db/pool)}
:app.storage/storage
{:pool (ig/ref :app.db/pool)
:executor (ig/ref :app.worker/executor)
:backend (:storage-backend config :fs)
:backends {:s3 (ig/ref [::main :app.storage.s3/backend])
:db (ig/ref [::main :app.storage.db/backend])
:fs (ig/ref [::main :app.storage.fs/backend])
:tmp (ig/ref [::tmp :app.storage.fs/backend])}}
[::main :app.storage.s3/backend]
{:region (:storage-s3-region config)
:bucket (:storage-s3-bucket config)}
[::main :app.storage.fs/backend]
{:directory (:storage-fs-directory config)}
[::tmp :app.storage.fs/backend]
{:directory "/tmp/penpot"}
[::main :app.storage.db/backend]
{:pool (ig/ref :app.db/pool)}}
(when (:telemetry-server-enabled config)
{:app.telemetry/handler
{:pool (ig/ref :app.db/pool)
:executor (ig/ref :app.worker/executor)}
:app.telemetry/server
{:port (:telemetry-server-port config 6063)
:handler (ig/ref :app.telemetry/handler)
:name "telemetry"}})))
[::main :app.storage.db/backend]
{:pool (ig/ref :app.db/pool)}})
(defmethod ig/init-key :default [_ data] data)
(defmethod ig/prep-key :default
@ -364,15 +304,14 @@
(defn start
[]
(let [system-config (build-system-config cfg/config)]
(ig/load-namespaces system-config)
(alter-var-root #'system (fn [sys]
(when sys (ig/halt! sys))
(-> system-config
(ig/prep)
(ig/init))))
(log/infof "welcome to penpot (version: '%s')"
(:full cfg/version))))
(ig/load-namespaces system-config)
(alter-var-root #'system (fn [sys]
(when sys (ig/halt! sys))
(-> system-config
(ig/prep)
(ig/init))))
(l/info :msg "welcome to penpot"
:version (:full cf/version)))
(defn stop
[]
@ -380,14 +319,6 @@
(when sys (ig/halt! sys))
nil)))
(prefer-method print-method
clojure.lang.IRecord
clojure.lang.IDeref)
(prefer-method pprint/simple-dispatch
clojure.lang.IPersistentMap
clojure.lang.IDeref)
(defn -main
[& _args]
(start))

7
backend/src/app/media.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.media
"Media postprocessing."
@ -15,7 +12,7 @@
[app.common.media :as cm]
[app.common.spec :as us]
[app.rlimits :as rlm]
[app.svgparse :as svg]
[app.rpc.queries.svg :as svg]
[clojure.spec.alpha :as s]
[cuerdas.core :as str]
[datoteka.core :as fs])

9
backend/src/app/metrics.clj
View file

@ -2,16 +2,13 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.metrics
(:require
[app.common.exceptions :as ex]
[app.util.logging :as l]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig])
(:import
io.prometheus.client.CollectorRegistry
@ -50,7 +47,7 @@
(defmethod ig/init-key ::metrics
[_ {:keys [definitions] :as cfg}]
(log/infof "Initializing prometheus registry and instrumentation.")
(l/info :action "initialize metrics")
(let [registry (create-registry)
definitions (reduce-kv (fn [res k v]
(->> (assoc v :registry registry)

8
backend/src/app/migrations.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.migrations
(:require
@ -166,6 +163,9 @@
{:name "0051-mod-file-library-rel-table"
:fn (mg/resource "app/migrations/sql/0051-mod-file-library-rel-table.sql")}
{:name "0052-del-legacy-user-and-team"
:fn (mg/resource "app/migrations/sql/0052-del-legacy-user-and-team.sql")}
])

5
backend/src/app/migrations/migration_0023.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.migrations.migration-0023
(:require

2
backend/src/app/migrations/sql/0052-del-legacy-user-and-team.sql Normal file
View file

@ -0,0 +1,2 @@
DELETE FROM team WHERE id = '00000000-0000-0000-0000-000000000000';
DELETE FROM profile WHERE id = '00000000-0000-0000-0000-000000000000';

32
backend/src/app/msgbus.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.msgbus
"The msgbus abstraction implemented using redis as underlying backend."
@ -14,10 +11,10 @@
[app.common.spec :as us]
[app.config :as cfg]
[app.util.blob :as blob]
[app.util.logging :as l]
[app.util.time :as dt]
[clojure.core.async :as a]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]
[promesa.core :as p])
(:import
@ -60,7 +57,8 @@
(defmethod ig/init-key ::msgbus
[_ {:keys [backend buffer-size] :as cfg}]
(log/debugf "initializing msgbus (backend=%s)" (name backend))
(l/debug :action "initialize msgbus"
:backend (name backend))
(let [cfg (init-backend cfg)
;; Channel used for receive publications from the application.
@ -165,13 +163,14 @@
(when-let [val (a/<! pub-ch)]
(let [result (a/<! (impl-redis-pub rac val))]
(when (ex/exception? result)
(log/error result "unexpected error on publish message to redis")))
(l/error :cause result
:hint "unexpected error on publish message to redis")))
(recur)))))
(defmethod init-sub-loop :redis
[{:keys [::sub-conn ::sub-ch buffer-size]}]
(let [rcv-ch (a/chan (a/dropping-buffer buffer-size))
chans (agent {} :error-handler #(log/error % "unexpected error on agent"))
chans (agent {} :error-handler #(l/error :cause % :hint "unexpected error on agent"))
rac (.async ^StatefulRedisPubSubConnection sub-conn)]
;; Add a unique listener to connection
@ -184,7 +183,7 @@
;; more messages that we can process.
(let [val {:topic topic :message (blob/decode message)}]
(when-not (a/offer! rcv-ch val)
(log/warn "dropping message on subscription loop"))))
(l/warn :msg "dropping message on subscription loop"))))
(psubscribed [it pattern count])
(punsubscribed [it pattern count])
(subscribed [it topic count])
@ -194,9 +193,12 @@
(let [nsubs (if (nil? nsubs) #{chan} (conj nsubs chan))]
(when (= 1 (count nsubs))
(let [result (a/<!! (impl-redis-sub rac topic))]
(log/tracef "opening subscription to %s" topic)
(l/trace :action "open subscription"
:topic topic)
(when (ex/exception? result)
(log/errorf result "unexpected exception on subscribing to '%s'" topic))))
(l/error :cause result
:hint "unexpected exception on subscribing"
:topic topic))))
nsubs))
(subscribe-to-topics [state topics chan]
@ -210,9 +212,12 @@
(let [nsubs (disj nsubs chan)]
(when (empty? nsubs)
(let [result (a/<!! (impl-redis-unsub rac topic))]
(log/tracef "closing subscription to %s" topic)
(l/trace :action "close subscription"
:topic topic)
(when (ex/exception? result)
(log/errorf result "unexpected exception on unsubscribing from '%s'" topic))))
(l/error :cause result
:hint "unexpected exception on unsubscribing"
:topic topic))))
nsubs))
(unsubscribe-channels [state pending]
@ -246,7 +251,6 @@
(recur (rest chans) pending)
(recur (rest chans) (conj pending ch)))
pending))]
;; (log/tracef "received message => pending: %s" (pr-str pending))
(some->> (seq pending)
(send-off chans unsubscribe-channels))

25
backend/src/app/notifications.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.notifications
"A websocket based notifications mechanism."
@ -14,12 +11,12 @@
[app.db :as db]
[app.metrics :as mtx]
[app.util.async :as aa]
[app.util.logging :as l]
[app.util.time :as dt]
[app.util.transit :as t]
[app.worker :as wrk]
[clojure.core.async :as a]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]
[ring.adapter.jetty9 :as jetty]
[ring.middleware.cookies :refer [wrap-cookies]]
@ -149,7 +146,7 @@
:out-ch out-ch
:sub-ch sub-ch)]
(log/tracef "on-connect %s" (:session-id cfg))
(l/trace :event "connect" :session (:session-id cfg))
;; Forward all messages from out-ch to the websocket
;; connection
@ -171,20 +168,22 @@
;; close subscription
(a/close! sub-ch))))
(on-error [_conn e]
(log/tracef "on-error %s (%s)" (:session-id cfg) (ex-message e))
(on-error [_conn _e]
(l/trace :event "error" :session (:session-id cfg))
(a/close! out-ch)
(a/close! rcv-ch))
(on-close [_conn _status _reason]
(log/tracef "on-close %s" (:session-id cfg))
(l/trace :event "close" :session (:session-id cfg))
(a/close! out-ch)
(a/close! rcv-ch))
(on-message [_ws message]
(let [message (t/decode-str message)]
(when-not (a/offer! rcv-ch message)
(log/warn "droping ws input message, channe full"))))]
(l/warn :msg "drop messages"))))]
{:on-connect on-connect
:on-error on-error
@ -254,12 +253,10 @@
(defmethod handle-message :connect
[cfg _]
;; (log/debugf "profile '%s' is connected to file '%s'" profile-id file-id)
(send-presence cfg :connect))
(defmethod handle-message :disconnect
[cfg _]
;; (log/debugf "profile '%s' is disconnected from '%s'" profile-id file-id)
(send-presence cfg :disconnect))
(defmethod handle-message :keepalive
@ -277,5 +274,7 @@
(defmethod handle-message :default
[_ws message]
(a/go
(log/warnf "received unexpected message: %s" message)))
(l/log :level :warn
:msg "received unexpected message"
:message message)))

5
backend/src/app/rlimits.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rlimits
"Resource usage limits (in other words: semaphores)."

23
backend/src/app/rpc.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rpc
(:require
@ -15,9 +12,9 @@
[app.db :as db]
[app.metrics :as mtx]
[app.rlimits :as rlm]
[app.util.logging :as l]
[app.util.services :as sv]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[cuerdas.core :as str]
[integrant.core :as ig]))
@ -33,10 +30,15 @@
(defn- rpc-query-handler
[methods {:keys [profile-id] :as request}]
(let [type (keyword (get-in request [:path-params :type]))
data (assoc (:params request) ::type type)
data (d/merge (:params request)
(:body-params request)
(:uploads request))
data (if profile-id
(assoc data :profile-id profile-id)
(dissoc data :profile-id))
result ((get methods type default-handler) data)
mdata (meta result)]
@ -76,7 +78,8 @@
(ex/raise :type :internal
:code :rlimit-not-configured
:hint (str/fmt "%s rlimit not configured" key)))
(log/tracef "adding rlimit to '%s' rpc handler" (::sv/name mdata))
(l/trace :action "add rlimit"
:handler (::sv/name mdata))
(fn [cfg params]
(rlm/execute rlinst (f cfg params))))
f))
@ -86,7 +89,8 @@
(let [f (wrap-with-rlimits cfg f mdata)
f (wrap-with-metrics cfg f mdata)
spec (or (::sv/spec mdata) (s/spec any?))]
(log/tracef "registering '%s' command to rpc service" (::sv/name mdata))
(l/trace :action "register"
:name (::sv/name mdata))
(fn [params]
(when (and (:auth mdata true) (not (uuid? (:profile-id params))))
(ex/raise :type :authentication
@ -115,7 +119,8 @@
'app.rpc.queries.comments
'app.rpc.queries.profile
'app.rpc.queries.recent-files
'app.rpc.queries.viewer)
'app.rpc.queries.viewer
'app.rpc.queries.svg)
(map (partial process-method cfg))
(into {}))))

5
backend/src/app/rpc/mutations/comments.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.mutations.comments
(:require

16
backend/src/app/rpc/mutations/demo.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.mutations.demo
"A demo specific mutations."
@ -16,8 +13,8 @@
[app.db :as db]
[app.rpc.mutations.profile :as profile]
[app.setup.initial-data :as sid]
[app.tasks :as tasks]
[app.util.services :as sv]
[app.worker :as wrk]
[buddy.core.codecs :as bc]
[buddy.core.nonce :as bn]
[clojure.spec.alpha :as s]))
@ -40,7 +37,7 @@
:password password
:props {:onboarding-viewed true}}]
(when-not (:allow-demo-users cfg/config)
(when-not (cfg/get :allow-demo-users)
(ex/raise :type :validation
:code :demo-users-not-allowed
:hint "Demo users are disabled by config."))
@ -51,9 +48,10 @@
(sid/load-initial-project! conn))
;; Schedule deletion of the demo profile
(tasks/submit! conn {:name "delete-profile"
:delay cfg/deletion-delay
:props {:profile-id id}})
(wrk/submit! {::wrk/task :delete-profile
::wrk/delay cfg/deletion-delay
::wrk/conn conn
:profile-id id})
{:email email
:password password})))

15
backend/src/app/rpc/mutations/files.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.mutations.files
(:require
@ -19,10 +16,10 @@
[app.rpc.permissions :as perms]
[app.rpc.queries.files :as files]
[app.rpc.queries.projects :as proj]
[app.tasks :as tasks]
[app.util.blob :as blob]
[app.util.services :as sv]
[app.util.time :as dt]
[app.worker :as wrk]
[clojure.spec.alpha :as s]))
;; --- Helpers & Specs
@ -126,9 +123,11 @@
(files/check-edition-permissions! conn profile-id id)
;; Schedule object deletion
(tasks/submit! conn {:name "delete-object"
:delay cfg/deletion-delay
:props {:id id :type :file}})
(wrk/submit! {::wrk/task :delete-object
::wrk/delay cfg/deletion-delay
::wrk/conn conn
:id id
:type :file})
(mark-file-deleted conn params)))

5
backend/src/app/rpc/mutations/ldap.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.mutations.ldap
(:require

5
backend/src/app/rpc/mutations/management.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.mutations.management
"Move & Duplicate RPC methods for files and projects."

5
backend/src/app/rpc/mutations/media.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.mutations.media
(:require

96
backend/src/app/rpc/mutations/profile.clj
View file

@ -2,28 +2,26 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.mutations.profile
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.config :as cfg]
[app.db :as db]
[app.emails :as emails]
[app.emails :as eml]
[app.media :as media]
[app.rpc.mutations.projects :as projects]
[app.rpc.mutations.teams :as teams]
[app.rpc.queries.profile :as profile]
[app.setup.initial-data :as sid]
[app.storage :as sto]
[app.tasks :as tasks]
[app.util.services :as sv]
[app.util.time :as dt]
[app.worker :as wrk]
[buddy.hashers :as hashers]
[clojure.spec.alpha :as s]
[cuerdas.core :as str]))
@ -117,16 +115,19 @@
;; Don't allow proceed in register page if the email is
;; already reported as permanent bounced
(when (emails/has-bounce-reports? conn (:email profile))
(when (eml/has-bounce-reports? conn (:email profile))
(ex/raise :type :validation
:code :email-has-permanent-bounces
:hint "looks like the email has one or many bounces reported"))
(emails/send! conn emails/register
{:to (:email profile)
:name (:fullname profile)
:token vtoken
:extra-data ptoken})
(eml/send! {::eml/conn conn
::eml/factory eml/register
:public-uri (:public-uri cfg)
:to (:email profile)
:name (:fullname profile)
:token vtoken
:extra-data ptoken})
(with-meta profile
{:before-complete (annotate-profile-register metrics profile)})))))
@ -303,16 +304,35 @@
(defn login-or-register
[{:keys [conn] :as cfg} {:keys [email backend] :as params}]
(letfn [(create-profile [conn {:keys [fullname email]}]
(letfn [(info->props [info]
(dissoc info :name :fullname :email :backend))
(info->lang [{:keys [locale] :as info}]
(when (and (string? locale)
(not (str/blank? locale)))
locale))
(create-profile [conn {:keys [email] :as info}]
(db/insert! conn :profile
{:id (uuid/next)
:fullname fullname
:fullname (:fullname info)
:email (str/lower email)
:lang (info->lang info)
:auth-backend backend
:is-active true
:password "!"
:props (db/tjson (info->props info))
:is-demo false}))
(update-profile [conn info profile]
(let [props (d/merge (:props profile)
(info->props info))]
(db/update! conn :profile
{:props (db/tjson props)
:modified-at (dt/now)}
{:id (:id profile)})
(assoc profile :props props)))
(register-profile [conn params]
(let [profile (->> (create-profile conn params)
(create-profile-relations conn))]
@ -321,7 +341,9 @@
(let [profile (profile/retrieve-profile-data-by-email conn email)
profile (if profile
(profile/populate-additional-data conn profile)
(->> profile
(update-profile conn params)
(profile/populate-additional-data conn))
(register-profile conn params))]
(profile/strip-private-attrs profile))))
@ -346,7 +368,6 @@
(update-profile conn params)
nil))
;; --- Mutation: Update Password
(declare validate-password!)
@ -439,7 +460,7 @@
{:changed true})
(defn- request-email-change
[{:keys [conn tokens]} {:keys [profile email] :as params}]
[{:keys [conn tokens] :as cfg} {:keys [profile email] :as params}]
(let [token (tokens :generate
{:iss :change-email
:exp (dt/in-future "15m")
@ -452,22 +473,24 @@
(when (not= email (:email profile))
(check-profile-existence! conn params))
(when-not (emails/allow-send-emails? conn profile)
(when-not (eml/allow-send-emails? conn profile)
(ex/raise :type :validation
:code :profile-is-muted
:hint "looks like the profile has reported repeatedly as spam or has permanent bounces."))
(when (emails/has-bounce-reports? conn email)
(when (eml/has-bounce-reports? conn email)
(ex/raise :type :validation
:code :email-has-permanent-bounces
:hint "looks like the email you invite has been repeatedly reported as spam or permanent bounce"))
(emails/send! conn emails/change-email
{:to (:email profile)
:name (:fullname profile)
:pending-email email
:token token
:extra-data ptoken})
(eml/send! {::eml/conn conn
::eml/factory eml/change-email
:public-uri (:public-uri cfg)
:to (:email profile)
:name (:fullname profile)
:pending-email email
:token token
:extra-data ptoken})
nil))
@ -493,16 +516,18 @@
(let [ptoken (tokens :generate-predefined
{:iss :profile-identity
:profile-id (:id profile)})]
(emails/send! conn emails/password-recovery
{:to (:email profile)
:token (:token profile)
:name (:fullname profile)
:extra-data ptoken})
(eml/send! {::eml/conn conn
::eml/factory eml/password-recovery
:public-uri (:public-uri cfg)
:to (:email profile)
:token (:token profile)
:name (:fullname profile)
:extra-data ptoken})
nil))]
(db/with-atomic [conn pool]
(when-let [profile (profile/retrieve-profile-data-by-email conn email)]
(when-not (emails/allow-send-emails? conn profile)
(when-not (eml/allow-send-emails? conn profile)
(ex/raise :type :validation
:code :profile-is-muted
:hint "looks like the profile has reported repeatedly as spam or has permanent bounces."))
@ -512,7 +537,7 @@
:code :profile-not-verified
:hint "the user need to validate profile before recover password"))
(when (emails/has-bounce-reports? conn (:email profile))
(when (eml/has-bounce-reports? conn (:email profile))
(ex/raise :type :validation
:code :email-has-permanent-bounces
:hint "looks like the email you invite has been repeatedly reported as spam or permanent bounce"))
@ -579,9 +604,10 @@
(check-can-delete-profile! conn profile-id)
;; Schedule a complete deletion of profile
(tasks/submit! conn {:name "delete-profile"
:delay cfg/deletion-delay
:props {:profile-id profile-id}})
(wrk/submit! {::wrk/task :delete-profile
::wrk/dalay cfg/deletion-delay
::wrk/conn conn
:profile-id profile-id})
(db/update! conn :profile
{:deleted-at (dt/now)}

15
backend/src/app/rpc/mutations/projects.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.mutations.projects
(:require
@ -16,9 +13,9 @@
[app.rpc.permissions :as perms]
[app.rpc.queries.projects :as proj]
[app.rpc.queries.teams :as teams]
[app.tasks :as tasks]
[app.util.services :as sv]
[app.util.time :as dt]
[app.worker :as wrk]
[clojure.spec.alpha :as s]))
;; --- Helpers & Specs
@ -128,9 +125,11 @@
(proj/check-edition-permissions! conn profile-id id)
;; Schedule object deletion
(tasks/submit! conn {:name "delete-object"
:delay cfg/deletion-delay
:props {:id id :type :project}})
(wrk/submit! {::wrk/task :delete-object
::wrk/delay cfg/deletion-delay
::wrk/conn conn
:id id
:type :project})
(db/update! conn :project
{:deleted-at (dt/now)}

37
backend/src/app/rpc/mutations/teams.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.mutations.teams
(:require
@ -15,16 +12,16 @@
[app.common.uuid :as uuid]
[app.config :as cfg]
[app.db :as db]
[app.emails :as emails]
[app.emails :as eml]
[app.media :as media]
[app.rpc.mutations.projects :as projects]
[app.rpc.permissions :as perms]
[app.rpc.queries.profile :as profile]
[app.rpc.queries.teams :as teams]
[app.storage :as sto]
[app.tasks :as tasks]
[app.util.services :as sv]
[app.util.time :as dt]
[app.worker :as wrk]
[clojure.spec.alpha :as s]
[datoteka.core :as fs]))
@ -139,9 +136,11 @@
:code :only-owner-can-delete-team))
;; Schedule object deletion
(tasks/submit! conn {:name "delete-object"
:delay cfg/deletion-delay
:props {:id id :type :team}})
(wrk/submit! {::wrk/task :delete-object
::wrk/delay cfg/deletion-delay
::wrk/conn conn
:id id
:type :team})
(db/update! conn :team
{:deleted-at (dt/now)}
@ -323,27 +322,29 @@
:code :insufficient-permissions))
;; First check if the current profile is allowed to send emails.
(when-not (emails/allow-send-emails? conn profile)
(when-not (eml/allow-send-emails? conn profile)
(ex/raise :type :validation
:code :profile-is-muted
:hint "looks like the profile has reported repeatedly as spam or has permanent bounces"))
(when (and member (not (emails/allow-send-emails? conn member)))
(when (and member (not (eml/allow-send-emails? conn member)))
(ex/raise :type :validation
:code :member-is-muted
:hint "looks like the profile has reported repeatedly as spam or has permanent bounces"))
;; Secondly check if the invited member email is part of the
;; global spam/bounce report.
(when (emails/has-bounce-reports? conn email)
(when (eml/has-bounce-reports? conn email)
(ex/raise :type :validation
:code :email-has-permanent-bounces
:hint "looks like the email you invite has been repeatedly reported as spam or permanent bounce"))
(emails/send! conn emails/invite-to-team
{:to email
:invited-by (:fullname profile)
:team (:name team)
:token itoken
:extra-data ptoken})
(eml/send! {::eml/conn conn
::eml/factory eml/invite-to-team
:public-uri (:public-uri cfg)
:to email
:invited-by (:fullname profile)
:team (:name team)
:token itoken
:extra-data ptoken})
nil)))

5
backend/src/app/rpc/mutations/verify_token.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.mutations.verify-token
(:require

5
backend/src/app/rpc/mutations/viewer.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.mutations.viewer
(:require

5
backend/src/app/rpc/permissions.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.permissions
"A permission checking helper factories."

6
backend/src/app/rpc/queries/comments.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.queries.comments
(:require
@ -131,7 +128,6 @@
(-> (db/exec-one! conn [sql profile-id file-id id])
(decode-row)))))
;; --- Query: Comments
(declare retrieve-comments)

5
backend/src/app/rpc/queries/files.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.queries.files
(:require

5
backend/src/app/rpc/queries/profile.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.queries.profile
(:require

5
backend/src/app/rpc/queries/projects.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.queries.projects
(:require

5
backend/src/app/rpc/queries/recent_files.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.queries.recent-files
(:require

58
backend/src/app/rpc/queries/svg.clj Normal file
View file

@ -0,0 +1,58 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.queries.svg
(:require
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.util.logging :as l]
[app.util.services :as sv]
[clojure.spec.alpha :as s]
[clojure.xml :as xml]
[cuerdas.core :as str])
(:import
javax.xml.XMLConstants
javax.xml.parsers.SAXParserFactory
org.apache.commons.io.IOUtils))
(defn- secure-parser-factory
[s ch]
(.. (doto (SAXParserFactory/newInstance)
(.setFeature XMLConstants/FEATURE_SECURE_PROCESSING true)
(.setFeature "http://apache.org/xml/features/disallow-doctype-decl" true))
(newSAXParser)
(parse s ch)))
(defn parse
[data]
(try
(with-open [istream (IOUtils/toInputStream data "UTF-8")]
(xml/parse istream secure-parser-factory))
(catch Exception e
(l/warn :hint "error on processing svg"
:message (ex-message e))
(ex/raise :type :validation
:code :invalid-svg-file
:cause e))))
(declare pre-process)
(s/def ::data ::us/string)
(s/def ::parsed-svg (s/keys :req-un [::data]))
(sv/defmethod ::parsed-svg
[_ {:keys [data] :as params}]
(->> data pre-process parse))
;; --- PROCESSORS
(defn strip-doctype
[data]
(cond-> data
(str/includes? data "<!DOCTYPE")
(str/replace #"<\!DOCTYPE[^>]+>" "")))
(def pre-process strip-doctype)

5
backend/src/app/rpc/queries/teams.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.queries.teams
(:require

5
backend/src/app/rpc/queries/viewer.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.rpc.queries.viewer
(:require

5
backend/src/app/setup.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.setup
"Initial data setup of instance."

5
backend/src/app/setup/initial_data.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.setup.initial-data
(:refer-clojure :exclude [load])

5
backend/src/app/srepl.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.srepl
"Server Repl."

23
backend/src/app/storage.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.storage
"File Storage abstraction layer."
@ -19,10 +16,10 @@
[app.storage.fs :as sfs]
[app.storage.impl :as impl]
[app.storage.s3 :as ss3]
[app.util.logging :as l]
[app.util.time :as dt]
[app.worker :as wrk]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[cuerdas.core :as str]
[datoteka.core :as fs]
[integrant.core :as ig]
@ -310,7 +307,9 @@
(run! (partial delete-in-bulk conn) groups)
(recur (+ n ^long total)))
(do
(log/infof "gc-deleted: processed %s items" n)
(l/info :task "gc-deleted"
:action "permanently delete items"
:count n)
{:deleted n})))))))
(def sql:retrieve-deleted-objects
@ -382,7 +381,12 @@
(recur (+ cntf (count to-freeze))
(+ cntd (count to-delete))))
(do
(log/infof "gc-touched: %s objects marked as freeze and %s marked to be deleted" cntf cntd)
(l/info :task "gc-touched"
:action "mark freeze"
:count cntf)
(l/info :task "gc-touched"
:action "mark for deletion"
:count cntd)
{:freeze cntf :delete cntd})))))))
(def sql:retrieve-touched-objects
@ -459,7 +463,10 @@
(recur (+ n (count all))
(+ d (count to-delete))))
(do
(log/infof "recheck: processed %s items, %s deleted" n d)
(l/info :task "recheck"
:action "recheck items"
:processed n
:deleted n)
{:processed n :deleted d})))))))
(def sql:retrieve-pending-to-recheck

5
backend/src/app/storage/db.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.storage.db
(:require

11
backend/src/app/storage/fs.clj
View file

@ -2,22 +2,19 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.storage.fs
(:require
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.common.uri :as u]
[app.storage.impl :as impl]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[cuerdas.core :as str]
[datoteka.core :as fs]
[integrant.core :as ig]
[lambdaisland.uri :as u])
[integrant.core :as ig])
(:import
java.io.InputStream
java.io.OutputStream
@ -43,7 +40,7 @@
:uri (u/uri (str "file://" dir))))))
(s/def ::type ::us/keyword)
(s/def ::uri #(instance? lambdaisland.uri.URI %))
(s/def ::uri u/uri?)
(s/def ::backend
(s/keys :req-un [::type ::directory ::uri]))

5
backend/src/app/storage/impl.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.storage.impl
"Storage backends abstraction layer."

9
backend/src/app/storage/s3.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.storage.s3
"Storage backends abstraction layer."
@ -13,12 +10,12 @@
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.common.uri :as u]
[app.storage.impl :as impl]
[app.util.time :as dt]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[integrant.core :as ig]
[lambdaisland.uri :as u])
[integrant.core :as ig])
(:import
java.time.Duration
java.util.Collection

72
backend/src/app/svgparse.clj
View file

@ -1,72 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) UXBOX Labs SL
(ns app.svgparse
(:require
[app.common.exceptions :as ex]
[app.metrics :as mtx]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[clojure.xml :as xml]
[integrant.core :as ig])
(:import
org.apache.commons.io.IOUtils))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Handler
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(declare handler)
(declare process-request)
(defmethod ig/pre-init-spec ::handler [_]
(s/keys :req-un [::mtx/metrics]))
(defmethod ig/init-key ::handler
[_ {:keys [metrics] :as cfg}]
(let [handler #(handler cfg %)]
(->> {:registry (:registry metrics)
:type :summary
:name "http_handler_svgparse_timing"
:help "svg parse timings"}
(mtx/instrument handler))))
(defn- handler
[_ {:keys [headers body] :as request}]
(when (not= "image/svg+xml" (get headers "content-type"))
(ex/raise :type :validation
:code :unsupported-mime-type
:mime (get headers "content-type")))
{:status 200
:body (process-request body)})
(defn secure-factory
[s ch]
(.. (doto (javax.xml.parsers.SAXParserFactory/newInstance)
(.setFeature javax.xml.XMLConstants/FEATURE_SECURE_PROCESSING true)
(.setFeature "http://apache.org/xml/features/disallow-doctype-decl" true))
(newSAXParser)
(parse s ch)))
(defn parse
[data]
(try
(with-open [istream (IOUtils/toInputStream data "UTF-8")]
(xml/parse istream secure-factory))
(catch Exception e
(log/warnf "error on processing svg: %s" (ex-message e))
(ex/raise :type :validation
:code :invalid-svg-file
:cause e))))
(defn process-request
[body]
(let [data (slurp body)]
(parse data)))

110
backend/src/app/tasks.clj
View file

@ -1,110 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
(ns app.tasks
(:require
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.db :as db]
[app.metrics :as mtx]
[app.util.time :as dt]
[app.worker]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]))
(s/def ::name ::us/string)
(s/def ::delay
(s/or :int ::us/integer
:duration dt/duration?))
(s/def ::queue ::us/string)
(s/def ::task-options
(s/keys :req-un [::name]
:opt-un [::delay ::props ::queue]))
(def ^:private sql:insert-new-task
"insert into task (id, name, props, queue, priority, max_retries, scheduled_at)
values (?, ?, ?, ?, ?, ?, clock_timestamp() + ?)
returning id")
(defn submit!
[conn {:keys [name delay props queue priority max-retries]
:or {delay 0 props {} queue "default" priority 100 max-retries 3}
:as options}]
(us/verify ::task-options options)
(let [duration (dt/duration delay)
interval (db/interval duration)
props (db/tjson props)
id (uuid/next)]
(log/debugf "submit task '%s' to be executed in '%s'" name (str duration))
(db/exec-one! conn [sql:insert-new-task id name props queue priority max-retries interval])
id))
(defn- instrument!
[registry]
(mtx/instrument-vars!
[#'submit!]
{:registry registry
:type :counter
:labels ["name"]
:name "tasks_submit_total"
:help "A counter of task submissions."
:wrap (fn [rootf mobj]
(let [mdata (meta rootf)
origf (::original mdata rootf)]
(with-meta
(fn [conn params]
(let [tname (:name params)]
(mobj :inc [tname])
(origf conn params)))
{::original origf})))})
(mtx/instrument-vars!
[#'app.worker/run-task]
{:registry registry
:type :summary
:quantiles []
:name "tasks_checkout_timing"
:help "Latency measured between scheduld_at and execution time."
:wrap (fn [rootf mobj]
(let [mdata (meta rootf)
origf (::original mdata rootf)]
(with-meta
(fn [tasks item]
(let [now (inst-ms (dt/now))
sat (inst-ms (:scheduled-at item))]
(mobj :observe (- now sat))
(origf tasks item)))
{::original origf})))}))
;; --- STATE INIT: REGISTRY
(s/def ::tasks
(s/map-of keyword? fn?))
(defmethod ig/pre-init-spec ::registry [_]
(s/keys :req-un [::mtx/metrics ::tasks]))
(defmethod ig/init-key ::registry
[_ {:keys [metrics tasks]}]
(instrument! (:registry metrics))
(let [mobj (mtx/create
{:registry (:registry metrics)
:type :summary
:labels ["name"]
:quantiles []
:name "tasks_timing"
:help "Background task execution timing."})]
(reduce-kv (fn [res k v]
(let [tname (name k)]
(log/debugf "registring task '%s'" tname)
(assoc res tname (mtx/wrap-summary v mobj [tname]))))
{}
tasks)))

11
backend/src/app/tasks/delete_object.clj
View file

@ -2,18 +2,16 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.tasks.delete-object
"Generic task for permanent deletion of objects."
(:require
[app.common.data :as d]
[app.common.spec :as us]
[app.db :as db]
[app.util.logging :as l]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]))
(declare handle-deletion)
@ -37,7 +35,8 @@
(defmethod handle-deletion :default
[_conn {:keys [type]}]
(log/warnf "no handler found for '%s'" type))
(l/warn :hint "no handler found"
:type (d/name type)))
(defmethod handle-deletion :file
[conn {:keys [id] :as props}]

13
backend/src/app/tasks/delete_profile.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.tasks.delete-profile
"Task for permanent deletion of profiles."
@ -13,8 +10,8 @@
[app.common.spec :as us]
[app.db :as db]
[app.db.sql :as sql]
[app.util.logging :as l]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]))
(declare delete-profile-data)
@ -47,7 +44,8 @@
(if (or (:is-demo profile)
(:deleted-at profile))
(delete-profile-data conn id)
(log/warnf "profile '%s' does not match constraints for deletion" id))))))
(l/warn :hint "profile does not match constraints for deletion"
:profile-id id))))))
;; --- IMPL
@ -70,7 +68,8 @@
(defn- delete-profile-data
[conn profile-id]
(log/debugf "proceding to delete all data related to profile '%s'" profile-id)
(l/debug :action "delete profile"
:profile-id profile-id)
(delete-teams conn profile-id)
(delete-profile conn profile-id)
true)

20
backend/src/app/tasks/file_media_gc.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.tasks.file-media-gc
"A maintenance task that is responsible to purge the unused media
@ -15,9 +12,9 @@
[app.common.pages.migrations :as pmg]
[app.db :as db]
[app.util.blob :as blob]
[app.util.logging :as l]
[app.util.time :as dt]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]))
(declare process-file)
@ -40,7 +37,7 @@
(run! (partial process-file cfg) files)
(recur (+ n (count files))))
(do
(log/debugf "finalized with total of %s processed files" n)
(l/debug :msg "finished processing files" :processed n)
{:processed n}))))))))
(def ^:private
@ -88,7 +85,10 @@
unused (->> (db/query conn :file-media-object {:file-id id})
(remove #(contains? used (:id %))))]
(log/debugf "processing file: id='%s' age='%s' to-delete=%s" id age (count unused))
(l/debug :action "processing file"
:id id
:age age
:to-delete (count unused))
;; Mark file as trimmed
(db/update! conn :file
@ -96,8 +96,10 @@
{:id id})
(doseq [mobj unused]
(log/debugf "deleting media object: id='%s' media-id='%s' thumb-id='%s'"
(:id mobj) (:media-id mobj) (:thumbnail-id mobj))
(l/debug :action "deleting media object"
:id (:id mobj)
:media-id (:media-id mobj)
:thumbnail-id (:thumbnail-id mobj))
;; NOTE: deleting the file-media-object in the database
;; automatically marks as toched the referenced storage objects.
(db/delete! conn :file-media-object {:id (:id mobj)}))

9
backend/src/app/tasks/file_xlog_gc.clj
View file

@ -2,19 +2,16 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.tasks.file-xlog-gc
"A maintenance task that performs a garbage collection of the file
change (transaction) log."
(:require
[app.db :as db]
[app.util.logging :as l]
[app.util.time :as dt]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]))
(declare sql:delete-files-xlog)
@ -31,7 +28,7 @@
(let [interval (db/interval max-age)
result (db/exec-one! conn [sql:delete-files-xlog interval])
result (:next.jdbc/update-count result)]
(log/debugf "removed %s rows from file-change table" result)
(l/debug :action "trim file-change table" :removed result)
result))))
(def ^:private

58
backend/src/app/tasks/sendmail.clj
View file

@ -1,58 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
(ns app.tasks.sendmail
(:require
[app.config :as cfg]
[app.util.emails :as emails]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]))
(declare send-console!)
(s/def ::username ::cfg/smtp-username)
(s/def ::password ::cfg/smtp-password)
(s/def ::tls ::cfg/smtp-tls)
(s/def ::ssl ::cfg/smtp-ssl)
(s/def ::host ::cfg/smtp-host)
(s/def ::port ::cfg/smtp-port)
(s/def ::default-reply-to ::cfg/smtp-default-reply-to)
(s/def ::default-from ::cfg/smtp-default-from)
(s/def ::enabled ::cfg/smtp-enabled)
(defmethod ig/pre-init-spec ::handler [_]
(s/keys :req-un [::enabled]
:opt-un [::username
::password
::tls
::ssl
::host
::port
::default-from
::default-reply-to]))
(defmethod ig/init-key ::handler
[_ cfg]
(fn [{:keys [props] :as task}]
(if (:enabled cfg)
(emails/send! cfg props)
(send-console! cfg props))))
(defn- send-console!
[cfg email]
(let [baos (java.io.ByteArrayOutputStream.)
mesg (emails/smtp-message cfg email)]
(.writeTo mesg baos)
(let [out (with-out-str
(println "email console dump:")
(println "******** start email" (:id email) "**********")
(println (.toString baos))
(println "******** end email "(:id email) "**********"))]
(log/info out))))

9
backend/src/app/tasks/tasks_gc.clj
View file

@ -2,19 +2,16 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.tasks.tasks-gc
"A maintenance task that performs a cleanup of already executed tasks
from the database table."
(:require
[app.db :as db]
[app.util.logging :as l]
[app.util.time :as dt]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]))
(declare sql:delete-completed-tasks)
@ -31,7 +28,7 @@
(let [interval (db/interval max-age)
result (db/exec-one! conn [sql:delete-completed-tasks interval])
result (:next.jdbc/update-count result)]
(log/debugf "removed %s rows from tasks-completed table" result)
(l/debug :action "trim completed tasks table" :removed result)
result))))
(def ^:private

28
backend/src/app/tasks/telemetry.clj
View file

@ -2,16 +2,14 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.tasks.telemetry
"A task that is reponsible to collect anonymous statistical
information about the current instance and send it to the telemetry
server."
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.config :as cfg]
@ -32,7 +30,6 @@
(s/def ::sprops
(s/keys :req-un [::instance-id]))
(defmethod ig/pre-init-spec ::handler [_]
(s/keys :req-un [::db/pool ::version ::uri ::sprops]))
@ -128,11 +125,16 @@
(defn- retrieve-stats
[{:keys [conn version]}]
(merge
{:version version
:with-taiga (:telemetry-with-taiga cfg/config false)
:total-teams (retrieve-num-teams conn)
:total-projects (retrieve-num-projects conn)
:total-files (retrieve-num-files conn)}
(retrieve-team-averages conn)
(retrieve-jvm-stats)))
(let [referer (if (cfg/get :telemetry-with-taiga)
"taiga"
(cfg/get :telemetry-referer))]
(-> {:version version
:referer referer
:total-teams (retrieve-num-teams conn)
:total-projects (retrieve-num-projects conn)
:total-files (retrieve-num-files conn)}
(d/merge
(retrieve-team-averages conn)
(retrieve-jvm-stats))
(d/without-nils))))

121
backend/src/app/telemetry.clj
View file

@ -1,121 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
(ns app.telemetry
(:require
[app.common.spec :as us]
[app.db :as db]
[app.http.middleware :refer [wrap-parse-request-body]]
[clojure.pprint :refer [pprint]]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[integrant.core :as ig]
[promesa.exec :as px]
[ring.middleware.keyword-params :refer [wrap-keyword-params]]
[ring.middleware.params :refer [wrap-params]]))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Migrations
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def sql:create-instance-table
"CREATE TABLE IF NOT EXISTS telemetry.instance (
id uuid PRIMARY KEY,
created_at timestamptz NOT NULL DEFAULT now()
);")
(def sql:create-info-table
"CREATE TABLE telemetry.info (
instance_id uuid,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
data jsonb NOT NULL,
PRIMARY KEY (instance_id, created_at)
) PARTITION BY RANGE(created_at);
CREATE TABLE telemetry.info_default (LIKE telemetry.info INCLUDING ALL);
ALTER TABLE telemetry.info
ATTACH PARTITION telemetry.info_default DEFAULT;")
(def migrations
[{:name "0001-add-telemetry-schema"
:fn #(db/exec! % ["CREATE SCHEMA IF NOT EXISTS telemetry;"])}
{:name "0002-add-instance-table"
:fn #(db/exec! % [sql:create-instance-table])}
{:name "0003-add-info-table"
:fn #(db/exec! % [sql:create-info-table])}
{:name "0004-del-instance-table"
:fn #(db/exec! % ["DROP TABLE telemetry.instance;"])}])
(defmethod ig/init-key ::migrations [_ _] migrations)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Router Handler
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(declare handler)
(declare process-request)
(defmethod ig/init-key ::handler
[_ cfg]
(-> (partial handler cfg)
(wrap-keyword-params)
(wrap-params)
(wrap-parse-request-body)))
(s/def ::instance-id ::us/uuid)
(s/def ::params (s/keys :req-un [::instance-id]))
(defn handler
[{:keys [executor] :as cfg} {:keys [params] :as request}]
(try
(let [params (us/conform ::params params)
cfg (assoc cfg
:instance-id (:instance-id params)
:data (dissoc params :instance-id))]
(px/run! executor (partial process-request cfg)))
(catch Exception e
;; We don't want notify user of a error, just log it for posible
;; future investigation.
(log/warn e (str "unexpected error on telemetry:\n"
(when-let [edata (ex-data e)]
(str "ex-data: \n"
(with-out-str (pprint edata))))
(str "params: \n"
(with-out-str (pprint params)))))))
{:status 200
:body "OK\n"})
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Request Processing
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def sql:insert-instance-info
"insert into telemetry.info (instance_id, data, created_at)
values (?, ?, date_trunc('day', now()))
on conflict (instance_id, created_at)
do update set data = ?")
(defn- process-request
[{:keys [pool instance-id data]}]
(try
(db/with-atomic [conn pool]
(let [data (db/json data)]
(db/exec! conn [sql:insert-instance-info
instance-id
data
data])))
(catch Exception e
(log/errorf e "error on procesing request"))))

5
backend/src/app/tokens.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.tokens
"Tokens generation service."

2
backend/src/app/util/async.clj
View file

@ -2,7 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) 2020 Andrey Antukh <niwi@niwi.nz>
;; Copyright (c) UXBOX Labs SL
(ns app.util.async
(:require

27
backend/src/app/util/blob.clj
View file

@ -2,16 +2,13 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2016-2020 Andrey Antukh <niwi@niwi.nz>
;; Copyright (c) UXBOX Labs SL
(ns app.util.blob
"A generic blob storage encoding. Mainly used for
page data, page options and txlog payload storage."
"A generic blob storage encoding. Mainly used for page data, page
options and txlog payload storage."
(:require
[app.config :as cfg]
[app.config :as cf]
[app.util.transit :as t]
[taoensso.nippy :as n])
(:import
@ -33,17 +30,15 @@
(declare encode-v2)
(declare encode-v3)
(def default-version
(:default-blob-version cfg/config 1))
(defn encode
([data] (encode data nil))
([data {:keys [version] :or {version default-version}}]
(case (long version)
1 (encode-v1 data)
2 (encode-v2 data)
3 (encode-v3 data)
(throw (ex-info "unsupported version" {:version version})))))
([data {:keys [version]}]
(let [version (or version (cf/get :default-blob-version 1))]
(case (long version)
1 (encode-v1 data)
2 (encode-v2 data)
3 (encode-v3 data)
(throw (ex-info "unsupported version" {:version version}))))))
(defn decode
"A function used for decode persisted blobs in the database."

2
backend/src/app/util/closeable.clj
View file

@ -2,7 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) 2016 Andrey Antukh <niwi@niwi.nz>
;; Copyright (c) UXBOX Labs SL
(ns app.util.closeable
"A closeable abstraction. A drop in replacement for

54
backend/src/app/util/data.clj
View file

@ -1,54 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) 2016 Andrey Antukh <niwi@niwi.nz>
(ns app.util.data
"Data transformations utils."
(:require [clojure.walk :as walk]
[cuerdas.core :as str]))
;; TODO: move to app.common.helpers
(defn dissoc-in
[m [k & ks]]
(if ks
(if-let [nextmap (get m k)]
(let [newmap (dissoc-in nextmap ks)]
(if (seq newmap)
(assoc m k newmap)
(dissoc m k)))
m)
(dissoc m k)))
(defn normalize-attrs
"Recursively transforms all map keys from strings to keywords."
[m]
(letfn [(tf [[k v]]
(let [ks (-> (name k)
(str/replace "_" "-"))]
[(keyword ks) v]))
(walker [x]
(if (map? x)
(into {} (map tf) x)
x))]
(walk/postwalk walker m)))
(defn strip-delete-attrs
[m]
(dissoc m :deleted-at))
(defn normalize
"Perform a common normalization transformation
for a entity (database retrieved) data structure."
[m]
(-> m normalize-attrs strip-delete-attrs))
(defn deep-merge
[& maps]
(letfn [(merge' [& maps]
(if (every? map? maps)
(apply merge-with merge' maps)
(last maps)))]
(apply merge' (remove nil? maps))))

95
backend/src/app/util/dispatcher.clj
View file

@ -1,95 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) 2019 Andrey Antukh <niwi@niwi.nz>
(ns app.util.dispatcher
"A generic service dispatcher implementation."
(:refer-clojure :exclude [defmethod])
(:require
[app.common.exceptions :as ex]
[clojure.spec.alpha :as s])
(:import
java.util.HashMap
java.util.Map))
(definterface IDispatcher
(^void add [key f]))
(deftype Dispatcher [reg attr wrap]
IDispatcher
(add [this key f]
(.put ^Map reg key (wrap f))
this)
clojure.lang.IDeref
(deref [_]
{:registry reg
:attr attr
:wrap wrap})
clojure.lang.IFn
(invoke [_ params]
(let [key (get params attr)
f (.get ^Map reg key)]
(when (nil? f)
(ex/raise :type :method-not-found
:hint "No method found for the current request."
:context {:key key}))
(f params))))
(defn dispatcher?
[v]
(instance? IDispatcher v))
(defmacro defservice
[sname & {:keys [dispatch-by wrap]}]
`(def ~sname (Dispatcher. (HashMap.) ~dispatch-by ~wrap)))
(defn parse-defmethod
[args]
(loop [r {}
s 0
v (first args)
n (rest args)]
(case s
0 (if (symbol? v)
(recur (assoc r :sym v) 1 (first n) (rest n))
(throw (ex-info "first arg to `defmethod` should be a symbol" {})))
1 (if (qualified-keyword? v)
(recur (-> r
(assoc :key (keyword (name v)))
(assoc :meta {:spec v :doc nil}))
3 (first n) (rest n))
(recur r (inc s) v n))
2 (if (simple-keyword? v)
(recur (-> r
(assoc :key v)
(assoc :meta {:doc nil}))
3 (first n) (rest n))
(throw (ex-info "second arg to `defmethod` should be a keyword" {})))
3 (if (string? v)
(recur (update r :meta assoc :doc v) (inc s) (first n) (rest n))
(recur r 4 v n))
4 (if (map? v)
(recur (update r :meta merge v) (inc s) (first n) (rest n))
(recur r 5 v n))
5 (if (vector? v)
(assoc r :args v :body n)
(throw (ex-info "missing arguments vector" {}))))))
(defn add-method
[^Dispatcher dsp key f meta]
(let [f (with-meta f meta)]
(.add dsp key f)
dsp))
(defmacro defmethod
[& args]
(let [{:keys [key meta sym args body]} (parse-defmethod args)
f `(fn ~args ~@body)]
`(do
(s/assert dispatcher? ~sym)
(add-method ~sym ~key ~f ~meta))))

5
backend/src/app/util/emails.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.util.emails
(:require

5
backend/src/app/util/http.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.util.http
"Http client abstraction layer."

5
backend/src/app/util/json.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.util.json
(:refer-clojure :exclude [read])

27
backend/src/app/util/log4j.clj
View file

@ -1,27 +0,0 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2021 UXBOX Labs SL
(ns app.util.log4j
(:require
[clojure.pprint :refer [pprint]])
(:import
org.apache.logging.log4j.ThreadContext))
(defn update-thread-context!
[data]
(run! (fn [[key val]]
(ThreadContext/put
(name key)
(cond
(coll? val)
(binding [clojure.pprint/*print-right-margin* 120]
(with-out-str (pprint val)))
(instance? clojure.lang.Named val) (name val)
:else (str val))))
data))

106
backend/src/app/util/logging.clj Normal file
View file

@ -0,0 +1,106 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) UXBOX Labs SL
(ns app.util.logging
(:require
[clojure.pprint :refer [pprint]])
(:import
org.apache.logging.log4j.Level
org.apache.logging.log4j.LogManager
org.apache.logging.log4j.Logger
org.apache.logging.log4j.ThreadContext
org.apache.logging.log4j.message.MapMessage
org.apache.logging.log4j.spi.LoggerContext))
(defn build-map-message
[m]
(let [message (MapMessage. (count m))]
(reduce-kv #(.with ^MapMessage %1 (name %2) %3) message m)))
(defprotocol ILogger
(-enabled? [logger level])
(-write! [logger level throwable message]))
(def logger-context
(LogManager/getContext false))
(def logging-agent
(agent nil :error-mode :continue))
(defn get-logger
[lname]
(.getLogger ^LoggerContext logger-context ^String lname))
(defn get-level
[level]
(case level
:trace Level/TRACE
:debug Level/DEBUG
:info Level/INFO
:warn Level/WARN
:error Level/ERROR
:fatal Level/FATAL))
(defn enabled?
[logger level]
(.isEnabled ^Logger logger ^Level level))
(defn write-log!
[logger level e msg]
(if e
(.log ^Logger logger
^Level level
^Object msg
^Throwable e)
(.log ^Logger logger
^Level level
^Object msg)))
(defmacro log
[& {:keys [level cause ::logger ::async] :as props}]
(let [props (dissoc props :level :cause ::logger ::async)
logger (or logger (str *ns*))
logger-sym (gensym "log")
level-sym (gensym "log")]
`(let [~logger-sym (get-logger ~logger)
~level-sym (get-level ~level)]
(if (enabled? ~logger-sym ~level-sym)
~(if async
`(send-off logging-agent (fn [_#] (write-log! ~logger-sym ~level-sym ~cause (build-map-message ~props))))
`(write-log! ~logger-sym ~level-sym ~cause (build-map-message ~props)))))))
(defmacro info
[& params]
`(log :level :info ~@params))
(defmacro error
[& params]
`(log :level :error ~@params))
(defmacro warn
[& params]
`(log :level :warn ~@params))
(defmacro debug
[& params]
`(log :level :debug ~@params))
(defmacro trace
[& params]
`(log :level :trace ~@params))
(defn update-thread-context!
[data]
(run! (fn [[key val]]
(ThreadContext/put
(name key)
(cond
(coll? val)
(binding [clojure.pprint/*print-right-margin* 120]
(with-out-str (pprint val)))
(instance? clojure.lang.Named val) (name val)
:else (str val))))
data))

10
backend/src/app/util/migrations.clj
View file

@ -2,17 +2,13 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.util.migrations
(:require
[app.util.logging :as l]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[cuerdas.core :as str]
[next.jdbc :as jdbc]))
(s/def ::name string?)
@ -40,7 +36,7 @@
(defn- impl-migrate-single
[pool modname {:keys [name] :as migration}]
(when-not (registered? pool modname (:name migration))
(log/info (str/format "applying migration %s/%s" modname name))
(l/info :action "apply migration" :module modname :name name)
(register! pool modname name)
((:fn migration) pool)))

5
backend/src/app/util/services.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 Andrey Antukh <niwi@niwi.nz>
;; Copyright (c) UXBOX Labs SL
(ns app.util.services
"A helpers and macros for define rpc like registry based services."

198
backend/src/app/util/sql.clj
View file

@ -1,198 +0,0 @@
;; Copyright (c) 2019 Andrey Antukh <niwi@niwi.nz>
;; All rights reserved.
;;
;; Redistribution and use in source and binary forms, with or without
;; modification, are permitted provided that the following conditions are met:
;;
;; * Redistributions of source code must retain the above copyright notice, this
;; list of conditions and the following disclaimer.
;;
;; * Redistributions in binary form must reproduce the above copyright notice,
;; this list of conditions and the following disclaimer in the documentation
;; and/or other materials provided with the distribution.
;;
;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
(ns app.util.sql
"A composable sql helpers."
(:refer-clojure :exclude [test update set format])
(:require [clojure.core :as c]
[cuerdas.core :as str]))
;; --- Low Level Helpers
(defn raw-expr
[m]
(cond
(string? m)
{::type :raw-expr
:sql m
:params []}
(vector? m)
{::type :raw-expr
:sql (first m)
:params (vec (rest m))}
(and (map? m)
(= :raw-expr (::type m)))
m
:else
(throw (ex-info "unexpected input" {:m m}))))
(defn alias-expr
[m]
(cond
(string? m)
{::type :alias-expr
:sql m
:alias nil
:params []}
(vector? m)
{::type :alias-expr
:sql (first m)
:alias (second m)
:params (vec (drop 2 m))}
:else
(throw (ex-info "unexpected input" {:m m}))))
;; --- SQL API (Select only)
(defn from
[name]
{::type :query
::from [(alias-expr name)]
::order []
::select []
::join []
::where []})
(defn select
[m & fields]
(c/update m ::select into (map alias-expr fields)))
(defn limit
[m n]
(assoc m ::limit [(raw-expr ["LIMIT ?" n])]))
(defn offset
[m n]
(assoc m ::offset [(raw-expr ["OFFSET ?" n])]))
(defn order
[m e]
(c/update m ::order conj (raw-expr e)))
(defn- join*
[m type table condition]
(c/update m ::join conj
{::type :join-expr
:type type
:table (alias-expr table)
:condition (raw-expr condition)}))
(defn join
[m table condition]
(join* m :inner table condition))
(defn ljoin
[m table condition]
(join* m :left table condition))
(defn rjoin
[m table condition]
(join* m :right table condition))
(defn where
[m & conditions]
(->> (filter identity conditions)
(reduce #(c/update %1 ::where conj (raw-expr %2)) m)))
;; --- Formating
(defmulti format-expr ::type)
(defmethod format-expr :raw-expr
[{:keys [sql params]}]
[sql params])
(defmethod format-expr :alias-expr
[{:keys [sql alias params]}]
(if alias
[(str sql " AS " alias) params]
[sql params]))
(defmethod format-expr :join-expr
[{:keys [table type condition]}]
(let [[csql cparams] (format-expr condition)
[tsql tparams] (format-expr table)
prefix (str/upper (name type))]
[(str prefix " JOIN " tsql " ON (" csql ")") (into cparams tparams)]))
(defn- format-exprs
([items] (format-exprs items {}))
([items {:keys [prefix suffix join-with]
:or {prefix ""
suffix ""
join-with ","}}]
(loop [rs []
rp []
v (first items)
n (rest items)]
(if v
(let [[s p] (format-expr v)]
(recur (conj rs s)
(into rp p)
(first n)
(rest n)))
(if (empty? rs)
["" []]
[(str prefix (str/join join-with rs) suffix) rp])))))
(defn- process-param-tokens
[sql]
(let [cnt (java.util.concurrent.atomic.AtomicInteger. 1)]
(str/replace sql #"\?" (fn [& _args]
(str "$" (.getAndIncrement cnt))))))
(def ^:private select-formatters
[#(format-exprs (::select %) {:prefix "SELECT "})
#(format-exprs (::from %) {:prefix "FROM "})
#(format-exprs (::join %) {:join-with " "})
#(format-exprs (::where %) {:prefix "WHERE ("
:join-with ") AND ("
:suffix ")"})
#(format-exprs (::order %) {:prefix "ORDER BY "} )
#(format-exprs (::limit %))
#(format-exprs (::offset %))])
(defn- collect
[formatters qdata]
(loop [sqls []
params []
f (first formatters)
r (rest formatters)]
(if (fn? f)
(let [[s p] (f qdata)]
(recur (conj sqls s)
(into params p)
(first r)
(rest r)))
[(str/join " " sqls) params])))
(defn fmt
[qdata]
(let [[sql params] (collect select-formatters qdata)]
(into [(process-param-tokens sql)] params)))

5
backend/src/app/util/template.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.util.template
(:require

6
backend/src/app/util/time.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020-2021 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.util.time
(:require
@ -60,7 +57,6 @@
[t1 t2]
(Duration/between t1 t2))
(letfn [(conformer [v]
(cond
(duration? v) v

5
backend/src/app/util/transit.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.util.transit
(:require

237
backend/src/app/worker.clj
View file

@ -2,24 +2,22 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 Andrey Antukh <niwi@niwi.nz>
;; Copyright (c) UXBOX Labs SL
(ns app.worker
"Async tasks abstraction (impl)."
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.db :as db]
[app.metrics :as mtx]
[app.util.async :as aa]
[app.util.log4j :refer [update-thread-context!]]
[app.util.logging :as l]
[app.util.time :as dt]
[clojure.core.async :as a]
[clojure.spec.alpha :as s]
[clojure.tools.logging :as log]
[cuerdas.core :as str]
[integrant.core :as ig]
[promesa.exec :as px])
@ -35,21 +33,13 @@
;; Executor
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(s/def ::name ::us/string)
(s/def ::name keyword?)
(s/def ::min-threads ::us/integer)
(s/def ::max-threads ::us/integer)
(s/def ::idle-timeout ::us/integer)
(defmethod ig/pre-init-spec ::executor [_]
(s/keys :opt-un [::min-threads ::max-threads ::idle-timeout ::name]))
(defmethod ig/prep-key ::executor
[_ cfg]
(merge {:min-threads 0
:max-threads 256
:idle-timeout 60000
:name "worker"}
cfg))
(s/keys :req-un [::min-threads ::max-threads ::idle-timeout ::name]))
(defmethod ig/init-key ::executor
[_ {:keys [min-threads max-threads idle-timeout name]}]
@ -57,28 +47,29 @@
(int min-threads)
(int idle-timeout))
(.setStopTimeout 500)
(.setName name)
(.setName (d/name name))
(.start)))
(defmethod ig/halt-key! ::executor
[_ instance]
(.stop ^QueuedThreadPool instance))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Worker
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(declare event-loop-fn)
(declare instrument-tasks)
(s/def ::queue ::us/string)
(s/def ::queue keyword?)
(s/def ::parallelism ::us/integer)
(s/def ::batch-size ::us/integer)
(s/def ::tasks (s/map-of string? fn?))
(s/def ::tasks (s/map-of keyword? fn?))
(s/def ::poll-interval ::dt/duration)
(defmethod ig/pre-init-spec ::worker [_]
(s/keys :req-un [::executor
::mtx/metrics
::db/pool
::batch-size
::name
@ -88,47 +79,50 @@
(defmethod ig/prep-key ::worker
[_ cfg]
(merge {:batch-size 2
:name "worker"
:poll-interval (dt/duration {:seconds 5})
:queue "default"}
cfg))
(d/merge {:batch-size 2
:name :worker
:poll-interval (dt/duration {:seconds 5})
:queue :default}
(d/without-nils cfg)))
(defmethod ig/init-key ::worker
[_ {:keys [pool poll-interval name queue] :as cfg}]
(log/infof "starting worker '%s' on queue '%s'" name queue)
(let [cch (a/chan 1)
poll-ms (inst-ms poll-interval)]
(l/info :action "start worker"
:name (d/name name)
:queue (d/name queue))
(let [close-ch (a/chan 1)
poll-ms (inst-ms poll-interval)]
(a/go-loop []
(let [[val port] (a/alts! [cch (event-loop-fn cfg)] :priority true)]
(let [[val port] (a/alts! [close-ch (event-loop-fn cfg)] :priority true)]
(cond
;; Terminate the loop if close channel is closed or
;; event-loop-fn returns nil.
(or (= port cch) (nil? val))
(log/infof "stop condition found; shutdown worker: '%s'" name)
(or (= port close-ch) (nil? val))
(l/debug :msg "stop condition found")
(db/pool-closed? pool)
(do
(log/info "worker eventloop is aborted because pool is closed")
(a/close! cch))
(l/debug :msg "eventloop aborted because pool is closed")
(a/close! close-ch))
(and (instance? java.sql.SQLException val)
(contains? #{"08003" "08006" "08001" "08004"} (.getSQLState ^java.sql.SQLException val)))
(do
(log/error "connection error, trying resume in some instants")
(l/error :hint "connection error, trying resume in some instants")
(a/<! (a/timeout poll-interval))
(recur))
(and (instance? java.sql.SQLException val)
(= "40001" (.getSQLState ^java.sql.SQLException val)))
(do
(log/debug "serialization failure (retrying in some instants)")
(l/debug :msg "serialization failure (retrying in some instants)")
(a/<! (a/timeout poll-ms))
(recur))
(instance? Exception val)
(do
(log/errorf val "unexpected error ocurried on polling the database (will resume in some instants)")
(l/error :cause val
:hint "unexpected error ocurried on polling the database (will resume in some instants)")
(a/<! (a/timeout poll-ms))
(recur))
@ -143,13 +137,57 @@
(reify
java.lang.AutoCloseable
(close [_]
(a/close! cch)))))
(a/close! close-ch)))))
(defmethod ig/halt-key! ::worker
[_ instance]
(.close ^java.lang.AutoCloseable instance))
;; --- SUBMIT
(s/def ::task keyword?)
(s/def ::delay (s/or :int ::us/integer :duration dt/duration?))
(s/def ::conn some?)
(s/def ::priority ::us/integer)
(s/def ::max-retries ::us/integer)
(s/def ::submit-options
(s/keys :req [::task ::conn]
:opt [::delay ::queue ::priority ::max-retries]))
(def ^:private sql:insert-new-task
"insert into task (id, name, props, queue, priority, max_retries, scheduled_at)
values (?, ?, ?, ?, ?, ?, clock_timestamp() + ?)
returning id")
(defn- extract-props
[options]
(persistent!
(reduce-kv (fn [res k v]
(cond-> res
(not (qualified-keyword? k))
(assoc! k v)))
(transient {})
options)))
(defn submit!
[{:keys [::task ::delay ::queue ::priority ::max-retries ::conn]
:or {delay 0 queue :default priority 100 max-retries 3}
:as options}]
(us/verify ::submit-options options)
(let [duration (dt/duration delay)
interval (db/interval duration)
props (-> options extract-props db/tjson)
id (uuid/next)]
(l/debug :action "submit task"
:name (d/name task)
:in duration)
(db/exec-one! conn [sql:insert-new-task id (d/name task) props (d/name queue) priority max-retries interval])
id))
;; --- RUNNER
(def ^:private
sql:mark-as-retry
@ -194,17 +232,19 @@
nil))
(defn- decode-task-row
[{:keys [props] :as row}]
[{:keys [props name] :as row}]
(when row
(cond-> row
(db/pgobject? props) (assoc :props (db/decode-transit-pgobject props)))))
(db/pgobject? props) (assoc :props (db/decode-transit-pgobject props))
(string? name) (assoc :name (keyword name)))))
(defn- handle-task
[tasks {:keys [name] :as item}]
(let [task-fn (get tasks name)]
(if task-fn
(task-fn item)
(log/warnf "no task handler found for '%s'" (pr-str name)))
(l/warn :msg "no task handler found"
:name (d/name name)))
{:status :completed :task item}))
(defn get-error-context
@ -228,21 +268,32 @@
(assoc :inc-by 0))
(let [cdata (get-error-context error item)]
(update-thread-context! cdata)
(log/errorf error "unhandled exception on task (id: '%s')" (:id cdata))
(l/update-thread-context! cdata)
(l/error :cause error
:hint "unhandled exception on task"
:id (:id cdata))
(if (>= (:retry-num item) (:max-retries item))
{:status :failed :task item :error error}
{:status :retry :task item :error error})))))
(defn- run-task
[{:keys [tasks]} item]
(try
(log/debugf "started task '%s/%s/%s'" (:name item) (:id item) (:retry-num item))
(handle-task tasks item)
(catch Exception e
(handle-exception e item))
(finally
(log/debugf "finished task '%s/%s/%s'" (:name item) (:id item) (:retry-num item)))))
(let [name (d/name (:name item))]
(try
(l/debug :action "start task"
:name name
:id (:id item)
:retry (:retry-num item))
(handle-task tasks item)
(catch Exception e
(handle-exception e item))
(finally
(l/debug :action "end task"
:name name
:id (:id item)
:retry (:retry-num item))))))
(def sql:select-next-tasks
"select * from task as t
@ -256,7 +307,7 @@
(defn- event-loop-fn*
[{:keys [pool executor batch-size] :as cfg}]
(db/with-atomic [conn pool]
(let [queue (:queue cfg)
(let [queue (name (:queue cfg))
items (->> (db/exec! conn [sql:select-next-tasks queue batch-size])
(map decode-task-row)
(seq))
@ -288,16 +339,16 @@
(declare synchronize-schedule)
(s/def ::fn (s/or :var var? :fn fn?))
(s/def ::id ::us/string)
(s/def ::id keyword?)
(s/def ::cron dt/cron?)
(s/def ::props (s/nilable map?))
(s/def ::task keyword?)
(s/def ::scheduled-task-spec
(s/keys :req-un [::id ::cron ::task]
:opt-un [::props]))
(s/def ::scheduled-task
(s/keys :req-un [::cron ::task]
:opt-un [::props ::id]))
(s/def ::schedule (s/coll-of (s/nilable ::scheduled-task-spec)))
(s/def ::schedule (s/coll-of (s/nilable ::scheduled-task)))
(defmethod ig/pre-init-spec ::scheduler [_]
(s/keys :req-un [::executor ::db/pool ::schedule ::tasks]))
@ -307,8 +358,13 @@
(let [scheduler (Executors/newScheduledThreadPool (int 1))
schedule (->> schedule
(filter some?)
;; If id is not defined, use the task as id.
(map (fn [{:keys [id task] :as item}]
(if (some? id)
(assoc item :id (d/name id))
(assoc item :id (d/name task)))))
(map (fn [{:keys [task] :as item}]
(let [f (get tasks (name task))]
(let [f (get tasks task)]
(when-not f
(ex/raise :type :internal
:code :task-not-found
@ -342,7 +398,7 @@
(defn- synchronize-schedule-item
[conn {:keys [id cron]}]
(let [cron (str cron)]
(log/infof "initialize scheduled task '%s' (cron: '%s')" id cron)
(l/debug :action "initialize scheduled task" :id id :cron cron)
(db/exec-one! conn [sql:upsert-scheduled-task id cron cron])))
(defn- synchronize-schedule
@ -362,8 +418,8 @@
[{:keys [executor pool] :as cfg} {:keys [id] :as task}]
(letfn [(run-task [conn]
(try
(when (db/exec-one! conn [sql:lock-scheduled-task id])
(log/debugf "executing scheduled task '%s'" id)
(when (db/exec-one! conn [sql:lock-scheduled-task (d/name id)])
(l/debug :action "execute scheduled task" :id id)
((:fn task) task))
(catch Throwable e
e)))
@ -372,7 +428,9 @@
(db/with-atomic [conn pool]
(let [result (run-task conn)]
(when (ex/exception? result)
(log/errorf result "unhandled exception on scheduled task '%s'" id)))))]
(l/error :cause result
:hint "unhandled exception on scheduled task"
:id id)))))]
(try
(px/run! executor handle-task)
@ -390,3 +448,62 @@
[{:keys [scheduler] :as cfg} {:keys [cron] :as task}]
(let [ms (ms-until-valid cron)]
(px/schedule! scheduler ms (partial execute-scheduled-task cfg task))))
;; --- INSTRUMENTATION
(defn instrument!
[registry]
(mtx/instrument-vars!
[#'submit!]
{:registry registry
:type :counter
:labels ["name"]
:name "tasks_submit_total"
:help "A counter of task submissions."
:wrap (fn [rootf mobj]
(let [mdata (meta rootf)
origf (::original mdata rootf)]
(with-meta
(fn [conn params]
(let [tname (:name params)]
(mobj :inc [tname])
(origf conn params)))
{::original origf})))})
(mtx/instrument-vars!
[#'app.worker/run-task]
{:registry registry
:type :summary
:quantiles []
:name "tasks_checkout_timing"
:help "Latency measured between scheduld_at and execution time."
:wrap (fn [rootf mobj]
(let [mdata (meta rootf)
origf (::original mdata rootf)]
(with-meta
(fn [tasks item]
(let [now (inst-ms (dt/now))
sat (inst-ms (:scheduled-at item))]
(mobj :observe (- now sat))
(origf tasks item)))
{::original origf})))}))
(defmethod ig/pre-init-spec ::registry [_]
(s/keys :req-un [::mtx/metrics ::tasks]))
(defmethod ig/init-key ::registry
[_ {:keys [metrics tasks]}]
(let [mobj (mtx/create
{:registry (:registry metrics)
:type :summary
:labels ["name"]
:quantiles []
:name "tasks_timing"
:help "Background task execution timing."})]
(reduce-kv (fn [res k v]
(let [tname (name k)]
(l/debug :action "register task" :name tname)
(assoc res k (mtx/wrap-summary v mobj [tname]))))
{}
tasks)))

32
backend/tests/app/tests/helpers.clj
View file

@ -2,10 +2,7 @@
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) 2020 UXBOX Labs SL
;; Copyright (c) UXBOX Labs SL
(ns app.tests.helpers
(:require
@ -13,7 +10,7 @@
[app.common.pages :as cp]
[app.common.spec :as us]
[app.common.uuid :as uuid]
[app.config :as cfg]
[app.config :as cf]
[app.db :as db]
[app.main :as main]
[app.media]
@ -38,16 +35,23 @@
(def ^:dynamic *system* nil)
(def ^:dynamic *pool* nil)
(def defaults
{:database-uri "postgresql://postgres/penpot_test"
:redis-uri "redis://redis/1"})
(def config
(merge {:redis-uri "redis://redis/1"
:database-uri "postgresql://postgres/penpot_test"
:storage-fs-directory "/tmp/app/storage"
:migrations-verbose false}
cfg/config))
(->> (cf/read-env "penpot-test")
(merge cf/defaults defaults)
(us/conform ::cf/config)))
(defn state-init
[next]
(let [config (-> (main/build-system-config config)
(let [config (-> main/system-config
(assoc-in [:app.msgbus/msgbus :redis-uri] (:redis-uri config))
(assoc-in [:app.db/pool :uri] (:database-uri config))
(assoc-in [:app.db/pool :username] (:database-username config))
(assoc-in [:app.db/pool :password] (:database-password config))
(assoc-in [[:app.main/main :app.storage.fs/backend] :directory] "/tmp/app/storage")
(dissoc :app.srepl/server
:app.http/server
:app.http/router
@ -328,8 +332,10 @@
"Helper for mock app.config/get"
[data]
(fn
([key] (get (merge config data) key))
([key default] (get (merge config data) key default))))
([key]
(get data key (get @cf/config key)))
([key default]
(get data key (get @cf/config key default)))))
(defn reset-mock!
[m]

Some files were not shown because too many files have changed in this diff Show more