Merge pull request #2635 from penpot/niwinz-backend-webhooks-3

🎉 Add webhook RPC API

backend/scripts/kill-repl.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -x
+
+jcmd |grep "rebel" |sed -nE 's/^([0-9]+).*$/\1/p' | xargs kill -9

backend/scripts/run.template.sh

@@ -1,19 +1,21 @@
 #!/usr/bin/env bash
-set +e
-JAVA_CMD=$(type -p java)
 
-set -e
 if [[ ! -n "$JAVA_CMD" ]]; then
-  if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then
-    JAVA_CMD="$JAVA_HOME/bin/java"
-  else
-    >&2 echo "Couldn't find 'java'. Please set JAVA_HOME."
-    exit 1
-  fi
+    if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then
+        JAVA_CMD="$JAVA_HOME/bin/java"
+    else
+        set +e
+        JAVA_CMD=$(type -p java)
+        set -e
+        if [[ ! -n "$JAVA_CMD" ]]; then
+            >&2 echo "Couldn't find 'java'. Please set JAVA_HOME."
+            exit 1
+        fi
+    fi
 fi
 
 if [ -f ./environ ]; then
-   source ./environ
+    source ./environ
 fi
 
 set -x

backend/src/app/db.clj

@@ -296,6 +296,7 @@
    (let [row (get* ds table params opts)]
      (when (and (not row) check-deleted?)
        (ex/raise :type :not-found
+                 :code :object-not-found
                  :table table
                  :hint "database object not found"))
      row)))
@@ -308,6 +309,7 @@
    (let [row (get* ds table params (assoc opts :check-deleted? check-not-found))]
      (when (and (not row) check-not-found)
        (ex/raise :type :not-found
+                 :code :object-not-found
                  :table table
                  :hint "database object not found"))
      row)))

backend/src/app/main.clj

@@ -320,20 +320,23 @@
    :app.rpc/methods
    {::audit/collector    (ig/ref ::audit/collector)
     ::http.client/client (ig/ref ::http.client/client)
-    :pool             (ig/ref ::db/pool)
-    :session          (ig/ref :app.http.session/manager)
-    :sprops           (ig/ref :app.setup/props)
-    :metrics          (ig/ref ::mtx/metrics)
-    :storage          (ig/ref ::sto/storage)
-    :msgbus           (ig/ref :app.msgbus/msgbus)
-    :public-uri       (cf/get :public-uri)
-    :redis            (ig/ref ::rds/redis)
-    :ldap             (ig/ref :app.auth.ldap/provider)
-    :http-client      (ig/ref ::http.client/client)
-    :climit           (ig/ref :app.rpc/climit)
-    :rlimit           (ig/ref :app.rpc/rlimit)
-    :executor         (ig/ref ::wrk/executor)
-    :templates        (ig/ref :app.setup/builtin-templates)
+    ::db/pool            (ig/ref ::db/pool)
+    ::wrk/executor       (ig/ref ::wrk/executor)
+
+    :pool                (ig/ref ::db/pool)
+    :session             (ig/ref :app.http.session/manager)
+    :sprops              (ig/ref :app.setup/props)
+    :metrics             (ig/ref ::mtx/metrics)
+    :storage             (ig/ref ::sto/storage)
+    :msgbus              (ig/ref :app.msgbus/msgbus)
+    :public-uri          (cf/get :public-uri)
+    :redis               (ig/ref ::rds/redis)
+    :ldap                (ig/ref :app.auth.ldap/provider)
+    :http-client         (ig/ref ::http.client/client)
+    :climit              (ig/ref :app.rpc/climit)
+    :rlimit              (ig/ref :app.rpc/rlimit)
+    :executor            (ig/ref ::wrk/executor)
+    :templates           (ig/ref :app.setup/builtin-templates)
     }
 
    :app.rpc.doc/routes

backend/src/app/migrations.clj

@@ -262,6 +262,9 @@
 
    {:name "0084-add-features-column-to-file-change-table"
     :fn (mg/resource "app/migrations/sql/0084-add-features-column-to-file-change-table.sql")}
+
+   {:name "0085-add-webhook-table"
+    :fn (mg/resource "app/migrations/sql/0085-add-webhook-table.sql")}
    ])
 
 

backend/src/app/migrations/sql/0085-add-webhook-table.sql

@@ -0,0 +1,25 @@
+CREATE TABLE webhook (
+  id uuid PRIMARY KEY,
+  team_id uuid NOT NULL REFERENCES team(id) ON DELETE CASCADE DEFERRABLE,
+
+  created_at timestamptz NOT NULL DEFAULT now(),
+  updated_at timestamptz NOT NULL DEFAULT now(),
+
+  uri text NOT NULL,
+  mtype text NOT NULL,
+
+  error_code text NULL,
+  error_count smallint DEFAULT 0,
+
+  is_active boolean DEFAULT true,
+  secret_key text NULL
+);
+
+ALTER TABLE webhook
+  ALTER COLUMN uri SET STORAGE external,
+  ALTER COLUMN mtype SET STORAGE external,
+  ALTER COLUMN error_code SET STORAGE external,
+  ALTER COLUMN secret_key SET STORAGE external;
+
+
+CREATE INDEX webhook__team_id__idx ON webhook (team_id);

backend/src/app/rpc.clj

@@ -253,6 +253,7 @@
                      'app.rpc.commands.auth
                      'app.rpc.commands.ldap
                      'app.rpc.commands.demo
+                     'app.rpc.commands.webhooks
                      'app.rpc.commands.files
                      'app.rpc.commands.files.update
                      'app.rpc.commands.files.create
@@ -270,7 +271,9 @@
 
 (defmethod ig/pre-init-spec ::methods [_]
   (s/keys :req [::audit/collector
-                ::http.client/client]
+                ::http.client/client
+                ::db/pool
+                ::wrk/executor]
           :req-un [::sto/storage
                    ::http.session/session
                    ::sprops

backend/src/app/rpc/commands/webhooks.clj

@@ -0,0 +1,120 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns app.rpc.commands.webhooks
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.spec :as us]
+   [app.common.uuid :as uuid]
+   [app.db :as db]
+   [app.http.client :as http]
+   [app.rpc.doc :as-alias doc]
+   [app.rpc.queries.teams :refer [check-edition-permissions!]]
+   [app.util.services :as sv]
+   [app.util.time :as dt]
+   [app.worker :as-alias wrk]
+   [clojure.spec.alpha :as s]
+   [cuerdas.core :as str]
+   [promesa.core :as p]))
+
+;; --- Mutation: Create Webhook
+
+(s/def ::profile-id ::us/uuid)
+(s/def ::team-id ::us/uuid)
+(s/def ::uri ::us/not-empty-string)
+(s/def ::is-active ::us/boolean)
+(s/def ::mtype
+  #{"application/json"
+    "application/x-www-form-urlencoded"
+    "application/transit+json"})
+
+(s/def ::create-webhook
+  (s/keys :req-un [::profile-id ::team-id ::uri ::mtype]
+          :opt-un [::is-active]))
+
+;; FIXME: validate
+;; FIXME: default ratelimit
+;; FIXME: quotes
+
+(defn- validate-webhook!
+  [cfg whook params]
+  (letfn [(handle-exception [exception]
+            (cond
+              (instance? java.util.concurrent.CompletionException exception)
+              (handle-exception (ex/cause exception))
+
+              (instance? javax.net.ssl.SSLHandshakeException exception)
+              (ex/raise :type :validation
+                        :code :webhook-validation
+                        :hint "ssl-validaton")
+
+              :else
+              (ex/raise :type :validation
+                        :code :webhook-validation
+                        :hint "unknown"
+                        :cause exception)))
+
+          (handle-response [{:keys [status] :as response}]
+            (when (not= status 200)
+              (ex/raise :type :validation
+                        :code :webhook-validation
+                        :hint (str/ffmt "unexpected-status-%" (:status response)))))]
+
+    (if (not= (:uri whook) (:uri params))
+      (->> (http/req! cfg {:method :head
+                           :uri (:uri params)
+                           :timeout (dt/duration "2s")})
+           (p/hmap (fn [response exception]
+                     (if exception
+                       (handle-exception exception)
+                       (handle-response response)))))
+      (p/resolved nil))))
+
+(sv/defmethod ::create-webhook
+  {::doc/added "1.17"}
+  [{:keys [::db/pool ::wrk/executor] :as cfg} {:keys [profile-id team-id uri mtype is-active] :as params}]
+  (check-edition-permissions! pool profile-id team-id)
+  (letfn [(insert-webhook [_]
+            (db/insert! pool :webhook
+                        {:id (uuid/next)
+                         :team-id team-id
+                         :uri uri
+                         :is-active is-active
+                         :mtype mtype}))]
+    (->> (validate-webhook! cfg nil params)
+         (p/fmap executor insert-webhook))))
+
+(s/def ::update-webhook
+  (s/keys :req-un [::id ::uri ::mtype ::is-active]))
+
+(sv/defmethod ::update-webhook
+  {::doc/added "1.17"}
+  [{:keys [::db/pool ::wrk/executor] :as cfg} {:keys [profile-id id uri mtype is-active] :as params}]
+  (let [whook     (db/get pool :webhook {:id id})
+        update-fn (fn [_]
+                    (db/update! pool :webhook
+                                {:uri uri
+                                 :is-active is-active
+                                 :mtype mtype
+                                 :error-code nil
+                                 :error-count 0}
+                                {:id id}))]
+    (check-edition-permissions! pool profile-id (:team-id whook))
+
+    (->> (validate-webhook! cfg whook params)
+         (p/fmap executor update-fn))))
+
+(s/def ::delete-webhook
+  (s/keys :req-un [::profile-id ::id]))
+
+(sv/defmethod ::delete-webhook
+  {::doc/added "1.17"}
+  [{:keys [::db/pool] :as cfg} {:keys [profile-id id]}]
+  (db/with-atomic [conn pool]
+    (let [whook (db/get conn :webhook {:id id})]
+      (check-edition-permissions! conn profile-id (:team-id whook))
+      (db/delete! conn :webhook {:id id})
+      nil)))

backend/test/backend_tests/helpers.clj

@@ -431,4 +431,10 @@
 
 (defn reset-mock!
   [m]
-  (reset! m @(mk/make-mock {})))
+  (swap! m (fn [m]
+             (-> m
+                 (assoc :called? false)
+                 (assoc :call-count 0)
+                 (assoc :return-list [])
+                 (assoc :call-args nil)
+                 (assoc :call-args-list [])))))

backend/test/backend_tests/rpc_webhooks_test.clj

@@ -0,0 +1,146 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns backend-tests.rpc-webhooks-test
+  (:require
+   [app.common.uuid :as uuid]
+   [app.db :as db]
+   [app.http :as http]
+   [app.storage :as sto]
+   [backend-tests.helpers :as th]
+   [clojure.test :as t]
+   [datoteka.fs :as fs]
+   [datoteka.io :as io]
+   [mockery.core :refer [with-mocks]]))
+
+(t/use-fixtures :once th/state-init)
+(t/use-fixtures :each th/database-reset)
+
+(t/deftest webhook-crud
+  (with-mocks [http-mock {:target 'app.http.client/req!
+                          :return {:status 200}}]
+
+    (let [prof    (th/create-profile* 1 {:is-active true})
+          team-id (:default-team-id prof)
+          proj-id (:default-project-id prof)
+          whook   (volatile! nil)]
+
+      (t/testing "create webhook"
+        (let [params {::th/type :create-webhook
+                      :profile-id (:id prof)
+                      :team-id team-id
+                      :uri "http://example.com"
+                      :mtype "application/json"}
+              out    (th/command! params)]
+
+          (t/is (nil? (:error out)))
+          (t/is (= 1 (:call-count @http-mock)))
+
+          (let [result (:result out)]
+            (t/is (contains? result :id))
+            (t/is (contains? result :team-id))
+            (t/is (contains? result :created-at))
+            (t/is (contains? result :updated-at))
+            (t/is (contains? result :uri))
+            (t/is (contains? result :mtype))
+
+            (t/is (= (:uri params) (:uri result)))
+            (t/is (= (:team-id params) (:team-id result)))
+            (t/is (= (:mtype params) (:mtype result)))
+            (vreset! whook result))))
+
+
+      (th/reset-mock! http-mock)
+
+      (t/testing "update webhook 1 (success)"
+        (let [params {::th/type :update-webhook
+                      :profile-id (:id prof)
+                      :id (:id @whook)
+                      :uri (:uri @whook)
+                      :mtype "application/transit+json"
+                      :is-active false}
+              out    (th/command! params)]
+
+          (t/is (nil? (:error out)))
+          (t/is (= 0 (:call-count @http-mock)))
+
+          (let [result (:result out)]
+            (t/is (contains? result :id))
+            (t/is (contains? result :team-id))
+            (t/is (contains? result :created-at))
+            (t/is (contains? result :updated-at))
+            (t/is (contains? result :uri))
+            (t/is (contains? result :mtype))
+
+            (t/is (= (:id params)  (:id result)))
+            (t/is (= (:id @whook)  (:id result)))
+            (t/is (= (:uri params) (:uri result)))
+            (t/is (= (:team-id @whook) (:team-id result)))
+            (t/is (= (:mtype params) (:mtype result))))))
+
+      (th/reset-mock! http-mock)
+
+      (t/testing "update webhook 2 (change uri)"
+        (let [params {::th/type :update-webhook
+                      :profile-id (:id prof)
+                      :id (:id @whook)
+                      :uri (str (:uri @whook) "/test")
+                      :mtype "application/transit+json"
+                      :is-active false}
+              out    (th/command! params)]
+
+          (t/is (nil? (:error out)))
+          (t/is (map? (:result out)))
+          (t/is (= 1 (:call-count @http-mock)))))
+
+      (th/reset-mock! http-mock)
+
+      (t/testing "update webhook 3 (not authorized)"
+        (let [params {::th/type :update-webhook
+                      :profile-id uuid/zero
+                      :id (:id @whook)
+                      :uri (str (:uri @whook) "/test")
+                      :mtype "application/transit+json"
+                      :is-active false}
+              out    (th/command! params)]
+
+          (t/is (= 0 (:call-count @http-mock)))
+          (let [error      (:error out)
+                error-data (ex-data error)]
+            (t/is (th/ex-info? error))
+            (t/is (= (:type error-data) :not-found))
+            (t/is (= (:code error-data) :object-not-found)))))
+
+      (th/reset-mock! http-mock)
+
+      (t/testing "delete webhook (success)"
+        (let [params {::th/type :delete-webhook
+                      :profile-id (:id prof)
+                      :id (:id @whook)}
+              out    (th/command! params)]
+
+          (t/is (= 0 (:call-count @http-mock)))
+          (t/is (nil? (:error out)))
+          (t/is (nil? (:result out)))
+
+          (let [rows (th/db-exec! ["select * from webhook"])]
+            (t/is (= 0 (count rows))))))
+
+      (t/testing "delete webhook (unauthorozed)"
+        (let [params {::th/type :delete-webhook
+                      :profile-id uuid/zero
+                      :id (:id @whook)}
+              out    (th/command! params)]
+
+          ;; (th/print-result! out)
+          (t/is (= 0 (:call-count @http-mock)))
+          (let [error      (:error out)
+                error-data (ex-data error)]
+            (t/is (th/ex-info? error))
+            (t/is (= (:type error-data) :not-found))
+            (t/is (= (:code error-data) :object-not-found)))))
+
+      )))