♻️ Refactor task worker.

backend/bin/repl

@@ -1,3 +1,4 @@
 #!/usr/bin/env bash
 set -ex
-clojure -Ojmx-remote -A:dev -e "(set! *warn-on-reflection* true)" -m rebel-readline.main
+#clojure -Ojmx-remote -A:dev -e "(set! *warn-on-reflection* true)" -m rebel-readline.main
+clojure -Ojmx-remote -A:dev -m rebel-readline.main

backend/resources/log4j2.xml

@@ -6,8 +6,10 @@
     </Console>
   </Appenders>
   <Loggers>
-    <Logger name="io.vertx.sqlclient.impl.SocketConnectionBase" level="ERROR"/>
+    <Logger name="com.zaxxer.hikari" level="info" additivity="false"></Logger>
-    <Root level="info">
+    <Logger name="io.lettuce.core" level="info" additivity="false"></Logger>
+    <Logger name="io.netty" level="info" additivity="false"></Logger>
+    <Root level="debug">
       <AppenderRef ref="console"/>
     </Root>
   </Loggers>

backend/resources/migrations/0015-improve-tasks-tables.sql

@@ -0,0 +1,29 @@
+DROP TABLE task;
+
+CREATE TABLE task (
+  id uuid DEFAULT uuid_generate_v4(),
+  created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
+  modified_at timestamptz NOT NULL DEFAULT clock_timestamp(),
+  completed_at timestamptz NULL DEFAULT NULL,
+  scheduled_at timestamptz NOT NULL,
+  priority smallint DEFAULT 100,
+
+  queue text NOT NULL,
+
+  name text NOT NULL,
+  props jsonb NOT NULL,
+
+  error text NULL DEFAULT NULL,
+  retry_num smallint NOT NULL DEFAULT 0,
+  max_retries smallint NOT NULL DEFAULT 3,
+  status text NOT NULL DEFAULT 'new',
+
+  PRIMARY KEY (id, status)
+) PARTITION BY list(status);
+
+CREATE TABLE task_completed partition OF task FOR VALUES IN ('completed', 'failed');
+CREATE TABLE task_default partition OF task default;
+
+CREATE INDEX task__scheduled_at__queue__idx
+    ON task (scheduled_at, queue)
+ WHERE status = 'new' or status = 'retry';

backend/resources/migrations/0016-truncate-and-alter-tokens-table.sql

@@ -0,0 +1,3 @@
+delete from generic_token;
+alter table generic_token drop column content;
+alter table generic_token add column content jsonb not null;

backend/src/uxbox/db.clj

@@ -6,6 +6,7 @@
 
 (ns uxbox.db
   (:require
+   [clojure.spec.alpha :as s]
    [clojure.data.json :as json]
    [clojure.string :as str]
    [clojure.tools.logging :as log]
@@ -17,16 +18,23 @@
    [next.jdbc.result-set :as jdbc-rs]
    [next.jdbc.sql :as jdbc-sql]
    [next.jdbc.sql.builder :as jdbc-bld]
-   [uxbox.metrics :as mtx]
    [uxbox.common.exceptions :as ex]
    [uxbox.config :as cfg]
+   [uxbox.metrics :as mtx]
+   [uxbox.util.time :as dt]
+   [uxbox.util.transit :as t]
    [uxbox.util.data :as data])
   (:import
    org.postgresql.util.PGobject
+   org.postgresql.util.PGInterval
    com.zaxxer.hikari.metrics.prometheus.PrometheusMetricsTrackerFactory
    com.zaxxer.hikari.HikariConfig
    com.zaxxer.hikari.HikariDataSource))
 
+(def initsql
+  (str "SET statement_timeout = 10000;\n"
+       "SET idle_in_transaction_session_timeout = 30000;"))
+
 (defn- create-datasource-config
   [cfg]
   (let [dburi    (:database-uri cfg)
@@ -39,17 +47,28 @@
       (.setPoolName "main")
       (.setAutoCommit true)
       (.setReadOnly false)
-      (.setConnectionTimeout 30000) ;; 30seg
+      (.setConnectionTimeout 8000)  ;; 8seg
-      (.setValidationTimeout 5000)  ;; 5seg
+      (.setValidationTimeout 4000)  ;; 4seg
-      (.setIdleTimeout 900000)      ;; 15min
+      (.setIdleTimeout 300000)      ;; 5min
       (.setMaxLifetime 900000)      ;; 15min
-      (.setMinimumIdle 5)
+      (.setMinimumIdle 0)
-      (.setMaximumPoolSize 10)
+      (.setMaximumPoolSize 15)
+      (.setConnectionInitSql initsql)
       (.setMetricsTrackerFactory mfactory))
     (when username (.setUsername config username))
     (when password (.setPassword config password))
     config))
 
+(defn pool?
+  [v]
+  (instance? javax.sql.DataSource v))
+
+(s/def ::pool pool?)
+
+(defn pool-closed?
+  [pool]
+  (.isClosed ^com.zaxxer.hikari.HikariDataSource pool))
+
 (defn- create-pool
   [cfg]
   (let [dsc (create-datasource-config cfg)]
@@ -135,6 +154,33 @@
   [v]
   (instance? PGobject v))
 
+(defn pginterval?
+  [v]
+  (instance? PGInterval v))
+
+(defn pginterval
+  [data]
+  (org.postgresql.util.PGInterval. ^String data))
+
+(defn interval
+  [data]
+  (cond
+    (integer? data)
+    (->> (/ data 1000.0)
+         (format "%s seconds")
+         (pginterval))
+
+    (string? data)
+    (pginterval data)
+
+    (dt/duration? data)
+    (->> (/ (.toMillis data) 1000.0)
+         (format "%s seconds")
+         (pginterval))
+
+    :else
+    (ex/raise :type :not-implemented)))
+
 (defn decode-pgobject
   [^PGobject obj]
   (let [typ (.getType obj)
@@ -144,6 +190,38 @@
       (json/read-str val)
       val)))
 
+(defn decode-json-pgobject
+  [^PGobject o]
+  (let [typ (.getType o)
+        val (.getValue o)]
+    (if (or (= typ "json")
+            (= typ "jsonb"))
+      (json/read-str val :key-fn keyword)
+      val)))
+
+(defn decode-transit-pgobject
+  [^PGobject o]
+  (let [typ (.getType o)
+        val (.getValue o)]
+    (if (or (= typ "json")
+            (= typ "jsonb"))
+      (t/decode-str val)
+      val)))
+
+(defn tjson
+  "Encode as transit json."
+  [data]
+  (doto (org.postgresql.util.PGobject.)
+    (.setType "jsonb")
+    (.setValue (t/encode-verbose-str data))))
+
+(defn json
+  "Encode as plain json."
+  [data]
+  (doto (org.postgresql.util.PGobject.)
+    (.setType "jsonb")
+    (.setValue (json/write-str data))))
+
 ;; Instrumentation
 
 (mtx/instrument-with-counter!

backend/src/uxbox/emails.clj

@@ -46,6 +46,7 @@
          email (email-factory data)]
      (tasks/submit! conn {:name "sendmail"
                           :delay 0
+                          :priority 200
                           :props email}))))
 
 ;; --- Emails

backend/src/uxbox/http/session.clj

@@ -23,7 +23,7 @@
 
 (defn create
   [profile-id user-agent]
-  (let [id (tokens/next)]
+  (let [id (tokens/next-token)]
     (db/insert! db/pool :http-session {:id id
                                        :profile-id profile-id
                                        :user-agent user-agent})

backend/src/uxbox/migrations.clj

@@ -71,7 +71,15 @@
 
     {:desc "Refactor media storage"
      :name "0014-refactor-media-storage.sql"
-     :fn (mg/resource "migrations/0014-refactor-media-storage.sql")}]})
+     :fn (mg/resource "migrations/0014-refactor-media-storage.sql")}
+
+    {:desc "Improve and partition task related tables"
+     :name "0015-improve-tasks-tables"
+     :fn (mg/resource "migrations/0015-improve-tasks-tables.sql")}
+
+    {:desc "Truncate & alter tokens tables"
+     :name "0016-truncate-and-alter-tokens-table"
+     :fn (mg/resource "migrations/0016-truncate-and-alter-tokens-table.sql")}]})
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Entry point

backend/src/uxbox/services/tokens.clj

@@ -8,7 +8,6 @@
 ;; Copyright (c) 2020 UXBOX Labs SL
 
 (ns uxbox.services.tokens
-  (:refer-clojure :exclude [next])
   (:require
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]
@@ -16,14 +15,11 @@
    [sodi.util]
    [uxbox.common.exceptions :as ex]
    [uxbox.common.spec :as us]
-   [uxbox.common.uuid :as uuid]
-   [uxbox.config :as cfg]
    [uxbox.util.time :as dt]
-   [uxbox.util.blob :as blob]
    [uxbox.db :as db]))
 
-(defn next
+(defn next-token
-  ([] (next 64))
+  ([] (next-token 64))
   ([n]
    (-> (sodi.prng/random-bytes n)
        (sodi.util/bytes->b64s))))
@@ -35,15 +31,16 @@
   [{:keys [content] :as row}]
   (when row
     (cond-> row
-      content (assoc :content (blob/decode content)))))
+      (db/pgobject? content)
+      (assoc :content (db/decode-transit-pgobject content)))))
 
 (defn create!
   ([conn payload] (create! conn payload {}))
   ([conn payload {:keys [valid] :or {valid default-duration}}]
-   (let [token (next)
+   (let [token (next-token)
          until (dt/plus (dt/now) (dt/duration valid))]
      (db/insert! conn :generic-token
-                 {:content (blob/encode payload)
+                 {:content (db/tjson payload)
                   :token token
                   :valid-until until})
      token)))

backend/src/uxbox/tasks.clj

@@ -23,24 +23,12 @@
    [uxbox.tasks.delete-profile]
    [uxbox.tasks.delete-object]
    [uxbox.tasks.impl :as impl]
-   [uxbox.util.time :as dt])
+   [uxbox.util.time :as dt]))
-  (:import
-   java.util.concurrent.ScheduledExecutorService
-   java.util.concurrent.Executors))
 
 ;; --- Scheduler Executor Initialization
 
-(defstate scheduler
-  :start (Executors/newScheduledThreadPool (int 1))
-  :stop (.shutdownNow ^ScheduledExecutorService scheduler))
-
 ;; --- State initialization
 
-;; TODO: missing self maintanance task; when the queue table is full
-;; of completed/failed task, the performance starts degrading
-;; linearly, so after some arbitrary number of tasks is processed, we
-;; need to perform a maintenance and delete some old tasks.
-
 (def ^:private tasks
   {"delete-profile" #'uxbox.tasks.delete-profile/handler
    "delete-object" #'uxbox.tasks.delete-object/handler
@@ -52,14 +40,12 @@
     :cron (dt/cron "1 1 */1 * * ? *")
     :fn #'uxbox.tasks.gc/remove-media}])
 
-(defstate tasks-worker
+(defstate worker
-  :start (impl/start-worker! {:tasks tasks
+  :start (impl/start-worker! {:tasks tasks :name "worker1"})
-                              :xtor scheduler})
+  :stop (impl/stop! worker))
-  :stop (impl/stop! tasks-worker))
 
 (defstate scheduler-worker
-  :start (impl/start-scheduler-worker! {:schedule schedule
+  :start (impl/start-scheduler-worker! {:schedule schedule})
-                                        :xtor scheduler})
   :stop (impl/stop! scheduler-worker))
 
 ;; --- Public API

backend/src/uxbox/tasks/impl.clj

@@ -10,6 +10,7 @@
 (ns uxbox.tasks.impl
   "Async tasks implementation."
   (:require
+   [cuerdas.core :as str]
    [clojure.core.async :as a]
    [clojure.spec.alpha :as s]
    [clojure.tools.logging :as log]
@@ -18,6 +19,7 @@
    [uxbox.common.uuid :as uuid]
    [uxbox.config :as cfg]
    [uxbox.db :as db]
+   [uxbox.util.async :as aa]
    [uxbox.util.blob :as blob]
    [uxbox.util.time :as dt])
   (:import
@@ -40,6 +42,7 @@
   sql:mark-as-retry
   "update task
       set scheduled_at = clock_timestamp() + '5 seconds'::interval,
+          modified_at = clock_timestamp(),
           error = ?,
           status = 'retry',
           retry_num = retry_num + 1
@@ -57,25 +60,29 @@
   (let [explain (ex-message error)]
     (db/update! conn :task
                 {:error explain
+                 :modified-at (dt/now)
                  :status "failed"}
                 {:id (:id task)})
     nil))
 
 (defn- mark-as-completed
   [conn task]
-  (db/update! conn :task
+  (let [now (dt/now)]
-              {:completed-at (dt/now)
+    (db/update! conn :task
-               :status "completed"}
+                {:completed-at now
-              {:id (:id task)})
+                 :modified-at now
-  nil)
+                 :status "completed"}
+                {:id (:id task)})
+    nil))
+
 
 (def ^:private
   sql:select-next-task
   "select * from task as t
     where t.scheduled_at <= now()
       and t.queue = ?
-      and (t.status = 'new' or (t.status = 'retry' and t.retry_num <= ?))
+      and (t.status = 'new' or t.status = 'retry')
-    order by t.scheduled_at
+    order by t.priority desc, t.scheduled_at
     limit 1
       for update skip locked")
 
@@ -83,12 +90,13 @@
   [{:keys [props] :as row}]
   (when row
     (cond-> row
-      props (assoc :props (blob/decode props)))))
+      (db/pgobject? props) (assoc :props (db/decode-transit-pgobject props)))))
+
 
 (defn- log-task-error
   [item err]
-  (log/error "Unhandled exception on task '" (:name item)
+  (log/error (str/format "Unhandled exception on task '%s' (retry: %s)\n" (:name item) (:retry-num item))
-             "' (retry:" (:retry-num item) ") \n"
+             (str/format "Props: %s\n" (pr-str (:props item)))
              (with-out-str
                (.printStackTrace ^Throwable err (java.io.PrintWriter. *out*)))))
 
@@ -101,38 +109,107 @@
         (log/warn "no task handler found for" (pr-str name))
         nil))))
 
-(defn- event-loop-fn
+(defn- run-task
-  [{:keys [tasks] :as options}]
+  [{:keys [tasks conn]} item]
-  (let [queue (:queue options "default")
+  (try
-        max-retries (:max-retries options 3)]
+    (log/debug (str/format "Started task '%s/%s'." (:name item) (:id item)))
-    (db/with-atomic [conn db/pool]
+    (handle-task tasks item)
-      (let [item (-> (db/exec-one! conn [sql:select-next-task queue max-retries])
+    (log/debug (str/format "Finished task '%s/%s'." (:name item) (:id item)))
-                     (decode-task-row))]
+    (mark-as-completed conn item)
-        (when item
+    (catch Exception e
-          (log/info "Execute task" (:name item))
+      (log-task-error item e)
-          (try
+      (if (>= (:retry-num item) (:max-retries item))
-            (handle-task tasks item)
+        (mark-as-failed conn item e)
-            (mark-as-completed conn item)
+        (mark-as-retry conn item e)))))
-            ::handled
-            (catch Throwable e
-              (log-task-error item e)
-              (if (>= (:retry-num item) max-retries)
-                (mark-as-failed conn item e)
-                (mark-as-retry conn item e)))))))))
 
-(defn- execute-worker-task
+(defn- event-loop-fn
-  [{:keys [::stop ::xtor poll-interval]
+  [{:keys [tasks] :as opts}]
+  (aa/thread-try
+   (db/with-atomic [conn db/pool]
+     (let [queue (:queue opts "default")
+           item  (-> (db/exec-one! conn [sql:select-next-task queue])
+                     (decode-task-row))
+           opts  (assoc opts :conn conn)]
+
+       (cond
+         (nil? item)
+         ::empty
+
+         (or (= "new" (:status item))
+             (= "retry" (:status item)))
+         (do
+           (run-task opts item)
+           ::handled)
+
+         :else
+         (do
+           (log/warn "Unexpected condition on worker event loop:" (pr-str item))
+           ::handled))))))
+
+(s/def ::poll-interval ::us/integer)
+(s/def ::fn (s/or :var var? :fn fn?))
+(s/def ::tasks (s/map-of string? ::fn))
+(s/def ::start-worker-params
+  (s/keys :req-un [::tasks]
+          :opt-un [::poll-interval]))
+
+(defn start-worker!
+  [{:keys [poll-interval]
     :or {poll-interval 5000}
     :as opts}]
-  (try
+  (us/assert ::start-worker-params opts)
-    (when-not @stop
+  (log/info (str/format "Starting worker '%s' on queue '%s'."
-      (let [res (event-loop-fn opts)]
+                        (:name opts "anonymous")
-        (if (= res ::handled)
+                        (:queue opts "default")))
-          (px/schedule! xtor 0 (partial execute-worker-task opts))
+  (let [cch (a/chan 1)]
-          (px/schedule! xtor poll-interval (partial execute-worker-task opts)))))
+    (a/go-loop []
-    (catch Throwable e
+      (let [[val port] (a/alts! [cch (event-loop-fn opts)] :priority true)]
-      (log/error "unexpected exception:" e)
+        (cond
-      (px/schedule! xtor poll-interval (partial execute-worker-task opts)))))
+          ;; Terminate the loop if close channel is closed or
+          ;; event-loop-fn returns nil.
+          (or (= port cch) (nil? val))
+          (log/info (str/format "Stop condition found. Shutdown worker: '%s'"
+                                (:name opts "anonymous")))
+
+          (db/pool-closed? db/pool)
+          (do
+            (log/info "Worker eventloop is aborted because pool is closed.")
+            (a/close! cch))
+
+          (and (instance? java.sql.SQLException val)
+               (contains? #{"08003" "08006" "08001" "08004"} (.getSQLState val)))
+          (do
+            (log/error "Connection error, trying resume in some instants.")
+            (a/<! (a/timeout poll-interval))
+            (recur))
+
+          (and (instance? java.sql.SQLException val)
+               (= "40001" (.getSQLState ^java.sql.SQLException val)))
+          (do
+            (log/debug "Serialization failure (retrying in some instants).")
+            (a/<! (a/timeout 1000))
+            (recur))
+
+          (instance? Exception val)
+          (do
+            (log/error "Unexpected error ocurried on polling the database." val)
+            (log/info "Trying resume operations in some instants.")
+            (a/<! (a/timeout poll-interval))
+            (recur))
+
+          (= ::handled val)
+          (recur)
+
+          (= ::empty val)
+          (do
+            (a/<! (a/timeout poll-interval))
+            (recur)))))
+
+    (reify
+      java.lang.AutoCloseable
+      (close [_]
+        (a/close! cch)))))
+
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Scheduled Tasks
@@ -178,7 +255,7 @@
         (log/info "Executing scheduled task" id)
         ((:fn task) task)))
 
-    (catch Throwable e
+    (catch Exception e
       (log-scheduled-task-error task e))
     (finally
       (schedule-task! xtor task))))
@@ -196,61 +273,34 @@
         task  (assoc task ::xtor xtor)]
     (px/schedule! xtor ms (partial execute-scheduled-task task))))
 
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Public API
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(s/def ::id string?)
-(s/def ::name string?)
-(s/def ::cron dt/cron?)
 (s/def ::fn (s/or :var var? :fn fn?))
+(s/def ::id string?)
+(s/def ::cron dt/cron?)
+;; (s/def ::xtor #(instance? ScheduledExecutorService %))
 (s/def ::props (s/nilable map?))
-(s/def ::xtor #(instance? ScheduledExecutorService %))
-
 (s/def ::scheduled-task
   (s/keys :req-un [::id ::cron ::fn]
           :opt-un [::props]))
 
-(s/def ::tasks (s/map-of string? ::fn))
 (s/def ::schedule (s/coll-of ::scheduled-task))
+(s/def ::start-scheduler-worker-params
+  (s/keys :req-un [::schedule]))
 
 (defn start-scheduler-worker!
-  [{:keys [schedule xtor] :as opts}]
+  [{:keys [schedule] :as opts}]
-  (us/assert ::xtor xtor)
+  (us/assert ::start-scheduler-worker-params opts)
-  (us/assert ::schedule schedule)
+  (let [xtor (Executors/newScheduledThreadPool (int 1))]
-  (let [stop (atom false)]
     (synchronize-schedule! schedule)
     (run! (partial schedule-task! xtor) schedule)
     (reify
       java.lang.AutoCloseable
       (close [_]
-        (reset! stop true)))))
+        (.shutdownNow ^ScheduledExecutorService xtor)))))
-
-(defn start-worker!
-  [{:keys [tasks xtor poll-interval]
-    :or {poll-interval 5000}
-    :as opts}]
-  (us/assert ::tasks tasks)
-  (us/assert ::xtor xtor)
-  (us/assert number? poll-interval)
-  (let [stop (atom false)
-        opts (assoc opts
-                    ::xtor xtor
-                    ::stop stop)]
-    (px/schedule! xtor poll-interval (partial execute-worker-task opts))
-    (reify
-      java.lang.AutoCloseable
-      (close [_]
-        (reset! stop true)))))
 
 (defn stop!
   [worker]
   (.close ^java.lang.AutoCloseable worker))
 
-
-
-
-
 ;; --- Submit API
 
 (s/def ::name ::us/string)
@@ -258,30 +308,25 @@
   (s/or :int ::us/integer
         :duration dt/duration?))
 (s/def ::queue ::us/string)
+
 (s/def ::task-options
   (s/keys :req-un [::name]
           :opt-un [::delay ::props ::queue]))
 
 (def ^:private sql:insert-new-task
-  "insert into task (id, name, props, queue, scheduled_at)
+  "insert into task (id, name, props, queue, priority, max_retries, scheduled_at)
-   values (?, ?, ?, ?, clock_timestamp()+cast(?::text as interval))
+   values (?, ?, ?, ?, ?, ?, clock_timestamp() + ?)
    returning id")
 
-(defn- duration->pginterval
-  [^Duration d]
-  (->> (/ (.toMillis d) 1000.0)
-       (format "%s seconds")))
-
 (defn submit!
-  [conn {:keys [name delay props queue key]
+  [conn {:keys [name delay props queue priority max-retries key]
-         :or {delay 0 props {} queue "default"}
+         :or {delay 0 props {} queue "default" priority 100 max-retries 3}
          :as options}]
   (us/verify ::task-options options)
-  (let [duration   (dt/duration delay)
+  (let [duration  (dt/duration delay)
-        pginterval (duration->pginterval duration)
+        interval  (db/interval duration)
-        props      (blob/encode props)
+        props     (db/tjson props)
-        id         (uuid/next)]
+        id        (uuid/next)]
-    (log/info "Submit task" name "to be executed in" (str duration))
+    (log/info (str/format "Submit task '%s' to be executed in '%s'." name (str duration)))
-    (db/exec-one! conn [sql:insert-new-task
+    (db/exec-one! conn [sql:insert-new-task id name props queue priority max-retries interval])
-                        id name props queue pginterval])
     id))

backend/src/uxbox/util/async.clj

@@ -0,0 +1,31 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) 2020 Andrey Antukh <niwi@niwi.nz>
+
+(ns uxbox.util.async
+  (:require [clojure.core.async :as a]))
+
+(defmacro go-try
+  [& body]
+  `(a/go
+     (try
+       ~@body
+       (catch Throwable e# e#))))
+
+(defmacro <?
+  [ch]
+  `(let [r# (a/<! ~ch)]
+     (if (instance? Throwable r#)
+       (throw r#)
+       r#)))
+
+(defmacro thread-try
+  [& body]
+  `(a/thread
+     (try
+       ~@body
+       (catch Throwable e#
+         e#))))
+

backend/src/uxbox/util/transit.clj

@@ -122,6 +122,11 @@
   (->> (encode message)
        (bytes->str)))
 
+(defn encode-verbose-str
+  [message]
+  (->> (encode message {:type :json-verbose})
+       (bytes->str)))
+
 ;; --- Helpers
 
 (defn str->bytes