From ebe8fdcba81f71aecf38ce55410bb46cdea80952 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 29 Jun 2022 14:39:40 +0200 Subject: [PATCH] :recycle: Refactor temporal files management on exporter --- exporter/src/app/core.cljs | 2 +- exporter/src/app/handlers/export_frames.cljs | 5 +-- exporter/src/app/handlers/export_shapes.cljs | 4 -- exporter/src/app/handlers/resources.cljs | 4 +- exporter/src/app/renderer/bitmap.cljs | 5 +-- exporter/src/app/renderer/pdf.cljs | 3 +- exporter/src/app/renderer/svg.cljs | 32 +++++---------- exporter/src/app/util/shell.cljs | 42 ++++++++++++++++++-- 8 files changed, 57 insertions(+), 40 deletions(-) diff --git a/exporter/src/app/core.cljs b/exporter/src/app/core.cljs index c3ebbd46a..79597412d 100644 --- a/exporter/src/app/core.cljs +++ b/exporter/src/app/core.cljs @@ -8,10 +8,10 @@ (:require ["process" :as proc] [app.browser :as bwr] - [app.redis :as redis] [app.common.logging :as l] [app.config] [app.http :as http] + [app.redis :as redis] [promesa.core :as p])) (enable-console-print!) diff --git a/exporter/src/app/handlers/export_frames.cljs b/exporter/src/app/handlers/export_frames.cljs index a8a4a0c85..716cc0b30 100644 --- a/exporter/src/app/handlers/export_frames.cljs +++ b/exporter/src/app/handlers/export_frames.cljs @@ -128,8 +128,8 @@ (defn- join-pdf [file-id paths] - (p/let [tmpdir (sh/mktmpdir! "join-pdf") - path (path/join tmpdir (str/concat file-id ".pdf"))] + (p/let [prefix (str/concat "penpot.tmp.pdfunite." file-id ".") + path (sh/tempfile :prefix prefix :suffix ".pdf")] (sh/run-cmd! (str "pdfunite " (str/join " " paths) " " path)) path)) @@ -137,5 +137,4 @@ [{:keys [path] :as resource} output-path] (p/do (sh/move! output-path path) - (sh/rmdir! (path/dirname output-path)) resource)) diff --git a/exporter/src/app/handlers/export_shapes.cljs b/exporter/src/app/handlers/export_shapes.cljs index c6ac2f05d..d50dfc7c3 100644 --- a/exporter/src/app/handlers/export_shapes.cljs +++ b/exporter/src/app/handlers/export_shapes.cljs @@ -102,8 +102,6 @@ total (count exports) topic (str profile-id) - to-delete (atom #{}) - on-progress (fn [{:keys [done]}] (when-not wait (let [data {:type :export-update @@ -137,7 +135,6 @@ :on-progress on-progress) append (fn [{:keys [filename path] :as object}] - (swap! to-delete conj path) (rsc/add-to-zip! zip path filename)) proc (-> (p/do @@ -146,7 +143,6 @@ (p/let [proc (rd/render export append)] (p/recur (rest exports))))) (.finalize zip)) - (p/then (fn [_] (p/run! #(sh/rmdir! (path/dirname %)) @to-delete))) (p/then (constantly resource)) (p/catch on-error)) ] diff --git a/exporter/src/app/handlers/resources.cljs b/exporter/src/app/handlers/resources.cljs index e02705610..02ccfe0b1 100644 --- a/exporter/src/app/handlers/resources.cljs +++ b/exporter/src/app/handlers/resources.cljs @@ -14,15 +14,15 @@ [app.common.data :as d] [app.common.exceptions :as ex] [app.common.uuid :as uuid] - [app.util.shell :as sh] [app.util.mime :as mime] + [app.util.shell :as sh] [cljs.core :as c] [cuerdas.core :as str] [promesa.core :as p])) (defn- get-path [type id] - (path/join (os/tmpdir) (str/concat "exporter-resource." (c/name type) "." id))) + (path/join sh/tmpdir (str/concat "penpot.resource." (c/name type) "." id))) (defn create "Generates ephimeral resource object." diff --git a/exporter/src/app/renderer/bitmap.cljs b/exporter/src/app/renderer/bitmap.cljs index eaee73346..bacde909b 100644 --- a/exporter/src/app/renderer/bitmap.cljs +++ b/exporter/src/app/renderer/bitmap.cljs @@ -36,9 +36,8 @@ :userAgent bw/default-user-agent}) (render-object [page {:keys [id] :as object}] - (p/let [tmpdir (sh/mktmpdir! "bitmap-render") - path (path/join tmpdir (str/concat id (mime/get-extension type))) - node (bw/select page (str/concat "#screenshot-" id))] + (p/let [path (sh/tempfile :prefix "penpot.tmp.render.bitmap." :suffix (mime/get-extension type)) + node (bw/select page (str/concat "#screenshot-" id))] (bw/wait-for node) (case type :png (bw/screenshot node {:omit-background? true :type type :path path}) diff --git a/exporter/src/app/renderer/pdf.cljs b/exporter/src/app/renderer/pdf.cljs index 124e545ad..2ba6926e8 100644 --- a/exporter/src/app/renderer/pdf.cljs +++ b/exporter/src/app/renderer/pdf.cljs @@ -44,8 +44,7 @@ (render-object [page base-uri {:keys [id] :as object}] (p/let [uri (prepare-uri base-uri id) - tmp (sh/mktmpdir! "pdf-render") - path (path/join tmp (str/concat id (mime/get-extension type)))] + path (sh/tempfile :prefix "penpot.tmp.render.pdf." :suffix (mime/get-extension type))] (l/info :uri uri) (bw/nav! page uri) (p/let [dom (bw/select page (dm/str "#screenshot-" id))] diff --git a/exporter/src/app/renderer/svg.cljs b/exporter/src/app/renderer/svg.cljs index 825c51ad9..d1c4e3fcc 100644 --- a/exporter/src/app/renderer/svg.cljs +++ b/exporter/src/app/renderer/svg.cljs @@ -116,24 +116,20 @@ (defn render [{:keys [page-id file-id objects token scale type]} on-object] (letfn [(convert-to-ppm [pngpath] - (l/trace :fn :convert-to-ppm) - (let [basepath (path/dirname pngpath) - ppmpath (path/join basepath "origin.ppm")] + (let [ppmpath (str/concat pngpath "origin.ppm")] + (l/trace :fn :convert-to-ppm :path ppmpath) (-> (sh/run-cmd! (str "convert " pngpath " " ppmpath)) (p/then (constantly ppmpath))))) (trace-color-mask [pbmpath] (l/trace :fn :trace-color-mask :pbmpath pbmpath) - (let [basepath (path/dirname pbmpath) - basename (path/basename pbmpath ".pbm") - svgpath (path/join basepath (str basename ".svg"))] + (let [svgpath (str/concat pbmpath ".svg")] (-> (sh/run-cmd! (str "potrace --flat -b svg " pbmpath " -o " svgpath)) (p/then (constantly svgpath))))) (generate-color-layer [ppmpath color] (l/trace :fn :generate-color-layer :ppmpath ppmpath :color color) - (let [basepath (path/dirname ppmpath) - pbmpath (path/join basepath (str "mask-" (subs color 1) ".pbm"))] + (let [pbmpath (str/concat ppmpath ".mask-" (subs color 1) ".pbm")] (-> (sh/run-cmd! (str/format "ppmcolormask \"%s\" %s" color ppmpath)) (p/then (fn [stdout] (-> (sh/write-file! pbmpath stdout) @@ -247,15 +243,14 @@ (trace-node [{:keys [data] :as node}] (l/trace :fn :trace-node) - (p/let [tdpath (sh/mktmpdir! "svgexport") - pngpath (path/join tdpath "origin.png") + (p/let [pngpath (sh/tempfile :prefix "penpot.tmp.render.svg.parse." + :suffix ".origin.png") _ (sh/write-file! pngpath data) ppmpath (convert-to-ppm pngpath) svgdata (convert-to-svg ppmpath node)] (-> node (dissoc :data) - (assoc :tempdir tdpath - :svgdata svgdata)))) + (assoc :svgdata svgdata)))) (extract-element-attrs [^js element] (let [^js attrs (.. element -attributes) @@ -289,17 +284,11 @@ shot (bw/screenshot text-node {:omit-background? true :type "png"})] [shot node])) - (clean-temp-data [{:keys [tempdir] :as node}] - (p/do! - (sh/rmdir! tempdir) - (dissoc node :tempdir))) - (extract-txt-node [page item] (-> (p/resolved item) (p/then (partial resolve-text-node page)) (p/then extract-single-node) - (p/then trace-node) - (p/then clean-temp-data))) + (p/then trace-node))) (extract-txt-nodes [page {:keys [id] :as objects}] (l/trace :fn :process-text-nodes) @@ -323,9 +312,8 @@ :userAgent bw/default-user-agent}) (render-object [page {:keys [id] :as object}] - (p/let [tmpdir (sh/mktmpdir! "svg-render") - path (path/join tmpdir (str/concat id (mime/get-extension type))) - node (bw/select page (str/concat "#screenshot-" id))] + (p/let [path (sh/tempfile :prefix "penpot.tmp.render.svg." :suffix (mime/get-extension type)) + node (bw/select page (str/concat "#screenshot-" id))] (bw/wait-for node) (p/let [xmldata (extract-svg page object) txtdata (extract-txt-nodes page object) diff --git a/exporter/src/app/util/shell.cljs b/exporter/src/app/util/shell.cljs index 93b5333ed..559174167 100644 --- a/exporter/src/app/util/shell.cljs +++ b/exporter/src/app/util/shell.cljs @@ -12,13 +12,49 @@ ["os" :as os] ["path" :as path] [app.common.logging :as l] + [app.common.uuid :as uuid] + [app.common.exceptions :as ex] + [cuerdas.core :as str] [promesa.core :as p])) (l/set-level! :trace) -(defn mktmpdir! - [prefix] - (.mkdtemp fs/promises (path/join (os/tmpdir) prefix))) +(def tempfile-minage (* 1000 60 60 1)) ;; 1h + +(def tmpdir + (let [path (path/join (os/tmpdir) "penpot")] + (when-not (fs/existsSync path) + (fs/mkdirSync path #js {:recursive true})) + path)) + + +(defn- schedule-deletion! + [path] + (letfn [(remote-tempfile [] + (when (fs/existsSync path) + (l/trace :hint "permanently remove tempfile" :path path) + (fs/rmSync path #js {:recursive true})))] + (let [ts (js/Date.now)] + (l/trace :hint "schedule tempfile deletion" + :path path + :scheduled-at (.. (js/Date. (+ (js/Date.now) tempfile-minage)) toString)) + (js/setTimeout remote-tempfile tempfile-minage)))) + +(defn tempfile + [& {:keys [prefix suffix] + :or {prefix "penpot." + suffix ".tmp"}}] + (loop [i 0] + (if (< i 1000) + (let [path (path/join tmpdir (str/concat prefix (uuid/next) "-" i suffix))] + (if (fs/existsSync path) + (recur (inc i)) + (do + (schedule-deletion! path) + path))) + (ex/raise :type :internal + :code :unable-to-locate-temporal-file + :hint "unable to find a tempfile candidate")))) (defn move! [origin-path dest-path]