🎉 Add support for alternative S3 compatible services

And also add support for all AWS regions (prevoiosly onlu eu-central-1)
was supported.

CHANGES.md

@@ -24,6 +24,7 @@
 - Redesign of workspace toolbars [Taiga #2319](https://tree.taiga.io/project/penpot/us/2319)
 - Graphic Tablet usability improvements [Taiga #1913](https://tree.taiga.io/project/penpot/us/1913)
 - Improved mouse collision detection for groups and text shapes [Taiga #2452](https://tree.taiga.io/project/penpot/us/2452), [Taiga #2453](https://tree.taiga.io/project/penpot/us/2453)
+- Add support for alternative S3 storage providers and all aws regions [#1267](https://github.com/penpot/penpot/issues/1267)
 
 ### :bug: Bugs fixed
 

backend/scripts/repl

@@ -10,6 +10,18 @@
 # export PENPOT_DATABASE_PASSWORD="penpot_pre"
 # export PENPOT_FLAGS="enable-asserts enable-audit-log $PENPOT_FLAGS"
 
+# Initialize MINIO config
+# mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin
+# mc admin user add penpot-s3 penpot-devenv penpot-devenv
+# mc admin policy set penpot-s3 readwrite user=penpot-devenv
+# mc mb penpot-s3/penpot -p
+# export AWS_ACCESS_KEY_ID=penpot-devenv
+# export AWS_SECRET_ACCESS_KEY=penpot-devenv
+# export PENPOT_ASSETS_STORAGE_BACKEND=assets-s3
+# export PENPOT_STORAGE_ASSETS_S3_ENDPOINT=http://minio:9000
+# export PENPOT_STORAGE_ASSETS_S3_REGION=eu-central-1
+# export PENPOT_STORAGE_ASSETS_S3_BUCKET=penpot
+
 export OPTIONS="
        -A:dev \
        -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \

backend/src/app/config.clj

@@ -181,9 +181,11 @@
 (s/def ::storage-assets-fs-directory ::us/string)
 (s/def ::storage-assets-s3-bucket ::us/string)
 (s/def ::storage-assets-s3-region ::us/keyword)
+(s/def ::storage-assets-s3-endpoint ::us/string)
 (s/def ::storage-fdata-s3-bucket ::us/string)
 (s/def ::storage-fdata-s3-region ::us/keyword)
 (s/def ::storage-fdata-s3-prefix ::us/string)
+(s/def ::storage-fdata-s3-endpoint ::us/string)
 (s/def ::telemetry-uri ::us/string)
 (s/def ::telemetry-with-taiga ::us/boolean)
 (s/def ::tenant ::us/string)
@@ -278,10 +280,12 @@
                    ::storage-assets-fs-directory
                    ::storage-assets-s3-bucket
                    ::storage-assets-s3-region
+                   ::storage-assets-s3-endpoint
                    ::fdata-storage-backend
                    ::storage-fdata-s3-bucket
                    ::storage-fdata-s3-region
                    ::storage-fdata-s3-prefix
+                   ::storage-fdata-s3-endpoint
                    ::telemetry-enabled
                    ::telemetry-uri
                    ::telemetry-referer

backend/src/app/http/assets.clj

@@ -52,10 +52,10 @@
        :body (sto/get-object-bytes storage obj)}
 
       :s3
-      (let [url (sto/get-object-url storage obj {:max-age signature-max-age})]
+      (let [{:keys [host port] :as url} (sto/get-object-url storage obj {:max-age signature-max-age})]
         {:status 307
          :headers {"location" (str url)
-                   "x-host"   (:host url)
+                   "x-host"   (cond-> host port (str ":" port))
                    "cache-control" (str "max-age=" (inst-ms cache-max-age))}
          :body ""})
 

backend/src/app/main.clj

@@ -309,13 +309,15 @@
      :fs        (ig/ref [::assets :app.storage.fs/backend])}}
 
    [::fdata :app.storage.s3/backend]
-   {:region (cf/get :storage-fdata-s3-region)
+   {:region   (cf/get :storage-fdata-s3-region)
-    :bucket (cf/get :storage-fdata-s3-bucket)
+    :bucket   (cf/get :storage-fdata-s3-bucket)
-    :prefix (cf/get :storage-fdata-s3-prefix)}
+    :endpoint (cf/get :storage-fdata-s3-endpoint)
+    :prefix   (cf/get :storage-fdata-s3-prefix)}
 
    [::assets :app.storage.s3/backend]
-   {:region (cf/get :storage-assets-s3-region)
+   {:region   (cf/get :storage-assets-s3-region)
-    :bucket (cf/get :storage-assets-s3-bucket)}
+    :endpoint (cf/get :storage-assets-s3-endpoint)
+    :bucket   (cf/get :storage-assets-s3-bucket)}
 
    [::assets :app.storage.fs/backend]
    {:directory (cf/get :storage-assets-fs-directory)}

backend/src/app/storage/s3.clj

@@ -56,9 +56,10 @@
 (s/def ::region #{:eu-central-1})
 (s/def ::bucket ::us/string)
 (s/def ::prefix ::us/string)
+(s/def ::endpoint ::us/string)
 
 (defmethod ig/pre-init-spec ::backend [_]
-  (s/keys :opt-un [::region ::bucket ::prefix]))
+  (s/keys :opt-un [::region ::bucket ::prefix ::endpoint]))
 
 (defmethod ig/prep-key ::backend
   [_ {:keys [prefix] :as cfg}]
@@ -119,20 +120,31 @@
 
 (defn- ^Region lookup-region
   [region]
-  (case region
+  (Region/of (name region)))
-    :eu-central-1 Region/EU_CENTRAL_1))
 
 (defn build-s3-client
-  [{:keys [region]}]
+  [{:keys [region endpoint]}]
-  (.. (S3Client/builder)
+  (if (string? endpoint)
-      (region (lookup-region region))
+    (let [uri (java.net.URI. endpoint)]
-      (build)))
+      (.. (S3Client/builder)
+          (endpointOverride uri)
+          (region (lookup-region region))
+          (build)))
+    (.. (S3Client/builder)
+        (region (lookup-region region))
+        (build))))
 
 (defn build-s3-presigner
-  [{:keys [region]}]
+  [{:keys [region endpoint]}]
-  (.. (S3Presigner/builder)
+  (if (string? endpoint)
-      (region (lookup-region region))
+    (let [uri (java.net.URI. endpoint)]
-      (build)))
+      (.. (S3Presigner/builder)
+          (endpointOverride uri)
+          (region (lookup-region region))
+          (build)))
+    (.. (S3Presigner/builder)
+        (region (lookup-region region))
+        (build))))
 
 (defn put-object
   [{:keys [client bucket prefix]} {:keys [id] :as object} content]

docker/devenv/docker-compose.yaml

@@ -10,6 +10,7 @@ networks:
 volumes:
   postgres_data:
   user_data:
+  minio_data:
 
 services:
   main:
@@ -66,6 +67,22 @@ services:
       - PENPOT_LDAP_ATTRS_FULLNAME=cn
       - PENPOT_LDAP_ATTRS_PHOTO=jpegPhoto
 
+  minio:
+    profiles: ["full"]
+    image: "minio/minio:latest"
+    command: minio server /mnt/data --console-address ":9001"
+
+    volumes:
+      - "minio_data:/mnt/data"
+
+    environment:
+      - MINIO_ROOT_USER=minioadmin
+      - MINIO_ROOT_PASSWORD=minioadmin
+
+    ports:
+      - 9000:9000
+      - 9001:9001
+
   backend:
     profiles: ["backend"]
     privileged: true
@@ -91,6 +108,7 @@ services:
     environment:
       - EXTERNAL_UID=${CURRENT_USER_ID}
       - PENPOT_SECRET_KEY=super-secret-devenv-key
+
       # SMTP setup
       - PENPOT_SMTP_ENABLED=true
       - PENPOT_SMTP_DEFAULT_FROM=no-reply@example.com