From 350663b7ce763d77e8b17d5844550d4962f652d0 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Sat, 12 Feb 2022 12:14:55 +0100 Subject: [PATCH] :tada: Add support for alternative S3 compatible services And also add support for all AWS regions (prevoiosly onlu eu-central-1) was supported. --- CHANGES.md | 1 + backend/scripts/repl | 12 +++++++++++ backend/src/app/config.clj | 4 ++++ backend/src/app/http/assets.clj | 4 ++-- backend/src/app/main.clj | 12 ++++++----- backend/src/app/storage/s3.clj | 34 +++++++++++++++++++++---------- docker/devenv/docker-compose.yaml | 18 ++++++++++++++++ 7 files changed, 67 insertions(+), 18 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 81d16cf7d..4555cc071 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -24,6 +24,7 @@ - Redesign of workspace toolbars [Taiga #2319](https://tree.taiga.io/project/penpot/us/2319) - Graphic Tablet usability improvements [Taiga #1913](https://tree.taiga.io/project/penpot/us/1913) - Improved mouse collision detection for groups and text shapes [Taiga #2452](https://tree.taiga.io/project/penpot/us/2452), [Taiga #2453](https://tree.taiga.io/project/penpot/us/2453) +- Add support for alternative S3 storage providers and all aws regions [#1267](https://github.com/penpot/penpot/issues/1267) ### :bug: Bugs fixed diff --git a/backend/scripts/repl b/backend/scripts/repl index 22bebe8c7..ebb3e1554 100755 --- a/backend/scripts/repl +++ b/backend/scripts/repl @@ -10,6 +10,18 @@ # export PENPOT_DATABASE_PASSWORD="penpot_pre" # export PENPOT_FLAGS="enable-asserts enable-audit-log $PENPOT_FLAGS" +# Initialize MINIO config +# mc alias set penpot-s3/ http://minio:9000 minioadmin minioadmin +# mc admin user add penpot-s3 penpot-devenv penpot-devenv +# mc admin policy set penpot-s3 readwrite user=penpot-devenv +# mc mb penpot-s3/penpot -p +# export AWS_ACCESS_KEY_ID=penpot-devenv +# export AWS_SECRET_ACCESS_KEY=penpot-devenv +# export PENPOT_ASSETS_STORAGE_BACKEND=assets-s3 +# export PENPOT_STORAGE_ASSETS_S3_ENDPOINT=http://minio:9000 +# export PENPOT_STORAGE_ASSETS_S3_REGION=eu-central-1 +# export PENPOT_STORAGE_ASSETS_S3_BUCKET=penpot + export OPTIONS=" -A:dev \ -J-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \ diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index b0030a2d1..7083b490e 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -181,9 +181,11 @@ (s/def ::storage-assets-fs-directory ::us/string) (s/def ::storage-assets-s3-bucket ::us/string) (s/def ::storage-assets-s3-region ::us/keyword) +(s/def ::storage-assets-s3-endpoint ::us/string) (s/def ::storage-fdata-s3-bucket ::us/string) (s/def ::storage-fdata-s3-region ::us/keyword) (s/def ::storage-fdata-s3-prefix ::us/string) +(s/def ::storage-fdata-s3-endpoint ::us/string) (s/def ::telemetry-uri ::us/string) (s/def ::telemetry-with-taiga ::us/boolean) (s/def ::tenant ::us/string) @@ -278,10 +280,12 @@ ::storage-assets-fs-directory ::storage-assets-s3-bucket ::storage-assets-s3-region + ::storage-assets-s3-endpoint ::fdata-storage-backend ::storage-fdata-s3-bucket ::storage-fdata-s3-region ::storage-fdata-s3-prefix + ::storage-fdata-s3-endpoint ::telemetry-enabled ::telemetry-uri ::telemetry-referer diff --git a/backend/src/app/http/assets.clj b/backend/src/app/http/assets.clj index 18c14462e..550bfcdfb 100644 --- a/backend/src/app/http/assets.clj +++ b/backend/src/app/http/assets.clj @@ -52,10 +52,10 @@ :body (sto/get-object-bytes storage obj)} :s3 - (let [url (sto/get-object-url storage obj {:max-age signature-max-age})] + (let [{:keys [host port] :as url} (sto/get-object-url storage obj {:max-age signature-max-age})] {:status 307 :headers {"location" (str url) - "x-host" (:host url) + "x-host" (cond-> host port (str ":" port)) "cache-control" (str "max-age=" (inst-ms cache-max-age))} :body ""}) diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index aaa7d521b..fabf61c4d 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -309,13 +309,15 @@ :fs (ig/ref [::assets :app.storage.fs/backend])}} [::fdata :app.storage.s3/backend] - {:region (cf/get :storage-fdata-s3-region) - :bucket (cf/get :storage-fdata-s3-bucket) - :prefix (cf/get :storage-fdata-s3-prefix)} + {:region (cf/get :storage-fdata-s3-region) + :bucket (cf/get :storage-fdata-s3-bucket) + :endpoint (cf/get :storage-fdata-s3-endpoint) + :prefix (cf/get :storage-fdata-s3-prefix)} [::assets :app.storage.s3/backend] - {:region (cf/get :storage-assets-s3-region) - :bucket (cf/get :storage-assets-s3-bucket)} + {:region (cf/get :storage-assets-s3-region) + :endpoint (cf/get :storage-assets-s3-endpoint) + :bucket (cf/get :storage-assets-s3-bucket)} [::assets :app.storage.fs/backend] {:directory (cf/get :storage-assets-fs-directory)} diff --git a/backend/src/app/storage/s3.clj b/backend/src/app/storage/s3.clj index 10c5710e0..22b3d88bd 100644 --- a/backend/src/app/storage/s3.clj +++ b/backend/src/app/storage/s3.clj @@ -56,9 +56,10 @@ (s/def ::region #{:eu-central-1}) (s/def ::bucket ::us/string) (s/def ::prefix ::us/string) +(s/def ::endpoint ::us/string) (defmethod ig/pre-init-spec ::backend [_] - (s/keys :opt-un [::region ::bucket ::prefix])) + (s/keys :opt-un [::region ::bucket ::prefix ::endpoint])) (defmethod ig/prep-key ::backend [_ {:keys [prefix] :as cfg}] @@ -119,20 +120,31 @@ (defn- ^Region lookup-region [region] - (case region - :eu-central-1 Region/EU_CENTRAL_1)) + (Region/of (name region))) (defn build-s3-client - [{:keys [region]}] - (.. (S3Client/builder) - (region (lookup-region region)) - (build))) + [{:keys [region endpoint]}] + (if (string? endpoint) + (let [uri (java.net.URI. endpoint)] + (.. (S3Client/builder) + (endpointOverride uri) + (region (lookup-region region)) + (build))) + (.. (S3Client/builder) + (region (lookup-region region)) + (build)))) (defn build-s3-presigner - [{:keys [region]}] - (.. (S3Presigner/builder) - (region (lookup-region region)) - (build))) + [{:keys [region endpoint]}] + (if (string? endpoint) + (let [uri (java.net.URI. endpoint)] + (.. (S3Presigner/builder) + (endpointOverride uri) + (region (lookup-region region)) + (build))) + (.. (S3Presigner/builder) + (region (lookup-region region)) + (build)))) (defn put-object [{:keys [client bucket prefix]} {:keys [id] :as object} content] diff --git a/docker/devenv/docker-compose.yaml b/docker/devenv/docker-compose.yaml index ee6b1d7df..5b92d0ed4 100644 --- a/docker/devenv/docker-compose.yaml +++ b/docker/devenv/docker-compose.yaml @@ -10,6 +10,7 @@ networks: volumes: postgres_data: user_data: + minio_data: services: main: @@ -66,6 +67,22 @@ services: - PENPOT_LDAP_ATTRS_FULLNAME=cn - PENPOT_LDAP_ATTRS_PHOTO=jpegPhoto + minio: + profiles: ["full"] + image: "minio/minio:latest" + command: minio server /mnt/data --console-address ":9001" + + volumes: + - "minio_data:/mnt/data" + + environment: + - MINIO_ROOT_USER=minioadmin + - MINIO_ROOT_PASSWORD=minioadmin + + ports: + - 9000:9000 + - 9001:9001 + backend: profiles: ["backend"] privileged: true @@ -91,6 +108,7 @@ services: environment: - EXTERNAL_UID=${CURRENT_USER_ID} - PENPOT_SECRET_KEY=super-secret-devenv-key + # SMTP setup - PENPOT_SMTP_ENABLED=true - PENPOT_SMTP_DEFAULT_FROM=no-reply@example.com