Merge remote-tracking branch 'origin/main' into develop

CHANGES.md

@@ -41,6 +41,24 @@
 - Cleanup unused static images (by @rhcarvalho) [#1561](https://github.com/penpot/penpot/pull/1561)
 - Compress static images to save space (by @rhcarvalho) [#1562](https://github.com/penpot/penpot/pull/1562)
 
+## 1.11.2-beta
+
+### :bug: Bugs fixed
+
+- Fix issue on handling empty content on boolean shapes
+- Fix race condition issue on component renaming
+- Handle EOF errors on writting streamed response
+- Handle EOF errors on websocket send/ping methods
+- Disable parallel upload of file media on import (causes too much
+  contention on the rlimit subsistem that does not works as expected
+  on high load).
+
+### :sparkles: New features
+
+- Add health check endpoint on API
+- Increase default max connection pool size to 60
+- Reduce resource usage of the error reporter.
+
 ## 1.11.1-beta
 
 ### :bug: Bugs fixed

backend/src/app/http/errors.clj

@@ -10,7 +10,6 @@
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
    [app.common.spec :as us]
-   [app.common.uuid :as uuid]
    [clojure.spec.alpha :as s]
    [cuerdas.core :as str]))
 
@@ -24,8 +23,7 @@
   [request error]
   (let [data (ex-data error)]
     (merge
-     {:id            (uuid/next)
+     {:path          (:uri request)
-      :path          (:uri request)
       :method        (:request-method request)
       :hint          (ex-message error)
       :params        (:params request)

backend/src/app/http/session.clj

@@ -40,8 +40,12 @@
           token      (tokens :generate {:iss "authentication"
                                         :iat (dt/now)
                                         :uid profile-id})
+
+          now        (dt/now)
           params     {:user-agent user-agent
                       :profile-id profile-id
+                      :created-at now
+                      :updated-at now
                       :id token}]
       (db/insert! pool :http-session params)
       token))
@@ -146,8 +150,7 @@
 
 (defmethod ig/prep-key ::session
   [_ cfg]
-  (d/merge {:buffer-size 64}
+  (d/merge {:buffer-size 128} (d/without-nils cfg)))
-           (d/without-nils cfg)))
 
 (defmethod ig/init-key ::session
   [_ {:keys [pool tokens] :as cfg}]
@@ -222,7 +225,7 @@
 
             (= :size reason)
             (l/debug :task "updater"
-                     :action "update sessions"
+                     :hint "update sessions"
                      :reason (name reason)
                      :count result))
           (recur))))))
@@ -251,17 +254,20 @@
 
 (defmethod ig/init-key ::gc-task
   [_ {:keys [pool max-age] :as cfg}]
+  (l/debug :hint "initializing session gc task" :max-age max-age)
   (fn [_]
     (db/with-atomic [conn pool]
       (let [interval (db/interval max-age)
-            result   (db/exec-one! conn [sql:delete-expired interval])
+            result   (db/exec-one! conn [sql:delete-expired interval interval])
             result   (:next.jdbc/update-count result)]
         (l/debug :task "gc"
-                 :action "clean http sessions"
+                 :hint "clean http sessions"
-                 :count result)
+                 :deleted result)
         result))))
 
 (def ^:private
   sql:delete-expired
   "delete from http_session
-    where updated_at < now() - ?::interval")
+    where updated_at < now() - ?::interval
+       or (updated_at is null and
+           created_at < now() - ?::interval)")

backend/src/app/loggers/database.clj

@@ -29,9 +29,7 @@
 (defn- persist-on-database!
   [{:keys [pool] :as cfg} {:keys [id] :as event}]
   (when-not (db/read-only? pool)
-    (db/with-atomic [conn pool]
+    (db/insert! pool :server-error-report {:id id :content (db/tjson event)})))
-      (db/insert! conn :server-error-report
-                  {:id id :content (db/tjson event)}))))
 
 (defn- parse-event-data
   [event]
@@ -52,7 +50,7 @@
       (assoc :host (cf/get :host))
       (assoc :public-uri (cf/get :public-uri))
       (assoc :version (:full cf/version))
-      (update :id (fn [id] (or id (uuid/next))))))
+      (assoc :id (uuid/next))))
 
 (defn handle-event
   [{:keys [executor] :as cfg} event]
@@ -60,12 +58,13 @@
     (try
       (let [event (parse-event event)
             uri   (cf/get :public-uri)]
+
         (l/debug :hint "registering error on database" :id (:id event)
                  :uri (str uri "/dbg/error/" (:id event)))
+
         (persist-on-database! cfg event))
-      (catch Exception e
+      (catch Exception cause
-        (l/warn :hint "unexpected exception on database error logger"
+        (l/warn :hint "unexpected exception on database error logger" :cause cause)))))
-                :cause e)))))
 
 (defmethod ig/pre-init-spec ::reporter [_]
   (s/keys :req-un [::wrk/executor ::db/pool ::receiver]))
@@ -77,8 +76,7 @@
 (defmethod ig/init-key ::reporter
   [_ {:keys [receiver] :as cfg}]
   (l/info :msg "initializing database error persistence")
-  (let [output (a/chan (a/sliding-buffer 5)
+  (let [output (a/chan (a/sliding-buffer 5) (filter error-event?))]
-                       (filter error-event?))]
     (receiver :sub output)
     (a/go-loop []
       (let [msg (a/<! output)]

backend/src/app/main.clj

@@ -177,7 +177,7 @@
         :task :file-offload})
 
      (when (contains? cf/flags :audit-log-archive)
-       {:cron #app/cron "0 */3 * * * ?" ;; every 3m
+       {:cron #app/cron "0 */5 * * * ?" ;; every 5m
         :task :audit-log-archive})
 
      (when (contains? cf/flags :audit-log-gc)
@@ -186,7 +186,7 @@
 
      (when (or (contains? cf/flags :telemetry)
                (cf/get :telemetry-enabled))
-       {:cron #app/cron "0 0 */6 * * ?" ;; every 6h
+       {:cron #app/cron "0 30 */3,23 * * ?"
         :task :telemetry})]}
 
    :app.worker/registry

backend/src/app/storage.clj

@@ -305,7 +305,7 @@
               (recur (+ n ^long total)))
             (do
               (l/info :task "gc-deleted"
-                      :action "permanently delete items"
+                      :hint "permanently delete items"
                       :count n)
               {:deleted n})))))))
 
@@ -379,10 +379,10 @@
                      (+ cntd (count to-delete))))
             (do
               (l/info :task "gc-touched"
-                      :action "mark freeze"
+                      :hint "mark freeze"
                       :count cntf)
               (l/info :task "gc-touched"
-                      :action "mark for deletion"
+                      :hint "mark for deletion"
                       :count cntd)
               {:freeze cntf :delete cntd})))))))
 
@@ -461,7 +461,7 @@
                      (+ d (count to-delete))))
             (do
               (l/info :task "recheck"
-                      :action "recheck items"
+                      :hint "recheck items"
                       :processed n
                       :deleted d)
               {:processed n :deleted d})))))))

backend/src/app/storage/s3.clj

@@ -33,6 +33,7 @@
    software.amazon.awssdk.services.s3.model.GetObjectRequest
    software.amazon.awssdk.services.s3.model.ObjectIdentifier
    software.amazon.awssdk.services.s3.model.PutObjectRequest
+   software.amazon.awssdk.services.s3.model.S3Error
    ;; software.amazon.awssdk.services.s3.model.GetObjectResponse
    software.amazon.awssdk.services.s3.presigner.S3Presigner
    software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest
@@ -231,6 +232,9 @@
                              ^DeleteObjectsRequest dor)]
     (when (.hasErrors ^DeleteObjectsResponse dres)
       (let [errors (seq (.errors ^DeleteObjectsResponse dres))]
-        (ex/raise :type :s3-error
+        (ex/raise :type :internal
-                  :code :error-on-bulk-delete
+                  :code :error-on-s3-bulk-delete
-                  :context errors)))))
+                  :s3-errors (mapv (fn [^S3Error error]
+                                     {:key (.key error)
+                                      :msg (.message error)})
+                                   errors))))))

backend/src/app/tasks/file_media_gc.clj

@@ -106,7 +106,7 @@
           unused (->> (db/query conn :file-media-object {:file-id id})
                       (remove #(contains? used (:id %))))]
 
-      (l/debug :action "processing file"
+      (l/debug :hint "processing file"
                :id id
                :age age
                :to-delete (count unused))
@@ -117,7 +117,7 @@
                   {:id id})
 
       (doseq [mobj unused]
-        (l/debug :action "deleting media object"
+        (l/debug :hint "deleting media object"
                  :id (:id mobj)
                  :media-id (:media-id mobj)
                  :thumbnail-id (:thumbnail-id mobj))

backend/src/app/tasks/file_offload.clj

@@ -29,7 +29,7 @@
 
 (defn- offload-candidate
   [{:keys [storage conn backend] :as cfg} {:keys [id data] :as file}]
-  (l/debug :action "offload file data" :id id)
+  (l/debug :hint "offload file data" :id id)
   (let [backend (simpl/resolve-backend storage backend)]
     (->> (simpl/content data)
          (simpl/put-object backend file))

backend/src/app/tasks/file_xlog_gc.clj

@@ -28,7 +28,8 @@
       (let [interval (db/interval max-age)
             result   (db/exec-one! conn [sql:delete-files-xlog interval])
             result   (:next.jdbc/update-count result)]
-        (l/debug :removed result :hint "remove old file changes")
+        (l/info :hint "remove old file changes"
+                :removed result)
         result))))
 
 (def ^:private

backend/src/app/tasks/objects_gc.clj

@@ -48,7 +48,7 @@
         result  (db/exec! conn [sql max-age])]
 
     (doseq [{:keys [id] :as item} result]
-      (l/trace :action "delete object" :table table :id id))
+      (l/trace :hint "delete object" :table table :id id))
 
     (count result)))
 
@@ -63,7 +63,7 @@
         backend (simpl/resolve-backend storage (cf/get :fdata-storage-backend))]
 
     (doseq [{:keys [id] :as item} result]
-      (l/trace :action "delete object" :table table :id id)
+      (l/trace :hint "delete object" :table table :id id)
       (when backend
         (simpl/del-object backend item)))
 
@@ -78,7 +78,7 @@
         fonts   (db/exec! conn [sql max-age])
         storage (assoc storage :conn conn)]
     (doseq [{:keys [id] :as font} fonts]
-      (l/trace :action "delete object" :table table :id id)
+      (l/trace :hint "delete object" :table table :id id)
       (some->> (:woff1-file-id font) (sto/del-object storage))
       (some->> (:woff2-file-id font) (sto/del-object storage))
       (some->> (:otf-file-id font)   (sto/del-object storage))
@@ -95,7 +95,7 @@
         storage (assoc storage :conn conn)]
 
     (doseq [{:keys [id] :as team} teams]
-      (l/trace :action "delete object" :table table :id id)
+      (l/trace :hint "delete object" :table table :id id)
       (some->> (:photo-id team) (sto/del-object storage)))
 
     (count teams)))
@@ -127,7 +127,7 @@
         storage  (assoc storage :conn conn)]
 
     (doseq [{:keys [id] :as profile} profiles]
-      (l/trace :action "delete object" :table table :id id)
+      (l/trace :hint "delete object" :table table :id id)
 
       ;; Mark the owned teams as deleted; this enables them to be processed
       ;; in the same transaction in the "team" table step.

backend/src/app/tasks/tasks_gc.clj

@@ -28,7 +28,7 @@
       (let [interval (db/interval max-age)
             result   (db/exec-one! conn [sql:delete-completed-tasks interval])
             result   (:next.jdbc/update-count result)]
-        (l/debug :action "trim completed tasks table" :removed result)
+        (l/debug :hint "trim completed tasks table" :removed result)
         result))))
 
 (def ^:private

backend/src/app/tasks/telemetry.clj

@@ -14,12 +14,17 @@
    [app.common.spec :as us]
    [app.config :as cfg]
    [app.db :as db]
+   [app.util.async :refer [thread-sleep]]
    [app.util.http :as http]
    [app.util.json :as json]
    [clojure.spec.alpha :as s]
    [integrant.core :as ig]))
 
-(declare retrieve-stats)
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; TASK ENTRY POINT
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(declare get-stats)
 (declare send!)
 
 (s/def ::version ::us/string)
@@ -34,11 +39,18 @@
 (defmethod ig/init-key ::handler
   [_ {:keys [pool sprops version] :as cfg}]
   (fn [_]
+    ;; Sleep randomly between 0 to 10s
+    (thread-sleep (rand-int 10000))
+
     (let [instance-id (:instance-id sprops)]
-      (-> (retrieve-stats pool version)
+      (-> (get-stats pool version)
           (assoc :instance-id instance-id)
           (send! cfg)))))
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; IMPL
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
 (defn- send!
   [data cfg]
   (let [response (http/send! {:method :post
@@ -63,6 +75,20 @@
   [conn]
   (-> (db/exec-one! conn ["select count(*) as count from file;"]) :count))
 
+(defn- retrieve-num-file-changes
+  [conn]
+  (let [sql (str "select count(*) as count "
+                 "  from file_change "
+                 " where date_trunc('day', created_at) = date_trunc('day', now())")]
+    (-> (db/exec-one! conn [sql]) :count)))
+
+(defn- retrieve-num-touched-files
+  [conn]
+  (let [sql (str "select count(distinct file_id) as count "
+                 "  from file_change "
+                 " where date_trunc('day', created_at) = date_trunc('day', now())")]
+    (-> (db/exec-one! conn [sql]) :count)))
+
 (defn- retrieve-num-users
   [conn]
   (-> (db/exec-one! conn ["select count(*) as count from profile;"]) :count))
@@ -118,7 +144,7 @@
      :jvm-heap-max     (.maxMemory runtime)
      :jvm-cpus         (.availableProcessors runtime)}))
 
-(defn retrieve-stats
+(defn get-stats
   [conn version]
   (let [referer (if (cfg/get :telemetry-with-taiga)
                   "taiga"
@@ -130,7 +156,9 @@
          :total-files    (retrieve-num-files conn)
          :total-users    (retrieve-num-users conn)
          :total-fonts    (retrieve-num-fonts conn)
-         :total-comments (retrieve-num-comments conn)}
+         :total-comments (retrieve-num-comments conn)
+         :total-file-changes  (retrieve-num-file-changes conn)
+         :total-touched-files (retrieve-num-touched-files conn)}
         (d/merge
          (retrieve-team-averages conn)
          (retrieve-jvm-stats))

backend/src/app/util/time.clj

@@ -295,6 +295,11 @@
   (s/assert cron? cron)
   (.toInstant (.getNextValidTimeAfter cron (Date/from now))))
 
+(defn get-next
+  [cron tnow]
+  (let [nt (next-valid-instant-from cron tnow)]
+    (cons nt (lazy-seq (get-next cron nt)))))
+
 (defmethod print-method CronExpression
   [mv ^java.io.Writer writer]
   (.write writer (str "#app/cron \"" (.toString ^CronExpression mv) "\"")))
@@ -302,3 +307,8 @@
 (defmethod print-dup CronExpression
   [o w]
   (print-ctor o (fn [o w] (print-dup (.toString ^CronExpression o) w)) w))
+
+(extend-protocol fez/IEdn
+  CronExpression
+  (-edn [o] (pr-str o)))
+

backend/src/app/worker.clj

@@ -106,7 +106,7 @@
           (and (instance? java.sql.SQLException val)
                (contains? #{"08003" "08006" "08001" "08004"} (.getSQLState ^java.sql.SQLException val)))
           (do
-            (l/error :hint "connection error, trying resume in some instants")
+            (l/warn :hint "connection error, trying resume in some instants")
             (a/<! (a/timeout poll-interval))
             (recur))
 
@@ -119,7 +119,7 @@
 
           (instance? Exception val)
           (do
-            (l/error :cause val
+            (l/warn :cause val
                     :hint "unexpected error ocurried on polling the database (will resume in some instants)")
             (a/<! (a/timeout poll-ms))
             (recur))
@@ -262,8 +262,7 @@
   [error item]
   (let [data (ex-data error)]
     (merge
-     {:id            (uuid/next)
+     {:hint          (ex-message error)
-      :hint          (ex-message error)
       :spec-problems (some->> data ::s/problems (take 10) seq vec)
       :spec-value    (some->> data ::s/value)
       :data          (some-> data (dissoc ::s/problems ::s/value ::s/spec))
@@ -429,21 +428,19 @@
 (defn- execute-scheduled-task
   [{:keys [executor pool] :as cfg} {:keys [id] :as task}]
   (letfn [(run-task [conn]
-            (try
             (when (db/exec-one! conn [sql:lock-scheduled-task (d/name id)])
               (l/debug :action "execute scheduled task" :id id)
-                ((:fn task) task))
+              ((:fn task) task)))
-              (catch Throwable e
-                e)))
 
           (handle-task []
+            (try
               (db/with-atomic [conn pool]
-              (let [result (run-task conn)]
+                (run-task conn))
-                (when (ex/exception? result)
+              (catch Throwable cause
-                  (l/error :cause result
+                (l/error :hint "unhandled exception on scheduled task"
-                           :hint "unhandled exception on scheduled task"
+                         ::l/context (get-error-context cause task)
-                           :id id)))))]
+                         :task-id id
-
+                         :cause cause))))]
     (try
       (px/run! executor handle-task)
       (finally

common/src/app/common/path/bool.cljc

@@ -326,6 +326,9 @@
   [bool-type contents]
   ;; We apply the boolean operation in to each pair and the result to the next
   ;; element
+  (if (seq contents)
     (->> contents
          (reduce (partial content-bool-pair bool-type))
-       (into [])))
+         (into []))
+    []))
+

frontend/src/app/main/data/workspace/libraries.cljs

@@ -317,10 +317,13 @@
   (ptk/reify ::rename-component
     ptk/WatchEvent
     (watch [it state _]
-      (let [[path name] (cph/parse-path-name new-name)
+      ;; NOTE: we need to ensure the component exists, because there
-            component (get-in state [:workspace-data :components id])
+      ;; are small posibilities of race conditions with component
+      ;; deletion.
+      (when-let [component (get-in state [:workspace-data :components id])]
+        (let [[path name] (cp/parse-path-name new-name)
               objects     (get component :objects)
-            ; Give the same name to the root shape
+              ;; Give the same name to the root shape
               new-objects (assoc-in objects
                                     [(:id component) :name]
                                     name)
@@ -336,10 +339,9 @@
                          :name (:name component)
                          :path (:path component)
                          :objects objects}]]
-
           (rx/of (dch/commit-changes {:redo-changes rchanges
                                       :undo-changes uchanges
-                                    :origin it}))))))
+                                      :origin it})))))))
 
 (defn duplicate-component
   "Create a new component copied from the one with the given id."