Skip to content

Commit

Permalink
support large file upload
Browse files Browse the repository at this point in the history
  • Loading branch information
@sifbiri
sifbiri committed Oct 25, 2024
1 parent 97c8767 commit 7e06486
Show file tree
Hide file tree
Showing 5 changed files with 78 additions and 30 deletions.
2 changes: 2 additions & 0 deletions config.namespaced-example.edn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
:triangulum.handler/truncate-request false
:triangulum.handler/private-request-keys #{:base64Image :plotFileBase64 :sampleFileBase64}
:triangulum.handler/private-response-keys #{}
:triangulum.handler/upload-max-size-mb 100
:triangulum.handler/upload-max-file-count 10

;; workers (server)
:triangulum.worker/workers [{:triangulum.worker/name "scheduler"
Expand Down
2 changes: 2 additions & 0 deletions config.nested-example.edn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
:truncate-request false
:private-request-keys #{:base64Image :plotFileBase64 :sampleFileBase64}
:private-response-keys #{}
:upload-max-size-mb 100
:upload-max-file-count 10

;; workers
:workers {:scheduler {:start product-ns.jobs/start-scheduled-jobs!
Expand Down
4 changes: 3 additions & 1 deletion src/triangulum/config_namespaced_spec.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
#(no-keys-of-ns? % "triangulum.server")
:server-keys
(s/keys :req [:triangulum.server/http-port
:triangulum.server/handler]
:triangulum.server/handler]
:opt [:triangulum.server/https-port
:triangulum.server/nrepl
:triangulum.server/nrepl-port
Expand All @@ -37,6 +37,8 @@
:triangulum.handler/private-request-keys
:triangulum.handler/private-response-keys
:triangulum.handler/bad-tokens
:triangulum.handler/upload-max-size-mb
:triangulum.handler/upload-max-file-count
:triangulum.worker/workers
:triangulum.response/response-type])))

Expand Down
2 changes: 2 additions & 0 deletions src/triangulum/config_nested_spec.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
:triangulum.handler/truncate-request
:triangulum.handler/private-request-keys
:triangulum.handler/private-response-keys
:triangulum.handler/upload-max-size-mb
:triangulum.handler/upload-max-file-count
:triangulum.worker/workers
:triangulum.response/response-type]))

Expand Down
98 changes: 69 additions & 29 deletions src/triangulum/handler.clj
View file
Original file line number Diff line number Diff line change
@@ -1,32 +1,33 @@
(ns triangulum.handler
(:require [clojure.data.json :as json]
[clojure.edn :as edn]
[clojure.spec.alpha :as s]
[clojure.string :as str]
[ring.middleware.absolute-redirects :refer [wrap-absolute-redirects]]
[ring.middleware.content-type :refer [wrap-content-type]]
[ring.middleware.default-charset :refer [wrap-default-charset]]
[ring.middleware.gzip :refer [wrap-gzip]]
[ring.middleware.json :refer [wrap-json-params]]
[ring.middleware.keyword-params :refer [wrap-keyword-params]]
[ring.middleware.multipart-params :refer [wrap-multipart-params]]
[ring.middleware.nested-params :refer [wrap-nested-params]]
[ring.middleware.not-modified :refer [wrap-not-modified]]
[ring.middleware.params :refer [wrap-params]]
[ring.middleware.reload :refer [wrap-reload]]
[ring.middleware.resource :refer [wrap-resource]]
[ring.middleware.session :refer [wrap-session]]
[ring.middleware.session.cookie :refer [cookie-store]]
[ring.middleware.ssl :refer [wrap-ssl-redirect]]
[ring.util.codec :refer [url-decode]]
[ring.middleware.x-headers :refer [wrap-content-type-options
wrap-frame-options
wrap-xss-protection]]
[triangulum.config :as config :refer [get-config]]
[triangulum.logging :refer [log log-str]]
[triangulum.errors :refer [nil-on-error]]
[triangulum.utils :refer [resolve-foreign-symbol]]
[triangulum.response :refer [forbidden-response data-response]]))
(:require [clojure.data.json :as json]
[clojure.edn :as edn]
[clojure.spec.alpha :as s]
[clojure.string :as str]
[ring.middleware.absolute-redirects :refer [wrap-absolute-redirects]]
[ring.middleware.content-type :refer [wrap-content-type]]
[ring.middleware.default-charset :refer [wrap-default-charset]]
[ring.middleware.gzip :refer [wrap-gzip]]
[ring.middleware.json :refer [wrap-json-params]]
[ring.middleware.keyword-params :refer [wrap-keyword-params]]
[ring.middleware.multipart-params :refer [wrap-multipart-params]]
[ring.middleware.multipart-params.temp-file :refer [temp-file-store]]
[ring.middleware.nested-params :refer [wrap-nested-params]]
[ring.middleware.not-modified :refer [wrap-not-modified]]
[ring.middleware.params :refer [wrap-params]]
[ring.middleware.reload :refer [wrap-reload]]
[ring.middleware.resource :refer [wrap-resource]]
[ring.middleware.session :refer [wrap-session]]
[ring.middleware.session.cookie :refer [cookie-store]]
[ring.middleware.ssl :refer [wrap-ssl-redirect]]
[ring.util.codec :refer [url-decode]]
[ring.middleware.x-headers :refer [wrap-content-type-options
wrap-frame-options
wrap-xss-protection]]
[triangulum.config :as config :refer [get-config]]
[triangulum.logging :refer [log log-str]]
[triangulum.errors :refer [nil-on-error]]
[triangulum.utils :refer [resolve-foreign-symbol]]
[triangulum.response :refer [forbidden-response data-response]]))

;; spec

Expand All @@ -38,6 +39,14 @@
(s/def ::truncate-request boolean?)
(s/def ::private-request-keys (s/coll-of keyword :kind set?))
(s/def ::private-response-keys (s/coll-of keyword :kind set?))
(s/def ::upload-max-size-mb
^{:doc "Maximum allowed file size in megabytes for uploads.
Must be a positive integer not exceeding 1000MB (1GB)."}
(s/and pos-int? #(<= % 1000)))
(s/def ::upload-max-file-count
^{:doc "Maximum number of files allowed in a single upload request.
Must be a positive integer not exceeding 100 files."}
(s/and pos-int? #(<= % 100)))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Routing Handler
Expand Down Expand Up @@ -173,6 +182,37 @@
(cookie-store {:key (-> (random-string 16)
(string-to-bytes))}))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Upload Configuration
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(def ^:private mb-to-bytes
"Convert megabytes to bytes."
(partial * 1024 1024))

(defn- calculate-progress-interval
"Calculate bytes between progress updates, aiming for ~10 updates."
[max-size-mb]
(mb-to-bytes (max 1 (quot max-size-mb 10))))

(defn- make-upload-config
"Creates upload configuration from config.edn or defaults."
[]
(let [max-size-mb (or (get-config ::upload-max-size-mb) 100)
max-file-count (or (get-config ::upload-max-file-count) 10)]
{:encoding "UTF-8"
:error-handler (fn [_]
(data-response
(str "File upload exceeded limits. Maximum file size is " max-size-mb "MB.")
{:status 413}))
:max-file-count max-file-count
:max-file-size (mb-to-bytes max-size-mb)
:progress-fn (fn [_ bytes-read content-length item-count]
(when (zero? (mod bytes-read (calculate-progress-interval max-size-mb)))
(log (str "Upload progress: " bytes-read "/" content-length
" bytes, items: " item-count))))
:store (temp-file-store)}))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Handler Stack
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
Expand All @@ -195,7 +235,7 @@
wrap-json-params
wrap-edn-params
wrap-nested-params
wrap-multipart-params
(wrap-multipart-params (make-upload-config))
wrap-params
(wrap-session {:store (get-cookie-store)})
wrap-absolute-redirects
Expand Down

0 comments on commit 7e06486

Please sign in to comment.