From 5fc7dd95f7dbdcfd8669de36fe0aa66b94c8d429 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 22 Nov 2016 21:41:12 +0100 Subject: [PATCH] Fix naming consistency on storages module. --- backend/test/storages/tests.clj | 24 +++++++------- .../vendor/storages/{fs => backend}/local.clj | 32 +++++++++---------- .../vendor/storages/{fs => backend}/misc.clj | 14 ++++---- backend/vendor/storages/core.clj | 7 ---- backend/vendor/storages/{util.clj => fs.clj} | 26 +++++++++++++-- backend/vendor/storages/impl.clj | 6 ++-- 6 files changed, 61 insertions(+), 48 deletions(-) rename backend/vendor/storages/{fs => backend}/local.clj (80%) rename backend/vendor/storages/{fs => backend}/misc.clj (91%) rename backend/vendor/storages/{util.clj => fs.clj} (86%) diff --git a/backend/test/storages/tests.clj b/backend/test/storages/tests.clj index c29e26eb0..a10004b34 100644 --- a/backend/test/storages/tests.clj +++ b/backend/test/storages/tests.clj @@ -1,8 +1,8 @@ (ns storages.tests (:require [clojure.test :as t] [storages.core :as st] - [storages.fs.local :as fs] - [storages.fs.misc :as misc]) + [storages.backend.local :as local] + [storages.backend.misc :as misc]) (:import java.io.File org.apache.commons.io.FileUtils)) @@ -21,7 +21,7 @@ ;; --- Tests: FileSystemStorage (t/deftest test-localfs-store-and-lookup - (let [storage (fs/filesystem {:basedir "/tmp/catacumba/test" + (let [storage (local/localfs {:basedir "/tmp/catacumba/test" :baseuri "http://localhost:5050/"}) rpath @(st/save storage "test.txt" "my content") fpath @(st/lookup storage rpath) @@ -30,14 +30,14 @@ (t/is (= "my content" fdata)))) (t/deftest test-localfs-store-and-get-public-url - (let [storage (fs/filesystem {:basedir "/tmp/catacumba/test" + (let [storage (local/localfs {:basedir "/tmp/catacumba/test" :baseuri "http://localhost:5050/"}) rpath @(st/save storage "test.txt" "my content") ruri (st/public-url storage rpath)] (t/is (= (str ruri) "http://localhost:5050/test.txt")))) (t/deftest test-localfs-store-and-lookup-with-subdirs - (let [storage (fs/filesystem {:basedir "/tmp/catacumba/test" + (let [storage (local/localfs {:basedir "/tmp/catacumba/test" :baseuri "http://localhost:5050/"}) rpath @(st/save storage "somepath/test.txt" "my content") fpath @(st/lookup storage rpath) @@ -46,21 +46,21 @@ (t/is (= "my content" fdata)))) (t/deftest test-localfs-store-and-delete-and-check - (let [storage (fs/filesystem {:basedir "/tmp/catacumba/test" + (let [storage (local/localfs {:basedir "/tmp/catacumba/test" :baseuri "http://localhost:5050/"}) rpath @(st/save storage "test.txt" "my content")] (t/is @(st/delete storage rpath)) (t/is (not @(st/exists? storage rpath))))) (t/deftest test-localfs-store-duplicate-file-raises-exception - (let [storage (fs/filesystem {:basedir "/tmp/catacumba/test" + (let [storage (local/localfs {:basedir "/tmp/catacumba/test" :baseuri "http://localhost:5050/"})] (t/is @(st/save storage "test.txt" "my content")) (t/is (thrown? java.util.concurrent.ExecutionException @(st/save storage "test.txt" "my content"))))) (t/deftest test-localfs-access-unauthorized-path - (let [storage (fs/filesystem {:basedir "/tmp/catacumba/test" + (let [storage (local/localfs {:basedir "/tmp/catacumba/test" :baseuri "http://localhost:5050/"})] (t/is (thrown? java.util.concurrent.ExecutionException @(st/lookup storage "../test.txt"))) @@ -70,7 +70,7 @@ ;; --- Tests: ScopedPathStorage (t/deftest test-localfs-scoped-store-and-lookup - (let [storage (fs/filesystem {:basedir "/tmp/catacumba/test" + (let [storage (local/localfs {:basedir "/tmp/catacumba/test" :baseuri "http://localhost:5050/"}) storage (misc/scoped storage "some/prefix") rpath @(st/save storage "test.txt" "my content") @@ -80,7 +80,7 @@ (t/is (= "my content" fdata)))) (t/deftest test-localfs-scoped-store-and-delete-and-check - (let [storage (fs/filesystem {:basedir "/tmp/catacumba/test" + (let [storage (local/localfs {:basedir "/tmp/catacumba/test" :baseuri "http://localhost:5050/"}) storage (misc/scoped storage "some/prefix") rpath @(st/save storage "test.txt" "my content")] @@ -88,7 +88,7 @@ (t/is (not @(st/exists? storage rpath))))) (t/deftest test-localfs-scoped-store-duplicate-file-raises-exception - (let [storage (fs/filesystem {:basedir "/tmp/catacumba/test" + (let [storage (local/localfs {:basedir "/tmp/catacumba/test" :baseuri "http://localhost:5050/"}) storage (misc/scoped storage "some/prefix")] (t/is @(st/save storage "test.txt" "my content")) @@ -96,7 +96,7 @@ @(st/save storage "test.txt" "my content"))))) (t/deftest test-localfs-scoped-access-unauthorized-path - (let [storage (fs/filesystem {:basedir "/tmp/catacumba/test" + (let [storage (local/localfs {:basedir "/tmp/catacumba/test" :baseuri "http://localhost:5050/"}) storage (misc/scoped storage "some/prefix")] (t/is (thrown? java.util.concurrent.ExecutionException diff --git a/backend/vendor/storages/fs/local.clj b/backend/vendor/storages/backend/local.clj similarity index 80% rename from backend/vendor/storages/fs/local.clj rename to backend/vendor/storages/backend/local.clj index a6d964cde..746612e8d 100644 --- a/backend/vendor/storages/fs/local.clj +++ b/backend/vendor/storages/backend/local.clj @@ -4,14 +4,14 @@ ;; ;; Copyright (c) 2016 Andrey Antukh -(ns storages.fs.local +(ns storages.backend.local "A local filesystem storage implementation." (:require [promesa.core :as p] [clojure.java.io :as io] [executors.core :as exec] [storages.proto :as pt] [storages.impl :as impl] - [storages.util :as util]) + [storages.fs :as fs]) (:import java.io.InputStream java.io.OutputStream java.net.URI @@ -20,7 +20,7 @@ (defn normalize-path [^Path base ^Path path] - (if (util/absolute? path) + (if (fs/absolute? path) (throw (ex-info "Suspicios operation: absolute path not allowed." {:path (str path)})) (let [^Path fullpath (.resolve base path) @@ -34,11 +34,11 @@ [base path content] (let [^Path path (pt/-path path) ^Path fullpath (normalize-path base path)] - (when-not (util/exists? (.getParent fullpath)) - (util/create-dir! (.getParent fullpath))) + (when-not (fs/exists? (.getParent fullpath)) + (fs/create-dir! (.getParent fullpath))) (with-open [^InputStream source (pt/-input-stream content) ^OutputStream dest (Files/newOutputStream - fullpath util/write-open-opts)] + fullpath fs/write-open-opts)] (io/copy source dest) path))) @@ -48,7 +48,7 @@ (normalize-path base))] (Files/deleteIfExists ^Path path))) -(defrecord FileSystemStorage [^Path base ^URI baseuri] +(defrecord LocalFileSystemBackend [^Path base ^URI baseuri] pt/IPublicStorage (-public-uri [_ path] (.resolve baseuri (str path))) @@ -65,14 +65,14 @@ (p/resolved (let [path (->> (pt/-path path) (normalize-path base))] - (util/exists? path))) + (fs/exists? path))) (catch Exception e (p/rejected e)))) pt/IClearableStorage (-clear [_] - (util/delete-dir! base) - (util/create-dir! base)) + (fs/delete-dir! base) + (fs/create-dir! base)) pt/ILocalStorage (-lookup [_ path'] @@ -83,7 +83,7 @@ (catch Exception e (p/rejected e))))) -(defn filesystem +(defn localfs "Create an instance of local FileSystem storage providing an absolute base path. @@ -93,12 +93,12 @@ [{:keys [basedir baseuri] :as keys}] (let [^Path basepath (pt/-path basedir) ^URI baseuri (pt/-uri baseuri)] - (when (and (util/exists? basepath) - (not (util/directory? basepath))) + (when (and (fs/exists? basepath) + (not (fs/directory? basepath))) (throw (ex-info "File already exists." {}))) - (when-not (util/exists? basepath) - (util/create-dir! basepath)) + (when-not (fs/exists? basepath) + (fs/create-dir! basepath)) - (->FileSystemStorage basepath baseuri))) + (->LocalFileSystemBackend basepath baseuri))) diff --git a/backend/vendor/storages/fs/misc.clj b/backend/vendor/storages/backend/misc.clj similarity index 91% rename from backend/vendor/storages/fs/misc.clj rename to backend/vendor/storages/backend/misc.clj index 4dab9bf1f..164b0d30d 100644 --- a/backend/vendor/storages/fs/misc.clj +++ b/backend/vendor/storages/backend/misc.clj @@ -4,7 +4,7 @@ ;; ;; Copyright (c) 2016 Andrey Antukh -(ns storages.fs.misc +(ns storages.backend.misc "A local filesystem storage implementation." (:require [promesa.core :as p] [cuerdas.core :as str] @@ -14,7 +14,7 @@ [buddy.core.hash :as hash] [storages.proto :as pt] [storages.impl :as impl] - [storages.fs.local :as localfs]) + [storages.backend.local :as local]) (:import java.io.InputStream java.io.OutputStream java.nio.file.Path @@ -22,7 +22,7 @@ ;; --- Scoped Storage -(defrecord ScopedPathStorage [storage ^Path prefix] +(defrecord ScopedBackend [storage ^Path prefix] pt/IPublicStorage (-public-uri [_ path] (let [^Path path (pt/-path [prefix path])] @@ -49,7 +49,7 @@ (p/map (fn [^Path base] (let [base (pt/-path [base prefix])] (->> (pt/-path path) - (localfs/normalize-path base)))))))) + (local/normalize-path base)))))))) (defn scoped "Create a composed storage instance that automatically prefixes @@ -60,7 +60,7 @@ uploads." [storage prefix] (let [prefix (pt/-path prefix)] - (->ScopedPathStorage storage prefix))) + (->ScopedBackend storage prefix))) ;; --- Hashed Storage @@ -78,7 +78,7 @@ frest (apply str rest-tokens)] (pt/-path (list path frest name)))) -(defrecord HashedStorage [storage] +(defrecord HashedBackend [storage] pt/IPublicStorage (-public-uri [_ path] (pt/-public-uri storage path)) @@ -107,5 +107,5 @@ This is usefull when you want to store files with not predictable uris." [storage] - (->HashedStorage storage)) + (->HashedBackend storage)) diff --git a/backend/vendor/storages/core.clj b/backend/vendor/storages/core.clj index 01aad5fcf..f817801be 100644 --- a/backend/vendor/storages/core.clj +++ b/backend/vendor/storages/core.clj @@ -42,13 +42,6 @@ [storage] (pt/-clear storage)) -(defn path - "Create path from string or more than one string." - ([fst] - (pt/-path fst)) - ([fst & more] - (pt/-path (cons fst more)))) - (defn public-url [storage path] (pt/-public-uri storage path)) diff --git a/backend/vendor/storages/util.clj b/backend/vendor/storages/fs.clj similarity index 86% rename from backend/vendor/storages/util.clj rename to backend/vendor/storages/fs.clj index b418c050d..7e47e44d8 100644 --- a/backend/vendor/storages/util.clj +++ b/backend/vendor/storages/fs.clj @@ -4,15 +4,17 @@ ;; ;; Copyright (c) 2016 Andrey Antukh -(ns storages.util - "FileSystem related utils." +(ns storages.fs + "File System helpers." (:refer-clojure :exclude [name resolve]) (:require [storages.proto :as pt]) (:import java.nio.file.Path java.nio.file.Files java.nio.file.LinkOption java.nio.file.OpenOption + java.nio.file.CopyOption java.nio.file.StandardOpenOption + java.nio.file.StandardCopyOption java.nio.file.SimpleFileVisitor java.nio.file.FileVisitResult java.nio.file.attribute.FileAttribute @@ -22,7 +24,7 @@ ;; --- Constants (def write-open-opts - (->> [#_StandardOpenOption/CREATE_NEW + (->> [StandardOpenOption/TRUNCATE_EXISTING StandardOpenOption/CREATE StandardOpenOption/WRITE] (into-array OpenOption))) @@ -31,6 +33,11 @@ (->> [StandardOpenOption/READ] (into-array OpenOption))) +(def move-opts + (->> [StandardCopyOption/ATOMIC_MOVE + StandardCopyOption/REPLACE_EXISTING] + (into-array CopyOption))) + (def follow-link-opts (into-array LinkOption [LinkOption/NOFOLLOW_LINKS])) @@ -128,6 +135,13 @@ ;; --- Side-Effectfull Operations +(defn create-dir! + "Create a new directory." + [path] + (let [^Path path (pt/-path path) + attrs (make-file-attrs "rwxr-xr-x")] + (Files/createDirectories path attrs))) + (defn create-dir! "Create a new directory." [path] @@ -146,3 +160,9 @@ (Files/delete dir) FileVisitResult/CONTINUE))] (Files/walkFileTree path visitor))) + +(defn create-tempfile + "Create a temporal file." + [& {:keys [suffix prefix]}] + (->> (make-file-attrs "rwxr-xr-x") + (Files/createTempFile prefix suffix))) diff --git a/backend/vendor/storages/impl.clj b/backend/vendor/storages/impl.clj index 2ffdc0d70..cda035e3c 100644 --- a/backend/vendor/storages/impl.clj +++ b/backend/vendor/storages/impl.clj @@ -7,7 +7,7 @@ (ns storages.impl "Implementation details and helpers." (:require [storages.proto :as pt] - [storages.util :as util] + [storages.fs :as fs] [buddy.core.codecs :as codecs] [clojure.java.io :as io]) (:import java.io.File @@ -79,11 +79,11 @@ (defn- path->input-stream [^Path path] - (Files/newInputStream path util/read-open-opts)) + (Files/newInputStream path fs/read-open-opts)) (defn- path->output-stream [^Path path] - (Files/newOutputStream path util/write-open-opts)) + (Files/newOutputStream path fs/write-open-opts)) (extend-type Path io/IOFactory