Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Storage] Add storage user #539

Merged
merged 3 commits into from
Nov 26, 2024
Merged

[Storage] Add storage user #539

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0ccb55b
[Storage] Add storage user
nezaj Nov 25, 2024
7590603
code review
nezaj Nov 26, 2024
90697e8
Update old storage example
nezaj Nov 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions server/resources/config/dev.edn
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@
:json "{\"encryptedKeyset\":\"AQICAHhpp/gyLqbSj/AUco4gftFgpQ99AbOx+UEx1DTG6st//AGkXTx5+GQplpY1CB0Ofh/zAAABAjCB/wYJKoZIhvcNAQcGoIHxMIHuAgEAMIHoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDEazOfwcJQYLMycv8gIBEICBuj/ddFoWosiu5V/usv/tHg+tsG1DHE5QZSAbihIGtzptpb9lHV1H+H5sLJ3dajeWBDfSkSm7WR7LGbZQbYQEyV7UamkjtWZ1X91inEzbceEph0lvVUyfP4vDTqJ1TwKrTea5vAP/rdZQBCIfk1i+oli9lsCkqsXuFXcTP8ZIUE3lopCKMeezT7KsqRRyMaFkgXg+duTZvOLzMTbwWZh6Xsg/u2Hv4RY15fLhyKRRnraOeaaEsxm58lTfBQ==\",\"keysetInfo\":{\"primaryKeyId\":1980914119,\"keyInfo\":[{\"typeUrl\":\"type.googleapis.com/google.crypto.tink.HpkePrivateKey\",\"status\":\"ENABLED\",\"keyId\":1980914119,\"outputPrefixType\":\"TINK\"}]}}"
:public-key-json "{\"primaryKeyId\":1980914119,\"key\":[{\"keyData\":{\"typeUrl\":\"type.googleapis.com/google.crypto.tink.HpkePublicKey\",\"value\":\"EgYIARABGAIaIBRh06KLJyUSH0NhTIfOdcRStRQUNqq2n69jUR5P3CU0\",\"keyMaterialType\":\"ASYMMETRIC_PUBLIC\"},\"status\":\"ENABLED\",\"keyId\":1980914119,\"outputPrefixType\":\"TINK\"}]}"}

:s3-storage-access-key {:enc "01761259c7282d2eae245762ed962d31c2ad90d62cd542a3d9db59fdedb7e1728d29e5f40c60f0f0fb6d9d5435823e2f5eb2c9f86677b76c4fb5333e4a545983a223abe246ce5275ff"}

:s3-storage-secret-key {:enc "01761259c74e2ec5eae423593e9c9aaf0aacb9aa9f6de632ce8aa7e4903c6e0b44af5b2f38a09613dc90e1de034f3c92ba50d252266e12a0c2bdeb86ddb894c77710e1fee02136448b9ffde5748d9735b5cf09820ca2774d0b385dca1f"}

:postmark-token {:enc "01761259c7a2c9e6fe9aaafaaa9ef753dac678962ea02a05c6415a87a3d868934132e06a281d351b1b23f5894b5dbfebc4ef9c02844051de78cc6464d903ee93300d906e62ca815d02062789a846eff55c866b3ff2dfda7079"}

:postmark-account-token {:enc "01761259c72b84fed42432ed258cd9b6db8dc12a0ede11e85350e54511fc54ca865a4f9c5aa7dc9956af5fad4f0ef07c5cc1f71df646cbf664ec2ad14601d92431153375106c5c038d2235f1743b02ffb695777be5648bbc12"}
Expand Down
4 changes: 4 additions & 0 deletions server/resources/config/prod.edn
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@
:json "{\"encryptedKeyset\":\"AQICAHhpp/gyLqbSj/AUco4gftFgpQ99AbOx+UEx1DTG6st//AFoN+9a7FNl1rjvox4RwL97AAABAjCB/wYJKoZIhvcNAQcGoIHxMIHuAgEAMIHoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDN1Uv+fy9Ru9r+3mTQIBEICBur72urR4FwbzDJ9IVe3q8Uv/uqpoBRTj3CsQnlHk4U97MjLf1W3oYVE2jdM0WOpLdvWhZfdWTcPhpgonP6CyyoHspFZ8MC8LrSheC8AVX6CRBEQSbNLiJ11Mbcjq6MHgH9pwxZAgmWdY7MJgmOqJw6+VA9otMG2LA4ssnIypuqewdLHWHdhp1EQNmfF5dKxcLe+OL/ee9GLfokI+AP/a91DeBHFo+32Co1ECjEd9CHIK/f3AhCVPtPvF2A==\",\"keysetInfo\":{\"primaryKeyId\":1349240052,\"keyInfo\":[{\"typeUrl\":\"type.googleapis.com/google.crypto.tink.HpkePrivateKey\",\"status\":\"ENABLED\",\"keyId\":1349240052,\"outputPrefixType\":\"TINK\"}]}}"
:public-key-json "{\"primaryKeyId\":1349240052,\"key\":[{\"keyData\":{\"typeUrl\":\"type.googleapis.com/google.crypto.tink.HpkePublicKey\",\"value\":\"EgYIARABGAIaIML+zPIaSSQvRzuOODD1ZKz/9QWHXj3Y/+A5btFCwxpz\",\"keyMaterialType\":\"ASYMMETRIC_PUBLIC\"},\"status\":\"ENABLED\",\"keyId\":1349240052,\"outputPrefixType\":\"TINK\"}]}"}

:s3-storage-access-key {:enc "01506bc4f451bbb565bfcb437ae2c305c1a3170c189c29c3e8cceca363a09b0d66dd26c5769b03e7351606075de558a8417b88a39f8640fdd0fd29faf4f3bea6f71646a7e3b138b452"}

:s3-storage-secret-key {:enc "01506bc4f4972c313c0bf2ec5be9f42a7f7389e4866ada94ae131b2cc5aa3c4ccd8007c876070813e1ce5d63485ee54d86471548322bc3af5162b0c45713cc9047dee6b5aea2c52181a37b323d3438b76266274afaccce01a935d00e62"}

:postmark-token {:enc "01506bc4f4f166d70857fd4d420ef1405c0b2d5a30cabaf073b1aa608d3618f4faca8bc168b7495f9175509e06dfa7d52f48ba76408cd346e562cab0afb1cf1b040caf358a3025fa37a450c6a6a387ddc463049d0f2b41031d"}

:postmark-account-token {:enc "01506bc4f4c67e78161ce519cf3bf14272614d38a7931e5606406e59a488084f42449f2401c89976d49d0d9d8c72745ac1991bb9d9c30f302067514a92491639fedc74aa34fb92508359255853d114a7b5f3f8fb3a3e6355de"}
Expand Down
6 changes: 6 additions & 0 deletions server/src/instant/config.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@
(defn instant-config-app-id []
(-> @config-map :instant-config-app-id))

(defn s3-storage-access-key []
(some-> @config-map :s3-storage-access-key crypt-util/secret-value))

(defn s3-storage-secret-key []
(some-> @config-map :s3-storage-secret-key crypt-util/secret-value))

(defn postmark-token []
(some-> @config-map :postmark-token crypt-util/secret-value))

Expand Down
6 changes: 6 additions & 0 deletions server/src/instant/config_edn.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
(s/def ::oauth-client (s/keys :req-un [::client-id
::client-secret]))

(s/def ::s3-storage-access-key ::config-value)
(s/def ::s3-storage-secret-key ::config-value)
(s/def ::postmark-token ::config-value)
(s/def ::postmark-account-token ::config-value)
(s/def ::secret-discord-token ::config-value)
Expand All @@ -50,6 +52,8 @@
::public-key-json]))

(s/def ::config (s/keys :opt-un [::instant-config-app-id
::s3-storage-access-key
::s3-storage-secret-key
::database-url
::postmark-token
::postmark-account-token
Expand All @@ -64,6 +68,8 @@
;; Prod config is more restrictive because we don't want to accidentally
;; forget to set one of these variables in prod
(s/def ::config-prod (s/keys :req-un [::aead-keyset
::s3-storage-access-key
::s3-storage-secret-key
::database-url
::postmark-token
::postmark-account-token
Expand Down
23 changes: 15 additions & 8 deletions server/src/instant/storage/s3.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
(ns instant.storage.s3
(:require [clojure.java.io :as io]
[clj-http.client :as clj-http]
[amazonica.aws.s3 :as s3]))
[amazonica.aws.s3 :as s3]
[instant.config :as config]))

(def default-bucket "instant-storage")

Expand Down Expand Up @@ -54,19 +55,25 @@
(defn signed-upload-url
([object-key] (signed-upload-url default-bucket object-key))
([bucket-name object-key]
(s3/generate-presigned-url {:method :put
:bucket-name bucket-name
:key object-key})))
(s3/generate-presigned-url
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we make this optional so that people who get this working on OSS won't have to set up the access keys if they don't want to?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done!

{:access-key (config/s3-storage-access-key)
:secret-key (config/s3-storage-secret-key)}
{:method :put
:bucket-name bucket-name
:key object-key})))

(defn signed-download-url
([object-key] (let [expiration (+ (System/currentTimeMillis) (* 1000 60 60 24 7))] ;; 7 days
(signed-download-url default-bucket object-key expiration)))
([object-key expiration] (signed-download-url default-bucket object-key expiration))
([bucket-name object-key expiration]
(s3/generate-presigned-url {:method :get
:bucket-name bucket-name
:key object-key
:expiration expiration})))
(s3/generate-presigned-url
{:access-key (config/s3-storage-access-key)
:secret-key (config/s3-storage-secret-key)}
{:method :get
:bucket-name bucket-name
:key object-key
:expiration expiration})))

(defn upload-image-to-s3
([object-key image-url] (upload-image-to-s3 default-bucket object-key image-url))
Expand Down