From bd97abf9975eaf247ec1cf64e048703473950f5a Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sun, 3 Apr 2022 03:46:45 +0200 Subject: [PATCH] next big update: - resuming interupted uploads (closes #1) - dedup of already existing files - multi-part uploads - configurable expiration time - UTF-8 support - copy URLs to clipboard (closes #12) - a ton of other smaller fixes --- README.md | 28 +- cmd/main.go | 5 +- config.yaml | 4 +- frontend/css/chart.scss | 17 ++ frontend/css/custom-file-button.scss | 23 ++ frontend/css/index.scss | 102 +++++--- frontend/css/variables.scss | 1 - frontend/img/gose-logo.png | Bin 0 -> 8491 bytes frontend/index.html | 191 +++++++------- frontend/package-lock.json | 376 ++++++++++++++++++++++----- frontend/package.json | 11 +- frontend/src/api.ts | 6 +- frontend/src/chart.ts | 110 ++++++++ frontend/src/checksum.ts | 47 ---- frontend/src/config.ts | 6 +- frontend/src/dropzone.ts | 39 +++ frontend/src/file.ts | 3 - frontend/src/index.ts | 259 ++++++++++-------- frontend/src/progress-handler.ts | 69 +++-- frontend/src/upload.ts | 292 +++++++++++++++------ frontend/src/utils.ts | 36 ++- frontend/webpack.config.js | 16 +- go.mod | 3 +- go.sum | 4 +- pkg/config/config.go | 7 +- pkg/handlers/complete.go | 78 ++++-- pkg/handlers/config.go | 6 +- pkg/handlers/download.go | 48 ++-- pkg/handlers/initiate.go | 213 ++++++++------- pkg/handlers/part.go | 75 ++++++ pkg/handlers/types.go | 9 + pkg/notifier/notifier.go | 40 +-- pkg/server/server.go | 6 +- pkg/utils/etag.go | 26 ++ 34 files changed, 1508 insertions(+), 648 deletions(-) create mode 100644 frontend/css/chart.scss create mode 100644 frontend/css/custom-file-button.scss delete mode 100644 frontend/css/variables.scss create mode 100644 frontend/img/gose-logo.png create mode 100644 frontend/src/chart.ts delete mode 100644 frontend/src/checksum.ts create mode 100644 frontend/src/dropzone.ts delete mode 100644 frontend/src/file.ts create mode 100644 pkg/handlers/part.go create mode 100644 pkg/handlers/types.go create mode 100644 pkg/utils/etag.go diff --git a/README.md b/README.md index da22913..8023130 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@

GoSƐ logo -

GoSƐ - A tera-scale file-uploader

+

GoSƐ - A terascale file-uploader

@@ -18,20 +18,28 @@ GoSƐ is a modern file-uploader focusing on scalability and simplicity. It only ## Features +- De-duplication of uploaded files based on their content-hash + - Uploads of existing files will complete in no-time without re-upload +- S3 Multi-part uploads + - Resumption of interrupted uploads +- Drag & Drop of files +- Browser notifications about failed & completed uploads +- User-provided object expiration/retention time +- Copy URL of uploaded file to clip-board +- Detailed transfer statistics and progress-bar / chart - Installation via single binary or container + - JS/HTML/CSS Frontend is bundled into binary - Scalable to multiple replicas - - No other backend services apart from S3 storage are required -- Upload progress-bar and transfer statistics -- Direct upload to Amazon S3 via presigned-URLs -- Direct download from Amazon S3 -- Drag & Drop of files -- Multi-part / chunked upload -- File integrity checks after finished upload via using MD5 checksum & ETags + - All state is kept in the S3 storage backend + - No other database or cache is required +- Direct up & download to Amazon S3 via presigned-URLs + - Gose deployment does not see an significant traffic +- UTF-8 filenames +- Multiple user-selectable buckets / servers - Optional link shortening via an external service - Optional notification about new uploads via [shoutrrr](https://containrrr.dev/shoutrrr/v0.5/) - Mail notifications to user-provided recipient -- Browser notifications about failed & completed uploads -- User-provided object expiration/retention time + ## Roadmap diff --git a/cmd/main.go b/cmd/main.go index 413bf5d..587a3d9 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -74,9 +74,10 @@ func run(cfg *config.Config) { router.GET(apiBase+"/config", handlers.HandleConfig) router.POST(apiBase+"/initiate", handlers.HandleInitiate) + router.POST(apiBase+"/part", handlers.HandlePart) router.POST(apiBase+"/complete", handlers.HandleComplete) - router.GET(apiBase+"/download/:server/*key", handlers.HandleDownload) - router.HEAD(apiBase+"/download/:server/*key", handlers.HandleDownload) + router.GET(apiBase+"/download/:server/:etag/:filename", handlers.HandleDownload) + router.HEAD(apiBase+"/download/:server/:etag/:filename", handlers.HandleDownload) server := &http.Server{ Addr: cfg.Listen, diff --git a/config.yaml b/config.yaml index d345d0a..fdc322a 100644 --- a/config.yaml +++ b/config.yaml @@ -57,7 +57,9 @@ notification: File: {{.FileName}} Size: {{.FileSizeHuman}} Type: {{.FileType}} - IP: {{.UploaderIP}} ({{.UploaderHostname}}) + Uploaded at: {{.UploadDate.Format "Jan 02, 2006 15:04:05 UTC"}} + Uploaded by: {{.UploaderIP}} ({{.UploaderHostname}}) + Expires at: {{.ExpiryDate.Format "Jan 02, 2006 15:04:05 UTC"}} ({{.ExpiryRuleID}}) # For user notifications mail: diff --git a/frontend/css/chart.scss b/frontend/css/chart.scss new file mode 100644 index 0000000..c016ab9 --- /dev/null +++ b/frontend/css/chart.scss @@ -0,0 +1,17 @@ +$color-chart-fill: #f6bad1; +$color-chart-stroke: #eb71a0; + +#chart { + width: 100%; + height: 100%; + + svg { + path { + fill: $color-chart-fill; + stroke: $color-chart-stroke; + stroke-width: 0.4; + fill-opacity: 0.15; + stroke-opacity: 0.8; + } + } +} diff --git a/frontend/css/custom-file-button.scss b/frontend/css/custom-file-button.scss new file mode 100644 index 0000000..4d85c58 --- /dev/null +++ b/frontend/css/custom-file-button.scss @@ -0,0 +1,23 @@ +.custom-file-button { + input[type="file"] { + margin-left: -2px !important; + + &::-webkit-file-upload-button { + display: none; + } + + &::file-selector-button { + display: none; + } + + height: 52px; + line-height: 38px; + } + + &:hover { + label { + background-color: #dde0e3; + cursor: pointer; + } + } +} diff --git a/frontend/css/index.scss b/frontend/css/index.scss index 5c41e8a..c2df68d 100644 --- a/frontend/css/index.scss +++ b/frontend/css/index.scss @@ -1,4 +1,4 @@ -@import "variables.scss"; +$color-dragzone: #eb71a0; .dropzone { box-sizing: border-box; @@ -14,52 +14,96 @@ } .logo { - margin-left: 2em; - width: 200px; + margin-left: 2em; // Aligns goose feet with title text + max-width: 40%; } .title { - margin-top: 1em; font-family: "Lucida Console", Monaco, monospace; font-weight: bolder; } -.subtitle { - margin-bottom: 2em; +#copy { + margin-right: 0.5em; + cursor: pointer; } -.progress { - margin-top: 1.5em; - height: 20px; -} +@import "chart.scss"; +@import "custom-file-button.scss"; -#statistics table { - margin-bottom: 0; -} +$container-max-widths: ( + sm: 540px, + md: 600px, + lg: 640px, + xl: 680px +); -.custom-file-button input[type=file] { - margin-left: -2px !important; -} +$primary: #eb71a0; +$secondary: #73D2DE; +$success: #285943; +$info: #e5e5e5; +$warning: #773344; +$danger: #f22775; +$light: #eeeeee; +$dark: #23395B; +$warning: #f5ee81; -.custom-file-button input[type=file]::-webkit-file-upload-button { - display: none; +$spinner-height: 1em; +$spinner-width: 1em; +$spinner-animation-speed: 1s; + +@import "~bootstrap/scss/bootstrap"; + +.alert-success { + a { + color: $success; + } } -.custom-file-button input[type=file]::file-selector-button { - display: none; +#reset { + svg { + width: 2em; + height: 2em; + } + + color: lightgray; } -.custom-file-button:hover label { - background-color: #dde0e3; - cursor: pointer; +#reset:hover { + color: darkgray; } -@import "~bootstrap/scss/bootstrap"; +#statistics { + .progress { + height: 2em; + } -header { - margin-top: 4rem; -} + .progress-bar { + font-weight: 600; + } + + table { + margin-bottom: 0; + + th { + padding-left: 0; + font-weight: 500; + } + + tr > *:not(:first-child) { + text-align: right; + } + + th, td { + width: 25%; + padding-top: 0px; + padding-bottom: 0px; + } -.row { - margin-top: 1rem; + tr:last-child { + th, td { + border: 0; + } + } + } } diff --git a/frontend/css/variables.scss b/frontend/css/variables.scss deleted file mode 100644 index b41ac5d..0000000 --- a/frontend/css/variables.scss +++ /dev/null @@ -1 +0,0 @@ -$color-dragzone: #eb71a0; diff --git a/frontend/img/gose-logo.png b/frontend/img/gose-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..1d7618edb4d2d2817b7708ed3e6e1de40914c187 GIT binary patch literal 8491 zcmZX4byQqUllL&_5Zri@ROku_!eF zKo3xqk<|9hKU(!mC;PdG^=G%Pdd)W4K*C$&)sTs{v1K3Srn3r}O2W&HAzl#d^;gJ^ zm}(7gKSmDP4<2gs%nr*08B9%0&8;|`ddgDhd+015EI{2y@8{hEFM9L%%xkTm?Ok_| z6Tf`>j;paNi|UW&xFk}`rJ*>+a5l|@QkgmhcQFE}+liy|7?bqmOEMR$sQARn#+rBb zi{&YMPO;j-yOE?4`d9;R1jdc-!^rcFjEkTa)YXWi>EP*@U<$sXPXN}Jt|XyhfR|mq z%3Ps!fDqMUb--Wog zOsE)Dxv@&|Ii)B?sAIpSU|19IE02s3V*{SQuVD@57}?HR0MGZUh(sTNnC(ds@zz5Q z!c1&#;?mKZ+n9vYcdS&B_9qL8FK#F)T4=v+y(Ke_Aev_y=PYUUrQYjF*pq3B$R;Lk zewk4x;}=XI+CMbGPnuXsi%*kOzN^PxVNTmHs7W{yuOSQqzBY( zP6W4{yPREZY$5$W6G4U{MHgk)$CgtN>?h%HC!UfOuqp#TA16W`7YLXjE~t=3$03v* zdB~EyHqqgPtLTj$_wIfCdQV8ZgdMKx6DHo+x;Al%vKP9Px?d(4Xm^)6{2`a>K6EGP zzUrxGmRcryasB=aw^)C~4Mpg<^dswOG^8B?8#=~|TjfkSY`L3@c7J66y#A zq{tQ;{6qhpDUyB@#?lDYif zDiLUS^|hCJJ)Je4(;&uh4B!^O386Tqxwc+I2}XW88ZtJgTq|U`?BSyEk%&)SWivds zg-llu`ynHQ%^vg=OIK*S%->OeQI_C-{J4;#leN1ZJDl1rKJ$(I<*E0rxcU#=-BmwLUQ#{pJH~-;kaiyR;W@+OBo;IRjC6AcS-?2w2SZGZ&@@#o z1CyNFVG{USVQO6WQWWtMaQ5Uwmv{pnkm6sD)`PdC zP4Ej2TABkUL=w#J(L+CeoPPtXCp^xm7XOj1!B=M4s&*k38ZT?7m{MbsQ`mkrZYZ>} zkEw?-6hLlbf~x`bk#|kM6s((=A~%qVMkP#v97hkPdFvh&r&Gn^C9 zA!Fx;Lp~l-cd9pj{5|jaO3;V`k$932Hs7a1rG-;^`UXm!NJG2 z;+Q~NXv4|jS|8;Q5OkBW3MP%V_@(?5Y8Xw?-5G0CTp`m;xKpG&5GfaX@exbM4 z-}e+Z@j)dUR#D&TZ>sl%I6RII9+g2H9G#KRk@%zJtfVa7SLjrj_o{m)mH#8!-Wa%F z`|$K>IRM?Nr3sm|9ew@UI1rhs5t=(QO4S@1B%^J()v(n5&%xVL_A7crAt!z7J^3I5EK*$qGFIp22!XeMcn)10ZL(?0*zxlhwT7QZXFgVW%O58h7Pnb^ zr40>+!lONlo9e9OfHAwm&&6yDi$3YtAFO4??)cnzrcLR5CrbI8G0!uRQQdp zIhJI$_@C;6H|_@M=E3c2w@zu>{Z~vr@^|$5Z+Pis+$MJ*5CX}?=GD(-f_(m*fP#7| zxl_!o^W4=dBv&)XIX?&S{<#PI7Ljy$0~@yY6FqUzSfA7nnNVr}8-j4k zfuNKsFFZ4PP!2S0?FIE+$+Ig5T|2EkbK%$2a`9Fql$ED4PK}j5G|{wv;=Z zt!8CPQ$1X0AvSWHD`Rh^;d++{Ta1`2h=rQbVeKXlP2Ds7a=Vd{Kbn5NhTl+oS|>6; zAWKbec=+;fk{M>^H&M1?&gKZW>QlY%%&wh#wBEcb=m+h+(O!AFC0t80M{mjBxFR>S zV75Ox|M)eNq{D(WbJ;@%%^0U@!srPr&0=gO;3R$lF7&qyn}lYX?u8 zt1%Z2!e90U2e-~j*yjDxa44;vjhDZDvdJE6@9Uo(=ZpAQ zCfn6pg90=T8g3zcW}NV76S1XxtlZ^&E~CN`thL8YVHm5^o{fs=Ar(RC5y-gWd9g_P zPUht9dQg9IS3n8WLeM*dTP&@gT}0Exs1>s)=%Hpa@30A_o0fuiNJ}W%cApMfA7-bM=G3mAKoEf;Tj2B zSqhE5MXipH4D*~eAd#k@h_7~1e$;&us7PIS9r-V3_S79KQ`(9O!WMXn?iSAzNtu9A zeC#|~>wf)*eZi9k=@3zz@R2bta&zWGT?);8YM+HRr`4F^Ca?B)V<(AHJ$<0$AbO(J zC3U}$--nChq29Z-3bj&uxF>G@2Z>Dg^=hWD=`KumIUCG=L)!(;vlcRdiNne{)@xwdfm_+F5_P zcS2TbY;`ZMhhNpCut?v$`iZaVwd`$)1qdv3TFzXh!DWBAZ|5|Jp|45kqeDtw6T-fn+Ngy_B5DP#*&}EvHE)wEC5Ur z0}}ip(E6BM!0DCxx`l#3Fzh+n#q$ElG!}AEFm7yjLMn-8y>0tCwaeh>a-2DzY4OzpMXl z{%ti1Wo`sHdwq9AY%y235v%;>SS`NmcQW!*)D2L=O4g28QHMW4cwm%fVmXo46%W@V zTEEwr^OLFj%I-}l$xE=19WF7;4}6D$yak1;|JtBeRwD2=oTd%(%`b}6V4t4Y#xi%^ z>^57cpX;}Y)^zIl=zB(mg99yhQjE%|vWYMBCKNU3jQIaPze<)XXVWdOzCuL<4+zwgsxvjQhT%PX=;hh{? z&lHX?%q6=PrDi-j8zx!u9$;)k(P(Ou_ax1A5_|P2Zq+%C) zJJ8Ioi)YgEqnC?;(=$+=$98JGtAuN8zfwH))p_k9he=>1a2q@fyc(|XhaG2g{@H3m zeEm~eR>HPgFi1~^#2X?^`a+VG*uzeRiv##vitcH<-pFtH)pI%~XM#`Xp>CD0I%F;6 z5vhI@Kj+h*T7A%e4|wypP(p;vns6Y3v|dh|&}#X(VQPv*=4}UYr33w7S~%2<8|zXW zXkaSB?XnQNl_$q7d6kh8rrBg_2pMRKa;&fCs>4O@I^90Q1YRX!B+a7gq+m_h4jQzG zY8{*Vd-w-gf$t=2fUvgMiK(2RFJ%~C17wbDb{0S5Gnp>_ZV;N-H125&kh^Tkxoo8l zg%*1PY}Xj!(spEky$<>p2ZLmU0;ZTj`Ueg9tnLx`*rv0N_c{nZ?Q?WST278=q%oco zML`K4F5m~@TiW-%U_l4G@fe9^T4-Ij&#p@~sed}%grbZP-g$G2)Ne zG%>skO+tG4%GSOVjd0O-z_f!0f z_qpW@6_^1FI);9o1An7c5A+EuStVO*iw|@8G4{_R()lZNSi`;_fIRU5-&R{$gj1Os zh=E`usmRW$A)3)-EH}W0L>E``$+d)iT;X$kiIGMcMHU|}^+m`KBWj_^e#yQe7c zgpD~cJ6SD_W&?6iOAapPqz1q~(M!Jg9Y)_p(w9*V$StMcYjBdiL-PQK!osOR`I5!7 zSi5rq8?K2Pn5@nO9_)}FcDRJzQhS7qEbt@OB%txclFUmx(Po?Z8S!uv#0eqO?|R(m zqqr}!ig&k04lYI6Alk@SF8InQC^-VrNCHBlCAdSEDQkW1eUd?CVas-%>9_lXrP!?e z@dVQuNmf$cD@ehrsFhIf4y<`L@+CI>B@3m=-Ap_QLTXqWWsI{xu?x(+ZM0#`)b+5Q zt+71o;8)o>1dyDnvqQOo|-&} z9ojH(S#Us!Z;%xr*^6iG(5*VL69Mv0CEebvtEMyk`6Ql^XYvIyu!=ev=l zSM*zqJfa`t^JY?4LQkTdJ9^7XwnvB(3_Eo&`o503Pc!(>G|`ME<=wLwq-3RdNk>fq zkJV{P>38RIT#JwiqVMat**GaqV-7o7vr>L_IBcY%-QX zIXgwDvCoKKhNO10igep`{-R1Fuv7h=^LVYMoO@1-3$Q0sjSI4ye#w0yG2;E`X72Nn z`%;Qa=@=ocPWDeNSpfqbE&yrcvW!d5EEHen5dD_DiJU<&MR+7{6rEh~Vsh;)Zjb2H zejSXDp#tnB0}oXV%MW*p`uRu`R^sdadd+l$(11%vxPSbPv!3>VAQZ$={^gpJ3ZuP<818Y?RkaFBj(Qnr|1!9C{A)5X#&od%p6Gb#gmK|3lWsYmuLAQLR(I zeN;Sc5e-t062^FTGg#Hb9YS6%pD*%{qf&Di?-oKWK3#;AN?#1MPqKREoy6m9zEYxK zs9AWX^h^0I(iwrzAuIaYbpomSuS_oq#p6xa5ugose3Md%h2a4HBE36Z#1Ng!KMGivMsn8q{{MzmygZaB+e3fL)Nym_X+?S~k6TPZeDO|Ea8b{uKS(R>@%D-VVx4Bn zy!ivlfL1@v5x9OWAtI_O7+|6iuf4aN>e#ABs^4SciNi_zM#}wwhp{>e@D~Hwgy{#k z266jNdP;-o+XD+q@TNhV(lXa%=_W1Go&=nE7 zQ*KBbpyq<9huN-Mh4#{&dqldw?^`L)x9GbV`wecB0Q0Wo^;@qxl4nyN*ANjisUIUS zIKVyTa-%AZ?vqsSxG<<+NX+Ep7V+iIFF^94ICs%NN*vr)2K_s){g=>x;?r7W- z1RjwA=lV6=_oHnob<{2!2I=?fsz9=9V6#tQp`L1uXwE8MktCv(R5A!&*j`qtw0_-K zO>30pqpF3C60jxo70?FgSPvZ4E8Zdf@re5g0jyI{JmjOBNmxYQlaMfJXz1pMx%FG$7nT#z50}^jNf>B+xir3OVEo-bdMIx3+S4cq) z{_41ztjGAsnN8Pbttxld{%%P2Sex>5(VKBwZ+|qeMn4k{3ZgxWt|EI6q$VJs*3bZH z1un_b+9}r^g|!LY88~8$b3+kV($m6+Gw(pyuSM&~6F4~aDE+zK|H)0imr^$TaM`to zvs#>dwU?+UF4ijYy3kM????=gDTZj=E_xRexQ~1hNX;Ae$wPU;wy&EdqaNg4`Hf_U zr^BB{BGVb)0?4H-731uA<*WgZah|ehgJ1c&?({WR6~z!(v}bcS=fG^y8!rx0%OpH> zI7`jO7~{xp+xZCbDSay&o#OaeF~QuGB;KNy`=A=V3F6{T4ve5L&{CXRZ}eNeq!sej zox~~pQfX$T@j+FEUjgQ)b3}lSjOkW3j0YNnuQJc-S}!!#IW%?Ci*`jdj9K9drC*H) z-rrRQqwUCgM{MZI%?=P<)G2z28@-uDyrT7!ijsiHUaLN~@6pDR&G#uwYn=e5?B$7^ ztzL0C%l`N^U;75_AR%*iN#>gnMtBV>OJ0x_FimfA{6Z}xZ~H{Q)x%bX>`f_Cvv`~+ z42h#Ow`zL`u-4v9zc`P9uVpO)9-#2f% z$Y2v!=t);mseFFi&j392?Mq;Lyl7xJcXw%4MQjM!J}dkEWN7|FxH=4rSDk~c=vIAN zQfB`rxCz6O$QfvUcPx#{3su5sa{-|PN|qegnCWgIWGnD}&ckn0iDE)RRqlMF?yyjV zK2;^>tsoZlZaZU_Vxe&2^LE}`j(kD-_mtOU8R{nV@MzKt2j`fp>7$DmH3OdCoZdYN z5F&Ov(r}Z01~0N=1l6?Q;!IiHZ2!Z~Yxjx#R(oLjnQcVuUpY3ceLpe$&6g-as_{pc@gZbHNA3e`bh`eBjRJI$}!6 zZ@Zi5Q~us5%C2*ga>Aut$r1mmeKqsb>P*kLzufF#-O<*IZQ;D21i0L+nNdLprZJV= zkPQ>MR)L_lJZx;@4%<#ADJ<3pz(KFXRbFdtBqwR1yc5*@8Plt~zoSYlj-9iyka^l#NR01~m?`bl+KDGf zb#i)QZgV02J`YG}4lrqAWY) zDVUiGV(rmbR&E&)zsHXy$7~tI%hbnj7=Eqt+*sZ*$zLw2_ou}2GO>)rq&xBGIe}(N(B8u`Ol6?ji;j6q;N;zOkHadf9n+9;Eh;q zmj~aJNw`hZPL|V-8B^b{7ZNazB`NLQ!(#a7#a0L!xe~!{2 zGyb$b4~x@3!Xp%uUqtIi{8~aR7SA~ z2!BB^P#AV~TFi|DfJxTwL+?Kc>S(={p91&cChoa$9W=|W-l%zlw2*}15 zcY#r>f52=lVrq*BIQ)7E-sETD-$KtF9T z6QkA_ZY#7E5=WQ+0}5AxyIF(6g4(6IfUdE7PzQ5xPM}qPcU6rnwxWCz7sHg;YH$JK zmIZ4P3w7L}j_YVSbriWAcs;9?mM^ZxEi2p=5pb0A*|np8UwYiLa1Y8pfbq)>bI0X; z*Yxc&Xa!XdBlm2k|HyK2LeV2gNBDy3j%D?0Q z9nV7RCZolN#XR8?Cl_5q!K{_uL%C_cauZ-1CLMpY5%#XC-25o zI&H=s3ji!BMP6;s$4x+jI34WvXkph~$&{kK_2p;Z=)1Um*Fe?-1^oz7FzM3<~59QV6 z+`ml8aIyh(yM1rfe+oDiVra0-Oa$bbzf<~~$KUH!6m+!YU#qy2Q+lp&su)Xh{F8Tb z+X_JUEK@rHdm0LOKEPmQ&1HKyB;1`5Vl=j7llyso;k(eQ+gP_uu5i0XfH+fo8>3jfK~o$s)r zl@PZbs!+?x??6Yt;O*ng`Ax1>CRiI-j7vI3%14kU09i5n*0jY5M%W(St6nO6e>G`4 zbd7dP*C!8wO$Y+!-rIBW&h2yb@S}Fzxfu0kBv3A8-^MTx78hV6NP>S#jEOeVWSyV< zTpyT(w#OEVjYbJ5#yL&=5gU~;TyN`G@9@UxI=g(HmX7X5AWTq3$EE<9&#b%2-V1`HE8RHn^UL9@e6KU@>$XI6yY%%-N_zW`}p9J>$PrvY_)h>{){zKsl zKb1Y$fAoU=3Z@+X?^cuCz=^Z&t8;e%qkiv0h6ndj~SMwG^Q9TTx^q`FVE3_x)v g;s(vZBk(7}O}ngWX6PCo%5Q+8teQ-{l==Jr2O2Thz5oCK literal 0 HcmV?d00001 diff --git a/frontend/index.html b/frontend/index.html index cd7edfb..92835e3 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -5,123 +5,140 @@ - GoSƐ - A tera-scale file uploader + GoSƐ - A terascale file uploader +
-
+
-

GoSƐ

-

A tera-scale file uploader

+

GoSƐ

+

A terascale file uploader

-
+
-
-
-
- - -
-
-
-
-
-
-
- -
-
-
-
-
-
-

- -

-
-
-
- - -
-
- - -
-
- - -
-
-
- - -
-
- - -
-
-
- - -
-
-
- - -
-
- - -
-
-
-
-
-
+
+
-
-
-
-
+
+
-
Transfer statistics
- +
Statistics
+
+
+
+
+
- - + + + + - - + + + + - - + + + + + + + + + + - + + + - - + + + +
Transferred0 BHashedUploadedTotal
Elapsed time0.0 sBytes0 B0 B0 B
Remaining time---
Time0.0 s0.0 s0.0 s
Time Left------
Average Speed---------
Chunk---Part---------
+
+
+ + +
+
+
+
+
+

+ +

+
+
+
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+
+ + +
+
+
+ + +
+
+
+ + +
+
+ + +
+
+
+
+
+
+
+
+
+ +
+
diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 2ad0af5..772c5de 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -9,9 +9,12 @@ "version": "1.0.0", "license": "GPL-3.0", "dependencies": { - "@types/crypto-js": "^4.1.1", + "@fortawesome/fontawesome-free": "^6.1.1", + "@popperjs/core": "^2.11.4", + "@types/bootstrap": "^5.1.9", + "@types/js-md5": "^0.4.3", "bootstrap": "^5.1.3", - "crypto-js": "^4.1.1", + "js-md5": "^0.7.3", "pretty-bytes": "^6.0.0", "pretty-ms": "^7.0.1" }, @@ -19,8 +22,8 @@ "@babel/core": "^7.17.8", "@babel/preset-env": "^7.16.11", "babel-loader": "^8.2.4", + "copy-webpack-plugin": "^10.2.4", "css-loader": "^6.7.1", - "html-loader": "^3.1.0", "html-webpack-plugin": "^5.5.0", "postcss": "^8.4.12", "postcss-loader": "^6.2.1", @@ -1611,6 +1614,15 @@ "node": ">=10.0.0" } }, + "node_modules/@fortawesome/fontawesome-free": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-free/-/fontawesome-free-6.1.1.tgz", + "integrity": "sha512-J/3yg2AIXc9wznaVqpHVX3Wa5jwKovVF0AMYSnbmcXTiL3PpRPfF58pzWucCwEiCJBp+hCNRLWClTomD8SseKg==", + "hasInstallScript": true, + "engines": { + "node": ">=6" + } + }, "node_modules/@jridgewell/resolve-uri": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.0.5.tgz", @@ -1675,7 +1687,6 @@ "version": "2.11.4", "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.4.tgz", "integrity": "sha512-q/ytXxO5NKvyT37pmisQAItCFqA7FD/vNb8dgaJy3/630Fsc+Mz9/9f2SziBoIZ30TJooXyTwZmhi1zjXmObYg==", - "peer": true, "funding": { "type": "opencollective", "url": "https://opencollective.com/popperjs" @@ -1700,6 +1711,15 @@ "@types/node": "*" } }, + "node_modules/@types/bootstrap": { + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/@types/bootstrap/-/bootstrap-5.1.9.tgz", + "integrity": "sha512-Tembe6lt7819EUzV5LSG9uuwULm4hdEGV9LZ8QBYpWc0J+a+9DdmJEwZ4FMaXGVJWwumTPSkJ8JQF0/KDAmXYg==", + "dependencies": { + "@popperjs/core": "^2.9.2", + "@types/jquery": "*" + } + }, "node_modules/@types/connect": { "version": "3.4.35", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", @@ -1719,11 +1739,6 @@ "@types/node": "*" } }, - "node_modules/@types/crypto-js": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/@types/crypto-js/-/crypto-js-4.1.1.tgz", - "integrity": "sha512-BG7fQKZ689HIoc5h+6D2Dgq1fABRa0RbBWKBd9SP/MVRVXROflpm5fhwyATX5duFmbStzyzyycPB8qUYKDH3NA==" - }, "node_modules/@types/eslint": { "version": "8.4.1", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.4.1.tgz", @@ -1788,6 +1803,19 @@ "@types/node": "*" } }, + "node_modules/@types/jquery": { + "version": "3.5.14", + "resolved": "https://registry.npmjs.org/@types/jquery/-/jquery-3.5.14.tgz", + "integrity": "sha512-X1gtMRMbziVQkErhTQmSe2jFwwENA/Zr+PprCkF63vFq+Yt5PZ4AlKqgmeNlwgn7dhsXEK888eIW2520EpC+xg==", + "dependencies": { + "@types/sizzle": "*" + } + }, + "node_modules/@types/js-md5": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@types/js-md5/-/js-md5-0.4.3.tgz", + "integrity": "sha512-BIga/WEqTi35ccnGysOuO4RmwVnpajv9oDB/sDQSY2b7/Ac7RyYR30bv7otZwByMvOJV9Vqq6/O1DFAnOzE4Pg==" + }, "node_modules/@types/json-schema": { "version": "7.0.10", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.10.tgz", @@ -1849,6 +1877,11 @@ "@types/node": "*" } }, + "node_modules/@types/sizzle": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/@types/sizzle/-/sizzle-2.3.3.tgz", + "integrity": "sha512-JYM8x9EGF163bEyhdJBpR2QX1R5naCJHC8ucJylJ3w9/CVBaskdQ8WqBf8MmQrd1kRvp/a4TS8HJ+bxzR7ZJYQ==" + }, "node_modules/@types/sockjs": { "version": "0.3.33", "resolved": "https://registry.npmjs.org/@types/sockjs/-/sockjs-0.3.33.tgz", @@ -2791,6 +2824,139 @@ "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=", "dev": true }, + "node_modules/copy-webpack-plugin": { + "version": "10.2.4", + "resolved": "https://registry.npmjs.org/copy-webpack-plugin/-/copy-webpack-plugin-10.2.4.tgz", + "integrity": "sha512-xFVltahqlsRcyyJqQbDY6EYTtyQZF9rf+JPjwHObLdPFMEISqkFkr7mFoVOC6BfYS/dNThyoQKvziugm+OnwBg==", + "dev": true, + "dependencies": { + "fast-glob": "^3.2.7", + "glob-parent": "^6.0.1", + "globby": "^12.0.2", + "normalize-path": "^3.0.0", + "schema-utils": "^4.0.0", + "serialize-javascript": "^6.0.0" + }, + "engines": { + "node": ">= 12.20.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.1.0" + } + }, + "node_modules/copy-webpack-plugin/node_modules/ajv": { + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", + "integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/copy-webpack-plugin/node_modules/ajv-keywords": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.3" + }, + "peerDependencies": { + "ajv": "^8.8.2" + } + }, + "node_modules/copy-webpack-plugin/node_modules/array-union": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-3.0.1.tgz", + "integrity": "sha512-1OvF9IbWwaeiM9VhzYXVQacMibxpXOMYVNIvMtKRyX9SImBXpKcFr8XvFDeEslCyuH/t6KRt7HEO94AlP8Iatw==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/copy-webpack-plugin/node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/copy-webpack-plugin/node_modules/globby": { + "version": "12.2.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-12.2.0.tgz", + "integrity": "sha512-wiSuFQLZ+urS9x2gGPl1H5drc5twabmm4m2gTR27XDFyjUHJUNsS8o/2aKyIF6IoBaR630atdher0XJ5g6OMmA==", + "dev": true, + "dependencies": { + "array-union": "^3.0.1", + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.7", + "ignore": "^5.1.9", + "merge2": "^1.4.1", + "slash": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/copy-webpack-plugin/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true + }, + "node_modules/copy-webpack-plugin/node_modules/schema-utils": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.0.0.tgz", + "integrity": "sha512-1edyXKgh6XnJsJSQ8mKWXnN/BVaIbFMLpouRUrXgVq7WYne5kw3MW7UPhO44uRXQSIpTSXoJbmrR2X0w9kUTyg==", + "dev": true, + "dependencies": { + "@types/json-schema": "^7.0.9", + "ajv": "^8.8.0", + "ajv-formats": "^2.1.1", + "ajv-keywords": "^5.0.0" + }, + "engines": { + "node": ">= 12.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/copy-webpack-plugin/node_modules/slash": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", + "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/core-js-compat": { "version": "3.21.1", "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.21.1.tgz", @@ -2850,11 +3016,6 @@ "node": ">= 8" } }, - "node_modules/crypto-js": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.1.1.tgz", - "integrity": "sha512-o2JlM7ydqd3Qk9CA0L4NL6mTzU2sdx96a+oOfPu8Mkl/PK51vSyoi8/rQ8NknZtk44vq15lmhAj9CIAGwgeWKw==" - }, "node_modules/css-loader": { "version": "6.7.1", "resolved": "https://registry.npmjs.org/css-loader/-/css-loader-6.7.1.tgz", @@ -3854,26 +4015,6 @@ "integrity": "sha512-c3Ab/url5ksaT0WyleslpBEthOzWhrjQbg75y7XUsfSzi3Dgzt0l8w5e7DylRn15MTlMMD58dTfzddNS2kcAjQ==", "dev": true }, - "node_modules/html-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/html-loader/-/html-loader-3.1.0.tgz", - "integrity": "sha512-ycMYFRiCF7YANcLDNP72kh3Po5pTcH+bROzdDwh00iVOAY/BwvpuZ1BKPziQ35Dk9D+UD84VGX1Lu/H4HpO4fw==", - "dev": true, - "dependencies": { - "html-minifier-terser": "^6.0.2", - "parse5": "^6.0.1" - }, - "engines": { - "node": ">= 12.13.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/webpack" - }, - "peerDependencies": { - "webpack": "^5.0.0" - } - }, "node_modules/html-minifier-terser": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/html-minifier-terser/-/html-minifier-terser-6.1.0.tgz", @@ -4382,6 +4523,11 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/js-md5": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/js-md5/-/js-md5-0.7.3.tgz", + "integrity": "sha512-ZC41vPSTLKGwIRjqDh8DfXoCrdQIyBgspJVPXHBGu4nZlAEvG3nf+jO9avM9RmLiGakg7vz974ms99nEV0tmTQ==" + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -5012,12 +5158,6 @@ "node": ">=6" } }, - "node_modules/parse5": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz", - "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==", - "dev": true - }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -8092,6 +8232,11 @@ "integrity": "sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==", "dev": true }, + "@fortawesome/fontawesome-free": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-free/-/fontawesome-free-6.1.1.tgz", + "integrity": "sha512-J/3yg2AIXc9wznaVqpHVX3Wa5jwKovVF0AMYSnbmcXTiL3PpRPfF58pzWucCwEiCJBp+hCNRLWClTomD8SseKg==" + }, "@jridgewell/resolve-uri": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.0.5.tgz", @@ -8143,8 +8288,7 @@ "@popperjs/core": { "version": "2.11.4", "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.4.tgz", - "integrity": "sha512-q/ytXxO5NKvyT37pmisQAItCFqA7FD/vNb8dgaJy3/630Fsc+Mz9/9f2SziBoIZ30TJooXyTwZmhi1zjXmObYg==", - "peer": true + "integrity": "sha512-q/ytXxO5NKvyT37pmisQAItCFqA7FD/vNb8dgaJy3/630Fsc+Mz9/9f2SziBoIZ30TJooXyTwZmhi1zjXmObYg==" }, "@types/body-parser": { "version": "1.19.2", @@ -8165,6 +8309,15 @@ "@types/node": "*" } }, + "@types/bootstrap": { + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/@types/bootstrap/-/bootstrap-5.1.9.tgz", + "integrity": "sha512-Tembe6lt7819EUzV5LSG9uuwULm4hdEGV9LZ8QBYpWc0J+a+9DdmJEwZ4FMaXGVJWwumTPSkJ8JQF0/KDAmXYg==", + "requires": { + "@popperjs/core": "^2.9.2", + "@types/jquery": "*" + } + }, "@types/connect": { "version": "3.4.35", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", @@ -8184,11 +8337,6 @@ "@types/node": "*" } }, - "@types/crypto-js": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/@types/crypto-js/-/crypto-js-4.1.1.tgz", - "integrity": "sha512-BG7fQKZ689HIoc5h+6D2Dgq1fABRa0RbBWKBd9SP/MVRVXROflpm5fhwyATX5duFmbStzyzyycPB8qUYKDH3NA==" - }, "@types/eslint": { "version": "8.4.1", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.4.1.tgz", @@ -8253,6 +8401,19 @@ "@types/node": "*" } }, + "@types/jquery": { + "version": "3.5.14", + "resolved": "https://registry.npmjs.org/@types/jquery/-/jquery-3.5.14.tgz", + "integrity": "sha512-X1gtMRMbziVQkErhTQmSe2jFwwENA/Zr+PprCkF63vFq+Yt5PZ4AlKqgmeNlwgn7dhsXEK888eIW2520EpC+xg==", + "requires": { + "@types/sizzle": "*" + } + }, + "@types/js-md5": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@types/js-md5/-/js-md5-0.4.3.tgz", + "integrity": "sha512-BIga/WEqTi35ccnGysOuO4RmwVnpajv9oDB/sDQSY2b7/Ac7RyYR30bv7otZwByMvOJV9Vqq6/O1DFAnOzE4Pg==" + }, "@types/json-schema": { "version": "7.0.10", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.10.tgz", @@ -8314,6 +8475,11 @@ "@types/node": "*" } }, + "@types/sizzle": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/@types/sizzle/-/sizzle-2.3.3.tgz", + "integrity": "sha512-JYM8x9EGF163bEyhdJBpR2QX1R5naCJHC8ucJylJ3w9/CVBaskdQ8WqBf8MmQrd1kRvp/a4TS8HJ+bxzR7ZJYQ==" + }, "@types/sockjs": { "version": "0.3.33", "resolved": "https://registry.npmjs.org/@types/sockjs/-/sockjs-0.3.33.tgz", @@ -9067,6 +9233,96 @@ "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=", "dev": true }, + "copy-webpack-plugin": { + "version": "10.2.4", + "resolved": "https://registry.npmjs.org/copy-webpack-plugin/-/copy-webpack-plugin-10.2.4.tgz", + "integrity": "sha512-xFVltahqlsRcyyJqQbDY6EYTtyQZF9rf+JPjwHObLdPFMEISqkFkr7mFoVOC6BfYS/dNThyoQKvziugm+OnwBg==", + "dev": true, + "requires": { + "fast-glob": "^3.2.7", + "glob-parent": "^6.0.1", + "globby": "^12.0.2", + "normalize-path": "^3.0.0", + "schema-utils": "^4.0.0", + "serialize-javascript": "^6.0.0" + }, + "dependencies": { + "ajv": { + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", + "integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + } + }, + "ajv-keywords": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.3" + } + }, + "array-union": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-3.0.1.tgz", + "integrity": "sha512-1OvF9IbWwaeiM9VhzYXVQacMibxpXOMYVNIvMtKRyX9SImBXpKcFr8XvFDeEslCyuH/t6KRt7HEO94AlP8Iatw==", + "dev": true + }, + "glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "requires": { + "is-glob": "^4.0.3" + } + }, + "globby": { + "version": "12.2.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-12.2.0.tgz", + "integrity": "sha512-wiSuFQLZ+urS9x2gGPl1H5drc5twabmm4m2gTR27XDFyjUHJUNsS8o/2aKyIF6IoBaR630atdher0XJ5g6OMmA==", + "dev": true, + "requires": { + "array-union": "^3.0.1", + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.7", + "ignore": "^5.1.9", + "merge2": "^1.4.1", + "slash": "^4.0.0" + } + }, + "json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true + }, + "schema-utils": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.0.0.tgz", + "integrity": "sha512-1edyXKgh6XnJsJSQ8mKWXnN/BVaIbFMLpouRUrXgVq7WYne5kw3MW7UPhO44uRXQSIpTSXoJbmrR2X0w9kUTyg==", + "dev": true, + "requires": { + "@types/json-schema": "^7.0.9", + "ajv": "^8.8.0", + "ajv-formats": "^2.1.1", + "ajv-keywords": "^5.0.0" + } + }, + "slash": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", + "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", + "dev": true + } + } + }, "core-js-compat": { "version": "3.21.1", "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.21.1.tgz", @@ -9115,11 +9371,6 @@ "which": "^2.0.1" } }, - "crypto-js": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.1.1.tgz", - "integrity": "sha512-o2JlM7ydqd3Qk9CA0L4NL6mTzU2sdx96a+oOfPu8Mkl/PK51vSyoi8/rQ8NknZtk44vq15lmhAj9CIAGwgeWKw==" - }, "css-loader": { "version": "6.7.1", "resolved": "https://registry.npmjs.org/css-loader/-/css-loader-6.7.1.tgz", @@ -9881,16 +10132,6 @@ "integrity": "sha512-c3Ab/url5ksaT0WyleslpBEthOzWhrjQbg75y7XUsfSzi3Dgzt0l8w5e7DylRn15MTlMMD58dTfzddNS2kcAjQ==", "dev": true }, - "html-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/html-loader/-/html-loader-3.1.0.tgz", - "integrity": "sha512-ycMYFRiCF7YANcLDNP72kh3Po5pTcH+bROzdDwh00iVOAY/BwvpuZ1BKPziQ35Dk9D+UD84VGX1Lu/H4HpO4fw==", - "dev": true, - "requires": { - "html-minifier-terser": "^6.0.2", - "parse5": "^6.0.1" - } - }, "html-minifier-terser": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/html-minifier-terser/-/html-minifier-terser-6.1.0.tgz", @@ -10242,6 +10483,11 @@ } } }, + "js-md5": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/js-md5/-/js-md5-0.7.3.tgz", + "integrity": "sha512-ZC41vPSTLKGwIRjqDh8DfXoCrdQIyBgspJVPXHBGu4nZlAEvG3nf+jO9avM9RmLiGakg7vz974ms99nEV0tmTQ==" + }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -10710,12 +10956,6 @@ "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz", "integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==" }, - "parse5": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz", - "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==", - "dev": true - }, "parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", diff --git a/frontend/package.json b/frontend/package.json index d9cff5f..009c34e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "gose", "version": "1.0.0", - "description": "A tera-scale file-uploader", + "description": "A terascale file-uploader", "private": true, "scripts": { "test": "echo \"Error: no test specified\" && exit 1", @@ -18,8 +18,8 @@ "@babel/core": "^7.17.8", "@babel/preset-env": "^7.16.11", "babel-loader": "^8.2.4", + "copy-webpack-plugin": "^10.2.4", "css-loader": "^6.7.1", - "html-loader": "^3.1.0", "html-webpack-plugin": "^5.5.0", "postcss": "^8.4.12", "postcss-loader": "^6.2.1", @@ -33,9 +33,12 @@ "webpack-dev-server": "^4.7.4" }, "dependencies": { - "@types/crypto-js": "^4.1.1", + "@fortawesome/fontawesome-free": "^6.1.1", + "@popperjs/core": "^2.11.4", + "@types/bootstrap": "^5.1.9", + "@types/js-md5": "^0.4.3", "bootstrap": "^5.1.3", - "crypto-js": "^4.1.1", + "js-md5": "^0.7.3", "pretty-bytes": "^6.0.0", "pretty-ms": "^7.0.1" } diff --git a/frontend/src/api.ts b/frontend/src/api.ts index 3b7d26e..7c0a9f8 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -10,9 +10,11 @@ export async function apiRequest(req: string, body: object, method = "POST") { body: method === "POST" ? JSON.stringify(body) : undefined }); + let json = await resp.json(); + if (resp.status !== 200) { - throw resp; + throw `Failed API request: ${json.error}`; } - return resp.json(); + return json; } diff --git a/frontend/src/chart.ts b/frontend/src/chart.ts new file mode 100644 index 0000000..bc0def7 --- /dev/null +++ b/frontend/src/chart.ts @@ -0,0 +1,110 @@ +type Line = { + length: number, + angle: number +}; + +type Point = number[]; + +type Bounds = { + xMin: number, + xMax: number, + yMin: number, + yMax: number +}; + +export class Chart { + element: HTMLDivElement; + smoothing: number = 0.15; + options: Bounds; + + constructor(elm: HTMLDivElement) { + this.element = elm; + } + + protected pointsPositions(points: Point[], bounds: Bounds): Point[] { + return points.map(e => { + const map = (value: number, inMin: number, inMax: number, outMin: number, outMax: number): number => { + return (value - inMin) * (outMax - outMin) / (inMax - inMin) + outMin; + }; + + const x = map(e[0], bounds.xMin, bounds.xMax, -1, 101) + const y = map(e[1], bounds.yMin, bounds.yMax, 27, 3) + + return [x, y]; + }) + } + + protected line(pointA: Point, pointB: Point): Line { + const lengthX: number = pointB[0] - pointA[0]; + const lengthY: number = pointB[1] - pointA[1]; + + return { + length: Math.sqrt(Math.pow(lengthX, 2) + Math.pow(lengthY, 2)), + angle: Math.atan2(lengthY, lengthX) + } + } + + protected controlPoint(current: Point, previous: Point, next: Point, reverse: boolean = false): Point { + const p = previous || current; + const n = next || current; + const l = this.line(p, n); + + const angle = l.angle + (reverse ? Math.PI : 0); + const length = l.length * this.smoothing; + const x = current[0] + Math.cos(angle) * length; + const y = current[1] + Math.sin(angle) * length; + + return [x, y]; + } + + protected bezierCommand(point: Point, i: number, a: Point[]): string { + const cps = this.controlPoint(a[i - 1], a[i - 2], point); + const cpe = this.controlPoint(point, a[i - 1], a[i + 1], true); + const close = i === a.length - 1 ? ' z':''; + + return `C ${cps[0]},${cps[1]} ${cpe[0]},${cpe[1]} ${point[0]},${point[1]}${close}`; + } + + protected svg(points: Point[]) { + const d = points.reduce((acc, e, i, a) => i === 0 + ? `M ${a[a.length - 1][0]},100 L ${e[0]},100 L ${e[0]},${e[1]}` + : `${acc} ${this.bezierCommand(e, i, a)}` + , ''); + + return ``; + } + + protected findBounds(points: Point[]): Bounds { + let options = { + xMin: 0, + xMax: 0, + yMin: 0, + yMax: 0 + }; + + for (let p of points) { + if (p[1] > options.yMax) + options.yMax = p[1]; + if (p[1] < options.yMin) + options.yMin = p[1]; + + if (p[0] > options.xMax) + options.xMax = p[0]; + if (p[0] < options.xMin) + options.xMin = p[0]; + } + + return options; + } + + public render(points: Point[]) { + if (points.length <= 1) { + return + } + + const bounds = this.findBounds(points); + const pointsPositions = this.pointsPositions(points, bounds); + + this.element.innerHTML = this.svg(pointsPositions); + } +} diff --git a/frontend/src/checksum.ts b/frontend/src/checksum.ts deleted file mode 100644 index 0bf163f..0000000 --- a/frontend/src/checksum.ts +++ /dev/null @@ -1,47 +0,0 @@ -import * as SHA256 from "crypto-js/sha256"; -import * as MD5 from "crypto-js/md5"; -import * as WordArray from "crypto-js/lib-typedarrays"; - -function wordToUintArray(wordArray: WordArray) { - const l = wordArray.sigBytes; - const words = wordArray.words; - const result = new Uint8Array(l); - var i = 0 /*dst*/ , - j = 0 /*src*/ ; - for (;;) { - // here i is a multiple of 4 - if (i === l) { - break; - } - var w = words[j++]; - result[i++] = (w & 0xff000000) >>> 24; - if (i === l) { - break; - } - result[i++] = (w & 0x00ff0000) >>> 16; - if (i === l) { - break; - } - result[i++] = (w & 0x0000ff00) >>> 8; - if (i === l) { - break; - } - result[i++] = (w & 0x000000ff); - } - return result; -} - -export async function sha256sum(buf: ArrayBuffer): Promise { - if (window.crypto.subtle !== undefined) { - let ab = await crypto.subtle.digest("SHA-256", buf); - } - - let wa = WordArray.create(buf as unknown as number[]); - return wordToUintArray(SHA256(wa)); -} - -export async function md5sum(buf: ArrayBuffer): Promise { - const wa = WordArray.create(buf as unknown as number[]); - const hash = MD5(wa); - return wordToUintArray(hash); -} diff --git a/frontend/src/config.ts b/frontend/src/config.ts index dbea796..becf245 100644 --- a/frontend/src/config.ts +++ b/frontend/src/config.ts @@ -8,14 +8,16 @@ export class Server { id: string = ""; title: string = ""; + part_size: number = 0; + max_upload_size: number = 0; + expiration: Array = []; } class Features { - shorten_link: boolean = false; + short_url: boolean = false; notify_mail: boolean = false; notify_browser: boolean = false; - encrypt: boolean = false; } export class Config { diff --git a/frontend/src/dropzone.ts b/frontend/src/dropzone.ts new file mode 100644 index 0000000..2ed7fed --- /dev/null +++ b/frontend/src/dropzone.ts @@ -0,0 +1,39 @@ + +export class Dropzone { + element: HTMLDivElement; + canDrop: (ev: DragEvent) => boolean; + handleDrop: (ev: DragEvent) => void; + + constructor(elm: HTMLDivElement, canDrop: (ev: DragEvent) => boolean, handleDrop: (ev: DragEvent) => void) { + this.element = elm; + this.canDrop = canDrop; + this.handleDrop = handleDrop; + + window.addEventListener("dragenter", (ev: DragEvent) => this.showDropZone(ev)); + this.element.addEventListener("dragenter", (ev: DragEvent) => this.allowDrag(ev)); + this.element.addEventListener("dragover", (ev: DragEvent) => this.allowDrag(ev)); + this.element.addEventListener("drop", (ev: DragEvent) => this.handleDrop(ev)); + this.element.addEventListener("dragleave", () => this.hideDropZone()); + } + + protected showDropZone(ev: DragEvent) { + if (!this.canDrop(ev)) { + return; + } + + this.element.style.display = "block"; + } + + protected hideDropZone() { + this.element.style.display = "none"; + } + + protected allowDrag(ev: DragEvent) { + if (!this.canDrop(ev)) { + return; + } + + ev.preventDefault(); + ev.dataTransfer.dropEffect = "copy"; + } +} diff --git a/frontend/src/file.ts b/frontend/src/file.ts deleted file mode 100644 index 854acc7..0000000 --- a/frontend/src/file.ts +++ /dev/null @@ -1,3 +0,0 @@ -export class ChecksummedFile extends File { - checksum: Uint8Array -} diff --git a/frontend/src/index.ts b/frontend/src/index.ts index 5c50519..d82302f 100644 --- a/frontend/src/index.ts +++ b/frontend/src/index.ts @@ -1,44 +1,117 @@ import "bootstrap"; +import { Tooltip } from "bootstrap"; import "../css/index.scss"; +import '@fortawesome/fontawesome-free/js/fontawesome'; +import '@fortawesome/fontawesome-free/js/solid'; + import prettyBytes from "pretty-bytes"; import * as prettyMilliseconds from "pretty-ms"; import { ProgressBar } from "./progress-bar"; import { Upload, UploadParams } from "./upload"; import { apiRequest } from "./api"; -import { sha256sum } from "./checksum"; import { Config, Server } from "./config"; -import { ChecksummedFile } from "./file"; +import { Chart } from "./chart"; +import { Dropzone } from "./dropzone"; -var statsTransferred: HTMLElement; -var statsElapsed: HTMLElement; -var statsEta: HTMLElement; -var statsSpeed: HTMLElement; -var statsParts: HTMLElement; var progressBar: ProgressBar; -var dropZone: HTMLElement; -var uploadInProgress: boolean = false; -var config: object; +var config: Config; +var chart: Chart; +var upload: Upload | null; +let points: Array = [] + +function reset() { + if (upload) { + upload.abort(); + } +} + +function alert(cls: string, msg: string, url?: string, icon?: string) { + let elm = document.getElementById("result"); + + elm.classList.remove("alert-danger", "alert-success", "alert-warning", "d-none"); + elm.classList.add("alert-" + cls); + + elm.innerHTML = ""; + + if (icon) { + if (icon === "spinner") { + elm.innerHTML += `
+ Loading... +
`; + } + else { + elm.innerHTML += ``; + } + } + + elm.innerHTML += `${msg}`; + + if (url) { + elm.innerHTML += ``; + elm.innerHTML += `${url}`; + + // Setup copy to clipboard + let btnCopy = document.getElementById("copy"); + let spanUrl = document.getElementById("upload-url"); + let tooltip = new Tooltip(btnCopy); + + btnCopy.addEventListener("click", async (ev: Event) => { + await navigator.clipboard.writeText(spanUrl.innerText); + + tooltip.dispose(); + btnCopy.title = "Copied! 🥳"; + tooltip = new Tooltip(btnCopy); + tooltip.show(); + + window.setTimeout(() => { + tooltip.dispose(); + btnCopy.title = "Copy to clipboard"; + tooltip = new Tooltip(btnCopy); + }, 1000) + }); + } +} function uploadStarted(upload: Upload) { + let msg: string = upload.stage == "hashing" + ? "Hashing in progress" + : "Uploading in progress"; + + alert("warning", msg, upload.url, "spinner"); + let p = upload.progress; - let resultElm = document.getElementById("result"); + let divStats = document.getElementById("statistics"); + divStats.classList.remove("d-none"); - resultElm.classList.remove("alert-danger", "alert-success", "d-none"); - resultElm.classList.add("alert-warning"); - resultElm.innerHTML = `Upload in progress: ${upload.url}`; + let btnReset = document.getElementById("reset"); + btnReset.classList.remove("d-none"); - progressBar.setMinMax(0, p.totalSize); + progressBar.setMinMax(0, upload.progress.totalSize); progressBar.set(0); + + let statsTotalBytes = document.getElementById("stats-total-bytes"); + let statsTotalParts = document.getElementById("stats-total-parts"); + + statsTotalBytes.textContent = prettyBytes(p.totalSize); + statsTotalParts.textContent = p.totalParts.toString(); + + points = []; } function uploadEnded(upload: Upload) { let p = upload.progress; - statsTransferred.textContent = prettyBytes(p.totalTransferred); + let statsBytes = document.getElementById(`stats-${upload.stage}-bytes`); + let statsTime = document.getElementById(`stats-${upload.stage}-time`); + let statsTimeETA = document.getElementById(`stats-${upload.stage}-eta`); + + statsBytes.textContent = prettyBytes(p.totalTransferred); + statsTime.textContent = prettyMilliseconds(p.totalElapsed, { compact: true }); + statsTimeETA.textContent = '0 s'; progressBar.set(p.totalSize); } @@ -46,50 +119,50 @@ function uploadEnded(upload: Upload) { function uploadProgressed(upload: Upload) { let p = upload.progress; - progressBar.set(p.transferred + upload.progress.totalTransferred); + let statsBytes = document.getElementById(`stats-${upload.stage}-bytes`); + let statsTime = document.getElementById(`stats-${upload.stage}-time`); + let statsTimeETA = document.getElementById(`stats-${upload.stage}-eta`); + let statsSpeed = document.getElementById(`stats-${upload.stage}-speed`); + let statsParts = document.getElementById(`stats-${upload.stage}-parts`); + let statsTotalTime = document.getElementById(`stats-total-time`); - statsTransferred.textContent = prettyBytes(p.transferred + p.totalTransferred) + " / " + prettyBytes(p.totalSize); - statsElapsed.textContent = prettyMilliseconds(p.elapsed + p.totalElapsed, { compact: true }); - statsEta.textContent = prettyMilliseconds(p.eta, { compact: true }); - statsSpeed.textContent = prettyBytes(p.speed, { bits: true }) + "/s"; - statsParts.textContent = `${p.part} / ${p.totalParts}`; + statsBytes.textContent = prettyBytes(p.transferred + p.totalTransferred); + statsTime.textContent = prettyMilliseconds(p.elapsed + p.totalElapsed); + statsTotalTime.textContent = prettyMilliseconds(p.elapsed + p.totalElapsed + p.overallElapsed); + + if (Number.isFinite(p.eta)) { + statsTimeETA.textContent = prettyMilliseconds(p.eta); + } + + statsSpeed.textContent = prettyBytes(p.averageSpeed, { bits: true }) + "/s"; + statsParts.textContent = p.part.toString(); + + progressBar.set(p.transferred + p.totalTransferred + p.totalSkipped); + + points.push([points.length, p.currentSpeed]); + chart.render(points); } async function startUpload(files: FileList) { - let resultElm = document.getElementById("result"); let params = getUploadParams(); try { - uploadInProgress = true; - if (files.length === 0) { - return; + throw "There are now files to upload"; } else if (files.length > 1) { - throw { - status: 400, - statusText: "Can only upload a single file" - }; + throw "Can only upload a single file"; } - let file = files[0] as ChecksummedFile; - let ab = await file.arrayBuffer(); - - file.checksum = await sha256sum(new Uint8Array(ab)); - - let upload = new Upload({ + upload = new Upload(files[0], { start: uploadStarted, end: uploadEnded, progress: uploadProgressed, }, params); + + let url = await upload.start(); - let url = await upload.upload(file); - - console.log("Upload succeeded", url); - - resultElm.classList.remove("alert-danger", "alert-warning"); - resultElm.classList.add("alert-success"); - resultElm.innerHTML = `Upload complete: ${url}`; + alert("success", "Upload completed", url, "circle-check"); if (params.notify_browser) { let dur = prettyMilliseconds(upload.progress.totalElapsed, { compact: true }); @@ -97,39 +170,37 @@ async function startUpload(files: FileList) { new Notification("Upload completed", { body: `Upload of ${size} for ${upload.file.name} has been completed in ${dur}.`, - icon: "gose-logo.svg", + icon: "/img/gose-logo.png", renotify: true, - tag: upload.uploadID + tag: upload.etag }); } - } catch (e) { - console.log("Upload failed", e); - - resultElm.classList.remove("alert-success", "alert-warning", "d-none"); - resultElm.classList.add("alert-danger"); - resultElm.textContent = `${e.status} - ${e.statusText}`; + } + catch (e) { + if (e === "Aborted") { + let divStats = document.getElementById("statistics"); + divStats.classList.add("d-none"); + + let btnReset = document.getElementById("reset"); + btnReset.classList.add("d-none"); + + let divResult = document.getElementById("result"); + divResult.classList.add("d-none"); + } else { + alert("danger", `Upload failed: ${e}`, null, "triangle-exclamation"); - if (params.notify_browser) { - new Notification("Upload failed", { - body: `Upload failed: ${e.status} - ${e.statusText}`, - icon: "gose-logo.svg", - }); + if (params.notify_browser) { + new Notification("Upload failed", { + body: `Upload failed: ${e}`, + icon: "/img/gose-logo.png", + }); + } } - } finally { - uploadInProgress = false; } } -function showDropZone(ev: DragEvent) { - dropZone.style.display = "block"; -} - -function hideDropZone() { - dropZone.style.display = "none"; -} - function canDrop(ev: DragEvent) { - if (uploadInProgress) { + if (upload && upload.inProgress) { return false; } @@ -144,22 +215,9 @@ function canDrop(ev: DragEvent) { return true; } -function allowDrag(ev: DragEvent) { - if (!canDrop(ev)) { - return; - } - - ev.preventDefault(); - ev.dataTransfer.dropEffect = "copy"; -} - function handleDrop(ev: DragEvent) { - if (!canDrop(ev)) { - return; - } - ev.preventDefault(); - hideDropZone(); + this.hideDropZone(); let inputElm = document.getElementById("file") as HTMLInputElement; inputElm.files = ev.dataTransfer.files; @@ -199,13 +257,13 @@ function updateExpiration(server: Server) { function getUploadParams(): UploadParams { let selServers = document.getElementById("servers") as HTMLSelectElement; let selExpiration = document.getElementById("expiration") as HTMLSelectElement; - let cbShortenLink = document.getElementById("shorten-link") as HTMLInputElement; + let cbShortURL = document.getElementById("shorten-link") as HTMLInputElement; let cbNotifyBrowser = document.getElementById("notify-browser") as HTMLInputElement; let cbNotifyMail = document.getElementById("notify-mail") as HTMLInputElement; let inpNotifyMail = document.getElementById("notify-mail-address") as HTMLInputElement; let params = new UploadParams(); - params.shorten_link = cbShortenLink.checked; + params.short_url = cbShortURL.checked; params.server = selServers.value; params.notify_browser = cbNotifyBrowser.checked; @@ -245,7 +303,7 @@ function onConfig(config: Config) { }); updateExpiration(config.servers[0]); - if (config.features.shorten_link) { + if (config.features.short_url) { let divShorten = document.getElementById("config-shorten"); divShorten.classList.remove("d-none"); } @@ -274,29 +332,25 @@ async function setupNotification(ev: Event) { } } -export async function load() { - statsTransferred = document.getElementById("stats-transferred"); - statsElapsed = document.getElementById("stats-elapsed"); - statsEta = document.getElementById("stats-eta"); - statsSpeed = document.getElementById("stats-speed"); - statsParts = document.getElementById("stats-parts"); - dropZone = document.getElementById("dropzone"); +async function load() { + const btnReset = document.getElementById("reset"); + btnReset.addEventListener("click", reset); + + const divDropzone = document.getElementById("dropzone") as HTMLDivElement; + new Dropzone(divDropzone, canDrop, handleDrop); + + const divChart = document.getElementById("chart") as HTMLDivElement; + chart = new Chart(divChart); - let progressElm = document.getElementById("progress") as HTMLProgressElement; + const progressElm = document.getElementById("progress") as HTMLProgressElement; progressBar = new ProgressBar(progressElm); - let inputElm = document.getElementById("file") as HTMLInputElement; + const inputElm = document.getElementById("file") as HTMLInputElement; inputElm.addEventListener("change", fileChanged); - window.addEventListener("dragenter", showDropZone); - dropZone.addEventListener("dragenter", allowDrag); - dropZone.addEventListener("dragover", allowDrag); - dropZone.addEventListener("drop", handleDrop); - dropZone.addEventListener("dragleave", hideDropZone); - // Toggle notification mail - let swNotifyMail = document.getElementById("notify-mail"); - let divNotifyMailAddress = document.getElementById("config-notify-mail-address"); + const swNotifyMail = document.getElementById("notify-mail"); + const divNotifyMailAddress = document.getElementById("config-notify-mail-address"); swNotifyMail.addEventListener("change", (ev) => { let cb = ev.target as HTMLInputElement; if (cb.checked) { @@ -319,8 +373,7 @@ export async function load() { } config = await apiRequest("config", {}, "GET"); - - onConfig(config as Config); + onConfig(config); } window.addEventListener("load", load); diff --git a/frontend/src/progress-handler.ts b/frontend/src/progress-handler.ts index 6ff5651..ed136b5 100644 --- a/frontend/src/progress-handler.ts +++ b/frontend/src/progress-handler.ts @@ -4,49 +4,65 @@ class Callbacks { export class ProgressHandler { callbacks: Callbacks; - totalSize: number; - totalParts: number; + + averageSpeed: number; + currentSpeed: number; + part: number; eta: number; - speed: number; started: number; elapsed: number; - total: number; transferred: number; + + overallElapsed: number; + + totalSize: number; + totalParts: number; totalElapsed: number; totalTransferred: number; + totalSkipped: number; + partStarted: number; + lastProgress: number; constructor(cbs: Callbacks, totalSize: number, totalParts: number) { this.callbacks = cbs; this.totalSize = totalSize; this.totalParts = totalParts; + + this.overallElapsed = 0; + + this.totalElapsed = 0; + this.totalTransferred = 0; + this.totalSkipped = 0; } - loadStart() { - this.started = Date.now(); + start() { + this.averageSpeed = 0; + this.currentSpeed = 0; this.part = 0; - this.speed = 0; this.eta = 0; - + this.started = Date.now(); this.elapsed = 0; this.transferred = 0; this.totalElapsed = 0; this.totalTransferred = 0; + this.totalSkipped = 0; this.callbacks.start(this); this.callbacks.progress(this); } - loadEnd() { + end() { this.callbacks.end(this); this.totalElapsed = Date.now() - this.started; + this.overallElapsed += this.totalElapsed; } - partLoadStart(ev: ProgressEvent) { + partStart(ev: ProgressEvent) { this.partStarted = Date.now(); this.part++; @@ -55,27 +71,36 @@ export class ProgressHandler { this.transferred = 0; } - partLoadEnd(ev: ProgressEvent) { + partEnd(ev: ProgressEvent) { this.totalTransferred += this.transferred; this.totalElapsed += this.elapsed; } - partProgress(ev: ProgressEvent) { - this.total = ev.total; - this.transferred = ev.loaded; + partProgress(ev: ProgressEvent) { + let incrElapsed = Date.now() - this.partStarted - this.elapsed; + let incrTransferred = ev.loaded - this.transferred; - this.elapsed = Date.now() - this.partStarted; + this.elapsed += incrElapsed; + this.transferred += incrTransferred; - this.update(); + let transferred = this.totalTransferred + this.transferred; + let elapsed = this.totalElapsed + this.elapsed; + if (incrElapsed > 0) { + this.currentSpeed = 8e3 * incrTransferred / incrElapsed; // b/s + } + + if (elapsed > 0) { + this.averageSpeed = 8e3 * transferred / elapsed; // b/s + } + + this.eta = 8e3 * (this.totalSize - this.totalSkipped - transferred) / this.averageSpeed; + this.callbacks.progress(this); } - update() { - let transferred = this.totalTransferred + this.transferred; - let elapsed = this.totalElapsed + this.elapsed; - - this.speed = 8e3 * transferred / elapsed; - this.eta = (this.totalSize - transferred) / this.speed; + partSkip(size: number) { + this.part++; + this.totalSkipped += size; } } diff --git a/frontend/src/upload.ts b/frontend/src/upload.ts index 2e0391e..7fabb8c 100644 --- a/frontend/src/upload.ts +++ b/frontend/src/upload.ts @@ -1,156 +1,276 @@ -import { md5sum } from "./checksum"; import { ProgressHandler } from "./progress-handler"; -import { buf2hex } from "./utils"; +import { buf2hex, hex2buf, arraybufferEqual } from "./utils"; import { apiRequest} from "./api"; -import { ChecksummedFile } from "./file"; +import * as md5 from "js-md5"; export class UploadParams { server: string expiration: string notify_mail: string notify_browser: boolean - shorten_link: boolean + short_url: boolean } -async function md5sumHex(blob: Blob) { - let ab = await blob.arrayBuffer(); +class Callbacks { + [key: string]: any +} - let hash = await md5sum(ab); +class Part { + number: number; + offset: number; + length: number; + etag: ArrayBuffer; - return buf2hex(hash); -} + constructor(num: number, etag: ArrayBuffer, len?: number, off?: number) { + this.number = num; + this.offset = off; + this.length = len; + this.etag = etag; + } -class Callbacks { - [key: string]: any + toJSON(): any { + return { + ...this, + etag: buf2hex(this.etag) + } + } + + static fromJSON(json: any): Part { + return new Part(json.number, hex2buf(json.etag), json.length); + } } export class Upload { + file: File | null = null; url: string; - uploadID: string | null; - callbacks: Callbacks; - params: UploadParams; progress: ProgressHandler | null = null; - file: ChecksummedFile | null = null; + etag: string; + stage: string; + inProgress: boolean = false; + + protected parts: Part[] = []; + protected callbacks: Callbacks; + protected params: UploadParams; + protected xhr: XMLHttpRequest; - constructor(cbs: Callbacks, params: UploadParams) { + readonly partSize = 6 << 20; + + constructor(file: File, cbs: Callbacks, params: UploadParams) { + this.file = file; this.callbacks = cbs; this.params = params; + + const partsCount = Math.ceil(this.file.size / this.partSize); + + this.progress = new ProgressHandler({ + start: () => this.callbacks.start(this), + end: () => this.callbacks.end(this), + progress: () => this.callbacks.progress(this), + }, this.file.size, partsCount); } - async upload(file: ChecksummedFile) { - this.file = file; + async start() { + try { + this.inProgress = true; + + if (this.file.size == 0) { + throw "Cannot upload empty file"; + } + + [this.parts, this.etag] = await this.hash(); + + return await this.upload(); + } catch(e) { + throw e; + } finally { + this.inProgress = false; + } + } + + async hash(): Promise<[ Part[], string ]> { + this.stage = "hashing"; + + this.progress.start(); + + let parts: Part[] = []; + let partNumber = 1; + for (let offset = 0; offset < this.file.size; offset += this.partSize) { + let length = this.partSize; + if (offset + length > this.file.size) { + length = this.file.size - offset; // handle last part + } + + let part = this.file.slice(offset, offset + length); + let partBuffer = await part.arrayBuffer(); + if (!this.file) { + throw "Aborted"; + } + + this.progress.partStart(new ProgressEvent("", { + loaded: 0, + total: length + })); + + let md = md5.create(); + // let chunkSize = 1<<20; + let chunkSize = this.partSize; + for (let chunkOffset = 0; chunkOffset < partBuffer.byteLength; chunkOffset += chunkSize) { + let chunkLength = chunkSize; + if (chunkOffset + chunkSize > partBuffer.byteLength) { + chunkLength = partBuffer.byteLength - chunkOffset; + } + + let chunkBuffer = partBuffer.slice(chunkOffset, chunkOffset+chunkLength); + + md.update(chunkBuffer); + + this.progress.partProgress(new ProgressEvent("", { + loaded: chunkOffset+chunkLength, + total: length + })); + } + + this.progress.partEnd(new ProgressEvent("", { + loaded: length, + total: length + })); + + let etag = md.arrayBuffer(); + + parts.push(new Part(partNumber++, etag, length, offset)); + } + + this.progress.end(); + + let etagBlob = new Blob(parts.map(p => p.etag)); + let etagBuf = await etagBlob.arrayBuffer(); + let etag = md5.arrayBuffer(etagBuf); + let etagStr = `${buf2hex(etag)}-${parts.length}`; + + return [parts, etagStr]; + } + + async upload() { + this.stage = "uploading"; let respInitiate = await apiRequest("initiate", { server: this.params.server, - filename: file.name, - shorten_link: this.params.shorten_link, - expiration: this.params.expiration, - content_length: file.size, - content_type: file.type, - checksum: buf2hex(file.checksum) + filename: this.file.name, + etag: this.etag, + short_url: this.params.short_url, }); + if (respInitiate.upload_id === undefined) { + return respInitiate.url; + } + this.url = respInitiate.url; - this.uploadID = respInitiate.upload_id; - this.progress = new ProgressHandler({ - start: () => this.callbacks.start(this), - end: () => this.callbacks.end(this), - progress: () => this.callbacks.progress(this), - }, file.size, respInitiate.parts.length); + let existingParts: {[x: number]: Part} = {} + for (let part of respInitiate.parts) { + existingParts[part.number] = Part.fromJSON(part); + } + + this.progress.start(); - this.progress.loadStart(); + for (let i = 0; i < this.parts.length; i++) { + let part = this.parts[i]; + if (part.number in existingParts) { + let existingPart = existingParts[part.number]; - let parts = []; - let etags = []; - for (let i = 0; i < respInitiate.parts.length; i++) { - let start = i * respInitiate.part_size; - let end = (i + 1) * respInitiate.part_size; - if (i === respInitiate.parts.length - 1) { - end = file.size; + if (arraybufferEqual(existingPart.etag, part.etag)) { + this.progress.partSkip(existingPart.length); + continue; + } } - let chunk = file.slice(start, end); - let url = respInitiate.parts[i]; - - let chunkBuffer = await chunk.arrayBuffer(); - let etag = await this.uploadPart(url, chunk); - let etagExpected = await md5sum(chunkBuffer); - if (etag !== "\"" + buf2hex(etagExpected) + "\"") { - throw { - status: 400, - statusText: "Checksum mismatch" - }; + let chunk = this.file.slice(part.offset, part.offset + part.length); + + let partResp = await apiRequest("part", { + server: this.params.server, + etag: respInitiate.etag, + upload_id: respInitiate.upload_id, + checksum: buf2hex(part.etag), + length: part.length, + number: part.number + }) + + let etag = await this.uploadPart(partResp.url, chunk); + if (!this.file) { + throw "Aborted"; } - etags.push(etagExpected); - parts.push({ - etag, - part_number: i + 1 - }); + if (etag !== "\"" + buf2hex(part.etag) + "\"") { + throw "Checksum mismatch"; + } } - this.progress.loadEnd(); + this.progress.end(); let respComplete = await apiRequest("complete", { server: this.params.server, - key: respInitiate.key, + etag: respInitiate.etag, upload_id: respInitiate.upload_id, - parts: parts, - notify_mail: this.params.notify_mail + parts: this.parts.map(p => p.toJSON()), + expiration: this.params.expiration, + notify_mail: this.params.notify_mail, }); - let etagBlob = new Blob(etags); - let objEtag = await md5sumHex(etagBlob) + `-${parts.length}`; - - if (respComplete.etag !== objEtag) { - throw { - status: 400, - statusText: "Final checksum mismatch" - }; + if (respComplete.etag !== this.etag) { + throw "Final checksum mismatch"; } - return respInitiate.url; + return respInitiate.url || respComplete.url; } async uploadPart(url: string, part: Blob) { let prom = new Promise((resolve, reject) => { - let xhr = new XMLHttpRequest(); - - xhr.open("PUT", url); - - xhr.onload = function() { - if (this.status >= 200 && this.status < 300) { - resolve(this); + this.xhr = new XMLHttpRequest(); + this.xhr.open("PUT", url); + this.xhr.onload = () => { + if (this.xhr.status >= 200 && this.xhr.status < 300) { + resolve(this.xhr); } else { reject({ - status: this.status, - statusText: xhr.statusText + status: this.xhr.status, + statusText: this.xhr.statusText }); } }; - xhr.onerror = function() { + this.xhr.onerror = () => { reject({ - status: this.status, - statusText: xhr.statusText + status: this.xhr.status, + statusText: this.xhr.statusText }); }; - xhr.upload.onprogress = (ev) => this.progress.partProgress(ev); - xhr.upload.onloadstart = (ev) => this.progress.partLoadStart(ev); - xhr.upload.onloadend = (ev) => this.progress.partLoadEnd(ev); + this.xhr.onabort = () => { + reject("Aborted"); + } + + this.xhr.upload.onprogress = (ev) => this.progress.partProgress(ev); + this.xhr.upload.onloadstart = (ev) => this.progress.partStart(ev); + this.xhr.upload.onloadend = (ev) => this.progress.partEnd(ev); - xhr.send(part); + this.xhr.send(part); }); let resp = await prom; if (resp.status !== 200) { - throw resp; + // TODO: Decode AWS S3 error here. + throw "Failed to upload part"; } - let etag = resp.getResponseHeader("etag"); + return resp.getResponseHeader("etag"); + } + + abort() { + if (this.xhr) { + this.xhr.abort(); + } - return etag; + // Used to signal hash() + this.file = null; } } diff --git a/frontend/src/utils.ts b/frontend/src/utils.ts index 3ef536b..0cc4a74 100644 --- a/frontend/src/utils.ts +++ b/frontend/src/utils.ts @@ -1,13 +1,31 @@ -export function buf2hex(buffer: ArrayBuffer): string { - return Array.prototype.map.call(new Uint8Array(buffer), (x: number) => ("00" + x.toString(16)).slice(-2)).join(""); +export function buf2hex(buf: ArrayBuffer): string { + return Array.prototype.map.call(new Uint8Array(buf), (x: number) => ("00" + x.toString(16)).slice(-2)).join(""); } -export function buf2base64(buffer: ArrayBuffer) { - let binary = ""; - let bytes = new Uint8Array(buffer); - let len = bytes.byteLength; - for (let i = 0; i < len; i++) { - binary += String.fromCharCode(bytes[i]); +export function hex2buf(hex: string): ArrayBuffer { + let a = new Uint8Array(hex.match(/.{1,2}/g).map(byte => parseInt(byte, 16))); + return a.buffer; +} + + +export function arraybufferEqual(a: ArrayBuffer, b: ArrayBuffer) { + if (a === b) { + return true; + } + + if (a.byteLength !== b.byteLength) { + return false; + } + + var view1 = new DataView(a); + var view2 = new DataView(b); + + var i = a.byteLength; + while (i--) { + if (view1.getUint8(i) !== view2.getUint8(i)) { + return false; + } } - return window.btoa(binary); + + return true; } diff --git a/frontend/webpack.config.js b/frontend/webpack.config.js index e056e81..46b1f2f 100644 --- a/frontend/webpack.config.js +++ b/frontend/webpack.config.js @@ -1,6 +1,6 @@ const path = require("path"); const HtmlWebpackPlugin = require("html-webpack-plugin"); - +const CopyPlugin = require("copy-webpack-plugin"); module.exports = { mode: "development", @@ -11,10 +11,14 @@ module.exports = { }, plugins: [ new HtmlWebpackPlugin({ - title: "GoS3 - A tera-scale file uploader", + title: "GoS3 - A terascale file uploader", template: "index.html", - favicon: "img/gose-logo.svg" - }) + }), + new CopyPlugin({ + patterns: [ + { from: "img/*", to: "" } + ], + }), ], devtool: "eval-source-map", devServer: { @@ -38,10 +42,6 @@ module.exports = { use: "ts-loader", exclude: /node_modules/, }, - { - test: /\.html$/i, - loader: "html-loader", - }, { test: /\.(scss)$/, use: [{ diff --git a/go.mod b/go.mod index 3bd0dce..5cefbd6 100644 --- a/go.mod +++ b/go.mod @@ -8,12 +8,13 @@ require ( github.com/docker/go-units v0.4.0 github.com/gin-contrib/static v0.0.1 github.com/gin-gonic/gin v1.7.7 - github.com/google/uuid v1.3.0 github.com/mitchellh/mapstructure v1.4.3 github.com/spf13/viper v1.10.1 gopkg.in/yaml.v2 v2.4.0 ) +require github.com/vfaronov/httpheader v0.1.0 + require ( github.com/fatih/color v1.13.0 // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect diff --git a/go.sum b/go.sum index c9f6224..89f1a2b 100644 --- a/go.sum +++ b/go.sum @@ -192,8 +192,6 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= @@ -378,6 +376,8 @@ github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/vfaronov/httpheader v0.1.0 h1:VdzetvOKRoQVHjSrXcIOwCV6JG5BCAW9rjbVbFPBmb0= +github.com/vfaronov/httpheader v0.1.0/go.mod h1:ZBxgbYu6nbN5V9Ptd1yYUUan0voD0O8nZLXHyxLgoLE= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/pkg/config/config.go b/pkg/config/config.go index a0355a6..5a24ded 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -44,7 +44,9 @@ type S3ServerConfig struct { ID string `mapstructure:"id" json:"id"` Title string `mapstructure:"title" json:"title"` - Expiration []Expiration `mapstructure:"expiration" json:"expiration"` + MaxUploadSize size `mapstructure:"max_upload_size" json:"max_upload_size"` + PartSize size `mapstructure:"part_size" json:"part_size"` + Expiration []Expiration `mapstructure:"expiration" json:"expiration"` } // S3Server describes an S3 server @@ -58,9 +60,6 @@ type S3Server struct { NoSSL bool `mapstructure:"no_ssl"` AccessKey string `mapstructure:"access_key"` SecretKey string `mapstructure:"secret_key"` - - MaxUploadSize size `mapstructure:"max_upload_size"` - PartSize size `mapstructure:"part_size"` } // ShortenerConfig contains Link-shortener specific configuration diff --git a/pkg/handlers/complete.go b/pkg/handlers/complete.go index 3a0ad0b..e67d3ee 100644 --- a/pkg/handlers/complete.go +++ b/pkg/handlers/complete.go @@ -12,17 +12,12 @@ import ( "github.com/stv0g/gose/pkg/config" "github.com/stv0g/gose/pkg/notifier" "github.com/stv0g/gose/pkg/server" + "github.com/stv0g/gose/pkg/utils" ) -type part struct { - PartNumber int64 `json:"part_number"` - Checksum string `json:"checksum"` - ETag string `json:"etag"` -} - type completionRequest struct { Server string `json:"server"` - Key string `json:"key"` + ETag string `json:"etag"` UploadID string `json:"upload_id"` Parts []part `json:"parts"` NotifyMail *string `json:"notify_mail"` @@ -31,6 +26,7 @@ type completionRequest struct { type completionResponse struct { ETag string `json:"etag"` + URL string `json:"url"` } // HandleComplete handles a completed upload @@ -50,35 +46,43 @@ func HandleComplete(c *gin.Context) { return } + if !utils.IsValidETag(req.ETag) { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid etag"}) + return + } + // Ceph's RadosGW does not yet support tagging during the initiation of multi-part uploads. // So we tag here with a separate request instead of the MPU initiate req. // See: https://github.com/ceph/ceph/pull/38275 - var expiration string + var exp *config.Expiration if req.Expiration == nil { if len(svr.Config.Expiration) > 0 { - expiration = svr.Config.Expiration[0].ID + exp = &svr.Config.Expiration[0] } } else { - if !svr.HasExpirationClass(*req.Expiration) { + if exp = svr.GetExpirationClass(*req.Expiration); exp == nil { c.JSON(http.StatusInternalServerError, gin.H{"error": "invalid expiration class"}) return } + } - expiration = *req.Expiration + if len(req.Parts) > int(svr.Config.MaxUploadSize/svr.Config.PartSize) { + c.JSON(http.StatusInternalServerError, gin.H{"error": "max upload size exceeded"}) + return } // Prepare MPU completion request parts := []*s3.CompletedPart{} for _, part := range req.Parts { parts = append(parts, &s3.CompletedPart{ - PartNumber: aws.Int64(part.PartNumber), + PartNumber: aws.Int64(part.Number), ETag: aws.String(part.ETag), }) } respCompleteMPU, err := svr.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{ Bucket: aws.String(svr.Config.Bucket), - Key: aws.String(req.Key), + Key: aws.String(req.ETag), UploadId: aws.String(req.UploadID), MultipartUpload: &s3.CompletedMultipartUpload{ Parts: parts, @@ -90,29 +94,48 @@ func HandleComplete(c *gin.Context) { } // Tag object with expiration tag here - if _, err := svr.PutObjectTagging(&s3.PutObjectTaggingInput{ - Bucket: aws.String(svr.Config.Bucket), - Key: aws.String(req.Key), - Tagging: &s3.Tagging{ - TagSet: []*s3.Tag{ - { - Key: aws.String("expiration"), - Value: aws.String(expiration), + if exp != nil { + if _, err := svr.PutObjectTagging(&s3.PutObjectTaggingInput{ + Bucket: aws.String(svr.Config.Bucket), + Key: aws.String(req.ETag), + Tagging: &s3.Tagging{ + TagSet: []*s3.Tag{ + { + Key: aws.String("expiration"), + Value: aws.String(exp.ID), + }, }, }, - }, - }); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to tag object"}) + }); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to tag object"}) + return + } + } + + // Retrieve meta-data + obj, err := svr.HeadObject(&s3.HeadObjectInput{ + Bucket: aws.String(svr.Config.Bucket), + Key: aws.String(req.ETag), + }) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to get object"}) return } + var url string + if u, ok := obj.Metadata["Original-Short-Url"]; ok { + url = *u + } else { + url = svr.GetObjectURL(req.ETag).String() + } + // Send notifications go func(key string) { if cfg.Notification != nil && cfg.Notification.Uploads { if notif, err := notifier.NewNotifier(cfg.Notification.Template, cfg.Notification.URLs...); err != nil { log.Fatalf("Failed to create notification sender: %s", err) } else { - if err := notif.Notify(svr, key, types.Params{ + if err := notif.Notify(url, obj, types.Params{ "Title": "New upload", }); err != nil { fmt.Printf("Failed to send notification: %s", err) @@ -125,16 +148,17 @@ func HandleComplete(c *gin.Context) { if notif, err := notifier.NewNotifier(cfg.Notification.Mail.Template, u); err != nil { log.Fatalf("Failed to create notification sender: %s", err) } else { - if err := notif.Notify(svr, key, types.Params{ + if err := notif.Notify(url, obj, types.Params{ "Title": "New upload", }); err != nil { fmt.Printf("Failed to send notification: %s", err) } } } - }(*respCompleteMPU.Key) + }(req.ETag) c.JSON(200, &completionResponse{ + URL: url, ETag: *respCompleteMPU.ETag, }) } diff --git a/pkg/handlers/config.go b/pkg/handlers/config.go index 3e131c3..2b9f834 100644 --- a/pkg/handlers/config.go +++ b/pkg/handlers/config.go @@ -7,10 +7,9 @@ import ( ) type featureResponse struct { - ShortenLink bool `json:"shorten_link"` + ShortURL bool `json:"short_url"` NotifyMail bool `json:"notify_mail"` NotifyBrowser bool `json:"notify_browser"` - Encrypt bool `json:"encrypt"` } type configResponse struct { @@ -31,10 +30,9 @@ func HandleConfig(c *gin.Context) { c.JSON(200, &configResponse{ Servers: svrsResp, Features: featureResponse{ - ShortenLink: cfg.Shortener != nil, + ShortURL: cfg.Shortener != nil, NotifyMail: cfg.Notification.Mail != nil, NotifyBrowser: true, - Encrypt: false, }, }) } diff --git a/pkg/handlers/download.go b/pkg/handlers/download.go index 3663f6d..477ccb3 100644 --- a/pkg/handlers/download.go +++ b/pkg/handlers/download.go @@ -4,17 +4,17 @@ import ( "fmt" "log" "net/http" - "strings" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/containrrr/shoutrrr/pkg/types" "github.com/gin-gonic/gin" - "github.com/google/uuid" "github.com/stv0g/gose/pkg/config" "github.com/stv0g/gose/pkg/notifier" "github.com/stv0g/gose/pkg/server" + "github.com/stv0g/gose/pkg/utils" + "github.com/vfaronov/httpheader" ) func HandleDownload(c *gin.Context) { @@ -23,29 +23,28 @@ func HandleDownload(c *gin.Context) { svrs := c.MustGet("servers").(server.List) cfg := c.MustGet("config").(*config.Config) - key := c.Param("key") - key = key[1:] + etag := c.Param("etag") + fileName := c.Param("filename") + svrName := c.Param("server") - svr, ok := svrs[c.Param("server")] + svr, ok := svrs[svrName] if !ok { c.JSON(http.StatusNotFound, gin.H{"error": "invalid server"}) return } - parts := strings.Split(key, "/") - if len(parts) != 2 { - c.JSON(http.StatusInternalServerError, gin.H{"error": "invalid key"}) + if !utils.IsValidETag(etag) { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid etag"}) return } - if _, err := uuid.Parse(parts[0]); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "invalid uuid in key"}) - return - } + // RFC8187 + contentDisposition := "attachment; filename*=" + httpheader.EncodeExtValue(fileName, "") req, _ := svr.GetObjectRequest(&s3.GetObjectInput{ - Bucket: aws.String(svr.Config.Bucket), - Key: aws.String(key), + Bucket: aws.String(svr.Config.Bucket), + Key: aws.String(etag), + ResponseContentDisposition: aws.String(contentDisposition), }) u, _, err := req.PresignRequest(10 * time.Second) @@ -54,19 +53,36 @@ func HandleDownload(c *gin.Context) { return } + // Retrieve meta-data + obj, err := svr.HeadObject(&s3.HeadObjectInput{ + Bucket: aws.String(svr.Config.Bucket), + Key: aws.String(etag), + }) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to get object"}) + return + } + + var url string + if u, ok := obj.Metadata["Original-Short-Url"]; ok { + url = *u + } else { + url = svr.GetObjectURL(etag).String() + } + go func(svr server.Server, key string) { if cfg.Notification != nil && cfg.Notification.Downloads { if notif, err := notifier.NewNotifier(cfg.Notification.Template, cfg.Notification.URLs...); err != nil { log.Fatalf("Failed to create notification sender: %s", err) } else { - if err := notif.Notify(svr, key, types.Params{ + if err := notif.Notify(url, obj, types.Params{ "Title": "New download", }); err != nil { fmt.Printf("Failed to send notification: %s", err) } } } - }(svr, key) + }(svr, etag) c.Redirect(http.StatusTemporaryRedirect, u) } diff --git a/pkg/handlers/initiate.go b/pkg/handlers/initiate.go index 1fcf7cb..c2832bd 100644 --- a/pkg/handlers/initiate.go +++ b/pkg/handlers/initiate.go @@ -1,36 +1,41 @@ package handlers import ( - "log" "net/http" "net/url" - "path" "path/filepath" - "time" + "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/gin-gonic/gin" - "github.com/google/uuid" "github.com/stv0g/gose/pkg/config" "github.com/stv0g/gose/pkg/server" "github.com/stv0g/gose/pkg/shortener" + "github.com/stv0g/gose/pkg/utils" +) + +const ( + MaxFileNameLength = 256 ) type initiateRequest struct { - Server string `json:"server"` - ContentLength int64 `json:"content_length"` - ContentType string `json:"content_type"` - Filename string `json:"filename"` - ShortenLink bool `json:"shorten_link"` + Server string `json:"server"` + ETag string `json:"etag"` + FileName string `json:"filename"` + ShortURL bool `json:"short_url"` } type initiateResponse struct { - Parts []string `json:"parts"` - Key string `json:"key"` - UploadID string `json:"upload_id"` - PartSize int64 `json:"part_size"` - URL string `json:"url"` + ETag string `json:"etag"` + + // We do not have a URL for resumed uploads due to limitations of the S3 API. + URL string `json:"url,omitempty"` + + // An empty UploadID indicate that the file already existed + UploadID string `json:"upload_id,omitempty"` + + Parts []part `json:"parts"` } // HandleInitiate initiates a new upload @@ -38,8 +43,8 @@ func HandleInitiate(c *gin.Context) { var err error svrs := c.MustGet("servers").(server.List) - cfg := c.MustGet("config").(*config.Config) shortener := c.MustGet("shortener").(*shortener.Shortener) + cfg := c.MustGet("config").(*config.Config) var req initiateRequest if err := c.BindJSON(&req); err != nil { @@ -53,99 +58,125 @@ func HandleInitiate(c *gin.Context) { return } - if req.ContentLength <= 0 { - c.JSON(http.StatusBadRequest, gin.H{"error": "invalid content length"}) - return - } - - if req.ContentLength > int64(svr.Config.MaxUploadSize) { - c.JSON(http.StatusBadRequest, gin.H{"error": "file is too large"}) - return - } - - // TODO: perform proper validation of filenames - // See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html - if len(req.Filename) > 128 { + if len(req.FileName) > MaxFileNameLength { c.JSON(http.StatusBadRequest, gin.H{"error": "invalid filename"}) return } - uid, err := uuid.NewRandom() - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to generate UUID"}) + if !utils.IsValidETag(req.ETag) { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid etag"}) return } - key := path.Join(uid.String(), req.Filename) - - u, _ := url.Parse(cfg.BaseURL) - u.Path += filepath.Join("api/v1/download", req.Server, key) - if req.ShortenLink { - if shortener == nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "shortened URL requested but nut supported"}) - return - } - - u, err = shortener.Shorten(u) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } + resp := initiateResponse{ + ETag: req.ETag, + Parts: []part{}, } - reqCreateMPU := &s3.CreateMultipartUploadInput{ + // Check if an object with this key already exists + respObj, err := svr.HeadObject(&s3.HeadObjectInput{ Bucket: aws.String(svr.Config.Bucket), - Key: aws.String(key), - Metadata: aws.StringMap(map[string]string{ - "uploaded-by": c.ClientIP(), - "url": u.String(), - }), - } - - log.Printf(" req: %+#v", reqCreateMPU) - - respCreateMPU, err := svr.CreateMultipartUpload(reqCreateMPU) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - - parts := []string{} - numParts := req.ContentLength / int64(svr.Config.PartSize) - if req.ContentLength%int64(svr.Config.PartSize) > 0 { - numParts++ - } + Key: aws.String(resp.ETag), + }) - for partNum := int64(1); partNum <= numParts; partNum++ { - partSize := int64(svr.Config.PartSize) - if partNum == numParts { - partSize = req.ContentLength % int64(svr.Config.PartSize) + u, _ := url.Parse(cfg.BaseURL) + u.Path += filepath.Join("api/v1/download", req.Server, resp.ETag, req.FileName) + + // Object already exists + if err == nil { + if req.ShortURL { + origShortURL, okURL := respObj.Metadata["Original-Short-Url"] + origFileName, okName := respObj.Metadata["Original-Filename"] + if okName && okURL && req.FileName == *origFileName { + // This file is uploaded with the same name + // So we can reuse the already shortened link + resp.URL = *origShortURL + } else { + if shortener == nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "shortened URL requested but nut supported"}) + return + } + + if u, err = shortener.Shorten(u); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + resp.URL = u.String() + } + } else { + resp.URL = u.String() } - - // For creating PutObject presigned URLs - req, _ := svr.UploadPartRequest(&s3.UploadPartInput{ - Bucket: aws.String(svr.Config.Bucket), - Key: aws.String(key), - ContentLength: aws.Int64(partSize), - UploadId: respCreateMPU.UploadId, - PartNumber: &partNum, - ChecksumAlgorithm: aws.String("SHA256"), + } else { + // Check if an upload has already been started + respUploads, err := svr.ListMultipartUploads(&s3.ListMultipartUploadsInput{ + Bucket: aws.String(svr.Config.Bucket), + Prefix: aws.String(resp.ETag), + MaxUploads: aws.Int64(1), }) - - u, _, err := req.PresignRequest(1 * time.Hour) if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + c.JSON(http.StatusBadRequest, gin.H{"error": "failed to get uploads"}) return } - parts = append(parts, u) + if len(respUploads.Uploads) > 0 { + upload := respUploads.Uploads[0] + + respParts, err := svr.ListParts(&s3.ListPartsInput{ + Bucket: aws.String(svr.Config.Bucket), + Key: aws.String(resp.ETag), + UploadId: upload.UploadId, + }) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "failed to get parts"}) + return + } + + for _, p := range respParts.Parts { + resp.Parts = append(resp.Parts, part{ + Number: *p.PartNumber, + ETag: strings.Trim(*p.ETag, "\""), + Length: int(*p.Size), + }) + } + + resp.UploadID = *upload.UploadId + } else { + meta := map[string]string{ + "Original-Uploader": c.ClientIP(), + "Original-Filename": req.FileName, + } + + // Shorten link + if req.ShortURL { + if shortener == nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "shortened URL requested but nut supported"}) + return + } + + u, err = shortener.Shorten(u) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + meta["Original-Short-Url"] = u.String() + } + + respCreateMPU, err := svr.CreateMultipartUpload(&s3.CreateMultipartUploadInput{ + Bucket: aws.String(svr.Config.Bucket), + Key: aws.String(resp.ETag), + Metadata: aws.StringMap(meta), + }) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + resp.URL = u.String() + resp.UploadID = *respCreateMPU.UploadId + } } - c.JSON(http.StatusOK, initiateResponse{ - URL: u.String(), - Parts: parts, - UploadID: *respCreateMPU.UploadId, - Key: key, - PartSize: int64(svr.Config.PartSize), - }) + c.JSON(http.StatusOK, resp) } diff --git a/pkg/handlers/part.go b/pkg/handlers/part.go new file mode 100644 index 0000000..8eca763 --- /dev/null +++ b/pkg/handlers/part.go @@ -0,0 +1,75 @@ +package handlers + +import ( + "net/http" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/gin-gonic/gin" + "github.com/stv0g/gose/pkg/server" + "github.com/stv0g/gose/pkg/utils" +) + +type partRequest struct { + Server string `json:"server"` + ETag string `json:"etag"` + UploadID string `json:"upload_id"` + Number int `json:"number"` + Length int `json:"length"` +} + +type partResponse struct { + URL string `json:"url"` +} + +// HandleInitiate initiates a new upload +func HandlePart(c *gin.Context) { + svrs := c.MustGet("servers").(server.List) + + var req partRequest + if err := c.BindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "malformed request"}) + return + } + + svr, ok := svrs[req.Server] + if !ok { + c.JSON(http.StatusNotFound, gin.H{"error": "invalid server"}) + return + } + + if req.Number <= 0 || req.Number >= utils.MaxPartCount { + c.JSON(http.StatusNotFound, gin.H{"error": "invalid part number"}) + return + } + + if !utils.IsValidETag(req.ETag) { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid etag"}) + return + } + + if req.Length > int(svr.Config.PartSize) { + c.JSON(http.StatusNotFound, gin.H{"error": "invalid part size"}) + return + } + + // For creating PutObject presigned URLs + partReq, _ := svr.UploadPartRequest(&s3.UploadPartInput{ + Bucket: aws.String(svr.Config.Bucket), + Key: aws.String(req.ETag), + UploadId: aws.String(req.UploadID), + ContentLength: aws.Int64(int64(req.Length)), + PartNumber: aws.Int64(int64(req.Number)), + }) + + u, _, err := partReq.PresignRequest(1 * time.Hour) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, partResponse{ + URL: u, + }) +} diff --git a/pkg/handlers/types.go b/pkg/handlers/types.go new file mode 100644 index 0000000..086f04b --- /dev/null +++ b/pkg/handlers/types.go @@ -0,0 +1,9 @@ +package handlers + +type part struct { + Number int64 `json:"number"` + ETag string `json:"etag"` + URL string `json:"url,omitempty"` + Length int `json:"length,omitempty"` + Offset uint64 `json:"offset,omitempty"` +} diff --git a/pkg/notifier/notifier.go b/pkg/notifier/notifier.go index 522da09..92ad0fa 100644 --- a/pkg/notifier/notifier.go +++ b/pkg/notifier/notifier.go @@ -5,15 +5,15 @@ import ( "fmt" "math" "net" - "path/filepath" + "net/http" + "regexp" "text/template" + "time" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/containrrr/shoutrrr" "github.com/containrrr/shoutrrr/pkg/router" "github.com/containrrr/shoutrrr/pkg/types" - "github.com/stv0g/gose/pkg/server" "github.com/stv0g/gose/pkg/utils" ) @@ -26,6 +26,9 @@ type notifierArgs struct { UploaderIP string UploaderHostname string Env map[string]string + ExpiryRuleID string + ExpiryDate time.Time + UploadDate time.Time } // Notifier sends notifications via various channels @@ -56,33 +59,38 @@ func NewNotifier(tpl string, urls ...string) (*Notifier, error) { } // Notify sends a notification -func (n *Notifier) Notify(svr server.Server, key string, params types.Params) error { - obj, err := svr.HeadObject(&s3.HeadObjectInput{ - Bucket: aws.String(svr.Config.Bucket), - Key: aws.String(key), - }) - if err != nil { - return err - } - +func (n *Notifier) Notify(url string, obj *s3.HeadObjectOutput, params types.Params) error { env, err := utils.EnvToMap() if err != nil { return fmt.Errorf("failed to get env: %w", err) } data := notifierArgs{ - FileName: filepath.Base(key), + FileName: *obj.Metadata["Original-Filename"], FileSize: *obj.ContentLength, FileSizeHuman: humanizeBytes(*obj.ContentLength), FileType: *obj.ContentType, Env: env, + URL: url, + UploadDate: *obj.LastModified, } - if u, ok := obj.Metadata["Url"]; ok { - data.URL = *u + if obj.Expiration != nil { + re := regexp.MustCompile(`([a-z-]+)="([^"]+)"`) + for _, m := range re.FindAllStringSubmatch(*obj.Expiration, -1) { + switch m[1] { + case "expiry-date": + if expiryTime, err := http.ParseTime(m[2]); err == nil { + data.ExpiryDate = expiryTime + } + + case "rule-id": + data.ExpiryRuleID = m[2] + } + } } - if upl, ok := obj.Metadata["Uploaded-By"]; ok { + if upl, ok := obj.Metadata["Original-Uploader"]; ok { data.UploaderIP = *upl if addrs, err := net.LookupAddr(data.UploaderIP); err != nil && len(addrs) > 0 { diff --git a/pkg/server/server.go b/pkg/server/server.go index 557b8d8..206b6f8 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -42,12 +42,12 @@ func (s *Server) GetObjectURL(key string) *url.URL { return u } -func (s *Server) HasExpirationClass(cls string) bool { +func (s *Server) GetExpirationClass(cls string) *config.Expiration { for _, c := range s.Config.Expiration { if c.ID == cls { - return true + return &c } } - return false + return nil } diff --git a/pkg/utils/etag.go b/pkg/utils/etag.go new file mode 100644 index 0000000..48b157c --- /dev/null +++ b/pkg/utils/etag.go @@ -0,0 +1,26 @@ +package utils + +import ( + "crypto/md5" + "encoding/hex" + "strconv" + "strings" +) + +const ( + MaxPartCount = 10000 +) + +func IsValidETag(et string) bool { + p := strings.SplitN(et, "-", 2) + + if etag, err := hex.DecodeString(p[0]); err != nil || len(etag) != md5.Size { + return false + } + + if num, err := strconv.Atoi(p[1]); err != nil || num > MaxPartCount || num <= 0 { + return false + } + + return true +}