From 5a2571c35f452ca99ddef107c184e94c52ad5051 Mon Sep 17 00:00:00 2001 From: Schahin Rouhanizadeh Date: Tue, 10 Sep 2024 19:01:21 +0200 Subject: [PATCH] Introduce `nativelink-bridge` This is a fast prototype for subscribing to the redis/dragonflydb "build_events" channel and decode them properly via protobuf and fires them via websocket to the browser. --- flake.nix | 3 + nativelink-config/examples/basic_bes.json | 177 ++++++++++++++++++++++ tools/pre-commit-hooks.nix | 1 + web/bridge/.env.example | 3 + web/bridge/.gitignore | 139 +++++++++++++++++ web/bridge/README.md | 78 ++++++++++ web/bridge/bun.lockb | Bin 0 -> 35076 bytes web/bridge/image.nix | 53 +++++++ web/bridge/index.ts | 3 + web/bridge/package.json | 22 +++ web/bridge/src/eventHandler.ts | 75 +++++++++ web/bridge/src/http.ts | 37 +++++ web/bridge/src/index.ts | 73 +++++++++ web/bridge/src/protobuf.ts | 98 ++++++++++++ web/bridge/src/redis.ts | 25 +++ web/bridge/src/types/buildTypes.ts | 20 +++ web/bridge/src/utils.ts | 28 ++++ web/bridge/src/websocket.ts | 42 +++++ web/bridge/tsconfig.json | 27 ++++ 19 files changed, 904 insertions(+) create mode 100644 nativelink-config/examples/basic_bes.json create mode 100644 web/bridge/.env.example create mode 100644 web/bridge/.gitignore create mode 100644 web/bridge/README.md create mode 100755 web/bridge/bun.lockb create mode 100644 web/bridge/image.nix create mode 100644 web/bridge/index.ts create mode 100644 web/bridge/package.json create mode 100644 web/bridge/src/eventHandler.ts create mode 100644 web/bridge/src/http.ts create mode 100644 web/bridge/src/index.ts create mode 100644 web/bridge/src/protobuf.ts create mode 100644 web/bridge/src/redis.ts create mode 100644 web/bridge/src/types/buildTypes.ts create mode 100644 web/bridge/src/utils.ts create mode 100644 web/bridge/src/websocket.ts create mode 100644 web/bridge/tsconfig.json diff --git a/flake.nix b/flake.nix index f9f70a972..c0c714183 100644 --- a/flake.nix +++ b/flake.nix @@ -279,6 +279,8 @@ }; }; + nativelink-bridge = pkgs.callPackage ./web/bridge/image.nix {inherit buildImage pullImage pkgs;}; + nativelink-worker-init = pkgs.callPackage ./tools/nativelink-worker-init.nix {inherit buildImage self nativelink-image;}; rbe-autogen = pkgs.callPackage ./local-remote-execution/rbe-autogen.nix { @@ -419,6 +421,7 @@ nativelink-worker-init nativelink-x86_64-linux publish-ghcr + nativelink-bridge ; default = nativelink; diff --git a/nativelink-config/examples/basic_bes.json b/nativelink-config/examples/basic_bes.json new file mode 100644 index 000000000..c4e91f098 --- /dev/null +++ b/nativelink-config/examples/basic_bes.json @@ -0,0 +1,177 @@ +{ + "stores": { + "AC_MAIN_STORE": { + "filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-ac", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-ac", + "eviction_policy": { + "max_bytes": 100000000000 + } + } + }, + "BEP_STORE": { + "redis_store": { + "addresses": [ + "redis://@localhost:6379/0" + ], + "response_timeout_s": 5, + "connection_timeout_s": 5, + "experimental_pub_sub_channel": "build_event", + "key_prefix": "nativelink:", + "mode": "standard" + } + }, + "WORKER_FAST_SLOW_STORE": { + "fast_slow": { + "fast": { + "filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", + "eviction_policy": { + "max_bytes": 100000000000 + } + } + }, + "slow": { + "noop": {} + } + } + } + }, + "schedulers": { + "MAIN_SCHEDULER": { + "simple": { + "supported_platform_properties": { + "cpu_count": "minimum", + "memory_kb": "minimum", + "network_kbps": "minimum", + "disk_read_iops": "minimum", + "disk_read_bps": "minimum", + "disk_write_iops": "minimum", + "disk_write_bps": "minimum", + "shm_size": "minimum", + "gpu_count": "minimum", + "gpu_model": "exact", + "cpu_vendor": "exact", + "cpu_arch": "exact", + "cpu_model": "exact", + "kernel_version": "exact", + "OSFamily": "priority", + "container-image": "priority" + } + } + } + }, + "workers": [ + { + "local": { + "worker_api_endpoint": { + "uri": "grpc://127.0.0.1:50062" + }, + "cas_fast_slow_store": "WORKER_FAST_SLOW_STORE", + "upload_action_result": { + "ac_store": "AC_MAIN_STORE" + }, + "work_directory": "/tmp/nativelink/work", + "platform_properties": { + "cpu_count": { + "values": [ + "16" + ] + }, + "memory_kb": { + "values": [ + "500000" + ] + }, + "network_kbps": { + "values": [ + "100000" + ] + }, + "cpu_arch": { + "values": [ + "x86_64" + ] + }, + "OSFamily": { + "values": [ + "" + ] + }, + "container-image": { + "values": [ + "" + ] + } + } + } + } + ], + "servers": [ + { + "name": "public", + "listener": { + "http": { + "socket_address": "0.0.0.0:50052" + } + }, + "services": { + "cas": { + "main": { + "cas_store": "WORKER_FAST_SLOW_STORE" + } + }, + "ac": { + "main": { + "ac_store": "AC_MAIN_STORE" + } + }, + "execution": { + "main": { + "cas_store": "WORKER_FAST_SLOW_STORE", + "scheduler": "MAIN_SCHEDULER" + } + }, + "capabilities": { + "main": { + "remote_execution": { + "scheduler": "MAIN_SCHEDULER" + } + } + }, + "bytestream": { + "cas_stores": { + "main": "WORKER_FAST_SLOW_STORE" + } + } + } + }, + { + "name": "private_workers_servers", + "listener": { + "http": { + "socket_address": "0.0.0.0:50062" + } + }, + "services": { + "experimental_prometheus": { + "path": "/metrics" + }, + "experimental_bep": { + "store": "BEP_STORE" + }, + "worker_api": { + "scheduler": "MAIN_SCHEDULER" + }, + "admin": {}, + "health": { + "path": "/status" + } + } + } + ], + "global": { + "max_open_files": 512 + } +} diff --git a/tools/pre-commit-hooks.nix b/tools/pre-commit-hooks.nix index e690341ae..e78747383 100644 --- a/tools/pre-commit-hooks.nix +++ b/tools/pre-commit-hooks.nix @@ -65,6 +65,7 @@ in { # Bun binary lockfile "web/platform/bun.lockb" + "web/bridge/bun.lockb" ]; enable = true; types = ["binary"]; diff --git a/web/bridge/.env.example b/web/bridge/.env.example new file mode 100644 index 000000000..0442389be --- /dev/null +++ b/web/bridge/.env.example @@ -0,0 +1,3 @@ +REDIS_URL=redis://localhost:6379 +NATIVELINK_PUB_SUB_CHANNEL=build_event +POSTGRES_URL= diff --git a/web/bridge/.gitignore b/web/bridge/.gitignore new file mode 100644 index 000000000..322378173 --- /dev/null +++ b/web/bridge/.gitignore @@ -0,0 +1,139 @@ +# Logs +logs +_.log +npm-debug.log_ +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Caches +.cache + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json + +# Runtime data +pids +_.pid +_.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional stylelint cache +.stylelintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files + +# Comment in the public line in if your project uses Gatsby and not Next.js + +# https://nextjs.org/blog/next-9-1#public-directory-support + +# public + +# vuepress build output +.vuepress/dist + +# vuepress v2.x temp and cache directory +.temp + +# Docusaurus cache and generated files +.docusaurus + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +# IntelliJ based IDEs +.idea + +# Finder (MacOS) folder config +.DS_Store diff --git a/web/bridge/README.md b/web/bridge/README.md new file mode 100644 index 000000000..6556be50c --- /dev/null +++ b/web/bridge/README.md @@ -0,0 +1,78 @@ +# NativeLink Bridge (Experimental) + +Make sure you are running an instance of Redis or DragonflyDB in your network. + +For DragonflyDB inside docker run: + +```bash +docker run \ + -d --name some-dragonfly \ + -p 6379:6379 \ + --ulimit memlock=-1 \ + docker.dragonflydb.io/dragonflydb/dragonfly + +``` + +For Redis inside docker run: + +```bash +docker run -d --name some-redis \ + -p 6379:6379 \ + redis +``` + +Set the Redis URL and the NativeLink pub sub channel ENV variables in `.env` as defined in the `.env.example` + + +The Redis URL format: 'redis://alice:foobared@awesome.redis.server:6380' + +The NativeLink pub sub channel ENV variable should match `experimental_pub_sub_channel` inside `nativelink-config/example/basic_bes.json`. + +Make sure you have set also the `key_prefix` in `nativelink-config/example/basic_bes.json` + +## You need 4 Components + Redis + +### 1. NativeLink + +Start an instance of NativeLink with the basic_bes.json inside the `nativelink-config/example/basic_bes.json`. + +### 2. NativeLink Web Bridge + +Install the dependencies and run the bridge: + +```bash +bun i && bun run index.ts +``` + +### 3. NativeLink Web UI + +Inside the web/ui directory run: + +```bash +bun i & bun dev +``` + +Now you can open http://localhost:4321. + + +### 4. Bazel + +Now you can run your Bazel build with NativeLink and see it in real-time going into the web app + +Include this in your .bazelrc +```bash +build --remote_instance_name=main +build --remote_cache=grpc://localhost:50051 +build --remote_executor=grpc://localhost:50051 +build --bes_backend=grpc://localhost:50061 +build --bes_results_url=http://localhost:4321/builds +build --bes_upload_mode=fully_async +build --build_event_publish_all_actions=true +``` + +Make sure to use the right IP, if it's not hosted on `localhost` + + +```bash +bazel build some-target +``` diff --git a/web/bridge/bun.lockb b/web/bridge/bun.lockb new file mode 100755 index 0000000000000000000000000000000000000000..e2abd5271b79d1a49b444c88c3dd303e50b4146e GIT binary patch literal 35076 zcmeHw2|QHa`~T3CrLs$#qS7M9nk5v9v>;0=34_7dhMBQMi*{*|D5((IY1c-IN~J|a zdsJwVR;9iCpL6FPbNhUKihN)H*YEp!b$UIWIrq8G`#k44&spxd*Exe#wZeH^EiZP6 z7Aqt|#Vb5S5=b+M;~nVB4q|Ega=5`vzGj5Bq$rI>b1!?UI9xZkxNo8TF3Tecs}^74 zY|%kp9K;9@qA9E}#s zg}5igbC^LvY#yIxB|@Y1fb=McWgy-_@>fFK9nuRRMtol~R)<&;(m|XMKln*|(nWCJ zEV|&nn~+90=gIge8D~Q*57*a0EDP~MGS-4v5z=gkdqeC3F}iON&z4v+80W$^|= zUPvDWGGmWw@%!e5AiVYxR7c#aaV+I-bCF5>n+yt2-xd#xVa$F(f6J)%DjFTZo`Ys~lP%?HW z<6tV&%0$6+9O_?OEi1$%O2T9>Bfw==4lsB zR@`DoWivPLPSkQ;J@kl~u>{BT{Bws*1>FNfww+lPcW=t3_|fK`ALi{WwKGhkZ8-eQ z$eTA}U6XjryC?B+&EsCyzc-Fs7Wp!&_x^ktTT}nhn|$X~rb-v>?sqXhM*GX4LbVY# zwNW8Ax(?EPn{(S@(3FIsnR8d9JLC_pI1n7~;^!r`^U8LIlxi=#Q-{VzEch{EgG{W2 z-t2u=^)K@K7O8x%SkiJN{cCT<5ShVd<08f09jnNcQ9d`k!9K_{tL0|m35y(S_H6Fh zVOx4Cjc-Wb7jeOHO!>(E%CbAt`)TJ_Hh8;U{17{E?zU-3NhU7_2dv%{=hf?*g!Dj# z59)Hm^+Z5kR2VBwR?WRD{-~!+LoI~$Rm9^q7#ajfm%d0}B zqd^{q6*OMbq}n44_h5NHkhdW7?+e$F9_%hz{OUIi4U=8tiI zR|b5a8^~LBLjQD-A4AGh%hlfe?^ERe#D0CD)3K)LkIU8G{5?T_3Pt{Ra+r_v$pv|2 zKb(f@Y*)bdzXEx4O8)rz??CW<3}|>pAdlJ&${Wntu7KrzL4GXABcUPfk`&y8<$u7V z2$t^$toGI+mXCr?dm`kI(qM`X>t9dGqw*udxbX9r0pB+kCZ4ur`H}5l?%%F}|5T8l+zI>DgFNzoRB3-=KTYY*$wzcj{&Xkh#o=MLGv#;ar2M8% z%GY&LUIP|GCUm0yd7YHc@1(p4IBjS2w*h&pPLzLjC*`kqQeFwxXF8+58_1hO`RUO9 z(W+iY$B$f)$ImYy-tK_=|Mwt2jeD&-h)&8Ec2b@Wiw3y= z|C91h?4I~EMEZf zqd*?XB0^(udjgh!1M(&mc`#?Y0+!eBPpsj_qI|mop*5`b1T4=7dG!2=WD%ixM0*3W zH6k=_!($XtezbmuYzWgrBD57FJHfPxxF7jEN~7ma6vJ>vltC3@6i|$=4?+nD9f?sH zhY(Wji@_8$b0B1c@jyu4nv7wNLZl}_i~===A&BOQE<*4m{{P>IkzJXjzSJ0%+Zzam zCR#Mf|6hzz`C>@@QH;`afl&G8lWFvR8i&9BEb!~U{``N_?>pwtKcs&kf&XRzJQ11z zmkLcU`@F5cxpME)A4|?nKVB4Eu++c)M~<@i<*kt~3mPo;?C&$BXNAPn)3da{ezdM1 zzH4=_^y;TA_F*c`%}w0Wer4elT+}vkp!1x~-j}h~oMG^mDf?>-oNDzV^TRgIEz{4N z2i(ndV&6MyyL+GGpEc z@z=IeG8Uf}uBgqEcwO`DMxW57h0CjVygoA|r@l7vfyP_iZ^@~b@~3`vcUc;FMG{&E>OIkO+=bP1`YO3C<`N7+)F1OS< z@4VBnw&MO4ix38jTazHVe{rTp;W3u;>7||%G+$6~(O8TFeffyRqhDD(4S#L8M83J) z*H8Y|L$yr$yUUB}@4P?THOl3*dQY91)0>hX+a_yMAfpm%`q!oX+V!zm>lD z6$O|2j7=YZ>3y$$9J!%O#u zd()D2`>9{sq}!M(9Y(>Gz{Z7`t}sqMOy^Fvfm>W2>zRL+$^gB-A-ww8B@O$Q`e@`V zP?|JametoLcF**KGm3bUj4ok)b>kWtn$4%U#@4e`bxbI@Xbl7hdjH_Xy$mEnl6mRMSHZyyTvcsY=j&1+I!pA6p~blwO&R)|Gg8ANW_#8z zWVpsiQgC6JzBSNO6?;isz2so0eRGuQfgl@+u$I`W!iOdoZ}td~dc#|0HgWsqL8%hG z<7z$5oUOWKCH{Cr?lza}i##XG4iw+p^JNY+GF)%e`67LN@tbXTRTk3ALVmbz^R~3T z9%!=Pz5kObD`RcDKd;mnKQ}H^z3&mlsN)ObkJ#v)tUT*m<|Zxwyv{!Q*&CzdWA9M( zMe{)%=m$&BKCNPVjtxH_HPS-#{95~b$AeX44dYq<%vB1T?(?^uF(~IP7?Z>IS9Q=X z5lKAD80zLhOUuet|K>5aMAn{2l~*dt4y9|@+~FzID{o-4RncBaPilBSz>Z! z?4#xgevaX;gQ`!auD827M5}Pg#+h@vx@pDDJ+w~sMCGIVR8yL|q5 zb<-n@)FE{H?voxLi{{MbS%k*E8g{|ti*eOK(<~dA9R@+Z7cv?$doQlpuWNKnEX==U zY~{<*{riS4m~X5_!G-1B)+bF$@-!-V=(A?{;}Z}arJGWOotSW1(A&|gYS{;tZX6%6|<(d z>%?Fg3ND(L<3LZWN}qSOdS9)VLWyI@0{xpcS_=CviT&`5^t*lPB=h(Nl{bqTH!W}+ zq4wcO%CrWLZHF3Mb~Em}Yr9#^i7bll(oDfc^L!lWJ#YRv~v zzoU|=bRzHZmt(dbH(h#-Zt}BvQ=A-p{D$Z|-r(x3RWCvut0JDAx3QBg%e&Kuf{UIT zaG-CPal?I6%AEY5_)!H_A6zAt$A#MYXSf+lIqN=Ve(8O*WdES5T$xWF=RxPQa8gQz zTHpYcQ(E7$AJW~%87-f*oq~&=LvWxcJd>1{WiY%BoHi?4D4%llP{h>WgKvgMFxTq@ z^qzm%AwoIhfXIbp{S#W3b1#WEJ0#klV)=}W?58lN>l3aNBa?!Q=2ke+6K8z;i@WfhY6mdz!zdM?`IbMA6K)gA99t$w(5_}M`p)BFCIT^T*}L1p*X&a*ci zT)is9(dwc7cM2|=tK&fTR&|Y^wCYLm`!dy;;?-|^Fy2M^F8p3C?r_>t#AK_oZ*`Tk zRdJAJ?er+)#nP^?T&pAVZZc!19!T3GuQq0O;ZO=LEYGwCx=3n&niOZtgVzt=JRdn& z_K91}(#^r9Mxq;94CPx&v~N4y(d(~i<6&h!LDFaY^|-<-7rRJpa~?YWpvl~!?=n&t z6kJ#~X$|zyQCl;2h4*>!J-7Noo%@OHPt10I8J}Gr+vqBDdY@hOIcIrSJGn(J#kWGa zSxeqV%vgQ&u~F&_DaRYN0}2KlRFtCNQlFpc&t&C2KTeuB;C_$LJl2=gBv#}hfOE_%-cQ@!aLq54Nej1nzZmz(I9AL%o0NA!CNEAIY>z>flhT)SyEtuh`{R;=3g(-k z#ofAmmh%nU`@Z60wfgpmsFRJ)x7f`HS906OQgDAfhEKs&#-I?>54<zPBH*%jd|kkIFxLH#C&)F>Pk$A5B@mY4?i8X(xLbd+RENMN#KFDi{=E zy8PpEiT-lIJF+DEah}^PJ|i}UGjQSIoL9pWFQlXnzwpL&#Bnc=(Q8dakB7cIdU2xl zPVYsVOcj@!UD(`P*OXm4kD{+C6*uhp!OW-5d(R(Qe$<3De|7lLvsZlVR;pRVGNktn za#|v{&bfK#7Kb~c6E5C#bb2e|VTXgj-#3h3C0#7W_W=-Zl+OMK)tDE)RfGP^^5Grn3_{sFGPPxy=X?&9Q zzPJAC_i+h*WnPRHx&2^3=9N8f5~k;8#dK5Y(c{AR47ZtjRS#!b&MnK|r+T{R)z#vm zKN{B%xO8F-O^u3sX>0jFhKzD!gLb!r%jC2@-)U7*OVi zGeXn#muajxx?n}3LbU&nPgnD2MfRMqalSo?i{`6Csknw?Myg#&ZxQ_^eA8s014^d6k-_&t?knj|iI z1{g-gE)=JMjT$QHh!%rM2a%)EH1hte^KK4b$&DD98 zMK5hn#2nwS^>)mX`kUsy5?2M|F`X6*n8T^K;WxWZ$kOzZ?3=Ul(@-B7e^KMZCmPCb z-FPTI+TgesJ6LufFJ)IBmEv{J<~cvkneIAGzvr^+##zIXPK~#px2Z<(+(Hvq0n8Cp z+?bT(%#jR>=1)8&-Nn*X;!X6ix&2g5=f=e7n3ZSg9Gu)WFfV$j-LNCOq@M?i=RaA! zcG3;D$>6OmgL$6oE!LbT>_8Jma~*Xm?$@iws=PI7Jn2UVtO%SDU$Hl6T(s)EOT6uJ z7n&NkJZ)OvV`k03T`y_tO8V?2i(Q~r(d*Q>XVtmt5?!;I0gLKBJ-lxaqw+Y|aAxrXYcKn`JrfJf zSK6#Fw$k8ENRa=iK-L?=g|*+-K)MW{!zR^D0>#}R3Pw~MYFW0dMjc|VGo$Cf{R zH&d_c@Z5{MJCix8$vfDa-|nZalwa96OXdfYS)(&pc_By8|I)?v9VWZz`^y($y&z@&(WXP7`zy-o#__2_O8^tgrNQ&MaKr$JxNcGi{ak5A3SG` z`HFA-*X)qcFxC;+R~-2-gNpmcvwD49X_uleWiB(+JnGW(%vbIyi@Uj~dz0Jxk_NHb zxcraBhVj~r+j`!zbxK@Jh|Pp!U->gy4x2o9cP78f$u6#= zvrkMWaK#D#)u!UQ{wVLJc~SaHg|V5b4b5rtvGuizAD)-Y&fi*Q`*O-j@4=(0&#ij# ziF2|hMozYonWSQwGk5(Zt?MNv1}aIz_6hoR!4kU;6*ums+t|o_hJ@V9) z$IQIer`&&Y)<}~$-QjPooaM5%E*k0*FI%5rm}Yq3@wkoqY#*v@oaH-Escu!?Gg4pV z5AZIbHPFW{-#be(f6zI1dy7%S!;LBq@4Qm{Y+&JC_l)RX*W^{l9DW~jvfutZ8~%Y0 znpp<}&fbl9dv}j<>7-amF%RdUm?;D zral!n(a^){TvpHb{R6)hny(nP*ml#&jMWm`v~n+GZzzhi>i7A8eW3O7g{Dh3`K~+I z-}2prUR=hz;KNQIUIxFJ{#s=YiE9p+@J^#O(AOH2jCbXGC6`ZgeeO_WWxHZS_8MNF z?KKq(=vAD{!>8t*zQQjIcx3FeQq@p2WAEeCbvkqFl=6nXT=_oC@Vj;ziHq!O(25Fn zaEQ;^-LQK}O3woh8i{2eS4$OtFw$Nfa6M4NwV09Y>|*VDfOd3BU5Wb6%u-7a#k~te ztFPMcw@c~i)z_~sCP6TN5GDEtLn>~|h6`&;cIG&!s25KAyzE`gRp!j-A+J13rk*nj z4d#n9A1f*}FdORi#DeX)ry3EQ^wUgTa;^+>Y;U+tFkugI-g zJeoBx=$_77YYMJ0758BMQ9HBzDT;R8y=`~_WidfVeO}cpkdx0l>egrQ&i*{RaR;`o zF!wlXsePw(^m$9wv7(QLU%&oF{?w95ehVc6u2FDDQ*p1&yO?rf4u4lwEUUZS)*y2Q zZLYKCkwCi0eOtaU#+7+ikI?G{c7m4eLX%4?ezfQzHcz z-chv%`asV07oSfCman||jQzf`?#uyZ{nxkU3)G!o*cOzeN8U8q^CMdOYrrSws<(3N zoLy?3soz~M1vb4fNl6d6G~#MB1=q9{75u@U9M4Aw4R+_U_sir7?! ztk_s9w=XY}G?J?wM=h5+Q?3v%R+&~nU-7!aNixS;Ct$?@3NCyD+ZyN$wQ9GX8u#N9 zt&Q{dx~H%`T&Tk$w`T4S%CleFwOJ;v< zIruF}arkiMu>NAtx-^C)O+WDM$H;w=meOKkUz7^NL)MbG2;YK=JJw23-)m5G)Sc6j zpUd`k8Osa4y>M(zs#V#ObsuwTm%VwfT(kVgzFxB}c6XO)kv~6jV&v6Q`Fq<(pRzwc zHGauHQxX^Tla^Fmv8(};x_RbJIp2MC_JM?rUax1~=O&)d=wlzLzU;G?Ote-`@pTVh zr6wo;P~)#3Xz^;ib^eny-CF8eWyg2Sv$(Ug zx1rG!_sJ`a;+NfybZ+!koV-wMyKP}zc50^HFe{7twF7h4fX`UGg>VR?EA~B(BNRm8jd>*fnhb zY0Keb`hUsTxMcXps!$(hNeze4S1^yVqT+t~w7u%yM@PTh+TbFeJ+XIoKUlJH{l@Z* zYd@+MIoir3%^BAx>F}(>ml^V>JR2qCoo-8pPTW)ba-B-<$K(r3`v{(o1@kEMJ`)Fe z+9LCN@1^5C%SS5*j_^y7c zUUY75ZBoFlqqU~Z7itCbK049=PQaiL)8&@bm9AQO!s!9O=|!Ju)4HF}+d0*$*H^Z= z_5#m?s_eNLEgSA%%}h6RpPD#u%_GL-sEi1^J(`8DYHu2Ue3Gk0ok!VFab@F`{Fcu> zU+(W*b4I@F#@madmaX}6-au)$+Lcikx=vQ!z9@THi*vDAuO*%o|vG z#(hqA$*eb~iv;#1=1~)=xHWcnGOhB*?NDEO+a_TDO}~S0x}}zMn^stv=vrH*K4VLk z{xkU-m3ySr%+1|I1|Ao&+SL3?v@v;V;uZJoUIQ-05V(TQ2fr6a@bo>>mlJ00yFYAd zvJQPkz~k=AH<_hPD!bopSb6lY*I)X*{uI+=;^O1#2^ZEg1Mg-(wT_w#2CTaKVM=S4HQn z6Z^u}$qC)C^uX=)+VdsUlLqY^bn2Duusc>)U+(Wc0^0!)r?jGizBRMF$DL@-7WGst zSk&m0{q^yqW5$Oz#bzz3@U$;e5cTpqm-3|cyy%KSm9saGue5UUO1i?lmTKu4U6PmP zqAW3k)EABO@J(53pxgE9TNltc)+1TPcf++cuSzSf)1S#Z&J`)Td*XwQ=U@rVk-5?3 zQTyi=9Q-iYq2ke*fqE-f9qvAFT=(wk7K!Vx^e6R2{exXAD(JgpkK)?gk`<%n7cDq` zGc)s`<@8>TgU=<@Ot-?UnZ1s#O(OFM-G^3vuj-Jg@8%qN#kjkJ zR@S+VJhowo$Z)4^GcQg&vEDguwegmS*m+oAb3mNhiVFJn-&I=N_h!@My%G}TCSwD_ zY@ax6baLok`*fRWkN1}~1}+Yi9At6(h1dSyV+=%=sBUUBZrZzR5xznq* zDpO>Uq<_Wx{FL!W(j#`C@f!3%?er(_6#;3J_DN}%N10xa-LsH_JB^C_Y$oqS-esTQ z(mVUFI;*Hf*?5=j^FJ@rHOb)cb7?m=U-z8c$vbBDD!W2Mo>OqAQ*kqnpN-kO@__QG8TK;oJ||V@)|>6v zt3IRU_LYR^y^n3oxpwWY^tJ&jW;H#2miSV?AT6NSD*xldbuM!9&l{rxixVigGpM*V zl@;s0k3VYOP&Pw%ZbdQ;_z|mapQ(m=_FEcXHs!>?;ibdW--lL`sHqK>+bfWlV2Nucw@zOY^bcbCSEQ7 zFmJ-LMvu42mt?QcIe$Uh@hbh8gv!@>=^osE$#-svE&mL-f5Tr3{F?<}WG22ZgLkO0 zu%QOSJAn6ZxBgeJqkQo`3;iH-l>dJ<}IwwmN3ujh7kDTo!ALimtW_j~&h8_-d%=t1yGuejyy+)<6CE(IN=Q8kRt4 zjRn8Y$7!_whu*~_9{M%`{at{HEBpsx$t*!EX&SDlI|efNm#mO1!a;xAhSq@4-;ANZ z??Qj;Y7S%pgz`i9=9DUo3zQa}lLVwSNz8{eQ zLIs1hYvS*+=RxcZrbe`r91z&NCXw69|24fW8Yr@9>cg(L49aK5WMgt*#G6F(Ah3XLbl?o8@Ekhs!piw~jKzcyBKsrF$ zKw3ZyAWa|*ppii8KqG*L0}TTj3Zw=!1ZXhOARuEPRKG|MRM)7^QQadSK)!%{0{I5= z5#%e#XORCOKSKV5{0R9R^1X3DsNGlqnFFD3v(R^6W zd@2_2`3xLwhOQw)hp;T-;9a3bk$VvV?|+W>jTUm$fsgXRJ6MZ~xEnI`kp_6rcf40N zaP%35h~o<~;AydV_iIrR9fmGq5bYXl5sUZJMjQizvw)PrJ7*&f{DZ42fsodCZ*9cU zW*`sQ0EE!+?%TqAvVjBs9gFwx7NE7I0az5l0c#XALK%?3dwL6Gkk{ee>+!DNq9S?> zV?t}Zzdha;TvP;1(`HG9`EW_Cu?;%M!8_aI9l@a%kl{dWyth5x6I{r_yWito!+`_U z!+?f{_s7Tkijy*69lVo1-ho`m!F%oFJ<16VcpRJt0NoVyM6q}mbD<30uOIJoPI5p7 z@92+rLKn*11T<7Xc&~IJ2cHgrcU2b^8DplShkOyAAAt8=2N|;c;FARK4(y^L^G7js z4ZsNhVo`9#e=kp{A5CK0D?bF*eeb$~!Mlw2tP~aRMD`*b*_7s)+@H6-2g)+f&cws?cC4G;Cl248Gibw{M z5a2oky|9ja^lftf;x1eQ4RhFhmL4s0P4PIs=7t2sLH#O>w&bY?xYZ*L>S<}T?J)Sj zI22pZbMuJl7i-4}HNZRP;~mF^oMRw^+9^Z&|1(4J4;2z;2z&^133S z%tw;*wpnA(GJ57`B!l`17&~D=g)uO;@X8Tu@l@+sLKy|%Ag_vW*66BMwsegkA0ueP zLx3X(oLL9NXRz2yf)r58(nnpA0dSCC50$6iPS5c>Ak=LlaG?K}Yw)VqNnkQ^CXkCeBL=_W$mmXn;UPa)d$i+zKIoGrlV4QO5c zCIvp|KJpPl6VzND$(blkTQ_y+!7x%c$ma@hV7?I>fBJ;Xz58`bg))yx&WQ16&1Fk= z5q1-J75**(@0$0dTQGit;~*R0z4rw&s6)V~Y2Z@`fTIgC z;Eebj3cPg3pf-a_~th7s(SQgdBX93O<8E$ib(r;8QAu z9DE)NKF31H!6&xhlP-iDe5MON3q#1kr@r9RGK8Eh&`qG;1)oPD^f!Dm2|m#QIQk4d zGzR~(&B=%?05hWAxCmn8RroXrA|GRhjxMcDA0;?#`Y6F^(?N1S* zi8}Zs5|~#RK_9Q9-G&UF3CRQ!10iiH8UMVd!ea$Tu(%AXC_alD!VGfabJ-z&V^nm6 z4^2j`4^4cYHvv`jRbe|fd94CE7zh87VT6PR2k;mit{+^2W3;q11+gYE0y6l1 z(dL5dUO^mAF!|d%h|LP&lLsJbpl9Zc4 z8%jzOJzHoaoqkaY5=5XP$P0deC=91a)H^7g2X(0#$eN?c3*zwoSRtU7w`M4Z6Z8vM z@>vT2$zNa~m=69%87er4Qj?m*Q`~Rl|EVldo}cw6jH3G+oS*6hKm?ToqUg!1UG_vZ z3xX&DF?a%QpBUjBsN~2=2vv33CrG#(2oiC-6cze7hzS###DW3J)LeK`i^+pTFq_As zUi4z}SO)skE8ZL*7E~Jb5-S2+pB2pJgA-A&`?C1p?vyNigE&k+^#VI2ES$~#DaTMI z|0fybgjA7mzOND4wnIRbU-&0_)`W7SNE0TTFqr?QS_M4@LIp7tX$tyK9epGT9SPf; ztx4SUi(IL<0Sq;Xv@n3a_cv-#42BAeKp-MK<+q+mkwu0BSrm0#7AgVBLPW^^wlPui zCF>N#$zL7Lop2M7Ct{F>N0koPjPN=njl8eJdAGI#KxqBlVH;3e1OQQ!9oC3!Z~#O8 z(j?Y~I$#k&BL)D$k9L&?FGYY9{t4Z){%^gPGQxr=C8ep&&?CniviXMkp`$U0_QZ+` zqOkbEmEG-|% z1KHD@&-U^14&sFSzz~9_GC?dJk5v30YB3{unmnICJpOWhc$x^sJcPvw!EjU%0@XK& z$z^Fpa=3vkE>B=BD1as_#E%`qf?*o^I7kX?_ls@`*91(0DD?s?+x%{dK*c&a5g?LRqF}CoukgY{I6O`2l}L7op6*Xqe3)Fg_|qi;G@GfX z+lmBj&Wq>s(_I3npYCG9M4!X<`AI8$jjDw&yb1m%Tx)qK`U|dwhI;?UHJ+Zac2ui& zs5kj1o(3Xlf#XTqLJx810ze`3Pz?o3igw60qm!AN5OuoKLGnF z-BB>AhX58e*-fGDXy10vdGD-ej1jPZ3Fc73V3MA}j0Y{kVD2%WN2N+?Zqc8%mLA^e} z2ooKJ5so?lBTRG@Mqmy>UA6x1C`jul397XI?kI@B6ar0JfBzYXz+(g;?e`bztPKi6 zO~Rs(kzo1)$GLNYB3PQCfqrPt^{aW@KW_!he_n!#&F>eR+M2-vmS8H24D|a)OFRw2 zlVTA6=Ovg?l210oD^Xn)Q=MN7k$8S2!~s3f!tYNrEG9cjljzlcf%K2T_LpNf&Y=Bp zzZkna4)%+&yCYz@Zw1f%sjnr{s6IOaM(Gv_u%FT$1)~nu;3CvyN1+6L2`(Zf)lneI zXbmb+QXK^nbO--3c6Y?&0-%2xyE_hqZPeOb8R&PkXxO9L9J@QB5%!)o$L>D?g2F+q zVOU6Yv}gj)AxBGLqN6av;TqHtCOQfusG~N=?v7YTI9v-`8JN7I+6c@csH@iB9R+C} zuR)d8-yH=J*hQd8>+e4U5qL~%W9TSI>yU%o4i7)=1R>s@qZ(m(h5Nyfk3RPL^^;;S zdaoADgOwk6wfUQS@p4)?df$Q;4bfXKqu&U%E-sQX#6qKBImCb%Y6Z*vk-wUUkQqlJ zU19N#cyUUM(}MSyf(5@{45>sG#Op`^5GLS7z$jQt1p^7*YQlHbj^R#xtMNXeXsfntpREAjsWYPgz2 literal 0 HcmV?d00001 diff --git a/web/bridge/image.nix b/web/bridge/image.nix new file mode 100644 index 000000000..889376944 --- /dev/null +++ b/web/bridge/image.nix @@ -0,0 +1,53 @@ +{ + pkgs, + buildImage, + ... +}: let + description = "A simple Bun environment image"; + title = "Bun Environment"; + + # Separately build the dependencies with the fixed-output + # to be able to download with bun + bunDeps = pkgs.stdenv.mkDerivation { + name = "bun-deps"; + src = ./.; + nativeBuildInputs = [pkgs.bun]; + buildPhase = '' + bun install + ''; + installPhase = '' + mkdir -p $out + cp -r $src/* $out + ''; + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + outputHash = "sha256-RHq0SxdXjryEnb7aOEm9Bp7Hcq9S+bgKbkCNU17fURg="; + }; + + # Use the bunDeps for the nativelink-image + nativelink-bridge = pkgs.stdenv.mkDerivation { + name = "nativelink-bridge"; + src = bunDeps; + buildInputs = [pkgs.bun]; + installPhase = '' + mkdir -p $out + cp -r ${bunDeps}/* $out + ''; + }; +in + buildImage { + name = "nativelink-bridge"; + + # Container configuration + config = { + WorkingDir = "${nativelink-bridge}"; + Entrypoint = ["${pkgs.bun}/bin/bun" "run" "index.ts"]; + ExposedPorts = { + "8080/tcp" = {}; + }; + Labels = { + "org.opencontainers.image.description" = description; + "org.opencontainers.image.title" = title; + }; + }; + } diff --git a/web/bridge/index.ts b/web/bridge/index.ts new file mode 100644 index 000000000..a50671e29 --- /dev/null +++ b/web/bridge/index.ts @@ -0,0 +1,3 @@ +import { start } from './src'; + +start().catch(err => console.error(err)); diff --git a/web/bridge/package.json b/web/bridge/package.json new file mode 100644 index 000000000..3fd75869d --- /dev/null +++ b/web/bridge/package.json @@ -0,0 +1,22 @@ +{ + "name": "nativelink-bridge", + "version": "0.5.3", + "module": "index.ts", + "type": "module", + "dependencies": { + "drizzle-orm": "^0.36.0", + "postgres": "^3.4.5", + "protobufjs": "^7.4.0", + "redis": "^4.7.0" + }, + "devDependencies": { + "@types/bun": "^1.1.8", + "drizzle-kit": "^0.27.1" + }, + "peerDependencies": { + "typescript": "^5.0.0" + }, + "trustedDependencies": [ + "protobufjs" + ] +} diff --git a/web/bridge/src/eventHandler.ts b/web/bridge/src/eventHandler.ts new file mode 100644 index 000000000..88d27ba91 --- /dev/null +++ b/web/bridge/src/eventHandler.ts @@ -0,0 +1,75 @@ +import type protobuf from 'protobufjs'; +import type { BuildEvent, Progress } from './types/buildTypes'; +import { commandOptions, type RedisClientType } from 'redis'; +import { constructRedisKey, parseMessage } from './utils'; +import { broadcastProgress } from './websocket'; + + +export async function handleEvent(message: string, commandClient: RedisClientType, types: { PublishBuildToolEventStreamRequest: protobuf.Type, PublishLifecycleEventRequest: protobuf.Type }) { + switch (parseMessage(message).eventType) { + case 'LifecycleEvent': + await fetchAndDecodeBuildData(constructRedisKey(parseMessage(message)), commandClient, types.PublishLifecycleEventRequest); + break; + case 'BuildToolEventStream': + await fetchAndDecodeBuildData(constructRedisKey(parseMessage(message)), commandClient, types.PublishBuildToolEventStreamRequest); + break; + default: + console.log('Unknown event type:', parseMessage(message).eventType); + } +} + +async function fetchAndDecodeBuildData(redisKey: string, commandClient: RedisClientType, messageType: protobuf.Type) { + try { + const buildData = await commandClient.get(commandOptions({ returnBuffers: true }), redisKey); + if (buildData) { + const decodedMessage = messageType.decode(new Uint8Array(Buffer.from(buildData))) as BuildEvent; + if(decodedMessage.orderedBuildEvent) { + const buildId = decodedMessage.orderedBuildEvent.streamId.buildId + const invocationId = decodedMessage.orderedBuildEvent.streamId.invocationId + console.log("Build ID: ", buildId) + console.log("Invocation ID: ", invocationId) + const eventTime = decodedMessage.orderedBuildEvent.event.eventTime; + const milliseconds = eventTime.seconds.low * 1000 + Math.floor(eventTime.nanos / 1000000); + const eventDate = new Date(milliseconds); + console.log("Event time nanos:", eventTime.nanos) + console.log("Event time seconds:", eventTime.seconds.low) + console.log("Event time:", eventDate.toISOString()); + const currentTime = new Date() + const elapsedTime = currentTime.getTime() - eventDate.getTime(); + console.log("Time Now: ", currentTime.toISOString()) + console.log(`Elapsed Time: ${elapsedTime} ms`); + } + if (decodedMessage?.orderedBuildEvent?.event?.bazelEvent) { + console.log("------------------") + decodeBazelEvent(decodedMessage.orderedBuildEvent.event.bazelEvent, messageType.root); + } + } + } catch (err) { + console.error(`Error fetching build data for key ${redisKey}:`, err); + } +} + +// TODO(SchahinRohani): Add Bazel Event Types +// biome-ignore lint/suspicious/noExplicitAny: Bazel Event Types are not known yet +function decodeBazelEvent(bazelEvent: any, root: protobuf.Root): any { + if (!bazelEvent || !bazelEvent.value) return null; + const messageType = root.lookupType(bazelEvent.typeUrl.split('/').pop()); + const decodedMessage = messageType.decode(new Uint8Array(Buffer.from(bazelEvent.value, 'base64'))); + const decodedObject = messageType.toObject(decodedMessage, { + longs: String, + enums: String, + bytes: String, + }); + if (decodedObject.progress) { + console.log("Processing progress information...\n\n"); + processProgress(decodedObject.progress); + } + return decodedObject; +} + +function processProgress(progress: Progress) { + if (progress.stderr) { + console.log(progress.stderr); + broadcastProgress(progress.stderr) + } +} diff --git a/web/bridge/src/http.ts b/web/bridge/src/http.ts new file mode 100644 index 000000000..e2e83bef3 --- /dev/null +++ b/web/bridge/src/http.ts @@ -0,0 +1,37 @@ +import { serve } from "bun"; + +export const startWebServer = () => { + console.log('\nHTTP server is running on http://localhost:3001\n'); + serve({ + port: 3001, + fetch(req) { + const url = new URL(req.url); + const handler = routes.get(url.pathname); + if (handler){ + return handler(); + } + return new Response("Not Found", { status: 404 }); + }, + }); +}; + +const routes = new Map Response>([ + ["/api", () => jsonResponse({ message: "Hello from API" })], + ["/health", () => jsonResponse({ status: "ok" })], + ["/readiness", () => jsonResponse({ status: "ready" })], + ]); + + +function jsonResponse(data: object, status: number = 200): Response { + const responseData = { + ...data, + timestamp: new Date().toISOString(), + }; + return new Response(JSON.stringify(responseData), { + headers: { + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*" + }, + status, + }); + } diff --git a/web/bridge/src/index.ts b/web/bridge/src/index.ts new file mode 100644 index 000000000..7296d255a --- /dev/null +++ b/web/bridge/src/index.ts @@ -0,0 +1,73 @@ +import { initializeRedisClients } from './redis'; +import { initializeProtobuf } from './protobuf'; +import { handleEvent } from './eventHandler'; +import { startWebSocket } from './websocket'; +import { startWebServer } from './http'; + + +export async function start() { + // Base URL + const github = "https://raw.githubusercontent.com" + + // NativeLink URL + const nativelinkRepo = "TraceMachina/nativelink" + const nativelinkBranch = "main" + const nativelinkProtoPath = `${github}/${nativelinkRepo}/${nativelinkBranch}/nativelink-proto/`; + + // Proto Remote Path + const protoRepo = "protocolbuffers/protobuf" + const protoBranch = "master" + const protoRepoPath = `${github}/${protoRepo}/${protoBranch}/main/src/google/protobuf`; + const protoDevToolsPath = `${github}/${protoRepo}/main/src/google/devtools/build/v1`; + + const googleProto = "googleapis/googleapis" + const googleProtoBranch = "master" + const googleProtoPath = `${github}/${googleProto}/${googleProtoBranch}/google/devtools/build/v1`; + + // Bazel Remote Path + const bazelRepo = "bazelbuild/bazel" + const bazelBranch = "master" + const bazelProtoPath = `${github}/${bazelRepo}/${bazelBranch}/src/main/java/com/google/devtools/build/lib/buildeventstream/proto`; + + // TODO(SchahinRohani): Add Buck2 Protos for future Buck2 support + // Buck2 Protos + // const buck2Repo = "facebook/buck2/main" + // const buck2Branch = "main" + // const buck2ProtoPath = `${github}/${buck2Repo}/${buck2Branch}/app/buck2_data/data.proto`; + + // Actual using Protos. + const PublishBuildEventProto =`${googleProtoPath}/publish_build_event.proto`; + const BazelBuildEventStreamProto = `${bazelProtoPath}/build_event_stream.proto`; + + const protos = [ PublishBuildEventProto, BazelBuildEventStreamProto ] + + console.info("Link to: \n") + console.info("Google Publish Build Events Proto:\n", PublishBuildEventProto, "\n"); + console.info("Bazel Build Event Stream Proto:\n", BazelBuildEventStreamProto, "\n") + + // Load Remote Bazel Proto Files + const protoTypes = await initializeProtobuf(protos) + + const { redisClient, commandClient } = await initializeRedisClients(); + + // Subscribe to the build_event channel + await redisClient.subscribe(process.env.NATIVELINK_PUB_SUB_CHANNEL || "build_event", async (message: string) => { + await handleEvent(message, commandClient, protoTypes); + }); + + const websocketServer = startWebSocket(); + const webServer = startWebServer(); + + process.on('SIGINT', async () => { + await redisClient.disconnect(); + await commandClient.disconnect(); + console.info("Received SIGINT. Shutdown gracefully.") + process.exit(); + }); + process.on('SIGTERM', async () => { + await redisClient.disconnect(); + await commandClient.disconnect(); + console.info("Received SIGTERM. Shutdown gracefully.") + process.exit(); + }); +} diff --git a/web/bridge/src/protobuf.ts b/web/bridge/src/protobuf.ts new file mode 100644 index 000000000..1aaace85e --- /dev/null +++ b/web/bridge/src/protobuf.ts @@ -0,0 +1,98 @@ +import protobuf from 'protobufjs'; + +export async function initializeProtobuf(protos: string[]) { + console.log("Loading Remote Proto Files"); + + // Create a new Root instance + const combinedRoot = new protobuf.Root(); + + // Track loaded files to avoid circular dependencies + const loadedFiles: Record = {}; + + // Track processed imports to avoid duplicates + const processedImports = new Set(); + + // Load all initial proto files + for (const proto of protos) { + await loadProto(loadedFiles, combinedRoot, proto, processedImports); + } + console.log("\nDone parsing all proto files.\n"); + // Now combinedRoot contains your parsed .proto content + // Example: Look up specific message types + const BazelBuildEvent = combinedRoot.lookupType("build_event_stream.BuildEvent"); + const PublishBuildToolEventStreamRequest = combinedRoot.lookupType("google.devtools.build.v1.PublishBuildToolEventStreamRequest"); + const PublishLifecycleEventRequest = combinedRoot.lookupType("google.devtools.build.v1.PublishLifecycleEventRequest"); + + console.log("Loaded Types:\n"); + console.log({ + PublishLifecycleEventRequest: PublishLifecycleEventRequest ? PublishLifecycleEventRequest.fullName : "Not found", + PublishBuildToolEventStreamRequest: PublishBuildToolEventStreamRequest ? PublishBuildToolEventStreamRequest.fullName : "Not found", + BazelBuildEvent: BazelBuildEvent ? BazelBuildEvent.fullName : "Not found" + }); + + return { + PublishLifecycleEventRequest, + PublishBuildToolEventStreamRequest, + BazelBuildEvent + }; +} + +function resolveImportPath(protoUrl: string, importPath: string): string { + // Handle googleapis imports + if (importPath.startsWith("google/api") || importPath.startsWith("google/devtools/build/v1")) { + return `https://raw.githubusercontent.com/googleapis/googleapis/master/${importPath}`; + } + + // Handle protocolbuffers imports + if (importPath.startsWith("google/protobuf")) { + return `https://raw.githubusercontent.com/protocolbuffers/protobuf/master/src/${importPath}`; + } + + // Handle specific case for bazel + if (importPath.includes("com/google/devtools/build/lib/packages/metrics") || importPath.startsWith("src/main/protobuf")) { + return `https://raw.githubusercontent.com/bazelbuild/bazel/master/${importPath}`; + } + + // Default behavior for other imports - resolve relative to protoUrl + return new URL(importPath, protoUrl).toString(); +} + +// Recursive function to fetch, parse, and handle imports +async function loadProto( + loadedFiles: Record, + root: protobuf.Root, + protoUrl: string, + processedImports: Set, + indentLevel = 0, +) { + if (loadedFiles[protoUrl]) { + // If already loaded, skip to prevent circular imports + return; + } + + // Fetch the .proto file content + const response = await fetch(protoUrl); + if (!response.ok) { + throw new Error(`Failed to fetch .proto file from ${protoUrl}: ${response.statusText}`); + } + + // Parse the proto content + const parsedProto = protobuf.parse(await response.text(), root); + // Mark this proto as loaded + loadedFiles[protoUrl] = true; + // Log the imports necessary for this proto file + if (indentLevel < 1) { + console.log(`\n${ ' '.repeat(indentLevel)} ${protoUrl}:`); + } + if (parsedProto.imports && parsedProto.imports.length > 0) { + for (const importPath of parsedProto.imports) { + const resolvedImportUrl = resolveImportPath(protoUrl, importPath); + if (!processedImports.has(resolvedImportUrl)) { + console.log(`${ ' '.repeat(indentLevel)} - ${importPath}`); + processedImports.add(resolvedImportUrl); + // Recursively handle the imports + await loadProto(loadedFiles, root, resolvedImportUrl, processedImports, indentLevel + 1,); + } + } + } +} diff --git a/web/bridge/src/redis.ts b/web/bridge/src/redis.ts new file mode 100644 index 000000000..8154e9512 --- /dev/null +++ b/web/bridge/src/redis.ts @@ -0,0 +1,25 @@ +import { createClient, type RedisClientType } from 'redis'; + +export async function initializeRedisClients() { + try { + const redisClient: RedisClientType = createClient({ + url: process.env.REDIS_URL, + }); + const commandClient = redisClient.duplicate(); + + redisClient.on('error', (err) => { + console.error('Redis Client Error:', err); + throw new Error('Failed to connect to Redis.'); + }); + + await redisClient.connect(); + await commandClient.connect(); + + console.log('\nRedis clients successfully connected.\n'); + + return { redisClient, commandClient }; + } catch (error) { + console.error('Error during Redis client initialization:', error); + throw new Error('Unable to initialize Redis clients. Please check your connection.'); + } +} diff --git a/web/bridge/src/types/buildTypes.ts b/web/bridge/src/types/buildTypes.ts new file mode 100644 index 000000000..fdde7c7fc --- /dev/null +++ b/web/bridge/src/types/buildTypes.ts @@ -0,0 +1,20 @@ +export interface BuildEvent extends protobuf.Message { + orderedBuildEvent: { + streamId: { + buildId: string; + invocationId: string; + }, + event: { + eventTime: { + seconds: protobuf.Long; + nanos: number; + }; + // biome-ignore lint/suspicious/noExplicitAny: Not known yet + bazelEvent?: any; + }; + }; + } + + export type Progress = { + stderr: string; + }; diff --git a/web/bridge/src/utils.ts b/web/bridge/src/utils.ts new file mode 100644 index 000000000..53face425 --- /dev/null +++ b/web/bridge/src/utils.ts @@ -0,0 +1,28 @@ +type ParsedMessage = { + prefix: string; + eventType: string; + eventID: string; + subEventID: string; + sequenceNumber: string; +} + +export function parseMessage(message: string) { + const parts = message.split(':'); + const [prefix, eventType, eventID, subEventID, sequenceNumber] = parts; + return { + prefix, + eventType, + eventID, + subEventID, + sequenceNumber + }; +} + +export function constructRedisKey(parsedMessage: ParsedMessage) { + console.log("\nNew Published Event: ") + console.log(" EventID: ", parsedMessage.eventID) + console.log(" Sequence Number: ", parsedMessage.sequenceNumber) + console.log(" Invocation ID: ", parsedMessage.subEventID) + console.log("------------------") + return `${parsedMessage.prefix}:${parsedMessage.eventType}:${parsedMessage.eventID}:${parsedMessage.subEventID}:${parsedMessage.sequenceNumber}`; +} diff --git a/web/bridge/src/websocket.ts b/web/bridge/src/websocket.ts new file mode 100644 index 000000000..68e9f7c35 --- /dev/null +++ b/web/bridge/src/websocket.ts @@ -0,0 +1,42 @@ +import type { ServerWebSocket } from "bun"; + +const clients = new Set>(); + +export const startWebSocket = () => { + console.log('\nWebSocket server is running on ws://localhost:8080\n'); + Bun.serve({ + port: 8080, + fetch(req, server) { + // Upgrade the request to a WebSocket + // Here we can also do the websocket auth/token auth + if (server.upgrade(req)) { + return; + } + return new Response("Upgrade failed", { status: 500 }); + }, + websocket: { + open(ws) { + console.log('New client connected'); + clients.add(ws); + ws.send("Hello Web Client") + }, + message(ws, message) { + console.log('Received message from web client:', message); + }, + close(ws) { + console.log('Web Client disconnected'); + clients.delete(ws); + }, + drain(ws) { + console.log('Ready to receive more data'); + }, + }, +});} + +export function broadcastProgress(progress: string) { + const buffer = Buffer.from(progress) + console.log(progress) + for (const ws of clients) { + ws.send(new Uint8Array(buffer)); + } +} diff --git a/web/bridge/tsconfig.json b/web/bridge/tsconfig.json new file mode 100644 index 000000000..238655f2c --- /dev/null +++ b/web/bridge/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + // Enable latest features + "lib": ["ESNext", "DOM"], + "target": "ESNext", + "module": "ESNext", + "moduleDetection": "force", + "jsx": "react-jsx", + "allowJs": true, + + // Bundler mode + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + + // Best practices + "strict": true, + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + + // Some stricter flags (disabled by default) + "noUnusedLocals": false, + "noUnusedParameters": false, + "noPropertyAccessFromIndexSignature": false + } +}