From bcbf293fff802f29587f7b62b5e5f526308e73ce Mon Sep 17 00:00:00 2001 From: Schahin Rouhanizadeh Date: Tue, 10 Sep 2024 19:01:21 +0200 Subject: [PATCH] Introduce `nativelink-bridge` This is a fast prototype for subscribing to the redis/dragonflydb "build_events" channel and decode them properly via protobuf and fires them via websocket to the browser. --- flake.nix | 3 + nativelink-config/examples/basic_bes.json | 177 ++++++++++++++++++++++ tools/pre-commit-hooks.nix | 1 + web/bridge/.env.example | 5 + web/bridge/.gitignore | 30 ++++ web/bridge/README.md | 79 ++++++++++ web/bridge/bun.lockb | Bin 0 -> 35076 bytes web/bridge/image.nix | 32 ++++ web/bridge/index.ts | 3 + web/bridge/package.json | 22 +++ web/bridge/src/db/index.ts | 9 ++ web/bridge/src/eventHandler.ts | 95 ++++++++++++ web/bridge/src/http.ts | 45 ++++++ web/bridge/src/index.ts | 53 +++++++ web/bridge/src/protobuf.ts | 76 ++++++++++ web/bridge/src/redis.ts | 25 +++ web/bridge/src/types/buildTypes.ts | 29 ++++ web/bridge/src/websocket.ts | 44 ++++++ web/bridge/tsconfig.json | 27 ++++ 19 files changed, 755 insertions(+) create mode 100644 nativelink-config/examples/basic_bes.json create mode 100644 web/bridge/.env.example create mode 100644 web/bridge/.gitignore create mode 100644 web/bridge/README.md create mode 100755 web/bridge/bun.lockb create mode 100644 web/bridge/image.nix create mode 100644 web/bridge/index.ts create mode 100644 web/bridge/package.json create mode 100644 web/bridge/src/db/index.ts create mode 100644 web/bridge/src/eventHandler.ts create mode 100644 web/bridge/src/http.ts create mode 100644 web/bridge/src/index.ts create mode 100644 web/bridge/src/protobuf.ts create mode 100644 web/bridge/src/redis.ts create mode 100644 web/bridge/src/types/buildTypes.ts create mode 100644 web/bridge/src/websocket.ts create mode 100644 web/bridge/tsconfig.json diff --git a/flake.nix b/flake.nix index f9f70a972..c0c714183 100644 --- a/flake.nix +++ b/flake.nix @@ -279,6 +279,8 @@ }; }; + nativelink-bridge = pkgs.callPackage ./web/bridge/image.nix {inherit buildImage pullImage pkgs;}; + nativelink-worker-init = pkgs.callPackage ./tools/nativelink-worker-init.nix {inherit buildImage self nativelink-image;}; rbe-autogen = pkgs.callPackage ./local-remote-execution/rbe-autogen.nix { @@ -419,6 +421,7 @@ nativelink-worker-init nativelink-x86_64-linux publish-ghcr + nativelink-bridge ; default = nativelink; diff --git a/nativelink-config/examples/basic_bes.json b/nativelink-config/examples/basic_bes.json new file mode 100644 index 000000000..c4e91f098 --- /dev/null +++ b/nativelink-config/examples/basic_bes.json @@ -0,0 +1,177 @@ +{ + "stores": { + "AC_MAIN_STORE": { + "filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-ac", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-ac", + "eviction_policy": { + "max_bytes": 100000000000 + } + } + }, + "BEP_STORE": { + "redis_store": { + "addresses": [ + "redis://@localhost:6379/0" + ], + "response_timeout_s": 5, + "connection_timeout_s": 5, + "experimental_pub_sub_channel": "build_event", + "key_prefix": "nativelink:", + "mode": "standard" + } + }, + "WORKER_FAST_SLOW_STORE": { + "fast_slow": { + "fast": { + "filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", + "eviction_policy": { + "max_bytes": 100000000000 + } + } + }, + "slow": { + "noop": {} + } + } + } + }, + "schedulers": { + "MAIN_SCHEDULER": { + "simple": { + "supported_platform_properties": { + "cpu_count": "minimum", + "memory_kb": "minimum", + "network_kbps": "minimum", + "disk_read_iops": "minimum", + "disk_read_bps": "minimum", + "disk_write_iops": "minimum", + "disk_write_bps": "minimum", + "shm_size": "minimum", + "gpu_count": "minimum", + "gpu_model": "exact", + "cpu_vendor": "exact", + "cpu_arch": "exact", + "cpu_model": "exact", + "kernel_version": "exact", + "OSFamily": "priority", + "container-image": "priority" + } + } + } + }, + "workers": [ + { + "local": { + "worker_api_endpoint": { + "uri": "grpc://127.0.0.1:50062" + }, + "cas_fast_slow_store": "WORKER_FAST_SLOW_STORE", + "upload_action_result": { + "ac_store": "AC_MAIN_STORE" + }, + "work_directory": "/tmp/nativelink/work", + "platform_properties": { + "cpu_count": { + "values": [ + "16" + ] + }, + "memory_kb": { + "values": [ + "500000" + ] + }, + "network_kbps": { + "values": [ + "100000" + ] + }, + "cpu_arch": { + "values": [ + "x86_64" + ] + }, + "OSFamily": { + "values": [ + "" + ] + }, + "container-image": { + "values": [ + "" + ] + } + } + } + } + ], + "servers": [ + { + "name": "public", + "listener": { + "http": { + "socket_address": "0.0.0.0:50052" + } + }, + "services": { + "cas": { + "main": { + "cas_store": "WORKER_FAST_SLOW_STORE" + } + }, + "ac": { + "main": { + "ac_store": "AC_MAIN_STORE" + } + }, + "execution": { + "main": { + "cas_store": "WORKER_FAST_SLOW_STORE", + "scheduler": "MAIN_SCHEDULER" + } + }, + "capabilities": { + "main": { + "remote_execution": { + "scheduler": "MAIN_SCHEDULER" + } + } + }, + "bytestream": { + "cas_stores": { + "main": "WORKER_FAST_SLOW_STORE" + } + } + } + }, + { + "name": "private_workers_servers", + "listener": { + "http": { + "socket_address": "0.0.0.0:50062" + } + }, + "services": { + "experimental_prometheus": { + "path": "/metrics" + }, + "experimental_bep": { + "store": "BEP_STORE" + }, + "worker_api": { + "scheduler": "MAIN_SCHEDULER" + }, + "admin": {}, + "health": { + "path": "/status" + } + } + } + ], + "global": { + "max_open_files": 512 + } +} diff --git a/tools/pre-commit-hooks.nix b/tools/pre-commit-hooks.nix index e690341ae..e78747383 100644 --- a/tools/pre-commit-hooks.nix +++ b/tools/pre-commit-hooks.nix @@ -65,6 +65,7 @@ in { # Bun binary lockfile "web/platform/bun.lockb" + "web/bridge/bun.lockb" ]; enable = true; types = ["binary"]; diff --git a/web/bridge/.env.example b/web/bridge/.env.example new file mode 100644 index 000000000..4b18a39aa --- /dev/null +++ b/web/bridge/.env.example @@ -0,0 +1,5 @@ +REDIS_URL=redis://localhost:6379 +NATIVELINK_PUB_SUB_CHANNEL=build_event +POSTGRES_URL=postgres://username:password@host:port/database +WEBSOCKET_PORT=8080 +HTTP_PORT=3001 diff --git a/web/bridge/.gitignore b/web/bridge/.gitignore new file mode 100644 index 000000000..666f92cdd --- /dev/null +++ b/web/bridge/.gitignore @@ -0,0 +1,30 @@ +# Logs +logs + +# Caches +.cache + +# Runtime data +pids +_.pid +_.seed +*.pid.lock + +# Dependency directories +node_modules/ + +# TypeScript cache +*.tsbuildinfo + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# distribution directory +dist + +# temporary files +.temp diff --git a/web/bridge/README.md b/web/bridge/README.md new file mode 100644 index 000000000..9a3f66245 --- /dev/null +++ b/web/bridge/README.md @@ -0,0 +1,79 @@ +# NativeLink Bridge (Experimental) + +Make sure you are running an instance of Redis or DragonflyDB in your network. + +For DragonflyDB inside docker run: + +```bash +docker run \ + -d --name some-dragonfly \ + -p 6379:6379 \ + --ulimit memlock=-1 \ + docker.dragonflydb.io/dragonflydb/dragonfly + +``` + +For Redis inside docker run: + +```bash +docker run -d --name some-redis \ + -p 6379:6379 \ + redis +``` + +Set the Redis URL and the NativeLink pub sub channel ENV variables in `.env` as defined in the `.env.example` + + +The Redis URL format: 'redis://alice:foobared@awesome.redis.server:6380' + +The NativeLink pub sub channel ENV variable should match `experimental_pub_sub_channel` inside `nativelink-config/example/basic_bes.json`. + +Make sure you have set also the `key_prefix` in `nativelink-config/example/basic_bes.json` + +## You need 4 Components + Redis + +### 1. NativeLink + +Start an instance of NativeLink with the basic_bes.json inside the `nativelink-config/example/basic_bes.json`. + +### 2. NativeLink Web Bridge + +Install the dependencies and run the bridge: + +```bash +bun i && bun run index.ts +``` + +### 3. NativeLink Web UI + +Inside the web/ui directory run: + +```bash +bun i & bun dev +``` + +Now you can open http://localhost:4321. + + +### 4. Bazel + +Now you can run your Bazel build with NativeLink and see it in real-time going into the web app + +Include this in your .bazelrc +```bash +bazel clean && bazel build \ + --remote_cache=grpc://localhost:50051 \ + --remote_executor=grpc://localhost:50051 \ + --bes_backend=grpc://localhost:50061 \ + --bes_results_url=http://localhost:4321/builds \ + --bes_upload_mode=fully_async \ + --build_event_publish_all_actions=true \ + //local-remote-execution/examples:hello_lre +``` + +Make sure to use the right IP, if it's not hosted on `localhost` + + +```bash +bazel build some-target +``` diff --git a/web/bridge/bun.lockb b/web/bridge/bun.lockb new file mode 100755 index 0000000000000000000000000000000000000000..9d6822e0b098d8143b169fc95c38497b55f47408 GIT binary patch literal 35076 zcmeHw2|QHa`~T3CrLs#ZMWscIH4zF$T2PWo!eB7AVP-5*(T*00k_w@nwolqfQK^(f zv_~Z^(yFwV|8wr#V{V_1Pm%BI|N4DiuTHP0Gv_|{d7tMz=Q+zg_c~{gnsz9UtL?!K z(q;vPsd|J4NdjpFa6J9J*#RsqZw@z*$=3?gkrbuTXs+c?6o={M7WXN%-(`6?Vbzk0 zoV-+(=KcAuy7@908r}G@a1#V)K|~Y(TO32CxwFY1G!HJ@%ZEjyaamq$9$d@@lB3aL zxDfY*cpft#fX(C6tVC$E9*`ahu?)l;Nd8KQyF)q}V#N0*V-1KUAsxU8@`0bU$6W;X z&7lkKyAElDbB>IUka0G|@^F0}#Ig`CB4cfc6(P-rxHrU35TpCXLyYtd=J5DFTo!Kt zBxAV&JFg8$I{ES^UwJHU(90);^LH$dDA;wmyOB;%cA zybfZd&q9bHuNXfvb|Pa-GG>r*A2RMn#!ZkZlDh{nDn}U^A1C7-WSk5!(swZ#2a~ZY z85cvj05WC^fFOI?LK@+kL5$M>)p8XBzc${-ad|xZ65cc{+)5`^6(cCz4zzK*qZu|-sC;6Dpk5@ci#)~Q97Ro7OD@o zsf!4@)^(uXo1B{#1E(Ym$($dTJ}rMx<$=I>Cm#=~on_mnrPO%Xo&0NTSoHVd8)RZE z^ylugs(+r}r%3f%<0f#)2FVOE8y7D2_Go3MjLO+z4fX-+B8wj-?vhuU@Zre@epBYy_XX_Z zrn?Nf1wK}kn&|B5dt-EeeQV~^$n^u=Y_^syRXQ_(tuo2*(9JKF|J8EU)}Bhs(JM4w zvxH{wpwGo~+w#s{JA5O-=3=+ug_Cs_y^DJ8b2(P7abWhPjq@`6uCg=KV>CxBV|hS# z+o^y?4HOK3^q{>!_e+DO)9!%f4}*ji$Rm9^q7#Z?MSOhkB=|9k%pYBEZyjR! zMexIzqJMkq5X2dSI31Ah3WDews>k*OEUyNc zjs|%cR?v7!lWLDJ+=JzPK;DAPzYknTdbB5C`CTB7%8%^cUfUylEMMPAc~wxDnm@+< zRT=PoE+B8&3H{SSehev3EmwQQk|Njo;$C36&YcTlwuYmA6`J=HHjeqS8SUwiy@%RO* z5OetUU|{*HAdknd-;q~?0U!7Ozat;iN%?|K%8SEdW@q%D0rGhK`knG`1$o?m|Bie^ zC*}2EQMWVsM|D#EY$xS=!lHL)^mpo{d`2hbpMpH@|9_|cwUs+3AJ$3v!cNN5VbK8h z|G!iIiJg>R(@FWNPRjR&hrZ5~e=f-Dk>ej4L*Q|=T>p(8OW z<6uIneKDAVW)6gGFdhiWTaz)&QHb;;h*6-%Fa*)u(M1UE#Q*<|7}=Fc>PwALxjlhk zXre`u{QqK%$`?iIk7AUb4}{9MkW8cZ(>VO`XMtb;@#p`We%~>F{x1C;3H&bu;EB)# zxKwC*`KN9D%vE}qeqVZa=CPu{f@Qw--*Z&NFKrEfQP5zqXTS24o|O_)PtDQ!^1-@( z*sj&R(rccy*oUY#H#c!h`<91Na8cXDfzER@dsog{bDF_huHvgXV5-&g%=gn&Xe#C9NCZ^Y!Xrbu~}5{J?Eims%b= z?!48ow({;4iy#JzTbm%de@Uih;Zc_3sb%gHw4PIN(O8TFeZ}x4qhDG)34JwcseE&V zw~zdd`|6qWx0e>z-+FhZYlPD&jh?!*XEr52vQ6Gt_tmb+cKx#O&xO5TPCu*n>_+;M zmlRy;Gd6wv#dp2>y3K9)D?W0FcC7#Ga}wuRpFx%cRMXnaF<^g`FJFP|v1)Tp8k75f8q-bFtg-=%ATpYg1%GWxU--CXPGdclha z6%}ojKA(8IrYG&j+D)RTHhi@eeXqyaajVCe0l9pp>O6@8uVU-hd|73w%@kbfe2U&M zC1v4A=P3%MdD+$vTV}t0w|kgf$|B9BoC^p1w%)ze7{B-48RbP)&Q>aEw5IZ)_b*%< z?#xKm>#K2PlU`%0bO;4k0vi`%y23d55ZznZhAy#rtf#(Ns{Qr*1o7(UmNe{J=B1ev ztu$%0EUS-A%$}JCXBF`z8C^p9=*2cNw3<(HjjiXX>6%b*(HaO2^nQU$dKpRvC3(+? z>T`9%%KjN2)<1~5GCO;Pz1uChbG`GFFRL}I^L8o_og?~rl*QRuO&JE8Gg3pt=DOD} zVmL=hQgC6JzBSNO6?;iszBtWJ=lV#~0|7P?AuTc0h4)P^T<_s0^_sWbY~uDy15+h> z$JV)>K2v?sO8n7=+-*)*7rRfE9U#89=ZhR@WVqg_^F{jl;@8`5t1hCK2Yq+m=4okr z)!$^lYrn@+R>s(Ne^#YAetvAQMxVoq5yztA58LRUs5;|a?jkM!?4f<+)7M7F#@?dn zi{^tk&<~cLc~Z@G9~*iuVuXe0xwZEB4hO5pj*4gbGFK^Vy35~o+OUEbJtl|mt2Rxi zL?rPHZ%o2Qwn3LM8@h&vIFHyw!A0{+9Oy%)t}=O&$+wi;Gc6=3Yk*UXf|IZQixQKv zu@9QV_&K9?9aMWfb-mr?!Pjy3?l* zmp47INF7YK?>_0#(MZmGo<(rX%c18@J{wmbG|jS+*wDq`qQK>Lz5p@oww`P zt!6?<&WV=4rcJ0{734am_$E)kD`W4ijiof{d;O%uKDKfXNvL$ZtuBbJ= zohJs$P;k+_90z)0b^3xcHT&v36iOU|q7AOsYAfu!DE8ew+~?-W6U<{9R9`P{+!XCF zT>bswlo<_f+x}{B+ReD_s^elcFT5zSOEU!*>`f_q_f+amFLpOAEs57nr@fa86}B z|CVZ|(($}UpO4zQU3cm=y2;1pb#Zdwv1_7ld4p=URzDANs1AF2&c;r*Jnxn=1s6Rx z;6UFn>zeDPlzI69@gobW-#bgJhz+*$&2TZ6a@2do{M`FU$^L=WxiTL=EP&2u(WI10 zb^rdVC$+z3->18bGg>ieI|Ua#hu}a@cq%C`%V2mMIAvD8NIvDrUtv=N55682#$2!K z*L&fiX<;fE2Sm;%8ywfZlzUOUd0L|VNtV}$@V*N3x<2MgF)}H*Xl{iAJ>f=Ln!?xX z?nJs+$>MyhPghrE^Vq-gRco)B|aIt`bd z$v$?8TDCdR)JSwg%P9Gl5}ljVZt3^avT?IApCIYA{c3Ds*@Z4r+Z>0CKWH+4$lHt* z1_c+EO+kU6NAWLR(heUHZlfv5NPnZb9At5K<_I20_mmqkzT za#uL7o#cA5qWUR4pMr~?IdPyX$@Um%IVpVwx6AZR5|ikKM`YJ6XKAIYe0x+#_%b)Di{=E`hj=m8Pb-aBV;#D z_W5h>hPgi6Jbr$^%TMbuxhZeY*W~(XmsBs#)LmkgzK$8?FhTCtblv;)EuWu8 zMdmM4@BeDN_lkw_p${}xKeOziG;M6t4GJ#$J{JeNgHfLy4I9=UQd`?w-~Py8-Ug*< zV@504#21`h?z*=xugjqQ`|o{-$GzwNT5-gLwQzOlkuzmpb}Q8_Vi?l< z22Nipx6ZM7=ay-=L?>Lh?lAq0h?j}T$6fVDY*OVOc5mq8CbEqfkBRScTh})bJUu+g zl^y2NKSTQRs^dQoZkoM+v*CMlrAB?oXpYp$QRQ)tZ*+5|)-;|;IKBMI;X@Oz zZaKZ(Vz%At*A)TjhY5WJxPz$r&ersO*55UF&CCG{CA*(6E_~>tVe|5y&NR*OJw1=_ zf9!9u=}?QFzJ<6%fWH6nr8=z1JnImH;amGwmTz^jzU@~{!5vJ+O$$AdzIA%;vvHaq zr9JPg|MG2ILLZsuqeX7s>z`S+=XJu&{H&;Msy%v~|CZr0Tfh4L9LxFT`TNvP6}`M% zJmh=h8UmM2tf8q>aW8JI7{HKGX>8Exc5u0zj{95f>Ppd~#tYlrQ`egBeA{z_Lg+(z z$>NOQbc5xZaYv%#5)~qSzkj@(KPSBBgpCXBNnA8v9YVz&HD-kRd9~gq<44Zzud}_U ze^&Dy)AwaEH}ZS8I2nb$)!ieuMr7q75nc6%iyKN88!#n3)~w;Z^iJ==$&TMsC8kB< zqGy1iR9ue2ggL2V)1&n#v+bw6{i|T`<4@Y@nTM*=^u73r14M4jikYCEvdYW8sJOW% zud3*U?eVB%8@AqzT3UbI+*{(ZU_7SNVgPd(6*u&H*9lo#9+G`>R(>4fCF3hM!Q8d@lpyGbHe6-qAv(}w{q<@_MtoX{k0plXo7F^_Q zmpk9oxaCRH`W~}u2kd%5TUVmIv!Ogm)@1A3pxd(8I^+>9JAVYj*?Z ztuc@L+HcJc2~A^Nfqli1|1zk!uib0bKP>H1^ts$=mb%-+^gQ#Gd&*<4FYeytvc9B2 ztS&bHL-DA19mY+4Pw(gF=0`See>hZSTaV@;X(rQ}ver5fxPs0{n~EDftYC`X*m%A8 zk;&DA=^C*?(_h4vx=OrF9PhW$xA`t3O>*a8FS(kO*r*_z2}i&1XSEzMx%c*TewPzn zoJVILpG@G26aK41#dZE((M{`u^yf-rGgBMd^vOrp*D1b#Rx&q#Yq{-iI{`iP|VR*+yoPs%6gn^%u3TmXsK(CJo&u=+^~H?7CFk*cUEe9@#f4ia4+XRW|j= zQ&$-?`-*af@8+x#Cb4?M-dH)xWo=zN#3^33K4Dbar~{A2ZQN&jUv=Xg?*U2=SLHn= z^+o;w?-E)Aee8<8b0qTzo^`di7&$D|sPfRxvf`%$3U9k+ME1HOuQuk;yQmX=_vhL0 z54_jPI_P)icG#QSdyGpb#Yl>|IR->cA#iEpsDIFJMFoG@UhaNwdYALY$8SU?C?D7? z)gTpCyjMf%jkHVY@QBkAB_?ZL@=Xr)7s=>XH|eVSnb^V1X`ge$ZVc)%W7R;*uSumO zZVX@=P;nDSxmlgf>iMpp|JOqExS>mIH=W2>EwN2I_k8w-qHwFepAOjjTd!DTx^$EG zx`X{J-%jYoWxNeMH2wXHz}GWhsm>#D%>fhMX|x9VTEmj@&U}yLiW$z&rqx>6#%;)6 z!&BZ~TNzES=3E*!HSbgzztHc2vDZqqQKA`pAEmC-o&QiNZ|IAa??Og>(@7(7k$nwY zQNa$=;|UDE^T0IC#PScTrHbDh>8$p<>aXcs%t&^0vUWZ|J2K^AiN?;%QcE|* zz0smIm+kl4rS$aZ3BW3%2{7hCCBh4l8Ex%F1Blle<>lh$w$Q699jAJ1;w zfo*Z-Zf7iYZk3KcXQ?(;^ue&JS6|DYTsp~Tk%ZqB3hrns?v(`>QjX8#@2ZYrb+_9Z zV6LFUwRlk9?LtBCxd&E_lNw@PCZFT=a?ktNezO?I6K%iV6l+Qwb>FaIeel&m$5w7? zq~O9ks@6asz`6ST(+U5ImDiuL-!(oweSlg2n3}?N9ug>`^a4N zMvk4cOWi&7oAX8grspOp=|LBVUyh{Unzo{XKlsx97VjUib*g7ox@4i`r|3Cd3(P;) zb9TffTOQjIR}mQQ(<7>|SI(o2KDTeljM7sI?sM7cQcKe%*X0|(_ZGh&M43mKQE}%_ zEXp4>{cCMj z&9!|SnIjz*4tOq+@rBS=Fpq+7En5Tq=B4y*$L-BEjYAzKZWnX(5Zg0Ir(#LZiX*CG zQyH>iW3601KTpz3u5lQ-Lh5vdLcCa2S^+)oRpoTa9BW;_xc(Ge_y)E$&>8AAEi)Sub!{nHIub}s`L51xf3K53*|->&*vOoB-U?@ z%I35|BJ&t%oDhMxSZEwFZ$$b!NnleFHI%t~SuY3SrOk+?B{IgW}ut+&>MnG@5Q``yZq?O0%O zYgzA6Mvq-5#~H;hzZve>=&3k)k=SJ{!#-38dofFDIfTA~d6X3u_v6Ry)ptHP_~h0F7J2Q7xwZS=(v9mk zR%~4RL9NKaRwilQIOU{6a}Hf%$e(m?l#ri(Q!;qsp1K$7RC7NhpI@ddcs>@)qtN?I z9O!9_&F{REj(4vZt>Qo2C(Yo=QRNS&W;?2%^FPFe?j31ed6{4M`J+h1tlN7|wYXfb zdCOjWc79!w->xHdrp@Q;1oJ*R(f>}spb*pLmOd<9wetA%d;F&7$}?tkKbN<2s#UKq zY;&Dx_k(Ke`57%6?q1GJALTkValo1fjL8uhVRn183SZV;H~#QASDQMIvZ3P2#w+=( zn18Oq*Rl4reAkUPmqaXI^ZA^i(r)##k>|TkR@uHddwGjvv00$rSd}^Isb+OKM-ML; zPP6S3OV{8F?rd1_*rYj&^x7o!MVLFa?t3nO^?9_dR7v-jN{ zIyG6B9_RO{`-)9wX_LzDb{kp|IrP=%zOOz;^_aNim`1|+^-TZU*-kh1WoFIsd)_tC z!7cL0`jWgS1g>Bn1>ab;2KuL@rO(z6)~R#bth~r;qOD2)_Xc}x{NI!q6(1@%|Ln5p zymexq*}6Hw8kfO>hg>Iy@z8v0OFKZRM59}j_0`J&Do;v ziUo@sr)Ph8^x&xRUz=jGmR7pkmn(>R_?%67Tz5`1ZeZ2i&Eu=AoIH}sm{(FQ-6Ko# z(wtNzW|8`$aUQ-YYYlX}zI`70HI8*lR`uR+WzEad%B%FJ@(%Mwif$i&Z{t2lLTf~B zWJSdO1qBD+51Ll_;Pe3fm8%YQUoftFcMXfg^=17?eNq2l*NO`IF5RQJHn${hwEW`e zW7jh?4_eOblbQ0?mzOSTF|n`Q7MlV zGwf{dlDIK|3Eu#=2D-hYo|)?D3njNrB6-~`7Cg1r)w4-t9;SQIYVK73W$L{-M_w`Z z_P~`7T}B+;Fj!>R^lh^*Ogz5cF?O}_mav!wSYLBMoZ5;C`u5vZTHNP))1$o-5)~$6 z{X%RXPunoqgT`vw}Xdb#^ z6G7^W>^qH$D^}=!O)?~!#xXqFq<(E&l+w-~D?=q!FJIGB(mG~$#xirLuF31HuF4D7 z46PcGr!{PJcCY9cx>@NOvi1_W$*%?P83gkv2P*E+jaN2ZY0*$veX6efn$~o;8NGJS z*`l8v>00+$ZPcV$6DfUw1_eyTUWDmi(8AH5p zpH9UM@)Pw`Pm?Kp-BVOCus273YG{*>XXl5A6(noi&9 zRac!UvRKl$@?Cz)_`~U8yH9%zyr+KZqi397+N6C_Iu#M7S7Y`pqTtS;;y#_tJDzvR zE3ov|{>zT4>Jc`c<@9k_V{^w5=FElpa7@Na;F(8E2I3aD& zsl!oLHD1$`DY#BlT+>yGilPZ8hik?UdCD1a*SShtXS#es&Ga|b^cA;4$A%5x9~R^k z9sHc1^CdOO@X#}97dBt-tlWuPX7;MPf`gt>aA#6+Gmf2!+Pm_A%E?*wGH*X6)#TQj z?bxd^tL0`{!n58-H|AWqa$9;^|F}6#kDeyJFepg#E4Ir2aDSbXocy!K2>;>)3hpc_ zZf#ZNx^Lr;m^YNq(i<_*ben%SFOdYz-k)AVvzC6b+ta$c{pjRZ#_wNSu^k$!DzAvw z$luSKu)NXjP4Y$AtMksC*KxQ^KPsX6WkI?dw{P;T8)7Ry0q&ph#{&Om0T`Ky@5|sF zY7A_s!SM9s{oAeo)9WZ-yw5^k$Q z{e2huTUT=+3m}vq!bgA4fbu|pCxE``R|Z1g%A@bv(KqPmdvWw_H~J1+6$t%37y5og z0tgih)~<=a%U%GnCy*Bq3&SU4Z5SxdOQX9U$u=7hQz#v-&_d z2p8EC*%jFr*-Qio*$>qbvL&)9vMsJRR9~pxQ2n8MsD$ba)fK8URCfwM@<4JxvOvft z$TmHJdH^9?AsNIcUGV=~gudf40U8a2{K*Ii`4p-{148Y_0>~T)eVc{8`!WMU{)2o6*&O)~itm&Dan27C z5F}0K=o!upHG*!!u(j>#{-c{9_N&ey_w$tzaUIFfVSovcw*bZ99e~9}Ty+_`3@|9F zfAFaoyyr7;bQpT07`lXI5eM%IEs8k$h5(26Kgat<14o}>zyR}Wkon*ptbsEMIG_RE z^BwP%4IBf;D8%sw8Su0iy!*ANh%Q49F^F~zwur&|X(JBGpGJ!&W$@0~hy(xN>PjG_ zHQrkrada5SLpA^*G`#z^FrRGTfPcr}{ksKdZD{}&MR33x35`$&WbmHe0vY6Wc=vj| z>$j+gKEs&M8t-q9_XQUf0n@ZuQei$^Qfq93&T;V0_IO8ds0Cy=P#f=UkM{%@a`5i= zc-L^?K=m-7q2c}U@xJ1u3|I&6q>pzX7jp1k`*@FXf&(50rvX4W1wBy=-o;!fgZJyl z`<#;;kik3pkO)eqh)UC6FOh2#ODX#eb+&TY(MxU0lWjd zsK~;R3_U|I!oOG)T=Cz_6YK+(3gOYXx|1_j$jK>y2B>d?Q7(uTfnt@OF0@C(Q>_(5 zP>Vu=%VK)+Azu6BX07ttv&l$Cmk{!w$72P|1K*yaS|57)uDnnta2_ut0IX!-mQeDs zv0f3$KoSC6XP_6>;Sat}&R^1nOQ2y6o6pjxWv(e6$Jg4BfHMqNL@(K`l6xy{~G5m35O4?GA8Yq>33hRq5TglcKIjMu(vbjjMs=H2K-B zQ(f_}NGS7xtm1({X(BWA!iG4 zdIMTlpGp4rx{r8(&;&J?M{*`g)7DKLaxjF{4e}`i4$L=V;!hoyxpVj7GNH^Pk~4h# z8FSf^U4-2PUWLC)!29My`+-U@hTa#S5P^5r2aY}i+7^tT;5f(zc<+6I4C)Z@X&U$x z0^sO@3^*e`hXU{44;*qV!6$Fv6AOrZz^jDcObB&b2)Y?D^mOnk9r$zuLIY^sx-@)V z2R`QjIArgSPxQbiArLae(1g#*z-J85MaXPu_%scC3W1Qb7#afd9ehrKkb_U;z$Y09 zIrz*DeAa=GgHQFqry&SA_}mYCUV@N=PX@s!DhN6F>=1nBf{=qx7s0182s!xt5qxfg zkb_Ss!6!QiIhN4mkssi*AA}ry$_YLlLde1Apy2Z(gdBWQ$_4U-2_XlcrGn3(5OVNo zEBKTOAqStwg3qxKa`1^Q_@oOV2cPMJ&%zLL@To8OvlJ8OoC5z z0FD6z=3M{uYZ;MfFeB=Xiy%f`g-?SZ@-b%U>e1TtQG(N^j}n|VeU#v|>7xXvO&=vV zZTcv|Y12muPMbbTaN6`yg43pt5}Y=Dl;E`KqXef-A0;?#`Y6F^(?Q276t;7d*D9ccvCuW?+HBL8Dk1rN9*@NZw7FHPiyXWsfAoci zP&6}HYBaJ;Btq(Y45@)mtUx1pmWSsv-G6;NNZggj2YJxg0*nBh=fE=ga2@^G0iHb6GxcG!}OrBPck~kH_F}ec%!tqou7Sh_#3j zkiqwfG#6a=2;gu6$={v23g2J;|KU08| zq}&YJP*Pgx*+K{D^pjGMAOaOZUho4%VK_yio&ljes7oz>);uj<0Eh3x3Ie@6wSqaE zfSI{7@$VBB&e?MNeMs zvL~up5JVA(!4q)%#0cj=B}Yy|sH)pOLBicYkciu*sL;njOqkFj77S3P=E95GOdccx z**q5Yq6d@5GBluG@#J{1pwg(9SYhD$tUxv&oQQheo5crrr)1$Bz+v*K7uZ1|p=|CC zIR-QNKgb{_q>6;{y^YAW9R#ZU#6QuqCX^dRS}@s!!TcB1D(Eo~Du|&-OVEev8X!sN zNZ6iiE#ju1FjQCs0ukXUzw}IsEHWI(qNwAtPzgvDB0~0; zjfs*kS*IXQ{_1eS}UB%3p5A5C(LX-5vX>s520Fc05gaM(-&p{ zOWOwJ4w3@f{iIvMH35?#O1(e|>IJPuqk171 z7qkQuA?VAo5TC^i)b`+VVB%noZ$dyUULYK8%JgD!f0FaRNPreXad2s}WAtJ1wZOuh zAaAzM50ai7E=wzv&kp$Mbcw2+Py+H2Ccur+A{dmH#|Z_;&v8X>$f;1q2 zU{jKdD@4P^+yHh^XoMD%3(Z7;s4ch>0doa>g%=vc;b~E?gtLS6^?tbG#pJ@pA1(=i z*i3!BRwQV19z3rf?h-)#a2FFM`W&{`4_e`CR4u&WP4M60TFXPxUvMor*z+IPc>2aV z5v|&x-sJCi0`<8ll*bnKBg8fi` z0QN(=qhM4I0W4~=qfitF1SCqTqd*kz1|UkRqd)?8BAN~WQBuDLM6}CRAR_g9KtvmB z1tL-%1;QQ#y$k?RQXK`t-Xj2_q<$9&iUS&9AV_r-NZ8K;jxf|&kk(NWRB8R)Q4oPC1e&z|{yh+Z#|S{$?=RF@ z8x(|^ghe4E!Sn@=bLRwvv9yByebAiiXY;tf-wK$2zXTJTUoSSbHG>5#!BiF*=+}>y zcp8K!#UTFoOE9A(SJH@AqI#;Px<45r@%%`L1A3r^-yddJOm>78(X0Ig>2HJWPseVY zLHpr;GIn?dP)N5F933ZD5xUrVG>eRc$l(k&8TKcqVfMjfodMX1S+LJ9g3TtrH$ zqd=6=8dRdBItnD{4*q5A?uf|+K>sp!cN_@YsI|K?H0Ws2ut&8yc6UT0>^*Ic-M<3_ zg@andu#oC#(FC4Dj+VkiM`47+HK-#@bQDHVM{SPX9kGsZxE8iDFnLF{5tu_zSFOK0 z3eq}WgDS1RI|?GOi$Igs-@gYU@R-)d&{2@qAqTk~9)8*hLcBdkHNx@;^?@NDeeCt~ zC&fVYUM-LZD?jjR^B4Ex<+M=rz6C8BqPJc~zYuC&TqI?Pg+{@0h#@i53YPoBe>M*x zGY&_(!r~qA;*=Pt1@AEh3w}QtQi&{x*O34qOu&nPk+7Bu1`@p0gzu^yLZ|bs#(M>m W?>YIx_nklemK=QriZTAL#QzVv*qs{y literal 0 HcmV?d00001 diff --git a/web/bridge/image.nix b/web/bridge/image.nix new file mode 100644 index 000000000..4d8c40c08 --- /dev/null +++ b/web/bridge/image.nix @@ -0,0 +1,32 @@ +{ + pkgs, + buildImage, + ... +}: let + # NativeLink Bridge + nativelink-bridge = pkgs.stdenv.mkDerivation { + name = "nativelink-bridge"; + src = ./.; + buildInputs = [pkgs.bun]; + installPhase = '' + mkdir -p $out + cp -r $src/* $out + ''; + }; +in + buildImage { + name = "nativelink-bridge"; + + # Container configuration + config = { + WorkingDir = "${nativelink-bridge}"; + Entrypoint = ["${pkgs.bun}/bin/bun" "run" "index.ts"]; + ExposedPorts = { + "8080/tcp" = {}; + }; + Labels = { + "org.opencontainers.image.description" = "A simple Bun environment image"; + "org.opencontainers.image.title" = "Bun Environment"; + }; + }; + } diff --git a/web/bridge/index.ts b/web/bridge/index.ts new file mode 100644 index 000000000..a50671e29 --- /dev/null +++ b/web/bridge/index.ts @@ -0,0 +1,3 @@ +import { start } from './src'; + +start().catch(err => console.error(err)); diff --git a/web/bridge/package.json b/web/bridge/package.json new file mode 100644 index 000000000..3fd75869d --- /dev/null +++ b/web/bridge/package.json @@ -0,0 +1,22 @@ +{ + "name": "nativelink-bridge", + "version": "0.5.3", + "module": "index.ts", + "type": "module", + "dependencies": { + "drizzle-orm": "^0.36.0", + "postgres": "^3.4.5", + "protobufjs": "^7.4.0", + "redis": "^4.7.0" + }, + "devDependencies": { + "@types/bun": "^1.1.8", + "drizzle-kit": "^0.27.1" + }, + "peerDependencies": { + "typescript": "^5.0.0" + }, + "trustedDependencies": [ + "protobufjs" + ] +} diff --git a/web/bridge/src/db/index.ts b/web/bridge/src/db/index.ts new file mode 100644 index 000000000..fa046bce1 --- /dev/null +++ b/web/bridge/src/db/index.ts @@ -0,0 +1,9 @@ +import { drizzle } from "drizzle-orm/postgres-js"; + +const postgresConfig = process.env.POSTGRES_URL || "postgres://user:password@localhost:5432/postgres" + +const db = drizzle(postgresConfig); + +export const build_data = await db.execute('select * from build_data'); + +export { db }; diff --git a/web/bridge/src/eventHandler.ts b/web/bridge/src/eventHandler.ts new file mode 100644 index 000000000..270810045 --- /dev/null +++ b/web/bridge/src/eventHandler.ts @@ -0,0 +1,95 @@ +import type protobuf from 'protobufjs'; +import type { BuildEvent, Progress, ParsedMessage } from './types/buildTypes'; +import { commandOptions, type RedisClientType } from 'redis'; +import { broadcastProgress } from './websocket'; + + +export async function handleEvent(message: string, commandClient: RedisClientType, types: { PublishBuildToolEventStreamRequest: protobuf.Type, PublishLifecycleEventRequest: protobuf.Type }) { + switch (parseMessage(message).eventType) { + case 'LifecycleEvent': + await fetchAndDecodeBuildData(constructRedisKey(parseMessage(message)), commandClient, types.PublishLifecycleEventRequest); + break; + case 'BuildToolEventStream': + await fetchAndDecodeBuildData(constructRedisKey(parseMessage(message)), commandClient, types.PublishBuildToolEventStreamRequest); + break; + default: + console.log('Unknown event type:', parseMessage(message).eventType); + } +} + +async function fetchAndDecodeBuildData(redisKey: string, commandClient: RedisClientType, messageType: protobuf.Type) { + try { + const buildData = await commandClient.get(commandOptions({ returnBuffers: true }), redisKey); + if (buildData) { + const decodedMessage = messageType.decode(new Uint8Array(Buffer.from(buildData))) as BuildEvent; + if(decodedMessage.orderedBuildEvent) { + const buildId = decodedMessage.orderedBuildEvent.streamId.buildId + const invocationId = decodedMessage.orderedBuildEvent.streamId.invocationId + console.log("Build ID: ", buildId) + console.log("Invocation ID: ", invocationId) + const eventTime = decodedMessage.orderedBuildEvent.event.eventTime; + const milliseconds = eventTime.seconds.low * 1000 + Math.floor(eventTime.nanos / 1000000); + const eventDate = new Date(milliseconds); + console.log("Event time nanos:", eventTime.nanos) + console.log("Event time seconds:", eventTime.seconds.low) + console.log("Event time:", eventDate.toISOString()); + const currentTime = new Date() + const elapsedTime = currentTime.getTime() - eventDate.getTime(); + console.log("Time Now: ", currentTime.toISOString()) + console.log(`Elapsed Time: ${elapsedTime} ms`); + } + if (decodedMessage?.orderedBuildEvent?.event?.bazelEvent) { + console.log("------------------") + decodeBazelEvent(decodedMessage.orderedBuildEvent.event.bazelEvent, messageType.root); + } + } + } catch (err) { + console.error(`Error fetching build data for key ${redisKey}:`, err); + } +} + +// TODO(SchahinRohani): Add Bazel Event Types +// biome-ignore lint/suspicious/noExplicitAny: Bazel Event Types are not known yet +function decodeBazelEvent(bazelEvent: any, root: protobuf.Root): any { + if (!bazelEvent || !bazelEvent.value) return null; + const messageType = root.lookupType(bazelEvent.typeUrl.split('/').pop()); + const decodedMessage = messageType.decode(new Uint8Array(Buffer.from(bazelEvent.value, 'base64'))); + const decodedObject = messageType.toObject(decodedMessage, { + longs: String, + enums: String, + bytes: String, + }); + if (decodedObject.progress) { + console.log("Processing progress information...\n\n"); + processProgress(decodedObject.progress); + } + return decodedObject; +} + +function processProgress(progress: Progress) { + if (progress.stderr) { + console.log(progress.stderr); + broadcastProgress(progress.stderr) + } +} + +export function parseMessage(message: string) { + const parts = message.split(':'); + const [prefix, eventType, eventID, subEventID, sequenceNumber] = parts; + return { + prefix, + eventType, + eventID, + subEventID, + sequenceNumber + }; +} + +export function constructRedisKey(parsedMessage: ParsedMessage) { + console.log("\nNew Published Event: ") + console.log(" EventID: ", parsedMessage.eventID) + console.log(" Sequence Number: ", parsedMessage.sequenceNumber) + console.log(" Invocation ID: ", parsedMessage.subEventID) + console.log("------------------") + return `${parsedMessage.prefix}:${parsedMessage.eventType}:${parsedMessage.eventID}:${parsedMessage.subEventID}:${parsedMessage.sequenceNumber}`; +} diff --git a/web/bridge/src/http.ts b/web/bridge/src/http.ts new file mode 100644 index 000000000..6737ac481 --- /dev/null +++ b/web/bridge/src/http.ts @@ -0,0 +1,45 @@ +import { serve } from "bun"; +import { build_data } from "./db" + +const httpPort = Number(process.env.HTTP_PORT) || 3001; + +export const startWebServer = () => { + console.log(`\nHTTP server is running on http://localhost:${httpPort}\n`); + serve({ + port: httpPort, + fetch(req) { + const url = new URL(req.url); + const handler = routes.get(url.pathname); + if (handler){ + return handler(); + } + return new Response("Not Found", { status: 404 }); + }, + }); +}; + +const routes = new Map Promise>([ + ["/api", () => jsonResponse({ message: "Hello from API" })], + ["/builds", () => getBuilds()], + ["/health", () => jsonResponse({ status: "ok" })], + ["/readiness", () => jsonResponse({ status: "ready" })], + ]); + +async function getBuilds(): Promise { + const dbResult = await build_data; + return jsonResponse({ build_data: dbResult}) +} + +async function jsonResponse(data: object, status: number = 200): Promise { + const responseData = { + ...data, + timestamp: new Date().toISOString(), + }; + return new Response(JSON.stringify(responseData), { + headers: { + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*" + }, + status, + }); + } diff --git a/web/bridge/src/index.ts b/web/bridge/src/index.ts new file mode 100644 index 000000000..551f24861 --- /dev/null +++ b/web/bridge/src/index.ts @@ -0,0 +1,53 @@ +import { initializeRedisClients } from './redis'; +import { initializeProtobuf } from './protobuf'; +import { handleEvent } from './eventHandler'; +import { startWebSocket } from './websocket'; +import { startWebServer } from './http'; + + +export async function start() { + // Google Remote Path + const googleProtoPath = `https://raw.githubusercontent.com/googleapis/googleapis/1f2e5aab4f95b9bd38dd1ac8c7486657f93c1975/google/devtools/build/v1`; + + // Bazel Remote Path + const bazelProtoPath = `https://raw.githubusercontent.com/bazelbuild/bazel/9.0.0-pre.20241023.1/src/main/java/com/google/devtools/build/lib/buildeventstream/proto`; + + // TODO(SchahinRohani): Add Buck2 Protos for future Buck2 support + // const buck2ProtoPath = `https://raw.githubusercontent.com/facebook/buck2/2024-11-01/app/buck2_data/data.proto`; + + // Actual using Protos. + const PublishBuildEventProto =`${googleProtoPath}/publish_build_event.proto`; + const BazelBuildEventStreamProto = `${bazelProtoPath}/build_event_stream.proto`; + + const protos = [ PublishBuildEventProto, BazelBuildEventStreamProto ] + + console.info("Link to: \n") + console.info("Google Publish Build Events Proto:\n", PublishBuildEventProto, "\n"); + console.info("Bazel Build Event Stream Proto:\n", BazelBuildEventStreamProto, "\n") + + // Load Remote Bazel Proto Files + const protoTypes = await initializeProtobuf(protos) + + const { redisClient, commandClient } = await initializeRedisClients(); + + // Subscribe to the build_event channel + await redisClient.subscribe(process.env.NATIVELINK_PUB_SUB_CHANNEL || "build_event", async (message: string) => { + await handleEvent(message, commandClient, protoTypes); + }); + + const websocketServer = startWebSocket(); + const webServer = startWebServer(); + + process.on('SIGINT', async () => { + await redisClient.disconnect(); + await commandClient.disconnect(); + console.info("Received SIGINT. Shutdown gracefully.") + process.exit(); + }); + process.on('SIGTERM', async () => { + await redisClient.disconnect(); + await commandClient.disconnect(); + console.info("Received SIGTERM. Shutdown gracefully.") + process.exit(); + }); +} diff --git a/web/bridge/src/protobuf.ts b/web/bridge/src/protobuf.ts new file mode 100644 index 000000000..48e9329c7 --- /dev/null +++ b/web/bridge/src/protobuf.ts @@ -0,0 +1,76 @@ +import protobuf from 'protobufjs'; + +export async function initializeProtobuf(protos: string[]) { + console.log("Loading Remote Proto Files"); + + const combinedRoot = new protobuf.Root(); + const loadedFiles: Record = {}; + const processedImports = new Set(); + for (const proto of protos) { + await loadProto(loadedFiles, combinedRoot, proto, processedImports); + } + console.log("\nDone parsing all proto files.\n"); + const BazelBuildEvent = combinedRoot.lookupType("build_event_stream.BuildEvent"); + const PublishBuildToolEventStreamRequest = combinedRoot.lookupType("google.devtools.build.v1.PublishBuildToolEventStreamRequest"); + const PublishLifecycleEventRequest = combinedRoot.lookupType("google.devtools.build.v1.PublishLifecycleEventRequest"); + + console.log("Loaded Types:\n"); + console.log({ + PublishLifecycleEventRequest: PublishLifecycleEventRequest ? PublishLifecycleEventRequest.fullName : "Not found", + PublishBuildToolEventStreamRequest: PublishBuildToolEventStreamRequest ? PublishBuildToolEventStreamRequest.fullName : "Not found", + BazelBuildEvent: BazelBuildEvent ? BazelBuildEvent.fullName : "Not found" + }); + + return { + PublishLifecycleEventRequest, + PublishBuildToolEventStreamRequest, + BazelBuildEvent + }; +} + +function resolveImportPath(protoUrl: string, importPath: string): string { + if (importPath.startsWith("google/api") || importPath.startsWith("google/devtools/build/v1")) { + return `https://raw.githubusercontent.com/googleapis/googleapis/1f2e5aab4f95b9bd38dd1ac8c7486657f93c1975/${importPath}`; + } + + if (importPath.startsWith("google/protobuf")) { + return `https://raw.githubusercontent.com/protocolbuffers/protobuf/v29.0-rc2/src/${importPath}`; + } + + if (importPath.includes("com/google/devtools/build/lib/packages/metrics") || importPath.startsWith("src/main/protobuf")) { + return `https://raw.githubusercontent.com/bazelbuild/bazel/9.0.0-pre.20241023.1/${importPath}`; + } + return new URL(importPath, protoUrl).toString(); +} + +async function loadProto( + loadedFiles: Record, + root: protobuf.Root, + protoUrl: string, + processedImports: Set, + indentLevel = 0, +) { + if (loadedFiles[protoUrl]) { + return; + } + const response = await fetch(protoUrl); + if (!response.ok) { + throw new Error(`Failed to fetch .proto file from ${protoUrl}: ${response.statusText}`); + } + + const parsedProto = protobuf.parse(await response.text(), root); + loadedFiles[protoUrl] = true; + if (indentLevel < 1) { + console.log(`\n${ ' '.repeat(indentLevel)} ${protoUrl}:`); + } + if (parsedProto.imports && parsedProto.imports.length > 0) { + for (const importPath of parsedProto.imports) { + const resolvedImportUrl = resolveImportPath(protoUrl, importPath); + if (!processedImports.has(resolvedImportUrl)) { + console.log(`${ ' '.repeat(indentLevel)} - ${importPath}`); + processedImports.add(resolvedImportUrl); + await loadProto(loadedFiles, root, resolvedImportUrl, processedImports, indentLevel + 1,); + } + } + } +} diff --git a/web/bridge/src/redis.ts b/web/bridge/src/redis.ts new file mode 100644 index 000000000..8154e9512 --- /dev/null +++ b/web/bridge/src/redis.ts @@ -0,0 +1,25 @@ +import { createClient, type RedisClientType } from 'redis'; + +export async function initializeRedisClients() { + try { + const redisClient: RedisClientType = createClient({ + url: process.env.REDIS_URL, + }); + const commandClient = redisClient.duplicate(); + + redisClient.on('error', (err) => { + console.error('Redis Client Error:', err); + throw new Error('Failed to connect to Redis.'); + }); + + await redisClient.connect(); + await commandClient.connect(); + + console.log('\nRedis clients successfully connected.\n'); + + return { redisClient, commandClient }; + } catch (error) { + console.error('Error during Redis client initialization:', error); + throw new Error('Unable to initialize Redis clients. Please check your connection.'); + } +} diff --git a/web/bridge/src/types/buildTypes.ts b/web/bridge/src/types/buildTypes.ts new file mode 100644 index 000000000..04531d677 --- /dev/null +++ b/web/bridge/src/types/buildTypes.ts @@ -0,0 +1,29 @@ +export interface BuildEvent extends protobuf.Message { + orderedBuildEvent: { + streamId: { + buildId: string; + invocationId: string; + }, + event: { + eventTime: { + seconds: protobuf.Long; + nanos: number; + }; + // biome-ignore lint/suspicious/noExplicitAny: Not known yet + bazelEvent?: any; + }; + }; + } + + export type ParsedMessage = { + prefix: string; + eventType: string; + eventID: string; + subEventID: string; + sequenceNumber: string; +} + + + export type Progress = { + stderr: string; + }; diff --git a/web/bridge/src/websocket.ts b/web/bridge/src/websocket.ts new file mode 100644 index 000000000..d0c33b6c4 --- /dev/null +++ b/web/bridge/src/websocket.ts @@ -0,0 +1,44 @@ +import type { ServerWebSocket } from "bun"; + +const clients = new Set>(); + +const websocketPort = Number(process.env.WEBSOCKET_PORT) || 8080; + +export const startWebSocket = () => { + console.log(`\nWebSocket server is running on ws://localhost:${websocketPort}\n`); + Bun.serve({ + port: websocketPort, + fetch(req, server) { + // Upgrade the request to a WebSocket + // Here we can also do the websocket auth/token auth + if (server.upgrade(req)) { + return; + } + return new Response("Upgrade failed", { status: 500 }); + }, + websocket: { + open(ws) { + console.log('New client connected'); + clients.add(ws); + ws.send("Hello Web Client") + }, + message(ws, message) { + console.log('Received message from web client:', message); + }, + close(ws) { + console.log('Web Client disconnected'); + clients.delete(ws); + }, + drain(ws) { + console.log('Ready to receive more data'); + }, + }, +});} + +export function broadcastProgress(progress: string) { + const buffer = Buffer.from(progress) + console.log(progress) + for (const ws of clients) { + ws.send(new Uint8Array(buffer)); + } +} diff --git a/web/bridge/tsconfig.json b/web/bridge/tsconfig.json new file mode 100644 index 000000000..238655f2c --- /dev/null +++ b/web/bridge/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + // Enable latest features + "lib": ["ESNext", "DOM"], + "target": "ESNext", + "module": "ESNext", + "moduleDetection": "force", + "jsx": "react-jsx", + "allowJs": true, + + // Bundler mode + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + + // Best practices + "strict": true, + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + + // Some stricter flags (disabled by default) + "noUnusedLocals": false, + "noUnusedParameters": false, + "noPropertyAccessFromIndexSignature": false + } +}