Full auto chrome builds (#705)

* automated build * update to new chrome installer image * use the right chrome driver * use the right chrome driver * sleep after running * linode-cli create not working * final fixes
livekit · Jun 20, 2024 · cd077eb · cd077eb
1 parent 7d48aa2
commit cd077eb
Show file tree

Hide file tree

Showing 11 changed files with 156 additions and 123 deletions.
diff --git a/.github/workflows/publish-chrome.yaml b/.github/workflows/publish-chrome.yaml
@@ -3,8 +3,8 @@ name: Publish Chrome
 on:
   workflow_dispatch:
     inputs:
-      builder_ip:
-        description: "IP address of the builder"
+      chrome_version:
+        description: "Version of Chrome to build"
         required: true
         type: string
 
@@ -15,31 +15,116 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Configure SSH
-        run: |
-          mkdir -p ~/.ssh/
-          echo "$SSH_KEY" > ~/.ssh/builder.key
-          chmod 600 ~/.ssh/builder.key
-          cat >>~/.ssh/config <<END
-          Host builder
-            HostName $SSH_HOST
-            User root
-            IdentityFile ~/.ssh/builder.key
-            StrictHostKeyChecking no
-          END
-        env:
-          SSH_KEY: ${{ secrets.LINODE_SSH_KEY }}
-          SSH_HOST: ${{ inputs.builder_ip }}
+      - name: Install the Linode CLI
+        uses: linode/action-linode-cli@v1
+        with:
+          token: ${{ secrets.LINODE_PAT }}
+
+#      For some reason, Linodes created via linode-cli do not work with ssh,
+#      so we have to create the Linode manually and then pull the Linode ID
+#
+#      - name: Create Builder
+#        id: create_builder
+#        run: |
+#          builder_info="$(linode-cli linodes create \
+#            --authorized_users dclivekit \
+#            --backups_enabled false \
+#            --booted true \
+#            --image linode/ubuntu22.04 \
+#            --label chrome-builder \
+#            --private_ip false \
+#            --region us-west \
+#            --root_pass '${{ secrets.LINODE_ROOT_PASS }}' \
+#            --type g6-dedicated-4 \
+#            --json)"
+#          builder_id="$(echo $builder_info | jq -r '.[0].id')"
+#          builder_ip="$(echo $builder_info | jq -r '.[0].ipv4[0]')"
+#          echo "builder_id: $builder_id"
+#          echo "builder_ip: $builder_ip"
+#          echo "builder_id=$builder_id" >> $GITHUB_OUTPUT
+#          echo "builder_ip=$builder_ip" >> $GITHUB_OUTPUT
+#        env:
+#          LINODE_CLI_TOKEN: ${{ secrets.LINODE_PAT }}
 
-      - name: Create chrome user
-        run: ssh -t root@$SSH_HOST 'bash -s' < ./build/chrome/scripts/create-user.sh
+      - name: Get Builder
+        id: get_builder
+        run: |
+          builder_info="$(linode-cli linodes list --label chrome-builder --json)"
+          builder_id="$(echo $builder_info | jq -r '.[0].id')"
+          builder_ip="$(echo $builder_info | jq -r '.[0].ipv4[0]')"
+          echo "builder_id: $builder_id"
+          echo "builder_ip: $builder_ip"
+          echo "builder_id=$builder_id" >> $GITHUB_OUTPUT
+          echo "builder_ip=$builder_ip" >> $GITHUB_OUTPUT
         env:
-          SSH_HOST: ${{ inputs.builder_ip }}
+          LINODE_CLI_TOKEN: ${{ secrets.LINODE_PAT }}
 
-      - name: Build chrome
-        run: ssh -t chrome@$SSH_HOST 'bash -s' < ./build/chrome/scripts/build.sh
+      - name: Wait for Builder
+        run: |
+          status=$(linode-cli linodes view ${{ steps.get_builder.outputs.builder_id }} --json | jq -r '.[0].status')
+          while [ "$status" == "provisioning" ] || [ "$status" == "booting" ]; do \
+              echo "Builder status: $status"; \
+              sleep 5; \
+              status=$(linode-cli linodes view ${{ steps.get_builder.outputs.builder_id }} --json | jq -r '.[0].status'); \
+          done
+          echo "Builder status: $status"
         env:
-          SSH_HOST: ${{ inputs.builder_ip }}
+          LINODE_CLI_TOKEN: ${{ secrets.LINODE_PAT }}
+
+      - name: Write SSH keys
+        run: |
+          mkdir ~/.ssh
+          chmod 700 ~/.ssh
+          echo "${{ secrets.LINODE_SSH_PUBLIC_KEY }}" > ~/.ssh/linode_ed25519.pub
+          chmod 600 ~/.ssh/linode_ed25519.pub
+          echo "${{ secrets.LINODE_SSH_PRIVATE_KEY }}" > ~/.ssh/linode_ed25519
+          chmod 600 ~/.ssh/linode_ed25519
+          ssh-keyscan -H ${{ steps.get_builder.outputs.builder_ip }} > ~/.ssh/known_hosts
+
+      - name: Setup
+        run: |
+          ssh -i ~/.ssh/linode_ed25519 \
+              -o PasswordAuthentication=no \
+              -t root@${{ steps.get_builder.outputs.builder_ip }} \
+              'bash -s' < ./build/chrome/scripts/setup.sh
+
+      - name: Amd64
+        run: |
+          ssh -i ~/.ssh/linode_ed25519 \
+              -o PasswordAuthentication=no \
+              -t chrome@${{ steps.get_builder.outputs.builder_ip }} \
+              'bash -s ${{ inputs.chrome_version }}' < ./build/chrome/scripts/amd64.sh
+
+      - name: Arm64
+        run: |
+          ssh -i ~/.ssh/linode_ed25519 \
+              -o PasswordAuthentication=no \
+              -o ServerAliveInterval=60 \
+              -t chrome@${{ steps.get_builder.outputs.builder_ip }} \
+              'bash -s ${{ inputs.chrome_version }}' < ./build/chrome/scripts/arm64.sh
+
+      - name: Drivers
+        run: |
+          ssh -i ~/.ssh/linode_ed25519 \
+              -o PasswordAuthentication=no \
+              -o ServerAliveInterval=60 \
+              -t chrome@${{ steps.get_builder.outputs.builder_ip }} \
+              'bash -s ${{ inputs.chrome_version }}' < ./build/chrome/scripts/driver.sh
+
+      - name: Prepare artifacts
+        run: |
+          ssh -i ~/.ssh/linode_ed25519 \
+              -o PasswordAuthentication=no \
+              -t chrome@${{ steps.get_builder.outputs.builder_ip }} \
+              'zip -r output.zip ./output'
+
+      - name: Download artifacts
+        run: |
+          scp -i ~/.ssh/linode_ed25519 \
+              -o PasswordAuthentication=no \
+              chrome@${{ steps.get_builder.outputs.builder_ip }}:/home/chrome/output.zip \
+              ${{ github.workspace }}/build/chrome/output.zip
+          unzip ${{ github.workspace }}/build/chrome/output.zip -d ${{ github.workspace }}/build/chrome
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -53,8 +138,13 @@ jobs:
       - name: Build and push
         uses: docker/build-push-action@v5
         with:
-          context: .
+          context: ./build/chrome
           file: ./build/chrome/Dockerfile
           push: true
           platforms: linux/amd64,linux/arm64
-          tags: test
+          tags: livekit/chrome-installer:${{ inputs.chrome_version }}
+
+      - name: Delete Linode
+        run: linode-cli linodes delete ${{ steps.get_builder.outputs.builder_id }}
+        env:
+          LINODE_CLI_TOKEN: ${{ secrets.LINODE_PAT }}
diff --git a/build/chrome/.gitignore b/build/chrome/.gitignore
diff --git a/build/chrome/Dockerfile b/build/chrome/Dockerfile
@@ -14,5 +14,7 @@
 
 FROM ubuntu:22.04
 
-COPY arm64/ /chrome-installer/arm64/
-COPY install-chrome /chrome-installer/
+RUN mkdir /chrome-installer
+COPY output/arm64 /chrome-installer/arm64
+COPY output/amd64 /chrome-installer/amd64
+COPY install-chrome /chrome-installer/install-chrome
diff --git a/build/chrome/README.md b/build/chrome/README.md
@@ -10,10 +10,10 @@ To install chrome, add the following to your dockerfile:
 
 ```dockerfile
 ARG TARGETPLATFORM
-COPY --from=livekit:chrome-installer /chrome-installer /chrome-installer
-RUN /chrome-installer/install-chrome "$TARGETPLATFORM" && \
-    rm -rf /chrome-installer \
+COPY --from=livekit/chrome-installer:124.0.6367.201 /chrome-installer /chrome-installer
+RUN /chrome-installer/install-chrome "$TARGETPLATFORM"
 ENV PATH=${PATH}:/chrome
+ENV CHROME_DEVEL_SANDBOX=/usr/local/sbin/chrome-devel-sandbox
 ```
 
 ## Compilation 
@@ -31,74 +31,3 @@ Relevant docs:
 * 64+ CPU cores
 * 128GB+ RAM
 * 100GB+ disk space
-
-### Build steps
-
-```shell
-export CHROME_BUILDER={ip}
-ssh root@$CHROME_BUILDER
-```
-```shell
-adduser chrome
-adduser chrome sudo
-su - chrome
-```
-```shell
-sudo apt-get update
-sudo apt-get install -y \
-  apt-utils \
-  build-essential \
-  curl \
-  git \
-  python3 \
-  sudo \
-  zip
-git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git
-export PATH="$PATH:/home/chrome/depot_tools"
-mkdir chromium && cd chromium
-fetch --nohooks --no-history chromium
-echo 'solutions = [
-  {
-    "name": "src",
-    "url": "https://chromium.googlesource.com/chromium/src.git",
-    "managed": False,
-    "custom_deps": {},
-    "custom_vars": {
-      "checkout_pgo_profiles": True,
-    },
-    "target_cpu": "arm64",
-  },
-]' | tee '.gclient' > /dev/null
-cd src
-./build/install-build-deps.sh
-./build/linux/sysroot_scripts/install-sysroot.py --arch=arm64
-gclient runhooks
-gn gen out/default --args='target_cpu="arm64" proprietary_codecs=true ffmpeg_branding="Chrome" enable_nacl=false is_debug=false symbol_level=0 v8_symbol_level=0 dcheck_always_on=false is_official_build=true'
-autoninja -C out/default chrome chrome_sandbox
-cd out/default
-zip arm64.zip \
-  chrome \
-  chrome-wrapper \
-  chrome_sandbox \
-  chrome_100_percent.pak \
-  chrome_200_percent.pak \
-  chrome_crashpad_handler \
-  headless_lib_data.pak \
-  headless_lib_strings.pak \
-  icudtl.dat \
-  locales/en-US.pak \
-  libEGL.so \
-  libGLESv2.so \
-  resources.pak \
-  snapshot_blob.bin \
-  v8_context_snapshot.bin
-exit
-```
-```shell
-exit
-```
-```shell
-scp root@$CHROME_BUILDER:/home/chrome/chromium/src/out/default/arm64.zip ~/livekit/egress/build/chrome/arm64.zip
-cd ~/livekit/egress/build/chrome
-mkdir arm64 && unzip arm64.zip -d arm64 && rm arm64.zip
-```
diff --git a/build/chrome/install-chrome b/build/chrome/install-chrome
@@ -36,20 +36,16 @@ then
     libxrender1 \
     libxss1 \
     libxtst6
+  chmod +x /chrome-installer/arm64/chromedriver-mac-arm64/chromedriver
+  mv -f /chrome-installer/arm64/chromedriver-mac-arm64/chromedriver /usr/local/bin/chromedriver
   mv /chrome-installer/arm64/ /chrome
   cp /chrome/chrome_sandbox /usr/local/sbin/chrome-devel-sandbox
   chown root:root /usr/local/sbin/chrome-devel-sandbox
   chmod 4755 /usr/local/sbin/chrome-devel-sandbox
 else
-  wget https://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_124.0.6367.201-1_amd64.deb
-  apt-get install -y ./google-chrome-stable_124.0.6367.201-1_amd64.deb
-  rm google-chrome-stable_124.0.6367.201-1_amd64.deb
+  apt-get install -y /chrome-installer/amd64/google-chrome-stable_124.0.6367.201-1_amd64.deb
+  chmod +x /chrome-installer/amd64/chromedriver-linux64/chromedriver
+  mv -f /chrome-installer/amd64/chromedriver-linux64/chromedriver /usr/local/bin/chromedriver
 fi
 
-wget -N https://chromedriver.storage.googleapis.com/2.46/chromedriver_linux64.zip
-unzip chromedriver_linux64.zip
-chmod +x chromedriver
-mv -f chromedriver /usr/local/bin/chromedriver
-rm chromedriver_linux64.zip
-
 rm -rf /chrome-installer
diff --git a/build/chrome/scripts/amd64.sh b/build/chrome/scripts/amd64.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+set -xeuo pipefail
+
+wget https://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_"$1"-1_amd64.deb
+mkdir -p "$HOME/output/amd64"
+mv google-chrome-stable_"$1"-1_amd64.deb "$HOME/output/amd64"
diff --git a/build/chrome/scripts/build.sh → build/chrome/scripts/arm64.sh b/build/chrome/scripts/build.sh → build/chrome/scripts/arm64.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+set -xeuo pipefail
 
 sudo apt-get update
 sudo apt-get install -y \
@@ -10,7 +11,7 @@ sudo apt-get install -y \
   sudo \
   zip
 git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git
-export PATH="$PATH:/home/chrome/depot_tools"
+export PATH="$PATH:$HOME/depot_tools"
 mkdir chromium
 cd chromium || exit
 fetch --nohooks --no-history chromium
@@ -28,28 +29,28 @@ echo 'solutions = [
 ]' | tee '.gclient' > /dev/null
 cd src || exit
 git fetch --tags
-git checkout -b stable 124.0.6367.201
-gclient sync --with_branch_heads --with_tags
+git checkout -b stable "$1"
+gclient sync -D --with_branch_heads --with_tags
 ./build/install-build-deps.sh
 ./build/linux/sysroot_scripts/install-sysroot.py --arch=arm64
 gclient runhooks
 gn gen out/default --args='target_cpu="arm64" proprietary_codecs=true ffmpeg_branding="Chrome" enable_nacl=false is_debug=false symbol_level=0 v8_symbol_level=0 dcheck_always_on=false is_official_build=true'
 autoninja -C out/default chrome chrome_sandbox
 cd out/default || exit
-mkdir "$GITHUB_WORKSPACE"/build/chrome/arm64
-mv "$GITHUB_WORKSPACE"/build/chrome/arm64 \
-  chrome \
+mkdir -p "$HOME/output/arm64/locales"
+mv locales/en-US.pak "$HOME/output/arm64/locales/"
+mv chrome \
   chrome-wrapper \
-  chrome_sandbox \
   chrome_100_percent.pak \
   chrome_200_percent.pak \
   chrome_crashpad_handler \
+  chrome_sandbox \
   headless_lib_data.pak \
   headless_lib_strings.pak \
   icudtl.dat \
-  locales/en-US.pak \
   libEGL.so \
   libGLESv2.so \
   resources.pak \
   snapshot_blob.bin \
-  v8_context_snapshot.bin
+  v8_context_snapshot.bin \
+  "$HOME/output/arm64/"
diff --git a/build/chrome/scripts/driver.sh b/build/chrome/scripts/driver.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+set -xeuo pipefail
+
+wget https://storage.googleapis.com/chrome-for-testing-public/"$1"/linux64/chromedriver-linux64.zip
+unzip chromedriver-linux64.zip -d "$HOME/output/amd64"
+wget https://storage.googleapis.com/chrome-for-testing-public/"$1"/mac-arm64/chromedriver-mac-arm64.zip
+unzip chromedriver-mac-arm64.zip -d "$HOME/output/arm64"
diff --git a/build/chrome/scripts/create-user.sh → build/chrome/scripts/setup.sh b/build/chrome/scripts/create-user.sh → build/chrome/scripts/setup.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+set -xeuo pipefail
 
 useradd -m -d /home/chrome -s /bin/bash chrome
 mkdir /home/chrome/.ssh
@@ -8,3 +9,6 @@ chmod 700 /home/chrome/.ssh
 chmod 600 /home/chrome/.ssh/authorized_keys
 adduser chrome sudo
 sed -i '54i chrome ALL=(ALL:ALL) NOPASSWD: ALL' /etc/sudoers
+echo "ClientAliveInterval 60" >> /etc/ssh/sshd_config
+echo "ClientAliveCountMax 3" >> /etc/ssh/sshd_config
+systemctl restart ssh
diff --git a/build/egress/Dockerfile b/build/egress/Dockerfile
@@ -68,7 +68,7 @@ RUN apt-get update && \
     gstreamer1.0-plugins-base-
 
 # install chrome
-COPY --from=livekit/chrome-installer:124.0.6367.201 /chrome-installer /chrome-installer
+COPY --from=livekit/chrome-installer:124.0.6367.201.1 /chrome-installer /chrome-installer
 RUN /chrome-installer/install-chrome "$TARGETPLATFORM"
 
 # clean up

diff --git a/build/test/Dockerfile b/build/test/Dockerfile
@@ -73,7 +73,7 @@ RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then GOARCH=arm64; else GOARCH=amd
 ENV PATH="/usr/local/go/bin:${PATH}"
 
 # install chrome
-COPY --from=livekit/chrome-installer:124.0.6367.201 /chrome-installer /chrome-installer
+COPY --from=livekit/chrome-installer:124.0.6367.201.1 /chrome-installer /chrome-installer
 RUN /chrome-installer/install-chrome "$TARGETPLATFORM"
 
 # clean up