feat: add ci for linux building and publishing (#17)

* feat: ci testing (#1) --------- Co-authored-by: Hu Yueh-Wei <[email protected]>
TEN-framework · Sep 18, 2024 · 42dd8bc · 42dd8bc
1 parent 116f0c3
commit 42dd8bc
Show file tree

Hide file tree

Showing 8 changed files with 350 additions and 13 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -0,0 +1,21 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/ubuntu
+{
+	"name": "ten_framework",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "ghcr.io/ten-framework/ten_building_ubuntu2204",
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	"features": {
+		"ghcr.io/devcontainers/features/git:1": {}
+	},
+	"remoteEnv": {
+		"PATH": "${containerWorkspaceFolder}/core/ten_gn:${containerEnv:PATH}"
+	},
+	"postCreateCommand": "git submodule update --init"
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+	// Configure tool-specific properties.
+	// "customizations": {},
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}
diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml
@@ -0,0 +1,37 @@
+name: Build - docker
+
+on:
+  push:
+    branches:
+      - "**"
+    paths:
+      - "tools/docker_for_building/**"
+      - ".github/workflows/build_docker.yml"
+
+env:
+  IMAGE_NAME_PREFIX: ten_building_
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        include:
+          - image_name: "ubuntu2204"
+            workdir: "tools/docker_for_building/ubuntu/22.04"
+          - image_name: "ubuntu1804"
+            workdir: "tools/docker_for_building/ubuntu/18.04"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build & Publish to Github Container Registry
+        uses: elgohr/Publish-Docker-Github-Action@v5
+        with:
+          name: ${{ github.repository_owner }}/${{ env.IMAGE_NAME_PREFIX }}${{ matrix.image_name }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          registry: ghcr.io
+          workdir: ${{ matrix.workdir }}
+          snapshot: true
+          tags: "${{ github.ref == 'refs/heads/main' && 'latest,' || '' }}"
diff --git a/.github/workflows/build_linux_ubuntu1804.yml b/.github/workflows/build_linux_ubuntu1804.yml
@@ -0,0 +1,73 @@
+name: Build - linux ubuntu1804
+
+on:
+  push:
+    branches:
+      - "**"
+    tags:
+      - "**"
+    paths-ignore:
+      - "tools/**"
+      - "docs/**"
+      - ".vscode/**"
+      - ".devcontainer/**"
+      - ".github/configs/**"
+      - "**.md"
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+    strategy:
+      matrix:
+        build_type: [release]
+    container:
+      image: ghcr.io/${{ github.repository_owner }}/ten_building_ubuntu1804
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: "0"
+          submodules: "true"
+
+      - name: Build
+        run: |
+          export PATH=$(pwd)/core/ten_gn/:$PATH
+          echo $PATH
+          go env -w GOFLAGS="-buildvcs=false"
+          rustup default nightly
+          tgn gen linux x64 ${{ matrix.build_type }} -- is_clang=false log_level=1 enable_serialized_actions=true ten_enable_test=false ten_enable_package_manager=false
+          tgn build linux x64 ${{ matrix.build_type }}
+          tree -I 'gen|obj' out
+
+      - name: Upload ten_packages
+        uses: actions/upload-artifact@v3
+        with:
+          name: ten_packages-linux-ubuntu1804-x64-gcc-${{ matrix.build_type }}
+          path: |
+            out/linux/x64/ten_packages/system
+            out/linux/x64/ten_packages/extension_group
+            out/linux/x64/ten_packages/extension/default_extension_cpp
+            out/linux/x64/ten_packages/extension/default_extension_go
+            out/linux/x64/ten_packages/extension/default_extension_python
+            out/linux/x64/ten_packages/extension/py_init_extension_cpp
+
+      - name: Package assets
+        if: startsWith(github.ref, 'refs/tags/')
+        run: |
+          cd out/linux/x64
+          zip -vr ten_packages-linux-ubuntu1804-x64-gcc-${{ matrix.build_type }}.zip \
+            ten_packages/system \
+            ten_packages/extension_group \
+            ten_packages/extension/default_extension_cpp \
+            ten_packages/extension/default_extension_go \
+            ten_packages/extension/default_extension_python \
+            ten_packages/extension/py_init_extension_cpp
+
+      - name: Release assets
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          files: |
+            out/linux/x64/ten_packages-linux-ubuntu1804-x64-gcc-${{ matrix.build_type }}.zip
diff --git a/.github/workflows/build_linux_ubuntu2204.yml b/.github/workflows/build_linux_ubuntu2204.yml
@@ -0,0 +1,81 @@
+name: Build - linux ubuntu2204
+
+on:
+  push:
+    branches:
+      - "**"
+    tags:
+      - "**"
+    paths-ignore:
+      - "tools/**"
+      - "docs/**"
+      - ".vscode/**"
+      - ".devcontainer/**"
+      - ".github/configs/**"
+      - "**.md"
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        compiler: [gcc, clang]
+        build_type: [debug, release]
+    container:
+      image: ghcr.io/${{ github.repository_owner }}/ten_building_ubuntu2204
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: "0"
+          submodules: "true"
+
+      - name: Build
+        run: |
+          export PATH=$(pwd)/core/ten_gn:$PATH
+          echo $PATH
+          go env -w GOFLAGS="-buildvcs=false"
+          go1.20.12 download
+          rustup default nightly
+          tgn gen linux x64 ${{ matrix.build_type }} -- is_clang=${{ matrix.compiler == 'gcc' && 'false' || 'true' }} log_level=1 enable_serialized_actions=true ten_enable_test=false
+          tgn build linux x64 ${{ matrix.build_type }}
+          tree -I 'gen|obj' out
+
+      - name: Upload tman
+        uses: actions/upload-artifact@v4
+        with:
+          name: tman-linux-x64-${{ matrix.compiler }}-${{ matrix.build_type }}
+          path: out/linux/x64/ten_manager/bin/tman
+
+      - name: Upload ten_packages
+        uses: actions/upload-artifact@v4
+        with:
+          name: ten_packages-linux-x64-${{ matrix.compiler }}-${{ matrix.build_type }}
+          path: |
+            out/linux/x64/ten_packages/system
+            out/linux/x64/ten_packages/extension_group
+            out/linux/x64/ten_packages/extension/default_extension_cpp
+            out/linux/x64/ten_packages/extension/default_extension_go
+            out/linux/x64/ten_packages/extension/default_extension_python
+            out/linux/x64/ten_packages/extension/py_init_extension_cpp
+
+      - name: Package assets
+        if: startsWith(github.ref, 'refs/tags/')
+        run: |
+          cd out/linux/x64
+          zip -vr tman-linux-x64-${{ matrix.compiler }}-${{ matrix.build_type }}.zip ten_manager/bin/tman
+          zip -vr ten_packages-linux-x64-${{ matrix.compiler }}-${{ matrix.build_type }}.zip \
+            ten_packages/system \
+            ten_packages/extension_group \
+            ten_packages/extension/default_extension_cpp \
+            ten_packages/extension/default_extension_go \
+            ten_packages/extension/default_extension_python \
+            ten_packages/extension/py_init_extension_cpp
+
+      - name: Release assets
+        uses: softprops/action-gh-release@v2
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          files: |
+            out/linux/x64/tman-linux-x64-${{ matrix.compiler }}-${{ matrix.build_type }}.zip
+            out/linux/x64/ten_packages-linux-x64-${{ matrix.compiler }}-${{ matrix.build_type }}.zip
diff --git a/tools/docker_for_building/ubuntu/18.04/Dockerfile b/tools/docker_for_building/ubuntu/18.04/Dockerfile
@@ -0,0 +1,121 @@
+FROM ubuntu:18.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get clean && \
+  apt-get update && \
+  apt-get install -y --no-install-recommends apt-utils software-properties-common && \
+  add-apt-repository ppa:git-core/ppa && \
+  apt-get update && \
+  apt-get install -y --no-install-recommends curl \
+  lsb-release \
+  iwyu \
+  libssl-dev \
+  p7zip-full \
+  p7zip-rar \
+  git \
+  tree \
+  zip \
+  unzip \
+  libasan5 \
+  autoconf \
+  libtool \
+  uuid-dev \
+  libmsgpack-dev \
+  libmysqlclient-dev \
+  libmysqlcppconn-dev \
+  libcrypto++-dev \
+  libexpat1-dev \
+  libxdamage1 \
+  make \
+  vim \
+  libxcomposite-dev \
+  libdrm-dev \
+  libavformat-dev \
+  libavfilter-dev \
+  libx264-dev \
+  build-essential \
+  zlib1g-dev \
+  libncurses5-dev \
+  libgdbm-dev \
+  libnss3-dev \
+  libssl-dev \
+  libreadline-dev \
+  libffi-dev \
+  wget \
+  gpg-agent
+
+# =======================================
+# Installing cmake
+#
+# The default version of cmake of Ubuntu 18.04 is 3.10, but TEN needs cmake version greater than 3.14, so we need to install it from cmake official site.
+
+RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null \
+  && apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+  && apt-get update \
+  && apt-get install -y --no-install-recommends cmake
+
+# =======================================
+# Installing gcc13
+#
+# NOTE: Install gcc after cmake, while cmake will auto-install gcc7.
+# Install gcc13 as default.
+
+RUN apt-get purge -y gcc gcc-7 g++-7 && \
+  apt-add-repository -y ppa:ubuntu-toolchain-r/test && \
+  apt-get update && \
+  apt-get install -y --no-install-recommends gcc-13 g++-13 && \
+  apt-get install -y gcc-9-multilib g++-13-multilib && \
+  update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \
+  update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100
+
+# =======================================
+# Install golang
+
+# /root/go is GOPATH.
+
+ENV PATH="$PATH:/usr/local/go/bin:/root/go/bin"
+
+# TEN go binding needs to be compatible with GO 1.20, so we need to install GO
+# 1.20 to check the compatibility.
+RUN curl -OL https://go.dev/dl/go1.22.3.linux-amd64.tar.gz && \
+  rm -rf /usr/local/go && tar -C /usr/local -xvf go1.22.3.linux-amd64.tar.gz && rm go1.22.3.linux-amd64.tar.gz && \
+  go install golang.org/dl/go1.20.12@latest && go1.20.12 download
+
+# =======================================
+# Install cargo for rust.
+
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \
+  /root/.cargo/bin/rustup install nightly && \
+  /root/.cargo/bin/rustup default nightly && \
+  /root/.cargo/bin/cargo install --force cbindgen
+
+ENV PATH="/root/.cargo/bin:$PATH"
+
+# =======================================
+# Installing Python3.10 from source code.
+
+ENV PYTHON_VERSION=3.10.14
+
+RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz && \
+  tar -zxvf Python-${PYTHON_VERSION}.tgz && \
+  cd Python-${PYTHON_VERSION} && \
+  ./configure --with-lto --with-computed-gotos --with-system-ffi --prefix=/usr --disable-test-modules --enable-shared && \
+  make -j && \
+  make altinstall && \
+  rm -rf Python-{PYTHON_VERSION}*
+
+RUN ln -sf /usr/bin/python3.10-config /usr/bin/python3-config && \
+  ln -sf /usr/bin/python3.10 /usr/bin/python3 && \
+  ln -sf /usr/bin/python3.10 /usr/bin/python && \
+  ln -sf /usr/bin/pip3.10 /usr/bin/pip3 && \
+  cp /usr/lib/python3/dist-packages/lsb_release.py /usr/bin && \
+  pip3 install --upgrade setuptools && \
+  pip3 install --upgrade pip && \
+  pip3 install requests oss2 && \
+  pip3 install --use-pep517 python-dotenv jinja2
+
+# =======================================
+# Cleanup
+
+RUN apt-get clean && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/*