From b980c94b26cf69e435298bc69fbf5bb6804d4fb2 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 29 Feb 2024 14:21:40 -0500 Subject: [PATCH] Add initial support of ppc64le arch on ci-runner (#4336) Signed-off-by: Peter Zhu --- docker/ci/build-image-multi-arch.sh | 2 +- docker/ci/config/gh-setup.sh | 68 +++++++++++++++++++ docker/ci/config/jdk-setup.sh | 10 +++ docker/ci/config/yq-setup.sh | 7 +- ...ld.centos7.opensearch.x64.arm64.dockerfile | 10 +-- ...opensearch-dashboards.x64.arm64.dockerfile | 10 +-- ...ockylinux8.opensearch.x64.arm64.dockerfile | 10 +-- ...ubuntu2004.opensearch.x64.arm64.dockerfile | 12 +--- .../docker-builder.ubuntu2004.x64.dockerfile | 13 ++-- ...lease.centos7.clients.x64.arm64.dockerfile | 8 +-- ...opensearch-dashboards.x64.arm64.dockerfile | 28 ++++---- ...kylinux8.systemd-base.x64.arm64.dockerfile | 8 +-- ...untu2004.systemd-base.x64.arm64.dockerfile | 14 ++-- 13 files changed, 136 insertions(+), 64 deletions(-) create mode 100755 docker/ci/config/gh-setup.sh diff --git a/docker/ci/build-image-multi-arch.sh b/docker/ci/build-image-multi-arch.sh index 4da82fa463..efa84d6c3f 100755 --- a/docker/ci/build-image-multi-arch.sh +++ b/docker/ci/build-image-multi-arch.sh @@ -107,5 +107,5 @@ docker buildx ls | grep $BUILDER_NAME docker ps | grep $BUILDER_NAME # Build multi-arch images -docker buildx build --platform linux/amd64,linux/arm64 --build-arg VERSION=${TAG_NAME} --build-arg BUILD_DATE=`date -u +%Y-%m-%dT%H:%M:%SZ` --build-arg NOTES=${NOTES} -t "opensearchstaging/${REPO_NAME}:${TAG_NAME}" -f "${DOCKERFILE}" --push . +docker buildx build --platform linux/amd64,linux/arm64,linux/ppc64le --build-arg VERSION=${TAG_NAME} --build-arg BUILD_DATE=`date -u +%Y-%m-%dT%H:%M:%SZ` --build-arg NOTES=${NOTES} -t "opensearchstaging/${REPO_NAME}:${TAG_NAME}" -f "${DOCKERFILE}" --push . diff --git a/docker/ci/config/gh-setup.sh b/docker/ci/config/gh-setup.sh new file mode 100755 index 0000000000..74e81be954 --- /dev/null +++ b/docker/ci/config/gh-setup.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# This script is to install gh properly on nix* server + +set -ex + +PLATFORM_LIST=(linux) +for entry in "${PLATFORM_LIST[@]}"; do + if echo $OSTYPE | grep -o $entry; then + PLATFORM=$entry + break + fi +done +ARCH=`uname -m` +VERSION="2.42.0" + +# ppc64le specific +function gh_install_ppc64le { + set -e + GO_VERSION=`go version | cut -d ' ' -f3 | tr -d 'go'` + GO_REQUIRED_VERSION="1.21.0" + COMPARE_VERSION=`echo $GO_REQUIRED_VERSION $GO_VERSION | tr ' ' '\n' | sort -V | uniq | head -n 1` + if [ "$COMPARE_VERSION" != "$GCC_REQUIRED_VERSION" ]; then + VERSION=2.32.1 + echo "go version on this env is older than $GO_REQUIRED_VERSION, use gh $VERSION" + fi + + git clone --single-branch --branch v$VERSION https://github.com/cli/cli.git gh-cli + cd gh-cli + make install + cd ../ + rm -rf gh-cli + gh --version +} + +echo "$PLATFORM-$ARCH" + +case $PLATFORM-$ARCH in + linux-x86_64|linux-amd64) + GH_TYPE="gh_${VERSION}_linux_amd64.tar.gz" + ;; + linux-aarch64|linux-arm64) + GH_TYPE="gh_${VERSION}_linux_arm64.tar.gz" + ;; + linux-ppc64le) + gh_install_ppc64le + exit 0 + ;; + *) + echo "Unsupported combination: $PLATFORM-$ARCH" + exit 1 + ;; +esac + +curl -SL https://github.com/cli/cli/releases/download/v$VERSION/$GH_TYPE -o /tmp/$GH_TYPE +tar -xzf /tmp/$GH_TYPE --strip-component=1 -C /usr +rm -v /tmp/$GH_TYPE +chmod 755 /usr/bin/gh +gh --version + +echo "Installed gh $VERSION" diff --git a/docker/ci/config/jdk-setup.sh b/docker/ci/config/jdk-setup.sh index 569aa3741b..ae8754b661 100755 --- a/docker/ci/config/jdk-setup.sh +++ b/docker/ci/config/jdk-setup.sh @@ -38,6 +38,16 @@ case "${ARCH}" in JDKS+="43ad054f135a7894dc87ad5d10ad45d8e82846186515892acdbc17c2c5cd27e4@https://github.com/adoptium/temurin20-binaries/releases/download/jdk-20.0.1%2B9/OpenJDK20U-jdk_x64_linux_hotspot_20.0.1_9.tar.gz " JDKS+="1a6fa8abda4c5caed915cfbeeb176e7fbd12eb6b222f26e290ee45808b529aa1@https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.1%2B12/OpenJDK21U-jdk_x64_linux_hotspot_21.0.1_12.tar.gz " ;; + ppc64le) + # Use "@" format to collect all JDK platform specific distributions + JDKS+="9d9813d2840360ffdbc449c45e71124e8170c31a3b6cce9151fbb31352064406@https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u392-b08/OpenJDK8U-jdk_ppc64le_linux_hotspot_8u392b08.tar.gz " + JDKS+="262ff98d6d88a7c7cc522cb4ec4129491a0eb04f5b17dcca0da57cfcdcf3830d@https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.21%2B9/OpenJDK11U-jdk_ppc64le_linux_hotspot_11.0.21_9.tar.gz " + JDKS+="465a3b8e931896b8d95e452d479615c4bf543535c05b6ea246323ae114e67d7d@https://github.com/AdoptOpenJDK/openjdk14-binaries/releases/download/jdk-14.0.2%2B12/OpenJDK14U-jdk_ppc64le_linux_hotspot_14.0.2_12.tar.gz " + JDKS+="3ae4b254d5b720f94f986481e787fbd67f0667571140ba2e2ae5020ceddbc826@https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.9%2B9/OpenJDK17U-jdk_ppc64le_linux_hotspot_17.0.9_9.tar.gz " + JDKS+="173d1256dfb9d13d309b5390e6bdf72d143b512201b0868f9d349d5ed3d64072@https://github.com/adoptium/temurin19-binaries/releases/download/jdk-19.0.2%2B7/OpenJDK19U-jdk_ppc64le_linux_hotspot_19.0.2_7.tar.gz " + JDKS+="45dde71faf8cbb78fab3c976894259655c8d3de827347f23e0ebe5710921dded@https://github.com/adoptium/temurin20-binaries/releases/download/jdk-20%2B36/OpenJDK20U-jdk_ppc64le_linux_hotspot_20_36.tar.gz " + JDKS+="9574828ef3d735a25404ced82e09bf20e1614f7d6403956002de9cfbfcb8638f@https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.1%2B12/OpenJDK21U-jdk_ppc64le_linux_hotspot_21.0.1_12.tar.gz " + ;; *) echo "Unsupported arch: ${ARCH}" exit 1 diff --git a/docker/ci/config/yq-setup.sh b/docker/ci/config/yq-setup.sh index 7bd4130aaf..d5c7e4a69e 100755 --- a/docker/ci/config/yq-setup.sh +++ b/docker/ci/config/yq-setup.sh @@ -24,12 +24,15 @@ VERSION="v4.27.2" echo "$PLATFORM-$ARCH" case $PLATFORM-$ARCH in - linux-x86_64) + linux-x86_64|linux-amd64) YQ_TYPE="yq_linux_amd64" ;; - linux-aarch64) + linux-aarch64|linux-arm64) YQ_TYPE="yq_linux_arm64" ;; + linux-ppc64le) + YQ_TYPE="yq_linux_ppc64le" + ;; freebsd-x86_64) YQ_TYPE="yq_freebsd_amd64" ;; diff --git a/docker/ci/dockerfiles/current/build.centos7.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.centos7.opensearch.x64.arm64.dockerfile index 781aec6cf7..9af2096507 100644 --- a/docker/ci/dockerfiles/current/build.centos7.opensearch.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/build.centos7.opensearch.x64.arm64.dockerfile @@ -20,10 +20,10 @@ ARG CONTAINER_USER_HOME=/home/ci-runner USER 0 # Add normal dependencies -RUN yum clean all && yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo && \ +RUN yum clean all && \ yum install epel-release -y && \ yum update -y && \ - yum install -y which curl git gnupg2 tar net-tools procps-ng python3 python3-devel python3-pip zip unzip jq gh pigz + yum install -y which curl git gnupg2 tar net-tools procps-ng python3 python3-devel python3-pip zip unzip jq pigz # Create user group RUN groupadd -g 1000 $CONTAINER_USER && \ @@ -44,8 +44,8 @@ RUN yum install -y nss xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils RUN yum groupinstall -y "Development Tools" && yum clean all && rm -rf /var/cache/yum/* # Tools setup -COPY --chown=0:0 config/jdk-setup.sh config/yq-setup.sh /tmp/ -RUN /tmp/jdk-setup.sh && /tmp/yq-setup.sh +COPY --chown=0:0 config/jdk-setup.sh config/yq-setup.sh config/gh-setup.sh /tmp/ +RUN yum install -y go && /tmp/jdk-setup.sh && /tmp/yq-setup.sh && /tmp/gh-setup.sh # Install higher version of maven 3.8.x RUN export MAVEN_URL=`curl -s https://maven.apache.org/download.cgi | grep -Eo '["\047].*.bin.tar.gz["\047]' | tr -d '"' | uniq | head -n 1` && \ @@ -107,6 +107,6 @@ USER $CONTAINER_USER WORKDIR $CONTAINER_USER_HOME # Install fpm for opensearch dashboards core -RUN gem install fpm -v 1.14.2 +RUN gem install dotenv -v 2.8.1 && gem install fpm -v 1.14.2 ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH RUN fpm -v diff --git a/docker/ci/dockerfiles/current/build.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile index e996ef034a..86341d8103 100644 --- a/docker/ci/dockerfiles/current/build.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/build.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile @@ -18,15 +18,15 @@ ARG CONTAINER_USER_HOME=/home/ci-runner USER 0 # Add normal dependencies -RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo && \ +RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && \ dnf config-manager --set-enabled powertools && \ dnf install epel-release -y && dnf repolist && \ dnf update -y && \ - dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip jq gh + dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip jq # Tools setup -COPY --chown=0:0 config/yq-setup.sh /tmp -RUN /tmp/yq-setup.sh +COPY --chown=0:0 config/yq-setup.sh config/gh-setup.sh /tmp/ +RUN dnf install -y go && /tmp/yq-setup.sh && /tmp/gh-setup.sh # Create user group RUN groupadd -g 1000 $CONTAINER_USER && \ @@ -80,7 +80,7 @@ USER $CONTAINER_USER WORKDIR $CONTAINER_USER_HOME # Install fpm for opensearch dashboards core -RUN gem install fpm -v 1.14.2 +RUN gem install dotenv -v 2.8.1 && gem install fpm -v 1.14.2 ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH # Hard code node version and yarn version for now diff --git a/docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile index 70bec6eefd..6fb8c3c550 100644 --- a/docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile @@ -20,9 +20,9 @@ ARG CONTAINER_USER_HOME=/home/ci-runner USER 0 # Add normal dependencies -RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo && \ +RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && \ dnf update -y && \ - dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip jq gh + dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip jq # Create user group RUN groupadd -g 1000 $CONTAINER_USER && \ @@ -43,8 +43,8 @@ RUN dnf install -y nss xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils RUN dnf groupinstall -y "Development Tools" && dnf clean all && rm -rf /var/cache/dnf/* # Tools setup -COPY --chown=0:0 config/jdk-setup.sh config/yq-setup.sh /tmp/ -RUN /tmp/jdk-setup.sh && /tmp/yq-setup.sh +COPY --chown=0:0 config/jdk-setup.sh config/yq-setup.sh config/gh-setup.sh /tmp/ +RUN dnf install -y go && /tmp/jdk-setup.sh && /tmp/yq-setup.sh && /tmp/gh-setup.sh # Install higher version of maven 3.8.x RUN export MAVEN_URL=`curl -s https://maven.apache.org/download.cgi | grep -Eo '["\047].*.bin.tar.gz["\047]' | tr -d '"' | uniq | head -n 1` && \ @@ -93,6 +93,6 @@ USER $CONTAINER_USER WORKDIR $CONTAINER_USER_HOME # Install fpm for opensearch dashboards core -RUN gem install fpm -v 1.14.2 +RUN gem install dotenv -v 2.8.1 && gem install fpm -v 1.14.2 ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH RUN fpm -v diff --git a/docker/ci/dockerfiles/current/build.ubuntu2004.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.ubuntu2004.opensearch.x64.arm64.dockerfile index 4dee5647d6..6ac4232f75 100644 --- a/docker/ci/dockerfiles/current/build.ubuntu2004.opensearch.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/build.ubuntu2004.opensearch.x64.arm64.dockerfile @@ -15,7 +15,7 @@ ARG CONTAINER_USER=ci-runner ARG CONTAINER_USER_HOME=/home/ci-runner # Install python dependencies -RUN apt-get update -y && apt-get install -y software-properties-common +RUN apt-get update -y && apt-get install -y software-properties-common && add-apt-repository ppa:longsleep/golang-backports -y # Install python binaries RUN apt-get update -y && apt-get install python3 && \ @@ -43,8 +43,8 @@ RUN curl -o- https://www.aptly.info/pubkey.txt | apt-key add - && \ dpkg -r lintian # Tools setup -COPY --chown=0:0 config/jdk-setup.sh config/yq-setup.sh /tmp/ -RUN /tmp/jdk-setup.sh && /tmp/yq-setup.sh # Ubuntu has a bug where entrypoint=bash does not actually run .bashrc correctly +COPY --chown=0:0 config/jdk-setup.sh config/yq-setup.sh config/gh-setup.sh /tmp/ +RUN apt-get install -y golang-go && /tmp/jdk-setup.sh && /tmp/yq-setup.sh && /tmp/gh-setup.sh # Ubuntu has a bug where entrypoint=bash does not actually run .bashrc correctly # Create user group RUN groupadd -g 1000 $CONTAINER_USER && \ @@ -52,12 +52,6 @@ RUN groupadd -g 1000 $CONTAINER_USER && \ mkdir -p $CONTAINER_USER_HOME && \ chown -R 1000:1000 $CONTAINER_USER_HOME -# Install gh cli -RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg && \ - chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg && \ - echo "deb [arch=`dpkg --print-architecture` signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list && \ - apt-get update && apt-get install -y gh && apt-get clean - # Change User USER $CONTAINER_USER WORKDIR $CONTAINER_USER_HOME diff --git a/docker/ci/dockerfiles/current/docker-builder.ubuntu2004.x64.dockerfile b/docker/ci/dockerfiles/current/docker-builder.ubuntu2004.x64.dockerfile index 3da55bb289..18884c9d91 100644 --- a/docker/ci/dockerfiles/current/docker-builder.ubuntu2004.x64.dockerfile +++ b/docker/ci/dockerfiles/current/docker-builder.ubuntu2004.x64.dockerfile @@ -21,19 +21,13 @@ ARG CONTAINER_USER=ci-runner ARG CONTAINER_USER_HOME=/home/ci-runner # Import necessary repository for installing qemu 5.0 -RUN apt-get update -y && apt-get install -y software-properties-common && add-apt-repository ppa:jacob/virtualisation -y +RUN apt-get update -y && apt-get install -y software-properties-common && add-apt-repository ppa:jacob/virtualisation -y && add-apt-repository ppa:longsleep/golang-backports -y # Install necessary packages RUN apt-get update -y && apt-get upgrade -y && apt-get install -y binfmt-support qemu qemu-user qemu-user-static docker.io curl python3-pip && \ apt-get install -y debmake debhelper-compat && \ apt-get clean -y && pip3 install awscli==1.32.17 -# Install gh cli -RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg && \ - chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg && \ - echo "deb [arch=`dpkg --print-architecture` signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list && \ - apt-get update && apt-get install -y gh && apt-get clean - # Install trivy to scan the docker images RUN apt-get install -y apt-transport-https gnupg lsb-release && \ curl -o- https://aquasecurity.github.io/trivy-repo/deb/public.key | gpg --dearmor | tee /usr/share/keyrings/trivy.gpg > /dev/null && \ @@ -86,3 +80,8 @@ RUN curl -SL -o- https://apt.releases.hashicorp.com/gpg | gpg --dearmor > /usr/s apt-get install packer=1.8.7* && \ packer --version && \ apt-get clean + +# Tools setup +COPY --chown=0:0 config/yq-setup.sh config/gh-setup.sh /tmp/ +RUN apt-get install -y golang-go && /tmp/yq-setup.sh && /tmp/gh-setup.sh + diff --git a/docker/ci/dockerfiles/current/release.centos7.clients.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/release.centos7.clients.x64.arm64.dockerfile index a680179a55..c16441faf8 100644 --- a/docker/ci/dockerfiles/current/release.centos7.clients.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/release.centos7.clients.x64.arm64.dockerfile @@ -22,9 +22,9 @@ ARG CONTAINER_USER_HOME=/home/ci-runner USER 0 # Add normal dependencies -RUN yum clean all && yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo && \ +RUN yum clean all && \ yum update -y && \ - yum install -y which curl git gnupg2 tar net-tools procps-ng python3 python3-devel python3-pip zip unzip jq gh epel-release + yum install -y which curl git gnupg2 tar net-tools procps-ng python3 python3-devel python3-pip zip unzip jq epel-release # Add Python dependencies RUN yum install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils @@ -54,8 +54,8 @@ RUN if [[ `uname -m` = 'aarch64' ]]; then mkdir -p aarch64-builds && cd aarch64- ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib # Tools setup -COPY --chown=0:0 config/yq-setup.sh /tmp/ -RUN /tmp/yq-setup.sh +COPY --chown=0:0 config/yq-setup.sh config/gh-setup.sh /tmp/ +RUN yum install -y go && /tmp/yq-setup.sh && /tmp/gh-setup.sh # Install JDK RUN curl -SL https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.15%2B10/OpenJDK11U-jdk_x64_linux_hotspot_11.0.15_10.tar.gz -o /opt/jdk11.tar.gz && \ diff --git a/docker/ci/dockerfiles/current/test.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/test.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile index c1130df199..5c5bc2faf3 100644 --- a/docker/ci/dockerfiles/current/test.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/test.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile @@ -18,15 +18,15 @@ ARG CONTAINER_USER_HOME=/home/ci-runner USER 0 # Add normal dependencies -RUN yum clean all && \ - yum update -y && \ - yum install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip +RUN dnf clean all && \ + dnf update -y && \ + dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip # Add Dashboards dependencies (mainly for cypress) -RUN yum install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib +RUN dnf install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib # Add Yarn dependencies -RUN yum groupinstall -y "Development Tools" && yum install -y cmake && yum clean all && rm -rf /var/cache/yum/* +RUN dnf groupinstall -y "Development Tools" && dnf install -y cmake && dnf clean all && rm -rf /var/cache/dnf/* # Create user group RUN groupadd -g 1000 $CONTAINER_USER && \ @@ -86,9 +86,9 @@ ARG CONTAINER_USER_HOME=/home/ci-runner USER 0 # Add normal dependencies -RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo && \ +RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && \ dnf update -y && \ - dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip gh + dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip # Create user group RUN groupadd -g 1000 $CONTAINER_USER && \ @@ -109,16 +109,16 @@ ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH RUN source $NVM_DIR/nvm.sh && ls -al $CONTAINER_USER_HOME && echo $NODE_VERSION $NVM_DIR && nvm use $NODE_VERSION # Add Python dependencies -RUN yum install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils +RUN dnf install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils # Add Dashboards dependencies (mainly for cypress) -RUN yum install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib +RUN dnf install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib # Add Reports dependencies -RUN yum install -y nss xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc fontconfig freetype +RUN dnf install -y nss xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc fontconfig freetype # Add Yarn dependencies -RUN yum groupinstall -y "Development Tools" && yum clean all && rm -rf /var/cache/yum/* +RUN dnf groupinstall -y "Development Tools" && dnf clean all && rm -rf /var/cache/dnf/* # Setup Shared Memory RUN chmod -R 777 /dev/shm @@ -129,9 +129,13 @@ RUN update-alternatives --set python /usr/bin/python3.9 && \ pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.32.17 # Add other dependencies -RUN yum install -y epel-release && yum clean all && yum install -y jq && yum clean all && rm -rf /var/cache/yum/* && \ +RUN dnf install -y epel-release && dnf clean all && dnf install -y jq && dnf clean all && rm -rf /var/cache/dnf/* && \ pip3 install cmake==3.23.3 +# Tools setup +COPY --chown=0:0 config/yq-setup.sh config/gh-setup.sh /tmp/ +RUN dnf install -y go && /tmp/yq-setup.sh && /tmp/gh-setup.sh + # Change User USER $CONTAINER_USER WORKDIR $CONTAINER_USER_HOME diff --git a/docker/ci/dockerfiles/current/test.rockylinux8.systemd-base.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/test.rockylinux8.systemd-base.x64.arm64.dockerfile index 69450761e0..8160b8acf0 100644 --- a/docker/ci/dockerfiles/current/test.rockylinux8.systemd-base.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/test.rockylinux8.systemd-base.x64.arm64.dockerfile @@ -94,9 +94,9 @@ ARG CONTAINER_USER_HOME=/home/ci-runner USER 0 # Add normal dependencies -RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo && \ +RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && \ dnf update -y && \ - dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip jq gh + dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip jq # Create user group RUN dnf install -y sudo && \ @@ -138,8 +138,8 @@ RUN dnf install -y nss xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils RUN dnf groupinstall -y "Development Tools" && dnf clean all # Tools setup -COPY --chown=0:0 config/jdk-setup.sh config/yq-setup.sh /tmp/ -RUN /tmp/jdk-setup.sh && dnf remove -y "java-1.8.0*" && /tmp/yq-setup.sh +COPY --chown=0:0 config/jdk-setup.sh config/yq-setup.sh config/gh-setup.sh /tmp/ +RUN /tmp/jdk-setup.sh && dnf remove -y "java-1.8.0*" && dnf install -y go && /tmp/yq-setup.sh && /tmp/gh-setup.sh # Setup Shared Memory RUN chmod -R 777 /dev/shm diff --git a/docker/ci/dockerfiles/current/test.ubuntu2004.systemd-base.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/test.ubuntu2004.systemd-base.x64.arm64.dockerfile index 1ec567c164..aff4ab68ad 100644 --- a/docker/ci/dockerfiles/current/test.ubuntu2004.systemd-base.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/test.ubuntu2004.systemd-base.x64.arm64.dockerfile @@ -96,8 +96,8 @@ USER 0 ARG DEBIAN_FRONTEND=noninteractive -# Install python dependencies and chromium dependencies -RUN apt-get update -y && apt-get install -y software-properties-common && add-apt-repository ppa:saiarcot895/chromium-beta -y +# Install python dependencies +RUN apt-get update -y && apt-get install -y software-properties-common && add-apt-repository ppa:longsleep/golang-backports -y # Install python binaries RUN apt-get update -y && apt-get install python3 && \ @@ -144,14 +144,8 @@ ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH RUN source $NVM_DIR/nvm.sh && ls -al $CONTAINER_USER_HOME && echo $NODE_VERSION $NVM_DIR && nvm use $NODE_VERSION # Tools setup -COPY --chown=0:0 config/jdk-setup.sh config/yq-setup.sh /tmp/ -RUN /tmp/jdk-setup.sh && /tmp/yq-setup.sh - -# Install gh cli -RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg && \ - chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg && \ - echo "deb [arch=`dpkg --print-architecture` signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list && \ - apt-get update && apt-get install -y gh && apt-get clean +COPY --chown=0:0 config/jdk-setup.sh config/yq-setup.sh config/gh-setup.sh /tmp/ +RUN apt-get install -y golang-go && /tmp/jdk-setup.sh && /tmp/yq-setup.sh && /tmp/gh-setup.sh # Setup Shared Memory RUN chmod -R 777 /dev/shm