Onboard jenkins prod docker images to github actions

Signed-off-by: Peter Zhu <[email protected]>
peterzhuamazon · Oct 31, 2023 · 9986b82 · 9986b82
1 parent 568e17a
commit 9986b82
Show file tree

Hide file tree

Showing 9 changed files with 167 additions and 430 deletions.
diff --git a/.github/workflows/add-untriaged.yml b/.github/workflows/add-untriaged.yml
diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml
diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -9,7 +9,13 @@ on:
       - "*"
 
 jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
   linux-build:
+    needs: Get-CI-Image-Tag
     strategy:
       matrix:
         java:
@@ -18,7 +24,16 @@ jobs:
     # Job name
     name: Build Asynchronous Search
     # This job runs on Linux.
+    outputs:
+      build-test-linux: ${{ steps.step-build-test-linux.outputs.build-test-linux }}
     runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
     steps:
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
       - name: Set Up JDK ${{ matrix.java }}
@@ -29,129 +44,157 @@ jobs:
       - name: Checkout Branch
         uses: actions/checkout@v2
       - name: Build with Gradle
-        run: ./gradlew build
-      - name: Pull and Run Docker for security tests
-        run: |
-          version=`./gradlew properties -q | grep "opensearch_version:" | awk '{print $2}'`
-          IFS='-' read -r -a version_array <<< "$version"
-          plugin_version="${version_array[0]}.0"; for entry in ${version_array[@]:1}; do plugin_version+="-$entry"; done
-          docker_version="${version_array[0]}-${version_array[1]}"
-
-          echo version $version
-          echo plugin_version $plugin_version
-          echo docker_version $docker_version
-
-          pwd=`pwd`
-          echo $pwd
-          cd ..
-          pwd1=`pwd`
-          echo $pwd1
-          list_of_all_files=`ls asynchronous-search/build/distributions/`
-          echo "listing distributions"
-          echo $list_of_all_files
-
-          if docker pull opensearchstaging/opensearch:$docker_version
-          then
-            echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile
-            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-asynchronous-search ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-asynchronous-search; fi" >> Dockerfile
-            echo "ADD asynchronous-search/build/distributions/opensearch-asynchronous-search-$plugin_version.zip /tmp/" >> Dockerfile
-            echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/opensearch-asynchronous-search-$plugin_version.zip" >> Dockerfile
-            docker build -t opensearch-asynchronous-search:test .
-            echo "imagePresent=true" >> $GITHUB_ENV
-          else
-            echo "imagePresent=false" >> $GITHUB_ENV
-          fi
-      - name: Run Docker Image
-        if: env.imagePresent == 'true'
-        run: |
-          cd ..
-          docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-asynchronous-search:test
-          sleep 90
-      - name: Run Asynchronous Search Test
-        if: env.imagePresent == 'true'
-        run: |
-          security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opensearch-security|wc -l`
-          if [ $security -gt 0 ]
-          then
-            echo "Security plugin is available"
-            ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin
-          else
-            echo "Security plugin is NOT available"
-            ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster"
-          fi
-      - name: Upload failed logs
-        uses: actions/upload-artifact@v2
-        if: failure()
-        with:
-          name: logs
-          path: build/testclusters/integTest-*/logs/*
-      - name: Create Artifact Path
+        id: step-build-test-linux
         run: |
-          mkdir -p asynchronous-search-artifacts
-          cp ./build/distributions/*.zip asynchronous-search-artifacts     
-      - name: Uploads coverage
-        uses: codecov/[email protected]
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./gradlew assemble"
+          plugin=`basename $(ls build/distributions/*.zip)`
+          echo plugin $plugin
+          mv -v build/distributions/$plugin ./
+          echo "build-test-linux=$plugin" >> $GITHUB_OUTPUT
+        #run: ./gradlew build
+#      - name: Uploads coverage
+#        uses: codecov/[email protected]
+#        with:
+#          token: ${{ secrets.CODECOV_TOKEN }}
+      - uses: actions/upload-artifact@v3
         with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-      # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v1
-        with:
-          name: asynchronous-search-plugin-ubuntu
-          path: asynchronous-search-artifacts
-  windows-build:
-    # Job name
-    name: Build Asynchronous Search
-    # This job runs on Windows.
-    runs-on: windows-latest
-    steps:
-      # This step uses the setup-java Github action: https://github.com/actions/setup-java
-      - name: Set Up JDK 11
-        uses: actions/setup-java@v1
-        with:
-          java-version: 11
-      # This step uses the checkout Github action: https://github.com/actions/checkout
-      - name: Checkout Branch
-        uses: actions/checkout@v2
-      - name: Build with Gradle
-        run: ./gradlew.bat build -x integTest -x jacocoTestReport
-        env:
-          _JAVA_OPTIONS: -Xmx4096M
-      - name: Create Artifact Path
-        run: |
-          mkdir -p asynchronous-search-artifacts
-          cp ./build/distributions/*.zip asynchronous-search-artifacts
-      # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v1
-        with:
-          name: asynchronous-search-plugin-windows
-          path: asynchronous-search-artifacts
-  mac-os-build:
-    # Job name
-    name: Build Asynchronous Search
-    # This job runs on Mac OS.
-    runs-on: macos-latest
-    steps:
-      # This step uses the setup-java Github action: https://github.com/actions/setup-java
-      - name: Set Up JDK 11
-        uses: actions/setup-java@v1
-        with:
-          java-version: 11
-      # This step uses the checkout Github action: https://github.com/actions/checkout
-      - name: Checkout Branch
-        uses: actions/checkout@v2
-      - name: Build with Gradle
-        run: ./gradlew build -x integTest -x jacocoTestReport
-        env:
-          _JAVA_OPTIONS: -Xmx4096M
-      - name: Create Artifact Path
-        run: |
-          mkdir -p asynchronous-search-artifacts
-          cp ./build/distributions/*.zip asynchronous-search-artifacts
-      # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v1
-        with:
-          name: asynchronous-search-plugin-mac
-          path: asynchronous-search-artifacts
+          name: async-plugin-linux-${{ matrix.java }}
+          path: ${{ steps.step-build-test-linux.outputs.build-test-linux }}
+          if-no-files-found: error
+
+#  linux-test-docker:
+#    needs: linux-build
+#    strategy:
+#      matrix:
+#        java:
+#          - 11
+#          - 17
+#    # Job name
+#    name: Build Asynchronous Search
+#    # This job runs on Linux.
+#    runs-on: ubuntu-latest
+#    steps:
+#      - name: Pull and Run Docker for security tests
+#        run: |
+#          version=`./gradlew properties -q | grep "opensearch_version:" | awk '{print $2}'`
+#          IFS='-' read -r -a version_array <<< "$version"
+#          plugin_version="${version_array[0]}.0"; for entry in ${version_array[@]:1}; do plugin_version+="-$entry"; done
+#          docker_version="${version_array[0]}-${version_array[1]}"
+#
+#          echo version $version
+#          echo plugin_version $plugin_version
+#          echo docker_version $docker_version
+#
+#          pwd=`pwd`
+#          echo $pwd
+#          cd ..
+#          pwd1=`pwd`
+#          echo $pwd1
+#          list_of_all_files=`ls asynchronous-search/build/distributions/`
+#          echo "listing distributions"
+#          echo $list_of_all_files
+#
+#          if docker pull opensearchstaging/opensearch:$docker_version
+#          then
+#            echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile
+#            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-asynchronous-search ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-asynchronous-search; fi" >> Dockerfile
+#            echo "ADD asynchronous-search/build/distributions/opensearch-asynchronous-search-$plugin_version.zip /tmp/" >> Dockerfile
+#            echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/opensearch-asynchronous-search-$plugin_version.zip" >> Dockerfile
+#            docker build -t opensearch-asynchronous-search:test .
+#            echo "imagePresent=true" >> $GITHUB_ENV
+#          else
+#            echo "imagePresent=false" >> $GITHUB_ENV
+#          fi
+#      - name: Run Docker Image
+#        if: env.imagePresent == 'true'
+#        run: |
+#          cd ..
+#          docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-asynchronous-search:test
+#          sleep 90
+#      - name: Run Asynchronous Search Test
+#        if: env.imagePresent == 'true'
+#        run: |
+#          security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opensearch-security|wc -l`
+#          if [ $security -gt 0 ]
+#          then
+#            echo "Security plugin is available"
+#            ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin
+#          else
+#            echo "Security plugin is NOT available"
+#            ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster"
+#          fi
+#      - name: Upload failed logs
+#        uses: actions/upload-artifact@v2
+#        if: failure()
+#        with:
+#          name: logs
+#          path: build/testclusters/integTest-*/logs/*
+#      - name: Create Artifact Path
+#        run: |
+#          mkdir -p asynchronous-search-artifacts
+#          cp ./build/distributions/*.zip asynchronous-search-artifacts     
+#      # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact
+#      - name: Upload Artifacts
+#        uses: actions/upload-artifact@v1
+#        with:
+#          name: asynchronous-search-plugin-ubuntu
+#          path: asynchronous-search-artifacts
+#
+#  windows-build:
+#    # Job name
+#    name: Build Asynchronous Search
+#    # This job runs on Windows.
+#    runs-on: windows-latest
+#    steps:
+#      # This step uses the setup-java Github action: https://github.com/actions/setup-java
+#      - name: Set Up JDK 11
+#        uses: actions/setup-java@v1
+#        with:
+#          java-version: 11
+#      # This step uses the checkout Github action: https://github.com/actions/checkout
+#      - name: Checkout Branch
+#        uses: actions/checkout@v2
+#      - name: Build with Gradle
+#        run: ./gradlew.bat build -x integTest -x jacocoTestReport
+#        env:
+#          _JAVA_OPTIONS: -Xmx4096M
+#      - name: Create Artifact Path
+#        run: |
+#          mkdir -p asynchronous-search-artifacts
+#          cp ./build/distributions/*.zip asynchronous-search-artifacts
+#      # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact
+#      - name: Upload Artifacts
+#        uses: actions/upload-artifact@v1
+#        with:
+#          name: asynchronous-search-plugin-windows
+#          path: asynchronous-search-artifacts
+#
+#  mac-os-build:
+#    # Job name
+#    name: Build Asynchronous Search
+#    # This job runs on Mac OS.
+#    runs-on: macos-latest
+#    steps:
+#      # This step uses the setup-java Github action: https://github.com/actions/setup-java
+#      - name: Set Up JDK 11
+#        uses: actions/setup-java@v1
+#        with:
+#          java-version: 11
+#      # This step uses the checkout Github action: https://github.com/actions/checkout
+#      - name: Checkout Branch
+#        uses: actions/checkout@v2
+#      - name: Build with Gradle
+#        run: ./gradlew build -x integTest -x jacocoTestReport
+#        env:
+#          _JAVA_OPTIONS: -Xmx4096M
+#      - name: Create Artifact Path
+#        run: |
+#          mkdir -p asynchronous-search-artifacts
+#          cp ./build/distributions/*.zip asynchronous-search-artifacts
+#      # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact
+#      - name: Upload Artifacts
+#        uses: actions/upload-artifact@v1
+#        with:
+#          name: asynchronous-search-plugin-mac
+#          path: asynchronous-search-artifacts