From 76b1c0d0739ff31138e952012a3289a2bfe3e06c Mon Sep 17 00:00:00 2001 From: prasanthV <40450906+PrasanthV454@users.noreply.github.com> Date: Thu, 19 Jan 2023 17:26:26 +0530 Subject: [PATCH] Complete migration to SemaphoreCI (#926) Co-authored-by: Emanuele Sabellico --- .semaphore/semaphore.yml | 139 +++++++++++++++-------------- .semaphore/semaphore_commands.ps1 | 1 + .semaphore/semaphore_commands.sh | 7 ++ .travis.yml | 81 ----------------- mk/mingw-w64/msys2-dependencies.sh | 16 ++++ mk/mingw-w64/semaphore_commands.sh | 6 ++ mk/mingw-w64/setup-msys2.ps1 | 31 +++++++ mk/setup-go.ps1 | 7 ++ mk/setup-msys2-mingw64.sh | 16 ---- 9 files changed, 139 insertions(+), 165 deletions(-) create mode 100644 .semaphore/semaphore_commands.ps1 create mode 100755 .semaphore/semaphore_commands.sh delete mode 100644 .travis.yml create mode 100644 mk/mingw-w64/msys2-dependencies.sh create mode 100644 mk/mingw-w64/semaphore_commands.sh create mode 100644 mk/mingw-w64/setup-msys2.ps1 create mode 100644 mk/setup-go.ps1 delete mode 100644 mk/setup-msys2-mingw64.sh diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 7890252ed..96877e094 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -11,7 +11,7 @@ global_job_config: commands: - checkout blocks: - - name: "Go 1.16 OSX bundled librdkafka" + - name: "Go 1.17 OSX bundled librdkafka" dependencies: [ ] task: agent: @@ -19,31 +19,25 @@ blocks: type: s1-prod-macos prologue: commands: - - export GOLINT="$GOPATH/bin/golint" + - sem-version go 1.18 + - export PATH="$PATH:$GOPATH/bin" - export PKG_CONFIG_PATH="/usr/local/opt/openssl/lib/pkgconfig:$HOME/confluent-kafka-go/tmp-build/lib/pkgconfig" - export LD_LIBRARY_PATH="$HOME/confluent-kafka-go/tmp-build/lib" - export DYLD_LIBRARY_PATH="$HOME/confluent-kafka-go/tmp-build/lib" - - sem-version go 1.16 - rm -rf tmp-build - # - go install golang.org/x/lint/golint@latest && touch .do_lint + - go install golang.org/x/lint/golint@latest && touch .do_lint jobs: - name: "Static Build" - commands: - - for dir in kafka examples ; do (cd $dir && go get ./...) ; done - - for dir in kafka examples ; do (cd $dir && go install ./...) ; done - - if [[ -f .do_lint ]]; then $GOLINT -set_exit_status ./examples/... \ - ./kafka/... ./kafkatest/... ./soaktest/... ./schemaregistry/...; fi + env_vars: + - name: EXPECT_LINK_INFO + value: static + commands_file: semaphore_commands.sh - name: "Dynamic Build" - commands: - - echo - - echo $PKG_CONFIG_PATH - - bash mk/bootstrap-librdkafka.sh ${LIBRDKAFKA_VERSION} tmp-build - - for dir in kafka examples ; do (cd $dir && go get -tags dynamic ./...) ; done - - for dir in kafka examples ; do (cd $dir && go install -tags dynamic ./...); done - - if [[ -f .do_lint ]]; then $GOLINT -set_exit_status ./examples/... \ - ./kafka/... ./kafkatest/... ./soaktest/... ./schemaregistry/...; fi - - for dir in kafka schemaregistry ; do (cd $dir && go test -timeout 180s -v -tags dynamic ./...) ; done - - name: "Go 1.16 linux bundled librdkafka" + env_vars: + - name: EXPECT_LINK_INFO + value: dynamic + commands_file: semaphore_commands.sh + - name: "Go 1.17 linux bundled librdkafka" dependencies: [ ] task: agent: @@ -51,34 +45,29 @@ blocks: type: s1-prod-ubuntu20-04-amd64-2 prologue: commands: - - export GOLINT="$GOPATH/bin/golint" + - sem-version go 1.17 + - export PATH="$PATH:$GOPATH/bin" - export PKG_CONFIG_PATH="$HOME/confluent-kafka-go/tmp-build/lib/pkgconfig" - export LD_LIBRARY_PATH="$HOME/confluent-kafka-go/tmp-build/lib" - export DYLD_LIBRARY_PATH="$HOME/confluent-kafka-go/tmp-build/lib" - - sem-version go 1.16 - wget -qO - https://packages.confluent.io/deb/5.4/archive.key | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://packages.confluent.io/deb/5.4 stable main" - sudo apt-get update -q - sudo apt-get install confluent-librdkafka-plugins -y - rm -rf tmp-build - #- go install golang.org/x/lint/golint@latest && touch .do_lint + - go install -v golang.org/x/lint/golint@latest && touch .do_lint jobs: - name: "Static Build" - commands: - - for dir in kafka examples ; do (cd $dir && go get ./...) ; done - - for dir in kafka examples ; do (cd $dir && go install ./...) ; done - - if [[ -f .do_lint ]]; then $GOLINT -set_exit_status ./examples/... \ - ./kafka/... ./kafkatest/... ./soaktest/... ./schemaregistry/...; fi + env_vars: + - name: EXPECT_LINK_INFO + value: static + commands_file: semaphore_commands.sh - name: "Dynamic Build" - commands: - - echo - - bash mk/bootstrap-librdkafka.sh ${LIBRDKAFKA_VERSION} tmp-build - - for dir in kafka examples ; do (cd $dir && go get -tags dynamic ./...) ; done - - for dir in kafka examples ; do (cd $dir && go install -tags dynamic ./...); done - - if [[ -f .do_lint ]]; then $GOLINT -set_exit_status ./examples/... \ - ./kafka/... ./kafkatest/... ./soaktest/... ./schemaregistry/...; fi - - for dir in kafka schemaregistry ; do (cd $dir && go test -timeout 180s -v -tags dynamic ./...); done - - name: "Go 1.16 linux arm64 bundled librdkafka" + env_vars: + - name: EXPECT_LINK_INFO + value: dynamic + commands_file: semaphore_commands.sh + - name: "Go 1.17 linux arm64 bundled librdkafka" dependencies: [ ] task: agent: @@ -86,42 +75,56 @@ blocks: type: s1-prod-ubuntu20-04-arm64-1 prologue: commands: - - sem-version go 1.16 + - sem-version go 1.17 + - export PATH="$PATH:$GOPATH/bin" + - export PKG_CONFIG_PATH="$HOME/confluent-kafka-go/tmp-build/lib/pkgconfig" + - export LD_LIBRARY_PATH="$HOME/confluent-kafka-go/tmp-build/lib" + - export DYLD_LIBRARY_PATH="$HOME/confluent-kafka-go/tmp-build/lib" - wget -qO - https://packages.confluent.io/deb/5.4/archive.key | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://packages.confluent.io/deb/5.4 stable main" - sudo apt-get update -q #- sudo apt-get install confluent-librdkafka-plugins -y - rm -rf tmp-build - #- go install golang.org/x/lint/golint@latest && touch .do_lint + - go install golang.org/x/lint/golint@latest && touch .do_lint + jobs: + - name: "Static Build" + env_vars: + - name: EXPECT_LINK_INFO + value: static + commands_file: semaphore_commands.sh + - name: "Dynamic Build" + env_vars: + - name: EXPECT_LINK_INFO + value: dynamic + commands_file: semaphore_commands.sh + - name: "Go 1.17 Windows bundled librdkafka" + dependencies: [ ] + task: + agent: + machine: + type: s1-prod-windows + prologue: + commands: + # Install Go + - cache restore win-go-1.17 + - "& .\\mk\\setup-go.ps1" + - cache delete win-go-1.17 + - cache store win-go-1.17 ($env:USERPROFILE + '\go') + - cache restore msys2-x64 + # Set up msys2 + - ".\\mk\\mingw-w64\\setup-msys2.ps1" + - $env:PATH = 'C:\msys64\usr\bin;' + $env:PATH + - bash -lc './mk/mingw-w64/msys2-dependencies.sh' + - cache delete msys2-x64 + - cache store msys2-x64 c:/msys64 jobs: - name: "Static Build" - commands: - - for dir in kafka examples ; do (cd $dir && go get ./...) ; done - - for dir in kafka examples ; do (cd $dir && go install ./...) ; done - - if [[ -f .do_lint ]]; then $GOLINT -set_exit_status ./examples/... \ - ./kafka/... ./kafkatest/... ./soaktest/... ./schemaregistry/...; fi - - for dir in kafka schemaregistry ; do (cd $dir && go test -timeout 180s -v ./...); done - # TODO: Disabled because uses PowerShell instead of sh - # - name: "Go 1.16 Windows bundled librdkafka" - # dependencies: [ ] - # task: - # agent: - # machine: - # type: s1-prod-windows - # prologue: - # commands: - # - curl https://dl.google.com/go/go1.16.windows-amd64.msi -o go.msi - # - msiexec /i go.msi /quiet - # - setx /M PATH "%PATH%;%USERPROFILE%\go\bin" - # - rm -rf tmp-build - # - go install golang.org/x/lint/golint@latest && touch .do_lint - # - ./mk/setup-msys2-mingw64.sh - # - export PATH=/C/tools/msys64/mingw64/bin:$PATH - # - export MAKE=mingw32-make - # jobs: - # - name: "Static Build" - # commands: - # - for dir in kafka examples ; do (cd $dir && go get ./...) ; done - # - for dir in kafka examples ; do (cd $dir && go install ./...) ; done - # - if [[ -f .do_lint ]]; then $GOLINT -set_exit_status ./examples/... \ - # ./kafka/... ./kafkatest/... ./soaktest/... ./schemaregistry/...; fi + env_vars: + - name: EXPECT_LINK_INFO + value: static + - name: CHERE_INVOKING + value: 'yes' + - name: MSYSTEM + value: UCRT64 + commands_file: semaphore_commands.ps1 + diff --git a/.semaphore/semaphore_commands.ps1 b/.semaphore/semaphore_commands.ps1 new file mode 100644 index 000000000..84e6c0ed5 --- /dev/null +++ b/.semaphore/semaphore_commands.ps1 @@ -0,0 +1 @@ +bash -lc './mk/mingw-w64/semaphore_commands.sh' diff --git a/.semaphore/semaphore_commands.sh b/.semaphore/semaphore_commands.sh new file mode 100755 index 000000000..f2ddbf3e8 --- /dev/null +++ b/.semaphore/semaphore_commands.sh @@ -0,0 +1,7 @@ +if [ "$EXPECT_LINK_INFO" = "dynamic" ]; then export GO_TAGS="-tags dynamic"; bash mk/bootstrap-librdkafka.sh ${LIBRDKAFKA_VERSION} tmp-build; fi +for dir in kafka examples ; do (cd $dir && go install $GO_TAGS ./...) ; done +if [[ -f .do_lint ]]; then golint -set_exit_status ./examples/... ./kafka/... ./kafkatest/... ./soaktest/... ./schemaregistry/...; fi +for dir in kafka schemaregistry ; do (cd $dir && go test -timeout 180s -v $GO_TAGS ./...) ; done +go-kafkacat --help +library-version +(library-version | grep "$EXPECT_LINK_INFO") || (echo "Incorrect linkage, expected $EXPECT_LINK_INFO" ; false) diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index fae391f1c..000000000 --- a/.travis.yml +++ /dev/null @@ -1,81 +0,0 @@ -language: go -osx_image: xcode9.2 -go_import_path: github.com/confluentinc/confluent-kafka-go -env: - global: - - PATH="$PATH:$GOPATH/bin" - -jobs: - include: -# - name: "Go 1.16 OSX bundled librdkafka" -# go: "1.16" -# os: osx -# env: -# - EXPECT_LINK_INFO="static" -# - name: "Go 1.16 Linux bundled librdkafka" -# go: "1.16" -# os: linux -# env: -# - EXPECT_LINK_INFO="static" -# - name: "Go 1.16 Linux arm64 bundled librdkafka" -# if: tag is present -# go: "1.16" -# os: linux -# arch: arm64 -# env: -# - EXPECT_LINK_INFO="static" -# - name: "Go 1.16 OSX dynamic librdkafka" -# if: tag is present -# go: "1.16" -# os: osx -# env: -# - EXPECT_LINK_INFO="dynamic" -# - BUILD_TYPE='-tags dynamic' -# - PKG_CONFIG_PATH="/usr/local/opt/openssl/lib/pkgconfig:$HOME/gopath/src/github.com/confluentinc/confluent-kafka-go/tmp-build/lib/pkgconfig" -# - LD_LIBRARY_PATH="$HOME/gopath/src/github.com/confluentinc/confluent-kafka-go/tmp-build/lib" -# - DYLD_LIBRARY_PATH="$HOME/gopath/src/github.com/confluentinc/confluent-kafka-go/tmp-build/lib" -# - LIBRDKAFKA_VERSION=master -# - name: "Go 1.16 Linux dynamic librdkafka" -# if: tag is present -# go: "1.16" -# os: linux -# env: -# - EXPECT_LINK_INFO="dynamic" -# - BUILD_TYPE='-tags dynamic' -# - PKG_CONFIG_PATH="$HOME/gopath/src/github.com/confluentinc/confluent-kafka-go/tmp-build/lib/pkgconfig" -# - LD_LIBRARY_PATH="$HOME/gopath/src/github.com/confluentinc/confluent-kafka-go/tmp-build/lib" -# - DYLD_LIBRARY_PATH="$HOME/gopath/src/github.com/confluentinc/confluent-kafka-go/tmp-build/lib" -# - LIBRDKAFKA_VERSION=master - - name: "Go 1.16 Windows bundled librdkafka" - go: "1.16" - os: windows - env: - - EXPECT_LINK_INFO="static" - -before_install: - - if [[ $TRAVIS_OS_NAME == linux ]]; then wget -qO - https://packages.confluent.io/deb/5.4/archive.key | sudo apt-key add - ; fi - - if [[ $TRAVIS_OS_NAME == linux ]]; then sudo add-apt-repository "deb [arch=amd64] https://packages.confluent.io/deb/5.4 stable main" -y ; fi - - if [[ $TRAVIS_OS_NAME == linux ]]; then sudo apt-get update -q ; fi - - if [[ $TRAVIS_OS_NAME == linux ]]; then sudo apt-get install confluent-librdkafka-plugins -y ; fi - - rm -rf tmp-build - - if [[ -n $BUILD_TYPE ]]; then bash mk/bootstrap-librdkafka.sh ${LIBRDKAFKA_VERSION} tmp-build ; fi - - go get -u golang.org/x/lint/golint && touch .do_lint - - |- - case $TRAVIS_OS_NAME in - windows) - ./mk/setup-msys2-mingw64.sh - export PATH=/C/tools/msys64/mingw64/bin:$PATH - export MAKE=mingw32-make # so that Autotools can find it - esac - -install: - - for dir in kafka examples ; do (cd $dir && go get ${BUILD_TYPE} ./...) ; done - - for dir in kafka examples ; do (cd $dir && go install ${BUILD_TYPE} ./...) ; done - -script: - # should be replaced with golangci-lint - - if [[ -f .do_lint ]]; then golint -set_exit_status ./examples/... ./kafka/... ./kafkatest/... ./soaktest/... ; fi - - for dir in kafka ; do (cd $dir && go test -timeout 180s -v ${BUILD_TYPE} ./...) ; done - - go-kafkacat --help - - library-version - - (library-version | grep "$EXPECT_LINK_INFO") || (echo "Incorrect linkage, expected $EXPECT_LINK_INFO" ; false) diff --git a/mk/mingw-w64/msys2-dependencies.sh b/mk/mingw-w64/msys2-dependencies.sh new file mode 100644 index 000000000..8777c99d0 --- /dev/null +++ b/mk/mingw-w64/msys2-dependencies.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +export msys2='cmd //C RefreshEnv.cmd ' +export msys2+='& set MSYS=winsymlinks:nativestrict ' +export msys2+='& C:\\msys64\\msys2_shell.cmd -defterm -no-start' +export mingw64="$msys2 -mingw64 -full-path -here -c "\"\$@"\" --" +export msys2+=" -msys2 -c "\"\$@"\" --" + +# Have to update pacman first or choco upgrade will failure due to migration +# to zstd instead of xz compression +$msys2 pacman -Sy --noconfirm pacman + +## Install more MSYS2 packages from https://packages.msys2.org/base here +$msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-gcc diff --git a/mk/mingw-w64/semaphore_commands.sh b/mk/mingw-w64/semaphore_commands.sh new file mode 100644 index 000000000..571955ce3 --- /dev/null +++ b/mk/mingw-w64/semaphore_commands.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e +export HOME=/c/Users/$USER +export PATH=$HOME/go/bin:/C/msys64/mingw64/bin:$PATH +export MAKE=mingw32-make # so that Autotools can find it +source .semaphore/semaphore_commands.sh diff --git a/mk/mingw-w64/setup-msys2.ps1 b/mk/mingw-w64/setup-msys2.ps1 new file mode 100644 index 000000000..cf7285041 --- /dev/null +++ b/mk/mingw-w64/setup-msys2.ps1 @@ -0,0 +1,31 @@ +# Install (if necessary) and set up msys2. + + +$url="https://github.com/msys2/msys2-installer/releases/download/2022-10-28/msys2-base-x86_64-20221028.sfx.exe" +$sha256="e365b79b4b30b6f4baf34bd93f3d2a41c0a92801c7a96d79cddbfca1090a0554" + + +if (!(Test-Path -Path "c:\msys64\usr\bin\bash.exe")) { + echo "Downloading and installing msys2 to c:\msys64" + + (New-Object System.Net.WebClient).DownloadFile($url, './msys2-installer.exe') + + # Verify checksum + (Get-FileHash -Algorithm "SHA256" .\msys2-installer.exe).hash -eq $sha256 + + # Install msys2 + .\msys2-installer.exe -y -oc:\ + + Remove-Item msys2-installer.exe + + # Set up msys2 the first time + echo "Setting up msys" + c:\msys64\usr\bin\bash -lc ' ' + +} else { + echo "Using previously installed msys2" +} + +# Update packages +echo "Updating msys2 packages" +c:\msys64\usr\bin\bash -lc "pacman --noconfirm -Syuu --overwrite '*'" diff --git a/mk/setup-go.ps1 b/mk/setup-go.ps1 new file mode 100644 index 000000000..7dd91b3ef --- /dev/null +++ b/mk/setup-go.ps1 @@ -0,0 +1,7 @@ +if (!(Test-Path -Path ($env:USERPROFILE + "\go"))) { + echo "Downloading and installing Go 1.17" + curl https://go.dev/dl/go1.17.13.windows-amd64.zip -o ($env:USERPROFILE + '\go.zip') + tar -xf ($env:USERPROFILE + '\go.zip') -C $env:USERPROFILE +} else { + echo "Using previously installed Go 1.17" +} diff --git a/mk/setup-msys2-mingw64.sh b/mk/setup-msys2-mingw64.sh deleted file mode 100644 index a671fc6a3..000000000 --- a/mk/setup-msys2-mingw64.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -# From https://docs.travis-ci.com/user/reference/windows/#how-do-i-use-msys2 - -set -e - -[[ ! -f C:/tools/msys64/msys2_shell.cmd ]] && rm -rf C:/tools/msys64 -choco uninstall -y mingw -choco upgrade --no-progress -y msys2 -export msys2='cmd //C RefreshEnv.cmd ' -export msys2+='& set MSYS=winsymlinks:nativestrict ' -export msys2+='& C:\\tools\\msys64\\msys2_shell.cmd -defterm -no-start' -export mingw64="$msys2 -mingw64 -full-path -here -c "\"\$@"\" --" -export msys2+=" -msys2 -c "\"\$@"\" --" -$msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-toolchain -## Install more MSYS2 packages from https://packages.msys2.org/base here -taskkill //IM gpg-agent.exe //F || true # https://travis-ci.community/t/4967