Complete migration to SemaphoreCI (#926)

Co-authored-by: Emanuele Sabellico <[email protected]>

.semaphore/semaphore.yml

@@ -11,117 +11,120 @@ global_job_config:
     commands:
       - checkout
 blocks:
-  - name: "Go 1.16 OSX bundled librdkafka"
+  - name: "Go 1.17 OSX bundled librdkafka"
     dependencies: [ ]
     task:
       agent:
         machine:
           type: s1-prod-macos
       prologue:
         commands:
-          - export GOLINT="$GOPATH/bin/golint"
+          - sem-version go 1.18
+          - export PATH="$PATH:$GOPATH/bin"
           - export PKG_CONFIG_PATH="/usr/local/opt/openssl/lib/pkgconfig:$HOME/confluent-kafka-go/tmp-build/lib/pkgconfig"
           - export LD_LIBRARY_PATH="$HOME/confluent-kafka-go/tmp-build/lib"
           - export DYLD_LIBRARY_PATH="$HOME/confluent-kafka-go/tmp-build/lib"
-          - sem-version go 1.16
           - rm -rf tmp-build
-          # - go install golang.org/x/lint/golint@latest && touch .do_lint
+          - go install golang.org/x/lint/golint@latest && touch .do_lint
       jobs:
         - name: "Static Build"
-          commands:
-            - for dir in kafka examples ; do (cd $dir && go get ./...) ; done
-            - for dir in kafka examples ; do (cd $dir && go install ./...) ; done
-            - if [[ -f .do_lint ]]; then $GOLINT -set_exit_status ./examples/... \
-              ./kafka/... ./kafkatest/... ./soaktest/... ./schemaregistry/...; fi
+          env_vars:
+            - name: EXPECT_LINK_INFO
+              value: static
+          commands_file: semaphore_commands.sh
         - name: "Dynamic Build"
-          commands:
-            - echo
-            - echo $PKG_CONFIG_PATH
-            - bash mk/bootstrap-librdkafka.sh ${LIBRDKAFKA_VERSION} tmp-build
-            - for dir in kafka examples ; do (cd $dir && go get -tags dynamic ./...) ; done
-            - for dir in kafka examples ; do (cd $dir && go install -tags dynamic ./...); done
-            - if [[ -f .do_lint ]]; then $GOLINT -set_exit_status ./examples/... \
-              ./kafka/... ./kafkatest/... ./soaktest/... ./schemaregistry/...; fi
-            - for dir in kafka schemaregistry ; do (cd $dir && go test -timeout 180s -v -tags dynamic ./...) ; done
-  - name: "Go 1.16 linux bundled librdkafka"
+          env_vars:
+            - name: EXPECT_LINK_INFO
+              value: dynamic
+          commands_file: semaphore_commands.sh
+  - name: "Go 1.17 linux bundled librdkafka"
     dependencies: [ ]
     task:
       agent:
         machine:
           type: s1-prod-ubuntu20-04-amd64-2
       prologue:
         commands:
-          - export GOLINT="$GOPATH/bin/golint"
+          - sem-version go 1.17
+          - export PATH="$PATH:$GOPATH/bin"
           - export PKG_CONFIG_PATH="$HOME/confluent-kafka-go/tmp-build/lib/pkgconfig"
           - export LD_LIBRARY_PATH="$HOME/confluent-kafka-go/tmp-build/lib"
           - export DYLD_LIBRARY_PATH="$HOME/confluent-kafka-go/tmp-build/lib"
-          - sem-version go 1.16
           - wget -qO - https://packages.confluent.io/deb/5.4/archive.key | sudo apt-key add -
           - sudo add-apt-repository "deb [arch=amd64] https://packages.confluent.io/deb/5.4 stable main"
           - sudo apt-get update -q
           - sudo apt-get install confluent-librdkafka-plugins -y
           - rm -rf tmp-build
-          #- go install golang.org/x/lint/golint@latest && touch .do_lint
+          - go install -v golang.org/x/lint/golint@latest && touch .do_lint
       jobs:
         - name: "Static Build"
-          commands:
-            - for dir in kafka examples ; do (cd $dir && go get ./...) ; done
-            - for dir in kafka examples ; do (cd $dir && go install ./...) ; done
-            - if [[ -f .do_lint ]]; then $GOLINT -set_exit_status ./examples/... \
-              ./kafka/... ./kafkatest/... ./soaktest/... ./schemaregistry/...; fi
+          env_vars:
+            - name: EXPECT_LINK_INFO
+              value: static
+          commands_file: semaphore_commands.sh
         - name: "Dynamic Build"
-          commands:
-            - echo
-            - bash mk/bootstrap-librdkafka.sh ${LIBRDKAFKA_VERSION} tmp-build
-            - for dir in kafka examples ; do (cd $dir && go get -tags dynamic ./...) ; done
-            - for dir in kafka examples ; do (cd $dir && go install -tags dynamic ./...); done
-            - if [[ -f .do_lint ]]; then $GOLINT -set_exit_status ./examples/... \
-              ./kafka/... ./kafkatest/... ./soaktest/... ./schemaregistry/...; fi
-            - for dir in kafka schemaregistry ; do (cd $dir && go test -timeout 180s -v -tags dynamic ./...); done
-  - name: "Go 1.16 linux arm64 bundled librdkafka"
+          env_vars:
+            - name: EXPECT_LINK_INFO
+              value: dynamic
+          commands_file: semaphore_commands.sh
+  - name: "Go 1.17 linux arm64 bundled librdkafka"
     dependencies: [ ]
     task:
       agent:
         machine:
           type: s1-prod-ubuntu20-04-arm64-1
       prologue:
         commands:
-          - sem-version go 1.16
+          - sem-version go 1.17
+          - export PATH="$PATH:$GOPATH/bin"
+          - export PKG_CONFIG_PATH="$HOME/confluent-kafka-go/tmp-build/lib/pkgconfig"
+          - export LD_LIBRARY_PATH="$HOME/confluent-kafka-go/tmp-build/lib"
+          - export DYLD_LIBRARY_PATH="$HOME/confluent-kafka-go/tmp-build/lib"
           - wget -qO - https://packages.confluent.io/deb/5.4/archive.key | sudo apt-key add -
           - sudo add-apt-repository "deb [arch=amd64] https://packages.confluent.io/deb/5.4 stable main"
           - sudo apt-get update -q
           #- sudo apt-get install confluent-librdkafka-plugins -y
           - rm -rf tmp-build
-          #- go install golang.org/x/lint/golint@latest && touch .do_lint
+          - go install golang.org/x/lint/golint@latest && touch .do_lint
+      jobs:
+        - name: "Static Build"
+          env_vars:
+            - name: EXPECT_LINK_INFO
+              value: static
+          commands_file: semaphore_commands.sh
+        - name: "Dynamic Build"
+          env_vars:
+            - name: EXPECT_LINK_INFO
+              value: dynamic
+          commands_file: semaphore_commands.sh
+  - name: "Go 1.17 Windows bundled librdkafka"
+    dependencies: [ ]
+    task:
+      agent:
+        machine:
+          type: s1-prod-windows
+      prologue:
+        commands:
+          # Install Go
+          - cache restore win-go-1.17
+          - "& .\\mk\\setup-go.ps1"
+          - cache delete win-go-1.17
+          - cache store win-go-1.17 ($env:USERPROFILE + '\go')
+          - cache restore msys2-x64
+          # Set up msys2
+          - ".\\mk\\mingw-w64\\setup-msys2.ps1"
+          - $env:PATH = 'C:\msys64\usr\bin;' + $env:PATH
+          - bash -lc './mk/mingw-w64/msys2-dependencies.sh'
+          - cache delete msys2-x64
+          - cache store msys2-x64 c:/msys64
       jobs:
         - name: "Static Build"
-          commands:
-            - for dir in kafka examples ; do (cd $dir && go get ./...) ; done
-            - for dir in kafka examples ; do (cd $dir && go install ./...) ; done
-            - if [[ -f .do_lint ]]; then $GOLINT -set_exit_status ./examples/... \
-              ./kafka/... ./kafkatest/... ./soaktest/... ./schemaregistry/...; fi
-            - for dir in kafka schemaregistry ; do (cd $dir && go test -timeout 180s -v ./...); done
-  # TODO: Disabled because uses PowerShell instead of sh
-  # - name: "Go 1.16 Windows bundled librdkafka"
-  #   dependencies: [ ]
-  #   task:
-  #     agent:
-  #       machine:
-  #         type: s1-prod-windows
-  #     prologue:
-  #       commands:
-  #         - curl https://dl.google.com/go/go1.16.windows-amd64.msi -o go.msi
-  #         - msiexec /i go.msi /quiet
-  #         - setx /M PATH "%PATH%;%USERPROFILE%\go\bin"
-  #         - rm -rf tmp-build
-  #         - go install golang.org/x/lint/golint@latest && touch .do_lint
-  #         - ./mk/setup-msys2-mingw64.sh
-  #         - export PATH=/C/tools/msys64/mingw64/bin:$PATH
-  #         - export MAKE=mingw32-make
-  #     jobs:
-  #       - name: "Static Build"
-  #         commands:
-  #           - for dir in kafka examples ; do (cd $dir && go get ./...) ; done
-  #           - for dir in kafka examples ; do (cd $dir && go install ./...) ; done
-  #           - if [[ -f .do_lint ]]; then $GOLINT -set_exit_status ./examples/... \
-  #             ./kafka/... ./kafkatest/... ./soaktest/... ./schemaregistry/...; fi
+          env_vars:
+            - name: EXPECT_LINK_INFO
+              value: static
+            - name: CHERE_INVOKING
+              value: 'yes'
+            - name: MSYSTEM
+              value: UCRT64
+          commands_file: semaphore_commands.ps1
+

.semaphore/semaphore_commands.ps1

@@ -0,0 +1 @@
+bash -lc './mk/mingw-w64/semaphore_commands.sh'

.semaphore/semaphore_commands.sh

@@ -0,0 +1,7 @@
+if [ "$EXPECT_LINK_INFO" = "dynamic" ]; then export GO_TAGS="-tags dynamic"; bash mk/bootstrap-librdkafka.sh ${LIBRDKAFKA_VERSION} tmp-build; fi
+for dir in kafka examples ; do (cd $dir && go install $GO_TAGS ./...) ; done
+if [[ -f .do_lint ]]; then golint -set_exit_status ./examples/... ./kafka/... ./kafkatest/... ./soaktest/... ./schemaregistry/...; fi
+for dir in kafka schemaregistry ; do (cd $dir && go test -timeout 180s -v $GO_TAGS ./...) ; done
+go-kafkacat --help
+library-version
+(library-version | grep "$EXPECT_LINK_INFO") || (echo "Incorrect linkage, expected $EXPECT_LINK_INFO" ; false)

mk/mingw-w64/msys2-dependencies.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e
+
+export msys2='cmd //C RefreshEnv.cmd '
+export msys2+='& set MSYS=winsymlinks:nativestrict '
+export msys2+='& C:\\msys64\\msys2_shell.cmd -defterm -no-start'
+export mingw64="$msys2 -mingw64 -full-path -here -c "\"\$@"\" --"
+export msys2+=" -msys2 -c "\"\$@"\" --"
+
+# Have to update pacman first or choco upgrade will failure due to migration
+# to zstd instead of xz compression
+$msys2 pacman -Sy --noconfirm pacman
+
+## Install more MSYS2 packages from https://packages.msys2.org/base here
+$msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-gcc

mk/mingw-w64/semaphore_commands.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+export HOME=/c/Users/$USER
+export PATH=$HOME/go/bin:/C/msys64/mingw64/bin:$PATH
+export MAKE=mingw32-make  # so that Autotools can find it
+source .semaphore/semaphore_commands.sh

mk/mingw-w64/setup-msys2.ps1

@@ -0,0 +1,31 @@
+# Install (if necessary) and set up msys2.
+
+
+$url="https://github.com/msys2/msys2-installer/releases/download/2022-10-28/msys2-base-x86_64-20221028.sfx.exe"
+$sha256="e365b79b4b30b6f4baf34bd93f3d2a41c0a92801c7a96d79cddbfca1090a0554"
+
+
+if (!(Test-Path -Path "c:\msys64\usr\bin\bash.exe")) {
+    echo "Downloading and installing msys2 to c:\msys64"
+
+    (New-Object System.Net.WebClient).DownloadFile($url, './msys2-installer.exe')
+
+    # Verify checksum
+    (Get-FileHash -Algorithm "SHA256" .\msys2-installer.exe).hash -eq $sha256
+
+    # Install msys2
+    .\msys2-installer.exe -y -oc:\
+
+    Remove-Item msys2-installer.exe
+
+    # Set up msys2 the first time
+    echo "Setting up msys"
+    c:\msys64\usr\bin\bash -lc ' '
+
+} else {
+    echo "Using previously installed msys2"
+}
+
+# Update packages
+echo "Updating msys2 packages"
+c:\msys64\usr\bin\bash -lc "pacman --noconfirm -Syuu --overwrite '*'"

mk/setup-go.ps1

@@ -0,0 +1,7 @@
+if (!(Test-Path -Path ($env:USERPROFILE + "\go"))) {
+    echo "Downloading and installing Go 1.17"
+    curl https://go.dev/dl/go1.17.13.windows-amd64.zip -o ($env:USERPROFILE + '\go.zip')
+    tar -xf ($env:USERPROFILE + '\go.zip') -C $env:USERPROFILE
+} else {
+    echo "Using previously installed Go 1.17"
+}